def __init__(self):
Config.add_option('-j',
'--jobs',
dest='jobs',
help='comma separated list of jobs to execute',
action='store',
default='',
metavar='JOBS')
Config.add_option('',
'--help-jobs',
dest='show_help',
help='print help for installed jobs',
action='store_true',
default=False)
super(Imp, self).__init__(Config.get().imp)
self._conf = Config.get().imp
self._show_help = Config.get().options.show_help
if Config.get().options.jobs:
self._jobs = Config.get().options.jobs.split(',')
else:
self._jobs = []
if self._show_help:
self._conf.verbose = False
def __init__(self):
Config.add_option('-j', '--jobs',
dest='jobs',
help='comma separated list of jobs to execute',
action='store',
default='',
metavar='JOBS')
Config.add_option('', '--help-jobs',
dest='show_help',
help='print help for installed jobs',
action='store_true',
default=False)
super(Imp, self).__init__(Config.get().imp)
self._conf = Config.get().imp
self._show_help = Config.get().options.show_help
if Config.get().options.jobs:
self._jobs = Config.get().options.jobs.split(',')
else:
self._jobs = []
if self._show_help:
self._conf.verbose = False
def run(self):
"""
Process a message from :attr:`sys.stdin` and write the result to
:attr:`sys.stdout`.
The message will be processed according the the global settings.
.. note::
There is no need to import and call this method directly.
SynFu provides the wrapper script *synfu-reactor.py* for this job.
"""
if (self._is_cancel(self._mm)):
sys.exit(0)
if Config.get().options.filter_only:
self._mm._headers = self._filter_headers(
re.compile(''), self._mm._headers, self._conf.outlook_hacks,
self._conf.fix_dateline, 0,
Config.get().options.header_whitelist)
else:
self._mm = self._apply_blacklist(self._mm, 'reactor', 0)
if not self._mm:
# should not happen but who knows?
self._log('--- Message was dropped by blacklist')
sys.exit(0)
self._mm = self._process(self._mm)
self._mm.add_header('X-SynFU-Reactor',
Reactor.NOTICE,
version=Reactor.VERSION)
print_out = False
for p in str(self._mm).split('\n')[1:]:
if self._conf.strip_notes and not Config.get().options.filter_only:
if Reactor.SIGN_NOTICE[0].match(p) and not print_out:
print_out = True
continue
if Reactor.SIGN_NOTICE[1].match(p) and not print_out:
# mailman is kinky
print ''
print_out = True
continue
if Reactor.SIGN_NOTICE[2].match(p) and not print_out:
continue
if Reactor.BROKEN_MULTIPART.match(p):
# multipart *should be* seperated by a newline
p = p.replace('; boundary=', ';\n boundary=')
print p.replace('\x0d', '')
def run(self):
"""
Process a message from :attr:`sys.stdin` and write the result to
:attr:`sys.stdout`.
The message will be processed according the the global settings.
.. note::
There is no need to import and call this method directly.
SynFu provides the wrapper script *synfu-reactor.py* for this job.
"""
if (self._is_cancel(self._mm)):
sys.exit(0)
if Config.get().options.filter_only:
self._mm._headers = self._filter_headers(re.compile(''), self._mm._headers,
self._conf.outlook_hacks,
self._conf.fix_dateline, 0,
Config.get().options.header_whitelist)
else:
self._mm = self._apply_blacklist(self._mm, 'reactor', 0)
if not self._mm:
# should not happen but who knows?
self._log('--- Message was dropped by blacklist')
sys.exit(0)
self._mm = self._process(self._mm)
self._mm.add_header('X-SynFU-Reactor',
Reactor.NOTICE, version=Reactor.VERSION)
print_out = False
for p in str(self._mm).split('\n')[1:]:
if self._conf.strip_notes and not Config.get().options.filter_only:
if Reactor.SIGN_NOTICE[0].match(p) and not print_out:
print_out = True
continue
if Reactor.SIGN_NOTICE[1].match(p) and not print_out:
# mailman is kinky
print ''
print_out = True
continue
if Reactor.SIGN_NOTICE[2].match(p) and not print_out:
continue
if Reactor.BROKEN_MULTIPART.match(p):
# multipart *should be* seperated by a newline
p = p.replace('; boundary=', ';\n boundary=')
print p.replace('\x0d', '')
def job_config(self, name, defaults={}):
"""
Load job specific config object.
| This method will try to load a subsection of the Imp-configuration.
| The optional dictionary *defaults* contains a list of default key,value pairs.
:param: name: The name of the config subsection beneath Imp.jobs
:param: defaults: A :const:`dict` with key,value pairs for default parameters
:return: A JobConf instance with all top-level settings converted to attributes.
"""
class JobConf(object):
pass
conf = JobConf()
imp = Config.get().imp
defaults.update(getattr(imp, 'jobs', {}).get(name, {}))
for k,v in defaults.items():
setattr(conf, k, v)
setattr(conf, 'verbose' , imp.verbose)
setattr(conf, 'verbosity', imp.verbosity)
return conf
def __init__(self):
super(GroomNewsgroups, self).__init__()
empty_conf = {
'http_proxy': None,
'https_proxy': None,
'listinfo': {},
'newsgroups': None
}
self._conf = self.job_config('groom_newsgroups', empty_conf)
proxy_list = []
if self._conf.http_proxy:
proxy_list.append(
urllib2.ProxyHandler({'http': self._conf.http_proxy}))
if self._conf.https_proxy:
proxy_list.append(
urllib2.ProxyHandler({'https': self._conf.https_proxy}))
opener = urllib2.build_opener(*proxy_list)
urllib2.install_opener(opener)
# plug in filter settings from postfilter
self._conf.__setattr__('filters', Config.get().postfilter.filters)
def __init__(self):
super(GroomNewsgroups, self).__init__()
empty_conf = {
'http_proxy' : None,
'https_proxy' : None,
'listinfo' : {},
'newsgroups' : None
}
self._conf = self.job_config('groom_newsgroups', empty_conf)
proxy_list = []
if self._conf.http_proxy:
proxy_list.append(urllib2.ProxyHandler({ 'http' : self._conf.http_proxy }))
if self._conf.https_proxy:
proxy_list.append(urllib2.ProxyHandler({ 'https': self._conf.https_proxy }))
opener = urllib2.build_opener(*proxy_list)
urllib2.install_opener(opener)
# plug in filter settings from postfilter
self._conf.__setattr__('filters', Config.get().postfilter.filters)
def job_config(self, name, defaults={}):
"""
Load job specific config object.
| This method will try to load a subsection of the Imp-configuration.
| The optional dictionary *defaults* contains a list of default key,value pairs.
:param: name: The name of the config subsection beneath Imp.jobs
:param: defaults: A :const:`dict` with key,value pairs for default parameters
:return: A JobConf instance with all top-level settings converted to attributes.
"""
class JobConf(object):
pass
conf = JobConf()
imp = Config.get().imp
defaults.update(getattr(imp, 'jobs', {}).get(name, {}))
for k, v in defaults.items():
setattr(conf, k, v)
setattr(conf, 'verbose', imp.verbose)
setattr(conf, 'verbosity', imp.verbosity)
return conf
def __init__(self, mode=None):
if mode == 'news2mail':
Config.get().postfilter.log_filename = \
Config.get().postfilter.log_news2mail
elif mode == 'mail2news':
Config.get().postfilter.log_filename = \
Config.get().postfilter.log_mail2news
super(PostFilter, self).__init__(Config.get().postfilter)
self._conf = Config.get().postfilter
def __init__(self, mode=None):
if mode == 'news2mail':
Config.get().postfilter.log_filename = \
Config.get().postfilter.log_news2mail
elif mode == 'mail2news':
Config.get().postfilter.log_filename = \
Config.get().postfilter.log_mail2news
super(PostFilter, self).__init__(Config.get().postfilter)
self._conf = Config.get().postfilter
def __init__(self):
Config.add_option('-F', '--filter-only',
dest = 'filter_only',
action = 'store_true',
default = False,
help = 'Apply only header filter')
Config.add_option('-K', '--keep-header',
dest = 'header_whitelist',
action = 'append',
default = [],
help = 'Header whitelist (won\'t remove these)')
super(Reactor, self).__init__(Config.get().reactor)
self._conf = Config.get().reactor
self._mm = email.message_from_file(sys.stdin)
def __init__(self):
Config.add_option('-F',
'--filter-only',
dest='filter_only',
action='store_true',
default=False,
help='Apply only header filter')
Config.add_option('-K',
'--keep-header',
dest='header_whitelist',
action='append',
default=[],
help='Header whitelist (won\'t remove these)')
super(Reactor, self).__init__(Config.get().reactor)
self._conf = Config.get().reactor
self._mm = email.message_from_file(sys.stdin)
def __init__(self):
super(ImpJob, self).__init__(Config.get().imp)
class Imp(FUCore):
"""
Imp - the periodic Imp
| SynFU Imp is a background helper designed to run periodic maintenance jobs.
| At the time of this writing the following jobs are built-in:
- :ref:`GroomNewsgroups` -- update newsgroup descriptions via mailman
| Each job is provided as a separate python file containing arbitrary
| job definitions in the form of :class:`synfu.imp.ImpJob` subclasses.
"""
VERSION = '0.3'
def __init__(self):
Config.add_option('-j',
'--jobs',
dest='jobs',
help='comma separated list of jobs to execute',
action='store',
default='',
metavar='JOBS')
Config.add_option('',
'--help-jobs',
dest='show_help',
help='print help for installed jobs',
action='store_true',
default=False)
super(Imp, self).__init__(Config.get().imp)
self._conf = Config.get().imp
self._show_help = Config.get().options.show_help
if Config.get().options.jobs:
self._jobs = Config.get().options.jobs.split(',')
else:
self._jobs = []
if self._show_help:
self._conf.verbose = False
def run(self):
if self._show_help:
# extra printout, --help-plugins disables _log()
print('Installed jobs:')
self._log('--- begin')
self._log('--- loading plugins:')
plugin_path = [
os.path.join(os.path.dirname(os.path.abspath(__file__)),
'plugins'), self._conf.plugin_dir
]
for p in plugin_path:
if not os.path.exists(p):
self._log(
'!!! skipping "{0}" - no such file or directory'.format(
p))
continue
sys.path.append(p)
for (importer, name, ispkg) in pkgutil.walk_packages([p]):
if name.startswith('ImpJob'):
try:
__import__(name)
except Exception, e:
self._log(
'!!! unable to import "{0}": {1}: {2}'.format(
name, e.__class__.__name__, e))
if 'LOADED_JOBS' in dir(Imp):
for plugin in Imp.LOADED_JOBS:
self._log('--- loaded plugin "{0}"'.format(plugin.__name__),
verbosity=2)
for plugin in Imp.LOADED_JOBS:
if self._jobs and (not plugin.__name__ in self._jobs):
continue
try:
inst = plugin()
if self._show_help:
print '-- {0}:'.format(plugin.__name__)
inst.show_help(Config.get().optargs)
del inst
continue
do_run = inst.needs_run(Config.get().optargs)
self._log('--- {0}.needs_run() ='.format(
plugin.__name__, do_run))
if do_run:
self._log('--- executing job "{0}"'.format(
plugin.__name__))
res = inst.run()
self._log('--- job result: {0}'.format(res))
del inst
except Exception, e:
self._log('!!! uncaught exception {0}: {1}'.format(
e.__class__.__name__, e))
def __init__(self):
super(ImpJob, self).__init__(Config.get().imp)