def run_prependers(self, suspect):
"""Run prependers on suspect"""
plugcopy = self.plugins[:]
for plugin in self.prependers:
try:
self.logger.debug('Running prepender %s' % plugin)
self.set_workerstate("%s : Running Prepender %s" %
(suspect, plugin))
starttime = time.time()
result = plugin.pluginlist(suspect, plugcopy)
plugintime = time.time() - starttime
suspect.tags['scantimes'].append((plugin.section, plugintime))
if result is not None:
plugcopyset = set(plugcopy)
resultset = set(result)
removed = list(plugcopyset - resultset)
added = list(resultset - plugcopyset)
if len(removed) > 0:
self.logger.debug('Prepender %s removed plugins: %s' %
(plugin, list(map(str, removed))))
if len(added) > 0:
self.logger.debug('Prepender %s added plugins: %s' %
(plugin, list(map(str, added))))
plugcopy = result
except Exception:
CrashStore.store_exception()
exc = traceback.format_exc()
self.logger.error('Prepender plugin %s failed: %s' %
(str(plugin), exc))
return plugcopy
def _load_all(self, configstring):
"""load all plugins from config string. returns tuple ([list of loaded instances],allOk)"""
pluglist = []
config_re = re.compile(
"""^(?P<structured_name>[a-zA-Z0-9\.\_\-]+)(?:\((?P<config_override>[a-zA-Z0-9\.\_\-]+)\))?$""")
allOK = True
plugins = configstring.split(',')
for plug in plugins:
if plug == "":
continue
m = config_re.match(plug)
if m == None:
self.logger.error('Invalid Plugin Syntax: %s' % plug)
allOK = False
continue
structured_name, configoverride = m.groups()
structured_name = self.get_component_by_alias(structured_name)
try:
plugininstance = self._load_component(
structured_name, configsection=configoverride)
pluglist.append(plugininstance)
except (configparser.NoSectionError, configparser.NoOptionError):
CrashStore.store_exception()
self._logger().error(
"The plugin %s is accessing the config in __init__ -> can not load default values" % structured_name)
except Exception as e:
CrashStore.store_exception()
self._logger().error('Could not load plugin %s : %s' %
(structured_name, e))
exc = traceback.format_exc()
self._logger().error(exc)
allOK = False
return pluglist, allOK
def run_prependers(self, suspect):
"""Run prependers on suspect"""
plugcopy = self.plugins[:]
for plugin in self.prependers:
try:
self.logger.debug('Running prepender %s' % plugin)
self.set_threadinfo(
"%s : Running Prepender %s" % (suspect, plugin))
starttime = time.time()
result = plugin.pluginlist(suspect, plugcopy)
plugintime = time.time() - starttime
suspect.tags['scantimes'].append((plugin.section, plugintime))
if result != None:
plugcopyset = set(plugcopy)
resultset = set(result)
removed = list(plugcopyset - resultset)
added = list(resultset - plugcopyset)
if len(removed) > 0:
self.logger.debug(
'Prepender %s removed plugins: %s' % (plugin, list(map(str, removed))))
if len(added) > 0:
self.logger.debug(
'Prepender %s added plugins: %s' % (plugin, list(map(str, added))))
plugcopy = result
except Exception:
CrashStore.store_exception()
exc = traceback.format_exc()
self.logger.error(
'Prepender plugin %s failed: %s' % (str(plugin), exc))
return plugcopy
def _load_all(self, configstring):
"""load all plugins from config string. returns tuple ([list of loaded instances],allOk)"""
pluglist = []
config_re = re.compile(
"""^(?P<structured_name>[a-zA-Z0-9\.\_\-]+)(?:\((?P<config_override>[a-zA-Z0-9\.\_\-]+)\))?$""")
allOK = True
plugins = configstring.split(',')
for plug in plugins:
if plug == "":
continue
m = config_re.match(plug)
if m is None:
self.logger.error('Invalid Plugin Syntax: %s' % plug)
allOK = False
continue
structured_name, configoverride = m.groups()
structured_name = self.get_component_by_alias(structured_name)
try:
plugininstance = self._load_component(
structured_name, configsection=configoverride)
pluglist.append(plugininstance)
except (configparser.NoSectionError, configparser.NoOptionError):
CrashStore.store_exception()
self.logger.error("The plugin %s is accessing the config in __init__ -> can not load default values" % structured_name)
except Exception as e:
CrashStore.store_exception()
self.logger.error('Could not load plugin %s : %s' %
(structured_name, e))
exc = traceback.format_exc()
self.logger.error(exc)
allOK = False
return pluglist, allOK
def run_appenders(self, suspect, finaldecision):
"""Run appenders on suspect"""
if suspect.get_tag('noappenders'):
return
for plugin in self.appenders:
try:
self.logger.debug('Running appender %s' % plugin)
suspect.debug('Running appender %s' % plugin)
self.set_workerstate("%s : Running appender %s" %
(suspect, plugin))
starttime = time.time()
plugin.process(suspect, finaldecision)
plugintime = time.time() - starttime
suspect.tags['scantimes'].append((plugin.section, plugintime))
except Exception:
CrashStore.store_exception()
exc = traceback.format_exc()
self.logger.error('Appender plugin %s failed: %s' %
(str(plugin), exc))
def run_appenders(self, suspect, finaldecision):
"""Run appenders on suspect"""
if suspect.get_tag('noappenders'):
return
for plugin in self.appenders:
try:
self.logger.debug('Running appender %s' % plugin)
suspect.debug('Running appender %s' % plugin)
self.set_threadinfo(
"%s : Running appender %s" % (suspect, plugin))
starttime = time.time()
plugin.process(suspect, finaldecision)
plugintime = time.time() - starttime
suspect.tags['scantimes'].append((plugin.section, plugintime))
except Exception:
CrashStore.store_exception()
exc = traceback.format_exc()
self.logger.error(
'Appender plugin %s failed: %s' % (str(plugin), exc))
def lint(self):
errors = 0
fc = FunkyConsole()
self._lint_dependencies(fc)
print(fc.strcolor('Loading extensions...', 'magenta'))
exts = self.load_extensions()
for ext in exts:
(name, enabled, status) = ext
pname = fc.strcolor(name, 'cyan')
if enabled:
penabled = fc.strcolor('enabled', 'green')
else:
penabled = fc.strcolor('disabled', 'red')
print("%s: %s (%s)" % (pname, penabled, status))
print(fc.strcolor('Loading plugins...', 'magenta'))
if not self.load_plugins():
print(fc.strcolor('At least one plugin failed to load', 'red'))
print(fc.strcolor('Plugin loading complete', 'magenta'))
print("Linting ", fc.strcolor("main configuration", 'cyan'))
if not self.checkConfig():
print(fc.strcolor("ERROR", "red"))
else:
print(fc.strcolor("OK", "green"))
trashdir = self.config.get('main', 'trashdir').strip()
if trashdir != "":
if not os.path.isdir(trashdir):
print(
fc.strcolor("Trashdir %s does not exist" % trashdir, 'red'))
# sql config override
sqlconfigdbconnectstring = self.config.get(
'databaseconfig', 'dbconnectstring')
if sqlconfigdbconnectstring.strip() != '':
print("")
print("Linting ", fc.strcolor("sql configuration", 'cyan'))
try:
from fuglu.extensions.sql import get_session
sess = get_session(sqlconfigdbconnectstring)
tempsuspect = Suspect(
'*****@*****.**', '*****@*****.**', '/dev/null')
sqlvars = dict(
section='testsection', option='testoption', scope='$GLOBAL')
default_template_values(tempsuspect, sqlvars)
sess.execute(self.config.get('databaseconfig', 'sql'), sqlvars)
sess.remove()
print(fc.strcolor("OK", 'green'))
except Exception as e:
print(fc.strcolor("Failed %s" % str(e), 'red'))
allplugins = self.plugins + self.prependers + self.appenders
for plugin in allplugins:
print()
print("Linting Plugin ", fc.strcolor(str(plugin), 'cyan'),
'Config section:', fc.strcolor(str(plugin.section), 'cyan'))
try:
result = plugin.lint()
except Exception as e:
CrashStore.store_exception()
print("ERROR: %s" % e)
result = False
if result:
print(fc.strcolor("OK", "green"))
else:
errors = errors + 1
print(fc.strcolor("ERROR", "red"))
print("%s plugins reported errors." % errors)
if self.config.getboolean('main', 'versioncheck'):
check_version_status(lint=True)
def run_plugins(self, suspect, pluglist):
"""Run scannerplugins on suspect"""
suspect.debug('Will run plugins: %s' % pluglist)
for plugin in pluglist:
try:
self.logger.debug('Running plugin %s' % plugin)
self.set_workerstate("%s : Running Plugin %s" %
(suspect, plugin))
suspect.debug('Running plugin %s' % str(plugin))
starttime = time.time()
ans = plugin.examine(suspect)
plugintime = time.time() - starttime
suspect.tags['scantimes'].append((plugin.section, plugintime))
message = None
if type(ans) is tuple:
result, message = ans
else:
result = ans
if result is None:
result = DUNNO
suspect.tags['decisions'].append((plugin.section, result))
if result == DUNNO:
suspect.debug('Plugin makes no final decision')
elif result == ACCEPT:
suspect.debug(
'Plugin accepts the message - skipping all further tests'
)
self.logger.debug(
'Plugin says: ACCEPT. Skipping all other tests')
self.action = ACCEPT
break
elif result == DELETE:
suspect.debug(
'Plugin DELETES this message - no further tests')
self.logger.debug(
'Plugin says: DELETE. Skipping all other tests')
self.action = DELETE
self.message = message
self.trash(suspect, str(plugin))
break
elif result == REJECT:
suspect.debug(
'Plugin REJECTS this message - no further tests')
self.logger.debug(
'Plugin says: REJECT. Skipping all other tests')
self.action = REJECT
self.message = message
break
elif result == DEFER:
suspect.debug(
'Plugin DEFERS this message - no further tests')
self.logger.debug(
'Plugin says: DEFER. Skipping all other tests')
self.action = DEFER
self.message = message
break
else:
self.logger.error(
'Invalid Message action Code: %s. Using DUNNO' %
result)
except Exception as e:
CrashStore.store_exception()
exc = traceback.format_exc()
self.logger.error('Plugin %s failed: %s' % (str(plugin), exc))
suspect.debug(
'Plugin failed : %s . Please check fuglu log for more details'
% e)
def run_plugins(self, suspect, pluglist):
"""Run scannerplugins on suspect"""
suspect.debug('Will run plugins: %s' % pluglist)
for plugin in pluglist:
try:
self.logger.debug('Running plugin %s' % plugin)
self.set_threadinfo(
"%s : Running Plugin %s" % (suspect, plugin))
suspect.debug('Running plugin %s' % str(plugin))
starttime = time.time()
ans = plugin.examine(suspect)
plugintime = time.time() - starttime
suspect.tags['scantimes'].append((plugin.section, plugintime))
message = None
if type(ans) is tuple:
result, message = ans
else:
result = ans
if result == None:
result = DUNNO
suspect.tags['decisions'].append((plugin.section, result))
if result == DUNNO:
suspect.debug('Plugin makes no final decision')
elif result == ACCEPT:
suspect.debug(
'Plugin accepts the message - skipping all further tests')
self.logger.debug(
'Plugin says: ACCEPT. Skipping all other tests')
self.action = ACCEPT
break
elif result == DELETE:
suspect.debug(
'Plugin DELETES this message - no further tests')
self.logger.debug(
'Plugin says: DELETE. Skipping all other tests')
self.action = DELETE
self.message = message
self.trash(suspect, str(plugin))
break
elif result == REJECT:
suspect.debug(
'Plugin REJECTS this message - no further tests')
self.logger.debug(
'Plugin says: REJECT. Skipping all other tests')
self.action = REJECT
self.message = message
break
elif result == DEFER:
suspect.debug(
'Plugin DEFERS this message - no further tests')
self.logger.debug(
'Plugin says: DEFER. Skipping all other tests')
self.action = DEFER
self.message = message
break
else:
self.logger.error(
'Invalid Message action Code: %s. Using DUNNO' % result)
except Exception as e:
CrashStore.store_exception()
exc = traceback.format_exc()
self.logger.error('Plugin %s failed: %s' % (str(plugin), exc))
suspect.debug(
'Plugin failed : %s . Please check fuglu log for more details' % e)
def lint(self):
errors = 0
fc = FunkyConsole()
self._lint_dependencies(fc)
print(fc.strcolor('Loading extensions...', 'magenta'))
exts = self.load_extensions()
for ext in exts:
(name, enabled, status) = ext
pname = fc.strcolor(name, 'cyan')
if enabled:
penabled = fc.strcolor('enabled', 'green')
else:
penabled = fc.strcolor('disabled', 'red')
print("%s: %s (%s)" % (pname, penabled, status))
print(fc.strcolor('Loading plugins...', 'magenta'))
if not self.load_plugins():
print(fc.strcolor('At least one plugin failed to load', 'red'))
print(fc.strcolor('Plugin loading complete', 'magenta'))
print("Linting ", fc.strcolor("main configuration", 'cyan'))
if not self.checkConfig():
print(fc.strcolor("ERROR", "red"))
else:
print(fc.strcolor("OK", "green"))
trashdir = self.config.get('main', 'trashdir').strip()
if trashdir != "" and not os.path.isdir(trashdir):
print(fc.strcolor("Trashdir %s does not exist" % trashdir, 'red'))
# sql config override
sqlconfigdbconnectstring = self.config.get('databaseconfig',
'dbconnectstring')
if sqlconfigdbconnectstring.strip() != '':
print()
print("Linting ", fc.strcolor("sql configuration", 'cyan'))
try:
from fuglu.extensions.sql import get_session
sess = get_session(sqlconfigdbconnectstring)
tempsuspect = Suspect('*****@*****.**',
'*****@*****.**', '/dev/null')
sqlvars = dict(section='testsection',
option='testoption',
scope='$GLOBAL')
default_template_values(tempsuspect, sqlvars)
sess.execute(self.config.get('databaseconfig', 'sql'), sqlvars)
sess.remove()
print(fc.strcolor("OK", 'green'))
except Exception as e:
print(fc.strcolor("Failed %s" % str(e), 'red'))
allplugins = self.plugins + self.prependers + self.appenders
for plugin in allplugins:
print()
print("Linting Plugin ", fc.strcolor(str(plugin), 'cyan'),
'Config section:', fc.strcolor(str(plugin.section), 'cyan'))
try:
result = plugin.lint()
except Exception as e:
CrashStore.store_exception()
print("ERROR: %s" % e)
result = False
if result:
print(fc.strcolor("OK", "green"))
else:
errors = errors + 1
print(fc.strcolor("ERROR", "red"))
print("%s plugins reported errors." % errors)
if self.config.getboolean('main', 'versioncheck'):
check_version_status(lint=True)
def lint(self):
errors = 0
fc = FunkyConsole()
self._lint_dependencies(fc)
print(fc.strcolor('Loading extensions...', 'magenta'))
exts = self.load_extensions()
for ext in exts:
(name, enabled, status) = ext
pname = fc.strcolor(name, 'cyan')
if enabled:
penabled = fc.strcolor('enabled', 'green')
else:
penabled = fc.strcolor('disabled', 'red')
print("%s: %s (%s)" % (pname, penabled, status))
print(fc.strcolor('Loading plugins...', 'magenta'))
if not self.load_plugins():
print(fc.strcolor('At least one plugin failed to load', 'red'))
errors +=1
print(fc.strcolor('Plugin loading complete', 'magenta'))
print("Linting ", fc.strcolor("main configuration", 'cyan'))
if not self.checkConfig():
print(fc.strcolor("ERROR", "red"))
errors += 1
else:
print(fc.strcolor("OK", "green"))
trashdir = self.config.get('main', 'trashdir').strip()
if trashdir != "" and not os.path.isdir(trashdir):
print(fc.strcolor("Trashdir %s does not exist" % trashdir, 'red'))
errors += 1
# sql config override
sqlconfigdbconnectstring = self.config.get('databaseconfig', 'dbconnectstring')
if sqlconfigdbconnectstring.strip() != '':
print()
print("Linting ", fc.strcolor("sql configuration", 'cyan'))
try:
from fuglu.extensions.sql import get_session
sess = get_session(sqlconfigdbconnectstring)
tempsuspect = Suspect(
'*****@*****.**', '*****@*****.**', '/dev/null',
att_cachelimit=self.config.getint('performance','att_mgr_cachesize'))
sqlvars = dict(
section='testsection', option='testoption', scope='$GLOBAL')
default_template_values(tempsuspect, sqlvars)
sess.execute(self.config.get('databaseconfig', 'sql'), sqlvars)
sess.remove()
print(fc.strcolor("OK", 'green'))
except Exception as e:
print(fc.strcolor("Failed %s" % str(e), 'red'))
errors += 1
allplugins = self.plugins + self.prependers + self.appenders
perrors = 0
for plugin in allplugins:
print()
print("Linting Plugin ", fc.strcolor(str(plugin), 'cyan'),
'Config section:', fc.strcolor(str(plugin.section), 'cyan'))
try:
result = plugin.lint()
except Exception as e:
CrashStore.store_exception()
print("ERROR: %s" % e)
result = False
if result:
print(fc.strcolor("OK", "green"))
else:
perrors += 1
errors += 1
print(fc.strcolor("ERROR", "red"))
print("%s plugins reported errors." % perrors)
if "milter" in self.config.get('main', 'incomingport') \
and self.config.get('performance', 'backend') != 'process':
try:
minfreethreads = self.config.getint('performance', 'minfreethreads')
if minfreethreads < 1:
print(fc.strcolor('\nMilter enabled with "thread" backend but "minfreethreads < 1"', 'yellow'))
print("To keep milter responsive it is recommended to set minfreethreads >= 1\n"
"to make fuglu more resonsive.\n")
except (configparser.NoSectionError, configparser.NoOptionError):
print(fc.strcolor('\nMilter enabled with "thread" backend but "minfreethreads is not defined!"', 'yellow'))
print("To keep fuglu-milter responsive it is recommended to set minfreethreads >= 1\n")
if self.config.getboolean('main', 'versioncheck'):
check_version_status(lint=True)
return errors