def check_sql_blacklist(self, suspect, runtimeconfig=None): """Check this message against the SQL blacklist. returns highspamaction on hit, DUNNO otherwise""" #work in progress if not self.config.has_option( self.section, 'check_sql_blacklist') or not self.config.getboolean( self.section, 'check_sql_blacklist'): return DUNNO from fuglu.extensions.sql import ENABLED if not ENABLED: self.logger.error( 'Cannot check sql blacklist, SQLALCHEMY extension is not available' ) return DUNNO from fuglu.extensions.sql import get_session try: dbsession = get_session( self.config.get(self.section, 'sql_blacklist_dbconnectstring')) conf_sql = self.config.get(self.section, 'sql_blacklist_sql') sql, params = self._replace_sql_params(suspect, conf_sql) resultproxy = dbsession.execute(sql, params) except Exception, e: self.logger.error('Could not read blacklist from DB: %s' % e) suspect.debug('Blacklist check failed: %s' % e) return DUNNO
def lint(self): allok = self.checkConfig() if not HAVE_SRS: allok = False print 'SRS library not found' if self.config.get(self.section, 'secret') == '': allok = False print 'no secret set in config' if allok: srs = self._init_srs() forward_domain = self.config.get(self.section, 'forward_domain') srs.forward('*****@*****.**', forward_domain) sqlquery = self.config.get(self.section, 'domain_sql_query') if not sqlquery.lower().startswith('select '): allok = False print 'SQL statement must be a SELECT query' if not ENABLED: allok = False print 'SQLAlchemy not available, cannot use SQL backend' if allok: dbconnection = self.config.get(self.section, 'dbconnection') if dbconnection.strip() == '': print 'No DB connection defined. Disabling SQL backend, all addresses will be rewritten.' else: try: conn = get_session(dbconnection) conn.execute(sqlquery, {'domain': 'example.com'}) except Exception as e: allok = False print str(e) return allok
def lint_blacklist(self): if not self.config.has_option( self.section, 'check_sql_blacklist') or not self.config.getboolean( self.section, 'check_sql_blacklist'): return True from fuglu.extensions.sql import ENABLED, get_session if not ENABLED: print("SQL Blacklist requested but SQLALCHEMY is not enabled") return False session = get_session( self.config.get(self.section, 'sql_blacklist_dbconnectstring')) suspect = Suspect('*****@*****.**', '*****@*****.**', '/dev/null') conf_sql = self.config.get(self.section, 'sql_blacklist_sql') sql, params = self._replace_sql_params(suspect, conf_sql) try: session.execute(sql, params) print("Blacklist SQL Query OK") return True except Exception as e: print(e) return False
def get_domain_setting(self, domain, dbconnection, sqlquery, cache, cachename, default_value=None, logger=None): if logger is None: logger = logging.getLogger() cachekey = '%s-%s' % (cachename, domain) cached = cache.get_cache(cachekey) if cached is not None: logger.debug("got cached setting for %s" % domain) return cached settings = default_value try: session = get_session(dbconnection) # get domain settings dom = session.execute(sqlquery, {'domain': domain}).fetchall() if not dom and not dom[0] and len(dom[0]) == 0: logger.warning( "Can not load domain setting - domain %s not found. Using default settings." % domain) else: settings = dom[0][0] session.close() except Exception as e: logger.error("Exception while loading setting for %s : %s" % (domain, str(e))) cache.put_cache(cachekey, settings) logger.debug("refreshed setting for %s" % domain) return settings
def pluginlist(self, suspect, pluginlist): if not SQL_EXTENSION_AVAILABLE: self._logger().warn("SQLALCHEMY extension is not enabled, SQLSkipper will not run") return sqlsession = get_session(self.config.get(self.section,'dbconnectstring')) self._logger().debug("Checking database overrides for %s"%(suspect.recipients)) #if postfix->fuglu is not configured with xxx_destination_recipient_limit=1 the message might have multiple recipients user_configs = sqlsession.execute("SELECT recipient,antispam_enabled FROM spamconfig WHERE recipient IN :recipient",dict(recipient=tuple(suspect.recipients))) #if one recipient doesn't have a config, we assume antispam should run normally if user_configs.rowcount<len(suspect.recipients): self._logger().debug("Not all recipients have a database config - assuming normal run") return for row in user_configs: recipient, antispam_enabled = row self._logger().debug("Recipient %s anti spam enabled in database: %s"%(recipient,bool(antispam_enabled))) if antispam_enabled: #at least one user has anti spam enabled return # if we reach this point, all recipients in the message have antispam disabled self._logger().info("%s - antispam disabled by database override"%(suspect.id)) skippluginlist = ['SAPlugin', ] # add other plugins you want to skip here listcopy = pluginlist[:] for plug in pluginlist: name = plug.__class__.__name__ if name in skippluginlist: listcopy.remove(plug) return listcopy
def check_sql_blacklist(self, suspect, runtimeconfig=None): """Check this message against the SQL blacklist. returns highspamaction on hit, DUNNO otherwise""" #work in progress if not self.config.has_option(self.section, 'check_sql_blacklist') or not self.config.getboolean(self.section, 'check_sql_blacklist'): return DUNNO from fuglu.extensions.sql import ENABLED if not ENABLED: self.logger.error( 'Cannot check sql blacklist, SQLALCHEMY extension is not available') return DUNNO from fuglu.extensions.sql import get_session try: dbsession = get_session( self.config.get(self.section, 'sql_blacklist_dbconnectstring')) conf_sql = self.config.get(self.section, 'sql_blacklist_sql') sql, params = self._replace_sql_params(suspect, conf_sql) resultproxy = dbsession.execute(sql, params) except Exception, e: self.logger.error('Could not read blacklist from DB: %s' % e) suspect.debug('Blacklist check failed: %s' % e) return DUNNO
def check_sql_blacklist(self, suspect, runtimeconfig=None): """Check this message against the SQL blacklist. returns highspamaction on hit, DUNNO otherwise""" #work in progress if not self.config.has_option( self.section, 'check_sql_blacklist') or not self.config.getboolean( self.section, 'check_sql_blacklist'): return DUNNO if not SQL_EXTENSION_ENABLED: self.logger.error( 'Cannot check sql blacklist, SQLALCHEMY extension is not available' ) return DUNNO try: dbsession = get_session( self.config.get(self.section, 'sql_blacklist_dbconnectstring')) conf_sql = self.config.get(self.section, 'sql_blacklist_sql') sql, params = self._replace_sql_params(suspect, conf_sql) resultproxy = dbsession.execute(sql, params) except Exception as e: self.logger.error('Could not read blacklist from DB: %s' % e) suspect.debug('Blacklist check failed: %s' % e) return DUNNO for result in resultproxy: dbvalue = result[0] # this value might have multiple words allvalues = dbvalue.split() for blvalue in allvalues: self.logger.debug(blvalue) # build regex # translate glob to regexr # http://stackoverflow.com/questions/445910/create-regex-from-glob-expression regexp = re.escape(blvalue).replace(r'\?', '.').replace(r'\*', '.*?') self.logger.debug(regexp) pattern = re.compile(regexp) if pattern.search(suspect.from_address): self.logger.debug( '%s Blacklist match : %s for sa pref %s' % (suspect.id, suspect.from_address, blvalue)) confcheck = self.config if runtimeconfig is not None: confcheck = runtimeconfig configaction = string_to_actioncode( confcheck.get(self.section, 'highspamaction'), self.config) suspect.tags['spam']['SpamAssassin'] = True prependheader = self.config.get('main', 'prependaddedheaders') suspect.addheader("%sBlacklisted" % prependheader, blvalue) suspect.debug('Sender is Blacklisted: %s' % blvalue) if configaction is None: return DUNNO return configaction return DUNNO
def check_sql_blacklist(self, suspect, runtimeconfig=None): """Check this message against the SQL blacklist. returns highspamaction on hit, DUNNO otherwise""" #work in progress if not self.config.has_option(self.section, 'check_sql_blacklist') or not self.config.getboolean(self.section, 'check_sql_blacklist'): return DUNNO from fuglu.extensions.sql import ENABLED if not ENABLED: self.logger.error( 'Cannot check sql blacklist, SQLALCHEMY extension is not available') return DUNNO from fuglu.extensions.sql import get_session try: dbsession = get_session( self.config.get(self.section, 'sql_blacklist_dbconnectstring')) conf_sql = self.config.get(self.section, 'sql_blacklist_sql') sql, params = self._replace_sql_params(suspect, conf_sql) resultproxy = dbsession.execute(sql, params) except Exception as e: self.logger.error('Could not read blacklist from DB: %s' % e) suspect.debug('Blacklist check failed: %s' % e) return DUNNO for result in resultproxy: dbvalue = result[0] # this value might have multiple words allvalues = dbvalue.split() for blvalue in allvalues: self.logger.debug(blvalue) # build regex # translate glob to regexr # http://stackoverflow.com/questions/445910/create-regex-from-glob-expression regexp = re.escape(blvalue).replace( r'\?', '.').replace(r'\*', '.*?') self.logger.debug(regexp) pattern = re.compile(regexp) if pattern.search(suspect.from_address): self.logger.debug( 'Blacklist match : %s for sa pref %s' % (suspect.from_address, blvalue)) confcheck = self.config if runtimeconfig != None: confcheck = runtimeconfig configaction = string_to_actioncode( confcheck.get(self.section, 'highspamaction'), self.config) suspect.tags['spam']['SpamAssassin'] = True prependheader = self.config.get( 'main', 'prependaddedheaders') suspect.addheader("%sBlacklisted" % prependheader, blvalue) suspect.debug('Sender is Blacklisted: %s' % blvalue) if configaction == None: return DUNNO return configaction return DUNNO
def already_notified(self, vacation, recipient): """return true if this user has been notfied in the last 24 hours""" dbsession = get_session(self.config.get(self.section, 'dbconnectstring')) log = dbsession.query(VacationReply).filter_by(vacation=vacation).filter( VacationReply.sent > datetime.now() - timedelta(days=1)).filter_by(recipient=recipient).first() dbsession.expunge_all() if log != None: self.logger.debug( 'Sender %s already notfied at %s' % (log.recipient, log.sent)) return True return False
def log_bounce(self, suspect, vacation): """log a bounce so we know tho whom we already sent""" log = VacationReply() log.recipient = suspect.from_address log.sent = datetime.now() log.vacation = vacation dbsession = get_session(self.config.get(self.section, 'dbconnectstring')) dbsession.add(log) dbsession.flush() dbsession.expunge_all()
def pluginlist(self, suspect, pluginlist): if not SQL_EXTENSION_AVAILABLE: self._logger().warn( "SQLALCHEMY extension is not enabled, SQLSkipper will not run") return sqlsession = get_session( self.config.get(self.section, 'dbconnectstring')) self._logger().debug("Checking database overrides for %s" % (suspect.recipients)) # if postfix->fuglu is not configured with # xxx_destination_recipient_limit=1 the message might have multiple # recipients user_configs = sqlsession.execute( "SELECT recipient,antispam_enabled FROM spamconfig WHERE recipient IN :recipient", dict(recipient=tuple(suspect.recipients))) # if one recipient doesn't have a config, we assume antispam should run # normally if user_configs.rowcount < len(suspect.recipients): self._logger().debug( "Not all recipients have a database config - assuming normal run" ) return for row in user_configs: recipient, antispam_enabled = row self._logger().debug( "Recipient %s anti spam enabled in database: %s" % (recipient, bool(antispam_enabled))) if antispam_enabled: # at least one user has anti spam enabled return # if we reach this point, all recipients in the message have antispam # disabled self._logger().info("%s - antispam disabled by database override" % (suspect.id)) # add other plugins you want to skip here skippluginlist = [ 'SAPlugin', ] listcopy = pluginlist[:] for plug in pluginlist: name = plug.__class__.__name__ if name in skippluginlist: listcopy.remove(plug) return listcopy
def lint_sql(self): if not SQL_EXTENSION_AVAILABLE: print "SQLALCHEMY extension is not enabled" return False from sqlalchemy.sql.expression import func session = get_session(self.config.get(self.section, 'dbconnectstring')) try: dbtime = session.execute(func.current_timestamp()).scalar() print "DB connection successful. Server time: %s" % dbtime session.close() return True except Exception, e: print e return False
def lint_sql(self): if not SQL_EXTENSION_AVAILABLE: print("SQLALCHEMY extension is not enabled") return False from sqlalchemy.sql.expression import func session = get_session( self.config.get(self.section, 'dbconnectstring')) try: dbtime = session.execute(func.current_timestamp()).scalar() print("DB connection successful. Server time: %s" % dbtime) session.close() return True except Exception as e: print(e) return False
def _loadvacation(self): """loads all vacations from database, do not call directly, only through reloadifnecessary""" self.logger.debug('Reloading vacation...') self.lastreload = time.time() newvacations = {} dbsession = get_session(self.config.get('VacationPlugin', 'dbconnectstring'), expire_on_commit=False) vaccounter = 0 now = datetime.now() for vac in dbsession.query(Vacation).filter_by(enabled=True).filter(Vacation.start < now).filter(Vacation.end > now): vaccounter += 1 self.logger.debug(vac) newvacations[vac.awayuser] = vac # important to expunge or other sessions wont be able to use this # vacation object dbsession.expunge_all() self.vacations = newvacations self.logger.debug('%s vacations loaded' % vaccounter)
def lint_sql(self): if not SQL_EXTENSION_ENABLED: print("Vacation requires the fuglu sql extension to be enabled") return False try: dbsession = get_session(self.config.get(self.section, 'dbconnectstring')) bind = dbsession.get_bind(Vacation) bind.connect() now = datetime.now() allvacs = dbsession.query(Vacation).filter_by(enabled=True).filter( Vacation.start < now).filter(Vacation.end > now) for vac in allvacs: print(vac) except Exception as e: print("Database error: %s" % str(e)) return False return True
def process(self, suspect, decision): if not SQL_EXTENSION_ENABLED: self.logger.error("Fuglu SQL Extensions not enabled") return connstring = self.config.get(self.section, 'dbconnectstring') session = get_session(connstring) if session is None: self.logger.error("Could not create database session") return try: conn = session.connection() conn.connect() except Exception as e: self.logger.error("Database Connection failed: %s" % e) return statementlist = self.get_statements() for statement in statementlist: self.logger.debug("Template: %s" % statement) addvalues = { 'action': actioncode_to_string(decision), } from_header = suspect.get_message_rep()['from'] try: addvalues['header_from'] = self.stripAddress(from_header) except Exception: #use full from header addvalues['header_from'] = from_header replaced = apply_template(statement, suspect, values=addvalues, valuesfunction=self.sqlfix) self.logger.debug("Statement: %s" % replaced) try: result = session.execute(replaced) except Exception as e: self.logger.error("Statement failed: statement=%s , error=%s" % (replaced, str(e))) session.remove()
def lint_blacklist(self): if not self.config.has_option(self.section, 'check_sql_blacklist') or not self.config.getboolean(self.section,'check_sql_blacklist'): return True from fuglu.extensions.sql import ENABLED,get_session if not ENABLED: print "SQL Blacklist requested but SQLALCHEMY is not enabled" return False session=get_session(self.config.get(self.section,'sql_blacklist_dbconnectstring')) suspect=Suspect('*****@*****.**','*****@*****.**','/dev/null') conf_sql=self.config.get(self.section,'sql_blacklist_sql') sql,params=self._replace_sql_params(suspect, conf_sql) try: session.execute(sql,params) print "Blacklist SQL Query OK" return True except Exception,e: print e return False
def _lint_sql(self): lint_ok = True sqlquery = self.config.get(self.section, 'domain_sql_query') dbconnection = self.config.get(self.section, 'dbconnection').strip() if not ENABLED and dbconnection != '': print('SQLAlchemy not available, cannot use SQL backend') lint_ok = False elif dbconnection == '': print('No DB connection defined. Disabling SQL backend') else: if not sqlquery.lower().startswith('select '): lint_ok = False print('SQL statement must be a SELECT query') if lint_ok: try: conn = get_session(dbconnection) conn.execute(sqlquery, {'domain': 'example.com'}) except Exception as e: lint_ok = False print(str(e)) return lint_ok
def lint(self): if not SQL_EXTENSION_ENABLED: print("Fuglu SQL Extensions not enabled") return False connstring = self.config.get(self.section, 'dbconnectstring') session = get_session(connstring) if session is None: print("Could not create database session") return False try: conn = session.connection() conn.connect() except Exception as e: print("Database Connection failed: %s" % e) return False session.remove() return True
def lint(self): errors = 0 fc = FunkyConsole() self._lint_dependencies(fc) print(fc.strcolor('Loading extensions...', 'magenta')) exts = self.load_extensions() for ext in exts: (name, enabled, status) = ext pname = fc.strcolor(name, 'cyan') if enabled: penabled = fc.strcolor('enabled', 'green') else: penabled = fc.strcolor('disabled', 'red') print("%s: %s (%s)" % (pname, penabled, status)) print(fc.strcolor('Loading plugins...', 'magenta')) if not self.load_plugins(): print(fc.strcolor('At least one plugin failed to load', 'red')) print(fc.strcolor('Plugin loading complete', 'magenta')) print("Linting ", fc.strcolor("main configuration", 'cyan')) if not self.checkConfig(): print(fc.strcolor("ERROR", "red")) else: print(fc.strcolor("OK", "green")) trashdir = self.config.get('main', 'trashdir').strip() if trashdir != "": if not os.path.isdir(trashdir): print( fc.strcolor("Trashdir %s does not exist" % trashdir, 'red')) # sql config override sqlconfigdbconnectstring = self.config.get( 'databaseconfig', 'dbconnectstring') if sqlconfigdbconnectstring.strip() != '': print("") print("Linting ", fc.strcolor("sql configuration", 'cyan')) try: from fuglu.extensions.sql import get_session sess = get_session(sqlconfigdbconnectstring) tempsuspect = Suspect( '*****@*****.**', '*****@*****.**', '/dev/null') sqlvars = dict( section='testsection', option='testoption', scope='$GLOBAL') default_template_values(tempsuspect, sqlvars) sess.execute(self.config.get('databaseconfig', 'sql'), sqlvars) sess.remove() print(fc.strcolor("OK", 'green')) except Exception as e: print(fc.strcolor("Failed %s" % str(e), 'red')) allplugins = self.plugins + self.prependers + self.appenders for plugin in allplugins: print() print("Linting Plugin ", fc.strcolor(str(plugin), 'cyan'), 'Config section:', fc.strcolor(str(plugin.section), 'cyan')) try: result = plugin.lint() except Exception as e: CrashStore.store_exception() print("ERROR: %s" % e) result = False if result: print(fc.strcolor("OK", "green")) else: errors = errors + 1 print(fc.strcolor("ERROR", "red")) print("%s plugins reported errors." % errors) if self.config.getboolean('main', 'versioncheck'): check_version_status(lint=True)
def exec_sql(self, sql, values=None): if values == None: values = {} session = get_session(self.testdb) session.execute(sql, values) session.remove()
def lint(self): errors = 0 fc = FunkyConsole() self._lint_dependencies(fc) print(fc.strcolor('Loading extensions...', 'magenta')) exts = self.load_extensions() for ext in exts: (name, enabled, status) = ext pname = fc.strcolor(name, 'cyan') if enabled: penabled = fc.strcolor('enabled', 'green') else: penabled = fc.strcolor('disabled', 'red') print("%s: %s (%s)" % (pname, penabled, status)) print(fc.strcolor('Loading plugins...', 'magenta')) if not self.load_plugins(): print(fc.strcolor('At least one plugin failed to load', 'red')) errors +=1 print(fc.strcolor('Plugin loading complete', 'magenta')) print("Linting ", fc.strcolor("main configuration", 'cyan')) if not self.checkConfig(): print(fc.strcolor("ERROR", "red")) errors += 1 else: print(fc.strcolor("OK", "green")) trashdir = self.config.get('main', 'trashdir').strip() if trashdir != "" and not os.path.isdir(trashdir): print(fc.strcolor("Trashdir %s does not exist" % trashdir, 'red')) errors += 1 # sql config override sqlconfigdbconnectstring = self.config.get('databaseconfig', 'dbconnectstring') if sqlconfigdbconnectstring.strip() != '': print() print("Linting ", fc.strcolor("sql configuration", 'cyan')) try: from fuglu.extensions.sql import get_session sess = get_session(sqlconfigdbconnectstring) tempsuspect = Suspect( '*****@*****.**', '*****@*****.**', '/dev/null', att_cachelimit=self.config.getint('performance','att_mgr_cachesize')) sqlvars = dict( section='testsection', option='testoption', scope='$GLOBAL') default_template_values(tempsuspect, sqlvars) sess.execute(self.config.get('databaseconfig', 'sql'), sqlvars) sess.remove() print(fc.strcolor("OK", 'green')) except Exception as e: print(fc.strcolor("Failed %s" % str(e), 'red')) errors += 1 allplugins = self.plugins + self.prependers + self.appenders perrors = 0 for plugin in allplugins: print() print("Linting Plugin ", fc.strcolor(str(plugin), 'cyan'), 'Config section:', fc.strcolor(str(plugin.section), 'cyan')) try: result = plugin.lint() except Exception as e: CrashStore.store_exception() print("ERROR: %s" % e) result = False if result: print(fc.strcolor("OK", "green")) else: perrors += 1 errors += 1 print(fc.strcolor("ERROR", "red")) print("%s plugins reported errors." % perrors) if "milter" in self.config.get('main', 'incomingport') \ and self.config.get('performance', 'backend') != 'process': try: minfreethreads = self.config.getint('performance', 'minfreethreads') if minfreethreads < 1: print(fc.strcolor('\nMilter enabled with "thread" backend but "minfreethreads < 1"', 'yellow')) print("To keep milter responsive it is recommended to set minfreethreads >= 1\n" "to make fuglu more resonsive.\n") except (configparser.NoSectionError, configparser.NoOptionError): print(fc.strcolor('\nMilter enabled with "thread" backend but "minfreethreads is not defined!"', 'yellow')) print("To keep fuglu-milter responsive it is recommended to set minfreethreads >= 1\n") if self.config.getboolean('main', 'versioncheck'): check_version_status(lint=True) return errors
def _real_init(self, backendconfig): self.session = get_session(backendconfig) metadata.create_all(bind=self.session.bind)
def lint(self): errors = 0 fc = FunkyConsole() self._lint_dependencies(fc) print(fc.strcolor('Loading extensions...', 'magenta')) exts = self.load_extensions() for ext in exts: (name, enabled, status) = ext pname = fc.strcolor(name, 'cyan') if enabled: penabled = fc.strcolor('enabled', 'green') else: penabled = fc.strcolor('disabled', 'red') print("%s: %s (%s)" % (pname, penabled, status)) print(fc.strcolor('Loading plugins...', 'magenta')) if not self.load_plugins(): print(fc.strcolor('At least one plugin failed to load', 'red')) print(fc.strcolor('Plugin loading complete', 'magenta')) print("Linting ", fc.strcolor("main configuration", 'cyan')) if not self.checkConfig(): print(fc.strcolor("ERROR", "red")) else: print(fc.strcolor("OK", "green")) trashdir = self.config.get('main', 'trashdir').strip() if trashdir != "" and not os.path.isdir(trashdir): print(fc.strcolor("Trashdir %s does not exist" % trashdir, 'red')) # sql config override sqlconfigdbconnectstring = self.config.get('databaseconfig', 'dbconnectstring') if sqlconfigdbconnectstring.strip() != '': print() print("Linting ", fc.strcolor("sql configuration", 'cyan')) try: from fuglu.extensions.sql import get_session sess = get_session(sqlconfigdbconnectstring) tempsuspect = Suspect('*****@*****.**', '*****@*****.**', '/dev/null') sqlvars = dict(section='testsection', option='testoption', scope='$GLOBAL') default_template_values(tempsuspect, sqlvars) sess.execute(self.config.get('databaseconfig', 'sql'), sqlvars) sess.remove() print(fc.strcolor("OK", 'green')) except Exception as e: print(fc.strcolor("Failed %s" % str(e), 'red')) allplugins = self.plugins + self.prependers + self.appenders for plugin in allplugins: print() print("Linting Plugin ", fc.strcolor(str(plugin), 'cyan'), 'Config section:', fc.strcolor(str(plugin.section), 'cyan')) try: result = plugin.lint() except Exception as e: CrashStore.store_exception() print("ERROR: %s" % e) result = False if result: print(fc.strcolor("OK", "green")) else: errors = errors + 1 print(fc.strcolor("ERROR", "red")) print("%s plugins reported errors." % errors) if self.config.getboolean('main', 'versioncheck'): check_version_status(lint=True)