def lint_sql(self): dbconn = '' if self.config.has_option(self.section, 'dbconnectstring'): dbconn = self.config.get(self.section, 'dbconnectstring') if dbconn.strip() != '': print("Reading per user/domain attachment rules from database") if not SQL_EXTENSION_ENABLED: print( "Fuglu SQL Extension not available, cannot load attachment rules from database" ) return False query = self.config.get(self.section, 'query') dbfile = DBFile(dbconn, query) try: dbfile.getContent({ 'scope': 'lint', 'checktype': FUATT_CHECKTYPE_FN }) except Exception as e: import traceback print( "Could not get attachment rules from database. Exception: %s" % str(e)) print(traceback.format_exc()) return False else: print( "No database configured. Using per user/domain file configuration from %s" % self.config.get(self.section, 'rulesdir')) return True
def lint_sql(self): dbconn = '' if self.config.has_option(self.section, 'dbconnectstring'): dbconn = self.config.get(self.section, 'dbconnectstring') if dbconn.strip() != '': print("Reading per user/domain attachment rules from database") if not fuglu.extensions.sql.ENABLED: print( "Fuglu SQL Extension not available, cannot load attachment rules from database") return False query = self.config.get(self.section, 'query') dbfile = DBFile(dbconn, query) try: dbfile.getContent( {'scope': 'lint', 'checktype': FUATT_CHECKTYPE_FN}) except Exception as e: import traceback print( "Could not get attachment rules from database. Exception: %s" % str(e)) print(traceback.format_exc()) return False else: print("No database configured. Using per user/domain file configuration from %s" % self.config.get(self.section, 'rulesdir')) return True
def lint_sql(self): dbconn = '' if self.config.has_option(self.section, 'dbconnectstring'): dbconn = self.config.get(self.section, 'dbconnectstring') if dbconn.strip() != '': print "Reading per user/domain attachment rules from database" if not fuglu.extensions.sql.ENABLED: print "Fuglu SQL Extension not available, cannot load attachment rules from database" return False query = self.config.get(self.section, 'query') dbfile = DBFile(dbconn, query) try: dbfile.getContent( {'scope': 'lint', 'checktype': FUATT_CHECKTYPE_FN}) except Exception, e: import traceback print "Could not get attachment rules from database. Exception: %s" % str(e) print traceback.format_exc() return False
def walk(self, suspect): """walks through a message and checks each attachment according to the rulefile specified in the config""" blockaction = self.config.get(self.section, 'blockaction') blockactioncode = string_to_actioncode(blockaction) # try db rules first self.rulescache.reloadifnecessary() dbconn = '' if self.config.has_option(self.section, 'dbconnectstring'): dbconn = self.config.get(self.section, 'dbconnectstring') if dbconn.strip() != '': self.logger.debug('Loading attachment rules from database') query = self.config.get(self.section, 'query') dbfile = DBFile(dbconn, query) user_names = self.rulescache.get_rules_from_config_lines( dbfile.getContent({ 'scope': suspect.to_address, 'checktype': FUATT_CHECKTYPE_FN })) user_ctypes = self.rulescache.get_rules_from_config_lines( dbfile.getContent({ 'scope': suspect.to_address, 'checktype': FUATT_CHECKTYPE_CT })) user_archive_names = self.rulescache.get_rules_from_config_lines( dbfile.getContent({ 'scope': suspect.to_address, 'checktype': FUATT_CHECKTYPE_ARCHIVE_FN })) user_archive_ctypes = self.rulescache.get_rules_from_config_lines( dbfile.getContent({ 'scope': suspect.to_address, 'checktype': FUATT_CHECKTYPE_ARCHIVE_CT })) self.logger.debug( 'Found %s filename rules, %s content-type rules, %s archive filename rules, %s archive content rules for address %s' % (len(user_names), len(user_ctypes), len(user_archive_names), len(user_archive_ctypes), suspect.to_address)) domain_names = self.rulescache.get_rules_from_config_lines( dbfile.getContent({ 'scope': suspect.to_domain, 'checktype': FUATT_CHECKTYPE_FN })) domain_ctypes = self.rulescache.get_rules_from_config_lines( dbfile.getContent({ 'scope': suspect.to_domain, 'checktype': FUATT_CHECKTYPE_CT })) domain_archive_names = self.rulescache.get_rules_from_config_lines( dbfile.getContent({ 'scope': suspect.to_domain, 'checktype': FUATT_CHECKTYPE_ARCHIVE_FN })) domain_archive_ctypes = self.rulescache.get_rules_from_config_lines( dbfile.getContent({ 'scope': suspect.to_domain, 'checktype': FUATT_CHECKTYPE_ARCHIVE_CT })) self.logger.debug( 'Found %s filename rules, %s content-type rules, %s archive filename rules, %s archive content rules for domain %s' % (len(domain_names), len(domain_ctypes), len(domain_archive_names), len(domain_archive_ctypes), suspect.to_domain)) else: self.logger.debug( 'Loading attachment rules from filesystem dir %s' % (self.config.get(self.section, 'rulesdir'))) user_names = self.rulescache.getNAMERules(suspect.to_address) user_ctypes = self.rulescache.getCTYPERules(suspect.to_address) user_archive_names = self.rulescache.getARCHIVENAMERules( suspect.to_address) user_archive_ctypes = self.rulescache.getARCHIVECTYPERules( suspect.to_address) domain_names = self.rulescache.getNAMERules(suspect.to_domain) domain_ctypes = self.rulescache.getCTYPERules(suspect.to_domain) domain_archive_names = self.rulescache.getARCHIVENAMERules( suspect.to_domain) domain_archive_ctypes = self.rulescache.getARCHIVECTYPERules( suspect.to_domain) # always get defaults from file default_names = self.rulescache.getNAMERules(FUATT_DEFAULT) default_ctypes = self.rulescache.getCTYPERules(FUATT_DEFAULT) default_archive_names = self.rulescache.getARCHIVENAMERules( FUATT_DEFAULT) default_archive_ctypes = self.rulescache.getARCHIVECTYPERules( FUATT_DEFAULT) m = suspect.get_message_rep() for part in self.walk_all_parts(m): if part.is_multipart(): continue contenttype_mime = part.get_content_type() att_name = part.get_filename(None) if att_name: # some filenames are encoded, try to decode try: att_name = ''.join([x[0] for x in decode_header(att_name)]) except Exception: pass else: # workaround for mimetypes, it always takes .ksh for text/plain if part.get_content_type() == 'text/plain': ext = '.txt' else: ext = mimetypes.guess_extension(part.get_content_type()) if ext is None: ext = '' att_name = 'unnamed%s' % ext att_name = self.asciionly(att_name) res = self.matchMultipleSets( [user_names, domain_names, default_names], att_name, suspect, att_name) if res == ATTACHMENT_SILENTDELETE: self._debuginfo( suspect, "Attachment name=%s SILENT DELETE : blocked by name" % att_name) return DELETE if res == ATTACHMENT_BLOCK: self._debuginfo( suspect, "Attachment name=%s : blocked by name)" % att_name) message = suspect.tags['FiletypePlugin.errormessage'] return blockactioncode, message # go through content type rules res = self.matchMultipleSets( [user_ctypes, domain_ctypes, default_ctypes], contenttype_mime, suspect, att_name) if res == ATTACHMENT_SILENTDELETE: self._debuginfo( suspect, "Attachment name=%s content-type=%s SILENT DELETE: blocked by mime content type (message source)" % (att_name, contenttype_mime)) return DELETE if res == ATTACHMENT_BLOCK: self._debuginfo( suspect, "Attachment name=%s content-type=%s : blocked by mime content type (message source)" % (att_name, contenttype_mime)) message = suspect.tags['FiletypePlugin.errormessage'] return blockactioncode, message contenttype_magic = None if MAGIC_AVAILABLE: pl = part.get_payload(decode=True) contenttype_magic = self.getBuffertype(pl) res = self.matchMultipleSets( [user_ctypes, domain_ctypes, default_ctypes], contenttype_magic, suspect, att_name) if res == ATTACHMENT_SILENTDELETE: self._debuginfo( suspect, "Attachment name=%s content-type=%s SILENT DELETE: blocked by mime content type (magic)" % (att_name, contenttype_mime)) return DELETE if res == ATTACHMENT_BLOCK: self._debuginfo( suspect, "Attachment name=%s content-type=%s : blocked by mime content type (magic)" % (att_name, contenttype_mime)) message = suspect.tags['FiletypePlugin.errormessage'] return blockactioncode, message # archives if self.checkarchivenames or self.checkarchivecontent: # try guessing the archive type based on magic content type first # we don't need to check for MAGIC_AVAILABLE here, if it is available the contenttype_magic is not None archive_type = self.archive_type_from_content_type( contenttype_magic) # if it didn't work, try to guess by the filename extension, if it is enabled if archive_type is None: # sort by length, so tar.gz is checked before .gz for arext in sorted( self.supported_archive_extensions.keys(), key=lambda x: len(x), reverse=True): if att_name.lower().endswith('.%s' % arext): archive_type = self.supported_archive_extensions[ arext] break if archive_type is not None: self.logger.debug( "Extracting {attname} as {artype}".format( attname=att_name, artype=archive_type)) try: pl = BytesIO(part.get_payload(decode=True)) archive_handle = self._archive_handle(archive_type, pl) namelist = self._archive_namelist( archive_type, archive_handle) if self.checkarchivenames: for name in namelist: # rarfile returns unicode objects which mess up # generated bounces if sys.version_info[0] == 2: # Py3 defaults to unicode name = self.asciionly(name) res = self.matchMultipleSets([ user_archive_names, domain_archive_names, default_archive_names ], name, suspect, name) if res == ATTACHMENT_SILENTDELETE: self._debuginfo( suspect, "Blocked filename in archive %s SILENT DELETE" % att_name) return DELETE if res == ATTACHMENT_BLOCK: self._debuginfo( suspect, "Blocked filename in archive %s" % att_name) message = suspect.tags[ 'FiletypePlugin.errormessage'] return blockactioncode, message if MAGIC_AVAILABLE and self.checkarchivecontent: for name in namelist: safename = self.asciionly(name) extracted = self._archive_extract( archive_type, archive_handle, name) if extracted is None: self._debuginfo( suspect, '%s not extracted - too large' % (safename)) contenttype_magic = self.getBuffertype( extracted) res = self.matchMultipleSets([ user_archive_ctypes, domain_archive_ctypes, default_archive_ctypes ], contenttype_magic, suspect, name) if res == ATTACHMENT_SILENTDELETE: self._debuginfo( suspect, "Extracted file %s from archive %s content-type=%s SILENT DELETE: blocked by mime content type (magic)" % (safename, att_name, contenttype_magic)) return DELETE if res == ATTACHMENT_BLOCK: self._debuginfo( suspect, "Extracted file %s from archive %s content-type=%s : blocked by mime content type (magic)" % (safename, att_name, contenttype_magic)) message = suspect.tags[ 'FiletypePlugin.errormessage'] return blockactioncode, message if hasattr(archive_handle, 'close'): archive_handle.close() pl.close() except Exception: self.logger.warning( "archive scanning failed in attachment {attname}: {error}" .format(attname=att_name, error=traceback.format_exc())) return DUNNO
def walk(self, suspect): """walks through a message and checks each attachment according to the rulefile specified in the config""" blockaction = self.config.get(self.section, 'blockaction') blockactioncode = string_to_actioncode(blockaction) # try db rules first self.rulescache.reloadifnecessary() dbconn = '' if self.config.has_option(self.section, 'dbconnectstring'): dbconn = self.config.get(self.section, 'dbconnectstring') if dbconn.strip() != '': self.logger.debug('Loading attachment rules from database') query = self.config.get(self.section, 'query') dbfile = DBFile(dbconn, query) user_names = self.rulescache.get_rules_from_config_lines( dbfile.getContent({'scope': suspect.to_address, 'checktype': FUATT_CHECKTYPE_FN})) user_ctypes = self.rulescache.get_rules_from_config_lines( dbfile.getContent({'scope': suspect.to_address, 'checktype': FUATT_CHECKTYPE_CT})) user_archive_names = self.rulescache.get_rules_from_config_lines( dbfile.getContent({'scope': suspect.to_address, 'checktype': FUATT_CHECKTYPE_ARCHIVE_FN})) user_archive_ctypes = self.rulescache.get_rules_from_config_lines( dbfile.getContent({'scope': suspect.to_address, 'checktype': FUATT_CHECKTYPE_ARCHIVE_CT})) self.logger.debug('Found %s filename rules, %s content-type rules, %s archive filename rules, %s archive content rules for address %s' % (len(user_names), len(user_ctypes), len(user_archive_names), len(user_archive_ctypes), suspect.to_address)) domain_names = self.rulescache.get_rules_from_config_lines( dbfile.getContent({'scope': suspect.to_domain, 'checktype': FUATT_CHECKTYPE_FN})) domain_ctypes = self.rulescache.get_rules_from_config_lines( dbfile.getContent({'scope': suspect.to_domain, 'checktype': FUATT_CHECKTYPE_CT})) domain_archive_names = self.rulescache.get_rules_from_config_lines( dbfile.getContent({'scope': suspect.to_domain, 'checktype': FUATT_CHECKTYPE_ARCHIVE_FN})) domain_archive_ctypes = self.rulescache.get_rules_from_config_lines( dbfile.getContent({'scope': suspect.to_domain, 'checktype': FUATT_CHECKTYPE_ARCHIVE_CT})) self.logger.debug('Found %s filename rules, %s content-type rules, %s archive filename rules, %s archive content rules for domain %s' % (len(domain_names), len(domain_ctypes), len(domain_archive_names), len(domain_archive_ctypes), suspect.to_domain)) else: self.logger.debug('Loading attachment rules from filesystem') user_names = self.rulescache.getNAMERules(suspect.to_address) user_ctypes = self.rulescache.getCTYPERules(suspect.to_address) user_archive_names = self.rulescache.getARCHIVENAMERules( suspect.to_address) user_archive_ctypes = self.rulescache.getARCHIVECTYPERules( suspect.to_address) domain_names = self.rulescache.getNAMERules(suspect.to_domain) domain_ctypes = self.rulescache.getCTYPERules(suspect.to_domain) domain_archive_names = self.rulescache.getARCHIVENAMERules( suspect.to_domain) domain_archive_ctypes = self.rulescache.getARCHIVECTYPERules( suspect.to_domain) # always get defaults from file default_names = self.rulescache.getNAMERules(FUATT_DEFAULT) default_ctypes = self.rulescache.getCTYPERules(FUATT_DEFAULT) default_archive_names = self.rulescache.getARCHIVENAMERules( FUATT_DEFAULT) default_archive_ctypes = self.rulescache.getARCHIVECTYPERules( FUATT_DEFAULT) m = suspect.get_message_rep() for i in m.walk(): if i.is_multipart(): continue contenttype_mime = i.get_content_type() att_name = i.get_filename(None) if att_name: # some filenames are encoded, try to decode try: att_name = ''.join([x[0] for x in decode_header(att_name)]) except: pass else: # workaround for mimetypes, it always takes .ksh for text/plain if i.get_content_type() == 'text/plain': ext = '.txt' else: ext = mimetypes.guess_extension(i.get_content_type()) if ext == None: ext = '' att_name = 'unnamed%s' % ext res = self.matchMultipleSets( [user_names, domain_names, default_names], att_name, suspect, att_name) if res == ATTACHMENT_SILENTDELETE: self._debuginfo( suspect, "Attachment name=%s SILENT DELETE : blocked by name" % att_name) return DELETE if res == ATTACHMENT_BLOCK: self._debuginfo( suspect, "Attachment name=%s : blocked by name)" % att_name) message = suspect.tags['FiletypePlugin.errormessage'] return blockactioncode, message # go through content type rules res = self.matchMultipleSets( [user_ctypes, domain_ctypes, default_ctypes], contenttype_mime, suspect, att_name) if res == ATTACHMENT_SILENTDELETE: self._debuginfo( suspect, "Attachment name=%s content-type=%s SILENT DELETE: blocked by mime content type (message source)" % (att_name, contenttype_mime)) return DELETE if res == ATTACHMENT_BLOCK: self._debuginfo( suspect, "Attachment name=%s content-type=%s : blocked by mime content type (message source)" % (att_name, contenttype_mime)) message = suspect.tags['FiletypePlugin.errormessage'] return blockactioncode, message if MAGIC_AVAILABLE: pl = i.get_payload(decode=True) contenttype_magic = self.getBuffertype(pl) res = self.matchMultipleSets( [user_ctypes, domain_ctypes, default_ctypes], contenttype_magic, suspect, att_name) if res == ATTACHMENT_SILENTDELETE: self._debuginfo( suspect, "Attachment name=%s content-type=%s SILENT DELETE: blocked by mime content type (magic)" % (att_name, contenttype_mime)) return DELETE if res == ATTACHMENT_BLOCK: self._debuginfo( suspect, "Attachment name=%s content-type=%s : blocked by mime content type (magic)" % (att_name, contenttype_mime)) message = suspect.tags['FiletypePlugin.errormessage'] return blockactioncode, message # archives if self.config.getboolean(self.section, 'checkarchivenames') or self.config.getboolean(self.section, 'checkarchivecontent'): archive_type = None for arext in self.supported_archive_extensions: if att_name.lower().endswith('.%s' % arext): archive_type = arext break if archive_type != None: try: pl = StringIO(i.get_payload(decode=True)) archive_handle = self._archive_handle(archive_type, pl) namelist = self._archive_namelist( archive_type, archive_handle) if self.config.getboolean(self.section, 'checkarchivenames'): for name in namelist: # rarfile returns unicode objects which mess up # generated bounces if type(name) == unicode: name = name.encode("utf-8", "ignore") res = self.matchMultipleSets( [user_archive_names, domain_archive_names, default_archive_names], name, suspect, name) if res == ATTACHMENT_SILENTDELETE: self._debuginfo( suspect, "Blocked filename in archive %s SILENT DELETE" % att_name) return DELETE if res == ATTACHMENT_BLOCK: self._debuginfo( suspect, "Blocked filename in archive %s" % att_name) message = suspect.tags[ 'FiletypePlugin.errormessage'] return blockactioncode, message if MAGIC_AVAILABLE and self.config.getboolean(self.section, 'checkarchivecontent'): for name in namelist: safename = self.asciionly(name) extracted = self._archive_extract( archive_type, archive_handle, name) if extracted == None: self._debuginfo( suspect, '%s not extracted - too large' % (safename)) contenttype_magic = self.getBuffertype( extracted) res = self.matchMultipleSets( [user_archive_ctypes, domain_archive_ctypes, default_archive_ctypes], contenttype_magic, suspect, name) if res == ATTACHMENT_SILENTDELETE: self._debuginfo( suspect, "Extracted file %s from archive %s content-type=%s SILENT DELETE: blocked by mime content type (magic)" % (safename, att_name, contenttype_magic)) return DELETE if res == ATTACHMENT_BLOCK: self._debuginfo( suspect, "Extracted file %s from archive %s content-type=%s : blocked by mime content type (magic)" % (safename, att_name, contenttype_magic)) message = suspect.tags[ 'FiletypePlugin.errormessage'] return blockactioncode, message except Exception as e: self.logger.warning( "archive scanning failed in attachment %s: %s" % (att_name, str(e))) return DUNNO
def walk(self, suspect): """walks through a message and checks each attachment according to the rulefile specified in the config""" blockaction = self.config.get(self.section, 'blockaction') blockactioncode = string_to_actioncode(blockaction) # try db rules first self.rulescache.reloadifnecessary() dbconn = '' if self.config.has_option(self.section, 'dbconnectstring'): dbconn = self.config.get(self.section, 'dbconnectstring') if dbconn.strip() != '': self.logger.debug('Loading attachment rules from database') query = self.config.get(self.section, 'query') dbfile = DBFile(dbconn, query) user_names = self.rulescache.get_rules_from_config_lines( dbfile.getContent({'scope': suspect.to_address, 'checktype': FUATT_CHECKTYPE_FN})) user_ctypes = self.rulescache.get_rules_from_config_lines( dbfile.getContent({'scope': suspect.to_address, 'checktype': FUATT_CHECKTYPE_CT})) user_archive_names = self.rulescache.get_rules_from_config_lines( dbfile.getContent({'scope': suspect.to_address, 'checktype': FUATT_CHECKTYPE_ARCHIVE_FN})) user_archive_ctypes = self.rulescache.get_rules_from_config_lines( dbfile.getContent({'scope': suspect.to_address, 'checktype': FUATT_CHECKTYPE_ARCHIVE_CT})) self.logger.debug('Found %s filename rules, %s content-type rules, %s archive filename rules, %s archive content rules for address %s' % (len(user_names), len(user_ctypes), len(user_archive_names), len(user_archive_ctypes), suspect.to_address)) domain_names = self.rulescache.get_rules_from_config_lines( dbfile.getContent({'scope': suspect.to_domain, 'checktype': FUATT_CHECKTYPE_FN})) domain_ctypes = self.rulescache.get_rules_from_config_lines( dbfile.getContent({'scope': suspect.to_domain, 'checktype': FUATT_CHECKTYPE_CT})) domain_archive_names = self.rulescache.get_rules_from_config_lines( dbfile.getContent({'scope': suspect.to_domain, 'checktype': FUATT_CHECKTYPE_ARCHIVE_FN})) domain_archive_ctypes = self.rulescache.get_rules_from_config_lines( dbfile.getContent({'scope': suspect.to_domain, 'checktype': FUATT_CHECKTYPE_ARCHIVE_CT})) self.logger.debug('Found %s filename rules, %s content-type rules, %s archive filename rules, %s archive content rules for domain %s' % (len(domain_names), len(domain_ctypes), len(domain_archive_names), len(domain_archive_ctypes), suspect.to_domain)) else: self.logger.debug('Loading attachment rules from filesystem') user_names = self.rulescache.getNAMERules(suspect.to_address) user_ctypes = self.rulescache.getCTYPERules(suspect.to_address) user_archive_names = self.rulescache.getARCHIVENAMERules( suspect.to_address) user_archive_ctypes = self.rulescache.getARCHIVECTYPERules( suspect.to_address) domain_names = self.rulescache.getNAMERules(suspect.to_domain) domain_ctypes = self.rulescache.getCTYPERules(suspect.to_domain) domain_archive_names = self.rulescache.getARCHIVENAMERules( suspect.to_domain) domain_archive_ctypes = self.rulescache.getARCHIVECTYPERules( suspect.to_domain) # always get defaults from file default_names = self.rulescache.getNAMERules(FUATT_DEFAULT) default_ctypes = self.rulescache.getCTYPERules(FUATT_DEFAULT) default_archive_names = self.rulescache.getARCHIVENAMERules( FUATT_DEFAULT) default_archive_ctypes = self.rulescache.getARCHIVECTYPERules( FUATT_DEFAULT) m = suspect.get_message_rep() for i in m.walk(): if i.is_multipart(): continue contenttype_mime = i.get_content_type() att_name = i.get_filename(None) if not att_name: # workaround for mimetypes, it always takes .ksh for text/plain if i.get_content_type() == 'text/plain': ext = '.txt' else: ext = mimetypes.guess_extension(i.get_content_type()) if ext == None: ext = '' att_name = 'unnamed%s' % ext res = self.matchMultipleSets( [user_names, domain_names, default_names], att_name, suspect, att_name) if res == ATTACHMENT_SILENTDELETE: self._debuginfo( suspect, "Attachment name=%s SILENT DELETE : blocked by name" % att_name) return DELETE if res == ATTACHMENT_BLOCK: self._debuginfo( suspect, "Attachment name=%s : blocked by name)" % att_name) message = suspect.tags['FiletypePlugin.errormessage'] return blockactioncode, message # go through content type rules res = self.matchMultipleSets( [user_ctypes, domain_ctypes, default_ctypes], contenttype_mime, suspect, att_name) if res == ATTACHMENT_SILENTDELETE: self._debuginfo( suspect, "Attachment name=%s content-type=%s SILENT DELETE: blocked by mime content type (message source)" % (att_name, contenttype_mime)) return DELETE if res == ATTACHMENT_BLOCK: self._debuginfo( suspect, "Attachment name=%s content-type=%s : blocked by mime content type (message source)" % (att_name, contenttype_mime)) message = suspect.tags['FiletypePlugin.errormessage'] return blockactioncode, message if MAGIC_AVAILABLE: pl = i.get_payload(decode=True) contenttype_magic = self.getBuffertype(pl) res = self.matchMultipleSets( [user_ctypes, domain_ctypes, default_ctypes], contenttype_magic, suspect, att_name) if res == ATTACHMENT_SILENTDELETE: self._debuginfo( suspect, "Attachment name=%s content-type=%s SILENT DELETE: blocked by mime content type (magic)" % (att_name, contenttype_mime)) return DELETE if res == ATTACHMENT_BLOCK: self._debuginfo( suspect, "Attachment name=%s content-type=%s : blocked by mime content type (magic)" % (att_name, contenttype_mime)) message = suspect.tags['FiletypePlugin.errormessage'] return blockactioncode, message # archives if self.config.getboolean(self.section, 'checkarchivenames') or self.config.getboolean(self.section, 'checkarchivecontent'): archive_type=None for arext in self.supported_archive_extensions: if att_name.lower().endswith('.%s'%arext): archive_type=arext break if archive_type!=None: try: pl = StringIO(i.get_payload(decode=True)) archive_handle = self._archive_handle(archive_type,pl) namelist = self._archive_namelist(archive_type,archive_handle) if self.config.getboolean(self.section, 'checkarchivenames'): for name in namelist: if type(name)==unicode: #rarfile returns unicode objects which mess up generated bounces name=name.encode("utf-8","ignore") res = self.matchMultipleSets( [user_archive_names, domain_archive_names, default_archive_names], name, suspect, name) if res == ATTACHMENT_SILENTDELETE: self._debuginfo( suspect, "Blocked filename in archive %s SILENT DELETE" % att_name) return DELETE if res == ATTACHMENT_BLOCK: self._debuginfo( suspect, "Blocked filename in archive %s" % att_name) message = suspect.tags[ 'FiletypePlugin.errormessage'] return blockactioncode, message if MAGIC_AVAILABLE and self.config.getboolean(self.section, 'checkarchivecontent'): for name in namelist: safename = self.asciionly(name) extracted = self._archive_extract(archive_type,archive_handle,name) if extracted==None: self._debuginfo(suspect,'%s not extracted - too large'%(safename)) contenttype_magic = self.getBuffertype( extracted) res = self.matchMultipleSets( [user_archive_ctypes, domain_archive_ctypes, default_archive_ctypes], contenttype_magic, suspect, name) if res == ATTACHMENT_SILENTDELETE: self._debuginfo( suspect, "Extracted file %s from archive %s content-type=%s SILENT DELETE: blocked by mime content type (magic)" % (safename, att_name, contenttype_magic)) return DELETE if res == ATTACHMENT_BLOCK: self._debuginfo( suspect, "Extracted file %s from archive %s content-type=%s : blocked by mime content type (magic)" % (safename, att_name, contenttype_magic)) message = suspect.tags[ 'FiletypePlugin.errormessage'] return blockactioncode, message except Exception, e: self.logger.warning( "archive scanning failed in attachment %s: %s" % (att_name, str(e)))
def walk(self, suspect): """walks through a message and checks each attachment according to the rulefile specified in the config""" blockaction = self.config.get(self.section, 'blockaction') blockactioncode = string_to_actioncode(blockaction) # try db rules first self.rulescache.reloadifnecessary() dbconn = '' if self.config.has_option(self.section, 'dbconnectstring'): dbconn = self.config.get(self.section, 'dbconnectstring') if dbconn.strip() != '': self.logger.debug('%s Loading attachment rules from database' % suspect.id) query = self.config.get(self.section, 'query') dbfile = DBFile(dbconn, query) user_names = self.rulescache.get_rules_from_config_lines( dbfile.getContent({ 'scope': suspect.to_address, 'checktype': FUATT_CHECKTYPE_FN })) user_ctypes = self.rulescache.get_rules_from_config_lines( dbfile.getContent({ 'scope': suspect.to_address, 'checktype': FUATT_CHECKTYPE_CT })) user_archive_names = self.rulescache.get_rules_from_config_lines( dbfile.getContent({ 'scope': suspect.to_address, 'checktype': FUATT_CHECKTYPE_ARCHIVE_FN })) user_archive_ctypes = self.rulescache.get_rules_from_config_lines( dbfile.getContent({ 'scope': suspect.to_address, 'checktype': FUATT_CHECKTYPE_ARCHIVE_CT })) self.logger.debug( '%s Found %s filename rules, %s content-type rules, %s archive filename rules, %s archive content rules for address %s' % (suspect.id, len(user_names), len(user_ctypes), len(user_archive_names), len(user_archive_ctypes), suspect.to_address)) domain_names = self.rulescache.get_rules_from_config_lines( dbfile.getContent({ 'scope': suspect.to_domain, 'checktype': FUATT_CHECKTYPE_FN })) domain_ctypes = self.rulescache.get_rules_from_config_lines( dbfile.getContent({ 'scope': suspect.to_domain, 'checktype': FUATT_CHECKTYPE_CT })) domain_archive_names = self.rulescache.get_rules_from_config_lines( dbfile.getContent({ 'scope': suspect.to_domain, 'checktype': FUATT_CHECKTYPE_ARCHIVE_FN })) domain_archive_ctypes = self.rulescache.get_rules_from_config_lines( dbfile.getContent({ 'scope': suspect.to_domain, 'checktype': FUATT_CHECKTYPE_ARCHIVE_CT })) self.logger.debug( '%s Found %s filename rules, %s content-type rules, %s archive filename rules, %s archive content rules for domain %s' % (suspect.id, len(domain_names), len(domain_ctypes), len(domain_archive_names), len(domain_archive_ctypes), suspect.to_domain)) else: self.logger.debug( '%s Loading attachment rules from filesystem dir %s' % (suspect.id, self.config.get(self.section, 'rulesdir'))) user_names = self.rulescache.getNAMERules(suspect.to_address) user_ctypes = self.rulescache.getCTYPERules(suspect.to_address) user_archive_names = self.rulescache.getARCHIVENAMERules( suspect.to_address) user_archive_ctypes = self.rulescache.getARCHIVECTYPERules( suspect.to_address) domain_names = self.rulescache.getNAMERules(suspect.to_domain) domain_ctypes = self.rulescache.getCTYPERules(suspect.to_domain) domain_archive_names = self.rulescache.getARCHIVENAMERules( suspect.to_domain) domain_archive_ctypes = self.rulescache.getARCHIVECTYPERules( suspect.to_domain) # always get defaults from file default_names = self.rulescache.getNAMERules(FUATT_DEFAULT) default_ctypes = self.rulescache.getCTYPERules(FUATT_DEFAULT) default_archive_names = self.rulescache.getARCHIVENAMERules( FUATT_DEFAULT) default_archive_ctypes = self.rulescache.getARCHIVECTYPERules( FUATT_DEFAULT) # get mail attachment objects (only directly attached objects) for attObj in suspect.att_mgr.get_objectlist(): contenttype_mime = attObj.contenttype_mime att_name = attObj.filename if attObj.is_inline or attObj.is_attachment or not attObj.filename_generated: # process all attachments marked as "inline", "attachment" or parts # with filenames that are not auto-generated pass else: self.logger.debug( "%s Skip message object: %s (attachment: %s, inline: %s, auto-name: %s)" % (suspect.id, att_name, attObj.is_attachment, attObj.is_inline, attObj.filename_generated)) continue att_name = self.asciionly(att_name) res = self.matchMultipleSets( [user_names, domain_names, default_names], att_name, suspect, att_name) if res == ATTACHMENT_SILENTDELETE: self._debuginfo( suspect, "Attachment name=%s SILENT DELETE : blocked by name" % att_name) return DELETE if res == ATTACHMENT_BLOCK: self._debuginfo( suspect, "Attachment name=%s : blocked by name)" % att_name) message = suspect.tags['FiletypePlugin.errormessage'] return blockactioncode, message # go through content type rules res = self.matchMultipleSets( [user_ctypes, domain_ctypes, default_ctypes], contenttype_mime, suspect, att_name) if res == ATTACHMENT_SILENTDELETE: self._debuginfo( suspect, "Attachment name=%s content-type=%s SILENT DELETE: blocked by mime content type (message source)" % (att_name, contenttype_mime)) return DELETE if res == ATTACHMENT_BLOCK: self._debuginfo( suspect, "Attachment name=%s content-type=%s : blocked by mime content type (message source)" % (att_name, contenttype_mime)) message = suspect.tags['FiletypePlugin.errormessage'] return blockactioncode, message contenttype_magic = attObj.contenttype if contenttype_magic is not None: res = self.matchMultipleSets( [user_ctypes, domain_ctypes, default_ctypes], contenttype_magic, suspect, att_name) if res == ATTACHMENT_SILENTDELETE: self._debuginfo( suspect, "Attachment name=%s content-type=%s SILENT DELETE: blocked by mime content type (magic)" % (att_name, contenttype_magic)) return DELETE if res == ATTACHMENT_BLOCK: self._debuginfo( suspect, "Attachment name=%s content-type=%s : blocked by mime content type (magic)" % (att_name, contenttype_magic)) message = suspect.tags['FiletypePlugin.errormessage'] return blockactioncode, message # archives if self.checkarchivenames or self.checkarchivecontent: #if archive_type is not None: if attObj.is_archive: # check if extension was used to determine archive type and # if yes, check if extension is enabled. This code # is here to remain backward compatible in the behavior. It # is recommended to define inactive archive-types and -extensions # differently if attObj.atype_fromext() is not None: if not attObj.atype_fromext( ) in self.active_archive_extensions.keys(): # skip if extension is not in active list continue self.logger.debug( "%s Extracting %s as %s" % (suspect.id, att_name, attObj.archive_type)) archivecontentmaxsize = self.config.getint( self.section, 'archivecontentmaxsize') try: archiveextractlevel = self.config.getint( self.section, 'archiveextractlevel') if archiveextractlevel < 0: # value must be greater or equals 0 archiveextractlevel = None except Exception: archiveextractlevel = None try: if self.checkarchivenames: if self.checkarchivecontent: namelist = attObj.get_fileslist( 0, archiveextractlevel, archivecontentmaxsize) else: namelist = attObj.fileslist_archive for name in namelist: res = self.matchMultipleSets([ user_archive_names, domain_archive_names, default_archive_names ], name, suspect, name) if res == ATTACHMENT_SILENTDELETE: self._debuginfo( suspect, "Blocked filename in archive %s SILENT DELETE" % att_name) return DELETE if res == ATTACHMENT_BLOCK: self._debuginfo( suspect, "Blocked filename in archive %s" % att_name) message = suspect.tags[ 'FiletypePlugin.errormessage'] return blockactioncode, message if filetype_handler.available( ) and self.checkarchivecontent: nocheckinfo = NoExtractInfo() for archObj in attObj.get_objectlist( 0, archiveextractlevel, archivecontentmaxsize, noextractinfo=nocheckinfo): safename = self.asciionly(archObj.filename) contenttype_magic = archObj.contenttype # Keeping this check for backward compatibility # This could easily be removed since memory is used anyway if archivecontentmaxsize is not None and archObj.filesize > archivecontentmaxsize: nocheckinfo.append( archObj.filename, u"toolarge", u"already extracted but too large for check: %u > %u" % (archObj.filesize, archivecontentmaxsize)) continue res = self.matchMultipleSets([ user_archive_ctypes, domain_archive_ctypes, default_archive_ctypes ], contenttype_magic, suspect, safename) if res == ATTACHMENT_SILENTDELETE: self._debuginfo( suspect, "Extracted file %s from archive %s content-type=%s " "SILENT DELETE: blocked by mime content type (magic)" % (safename, att_name, contenttype_magic)) return DELETE if res == ATTACHMENT_BLOCK: self._debuginfo( suspect, "Extracted file %s from archive %s content-type=%s : " "blocked by mime content type (magic)" % (safename, att_name, contenttype_magic)) message = suspect.tags[ 'FiletypePlugin.errormessage'] return blockactioncode, message for item in nocheckinfo.get_filtered(): try: self._debuginfo( suspect, 'Archive File not checked: reason: %s -> %s' % (item[0], item[1])) except Exception as e: self._debuginfo( suspect, 'Archive File not checked: %s' % str(e)) except Exception as e: self.logger.error( "%s archive scanning failed in attachment %s: %s" % (suspect.id, att_name, str(e))) return DUNNO
def walk(self,suspect): """walks through a message and checks each attachment according to the rulefile specified in the config""" blockaction=self.config.get(self.section,'blockaction') blockactioncode=string_to_actioncode(blockaction) #try db rules first self.rulescache.reloadifnecessary() dbconn='' if self.config.has_option(self.section,'dbconnectstring'): dbconn=self.config.get(self.section,'dbconnectstring') if dbconn.strip()!='': self.logger.debug('Loading attachment rules from database') query=self.config.get(self.section,'query') dbfile=DBFile(dbconn, query) user_names=self.rulescache.get_rules_from_config_lines(dbfile.getContent({'scope':suspect.to_address,'checktype':FUATT_CHECKTYPE_FN})) user_ctypes=self.rulescache.get_rules_from_config_lines(dbfile.getContent({'scope':suspect.to_address,'checktype':FUATT_CHECKTYPE_CT})) self.logger.debug('Found %s filename rules, %s content-type rules for address %s'%(len(user_names),len(user_ctypes),suspect.to_address)) domain_names=self.rulescache.get_rules_from_config_lines(dbfile.getContent({'scope':suspect.to_domain,'checktype':FUATT_CHECKTYPE_FN})) domain_ctypes=self.rulescache.get_rules_from_config_lines(dbfile.getContent({'scope':suspect.to_domain,'checktype':FUATT_CHECKTYPE_CT})) self.logger.debug('Found %s filename rules, %s content-type rules for domain %s'%(len(domain_names),len(domain_ctypes),suspect.to_domain)) else: self.logger.debug('Loading attachment rules from filesystem') user_names=self.rulescache.getNAMERules(suspect.to_address) user_ctypes=self.rulescache.getCTYPERules(suspect.to_address) domain_names=self.rulescache.getNAMERules(suspect.to_domain) domain_ctypes=self.rulescache.getCTYPERules(suspect.to_domain) #always get defaults from file default_names=self.rulescache.getNAMERules(FUATT_DEFAULT) default_ctypes=self.rulescache.getCTYPERules(FUATT_DEFAULT) m=suspect.get_message_rep() for i in m.walk(): if i.is_multipart(): continue contenttype_mime=i.get_content_type() att_name = i.get_filename(None) if not att_name: #workaround for mimetypes, it always takes .ksh for text/plain if i.get_content_type()=='text/plain': ext='.txt' else: ext = mimetypes.guess_extension(i.get_content_type()) if ext==None: ext='' att_name = 'unnamed%s' % ext res=self.matchMultipleSets([user_names,domain_names,default_names], att_name,suspect,att_name) if res==ATTACHMENT_SILENTDELETE: self._debuginfo(suspect,"Attachment name=%s SILENT DELETE : blocked by name"%att_name) return DELETE if res==ATTACHMENT_BLOCK: self._debuginfo(suspect,"Attachment name=%s : blocked by name)"%att_name) message=suspect.tags['FiletypePlugin.errormessage'] return blockactioncode,message #go through content type rules res=self.matchMultipleSets([user_ctypes,domain_ctypes,default_ctypes], contenttype_mime,suspect,att_name) if res==ATTACHMENT_SILENTDELETE: self._debuginfo(suspect,"Attachment name=%s content-type=%s SILENT DELETE: blocked by mime content type (message source)"%(att_name,contenttype_mime)) return DELETE if res==ATTACHMENT_BLOCK: self._debuginfo(suspect,"Attachment name=%s content-type=%s : blocked by mime content type (message source)"%(att_name,contenttype_mime)) message=suspect.tags['FiletypePlugin.errormessage'] return blockactioncode,message if MAGIC_AVAILABLE: pl = i.get_payload(decode=True) contenttype_magic=self.getBuffertype(pl) res=self.matchMultipleSets([user_ctypes,domain_ctypes,default_ctypes], contenttype_magic,suspect,att_name) if res==ATTACHMENT_SILENTDELETE: self._debuginfo(suspect,"Attachment name=%s content-type=%s SILENT DELETE: blocked by mime content type (magic)"%(att_name,contenttype_mime)) return DELETE if res==ATTACHMENT_BLOCK: self._debuginfo(suspect,"Attachment name=%s content-type=%s : blocked by mime content type (magic)"%(att_name,contenttype_mime)) message=suspect.tags['FiletypePlugin.errormessage'] return blockactioncode,message return DUNNO
def walk(self, suspect): """walks through a message and checks each attachment according to the rulefile specified in the config""" blockaction = self.config.get(self.section, 'blockaction') blockactioncode = string_to_actioncode(blockaction) # try db rules first self.rulescache.reloadifnecessary() dbconn = '' if self.config.has_option(self.section, 'dbconnectstring'): dbconn = self.config.get(self.section, 'dbconnectstring') if dbconn.strip() != '': self.logger.debug('Loading attachment rules from database') query = self.config.get(self.section, 'query') dbfile = DBFile(dbconn, query) user_names = self.rulescache.get_rules_from_config_lines( dbfile.getContent({'scope': suspect.to_address, 'checktype': FUATT_CHECKTYPE_FN})) user_ctypes = self.rulescache.get_rules_from_config_lines( dbfile.getContent({'scope': suspect.to_address, 'checktype': FUATT_CHECKTYPE_CT})) user_archive_names = self.rulescache.get_rules_from_config_lines( dbfile.getContent({'scope': suspect.to_address, 'checktype': FUATT_CHECKTYPE_ARCHIVE_FN})) user_archive_ctypes = self.rulescache.get_rules_from_config_lines( dbfile.getContent({'scope': suspect.to_address, 'checktype': FUATT_CHECKTYPE_ARCHIVE_CT})) self.logger.debug('Found %s filename rules, %s content-type rules, %s archive filename rules, %s archive content rules for address %s' % (len(user_names), len(user_ctypes), len(user_archive_names), len(user_archive_ctypes), suspect.to_address)) domain_names = self.rulescache.get_rules_from_config_lines( dbfile.getContent({'scope': suspect.to_domain, 'checktype': FUATT_CHECKTYPE_FN})) domain_ctypes = self.rulescache.get_rules_from_config_lines( dbfile.getContent({'scope': suspect.to_domain, 'checktype': FUATT_CHECKTYPE_CT})) domain_archive_names = self.rulescache.get_rules_from_config_lines( dbfile.getContent({'scope': suspect.to_domain, 'checktype': FUATT_CHECKTYPE_ARCHIVE_FN})) domain_archive_ctypes = self.rulescache.get_rules_from_config_lines( dbfile.getContent({'scope': suspect.to_domain, 'checktype': FUATT_CHECKTYPE_ARCHIVE_CT})) self.logger.debug('Found %s filename rules, %s content-type rules, %s archive filename rules, %s archive content rules for domain %s' % (len(domain_names), len(domain_ctypes), len(domain_archive_names), len(domain_archive_ctypes), suspect.to_domain)) else: self.logger.debug('Loading attachment rules from filesystem dir %s'%(self.config.get(self.section,'rulesdir'))) user_names = self.rulescache.getNAMERules(suspect.to_address) user_ctypes = self.rulescache.getCTYPERules(suspect.to_address) user_archive_names = self.rulescache.getARCHIVENAMERules( suspect.to_address) user_archive_ctypes = self.rulescache.getARCHIVECTYPERules( suspect.to_address) domain_names = self.rulescache.getNAMERules(suspect.to_domain) domain_ctypes = self.rulescache.getCTYPERules(suspect.to_domain) domain_archive_names = self.rulescache.getARCHIVENAMERules( suspect.to_domain) domain_archive_ctypes = self.rulescache.getARCHIVECTYPERules( suspect.to_domain) # always get defaults from file default_names = self.rulescache.getNAMERules(FUATT_DEFAULT) default_ctypes = self.rulescache.getCTYPERules(FUATT_DEFAULT) default_archive_names = self.rulescache.getARCHIVENAMERules( FUATT_DEFAULT) default_archive_ctypes = self.rulescache.getARCHIVECTYPERules( FUATT_DEFAULT) m = suspect.get_message_rep() for part in self.walk_all_parts(m): if part.is_multipart(): continue contenttype_mime = part.get_content_type() att_name = part.get_filename(None) if att_name: # some filenames are encoded, try to decode try: att_name = ''.join([x[0] for x in decode_header(att_name)]) except Exception: pass else: # workaround for mimetypes, it always takes .ksh for text/plain if part.get_content_type() == 'text/plain': ext = '.txt' else: ext = mimetypes.guess_extension(part.get_content_type()) if ext is None: ext = '' att_name = 'unnamed%s' % ext att_name = self.asciionly(att_name) res = self.matchMultipleSets( [user_names, domain_names, default_names], att_name, suspect, att_name) if res == ATTACHMENT_SILENTDELETE: self._debuginfo( suspect, "Attachment name=%s SILENT DELETE : blocked by name" % att_name) return DELETE if res == ATTACHMENT_BLOCK: self._debuginfo( suspect, "Attachment name=%s : blocked by name)" % att_name) message = suspect.tags['FiletypePlugin.errormessage'] return blockactioncode, message # go through content type rules res = self.matchMultipleSets( [user_ctypes, domain_ctypes, default_ctypes], contenttype_mime, suspect, att_name) if res == ATTACHMENT_SILENTDELETE: self._debuginfo( suspect, "Attachment name=%s content-type=%s SILENT DELETE: blocked by mime content type (message source)" % (att_name, contenttype_mime)) return DELETE if res == ATTACHMENT_BLOCK: self._debuginfo( suspect, "Attachment name=%s content-type=%s : blocked by mime content type (message source)" % (att_name, contenttype_mime)) message = suspect.tags['FiletypePlugin.errormessage'] return blockactioncode, message contenttype_magic = None if MAGIC_AVAILABLE: pl = part.get_payload(decode=True) contenttype_magic = self.getBuffertype(pl) res = self.matchMultipleSets( [user_ctypes, domain_ctypes, default_ctypes], contenttype_magic, suspect, att_name) if res == ATTACHMENT_SILENTDELETE: self._debuginfo( suspect, "Attachment name=%s content-type=%s SILENT DELETE: blocked by mime content type (magic)" % (att_name, contenttype_mime)) return DELETE if res == ATTACHMENT_BLOCK: self._debuginfo( suspect, "Attachment name=%s content-type=%s : blocked by mime content type (magic)" % (att_name, contenttype_mime)) message = suspect.tags['FiletypePlugin.errormessage'] return blockactioncode, message # archives if self.checkarchivenames or self.checkarchivecontent: # try guessing the archive type based on magic content type first # we don't need to check for MAGIC_AVAILABLE here, if it is available the contenttype_magic is not None archive_type = self.archive_type_from_content_type(contenttype_magic) # if it didn't work, try to guess by the filename extension, if it is enabled if archive_type is None: # sort by length, so tar.gz is checked before .gz for arext in sorted(self.supported_archive_extensions.keys(), key=lambda x: len(x), reverse=True): if att_name.lower().endswith('.%s' % arext): archive_type = self.supported_archive_extensions[arext] break if archive_type is not None: self.logger.debug("Extracting {attname} as {artype}".format(attname=att_name,artype=archive_type)) try: pl = BytesIO(part.get_payload(decode=True)) archive_handle = self._archive_handle(archive_type, pl) namelist = self._archive_namelist(archive_type, archive_handle) if self.checkarchivenames: for name in namelist: # rarfile returns unicode objects which mess up # generated bounces if sys.version_info[0] == 2: # Py3 defaults to unicode name = self.asciionly(name) res = self.matchMultipleSets( [user_archive_names, domain_archive_names, default_archive_names], name, suspect, name) if res == ATTACHMENT_SILENTDELETE: self._debuginfo( suspect, "Blocked filename in archive %s SILENT DELETE" % att_name) return DELETE if res == ATTACHMENT_BLOCK: self._debuginfo( suspect, "Blocked filename in archive %s" % att_name) message = suspect.tags['FiletypePlugin.errormessage'] return blockactioncode, message if MAGIC_AVAILABLE and self.checkarchivecontent: for name in namelist: safename = self.asciionly(name) extracted = self._archive_extract(archive_type, archive_handle, name) if extracted is None: self._debuginfo( suspect, '%s not extracted - too large' % (safename)) contenttype_magic = self.getBuffertype( extracted) res = self.matchMultipleSets( [user_archive_ctypes, domain_archive_ctypes, default_archive_ctypes], contenttype_magic, suspect, name) if res == ATTACHMENT_SILENTDELETE: self._debuginfo( suspect, "Extracted file %s from archive %s content-type=%s SILENT DELETE: blocked by mime content type (magic)" % (safename, att_name, contenttype_magic)) return DELETE if res == ATTACHMENT_BLOCK: self._debuginfo( suspect, "Extracted file %s from archive %s content-type=%s : blocked by mime content type (magic)" % (safename, att_name, contenttype_magic)) message = suspect.tags['FiletypePlugin.errormessage'] return blockactioncode, message if hasattr(archive_handle, 'close'): archive_handle.close() pl.close() except Exception: self.logger.warning( "archive scanning failed in attachment {attname}: {error}".format(attname=att_name, error=traceback.format_exc() )) return DUNNO