def archive(self, suspect): archivedir = self.config.get(self.section, 'archivedir') if archivedir == "": self.logger.error('Archivedir is not specified') return subdirtemplate = self.config.get(self.section, 'subdirtemplate') if self.config.has_option(self.section, 'makedomainsubdir') and subdirtemplate == self.requiredvars['subdirtemplate']['default']: self.logger.warning( "Archive config is using deprecated 'makedomainsubdir' config option. Emulating old behaviour. Update your config(subdirtemplate)") if self.config.getboolean(self.section, 'makedomainsubdir'): subdirtemplate = "${to_domain}" else: subdirtemplate = "" # the archive root dir startdir = os.path.abspath(archivedir) # relative dir within archive root subdir = apply_template(subdirtemplate, suspect) if subdir.endswith('/'): subdir = subdir[:-1] # filename without dir filenametemplate = self.config.get(self.section, 'filenametemplate') filename = apply_template(filenametemplate, suspect) # make sure filename can't create new folders filename = filename.replace('/', '_') # full relative filepath within archive dir fpath = "%s/%s" % (subdir, filename) # absolute final filepath requested_path = os.path.abspath("%s/%s" % (startdir, fpath)) if not os.path.commonprefix([requested_path, startdir]).startswith(startdir): self.logger.error( "file path '%s' seems to be outside archivedir '%s' - storing to archivedir" % (requested_path, startdir)) requested_path = "%s/%s" % (startdir, filename) finaldir = os.path.dirname(requested_path) if not os.path.isdir(finaldir): os.makedirs(finaldir, 0o755) if self.config.getboolean(self.section, 'storeoriginal'): shutil.copy(suspect.tempfile, requested_path) else: with open(requested_path, 'w') as fp: fp.write(suspect.get_source()) chmod = self.config.get(self.section, 'chmod') chgrp = self.config.get(self.section, 'chgrp') chown = self.config.get(self.section, 'chown') if chmod or chgrp or chown: self.setperms(requested_path, chmod, chgrp, chown) self.logger.info('Message from %s to %s archived as %s' % ( suspect.from_address, suspect.to_address, requested_path)) return requested_path
def examine(self, suspect): if not DKIMPY_AVAILABLE: suspect.debug("dkimpy not available, can not check") self.logger.error("DKIM signing skipped - missing dkimpy library") return DUNNO message = suspect.get_source() domain = extract_from_domain(suspect) addvalues = dict(header_from_domain=domain) selector = apply_template(self.config.get(self.section, 'selector'), suspect, addvalues) if domain is None: self.logger.error( "%s: Failed to extract From-header domain for DKIM signing" % suspect.id) return DUNNO privkeyfile = apply_template( self.config.get(self.section, 'privatekeyfile'), suspect, addvalues) if not os.path.isfile(privkeyfile): self.logger.debug( "%s: DKIM signing failed for domain %s, private key not found: %s" % (suspect.id, domain, privkeyfile)) return DUNNO with open(privkeyfile, 'br') as f: privkeycontent = f.read() canH = Simple canB = Simple if self.config.get(self.section, 'canonicalizeheaders').lower() == 'relaxed': canH = Relaxed if self.config.get(self.section, 'canonicalizebody').lower() == 'relaxed': canB = Relaxed canon = (canH, canB) headerconfig = self.config.get(self.section, 'signheaders') if headerconfig is None or headerconfig.strip() == '': inc_headers = None else: inc_headers = headerconfig.strip().split(',') blength = self.config.getboolean(self.section, 'signbodylength') dkimhdr = sign(message, force_bString(selector), force_bString(domain), privkeycontent, canonicalize=canon, include_headers=inc_headers, length=blength, logger=suspect.get_tag('debugfile')) if dkimhdr.startswith(b'DKIM-Signature: '): dkimhdr = dkimhdr[16:] suspect.addheader('DKIM-Signature', dkimhdr, immediate=True)
def examine(self,suspect): if not DKIMPY_AVAILABLE: suspect.debug("dkimpy not available, can not check") suspect.set_tag('DKIMVerify.skipreason','dkimpy library not available') return DUNNO starttime=time.time() message=suspect.get_source() domain=self.get_header_from_domain(suspect) addvalues=dict(header_from_domain=domain) selector=apply_template(self.config.get(self.section,'selector'),suspect,addvalues) if domain==None: self._logger().error("%s: Failed to extract From-header domain for DKIM signing"%suspect.id) return DUNNO privkeyfile=apply_template(self.config.get(self.section,'privatekeyfile'), suspect,addvalues) if not os.path.isfile(privkeyfile): self._logger().error("%s: DKIM signing failed for domain %s, private key not found: %s"%(suspect.id,domain,privkeyfile)) return DUNNO privkeycontent=open(privkeyfile,'r').read() canH=Simple canB=Simple if self.config.get(self.section,'canonicalizeheaders').lower()=='relaxed': canH=Relaxed if self.config.get(self.section,'canonicalizebody').lower()=='relaxed': canB=Relaxed canon=(canH,canB) headerconfig=self.config.get(self.section,'signheaders') if headerconfig==None or headerconfig.strip()=='': inc_headers=None else: inc_headers=headerconfig.strip().split(',') blength=self.config.getboolean(self.section,'signbodylength') dkimhdr=sign(message, selector, domain, privkeycontent, canonicalize=canon, include_headers=inc_headers, length=blength, logger=suspect.get_tag('debugfile')) if dkimhdr.startswith('DKIM-Signature: '): dkimhdr=dkimhdr[16:] suspect.addheader('DKIM-Signature',dkimhdr,immediate=True) endtime=time.time() difftime=endtime-starttime suspect.tags['DKIMSign.time']="%.4f"%difftime
def examine(self, suspect): self.filelist.filename = self.config.get(self.section, 'domainsfile') checkdomains = self.filelist.get_list() envelope_sender_domain = suspect.from_domain.lower() header_from_domain = self.extract_from_domain(suspect) if header_from_domain == None: return if header_from_domain not in checkdomains: return # TODO: do we need a tag from dkim to check if the verified dkim domain # actually matches the header from domain? dkimresult = suspect.get_tag('DKIMVerify.sigvalid', False) if dkimresult == True: return DUNNO # DKIM failed, check SPF if envelope senderdomain belongs to header # from domain spfresult = suspect.get_tag('SPF.status', 'unknown') if (envelope_sender_domain == header_from_domain or envelope_sender_domain.endswith('.%s' % header_from_domain)) and spfresult == 'pass': return DUNNO failaction = self.config.get(self.section, 'failaction') actioncode = string_to_actioncode(failaction, self.config) values = dict( header_from_domain=header_from_domain) message = apply_template( self.config.get(self.section, 'rejectmessage'), suspect, values) return actioncode, message
def examine(self, suspect): if not self.should_we_check_this_domain(suspect): return DUNNO envelope_recipient_domain = suspect.to_domain.lower() envelope_sender_domain = suspect.from_domain.lower() if envelope_sender_domain == envelope_recipient_domain: return DUNNO # we only check the message if the env_sender_domain differs. If it's the same it will be caught by other means (like SPF) header_from_domain = extract_from_domain(suspect) if header_from_domain is None: self._logger().warn("%s: Could not extract header from domain for spearphish check" % suspect.id) return DUNNO if header_from_domain == envelope_recipient_domain: virusname = self.config.get(self.section, 'virusname') virusaction = self.config.get(self.section, 'virusaction') actioncode = string_to_actioncode(virusaction, self.config) logmsg = '%s: spear phish pattern detected, recipient=%s env_sender_domain=%s header_from_domain=%s' % ( suspect.id, suspect.to_address, envelope_sender_domain, header_from_domain) self._logger().info(logmsg) self.flag_as_phish(suspect, virusname) message = apply_template( self.config.get(self.section, 'rejectmessage'), suspect, {'virusname': virusname}) return actioncode, message return DUNNO
def examine(self, suspect): if not self.should_we_check_this_domain(suspect): return DUNNO envelope_recipient_domain = suspect.to_domain.lower() envelope_sender_domain = suspect.from_domain.lower() if envelope_sender_domain == envelope_recipient_domain: return DUNNO # we only check the message if the env_sender_domain differs. If it's the same it will be caught by other means (like SPF) header_from_domain = extract_from_domain(suspect) if header_from_domain is None: self._logger().warn( "%s: Could not extract header from domain for spearphish check" % suspect.id) return DUNNO if header_from_domain == envelope_recipient_domain: virusname = self.config.get(self.section, 'virusname') virusaction = self.config.get(self.section, 'virusaction') actioncode = string_to_actioncode(virusaction, self.config) logmsg = '%s: spear phish pattern detected, recipient=%s env_sender_domain=%s header_from_domain=%s' % ( suspect.id, suspect.to_address, envelope_sender_domain, header_from_domain) self._logger().info(logmsg) self.flag_as_phish(suspect, virusname) message = apply_template( self.config.get(self.section, 'rejectmessage'), suspect, {'virusname': virusname}) return actioncode, message return DUNNO
def examine(self, suspect): self.filelist.filename = self.config.get(self.section, 'domainsfile') checkdomains = self.filelist.get_list() envelope_sender_domain = suspect.from_domain.lower() header_from_domain = extract_from_domain(suspect) if header_from_domain is None: return if header_from_domain not in checkdomains: return # TODO: do we need a tag from dkim to check if the verified dkim domain # actually matches the header from domain? dkimresult = suspect.get_tag('DKIMVerify.sigvalid', False) if dkimresult == True: return DUNNO # DKIM failed, check SPF if envelope senderdomain belongs to header # from domain spfresult = suspect.get_tag('SPF.status', 'unknown') if (envelope_sender_domain == header_from_domain or envelope_sender_domain.endswith('.%s' % header_from_domain)) and spfresult == 'pass': return DUNNO failaction = self.config.get(self.section, 'failaction') actioncode = string_to_actioncode(failaction, self.config) values = dict( header_from_domain=header_from_domain) message = apply_template( self.config.get(self.section, 'rejectmessage'), suspect, values) return actioncode, message
def send_template_string(self, recipient, templatecontent, suspect, values): """Send a E-Mail Bounce Message recipient -- Message recipient ([email protected]) templatecontent -- Template to use suspect -- suspect that caused the bounce values -- Values to apply to the template If the suspect has the 'nobounce' tag set, the message will not be sent. The same happens if the global configuration 'disablebounces' is set. """ if suspect.get_tag('nobounce'): self.logger.info( 'Not sending bounce to %s - bounces disabled by plugin' % recipient) return message = apply_template(templatecontent, suspect, values) try: message = self._add_required_headers(recipient, message) except Exception as e: self.logger.warning( "Bounce message template could not be verified: %s" % str(e)) self.logger.debug('Sending bounce message to %s' % recipient) fromaddress = "<>" self._send(fromaddress, recipient, message)
def examine(self, suspect): starttime = time.time() if self.filter == None: self.filter = SuspectFilter( self.config.get(self.section, 'filterfile')) hits = self.filter.get_args(suspect, extended=True) if len(hits) == 0: return DUNNO #open file ofile = self.config.get(self.section, 'outputfile') if ofile.strip() == '': self._logger().error("No output file specified for headerwriter") return DUNNO fh = open(ofile, 'a') for hit in hits: (fieldname, matchedvalue, arg, regex) = hit if arg == None or arg == '': arg = self.config.get(self.section, 'defaultlinetemplate') addvalues = dict(fieldname=fieldname, matchedvalue=matchedvalue, regex=regex) outputline = apply_template(arg, suspect, addvalues) fh.write(outputline) fh.write('\n') fh.close()
def examine(self, suspect): enginename = self.config.get(self.section, 'enginename') if suspect.size > self.config.getint(self.section, 'maxsize'): self.logger.info('Not scanning - message too big') return content = suspect.get_source() for i in range(0, self.config.getint(self.section, 'retries')): try: viruses = self.scan_stream(content) if viruses != None: self.logger.info( "Virus found in message from %s : %s" % (suspect.from_address, viruses)) suspect.tags['virus'][enginename] = True suspect.tags['%s.virus' % enginename] = viruses suspect.debug('viruses found in message : %s' % viruses) else: suspect.tags['virus'][enginename] = False if viruses != None: virusaction = self.config.get(self.section, 'virusaction') actioncode = string_to_actioncode(virusaction, self.config) firstinfected, firstvirusname = viruses.items()[0] values = dict( infectedfile=firstinfected, virusname=firstvirusname) message = apply_template( self.config.get(self.section, 'rejectmessage'), suspect, values) return actioncode, message return DUNNO except Exception, e: self.logger.warning("Error encountered while contacting ICAP server (try %s of %s): %s" % ( i + 1, self.config.getint(self.section, 'retries'), str(e)))
def examine(self, suspect): urls = suspect.get_tag('black.uris', defaultvalue=[]) add_header_links = self.config.getboolean(self.section, 'addheaderlinks') add_header_count = self.config.getboolean(self.section, 'addheadercount') if len(urls) == 0: return DUNNO elif add_header_count is True and add_header_links is False: suspect.add_header('X-Black-Host-Count', str(len(urls)), immediate=True) elif add_header_count is True and add_header_links is True: suspect.add_header('X-Black-Host', "\t" + "\r\n\t\t\t ".join(urls), immediate=True) suspect.add_header('X-Black-Host-Count', str(len(urls)), immediate=True) elif add_header_count is False and add_header_links is True: suspect.add_header('X-Black-Host', "\t" + "\r\n\t\t\t ".join(urls), immediate=True) return string_to_actioncode( self.config.get('URIExtractPlugin', 'action'), self.config), apply_template( self.config.get('URIExtractPlugin', 'message'), suspect, dict(domain=urls[0], blacklist='tbd'))
def examine(self, suspect): if suspect.size > self.config.getint(self.section, 'maxsize'): self.logger.info('Not scanning - message too big') return content = suspect.get_source() for i in range(0, self.config.getint(self.section, 'retries')): try: viruses = self.scan_stream(content) if viruses != None: self.logger.info( "Virus found in message from %s : %s" % (suspect.from_address, viruses)) suspect.tags['virus']['ClamAV'] = True suspect.tags['ClamavPlugin.virus'] = viruses suspect.debug('viruses found in message : %s' % viruses) else: suspect.tags['virus']['ClamAV'] = False if viruses != None: virusaction = self.config.get(self.section, 'virusaction') actioncode = string_to_actioncode(virusaction, self.config) firstinfected, firstvirusname = viruses.items()[0] values = dict( infectedfile=firstinfected, virusname=firstvirusname) message = apply_template( self.config.get(self.section, 'rejectmessage'), suspect, values) return actioncode, message return DUNNO except Exception, e: self.logger.warning("Error encountered while contacting clamd (try %s of %s): %s" % ( i + 1, self.config.getint(self.section, 'retries'), str(e)))
def examine(self, suspect): if suspect.size > self.config.getint(self.section, 'maxsize'): self._logger().info('Not scanning - message too big (message %s bytes > config %s bytes )' % (suspect.size, self.config.getint(self.section, 'maxsize'))) return DUNNO try: viruses = self.scan_file(suspect.tempfile) if viruses is not None: self._logger().info("Virus found in message from %s : %s" % (suspect.from_address, viruses)) suspect.tags['virus'][self.config.get(self.section,'identifier')] = True suspect.tags['%s.virus'%self.config.get(self.section,'identifier')] = viruses suspect.debug('Viruses found in message : %s' % viruses) else: suspect.tags['virus'][self.config.get(self.section,'identifier')] = False if viruses != None: virusaction = self.config.get(self.section, 'virusaction') actioncode = string_to_actioncode(virusaction, self.config) firstinfected, firstvirusname = viruses.items()[0] values = dict( infectedfile=firstinfected, virusname=firstvirusname) message = apply_template( self.config.get(self.section, 'rejectmessage'), suspect, values) return actioncode, message else: return DUNNO except Exception, e: self._logger().warning("Error encountered while running cmdline av scan: %s" % str(e))
def examine(self, suspect): if suspect.get_tag('Dspam.skip') is True: self.logger.debug( '%s Skipping Dspam Plugin (requested by previous plugin)' % suspect.id) suspect.set_tag('Dspam.skipreason', 'requested by previous plugin') return DUNNO if self.config.getboolean(self.section, 'scanoriginal'): content = suspect.get_original_source() else: content = suspect.get_source() recipient = suspect.to_address try: results = self._check_dspam(content, recipient) except DspamClientError as e: self.logger.error('%s failed to process by dspam: %s' % (suspect.id, str(e))) return self._problemcode() action = DUNNO message = None isspam = self._is_spam(results) suspect.tags['spam']['Dspam'] = isspam suspect.tags['DSpam.report'] = self._report(results) if isspam: action = string_to_actioncode( self.config.get(self.section, 'spamaction'), self.config) message = apply_template( self.config.get(self.section, 'rejectmessage'), suspect, results) return action, message
def examine(self,suspect): starttime=time.time() if self.filter==None: self.filter=SuspectFilter(self.config.get(self.section,'filterfile')) hits=self.filter.get_args(suspect,extended=True) if len(hits)==0: return DUNNO #open file ofile=self.config.get(self.section,'outputfile') if ofile.strip()=='': self._logger().error("No output file specified for headerwriter") return DUNNO fh=open(ofile,'a') for hit in hits: (fieldname, matchedvalue, arg, regex)=hit if arg==None or arg=='': arg=self.config.get(self.section,'defaultlinetemplate') addvalues=dict(fieldname=fieldname,matchedvalue=matchedvalue,regex=regex) outputline=apply_template(arg, suspect, addvalues) fh.write(outputline) fh.write('\n') fh.close()
def commitback(self,suspect): injectanswer=self.re_inject(suspect) suspect.set_tag("injectanswer",injectanswer) values=dict(injectanswer=injectanswer) message=apply_template(self.config.get('smtpconnector','requeuetemplate'), suspect, values) self.sess.endsession(250, message) self.sess=None
def test_bounce(self): """Test bounce message, especially the encoding""" suspect = Suspect('*****@*****.**', '*****@*****.**', '/dev/null') # include non-ascii charset unicode characters to make sure the encoding/decoding # works correctly displayname = u"((testing placeholder for displayname -> äää))" asciirep = u"((testing placeholder for asciirep -> üüü))" description = u"((testing placeholder for description -> ööö))" blockinfo = ("%s %s: %s" % (displayname, asciirep, description)).strip() blockedfiletemplate = os.path.join( *[CONFDIR, "templates", "blockedfile.tmpl.dist"]) bounce = Bounce(self.config) bounce.send_template_file(suspect.from_address, blockedfiletemplate, suspect, dict(blockinfo=blockinfo)) # might be needed to wait for a bit to make sure answer is available counter = 0 while self.smtp.suspect is None and counter < 20: counter = counter + 1 time.sleep(0.05) # sleep is needed to gotback = self.smtp.suspect self.assertFalse(gotback == None, "Did not get message from dummy smtp server") # get message received by dummy smtp server msg = gotback.get_message_rep() receivedMsg = msg.get_payload(decode='utf-8') # Build the message according to what Bounce is doing so it can be compared # to what was received from DummySMTPServer with open(blockedfiletemplate) as fp: templatecontent = fp.read() blockinfo = ("%s %s: %s" % (displayname, asciirep, description)).strip() message = apply_template(templatecontent, suspect, dict(blockinfo=blockinfo)) messageB = force_bString(message) # modify received message to add header parts from template messageToCompare = force_bString("To: " + msg['To'] + "\nSubject: " + msg['Subject'] + "\n\n") + force_bString(receivedMsg) # make sure comparison will not fail because of newlines # For example, Python 2.6 has only one "\n" at the end of the received message, whereas Python 2.7 and 3 have to messageToCompare = messageToCompare.replace(b"\r", b"\n").replace( b"\n\n", b"\n") messageB = messageB.replace(b"\r", b"\n").replace(b"\n\n", b"\n") self.assertEqual(messageB, messageToCompare)
def commitback(self, suspect): injectanswer = self.re_inject(suspect) suspect.set_tag("injectanswer", injectanswer) values = dict(injectanswer=injectanswer) message = apply_template( self.config.get('smtpconnector', 'requeuetemplate'), suspect, values) self.sess.endsession(250, message) self.sess = None
def examine(self, suspect): starttime = time.time() if suspect.size > self.config.getint(self.section, 'maxsize'): self._logger().info( 'Not scanning - message too big (message %s bytes > config %s bytes )' % (suspect.size, self.config.getint(self.section, 'maxsize'))) return DUNNO content = suspect.get_message_rep().as_string() for i in range(0, self.config.getint(self.section, 'retries')): try: headerscannername = '-' + self.config.get( self.section, 'headerscannername') if headerscannername == '-': headerscannername = '' if self.config.getboolean(self.section, 'networkmode'): viruses = self.scan_stream(content) else: viruses = self.scan_file(suspect.tempfile) if viruses != None: self._logger().info("Virus found in message from %s : %s" % (suspect.from_address, viruses)) suspect.tags['virus']['F-Prot'] = True suspect.tags['FprotPlugin.virus'] = viruses suspect.debug('Viruses found in message : %s' % viruses) else: suspect.tags['virus']['F-Prot'] = False addheaderclean = self.config.get(self.section, 'addheaderclean') suspect.set_tag('fprot.clean', 'Clean') if viruses != None: virusaction = self.config.get(self.section, 'virusaction') actioncode = string_to_actioncode(virusaction, self.config) addheaderinfected = self.config.get( self.section, 'addheaderinfected') firstinfected, firstvirusname = list(viruses.items())[0] values = dict(infectedfile=firstinfected, virusname=firstvirusname) message = apply_template( self.config.get(self.section, 'rejectmessage'), suspect, values) suspect.set_tag('fprot.virus', firstvirusname) return actioncode, message else: return DUNNO except Exception as e: self._logger().warning( "Error encountered while contacting fpscand (try %s of %s): %s" % (i + 1, self.config.getint(self.section, 'retries'), str(e))) self._logger().error("fpscand failed after %s retries" % self.config.getint(self.section, 'retries')) content = None return self._problemcode()
def deliver_mbox(self, suspect): mbox_msg = mailbox.mboxMessage(suspect.get_message_rep()) mbox_path = apply_template(self.config.get(self.section, "path"), suspect) mbox = mailbox.mbox(mbox_path) try: mbox.lock() mbox.add(mbox_msg) mbox.flush() except Exception, e: self.logger.error("Could not store message %s to %s: %s" % (suspect.id, mbox_path, str(e)))
def deliver_mbox(self,suspect): mbox_msg=mailbox.mboxMessage(suspect.get_message_rep()) mbox_path=apply_template(self.config.get(self.section,'path'), suspect) mbox=mailbox.mbox( mbox_path) try: mbox.lock() mbox.add(mbox_msg) mbox.flush() except Exception,e: self.logger.error("Could not store message %s to %s: %s"%(suspect.id,mbox_path,str(e)))
def commitback(self, suspect): injectcode, injectanswer = self.re_inject(suspect) suspect.set_tag("injectanswer", injectanswer) values = dict(injectanswer=injectanswer) message = apply_template(self.config.get('esmtpconnector', 'queuetemplate'), suspect, values) if injectcode >= 200 and injectcode < 300: self.sess.endsession(250, message) else: self.sess.endsession(injectcode, injectanswer) self.sess = None
def commitback(self,suspect): injectcode,injectanswer=self.re_inject(suspect) suspect.set_tag("injectanswer",injectanswer) values=dict(injectanswer=injectanswer) message=apply_template(self.config.get('esmtpconnector','queuetemplate'), suspect, values) if injectcode>=200 and injectcode<300: self.sess.endsession(250, message) else: self.sess.endsession(injectcode,injectanswer) self.sess=None
def deliver_maildir(self, suspect): md_msg = mailbox.MaildirMessage(suspect.get_message_rep()) md_path = apply_template(self.config.get(self.section, "path"), suspect) if os.path.isfile(md_path): self.logger.error("%s seems to be a file - can not use as maildir" % md_path) return maildir = mailbox.Maildir(md_path) try: maildir.lock() maildir.add(md_msg) maildir.flush() except Exception, e: self.logger.error("Could not store message %s to %s: %s" % (suspect.id, md_path, str(e)))
def deliver_maildir(self,suspect): md_msg=mailbox.MaildirMessage(suspect.get_message_rep()) md_path=apply_template(self.config.get(self.section,'path'), suspect) if os.path.isfile(md_path): self.logger.error("%s seems to be a file - can not use as maildir"%md_path) return maildir=mailbox.Maildir(md_path) try: maildir.lock() maildir.add(md_msg) maildir.flush() except Exception,e: self.logger.error("Could not store message %s to %s: %s"%(suspect.id,md_path,str(e)))
def examine(self, suspect): enginename = 'sophos' if suspect.size > self.config.getint(self.section, 'maxsize'): self.logger.info('Not scanning - message too big') return content = suspect.get_source() for i in range(0, self.config.getint(self.section, 'retries')): try: viruses = self.scan_stream(content) headerscannername = '-' + self.config.get( self.section, 'headerscannername') if headerscannername == '-': headerscannername = '' if viruses is not None: self.logger.info("Virus found in message from %s : %s" % (suspect.from_address, viruses)) suspect.tags['virus'][enginename] = True suspect.tags['%s.virus' % enginename] = viruses suspect.debug('viruses found in message : %s' % viruses) else: suspect.tags['virus'][enginename] = False addheaderclean = self.config.get(self.section, 'addheaderclean') suspect.set_tag('sssp.clean', 'Clean') if viruses is not None: virusaction = self.config.get(self.section, 'virusaction') actioncode = string_to_actioncode(virusaction, self.config) addheaderinfected = self.config.get( self.section, 'addheaderinfected') firstinfected, firstvirusname = list(viruses.items())[0] values = dict(infectedfile=firstinfected, virusname=firstvirusname) message = apply_template( self.config.get(self.section, 'rejectmessage'), suspect, values) suspect.set_tag('sssp.virus', firstvirusname) return actioncode, message return DUNNO except Exception as e: self.logger.warning( "Error encountered while contacting SSSP server (try %s of %s): %s" % (i + 1, self.config.getint(self.section, 'retries'), str(e))) self.logger.error("SSSP scan failed after %s retries" % self.config.getint(self.section, 'retries')) return self._problemcode()
def examine(self,suspect): if not YARA_AVAILABLE: return self.reload_if_necessary() if not self.compiled_rules: return yarahits = None infectedfile = None m = suspect.get_message_rep() for i in m.walk(): if i.is_multipart(): continue att_name = i.get_filename(None) if att_name: # some filenames are encoded, try to decode try: att_name = ''.join([x[0] for x in decode_header(att_name)]) except: pass else: att_name='message' payload = StringIO(i.get_payload(decode=True)) yarahits = self.check_file(suspect,att_name, payload) if yarahits != None: infectedfile = att_name break if infectedfile and yarahits: self.logger.info( "YARA hit(s) in message from %s : %s" % (suspect.from_address, yarahits)) suspect.tags['virus']['YARA'] = True suspect.tags['YARAPlugin.virus'] = yarahits[0].rule suspect.debug('YARA hit found in message : %s' % yarahits) virusaction = self.config.get(self.section, 'virusaction') actioncode = string_to_actioncode(virusaction, self.config) values = dict( infectedfile=infectedfile, virusname=yarahits[0].rule) message = apply_template( self.config.get(self.section, 'rejectmessage'), suspect, values) return actioncode, message else: suspect.tags['virus']['YARA'] = False return DUNNO
def examine(self, suspect): if suspect.size > self.config.getint(self.section, 'maxsize'): self.logger.info('Not scanning - message too big') return content = suspect.get_source() for i in range(0, self.config.getint(self.section, 'retries')): try: viruses = self.scan_stream(content) headerscannername = '-' + self.config.get(self.section, 'headerscannername') if headerscannername == '-': headerscannername = '' if viruses != None: self.logger.info( "Virus found in message from %s : %s" % (suspect.from_address, viruses)) suspect.tags['virus']['ClamAV'] = True suspect.tags['ClamavPlugin.virus'] = viruses suspect.debug('viruses found in message : %s' % viruses) else: suspect.tags['virus']['ClamAV'] = False addheaderclean = self.config.get(self.section, 'addheaderclean') suspect.set_tag('clam.clean', 'Clean') if viruses != None: virusaction = self.config.get(self.section, 'virusaction') actioncode = string_to_actioncode(virusaction, self.config) addheaderinfected = self.config.get(self.section, 'addheaderinfected') firstinfected, firstvirusname = list(viruses.items())[0] values = dict( infectedfile=firstinfected, virusname=firstvirusname) message = apply_template( self.config.get(self.section, 'rejectmessage'), suspect, values) suspect.set_tag('clam.virus', firstvirusname) return actioncode, message return DUNNO except Exception as e: self.__invalidate_socket() # don't warn the first time if it's just a broken pipe which # can happen with the new pipelining protocol if not (i == 0 and isinstance(e, socket.error) and e.errno == errno.EPIPE): self.logger.warning("Error encountered while contacting clamd (try %s of %s): %s" % ( i + 1, self.config.getint(self.section, 'retries'), str(e))) self.logger.error("Clamdscan failed after %s retries" % self.config.getint(self.section, 'retries')) content = None return self._problemcode()
def examine(self, suspect): if not self.should_we_check_this_domain(suspect): return DUNNO envelope_recipient_domain = suspect.to_domain.lower() envelope_sender_domain = suspect.from_domain.lower() if envelope_sender_domain == envelope_recipient_domain: return DUNNO # we only check the message if the env_sender_domain differs. If it's the same it will be caught by other means (like SPF) header_from_domains = [] header_from_domain = extract_from_domain(suspect) if header_from_domain is None: self.logger.warn( "%s: Could not extract header from domain for spearphish check" % suspect.id) return DUNNO else: header_from_domains.append(header_from_domain) self.logger.debug( '%s: checking domain %s (source: From header address part)' % (suspect.id, header_from_domain)) if self.config.getboolean(self.section, 'check_display_part'): display_from_domain = extract_from_domain(suspect, False) if display_from_domain is not None and display_from_domain not in header_from_domains: header_from_domains.append(display_from_domain) self.logger.debug( '%s: checking domain %s (source: From header display part)' % (suspect.id, display_from_domain)) actioncode = DUNNO message = None for header_from_domain in header_from_domains: if header_from_domain == envelope_recipient_domain: virusname = self.config.get(self.section, 'virusname') virusaction = self.config.get(self.section, 'virusaction') actioncode = string_to_actioncode(virusaction, self.config) logmsg = '%s: spear phish pattern detected, env_rcpt_domain=%s env_sender_domain=%s header_from_domain=%s' % \ (suspect.id, envelope_recipient_domain, envelope_sender_domain, header_from_domain) self.logger.info(logmsg) self.flag_as_phish(suspect, virusname) message = apply_template( self.config.get(self.section, 'rejectmessage'), suspect, {'virusname': virusname}) break return actioncode, message
def test_template(self): """Test Basic Template function""" suspect = Suspect('*****@*****.**', '*****@*****.**', TESTDATADIR + '/helloworld.eml') suspect.tags['nobounce'] = True reason = "a three-headed monkey stole it" template = """Your message '${subject}' from ${from_address} to ${to_address} could not be delivered because ${reason}""" result = apply_template(template, suspect, dict(reason=reason)) expected = """Your message 'Hello world!' from [email protected] to [email protected] could not be delivered because a three-headed monkey stole it""" self.assertEqual( result, expected), "Got unexpected template result: %s" % result
def examine(self, suspect): if not YARA_AVAILABLE: return self.reload_if_necessary() if not self.compiled_rules: return yarahits = None infectedfile = None m = suspect.get_message_rep() for i in m.walk(): if i.is_multipart(): continue att_name = i.get_filename(None) if att_name: # some filenames are encoded, try to decode try: att_name = "".join([x[0] for x in decode_header(att_name)]) except: pass else: att_name = "message" payload = StringIO(i.get_payload(decode=True)) yarahits = self.check_file(suspect, att_name, payload) if yarahits != None: infectedfile = att_name break if infectedfile and yarahits: self.logger.info("YARA hit(s) in message from %s : %s" % (suspect.from_address, yarahits)) suspect.tags["virus"]["YARA"] = True suspect.tags["YARAPlugin.virus"] = yarahits[0].rule suspect.debug("YARA hit found in message : %s" % yarahits) virusaction = self.config.get(self.section, "virusaction") actioncode = string_to_actioncode(virusaction, self.config) values = dict(infectedfile=infectedfile, virusname=yarahits[0].rule) message = apply_template(self.config.get(self.section, "rejectmessage"), suspect, values) return actioncode, message else: suspect.tags["virus"]["YARA"] = False return DUNNO
def examine(self, suspect): if not DNSQUERY_EXTENSION_ENABLED: return DUNNO if not self.config.getboolean(self.section, 'check_always'): # save the lookup if mail is already tagged as virus or spam if suspect.is_virus() or suspect.is_spam() or suspect.is_blocked(): return DUNNO maxlookups = self.config.getint(self.section, 'maxlookups') emails = suspect.get_tag('emails', defaultvalue=[])[:maxlookups] emails = [self._email_normalise(email) for email in emails] emails = list(set(emails)) #if emails: # self.logger.debug('%s EBL checking addresses %s' % (suspect.id, ', '.join(emails))) listed = False action = DUNNO message = None email = None for email in emails: addr_hash = self._create_hash(email) listed, message = self._ebl_lookup(addr_hash) if listed: break suspect.tags['spam']['EBL'] = listed if listed: self.logger.debug('%s EBL hit for %s' % (suspect.id, email)) action = string_to_actioncode( self.config.get(self.section, 'action')) suspect.tags['EBL.email'] = email suspect.tags['EBL.reason'] = message if action != DUNNO: values = { 'dnszone': self.config.get(self.section, 'dnszone'), 'message': message, } message = apply_template( self.config.get(self.section, 'messagetemplate'), suspect, values) return action, message
def examine(self, suspect): if not DOMAINMAGIC_AVAILABLE: self.logger.info('Not scanning - Domainmagic not available') return DUNNO if self.rbllookup is None: self.rbllookup = RBLLookup() self.rbllookup.from_config( self.config.get(self.section, 'blacklistconfig')) self._init_tldmagic() urls = suspect.get_tag('body.uris', defaultvalue=[]) #self.logger.info("Body URIs to check: %s"%urls) domains = set(map(fqdn_from_uri, urls)) counter = 0 for domain in domains: counter += 1 if counter > self.config.getint(self.section, 'maxdomains'): self.logger.info("maximum number of domains reached") break tldcount = self.tldmagic.get_tld_count(domain) parts = domain.split('.') if self.config.getboolean(self.section, 'checksubdomains'): subrange = range(tldcount + 1, len(parts) + 1) else: subrange = [tldcount + 1] for subindex in subrange: subdomain = '.'.join(parts[-subindex:]) listings = self.rbllookup.listings(subdomain) for identifier, humanreadable in iter(listings.items()): self.logger.info( "%s : url host %s flagged as %s because %s" % (suspect.id, domain, identifier, humanreadable)) return string_to_actioncode( self.config.get(self.section, 'action'), self.config), apply_template( self.config.get(self.section, 'message'), suspect, dict(domain=domain, blacklist=identifier)) return DUNNO
def examine(self, suspect): starttime = time.time() if suspect.size > self.config.getint(self.section, 'maxsize'): self._logger().info('Not scanning - message too big (message %s bytes > config %s bytes )' % (suspect.size, self.config.getint(self.section, 'maxsize'))) return DUNNO content = suspect.get_message_rep().as_string() for i in range(0, self.config.getint(self.section, 'retries')): try: if self.config.getboolean(self.section, 'networkmode'): viruses = self.scan_stream(content) else: viruses = self.scan_file(suspect.tempfile) if viruses != None: self._logger().info("Virus found in message from %s : %s" % (suspect.from_address, viruses)) suspect.tags['virus']['F-Prot'] = True suspect.tags['FprotPlugin.virus'] = viruses suspect.debug('Viruses found in message : %s' % viruses) else: suspect.tags['virus']['F-Prot'] = False if viruses != None: virusaction = self.config.get(self.section, 'virusaction') actioncode = string_to_actioncode(virusaction, self.config) firstinfected, firstvirusname = list(viruses.items())[0] values = dict( infectedfile=firstinfected, virusname=firstvirusname) message = apply_template( self.config.get(self.section, 'rejectmessage'), suspect, values) return actioncode, message else: return DUNNO except Exception as e: self._logger().warning("Error encountered while contacting fpscand (try %s of %s): %s" % (i + 1, self.config.getint(self.section, 'retries'), str(e))) self._logger().error("fpscand failed after %s retries" % self.config.getint(self.section, 'retries')) content = None return self._problemcode()
def examine(self, suspect): if suspect.size > self.config.getint(self.section, 'maxsize'): self.logger.info('Not scanning - message too big') return content = suspect.get_source() for i in range(0, self.config.getint(self.section, 'retries')): try: viruses = self.scan_stream(content) if viruses != None: self.logger.info( "Virus found in message from %s : %s" % (suspect.from_address, viruses)) suspect.tags['virus']['ClamAV'] = True suspect.tags['ClamavPlugin.virus'] = viruses suspect.debug('viruses found in message : %s' % viruses) else: suspect.tags['virus']['ClamAV'] = False if viruses != None: virusaction = self.config.get(self.section, 'virusaction') actioncode = string_to_actioncode(virusaction, self.config) firstinfected, firstvirusname = list(viruses.items())[0] values = dict( infectedfile=firstinfected, virusname=firstvirusname) message = apply_template( self.config.get(self.section, 'rejectmessage'), suspect, values) return actioncode, message return DUNNO except Exception as e: self.__invalidate_socket() # don't warn the first time if it's just a broken pipe which # can happen with the new pipelining protocol if not (i == 0 and isinstance(e, socket.error) and e.errno == errno.EPIPE): self.logger.warning("Error encountered while contacting clamd (try %s of %s): %s" % ( i + 1, self.config.getint(self.section, 'retries'), str(e))) self.logger.error("Clamdscan failed after %s retries" % self.config.getint(self.section, 'retries')) content = None return self._problemcode()
def process(self, suspect, decision): if not SQL_EXTENSION_ENABLED: self.logger.error("Fuglu SQL Extensions not enabled") return connstring = self.config.get(self.section, 'dbconnectstring') session = get_session(connstring) if session is None: self.logger.error("Could not create database session") return try: conn = session.connection() conn.connect() except Exception as e: self.logger.error("Database Connection failed: %s" % e) return statementlist = self.get_statements() for statement in statementlist: self.logger.debug("Template: %s" % statement) addvalues = { 'action': actioncode_to_string(decision), } from_header = suspect.get_message_rep()['from'] try: addvalues['header_from'] = self.stripAddress(from_header) except Exception: #use full from header addvalues['header_from'] = from_header replaced = apply_template(statement, suspect, values=addvalues, valuesfunction=self.sqlfix) self.logger.debug("Statement: %s" % replaced) try: result = session.execute(replaced) except Exception as e: self.logger.error("Statement failed: statement=%s , error=%s" % (replaced, str(e))) session.remove()
def examine(self, suspect): if suspect.size > self.config.getint(self.section, 'maxsize'): self.logger.info( 'Not scanning - message too big (message %s bytes > config %s bytes )' % (suspect.size, self.config.getint(self.section, 'maxsize'))) return DUNNO content = suspect.get_message_rep().as_string() for i in range(0, self.config.getint(self.section, 'retries')): try: viruses = self.scan_stream(content) if viruses != None: self.logger.info("Virus found in message from %s : %s" % (suspect.from_address, viruses)) suspect.tags['virus']['drweb'] = True suspect.tags['DrWebPlugin.virus'] = viruses suspect.debug('Viruses found in message : %s' % viruses) else: suspect.tags['virus']['drweb'] = False if viruses != None: virusaction = self.config.get(self.section, 'virusaction') actioncode = string_to_actioncode(virusaction, self.config) firstinfected, firstvirusname = list(viruses.items())[0] values = dict(infectedfile=firstinfected, virusname=firstvirusname) message = apply_template( self.config.get(self.section, 'rejectmessage'), suspect, values) return actioncode, message else: return DUNNO except Exception as e: self.logger.warning( "Error encountered while contacting drweb (try %s of %s): %s" % (i + 1, self.config.getint(self.section, 'retries'), str(e))) self.logger.error("drweb scan failed after %s retries" % self.config.getint(self.section, 'retries')) content = None return self._problemcode()
def _virusreport(self, suspect, viruses): actioncode = DUNNO message = None if viruses is not None: self.logger.info("%s Virus found in message from %s : %s" % (suspect.id, suspect.from_address, viruses)) suspect.tags['virus']['ClamAV'] = True suspect.tags['ClamavPlugin.virus'] = viruses suspect.debug('viruses found in message : %s' % viruses) else: suspect.tags['virus']['ClamAV'] = False if viruses is not None: virusaction = self.config.get(self.section, 'virusaction') actioncode = string_to_actioncode(virusaction, self.config) firstinfected, firstvirusname = list(viruses.items())[0] values = dict( infectedfile=firstinfected, virusname=firstvirusname) message = apply_template( self.config.get(self.section, 'rejectmessage'), suspect, values) return actioncode, message
def examine(self, suspect): if not self.should_we_check_this_domain(suspect): return DUNNO envelope_recipient_domain = suspect.to_domain.lower() envelope_sender_domain = suspect.from_domain.lower() if envelope_sender_domain == envelope_recipient_domain: return DUNNO # we only check the message if the env_sender_domain differs. If it's the same it will be caught by other means (like SPF) header_from_domains = [] header_from_domain = extract_from_domain(suspect) if header_from_domain is None: self.logger.warn("%s: Could not extract header from domain for spearphish check" % suspect.id) return DUNNO else: header_from_domains.append(header_from_domain) self.logger.debug('%s: checking domain %s (source: From header address part)' % (suspect.id, header_from_domain)) if self.config.getboolean(self.section, 'check_display_part'): display_from_domain = extract_from_domain(suspect, False) if display_from_domain is not None and display_from_domain not in header_from_domains: header_from_domains.append(display_from_domain) self.logger.debug('%s: checking domain %s (source: From header display part)' % (suspect.id, display_from_domain)) actioncode = DUNNO message = None for header_from_domain in header_from_domains: if header_from_domain == envelope_recipient_domain: virusname = self.config.get(self.section, 'virusname') virusaction = self.config.get(self.section, 'virusaction') actioncode = string_to_actioncode(virusaction, self.config) logmsg = '%s: spear phish pattern detected, env_rcpt_domain=%s env_sender_domain=%s header_from_domain=%s' % \ (suspect.id, envelope_recipient_domain, envelope_sender_domain, header_from_domain) self.logger.info(logmsg) self.flag_as_phish(suspect, virusname) message = apply_template(self.config.get(self.section, 'rejectmessage'), suspect, {'virusname': virusname}) break return actioncode, message
def examine(self, suspect): if suspect.size > self.config.getint(self.section, 'maxsize'): self._logger().info( 'Not scanning - message too big (message %s bytes > config %s bytes )' % (suspect.size, self.config.getint(self.section, 'maxsize'))) return DUNNO try: viruses = self.scan_file(suspect.tempfile) if viruses is not None: self._logger().info("Virus found in message from %s : %s" % (suspect.from_address, viruses)) suspect.tags['virus'][self.config.get(self.section, 'identifier')] = True suspect.tags['%s.virus' % self.config.get( self.section, 'identifier')] = viruses suspect.debug('Viruses found in message : %s' % viruses) else: suspect.tags['virus'][self.config.get(self.section, 'identifier')] = False if viruses is not None: virusaction = self.config.get(self.section, 'virusaction') actioncode = string_to_actioncode(virusaction, self.config) firstinfected, firstvirusname = viruses.items()[0] values = dict(infectedfile=firstinfected, virusname=firstvirusname) message = apply_template( self.config.get(self.section, 'rejectmessage'), suspect, values) return actioncode, message else: return DUNNO except Exception as e: self._logger().warning( "Error encountered while running cmdline av scan: %s" % str(e)) return self._problemcode()
def _virusreport(self, suspect, viruses): actioncode = DUNNO message = None if viruses is not None: self.logger.info("%s Virus found in message from %s : %s" % (suspect.id, suspect.from_address, viruses)) suspect.tags['virus']['ClamAV'] = True suspect.tags['ClamavPlugin.virus'] = viruses suspect.debug('viruses found in message : %s' % viruses) else: suspect.tags['virus']['ClamAV'] = False if viruses is not None: virusaction = self.config.get(self.section, 'virusaction') actioncode = string_to_actioncode(virusaction, self.config) firstinfected, firstvirusname = list(viruses.items())[0] values = dict(infectedfile=firstinfected, virusname=firstvirusname) message = apply_template( self.config.get(self.section, 'rejectmessage'), suspect, values) return actioncode, message
def examine(self, suspect): enginename = "sophos" if suspect.size > self.config.getint(self.section, "maxsize"): self.logger.info("Not scanning - message too big") return content = suspect.get_source() for i in range(0, self.config.getint(self.section, "retries")): try: viruses = self.scan_stream(content) if viruses != None: self.logger.info("Virus found in message from %s : %s" % (suspect.from_address, viruses)) suspect.tags["virus"][enginename] = True suspect.tags["%s.virus" % enginename] = viruses suspect.debug("viruses found in message : %s" % viruses) else: suspect.tags["virus"][enginename] = False if viruses != None: virusaction = self.config.get(self.section, "virusaction") actioncode = string_to_actioncode(virusaction, self.config) firstinfected, firstvirusname = viruses.items()[0] values = dict(infectedfile=firstinfected, virusname=firstvirusname) message = apply_template(self.config.get(self.section, "rejectmessage"), suspect, values) return actioncode, message return DUNNO except Exception as e: self.logger.warning( "Error encountered while contacting SSSP server (try %s of %s): %s" % (i + 1, self.config.getint(self.section, "retries"), str(e)) ) self.logger.error("SSSP scan failed after %s retries" % self.config.getint(self.section, "retries")) content = None return self._problemcode()
def examine(self, suspect): # check if someone wants to skip sa checks if suspect.get_tag('SAPlugin.skip') == True: self.logger.debug( 'Skipping SA Plugin (requested by previous plugin)') suspect.set_tag( 'SAPlugin.skipreason', 'requested by previous plugin') return runtimeconfig = DBConfig(self.config, suspect) spamsize = suspect.size maxsize = self.config.getint(self.section, 'maxsize') spamheadername = self.config.get(self.section, 'spamheader') if spamsize > maxsize: self.logger.info('%s Size Skip, %s > %s' % (suspect.id, spamsize, maxsize)) suspect.debug('Too big for spamchecks. %s > %s' % (spamsize, maxsize)) prependheader = self.config.get('main', 'prependaddedheaders') suspect.addheader( "%sSA-SKIP" % prependheader, 'Too big for spamchecks. %s > %s' % (spamsize, maxsize)) suspect.set_tag('SAPlugin.skipreason', 'size skip') return self.check_sql_blacklist(suspect) forwardoriginal = self.config.getboolean( self.section, 'forwardoriginal') if self.config.getboolean(self.section, 'scanoriginal'): spam = suspect.get_original_source() else: spam = suspect.get_source() # prepend temporary headers set by other plugins tempheader = suspect.get_tag('SAPlugin.tempheader') if tempheader != None: if type(tempheader) == list: tempheader = "\r\n".join(tempheader) tempheader = tempheader.strip() if tempheader != '': spam = tempheader + '\r\n' + spam if forwardoriginal: ret = self.safilter_report(spam, suspect.to_address) if ret == None: suspect.debug('SA report Scan failed - please check error log') self.logger.error('%s SA report scan FAILED' % suspect.id) suspect.addheader( '%sSA-SKIP' % self.config.get('main', 'prependaddedheaders'), 'SA scan failed') suspect.set_tag('SAPlugin.skipreason', 'scan failed') return self._problemcode() isspam, spamscore, report = ret suspect.tags['SAPlugin.report'] = report else: filtered = self.safilter(spam, suspect.to_address) content = None if filtered == None: suspect.debug('SA Scan failed - please check error log') self.logger.error('%s SA scan FAILED' % suspect.id) suspect.addheader( '%sSA-SKIP' % self.config.get('main', 'prependaddedheaders'), 'SA scan failed') suspect.set_tag('SAPlugin.skipreason', 'scan failed') return self._problemcode() else: content = filtered newmsgrep = email.message_from_string(content) suspect.set_source(content) isspam, spamscore = self._extract_spamstatus( newmsgrep, spamheadername, suspect) action = DUNNO message = None if isspam: self.logger.debug('Message is spam') suspect.debug('Message is spam') configaction = string_to_actioncode( runtimeconfig.get(self.section, 'lowspamaction'), self.config) if configaction != None: action = configaction values = dict(spamscore=spamscore) message = apply_template( self.config.get(self.section, 'rejectmessage'), suspect, values) else: self.logger.debug('Message is not spam') suspect.debug('Message is not spam') suspect.tags['spam']['SpamAssassin'] = isspam suspect.tags['highspam']['SpamAssassin'] = False if spamscore != None: suspect.tags['SAPlugin.spamscore'] = spamscore highspamlevel = runtimeconfig.getint(self.section, 'highspamlevel') if spamscore >= highspamlevel: suspect.tags['highspam']['SpamAssassin'] = True configaction = string_to_actioncode( runtimeconfig.get(self.section, 'highspamaction'), self.config) if configaction != None: action = configaction return action, message
def examine(self,suspect): if self.rbllookup==None: self.rbllookup=domainmagic.rbl.RBLLookup() self.rbllookup.from_config(self.config.get(self.section,'blacklistconfig')) if self.tldmagic==None: self.tldmagic=domainmagic.tld.TLDMagic() urls=suspect.get_tag('body.uris',defaultvalue=[]) #self.logger.info("Body URIs to check: %s"%urls) domains=set(map(domainmagic.extractor.domain_from_uri,urls)) counter=0 for domain in domains: counter+=1 if counter>self.config.getint(self.section,'maxdomains'): self.logger.info("maximum number of domains reached") break tldcount=self.tldmagic.get_tld_count(domain) parts=domain.split('.') if self.config.getboolean(self.section,'checksubdomains'): subrange=range(tldcount+1,len(parts)+1) else: subrange=[tldcount+1] for subindex in subrange: subdomain='.'.join(parts[-subindex:]) listings=self.rbllookup.listings(subdomain) for identifier,humanreadable in listings.iteritems(): self.logger.info("%s : url host %s flagged as %s because %s"%(suspect.id,domain,identifier,humanreadable)) return string_to_actioncode(self.config.get(self.section,'action'), self.config),apply_template(self.config.get(self.section,'message'), suspect, dict(domain=domain,blacklist=identifier)) return DUNNO
def examine(self,suspect): #check if someone wants to skip sa checks if suspect.get_tag('SAPlugin.skip')==True: self.logger.debug('Skipping SA Plugin (requested by previous plugin)') suspect.set_tag('SAPlugin.skipreason','requested by previous plugin') return runtimeconfig=DBConfig(self.config, suspect) spamsize=suspect.size maxsize=self.config.getint(self.section, 'maxsize') spamheadername=self.config.get(self.section,'spamheader') if spamsize>maxsize: self.logger.info('%s Size Skip, %s > %s'%(suspect.id,spamsize,maxsize)) suspect.debug('Too big for spamchecks. %s > %s'%(spamsize,maxsize)) prependheader=self.config.get('main','prependaddedheaders') suspect.addheader("%sSA-SKIP"%prependheader, 'Too big for spamchecks. %s > %s'%(spamsize,maxsize)) suspect.set_tag('SAPlugin.skipreason','size skip') return self.check_sql_blacklist(suspect) starttime=time.time() forwardoriginal=self.config.getboolean(self.section,'forwardoriginal') if self.config.getboolean(self.section,'scanoriginal'): spam=suspect.getOriginalSource() else: spam=suspect.getSource() #prepend temporary headers set by other plugins tempheader=suspect.get_tag('SAPlugin.tempheader') if tempheader!=None: if type(tempheader)==list: tempheader="\r\n".join(tempheader) tempheader=tempheader.strip() if tempheader!='': spam=tempheader+'\r\n'+spam if forwardoriginal: ret=self.safilter_report(spam, suspect.to_address) if ret==None: suspect.debug('SA report Scan failed - please check error log') self.logger.error('%s SA report scan FAILED'%suspect.id) suspect.addheader('%sSA-SKIP'%self.config.get('main','prependaddedheaders'),'SA scan failed') suspect.set_tag('SAPlugin.skipreason','scan failed') return self._problemcode() isspam,spamscore,report=ret suspect.tags['SAPlugin.report']=report else: filtered=self.safilter(spam,suspect.to_address) content=None if filtered==None: suspect.debug('SA Scan failed - please check error log') self.logger.error('%s SA scan FAILED'%suspect.id) suspect.addheader('%sSA-SKIP'%self.config.get('main','prependaddedheaders'),'SA scan failed') suspect.set_tag('SAPlugin.skipreason','scan failed') return self._problemcode() else: content=filtered newmsgrep=email.message_from_string(content) suspect.setSource(content) isspam=False spamheader=newmsgrep[spamheadername] spamscore=None if spamheader==None: self.logger.warning('Did not find Header %s in returned message from SA'%spamheadername) else: if len(spamheader)>2 and 'yes' in spamheader.lower(): isspam=True patt=re.compile('Score=([\-\d\.]+)',re.IGNORECASE) m=patt.search(spamheader) if m !=None: spamscore=float(m.group(1)) self.logger.debug('Spamscore: %s'%spamscore) suspect.debug('Spamscore: %s'%spamscore) else: self.logger.warning('Could not extract spam score from header: %s'%spamheader) suspect.debug('Could not read spam score from header %s'%spamheader) action=DUNNO message=None if isspam: self.logger.debug('Message is spam') suspect.debug('Message is spam') configaction=string_to_actioncode(runtimeconfig.get(self.section,'lowspamaction'),self.config) if configaction!=None: action=configaction values=dict(spamscore=spamscore) message=apply_template(self.config.get(self.section,'rejectmessage'), suspect, values) else: self.logger.debug('Message is not spam') suspect.debug('Message is not spam') suspect.tags['spam']['SpamAssassin']=isspam suspect.tags['highspam']['SpamAssassin']=False if spamscore != None: suspect.tags['SAPlugin.spamscore']=spamscore highspamlevel=runtimeconfig.getint(self.section,'highspamlevel') if spamscore>=highspamlevel: suspect.tags['highspam']['SpamAssassin']=True configaction=string_to_actioncode(runtimeconfig.get(self.section,'highspamaction'),self.config) if configaction!=None: action=configaction endtime=time.time() difftime=endtime-starttime suspect.tags['SAPlugin.time']="%.4f"%difftime return action,message
def examine(self, suspect): # check if someone wants to skip sa checks if suspect.get_tag('SAPlugin.skip') is True: self.logger.debug('%s Skipping SA Plugin (requested by previous plugin)' % suspect.id) suspect.set_tag('SAPlugin.skipreason', 'requested by previous plugin') return DUNNO runtimeconfig = DBConfig(self.config, suspect) spamsize = suspect.size maxsize = self.config.getint(self.section, 'maxsize') strip_oversize = self.config.getboolean(self.section, 'strip_oversize') if spamsize > maxsize and not strip_oversize: self.logger.info('%s Size Skip, %s > %s' % (suspect.id, spamsize, maxsize)) suspect.debug('Too big for spamchecks. %s > %s' % (spamsize, maxsize)) prependheader = self.config.get('main', 'prependaddedheaders') suspect.addheader("%sSA-SKIP" % prependheader, 'Too big for spamchecks. %s > %s' % (spamsize, maxsize)) suspect.set_tag('SAPlugin.skipreason', 'size skip') return self.check_sql_blacklist(suspect) if self.config.getboolean(self.section, 'scanoriginal'): content = suspect.get_original_source() else: content = suspect.get_source() stripped = False if spamsize > maxsize: stripped = True # keep copy of original content before stripping content_orig = content # send maxsize-1 to be consistent with previous implementation content = suspect.source_stripped_attachments(content=content, maxsize=maxsize-1) self.logger.info('%s stripped attachments, body size reduced from %s to %s bytes' % (suspect.id, len(content_orig), len(content))) # stick to bytes content = force_bString(content) # prepend temporary headers set by other plugins tempheader = suspect.get_tag('SAPlugin.tempheader') if tempheader is not None: if isinstance(tempheader, list): tempheader = "\r\n".join(tempheader) tempheader = tempheader.strip() if tempheader != '': content = force_bString(tempheader + '\r\n') + content # add envelope sender information msgrep = suspect.get_message_rep() if not 'Return-Path' in msgrep.keys(): content = force_bString('Return-Path: %s' % suspect.from_address + '\r\n') + content forwardoriginal = self.config.getboolean(self.section, 'forwardoriginal') if forwardoriginal: ret = self.safilter_report(content, suspect.to_address) if ret is None: suspect.debug('SA report Scan failed - please check error log') self.logger.error('%s SA report scan FAILED' % suspect.id) suspect.addheader('%sSA-SKIP' % self.config.get('main', 'prependaddedheaders'), 'SA scan failed') suspect.set_tag('SAPlugin.skipreason', 'scan failed') return self._problemcode() isspam, spamscore, report = ret suspect.tags['SAPlugin.report'] = report else: filtered = self.safilter(content, suspect.to_address) if filtered is None: suspect.debug('SA Scan failed - please check error log') self.logger.error('%s SA scan FAILED' % suspect.id) suspect.addheader('%sSA-SKIP' % self.config.get('main', 'prependaddedheaders'), 'SA scan failed') suspect.set_tag('SAPlugin.skipreason', 'scan failed') return self._problemcode() else: if stripped: # create msgrep of filtered msg if isinstance(content,str): msgrep_filtered = email.message_from_string(filtered, _class=PatchedMessage) else: msgrep_filtered = email.message_from_bytes(filtered, _class=PatchedMessage) header_new = [] for h,v in msgrep_filtered.items(): header_new.append(force_uString(h).strip() + ': ' + force_uString(v).strip()) # add headers to msg sa_prepend = self.config.get(self.section, 'spamheader_prepend') for i in header_new: if sa_prepend == '' or sa_prepend is None: break if re.match('^' + sa_prepend + '[^:]+: ', i, re.I): # in case of stripped msg add header to original content content_orig = force_bString(i) + b'\r\n' + force_bString(content_orig) else: continue content = content_orig else: content = filtered if isinstance(content,str): newmsgrep = email.message_from_string(content, _class=PatchedMessage) else: newmsgrep = email.message_from_bytes(content, _class=PatchedMessage) # if original content is forwarded there's no need to reset the attachmant # manager. Only header have been changed. suspect.set_source(content,att_mgr_reset=(not forwardoriginal)) spamheadername = self.config.get(self.section, 'spamheader') isspam, spamscore, report = self._extract_spamstatus(newmsgrep, spamheadername, suspect) suspect.tags['SAPlugin.report'] = report self.logger.debug('suspect %s %s %s %s' % (suspect.id, isspam, spamscore, suspect.get_tag('SAPlugin.report'))) action = DUNNO message = None if isspam: self.logger.debug('%s Message is spam' % suspect.id) suspect.debug('Message is spam') configaction = string_to_actioncode( runtimeconfig.get(self.section, 'lowspamaction'), self.config) if configaction is not None: action = configaction values = dict(spamscore=spamscore) message = apply_template( self.config.get(self.section, 'rejectmessage'), suspect, values) else: self.logger.debug('%s Message is not spam' % suspect.id) suspect.debug('Message is not spam') suspect.tags['spam']['SpamAssassin'] = isspam suspect.tags['highspam']['SpamAssassin'] = False if spamscore is not None: suspect.tags['SAPlugin.spamscore'] = spamscore highspamlevel = runtimeconfig.getfloat(self.section, 'highspamlevel') if spamscore >= highspamlevel: suspect.tags['highspam']['SpamAssassin'] = True configaction = string_to_actioncode(runtimeconfig.get(self.section, 'highspamaction'), self.config) if configaction is not None: action = configaction return action, message
def examine(self, suspect): if self.limiters == None: filename = self.config.get(self.section, 'limiterfile') if not os.path.exists(filename): self.logger.error("Limiter config file %s not found" % filename) return limiterconfig = open(filename, 'r').read() limiters = self.load_limiter_config(limiterconfig) self.limiters = limiters self.logger.info("Found %s limiter configurations" % (len(limiters))) if self.backend_instance == None: btype = self.config.get(self.section, 'backendtype') if btype not in AVAILABLE_RATELIMIT_BACKENDS: self.logger.error('ratelimit backend %s not available' % (btype)) return self.backend_instance = AVAILABLE_RATELIMIT_BACKENDS[btype]( self.config.get(self.section, 'backendconfig')) skiplist = [] for limiter in self.limiters: if limiter.name in skiplist: # check if this limiter is skipped by a previous one self.logger.debug('limiter %s skipped due to previous match' % limiter.name) continue #get field values allfieldsavailable = True fieldvalues = [] for fieldname in limiter.fields: values = self.filter.get_field(suspect, fieldname) if len(values) < 1: allfieldsavailable = False self.logger.debug( 'Skipping limiter %s - field %s not available' % (limiter.name, fieldname)) break fieldvalues.append(values[0]) if not allfieldsavailable: #rate limit can not be applied continue checkval = ','.join(fieldvalues) if limiter.regex != None: if re.match(limiter.regex, checkval): if limiter.skip != None: skiplist.extend(limiter.skip) else: #no match, skip this limiter self.logger.debug( 'Skipping limiter %s - regex does not match' % (limiter.name)) continue #self.logger.debug("check %s"%str(limiter)) eventname = limiter.name + checkval timespan = limiter.timespan max = limiter.max if max < 0: #no limit continue event_count = self.backend_instance.check_count( eventname, timespan) self.logger.debug("Limiter event %s count: %s" % (eventname, event_count)) if event_count > max: return limiter.action, apply_template(limiter.message, suspect)
def archive(self, suspect): archivedir = self.config.get(self.section, 'archivedir') if archivedir == "": self.logger.error('Archivedir is not specified') return subdirtemplate = self.config.get(self.section, 'subdirtemplate') if self.config.has_option( self.section, 'makedomainsubdir' ) and subdirtemplate == self.requiredvars['subdirtemplate']['default']: self.logger.warning( "Archive config is using deprecated 'makedomainsubdir' config option. Emulating old behaviour. Update your config(subdirtemplate)" ) if self.config.getboolean(self.section, 'makedomainsubdir'): subdirtemplate = "${to_domain}" else: subdirtemplate = "" # the archive root dir startdir = os.path.abspath(archivedir) # relative dir within archive root subdir = apply_template(subdirtemplate, suspect) if subdir.endswith('/'): subdir = subdir[:-1] # filename without dir filenametemplate = self.config.get(self.section, 'filenametemplate') filename = apply_template(filenametemplate, suspect) # make sure filename can't create new folders filename = filename.replace('/', '_') # full relative filepath within archive dir fpath = "%s/%s" % (subdir, filename) # absolute final filepath requested_path = os.path.abspath("%s/%s" % (startdir, fpath)) if not os.path.commonprefix([requested_path, startdir ]).startswith(startdir): self.logger.error( "file path '%s' seems to be outside archivedir '%s' - storing to archivedir" % (requested_path, startdir)) requested_path = "%s/%s" % (startdir, filename) finaldir = os.path.dirname(requested_path) if not os.path.isdir(finaldir): os.makedirs(finaldir, 0o755) if self.config.getboolean(self.section, 'storeoriginal'): shutil.copy(suspect.tempfile, requested_path) else: with open(requested_path, 'w') as fp: fp.write(suspect.get_source()) chmod = self.config.get(self.section, 'chmod') chgrp = self.config.get(self.section, 'chgrp') chown = self.config.get(self.section, 'chown') if chmod or chgrp or chown: self.setperms(requested_path, chmod, chgrp, chown) self.logger.info( 'Message from %s to %s archived as %s' % (suspect.from_address, suspect.to_address, requested_path)) return requested_path
def examine(self, suspect): # check if someone wants to skip sa checks if suspect.get_tag('SAPlugin.skip') is True: self.logger.debug( '%s Skipping SA Plugin (requested by previous plugin)' % suspect.id) suspect.set_tag('SAPlugin.skipreason', 'requested by previous plugin') return DUNNO runtimeconfig = DBConfig(self.config, suspect) spamsize = suspect.size maxsize = self.config.getint(self.section, 'maxsize') strip_oversize = self.config.getboolean(self.section, 'strip_oversize') if spamsize > maxsize and not strip_oversize: self.logger.info('%s Size Skip, %s > %s' % (suspect.id, spamsize, maxsize)) suspect.debug('Too big for spamchecks. %s > %s' % (spamsize, maxsize)) prependheader = self.config.get('main', 'prependaddedheaders') suspect.addheader( "%sSA-SKIP" % prependheader, 'Too big for spamchecks. %s > %s' % (spamsize, maxsize)) suspect.set_tag('SAPlugin.skipreason', 'size skip') return self.check_sql_blacklist(suspect) if self.config.getboolean(self.section, 'scanoriginal'): content = suspect.get_original_source() else: content = suspect.get_source() stripped = False if spamsize > maxsize: stripped = True # keep copy of original content before stripping content_orig = content content = self._strip_attachments(content, maxsize) self.logger.info( '%s stripped attachments, body size reduced from %s to %s bytes' % (suspect.id, len(content_orig), len(content))) # stick to bytes content = force_bString(content) # prepend temporary headers set by other plugins tempheader = suspect.get_tag('SAPlugin.tempheader') if tempheader is not None: if isinstance(tempheader, list): tempheader = "\r\n".join(tempheader) tempheader = tempheader.strip() if tempheader != '': content = force_bString(tempheader + '\r\n') + content forwardoriginal = self.config.getboolean(self.section, 'forwardoriginal') if forwardoriginal: ret = self.safilter_report(content, suspect.to_address) if ret is None: suspect.debug('SA report Scan failed - please check error log') self.logger.error('%s SA report scan FAILED' % suspect.id) suspect.addheader( '%sSA-SKIP' % self.config.get('main', 'prependaddedheaders'), 'SA scan failed') suspect.set_tag('SAPlugin.skipreason', 'scan failed') return self._problemcode() isspam, spamscore, report = ret suspect.tags['SAPlugin.report'] = report else: filtered = self.safilter(content, suspect.to_address) if filtered is None: suspect.debug('SA Scan failed - please check error log') self.logger.error('%s SA scan FAILED' % suspect.id) suspect.addheader( '%sSA-SKIP' % self.config.get('main', 'prependaddedheaders'), 'SA scan failed') suspect.set_tag('SAPlugin.skipreason', 'scan failed') return self._problemcode() else: if stripped: # create msgrep of filtered msg msgrep_filtered = email.message_from_string(filtered) header_new = [] header_old = [] # create a msgrep from original msg msgrep_orig = email.message_from_string(content_orig) # read all headers from after-scan and before-scan for h, v in msgrep_filtered.items(): header_new.append(h.strip() + ': ' + v.strip()) for h, v in msgrep_orig.items(): header_old.append(h.strip() + ': ' + v.strip()) # create a list of headers added by spamd # header diff between before-scan and after-scan msg header_new = reversed(self.diff(header_new, header_old)) # add headers to msg for i in header_new: if re.match('^Received: ', i, re.I): continue # in case of stripped msg add header to original content content_orig = i + '\r\n' + content_orig content = content_orig else: content = filtered if sys.version_info > (3, ): # Python 3 and larger # the basic "str" type is unicode if isinstance(content, str): newmsgrep = email.message_from_string(content) else: newmsgrep = email.message_from_bytes(content) else: # Python 2.x newmsgrep = email.message_from_string(content) suspect.set_source(content) spamheadername = self.config.get(self.section, 'spamheader') isspam, spamscore, report = self._extract_spamstatus( newmsgrep, spamheadername, suspect) suspect.tags['SAPlugin.report'] = report self.logger.debug('suspect %s %s %s %s' % (suspect.id, isspam, spamscore, suspect.get_tag('SAPlugin.report'))) action = DUNNO message = None if isspam: self.logger.debug('%s Message is spam' % suspect.id) suspect.debug('Message is spam') configaction = string_to_actioncode( runtimeconfig.get(self.section, 'lowspamaction'), self.config) if configaction is not None: action = configaction values = dict(spamscore=spamscore) message = apply_template( self.config.get(self.section, 'rejectmessage'), suspect, values) else: self.logger.debug('%s Message is not spam' % suspect.id) suspect.debug('Message is not spam') suspect.tags['spam']['SpamAssassin'] = isspam suspect.tags['highspam']['SpamAssassin'] = False if spamscore is not None: suspect.tags['SAPlugin.spamscore'] = spamscore highspamlevel = runtimeconfig.getfloat(self.section, 'highspamlevel') if spamscore >= highspamlevel: suspect.tags['highspam']['SpamAssassin'] = True configaction = string_to_actioncode( runtimeconfig.get(self.section, 'highspamaction'), self.config) if configaction is not None: action = configaction return action, message
def examine(self, suspect): # check if someone wants to skip sa checks if suspect.get_tag('SAPlugin.skip') is True: self.logger.debug( '%s Skipping SA Plugin (requested by previous plugin)' % suspect.id) suspect.set_tag('SAPlugin.skipreason', 'requested by previous plugin') return DUNNO runtimeconfig = DBConfig(self.config, suspect) spamsize = suspect.size maxsize = self.config.getint(self.section, 'maxsize') spamheadername = self.config.get(self.section, 'spamheader') if spamsize > maxsize: self.logger.info('%s Size Skip, %s > %s' % (suspect.id, spamsize, maxsize)) suspect.debug('Too big for spamchecks. %s > %s' % (spamsize, maxsize)) prependheader = self.config.get('main', 'prependaddedheaders') suspect.addheader( "%sSA-SKIP" % prependheader, 'Too big for spamchecks. %s > %s' % (spamsize, maxsize)) suspect.set_tag('SAPlugin.skipreason', 'size skip') return self.check_sql_blacklist(suspect) forwardoriginal = self.config.getboolean(self.section, 'forwardoriginal') if self.config.getboolean(self.section, 'scanoriginal'): spam = suspect.get_original_source() else: spam = suspect.get_source() # prepend temporary headers set by other plugins tempheader = suspect.get_tag('SAPlugin.tempheader') if tempheader is not None: if type(tempheader) == list: tempheader = "\r\n".join(tempheader) tempheader = tempheader.strip() if tempheader != '': spam = tempheader + '\r\n' + spam if forwardoriginal: ret = self.safilter_report(spam, suspect.to_address) if ret is None: suspect.debug('SA report Scan failed - please check error log') self.logger.error('%s SA report scan FAILED' % suspect.id) suspect.addheader( '%sSA-SKIP' % self.config.get('main', 'prependaddedheaders'), 'SA scan failed') suspect.set_tag('SAPlugin.skipreason', 'scan failed') return self._problemcode() isspam, spamscore, report = ret suspect.tags['SAPlugin.report'] = report else: filtered = self.safilter(spam, suspect.to_address) if filtered is None: suspect.debug('SA Scan failed - please check error log') self.logger.error('%s SA scan FAILED' % suspect.id) suspect.addheader( '%sSA-SKIP' % self.config.get('main', 'prependaddedheaders'), 'SA scan failed') suspect.set_tag('SAPlugin.skipreason', 'scan failed') return self._problemcode() else: content = filtered newmsgrep = email.message_from_string(content) suspect.set_source(content) isspam, spamscore = self._extract_spamstatus( newmsgrep, spamheadername, suspect) action = DUNNO message = None if isspam: self.logger.debug('%s Message is spam' % suspect.id) suspect.debug('Message is spam') configaction = string_to_actioncode( runtimeconfig.get(self.section, 'lowspamaction'), self.config) if configaction is not None: action = configaction values = dict(spamscore=spamscore) message = apply_template( self.config.get(self.section, 'rejectmessage'), suspect, values) else: self.logger.debug('%s Message is not spam' % suspect.id) suspect.debug('Message is not spam') suspect.tags['spam']['SpamAssassin'] = isspam suspect.tags['highspam']['SpamAssassin'] = False if spamscore is not None: suspect.tags['SAPlugin.spamscore'] = spamscore highspamlevel = runtimeconfig.getfloat(self.section, 'highspamlevel') if spamscore >= highspamlevel: suspect.tags['highspam']['SpamAssassin'] = True configaction = string_to_actioncode( runtimeconfig.get(self.section, 'highspamaction'), self.config) if configaction is not None: action = configaction return action, message