def handlesession(self): line = force_uString(self.socket.recv(4096)).strip() if line == '': self.socket.close() return self.logger.debug('Control Socket command: %s' % line) answer = None try: if line.startswith("objgraph"): # special handling for objgraph # -> argument is a dict in json format # -> check attributes for commands, don't use list parts = line.split(maxsplit=1) if len(parts) == 2: argsdict = ControlSession.json_string_to_obj( parts[1], ForcedType=dict) else: argsdict = {} self.logger.debug('objgraph_growth: args dict: %s' % argsdict) answer = self.handle_command(parts[0], argsdict, checkattr=True) else: # default handling line = line.lower() parts = line.split() answer = self.handle_command(parts[0], parts[1:]) except Exception as e: if not answer: answer = force_uString(e) else: answer += force_uString(e) self.socket.sendall(force_bString(answer)) self.socket.close()
def dict_unicode(command_dict): commanddictstring = u"" if command_dict: for key, value in iter(command_dict.items()): commanddictstring += force_uString( key) + u": " + force_uString(value) + u", " return commanddictstring
def test_SMTPUTF8_E2E(self): """test if a UTF-8 message runs through""" # give fuglu time to start listener time.sleep(1) root = logging.getLogger() root.setLevel(logging.DEBUG) ch = logging.StreamHandler(sys.stdout) ch.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') ch.setFormatter(formatter) root.addHandler(ch) # send test message smtpclient = smtplib.SMTP('127.0.0.1', EndtoEndBaseTestCase.FUGLU_PORT) # smtpServer.set_debuglevel(1) (code, msg) = smtpclient.ehlo('test.e2e') msg = force_uString(msg.split()) self.assertEqual(250, code) print("%s"%msg) self.assertIn("SMTPUTF8", msg) testunicodemessage = u"""Hello Wörld!\r Don't där yü tschänsch äny of mai baits or iwen remüv ön!""" # TODO: this test fails if we don't put in the \r in there... (eg, # fuglu adds it) - is this a bug or wrong test? msg = MIMEText(testunicodemessage, _charset='utf-8') msg["Subject"] = "End to End Test" msgstring = msg.as_string() inbytes = len(msg.get_payload(decode=True)) # envelope sender/recipients env_sender = u'sä[email protected]' env_recipients = [u'rö[email protected]', u'récipiè[email protected]'] smtpclient.sendmail(force_uString(env_sender), force_uString(env_recipients), force_bString(msgstring), mail_options=["SMTPUTF8"]) smtpclient.quit() # get answer (wait to give time to create suspect) time.sleep(0.1) gotback = self.smtp.suspect self.assertFalse(gotback == None, "Did not get message from dummy smtp server") # check a few things on the received message msgrep = gotback.get_message_rep() self.assertTrue('X-Fuglutest-Spamstatus' in msgrep, "Fuglu SPAM Header not found in message") payload = msgrep.get_payload(decode=True) outbytes = len(payload) self.assertEqual(inbytes, outbytes,"Message size change: bytes in: %u, bytes out %u" % (inbytes, outbytes)) self.assertEqual(testunicodemessage, force_uString(payload), "Message body has been altered. In: %u bytes, Out: %u bytes, teststring=->%s<- result=->%s<-" % (inbytes, outbytes, testunicodemessage, force_uString(payload))) # check sender/recipients self.assertEqual(env_sender, gotback.from_address) self.assertEqual(env_recipients, gotback.recipients)
def remove_headers(self): """ Remove all original headers """ for key, value in self.sess.original_headers: self.logger.debug("Remove header-> %s: %s" % (force_uString(key), force_uString(value))) self.changeheader(key, b"") self.sess.original_headers = []
def lint_ping(self): try: s = self.__init_socket__(oneshot=True) except Exception as e: print("Could not contact clamd: %s" % (str(e))) return False s.sendall(force_bString('PING')) result = s.recv(20000) print("Got Pong: %s" % force_uString(result)) if result.strip() != b'PONG': print("Invalid PONG: %s" % force_uString(result)) return True
def test_reinject_error(self): """test if a reinject error is passed""" # give fuglu time to start listener time.sleep(1) import logging import sys root = logging.getLogger() root.setLevel(logging.DEBUG) ch = logging.StreamHandler(sys.stdout) ch.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') ch.setFormatter(formatter) root.addHandler(ch) # send test message smtpclient = smtplib.SMTP('127.0.0.1', ReinjectErrorTestCase.FUGLU_PORT) # smtpServer.set_debuglevel(1) (code, msg) = smtpclient.helo('test.e2e') self.assertEqual(250, code) testmessage = u"""Hello World!""" # TODO: this test fails if we don't put in the \r in there... (eg, # fuglu adds it) - is this a bug or wrong test? msg = MIMEText(testmessage) msg["Subject"] = "End to End Test" msgstring = msg.as_string() inbytes = len(msg.get_payload(decode=True)) # envelope sender/recipients env_sender = u'*****@*****.**' env_recipients = [u'*****@*****.**'] self.smtp.response_code = 554 # python 3 returnes bytes self.smtp.response_message = '5.4.0 Error: too many hops' try: smtpclient.sendmail(force_uString(env_sender), force_uString(env_recipients), force_bString(msgstring)) except smtplib.SMTPDataError as e: self.assertEqual(self.smtp.response_code, e.smtp_code) self.assertEqual(self.smtp.response_message, force_uString(e.smtp_error)) pass
def append(self, filename, cause, message): """ Append a new info about noExtraction Args: filename (str, unicode): filename causing the issue cause (str, unicode): reason for being added, must be in valid_causes message (str, unicode): an additional message """ assert cause in NoExtractInfo.valid_causes self.infolists[cause].append( (force_uString(filename), force_uString(message)))
def Dispatch(self, data): """Callback function for the milter socket server to handle a single milter command. Parses the milter command data, invokes the milter handler, and formats a suitable response for the server to send on the socket. Args: data: A (binary) string (consisting of a command code character followed by binary data for that command code). Returns: A binary string to write on the socket and return to sendmail. The string typically consists of a RESPONSE[] command character then some response-specific protocol data. Raises: PpyMilterCloseConnection: Indicating the (milter) connection should be closed. """ cmd, data = data[0], data[1:] try: if cmd not in COMMANDS: logging.warn('Unknown command code: "%s" ("%s")', force_uString(cmd), force_uString(data)) return RESPONSE['CONTINUE'] u_command = force_uString(COMMANDS[cmd]) # (unicode) parser_callback_name = '_Parse%s' % u_command handler_callback_name = 'On%s' % u_command if not hasattr(self, parser_callback_name): logging.error('No parser implemented for "%s"', u_command) return RESPONSE['CONTINUE'] if not hasattr(self.__milter, handler_callback_name): logging.warn( 'Unimplemented command in milter %s: "%s" ("%s")' % (self.__milter, u_command, data)) return RESPONSE['CONTINUE'] parser = getattr(self, parser_callback_name) callback = getattr(self.__milter, handler_callback_name) args = parser(cmd, data) return callback(*args) except PpyMilterTempFailure as e: logging.info('Temp Failure: %s', str(e)) return RESPONSE['TEMPFAIL'] except PpyMilterPermFailure as e: logging.info('Perm Failure: %s', str(e)) return RESPONSE['REJECT']
def test_decode2unicode(self): """Test if strings are correctly decoded to unicode string""" self.assertEqual(str, type(force_uString("bla")), "After conversion, type has to be unicode") self.assertEqual(str, type(force_uString(u"bla")), "After conversion, type has to be unicode") self.assertEqual(str, type(force_uString(b"bla")), "After conversion, type has to be unicode") mixedlist = ["bla", u"bla", b"bla"] for item in force_uString(mixedlist): self.assertEqual(str, type(item), "After conversion, type has to be unicode") self.assertEqual(u"bla", item, "String has to match the test string u\"bla\"")
def get_decoded_textparts_deprecated(self, suspect): """Returns a list of all text contents""" messagerep = suspect.get_message_rep() textparts = [] for part in messagerep.walk(): if part.is_multipart(): continue fname = part.get_filename(None) if fname is None: fname = "" fname = fname.lower() contenttype = part.get_content_type() if contenttype.startswith('text/') or fname.endswith( ".txt") or fname.endswith(".html") or fname.endswith( ".htm"): payload = part.get_payload(None, True) if payload is not None: # Try to decode using the given char set (or utf-8 by default) charset = part.get_content_charset("utf-8") payload = force_uString(payload, encodingGuess=charset) if 'html' in contenttype or '.htm' in fname: #remove newlines from html so we get uris spanning multiple lines payload = payload.replace('\n', '').replace('\r', '') try: payload = self.htmlparser.unescape(payload) except Exception: self.logger.debug('%s failed to unescape html entities' % suspect.id) textparts.append(payload) if contenttype == 'multipart/alternative': try: payload = part.get_payload(None, True) if payload is not None: # Try to decode using the given char set charset = part.get_content_charset("utf-8") text = force_uString(payload, encodingGuess=charset) textparts.append(text) except (UnicodeEncodeError, UnicodeDecodeError): self.logger.debug( '%s failed to convert alternative part to string' % suspect.id) return textparts
def lint_ping(self): """ping sa""" retries = self.config.getint(self.section, 'retries') for i in range(0, retries): try: self.logger.debug('Contacting spamd (Try %s of %s)' % (i + 1, retries)) s = self.__init_socket() s.sendall(b'PING SPAMC/1.2') s.sendall(b"\r\n") s.shutdown(socket.SHUT_WR) socketfile = s.makefile("rb") line = force_uString(socketfile.readline()) line = line.strip() answer = line.split() if len(answer) != 3: print("Invalid SPAMD PONG: %s" % line) return False if answer[2] != "PONG": print("Invalid SPAMD Pong: %s" % line) return False print("Got: %s" % line) return True except socket.timeout: print('SPAMD Socket timed out.') except socket.herror as h: print('SPAMD Herror encountered : %s' % str(h)) except socket.gaierror as g: print('SPAMD gaierror encountered: %s' % str(g)) except socket.error as e: print('SPAMD socket error: %s' % str(e)) time.sleep(1) return False
def process(self, suspect, decision): recipient = force_uString( suspect.to_domain) # work with unicode string if self.config.get(self.section, 'level') == 'email': recipient = suspect.to_address recipient = recipient.replace('.', '-') recipient = recipient.replace('@', '--') host = self.config.get(self.section, 'host') port = self.config.getint(self.section, 'port') buffer = "" if self.sock is None: addr_f = socket.getaddrinfo(host, 0)[0][0] self.sock = socket.socket(addr_f, socket.SOCK_DGRAM) if suspect.is_virus(): buffer = "%s%s.fuglu.recipient.%s.virus:1|c\n" % ( buffer, self.nodename, recipient) elif suspect.is_highspam(): buffer = "%s%s.fuglu.recipient.%s.highspam:1|c\n" % ( buffer, self.nodename, recipient) elif suspect.is_spam(): buffer = "%s%s.fuglu.recipient.%s.spam:1|c\n" % ( buffer, self.nodename, recipient) else: buffer = "%s%s.fuglu.recipient.%s.clean:1|c\n" % ( buffer, self.nodename, recipient) self.sock.sendto(force_bString(buffer), (host, port))
def scan_stream(self, content, suspectid='(NA)'): """ Scan a buffer content (string) : buffer to scan return either : - (dict) : {filename1: "virusname"} - None if no virus found """ s = self.__init_socket__() content = force_bString(content) buflen = len(content) s.sendall(force_bString('SCAN %s STREAM fu_stream SIZE %s' % (self.config.get(self.section, 'scanoptions'), buflen))) s.sendall(b'\n') self.logger.debug('%s Sending buffer (length=%s) to fpscand...' % (suspectid, buflen)) s.sendall(content) self.logger.debug('%s Sent %s bytes to fpscand, waiting for scan result' % (suspectid, buflen)) result = force_uString(s.recv(20000)) if len(result) < 1: self.logger.error('Got no reply from fpscand') s.close() return self._parse_result(result)
def archive_handle(self): """ (Cached Property-Getter) Create an archive handle to check, extract, ... files in the buffered archive. Internal: - archive_type: The archive type (already detected) - buffer: The file buffer containing the archive Returns: (Archivehandle) : The handle to work with the archive """ # make sure there's no buffered archive object when # the archive handle is created (or overwritten) self._buffer_archobj = {} handle = None if self.buffer is not None: try: handle = Archivehandle(self.archive_type, BytesIO(self.buffer), archivename=self.filename) except Exception as e: self.logger.error( "%s, Problem creating Archivehandle for file: " "%s using archive handler %s (message: %s) -> ignore" % (self.fugluid, self.filename, str( self.archive_type), force_uString(e))) return handle
def testSMIME(self): """test if S/MIME mails still pass the signature""" # give fuglu time to start listener time.sleep(1) # send test message smtpclient = smtplib.SMTP('127.0.0.1', SMIMETestCase.FUGLU_PORT) # smtpServer.set_debuglevel(1) smtpclient.helo('test.smime') inputfile = TESTDATADIR + '/smime/signedmessage.eml' (status, output) = self.verifyOpenSSL(inputfile) self.assertTrue( status == 0, "Testdata S/MIME verification failed: \n%s" % output) msgstring = open(inputfile, 'r').read() smtpclient.sendmail( '*****@*****.**', '*****@*****.**', force_uString(msgstring)) smtpclient.quit() # verify the smtp server stored the file correctly tmpfile = self.smtp.tempfilename #self.failUnlessEqual(msgstring, tmpcontent, "SMTP Server did not store the tempfile correctly: %s"%tmpfile) (status, output) = self.verifyOpenSSL(tmpfile) self.assertTrue( status == 0, "S/MIME verification failed: \n%s\n tmpfile is:%s" % (output, tmpfile))
def connect(self, hostname, family, ip, port, command_dict): self.log('Connect from %s:%d (%s) with family: %s, dict: %s' % (ip, port, hostname, family, str(command_dict))) self.store_info_from_dict(command_dict) if family not in (b'4', b'6'): # we don't handle unix socket self.logger.error('Return temporary fail since family is: %s' % force_uString(family)) self.logger.error(u'command dict is: %s' % MilterSession.dict_unicode(command_dict)) return lm.TEMPFAIL if hostname is None or force_uString( hostname) == u'[%s]' % force_uString(ip): hostname = u'unknown' self.rdns = hostname self.addr = ip return lm.CONTINUE
def get_cleaned_from_address(self): """Return from_address, without <> qualification or other MAIL FROM parameters""" from_address_cleaned = "" if self.from_address is not None: fromaddr = force_uString(self.from_address) fromaddr_split = fromaddr.split(u'\0', maxsplit=1) from_address_cleaned = fromaddr_split[0].strip(u'<>') return from_address_cleaned
def test_nonstringinput(self): self.assertEqual(str, type(force_uString(1)), "After conversion, type has to be unicode") self.assertEqual(str, type(force_uString(1.3e-2)), "After conversion, type has to be unicode") class WithUnicode(object): def __unicode__(self): return u"I have unicode" def __str__(self): return "I also have str" class WithStr(object): def __str__(self): return "I have str" print(force_uString(WithUnicode())) print(force_uString(WithStr())) self.assertEqual( str, type(force_uString(WithUnicode())), "Class has __unicode__ and __str__ (Py2: __unicode__ / Py3: __str__" ) self.assertEqual(str, type(force_uString(WithStr())), "Class has __str__ (Py2/3: __str__") for item in force_uString([int(1), "bla", 1.3e-2]): self.assertEqual(str, type(item), "After conversion, type has to be unicode")
def runArchiveChecks(self, handle): archive_flist = handle.namelist() self.assertEqual(["test.txt"], archive_flist) # file should not be extracted if maximum size to extract a file is 0 extracted = handle.extract(archive_flist[0], 0) self.assertEqual(None, extracted) extracted = handle.extract(archive_flist[0], 500000) self.assertEqual(u"This is a test\n", force_uString(extracted))
def lint_version(self): try: s = self.__init_socket__(oneshot=True) except Exception: return False s.sendall(b'VERSION') result = s.recv(20000) print("Got Version: %s" % force_uString(result)) return True
def remove_recipients(self): """ Remove all the original envelope recipients """ # use the recipient data from the session because # it has to match exactly for recipient in self.sess.recipients: self.logger.debug("Remove env recipient: %s" % force_uString(recipient)) self.sess.delRcpt(recipient) self.sess.recipients = []
def safilter_symbols(self, messagecontent, user): """Pass content to sa, return spamflag, score, rules""" ret = self._safilter_content(messagecontent, user, 'SYMBOLS') if ret is None: return None status, score, content = ret content = force_uString(content) rules = content.split(',') return status, score, rules
def _read_options(self, s): """ Reads a message which should be a list of options and transforms them into a dictionary :param s: the socket :return: dict: options supported by sophos """ resp = self._receive_msg(s) opts = {} for l in resp: parts = optionsyntax.findall(l) for p in parts: p0 = force_uString(p[0]) if p0 not in opts: opts[p0] = [] opts[p0].append(force_uString(p[1])) return opts
def objgraph_count_types(self, args): """ This function can be used to display the number of objects for one or several types of objects. For now this works best for fuglu with thread backend. Fuglu has to be running as a daemon. "fuglu_control" is used to communicate with the fuglu instance. Examples: (1) Count ans sum objects given by a list ----------------------------------------- $ fuglu_control objgraph_count_types '{"typelist":["Worker","Suspect","SessionHandler"]}' --------------- Count suspects: --------------- params: * typelist: Worker,Suspect,SessionHandler Object types found in memory: Worker : 2 Suspect : 0 SessionHandler : 1 """ res = u"---------------\n" \ + u"Count suspects:\n" \ + u"---------------\n\n" defaults = { "typelist": ["Suspect", "Mailattachment", "Mailattachment_mgr"] } if OBJGRAPH_EXTENSION_ENABLED: if not args: args = {} # fill filter lists and other vars from dict res, inputdict = ControlSession.prepare_objectgraph_list_from_dict( args, res, defaults) try: res += u"Object types found in memory:\n" for otype in inputdict["typelist"]: n_otypes = objgraph.count(otype) res += u"%s : %u\n" % (otype, n_otypes) except Exception as e: res += u"ERROR: %s" % force_uString(e) self.logger.exception(e) else: res += u"please install module 'objgraph'" return res
def get_cleaned_recipients(self): """Return recipient addresses, without <> qualification or other RCPT TO parameters""" to_addresses_cleaned = [] if self.recipients is not None: for rec in self.recipients: if rec is not None: recipient = force_uString(rec) recipient_split = recipient.split(u'\0', maxsplit=1) recipient_cleaned = recipient_split[0].strip(u'<>') to_addresses_cleaned.append(recipient_cleaned) return to_addresses_cleaned
def safilter(self, messagecontent, user): """pass content to sa, return sa-processed mail""" retries = self.config.getint(self.section, 'retries') peruserconfig = self.config.getboolean(self.section, 'peruserconfig') spamsize = len(messagecontent) for i in range(0, retries): try: self.logger.debug('Contacting spamd (Try %s of %s)' % (i + 1, retries)) s = self.__init_socket() s.sendall(force_bString('PROCESS SPAMC/1.2')) s.sendall(force_bString("\r\n")) s.sendall(force_bString("Content-length: %s" % spamsize)) s.sendall(force_bString("\r\n")) if peruserconfig: s.sendall(force_bString("User: %s" % user)) s.sendall(force_bString("\r\n")) s.sendall(force_bString("\r\n")) s.sendall(force_bString(messagecontent)) self.logger.debug('Sent %s bytes to spamd' % spamsize) s.shutdown(socket.SHUT_WR) socketfile = s.makefile("rb") line1_info = socketfile.readline() line1_info = force_uString(line1_info) # convert to unicode string self.logger.debug(line1_info) line2_contentlength = socketfile.readline() line3_empty = socketfile.readline() content = socketfile.read() self.logger.debug('Got %s message bytes from back from spamd' % len(content)) answer = line1_info.strip().split() if len(answer) != 3: self.logger.error("Got invalid status line from spamd: %s" % line1_info) continue version, number, status = answer if status != 'EX_OK': self.logger.error("Got bad status from spamd: %s" % status) continue return content except socket.timeout: self.logger.error('SPAMD Socket timed out.') except socket.herror as h: self.logger.error('SPAMD Herror encountered : %s' % str(h)) except socket.gaierror as g: self.logger.error('SPAMD gaierror encountered: %s' % str(g)) except socket.error as e: self.logger.error('SPAMD socket error: %s' % str(e)) except Exception as e: self.logger.error('SPAMD communication error: %s' % str(e)) time.sleep(1) return None
def store_info_from_dict(self, command_dict): """Extract and store additional info passed by dict""" if command_dict: if not self.queueid: queueid = command_dict.get(b'i', None) if queueid: self.queueid = force_uString(queueid) if not self.sasl_login: sasl_login = command_dict.get(b'auth_authen', None) if sasl_login: self.sasl_login = force_uString(sasl_login) if not self.sasl_sender: sasl_sender = command_dict.get(b'auth_author', None) if sasl_sender: self.sasl_sender = force_uString(sasl_sender) if not self.sasl_method: sasl_method = command_dict.get(b'auth_type', None) if sasl_method: self.sasl_method = force_uString(sasl_method)
def forwardCommand(self, command): """forward a esmtp command to outgoing postfix instance Args: command (str): command in unicode Returns (str): reply from outgoing server """ command = command.strip() if self.forwardconn is None: targethost = self.config.get('main', 'outgoinghost') if targethost == '${injecthost}': targethost = self.socket.getpeername()[0] self.forwardconn = smtplib.SMTP( force_uString(targethost), self.config.getint('main', 'outgoingport')) self.logger.debug("""SEND: "%s" """ % command) # docmd seems to have a normal string as input, so # I guess unicode will work for python 3 code, ans = self.forwardconn.docmd(command) ret = "%s %s" % (code, force_uString(ans)) if ret.find('\n'): temprv = [] parts = ret.split('\n') code = ret[:3] parts[0] = parts[0][3:] line = '' for line in parts: line = line.strip() temprv.append('%s-%s' % (code, line)) # replace - with space on last line temprv[-1] = '%s %s' % (code, line) ret = '\r\n'.join(temprv) self.logger.debug("""RECEIVE: "%s" """ % ret) return ret.strip()
def _parse_result(self, result): dr = {} result = force_uString(result) for line in result.strip().split('\n'): m = self.pattern.match(force_bString(line)) if m is None: self.logger.error('Could not parse line from f-prot: %s' % line) raise Exception('f-prot: Unparseable answer: %s' % result) status = force_uString(m.group(1)) text = force_uString(m.group(2)) details = force_uString(m.group(3)) status = int(status) self.logger.debug("f-prot scan status: %s" % status) self.logger.debug("f-prot scan text: %s" % text) if status == 0: continue if status > 3: self.logger.warning("f-prot: got unusual status %s (result: %s)" % (status, result)) # http://www.f-prot.com/support/helpfiles/unix/appendix_c.html if status & 1 == 1 or status & 2 == 2: # we have a infection if text[0:10] == "infected: ": text = text[10:] elif text[0:27] == "contains infected objects: ": text = text[27:] else: self.logger.warn("Unexpected reply from f-prot: %s" % text) continue dr[details] = text if len(dr) == 0: return None else: return dr
def re_inject(self, suspect): """Send message back to postfix""" if suspect.get_tag('noreinject'): # in esmtp sessions we don't want to provide info to the connecting # client return 250, 'OK' if suspect.get_tag('reinjectoriginal'): self.logger.info( 'Injecting original message source without modifications') msgcontent = suspect.get_original_source() else: msgcontent = buildmsgsource(suspect) code, answer = self.sess.forwardconn.data(force_bString(msgcontent)) answer = force_uString(answer) return code, answer