def decode(self, report, filetype): for k in sorted(report['flows'].keys()): l4proto = k.split(' - ')[2] if 'currtid' in report['flows'][k].keys(): del report['flows'][k]['currtid'] if report['flows'][k]['info'] and report['flows'][k]['info']['proto']: l7proto = report['flows'][k]['info']['proto'] else: l7proto = None if l4proto == 'TCP' and l7proto == 'HTTP' and 'transactions' in report['flows'][k].keys() and report['flows'][k]['transactions']: for tid in sorted(report['flows'][k]['transactions'].keys()): ctsbuf = report['flows'][k]['transactions'][tid]['ctsbuf'] stcbuf = report['flows'][k]['transactions'][tid]['stcbuf'] ctsbuflen = len(ctsbuf) if ctsbuf else 0 stcbuflen = len(stcbuf) if stcbuf else 0 if (ctsbuflen or stcbuflen) > 0: self.logger.debug('Decoding CTS: %s and STC: %s buffers as HTTP' % (utils.size_string(ctsbuflen), utils.size_string(stcbuflen))) report['flows'][k]['transactions'][tid]['ctsdecode'], report['flows'][k]['transactions'][tid]['stcdecode'] = self.decodeAsHTTP(ctsbuf, stcbuf) return dict(report)
def decodeAsHTTP(self, ctsbuf=None, stcbuf=None): decode = {'CTS': {},'STC': {}} matched = False if ctsbuf and len(ctsbuf) > 0: self.logger.debug("Decoding %s HTTP CTS buffer (RE Count: %d)" % (utils.size_string(len(ctsbuf)), len(self.config['decode_regexes']['HTTP']['CTS']))) matchcount = 0 for regex_type, regex in self.config['decode_regexes']['HTTP']['CTS'].iteritems(): for m in regex.finditer(ctsbuf): matched = True matchcount += 1 for key, value in m.groupdict().iteritems(): decode['CTS']['%s' % (key.replace('-', '_'))] = value if matched: self.logger.debug("Found %d HTTP decode matches for HTTP CTS buffer" % (matchcount)) if 'Request_Data' in decode['CTS'].keys() and 'Transfer_Encoding' in decode['CTS'].keys(): if decode['CTS']['Transfer_Encoding'] == 'chunked': if 'Request_Data' in decode['CTS'].keys(): predechunksize = len(decode['CTS']['Request_Data']) decode['CTS']['Request_Data'] = utils.remove_chunked(decode['CTS']['Request_Data']) postdechunksize = len(decode['CTS']['Request_Data']) self.logger.debug("Dechunked %dB HTTP CTS buffer to %dB" % (predechunksize, postdechunksize)) if 'Request_Data' in decode['CTS'].keys() and 'Content_Encoding' in decode['CTS'].keys(): if 'gzip' in decode['CTS']['Content_Encoding']: if 'Request_Data' in decode['CTS'].keys(): preungzipsize = len(decode['CTS']['Request_Data']) decode['CTS']['Request_Data'] = utils.expand_gzip(decode['CTS']['Request_Data']) postungzipsize = len(decode['CTS']['Request_Data']) self.logger.debug("Expanded gzipped %dB HTTP CTS buffer to %dB" % (preungzipsize, postungzipsize)) if 'deflate' in decode['CTS']['Content_Encoding']: if 'Request_Data' in decode['CTS'].keys(): preundeflatesize = len(decode['CTS']['Request_Data']) decode['CTS']['Request_Data'] = utils.expand_deflate(decode['CTS']['Request_Data']) postundeflatesize = len(decode['CTS']['Request_Data']) self.logger.debug("Expanded deflated %dB HTTP CTS buffer to %dB" % (preundeflatesize, postundeflatesize)) else: self.logger.debug("Could not decode %s buffer since none of the %d HTTP CTS regexes matched" % (utils.size_string(len(ctsbuf)), len(self.config['decode_regexes']['HTTP']['CTS']))) if stcbuf and len(stcbuf) > 0: self.logger.debug("Decoding %s HTTP STC buffer (RE Count: %d)" % (utils.size_string(len(stcbuf)), len(self.config['decode_regexes']['HTTP']['STC']))) matchcount = 0 for regex_type, regex in self.config['decode_regexes']['HTTP']['STC'].iteritems(): for m in regex.finditer(stcbuf): matched = True matchcount += 1 for key, value in m.groupdict().iteritems(): decode['STC']['%s' % (key.replace('-', '_'))] = value if matched: self.logger.debug("Found %d HTTP decode matches for HTTP STC buffer" % (matchcount)) if 'Response_Data' in decode['STC'].keys() and 'Transfer_Encoding' in decode['STC'].keys(): if 'chunked' in decode['STC']['Transfer_Encoding']: if 'Response_Data' in decode['STC'].keys(): predechunksize = len(decode['STC']['Response_Data']) decode['STC']['Response_Data'] = utils.remove_chunked(decode['STC']['Response_Data']) postdechunksize = len(decode['STC']['Response_Data']) self.logger.debug("Dechunked %dB HTTP STC buffer to %dB" % (predechunksize, postdechunksize)) if 'Response_Data' in decode['STC'].keys() and 'Content_Encoding' in decode['STC'].keys(): if 'gzip' in decode['STC']['Content_Encoding']: if 'Response_Data' in decode['STC'].keys(): preungzipsize = len(decode['STC']['Response_Data']) decode['STC']['Response_Data'] = utils.expand_gzip(decode['STC']['Response_Data']) postungzipsize = len(decode['STC']['Response_Data']) self.logger.debug("Expanded gzipped %dB HTTP STC buffer to %dB" % (preungzipsize, postungzipsize)) if 'deflate' in decode['STC']['Content_Encoding']: if 'Response_Data' in decode['STC'].keys(): preundeflatesize = len(decode['STC']['Response_Data']) decode['STC']['Response_Data'] = utils.expand_deflate(decode['STC']['Response_Data']) postundeflatesize = len(decode['STC']['Response_Data']) self.logger.debug("Expanded deflated %dB HTTP STC buffer to %dB" % (preundeflatesize, postundeflatesize)) else: self.logger.debug("Could not decode %s buffer since none of the %d HTTP STC regexes matched" % (utils.size_string(len(stcbuf)), len(self.config['decode_regexes']['HTTP']['STC']))) return decode['CTS'], decode['STC']
def identify(self, ctsbuf=None, stcbuf=None, udpbuf=None, tcpport=None, udpport=None): # check if protocol is HTTP if ctsbuf and len(ctsbuf) > 0 and self.config['protoregexes']['HTTP']['cts']: self.logger.debug('Testing if %s of CTS data is HTTP' % (utils.size_string(len(ctsbuf)))) if re.search(self.config['protoregexes']['HTTP']['cts'], ctsbuf): self.logger.debug('HTTP CTS regex: \'%s\' matches' % (self.config['protoregexes']['HTTP']['cts'].pattern)) return 'HTTP' if stcbuf and len(stcbuf) > 0 and self.config['protoregexes']['HTTP']['stc']: self.logger.debug('Testing if %s of STC data is HTTP' % (utils.size_string(len(stcbuf)))) if re.search(self.config['protoregexes']['HTTP']['stc'], stcbuf): self.logger.debug('HTTP STC regex: \'%s\' matches' % (self.config['protoregexes']['HTTP']['stc'].pattern)) return 'HTTP' # check if protocol is IMAP if stcbuf and len(stcbuf) > 0 and self.config['protoregexes']['IMAP']['stc']: self.logger.debug('Testing if %s of STC data is IMAP' % (utils.size_string(len(stcbuf)))) if re.search(self.config['protoregexes']['IMAP']['stc'], stcbuf): self.logger.debug('IMAP STC regex: \'%s\' matches' % (self.config['protoregexes']['IMAP']['stc'].pattern)) return 'IMAP' # check if protocol is SMTP if ctsbuf and len(ctsbuf) > 0 and self.config['protoregexes']['SMTP']['cts']: self.logger.debug('Testing if %s of CTS data is SMTP' % (utils.size_string(len(ctsbuf)))) if re.search(self.config['protoregexes']['SMTP']['cts'], ctsbuf): self.logger.debug('SMTP CTS regex: \'%s\' matches' % (self.config['protoregexes']['SMTP']['cts'].pattern)) return 'SMTP' if stcbuf and len(stcbuf) > 0 and self.config['protoregexes']['SMTP']['stc']: self.logger.debug('Testing if %s of STC data is SMTP' % (utils.size_string(len(stcbuf)))) if re.search(self.config['protoregexes']['SMTP']['stc'], stcbuf): self.logger.debug('SMTP STC regex: \'%s\' matches' % (self.config['protoregexes']['SMTP']['stc'].pattern)) return 'SMTP' # check if protocol is POP3 if stcbuf and len(stcbuf) > 0 and self.config['protoregexes']['POP3']['stc']: self.logger.debug('Testing if %s of STC data is POP3' % (utils.size_string(len(stcbuf)))) if re.search(self.config['protoregexes']['POP3']['stc'], stcbuf): self.logger.debug('POP3 STC regex: \'%s\' matches' % (self.config['protoregexes']['POP3']['stc'].pattern)) return 'POP3' # check if protocol is SIP if udpbuf and len(udpbuf) > 0 and self.config['protoregexes']['SIP']['any']: self.logger.debug('Testing if %s of UDP data is SIP' % (utils.size_string(len(udpbuf)))) if re.search(self.config['protoregexes']['SIP']['any'], udpbuf): self.logger.debug('SIP regex: \'%s\' matches' % (self.config['protoregexes']['SIP']['any'].pattern)) return 'SIP' # check if protocol is SSDP if udpbuf and len(udpbuf) > 0 and self.config['protoregexes']['SSDP']['any']: self.logger.debug('Testing if %s of UDP data is SSDP' % (utils.size_string(len(udpbuf)))) if re.search(self.config['protoregexes']['SSDP']['any'], udpbuf): self.logger.debug('SSDP regex: \'%s\' matches' % (self.config['protoregexes']['SSDP']['any'].pattern)) return 'SSDP' # if we're here it means that all of the above regexes checks was unsuccessful # we need to fallback on port based checks self.logger.debug('No regex matched for protoid. Using port mapping as a fallback for port %s' % (tcpport if tcpport else udpport)) if tcpport and tcpport in self.config['protoports']: self.logger.debug('Identified port %d as %s' % (tcpport, ', '.join(self.config['protoports'][tcpport]['tcp']))) return ', '.join(self.config['protoports'][tcpport]['tcp']) # if we're here it means even the port mapping based checks were unsuccessful # or we might have a FN # in any case we need to return empty-handed return None
def identify(self, ctsbuf=None, stcbuf=None, udpbuf=None, tcpport=None, udpport=None): # check if protocol is HTTP if ctsbuf and len( ctsbuf) > 0 and self.config['protoregexes']['HTTP']['cts']: self.logger.debug('Testing if %s of CTS data is HTTP' % (utils.size_string(len(ctsbuf)))) if re.search(self.config['protoregexes']['HTTP']['cts'], ctsbuf): self.logger.debug( 'HTTP CTS regex: \'%s\' matches' % (self.config['protoregexes']['HTTP']['cts'].pattern)) return 'HTTP' if stcbuf and len( stcbuf) > 0 and self.config['protoregexes']['HTTP']['stc']: self.logger.debug('Testing if %s of STC data is HTTP' % (utils.size_string(len(stcbuf)))) if re.search(self.config['protoregexes']['HTTP']['stc'], stcbuf): self.logger.debug( 'HTTP STC regex: \'%s\' matches' % (self.config['protoregexes']['HTTP']['stc'].pattern)) return 'HTTP' # check if protocol is IMAP if stcbuf and len( stcbuf) > 0 and self.config['protoregexes']['IMAP']['stc']: self.logger.debug('Testing if %s of STC data is IMAP' % (utils.size_string(len(stcbuf)))) if re.search(self.config['protoregexes']['IMAP']['stc'], stcbuf): self.logger.debug( 'IMAP STC regex: \'%s\' matches' % (self.config['protoregexes']['IMAP']['stc'].pattern)) return 'IMAP' # check if protocol is SMTP if ctsbuf and len( ctsbuf) > 0 and self.config['protoregexes']['SMTP']['cts']: self.logger.debug('Testing if %s of CTS data is SMTP' % (utils.size_string(len(ctsbuf)))) if re.search(self.config['protoregexes']['SMTP']['cts'], ctsbuf): self.logger.debug( 'SMTP CTS regex: \'%s\' matches' % (self.config['protoregexes']['SMTP']['cts'].pattern)) return 'SMTP' if stcbuf and len( stcbuf) > 0 and self.config['protoregexes']['SMTP']['stc']: self.logger.debug('Testing if %s of STC data is SMTP' % (utils.size_string(len(stcbuf)))) if re.search(self.config['protoregexes']['SMTP']['stc'], stcbuf): self.logger.debug( 'SMTP STC regex: \'%s\' matches' % (self.config['protoregexes']['SMTP']['stc'].pattern)) return 'SMTP' # check if protocol is POP3 if stcbuf and len( stcbuf) > 0 and self.config['protoregexes']['POP3']['stc']: self.logger.debug('Testing if %s of STC data is POP3' % (utils.size_string(len(stcbuf)))) if re.search(self.config['protoregexes']['POP3']['stc'], stcbuf): self.logger.debug( 'POP3 STC regex: \'%s\' matches' % (self.config['protoregexes']['POP3']['stc'].pattern)) return 'POP3' # check if protocol is SIP if udpbuf and len( udpbuf) > 0 and self.config['protoregexes']['SIP']['any']: self.logger.debug('Testing if %s of UDP data is SIP' % (utils.size_string(len(udpbuf)))) if re.search(self.config['protoregexes']['SIP']['any'], udpbuf): self.logger.debug( 'SIP regex: \'%s\' matches' % (self.config['protoregexes']['SIP']['any'].pattern)) return 'SIP' # check if protocol is SSDP if udpbuf and len( udpbuf) > 0 and self.config['protoregexes']['SSDP']['any']: self.logger.debug('Testing if %s of UDP data is SSDP' % (utils.size_string(len(udpbuf)))) if re.search(self.config['protoregexes']['SSDP']['any'], udpbuf): self.logger.debug( 'SSDP regex: \'%s\' matches' % (self.config['protoregexes']['SSDP']['any'].pattern)) return 'SSDP' # if we're here it means that all of the above regexes checks was unsuccessful # we need to fallback on port based checks self.logger.debug( 'No regex matched for protoid. Using port mapping as a fallback for port %s' % (tcpport if tcpport else udpport)) if tcpport and tcpport in self.config['protoports']: self.logger.debug('Identified port %d as %s' % (tcpport, ', '.join( self.config['protoports'][tcpport]['tcp']))) return ', '.join(self.config['protoports'][tcpport]['tcp']) # if we're here it means even the port mapping based checks were unsuccessful # or we might have a FN # in any case we need to return empty-handed return None