def decode(self, report, filetype):
for k in sorted(report['flows'].keys()):
l4proto = k.split(' - ')[2]
if 'currtid' in report['flows'][k].keys():
del report['flows'][k]['currtid']
if report['flows'][k]['info'] and report['flows'][k]['info']['proto']:
l7proto = report['flows'][k]['info']['proto']
else:
l7proto = None
if l4proto == 'TCP' and l7proto == 'HTTP' and 'transactions' in report['flows'][k].keys() and report['flows'][k]['transactions']:
for tid in sorted(report['flows'][k]['transactions'].keys()):
ctsbuf = report['flows'][k]['transactions'][tid]['ctsbuf']
stcbuf = report['flows'][k]['transactions'][tid]['stcbuf']
ctsbuflen = len(ctsbuf) if ctsbuf else 0
stcbuflen = len(stcbuf) if stcbuf else 0
if (ctsbuflen or stcbuflen) > 0:
self.logger.debug('Decoding CTS: %s and STC: %s buffers as HTTP' % (utils.size_string(ctsbuflen), utils.size_string(stcbuflen)))
report['flows'][k]['transactions'][tid]['ctsdecode'], report['flows'][k]['transactions'][tid]['stcdecode'] = self.decodeAsHTTP(ctsbuf, stcbuf)
return dict(report)
def decodeAsHTTP(self, ctsbuf=None, stcbuf=None):
decode = {'CTS': {},'STC': {}}
matched = False
if ctsbuf and len(ctsbuf) > 0:
self.logger.debug("Decoding %s HTTP CTS buffer (RE Count: %d)" % (utils.size_string(len(ctsbuf)), len(self.config['decode_regexes']['HTTP']['CTS'])))
matchcount = 0
for regex_type, regex in self.config['decode_regexes']['HTTP']['CTS'].iteritems():
for m in regex.finditer(ctsbuf):
matched = True
matchcount += 1
for key, value in m.groupdict().iteritems():
decode['CTS']['%s' % (key.replace('-', '_'))] = value
if matched:
self.logger.debug("Found %d HTTP decode matches for HTTP CTS buffer" % (matchcount))
if 'Request_Data' in decode['CTS'].keys() and 'Transfer_Encoding' in decode['CTS'].keys():
if decode['CTS']['Transfer_Encoding'] == 'chunked':
if 'Request_Data' in decode['CTS'].keys():
predechunksize = len(decode['CTS']['Request_Data'])
decode['CTS']['Request_Data'] = utils.remove_chunked(decode['CTS']['Request_Data'])
postdechunksize = len(decode['CTS']['Request_Data'])
self.logger.debug("Dechunked %dB HTTP CTS buffer to %dB" % (predechunksize, postdechunksize))
if 'Request_Data' in decode['CTS'].keys() and 'Content_Encoding' in decode['CTS'].keys():
if 'gzip' in decode['CTS']['Content_Encoding']:
if 'Request_Data' in decode['CTS'].keys():
preungzipsize = len(decode['CTS']['Request_Data'])
decode['CTS']['Request_Data'] = utils.expand_gzip(decode['CTS']['Request_Data'])
postungzipsize = len(decode['CTS']['Request_Data'])
self.logger.debug("Expanded gzipped %dB HTTP CTS buffer to %dB" % (preungzipsize, postungzipsize))
if 'deflate' in decode['CTS']['Content_Encoding']:
if 'Request_Data' in decode['CTS'].keys():
preundeflatesize = len(decode['CTS']['Request_Data'])
decode['CTS']['Request_Data'] = utils.expand_deflate(decode['CTS']['Request_Data'])
postundeflatesize = len(decode['CTS']['Request_Data'])
self.logger.debug("Expanded deflated %dB HTTP CTS buffer to %dB" % (preundeflatesize, postundeflatesize))
else:
self.logger.debug("Could not decode %s buffer since none of the %d HTTP CTS regexes matched" % (utils.size_string(len(ctsbuf)), len(self.config['decode_regexes']['HTTP']['CTS'])))
if stcbuf and len(stcbuf) > 0:
self.logger.debug("Decoding %s HTTP STC buffer (RE Count: %d)" % (utils.size_string(len(stcbuf)), len(self.config['decode_regexes']['HTTP']['STC'])))
matchcount = 0
for regex_type, regex in self.config['decode_regexes']['HTTP']['STC'].iteritems():
for m in regex.finditer(stcbuf):
matched = True
matchcount += 1
for key, value in m.groupdict().iteritems():
decode['STC']['%s' % (key.replace('-', '_'))] = value
if matched:
self.logger.debug("Found %d HTTP decode matches for HTTP STC buffer" % (matchcount))
if 'Response_Data' in decode['STC'].keys() and 'Transfer_Encoding' in decode['STC'].keys():
if 'chunked' in decode['STC']['Transfer_Encoding']:
if 'Response_Data' in decode['STC'].keys():
predechunksize = len(decode['STC']['Response_Data'])
decode['STC']['Response_Data'] = utils.remove_chunked(decode['STC']['Response_Data'])
postdechunksize = len(decode['STC']['Response_Data'])
self.logger.debug("Dechunked %dB HTTP STC buffer to %dB" % (predechunksize, postdechunksize))
if 'Response_Data' in decode['STC'].keys() and 'Content_Encoding' in decode['STC'].keys():
if 'gzip' in decode['STC']['Content_Encoding']:
if 'Response_Data' in decode['STC'].keys():
preungzipsize = len(decode['STC']['Response_Data'])
decode['STC']['Response_Data'] = utils.expand_gzip(decode['STC']['Response_Data'])
postungzipsize = len(decode['STC']['Response_Data'])
self.logger.debug("Expanded gzipped %dB HTTP STC buffer to %dB" % (preungzipsize, postungzipsize))
if 'deflate' in decode['STC']['Content_Encoding']:
if 'Response_Data' in decode['STC'].keys():
preundeflatesize = len(decode['STC']['Response_Data'])
decode['STC']['Response_Data'] = utils.expand_deflate(decode['STC']['Response_Data'])
postundeflatesize = len(decode['STC']['Response_Data'])
self.logger.debug("Expanded deflated %dB HTTP STC buffer to %dB" % (preundeflatesize, postundeflatesize))
else:
self.logger.debug("Could not decode %s buffer since none of the %d HTTP STC regexes matched" % (utils.size_string(len(stcbuf)), len(self.config['decode_regexes']['HTTP']['STC'])))
return decode['CTS'], decode['STC']
def identify(self, ctsbuf=None, stcbuf=None, udpbuf=None, tcpport=None, udpport=None):
# check if protocol is HTTP
if ctsbuf and len(ctsbuf) > 0 and self.config['protoregexes']['HTTP']['cts']:
self.logger.debug('Testing if %s of CTS data is HTTP' % (utils.size_string(len(ctsbuf))))
if re.search(self.config['protoregexes']['HTTP']['cts'], ctsbuf):
self.logger.debug('HTTP CTS regex: \'%s\' matches' % (self.config['protoregexes']['HTTP']['cts'].pattern))
return 'HTTP'
if stcbuf and len(stcbuf) > 0 and self.config['protoregexes']['HTTP']['stc']:
self.logger.debug('Testing if %s of STC data is HTTP' % (utils.size_string(len(stcbuf))))
if re.search(self.config['protoregexes']['HTTP']['stc'], stcbuf):
self.logger.debug('HTTP STC regex: \'%s\' matches' % (self.config['protoregexes']['HTTP']['stc'].pattern))
return 'HTTP'
# check if protocol is IMAP
if stcbuf and len(stcbuf) > 0 and self.config['protoregexes']['IMAP']['stc']:
self.logger.debug('Testing if %s of STC data is IMAP' % (utils.size_string(len(stcbuf))))
if re.search(self.config['protoregexes']['IMAP']['stc'], stcbuf):
self.logger.debug('IMAP STC regex: \'%s\' matches' % (self.config['protoregexes']['IMAP']['stc'].pattern))
return 'IMAP'
# check if protocol is SMTP
if ctsbuf and len(ctsbuf) > 0 and self.config['protoregexes']['SMTP']['cts']:
self.logger.debug('Testing if %s of CTS data is SMTP' % (utils.size_string(len(ctsbuf))))
if re.search(self.config['protoregexes']['SMTP']['cts'], ctsbuf):
self.logger.debug('SMTP CTS regex: \'%s\' matches' % (self.config['protoregexes']['SMTP']['cts'].pattern))
return 'SMTP'
if stcbuf and len(stcbuf) > 0 and self.config['protoregexes']['SMTP']['stc']:
self.logger.debug('Testing if %s of STC data is SMTP' % (utils.size_string(len(stcbuf))))
if re.search(self.config['protoregexes']['SMTP']['stc'], stcbuf):
self.logger.debug('SMTP STC regex: \'%s\' matches' % (self.config['protoregexes']['SMTP']['stc'].pattern))
return 'SMTP'
# check if protocol is POP3
if stcbuf and len(stcbuf) > 0 and self.config['protoregexes']['POP3']['stc']:
self.logger.debug('Testing if %s of STC data is POP3' % (utils.size_string(len(stcbuf))))
if re.search(self.config['protoregexes']['POP3']['stc'], stcbuf):
self.logger.debug('POP3 STC regex: \'%s\' matches' % (self.config['protoregexes']['POP3']['stc'].pattern))
return 'POP3'
# check if protocol is SIP
if udpbuf and len(udpbuf) > 0 and self.config['protoregexes']['SIP']['any']:
self.logger.debug('Testing if %s of UDP data is SIP' % (utils.size_string(len(udpbuf))))
if re.search(self.config['protoregexes']['SIP']['any'], udpbuf):
self.logger.debug('SIP regex: \'%s\' matches' % (self.config['protoregexes']['SIP']['any'].pattern))
return 'SIP'
# check if protocol is SSDP
if udpbuf and len(udpbuf) > 0 and self.config['protoregexes']['SSDP']['any']:
self.logger.debug('Testing if %s of UDP data is SSDP' % (utils.size_string(len(udpbuf))))
if re.search(self.config['protoregexes']['SSDP']['any'], udpbuf):
self.logger.debug('SSDP regex: \'%s\' matches' % (self.config['protoregexes']['SSDP']['any'].pattern))
return 'SSDP'
# if we're here it means that all of the above regexes checks was unsuccessful
# we need to fallback on port based checks
self.logger.debug('No regex matched for protoid. Using port mapping as a fallback for port %s' % (tcpport if tcpport else udpport))
if tcpport and tcpport in self.config['protoports']:
self.logger.debug('Identified port %d as %s' % (tcpport, ', '.join(self.config['protoports'][tcpport]['tcp'])))
return ', '.join(self.config['protoports'][tcpport]['tcp'])
# if we're here it means even the port mapping based checks were unsuccessful
# or we might have a FN
# in any case we need to return empty-handed
return None
def identify(self,
ctsbuf=None,
stcbuf=None,
udpbuf=None,
tcpport=None,
udpport=None):
# check if protocol is HTTP
if ctsbuf and len(
ctsbuf) > 0 and self.config['protoregexes']['HTTP']['cts']:
self.logger.debug('Testing if %s of CTS data is HTTP' %
(utils.size_string(len(ctsbuf))))
if re.search(self.config['protoregexes']['HTTP']['cts'], ctsbuf):
self.logger.debug(
'HTTP CTS regex: \'%s\' matches' %
(self.config['protoregexes']['HTTP']['cts'].pattern))
return 'HTTP'
if stcbuf and len(
stcbuf) > 0 and self.config['protoregexes']['HTTP']['stc']:
self.logger.debug('Testing if %s of STC data is HTTP' %
(utils.size_string(len(stcbuf))))
if re.search(self.config['protoregexes']['HTTP']['stc'], stcbuf):
self.logger.debug(
'HTTP STC regex: \'%s\' matches' %
(self.config['protoregexes']['HTTP']['stc'].pattern))
return 'HTTP'
# check if protocol is IMAP
if stcbuf and len(
stcbuf) > 0 and self.config['protoregexes']['IMAP']['stc']:
self.logger.debug('Testing if %s of STC data is IMAP' %
(utils.size_string(len(stcbuf))))
if re.search(self.config['protoregexes']['IMAP']['stc'], stcbuf):
self.logger.debug(
'IMAP STC regex: \'%s\' matches' %
(self.config['protoregexes']['IMAP']['stc'].pattern))
return 'IMAP'
# check if protocol is SMTP
if ctsbuf and len(
ctsbuf) > 0 and self.config['protoregexes']['SMTP']['cts']:
self.logger.debug('Testing if %s of CTS data is SMTP' %
(utils.size_string(len(ctsbuf))))
if re.search(self.config['protoregexes']['SMTP']['cts'], ctsbuf):
self.logger.debug(
'SMTP CTS regex: \'%s\' matches' %
(self.config['protoregexes']['SMTP']['cts'].pattern))
return 'SMTP'
if stcbuf and len(
stcbuf) > 0 and self.config['protoregexes']['SMTP']['stc']:
self.logger.debug('Testing if %s of STC data is SMTP' %
(utils.size_string(len(stcbuf))))
if re.search(self.config['protoregexes']['SMTP']['stc'], stcbuf):
self.logger.debug(
'SMTP STC regex: \'%s\' matches' %
(self.config['protoregexes']['SMTP']['stc'].pattern))
return 'SMTP'
# check if protocol is POP3
if stcbuf and len(
stcbuf) > 0 and self.config['protoregexes']['POP3']['stc']:
self.logger.debug('Testing if %s of STC data is POP3' %
(utils.size_string(len(stcbuf))))
if re.search(self.config['protoregexes']['POP3']['stc'], stcbuf):
self.logger.debug(
'POP3 STC regex: \'%s\' matches' %
(self.config['protoregexes']['POP3']['stc'].pattern))
return 'POP3'
# check if protocol is SIP
if udpbuf and len(
udpbuf) > 0 and self.config['protoregexes']['SIP']['any']:
self.logger.debug('Testing if %s of UDP data is SIP' %
(utils.size_string(len(udpbuf))))
if re.search(self.config['protoregexes']['SIP']['any'], udpbuf):
self.logger.debug(
'SIP regex: \'%s\' matches' %
(self.config['protoregexes']['SIP']['any'].pattern))
return 'SIP'
# check if protocol is SSDP
if udpbuf and len(
udpbuf) > 0 and self.config['protoregexes']['SSDP']['any']:
self.logger.debug('Testing if %s of UDP data is SSDP' %
(utils.size_string(len(udpbuf))))
if re.search(self.config['protoregexes']['SSDP']['any'], udpbuf):
self.logger.debug(
'SSDP regex: \'%s\' matches' %
(self.config['protoregexes']['SSDP']['any'].pattern))
return 'SSDP'
# if we're here it means that all of the above regexes checks was unsuccessful
# we need to fallback on port based checks
self.logger.debug(
'No regex matched for protoid. Using port mapping as a fallback for port %s'
% (tcpport if tcpport else udpport))
if tcpport and tcpport in self.config['protoports']:
self.logger.debug('Identified port %d as %s' % (tcpport, ', '.join(
self.config['protoports'][tcpport]['tcp'])))
return ', '.join(self.config['protoports'][tcpport]['tcp'])
# if we're here it means even the port mapping based checks were unsuccessful
# or we might have a FN
# in any case we need to return empty-handed
return None
