Exemple #1
0
  def decode(self, report, filetype):
    for k in sorted(report['flows'].keys()):
      l4proto = k.split(' - ')[2]

      if 'currtid' in report['flows'][k].keys():
        del report['flows'][k]['currtid']

      if report['flows'][k]['info'] and report['flows'][k]['info']['proto']:
        l7proto = report['flows'][k]['info']['proto']
      else:
        l7proto = None

      if l4proto == 'TCP' and l7proto == 'HTTP' and 'transactions' in report['flows'][k].keys() and report['flows'][k]['transactions']:
        for tid in sorted(report['flows'][k]['transactions'].keys()):
            ctsbuf = report['flows'][k]['transactions'][tid]['ctsbuf']
            stcbuf = report['flows'][k]['transactions'][tid]['stcbuf']

            ctsbuflen = len(ctsbuf) if ctsbuf else 0
            stcbuflen = len(stcbuf) if stcbuf else 0

            if (ctsbuflen or stcbuflen) > 0:
              self.logger.debug('Decoding CTS: %s and STC: %s buffers as HTTP' % (utils.size_string(ctsbuflen), utils.size_string(stcbuflen)))
              report['flows'][k]['transactions'][tid]['ctsdecode'], report['flows'][k]['transactions'][tid]['stcdecode'] = self.decodeAsHTTP(ctsbuf, stcbuf)

    return dict(report)
Exemple #2
0
  def decodeAsHTTP(self, ctsbuf=None, stcbuf=None):
    decode = {'CTS': {},'STC': {}}
    matched = False

    if ctsbuf and len(ctsbuf) > 0:
      self.logger.debug("Decoding %s HTTP CTS buffer (RE Count: %d)" % (utils.size_string(len(ctsbuf)), len(self.config['decode_regexes']['HTTP']['CTS'])))

      matchcount = 0
      for regex_type, regex in self.config['decode_regexes']['HTTP']['CTS'].iteritems():
        for m in regex.finditer(ctsbuf):
          matched = True
          matchcount += 1

          for key, value in m.groupdict().iteritems():
            decode['CTS']['%s' % (key.replace('-', '_'))] = value

      if matched:
        self.logger.debug("Found %d HTTP decode matches for HTTP CTS buffer" % (matchcount))

        if 'Request_Data' in decode['CTS'].keys() and 'Transfer_Encoding' in decode['CTS'].keys():
          if decode['CTS']['Transfer_Encoding'] == 'chunked':
            if 'Request_Data' in decode['CTS'].keys():
              predechunksize = len(decode['CTS']['Request_Data'])
              decode['CTS']['Request_Data'] = utils.remove_chunked(decode['CTS']['Request_Data'])
              postdechunksize = len(decode['CTS']['Request_Data'])
              self.logger.debug("Dechunked %dB HTTP CTS buffer to %dB" % (predechunksize, postdechunksize))

        if 'Request_Data' in decode['CTS'].keys() and 'Content_Encoding' in decode['CTS'].keys():
          if 'gzip' in decode['CTS']['Content_Encoding']:
            if 'Request_Data' in decode['CTS'].keys():
              preungzipsize = len(decode['CTS']['Request_Data'])
              decode['CTS']['Request_Data'] = utils.expand_gzip(decode['CTS']['Request_Data'])
              postungzipsize = len(decode['CTS']['Request_Data'])
              self.logger.debug("Expanded gzipped %dB HTTP CTS buffer to %dB" % (preungzipsize, postungzipsize))

          if 'deflate' in decode['CTS']['Content_Encoding']:
            if 'Request_Data' in decode['CTS'].keys():
              preundeflatesize = len(decode['CTS']['Request_Data'])
              decode['CTS']['Request_Data'] = utils.expand_deflate(decode['CTS']['Request_Data'])
              postundeflatesize = len(decode['CTS']['Request_Data'])
              self.logger.debug("Expanded deflated %dB HTTP CTS buffer to %dB" % (preundeflatesize, postundeflatesize))

      else:
        self.logger.debug("Could not decode %s buffer since none of the %d HTTP CTS regexes matched" % (utils.size_string(len(ctsbuf)), len(self.config['decode_regexes']['HTTP']['CTS'])))

    if stcbuf and len(stcbuf) > 0:
      self.logger.debug("Decoding %s HTTP STC buffer (RE Count: %d)" % (utils.size_string(len(stcbuf)), len(self.config['decode_regexes']['HTTP']['STC'])))

      matchcount = 0
      for regex_type, regex in self.config['decode_regexes']['HTTP']['STC'].iteritems():
        for m in regex.finditer(stcbuf):
          matched = True
          matchcount += 1

          for key, value in m.groupdict().iteritems():
            decode['STC']['%s' % (key.replace('-', '_'))] = value

      if matched:
        self.logger.debug("Found %d HTTP decode matches for HTTP STC buffer" % (matchcount))

        if 'Response_Data' in decode['STC'].keys() and 'Transfer_Encoding' in decode['STC'].keys():
          if 'chunked' in decode['STC']['Transfer_Encoding']:
            if 'Response_Data' in decode['STC'].keys():
              predechunksize = len(decode['STC']['Response_Data'])
              decode['STC']['Response_Data'] = utils.remove_chunked(decode['STC']['Response_Data'])
              postdechunksize = len(decode['STC']['Response_Data'])
              self.logger.debug("Dechunked %dB HTTP STC buffer to %dB" % (predechunksize, postdechunksize))

        if 'Response_Data' in decode['STC'].keys() and 'Content_Encoding' in decode['STC'].keys():
          if 'gzip' in decode['STC']['Content_Encoding']:
            if 'Response_Data' in decode['STC'].keys():
              preungzipsize = len(decode['STC']['Response_Data'])
              decode['STC']['Response_Data'] = utils.expand_gzip(decode['STC']['Response_Data'])
              postungzipsize = len(decode['STC']['Response_Data'])
              self.logger.debug("Expanded gzipped %dB HTTP STC buffer to %dB" % (preungzipsize, postungzipsize))

          if 'deflate' in decode['STC']['Content_Encoding']:
            if 'Response_Data' in decode['STC'].keys():
              preundeflatesize = len(decode['STC']['Response_Data'])
              decode['STC']['Response_Data'] = utils.expand_deflate(decode['STC']['Response_Data'])
              postundeflatesize = len(decode['STC']['Response_Data'])
              self.logger.debug("Expanded deflated %dB HTTP STC buffer to %dB" % (preundeflatesize, postundeflatesize))

      else:
        self.logger.debug("Could not decode %s buffer since none of the %d HTTP STC regexes matched" % (utils.size_string(len(stcbuf)), len(self.config['decode_regexes']['HTTP']['STC'])))

    return decode['CTS'], decode['STC']
Exemple #3
0
  def identify(self, ctsbuf=None, stcbuf=None, udpbuf=None, tcpport=None, udpport=None):
    # check if protocol is HTTP
    if ctsbuf and len(ctsbuf) > 0 and self.config['protoregexes']['HTTP']['cts']:
      self.logger.debug('Testing if %s of CTS data is HTTP' % (utils.size_string(len(ctsbuf))))
      if re.search(self.config['protoregexes']['HTTP']['cts'], ctsbuf):
        self.logger.debug('HTTP CTS regex: \'%s\' matches' % (self.config['protoregexes']['HTTP']['cts'].pattern))
        return 'HTTP'
    if stcbuf and len(stcbuf) > 0 and self.config['protoregexes']['HTTP']['stc']:
      self.logger.debug('Testing if %s of STC data is HTTP' % (utils.size_string(len(stcbuf))))
      if re.search(self.config['protoregexes']['HTTP']['stc'], stcbuf):
        self.logger.debug('HTTP STC regex: \'%s\' matches' % (self.config['protoregexes']['HTTP']['stc'].pattern))
        return 'HTTP'

    # check if protocol is IMAP
    if stcbuf and len(stcbuf) > 0 and self.config['protoregexes']['IMAP']['stc']:
      self.logger.debug('Testing if %s of STC data is IMAP' % (utils.size_string(len(stcbuf))))
      if re.search(self.config['protoregexes']['IMAP']['stc'], stcbuf):
        self.logger.debug('IMAP STC regex: \'%s\' matches' % (self.config['protoregexes']['IMAP']['stc'].pattern))
        return 'IMAP'

    # check if protocol is SMTP
    if ctsbuf and len(ctsbuf) > 0 and self.config['protoregexes']['SMTP']['cts']:
      self.logger.debug('Testing if %s of CTS data is SMTP' % (utils.size_string(len(ctsbuf))))
      if re.search(self.config['protoregexes']['SMTP']['cts'], ctsbuf):
        self.logger.debug('SMTP CTS regex: \'%s\' matches' % (self.config['protoregexes']['SMTP']['cts'].pattern))
        return 'SMTP'
    if stcbuf and len(stcbuf) > 0 and self.config['protoregexes']['SMTP']['stc']:
      self.logger.debug('Testing if %s of STC data is SMTP' % (utils.size_string(len(stcbuf))))
      if re.search(self.config['protoregexes']['SMTP']['stc'], stcbuf):
        self.logger.debug('SMTP STC regex: \'%s\' matches' % (self.config['protoregexes']['SMTP']['stc'].pattern))
        return 'SMTP'

    # check if protocol is POP3
    if stcbuf and len(stcbuf) > 0 and self.config['protoregexes']['POP3']['stc']:
      self.logger.debug('Testing if %s of STC data is POP3' % (utils.size_string(len(stcbuf))))
      if re.search(self.config['protoregexes']['POP3']['stc'], stcbuf):
        self.logger.debug('POP3 STC regex: \'%s\' matches' % (self.config['protoregexes']['POP3']['stc'].pattern))
        return 'POP3'

    # check if protocol is SIP
    if udpbuf and len(udpbuf) > 0 and self.config['protoregexes']['SIP']['any']:
      self.logger.debug('Testing if %s of UDP data is SIP' % (utils.size_string(len(udpbuf))))
      if re.search(self.config['protoregexes']['SIP']['any'], udpbuf):
        self.logger.debug('SIP regex: \'%s\' matches' % (self.config['protoregexes']['SIP']['any'].pattern))
        return 'SIP'

    # check if protocol is SSDP
    if udpbuf and len(udpbuf) > 0 and self.config['protoregexes']['SSDP']['any']:
      self.logger.debug('Testing if %s of UDP data is SSDP' % (utils.size_string(len(udpbuf))))
      if re.search(self.config['protoregexes']['SSDP']['any'], udpbuf):
        self.logger.debug('SSDP regex: \'%s\' matches' % (self.config['protoregexes']['SSDP']['any'].pattern))
        return 'SSDP'

    # if we're here it means that all of the above regexes checks was unsuccessful
    # we need to fallback on port based checks
    self.logger.debug('No regex matched for protoid. Using port mapping as a fallback for port %s' % (tcpport if tcpport else udpport))
    if tcpport and tcpport in self.config['protoports']:
      self.logger.debug('Identified port %d as %s' % (tcpport, ', '.join(self.config['protoports'][tcpport]['tcp'])))
      return ', '.join(self.config['protoports'][tcpport]['tcp'])

    # if we're here it means even the port mapping based checks were unsuccessful
    # or we might have a FN
    # in any case we need to return empty-handed
    return None