def handle_tcp(self, tcp):
if tcp.nids_state == nids.NIDS_JUST_EST:
((src, sport), (dst, dport)) = tcp.addr
tcp.client.collect = 1
tcp.server.collect = 1
tcp.start_ts = nids.get_pkt_ts()
elif tcp.nids_state == nids.NIDS_DATA:
tcp.discard(0)
elif tcp.nids_state in end_states:
((src, sport), (dst, dport)) = tcp.addr
tcp.stop_ts = nids.get_pkt_ts()
print "[+](%s-%s) %s:%s - %s:%s (CTS: %dB | STC: %dB)" % (
tcp.start_ts, tcp.stop_ts, src, sport, dst, dport,
len(tcp.server.data[:tcp.server.count]),
len(tcp.client.data[:tcp.client.count]))
# pprint(getmembers(tcp.client))
print(binascii.hexlify(tcp.server.data))
print(len(tcp.server.data))
raw_input("Enter to continue")
msg = TcpMessage(tcp.client.data, tcp.server.data,
(src, sport, dst, dport), tcp.start_ts,
tcp.stop_ts)
self.lock.acquire()
self.connection_list.append(msg)
self.lock.release()
def __handleTCPStream(self, tcp):
"""
:param tcp:
:return:
"""
global ts_start, ts_end
((src_ip, src_port), (dst_ip, dst_port)) = tcp.addr
if self.dst_tcp_port_filter and dst_port not in self.dst_tcp_port_filter:
return
if self.dst_tcp_ip_filter and dst_ip not in self.dst_tcp_ip_filter:
return
if self.src_tcp_port_filter and src_port not in self.src_tcp_port_filter:
return
if self.src_tcp_ip_filter and src_ip not in self.src_tcp_ip_filter:
return
if tcp.nids_state == nids.NIDS_JUST_EST:
if self.device:
ts_start = mills.getCurrenttimestamp()
else:
ts_start = nids.get_pkt_ts()
tcp.client.collect = 1
tcp.server.collect = 1
elif tcp.nids_state == nids.NIDS_DATA:
if self.device:
ts_end = mills.getCurrenttimestamp()
else:
ts_end = nids.get_pkt_ts()
# keep all of the stream's new data
tcp.discard(0)
data_c2s = tcp.server.data[tcp.server.count -
tcp.server.count_new:tcp.server.count]
data_s2c = tcp.client.data[tcp.client.count -
tcp.client.count_new:tcp.client.count]
result = (ts_start, ts_end, src_ip, src_port, dst_ip, dst_port,
data_c2s, data_s2c)
if self.data_stream_direct == 1:
self.__outputTCP(result,
direct=self.data_stream_direct,
level=self.data_level)
elif tcp.nids_state in StreamHandler.__END_STATES:
if self.device:
ts_end = mills.getCurrenttimestamp()
else:
ts_end = nids.get_pkt_ts()
data_c2s_session = tcp.server.data[:tcp.server.count]
data_s2c_session = tcp.client.data[:tcp.client.count]
result = (ts_start, ts_end, src_ip, src_port, dst_ip, dst_port,
data_c2s_session, data_s2c_session)
if self.data_stream_direct == 2:
self.__outputTCP(result,
direct=self.data_stream_direct,
level=self.data_level)
def handle_tcp_stream(self, tcp_stream):
# print tcp_stream.addr, tcp_stream.nids_state, (nids.NIDS_JUST_EST, nids.NIDS_DATA, end_states)
if tcp_stream.nids_state == nids.NIDS_JUST_EST:
# new to us, but do we care?
((src, sport), (dst, dport)) = tcp_stream.addr
if self._dst is not None and dst != self._dst:
return
if self._dest_port is not None and dport != self._dest_port:
return
if self._src is not None and src != self._src:
return
if self._src_port is not None and sport != self._src_port:
return
dt = datetime.utcfromtimestamp(nids.get_pkt_ts())
if self._start_date is not None and dt < self._start_date:
return
tcp_stream.client.collect = 1
tcp_stream.server.collect = 1
if tcp_stream.addr in self.stream_parsers:
raise Exception("Addr already known in streams?!")
stream_parser = parser_adapter(self._proto_uri, self._proto_opts,
tcp_stream, self.output_dir, dt)
self.configure_stream_logging(stream_parser)
if stream_parser is None:
raise Exception("Unknown proto '{}'".format(self._proto_uri))
self.stream_parsers[tcp_stream.addr] = stream_parser
elif tcp_stream.nids_state == nids.NIDS_DATA:
client_stream = tcp_stream.client
server_stream = tcp_stream.server
ts = nids.get_pkt_ts()
stream_parser = self.stream_parsers[tcp_stream.addr]
if client_stream.count_new > 0:
stream_parser.write_client(
client_stream.data[:client_stream.count_new], ts)
if server_stream.count_new > 0:
stream_parser.write_server(
server_stream.data[:server_stream.count_new], ts)
stream_parser.last_packet_ts = ts
elif tcp_stream.nids_state in end_states:
stream_parser = self.stream_parsers[tcp_stream.addr]
dt = datetime.utcfromtimestamp(nids.get_pkt_ts())
gevent.spawn(self.finish_complete_stream, stream_parser, dt)
del self.stream_parsers[tcp_stream.addr]
def tcp_callback(self, tcp):
# Established
if tcp.nids_state == nids.NIDS_JUST_EST:
(saddr, sport), (daddr, dport) = tcp.addr
tcp.client.collect = 1
tcp.server.collect = 1
flow = hashlib.sha1(str(tcp.addr)).hexdigest()
self.flows[flow] = Conversation()
self.flows[flow].addresses = tcp.addr
if self.tcp_debug:
print "Conversation established at -> {}".format(
nids.get_pkt_ts())
# Data received
elif tcp.nids_state == nids.NIDS_DATA:
(saddr, sport), (daddr, dport) = tcp.addr
flow = hashlib.sha1(str(tcp.addr)).hexdigest()
if tcp.client.data[:tcp.client.count_new]:
'''This is data from the Server to the Client'''
client_data = len(tcp.client.data[:tcp.client.count_new])
self.flows[flow].bpackets.append(client_data)
self.flows[flow].btimestamps.append(nids.get_pkt_ts())
elif tcp.server.data[:tcp.server.count_new]:
'''This is data from the Client to the Server'''
server_data = len(tcp.server.data[:tcp.server.count_new])
self.flows[flow].fpackets.append(server_data)
self.flows[flow].ftimestamps.append(nids.get_pkt_ts())
tcp.discard(0)
if self.tcp_debug:
print "Data sent at {}: {} -> {}".format(
nids.get_pkt_ts(), saddr, daddr)
elif tcp.nids_state in self.END_STATES:
(saddr, sport), (daddr, dport) = tcp.addr
flow = hashlib.sha1(str(tcp.addr)).hexdigest()
self.flows[flow].fdata = tcp.server.data[:tcp.server.count]
self.flows[flow].bdata = tcp.client.data[:tcp.client.count]
self.emit(self.flows[flow], tcp.addr)
if self.tcp_debug:
print "Conversation ended at {} {} -> {} ({}) Server -> {} Client -> {}".format(
nids.get_pkt_ts(), saddr, daddr, dport,
len(tcp.client.data[:tcp.client.count]),
len(tcp.server.data[:tcp.server.count]))
def __handleUDPDatagram(self, addrs, payload, pkt):
"""
Args:
addrs:
payload:
pkt:
Returns:
"""
global ts_start, ts_end
((src_ip, src_port), (dst_ip, dst_port)) = addrs
if self.udp_port_filter and ((src_port not in self.udp_port_filter) and
(dst_port not in self.udp_port_filter)):
return
if self.device:
ts = mills.getCurrenttimestamp()
else:
ts = nids.get_pkt_ts()
result = (ts, src_ip, src_port, dst_ip, dst_port, payload)
self.__outputUDP(result, level=self.data_level)
def artifact_ready(self, address, kind, key):
self.times[key]['finish'] = nids.get_pkt_ts()
#print "times:", self.times[key]
#artifact = (key, self.data[key])
artifact = HttpArtifact(address, kind, self.data[key], self.times[key]['begin'], self.times[key]['finish'])
self.pipe.send(artifact)
del self.data[key]
del self.times[key]
def tcp_callback(self, tcp):
# Established
if tcp.nids_state == nids.NIDS_JUST_EST:
(saddr, sport), (daddr, dport) = tcp.addr
tcp.client.collect = 1
tcp.server.collect = 1
flow = hashlib.sha1(str(tcp.addr)).hexdigest()
self.flows[flow] = Conversation()
self.flows[flow].addresses = tcp.addr
if self.tcp_debug:
print "Conversation established at -> {}".format(nids.get_pkt_ts())
# Data received
elif tcp.nids_state == nids.NIDS_DATA:
(saddr, sport), (daddr, dport) = tcp.addr
flow = hashlib.sha1(str(tcp.addr)).hexdigest()
if tcp.client.data[:tcp.client.count_new]:
'''This is data from the Server to the Client'''
client_data = len(tcp.client.data[:tcp.client.count_new])
self.flows[flow].bpackets.append(client_data)
self.flows[flow].btimestamps.append(nids.get_pkt_ts())
elif tcp.server.data[:tcp.server.count_new]:
'''This is data from the Client to the Server'''
server_data = len(tcp.server.data[:tcp.server.count_new])
self.flows[flow].fpackets.append(server_data)
self.flows[flow].ftimestamps.append(nids.get_pkt_ts())
tcp.discard(0)
if self.tcp_debug:
print "Data sent at {}: {} -> {}".format(nids.get_pkt_ts(), saddr, daddr)
elif tcp.nids_state in self.END_STATES:
(saddr, sport), (daddr, dport) = tcp.addr
flow = hashlib.sha1(str(tcp.addr)).hexdigest()
self.flows[flow].fdata = tcp.server.data[:tcp.server.count]
self.flows[flow].bdata = tcp.client.data[:tcp.client.count]
self.emit(self.flows[flow], tcp.addr)
if self.tcp_debug:
print "Conversation ended at {} {} -> {} ({}) Server -> {} Client -> {}".format(nids.get_pkt_ts(), saddr, daddr, dport,
len(tcp.client.data[:tcp.client.count]), len(tcp.server.data[:tcp.server.count]))
def handleUdpDatagrams(addr, data, ip):
global ptimestamp
global metadata
ptimestamp = nids.get_pkt_ts()
((src,sport),(dst,dport)) = addr
if src < dst:
f_string = src + ":" + str(sport) + "-" + dst + ":" + str(dport)
else:
f_string = dst + ":" + str(dport) + "-" + src + ":" + str(sport)
metadata['proto'] = 'udp'
metadata['time'] = ptimestamp
metadata['addr'] = { 'src' : src,
'dst' : dst,
'sport' : sport,
'dport' : dport
}
stopped = False
for module in udp_modules:
if f_string in module.streaminfo and module.streaminfo[f_string] == None:
# This module called udp.stop() for this f_string
continue
# Create new udp object
udpd = copy_udp_data(addr, data, ip)
udpd.timestamp = ptimestamp
udpd.module_data = module.module_data
if f_string not in module.streaminfo:
# First time this module has seen this f_string.
# Create a new stream_data object. Will save it later.
module.streaminfo[f_string] = stream_meta()
udpd.stream_data = stream_meta().stream_data
else:
udpd.stream_data = module.streaminfo[f_string].stream_data
try:
module.handleDatagram(udpd)
except Exception, e:
exc = traceback.format_exc()
chop.prettyprnt("YELLOW", "Exception in module %s -- Traceback: \n%s" % (module.moduleName, exc))
sys.exit(-1)
#Have to copy the information back now
module.module_data = udpd.module_data
if udpd.sval: #we were told by this module to stop collecting
del udpd
module.streaminfo[f_string] = None
stopped = True
continue
#else we continue on since this module is still collecting
module.streaminfo[f_string].stream_data = udpd.stream_data
del udpd
def handleTcpStream(tcp):
#print "tcps -", str(tcp.addr), " state:", tcp.nids_state
global _tcp_pkts
_tcp_pkts += 1
if tcp.nids_state == nids.NIDS_JUST_EST:
((src, sport), (dst, dport)) = tcp.addr
if tcp.addr not in _tcp_bag:
_tcp_bag[tcp.addr] = [None, None, 1, None]
_tcp_bag[tcp.addr][0] = nids.get_pkt_ts()
tcp.client.collect = 1 # do not store data
tcp.server.collect = 1
tcp.client.collect_urg = 1
tcp.server.collect_urg = 1
elif tcp.nids_state == nids.NIDS_DATA:
if tcp.addr in _tcp_bag:
_tcp_bag[tcp.addr][2] += 1
elif tcp.nids_state in end_states:
if tcp.addr in _tcp_bag:
_tcp_bag[tcp.addr][1] = nids.get_pkt_ts()
_tcp_bag[tcp.addr][2] += 1
_tcp_bag[tcp.addr][3] = tcp.nids_state
def handleTcpStream(tcp):
global streams_, session_start_, session_end_
if tcp.nids_state == nids.NIDS_JUST_EST:
# new to us, but do we care?
session_start_ = nids.get_pkt_ts()
((src, sport), (dst, dport)) = tcp.addr
if not dports_ or dport in dports_:
tcp.client.collect = 1
tcp.server.collect = 1
elif tcp.nids_state == nids.NIDS_DATA:
# keep all of the stream's new data
tcp.discard(0)
elif tcp.nids_state in end_states:
session_end_ = nids.get_pkt_ts()
clientServerMessages = http_stream_to_array(tcp.server.data[:tcp.server.count])
serverClientMessages = http_stream_to_array(tcp.client.data[:tcp.client.count])
streams_.append({'clientServerMessages': clientServerMessages,
'serverClientMessages': serverClientMessages,
'sessionStart': session_start_,
'sessionEnd': session_end_})
return streams_
def handleTcpStreams(tcp):
end_states = (nids.NIDS_CLOSE, nids.NIDS_TIMEOUT, nids.NIDS_RESET)
client_direction = False
if tcp.server.count_new == 0:
smallest_discard = tcp.client.count_new
client_direction = True
else:
smallest_discard = tcp.server.count_new
global ptimestamp
global metadata
ptimestamp = nids.get_pkt_ts()
((src, sport), (dst, dport)) = tcp.addr
f_string = src + ":" + str(sport) + "-" + dst + ":" + str(dport)
metadata['proto'] = 'tcp'
metadata['time'] = ptimestamp
metadata['addr'] = {'src': src, 'dst': dst, 'sport': sport, 'dport': dport}
if tcp.nids_state == nids.NIDS_JUST_EST: #Implement tasting
for module in tcp_modules:
code = module.code
collecting = False
try:
temp_info = stream_meta(0, 0)
tcpd = copy_tcp_data(tcp, temp_info, 0)
tcpd.timestamp = ptimestamp
tcpd.module_data = module.module_data
#Create a temporary stream_data in case the module needs it -- it'll be saved if the module decides to collect
tcpd.stream_data = stream_meta(
).stream_data #Yes I could probably do = {} but this is more descriptive
collecting = code.taste(tcpd)
except Exception, e:
chop.prettyprnt(
"YELLOW", "Module %s error in taste function: %s" %
(code.moduleName, str(e)))
sys.exit(-1)
module.module_data = tcpd.module_data
if collecting:
module.streaminfo['tcp'][f_string] = stream_meta()
module.streaminfo['tcp'][
f_string].stream_data = tcpd.stream_data
tcp.client.collect = 1
tcp.server.collect = 1
del tcpd
def handleTcpStreams(tcp):
end_states = (nids.NIDS_CLOSE, nids.NIDS_TIMEOUT, nids.NIDS_RESET)
client_direction = False
if tcp.server.count_new == 0:
smallest_discard = tcp.client.count_new
client_direction = True
else:
smallest_discard = tcp.server.count_new
global ptimestamp
global metadata
ptimestamp = nids.get_pkt_ts()
((src,sport),(dst,dport)) = tcp.addr
f_string = src + ":" + str(sport) + "-" + dst + ":" + str(dport)
metadata['proto'] = 'tcp'
metadata['time'] = ptimestamp
metadata['addr'] = { 'src' : src,
'dst' : dst,
'sport' : sport,
'dport' : dport
}
if tcp.nids_state == nids.NIDS_JUST_EST: #Implement tasting
for module in tcp_modules:
code = module.code
collecting = False
try:
temp_info = stream_meta(0,0)
tcpd = copy_tcp_data(tcp,temp_info,0)
tcpd.timestamp = ptimestamp
tcpd.module_data = module.module_data
#Create a temporary stream_data in case the module needs it -- it'll be saved if the module decides to collect
tcpd.stream_data = stream_meta().stream_data #Yes I could probably do = {} but this is more descriptive
collecting = code.taste(tcpd)
except Exception, e:
chop.prettyprnt("YELLOW", "Module %s error in taste function: %s" % (code.moduleName, str(e)))
sys.exit(-1)
module.module_data = tcpd.module_data
if collecting:
module.streaminfo['tcp'][f_string] = stream_meta()
module.streaminfo['tcp'][f_string].stream_data = tcpd.stream_data
tcp.client.collect = 1
tcp.server.collect = 1
del tcpd
def nidsCallback(self, tcp):
if tcp.nids_state == nids.NIDS_JUST_EST:
((src, sport), (dst, dport)) = tcp.addr
if src == "127.0.0.1":
return
print tcp.addr
tcp.client.collect = 1
tcp.server.collect = 1
elif tcp.nids_state == nids.NIDS_DATA:
tcp.discard(0)
elif tcp.nids_state in end_states:
((src, sport), (dst, dport)) = tcp.addr
if (len(tcp.server.data[:tcp.server.count]) > 0
and len(tcp.client.data[:tcp.client.count]) > 0):
self.insertPacketIntoDatabase(
src, sport, tcp.client.data[:tcp.client.count], dst, dport,
tcp.server.data[:tcp.server.count], 'tcp',
nids.get_pkt_ts() * 1000)
elif (len(tcp.server.data[:tcp.server.count]) > 0
and len(tcp.client.data[:tcp.client.count])) == 0:
self.insertPacketIntoDatabase(
src, sport, None, dst, dport,
tcp.server.data[:tcp.server.count], 'tcp',
nids.get_pkt_ts() * 1000)
elif (len(tcp.server.data[:tcp.server.count]) == 0
and len(tcp.client.data[:tcp.client.count]) > 0):
self.insertPacketIntoDatabase(
src, sport, tcp.client.data[:tcp.client.count], dst, dport,
None, 'tcp',
nids.get_pkt_ts() * 1000)
else:
self.insertPacketIntoDatabase(src, sport, None, dst, dport,
None, 'tcp',
nids.get_pkt_ts() * 1000)
def handleIpPackets(pkt):
global timestamp
global metadata
global once
ptimestamp = nids.get_pkt_ts()
if len(pkt) >= 20: #packets should have at least a 20 byte header
#nids should take care of this, but better safe than sorry, I guess
ip = process_ip_data(pkt)
metadata['proto'] = 'ip'
metadata['time'] = ptimestamp
metadata['addr'] = {
'src': ip.src,
'dst': ip.dst,
'dport': '',
'sport': ''
}
for module in ip_modules:
code = module.code
#TODO do we need a shallow or deep copy?
ipd = copy.copy(ip)
ipd.timestamp = ptimestamp
ipd.module_data = module.module_data
try:
output = code.handlePacket(ipd)
except Exception, e:
exc = traceback.format_exc()
chop.prettyprnt(
"YELLOW", "Exception in module %s -- Traceback: \n%s" %
(code.moduleName, exc))
sys.exit(-1)
module.module_data = ipd.module_data
#Handle Children
if not module.legacy:
if output is not None:
ipd.unique = ipd.src + "-" + ipd.dst
ipd.type = 'ip'
handleChildren(module, ipd, output)
del ipd
def handleIpPackets(pkt):
global timestamp
global metadata
global once
ptimestamp = nids.get_pkt_ts()
if len(pkt) >= 20:#packets should have at least a 20 byte header
#nids should take care of this, but better safe than sorry, I guess
ip = process_ip_data(pkt)
metadata['proto'] = 'ip'
metadata['time'] = ptimestamp
metadata['addr'] = { 'src': ip.src,
'dst': ip.dst,
'dport': '',
'sport': ''
}
for module in ip_modules:
code = module.code
#TODO do we need a shallow or deep copy?
ipd = copy.copy(ip)
ipd.timestamp = ptimestamp
ipd.module_data = module.module_data
try:
output = code.handlePacket(ipd)
except Exception, e:
exc = traceback.format_exc()
chop.prettyprnt("YELLOW", "Exception in module %s -- Traceback: \n%s" % (code.moduleName, exc))
sys.exit(-1)
module.module_data = ipd.module_data
#Handle Children
if not module.legacy:
if output is not None:
ipd.unique = ipd.src + "-" + ipd.dst
ipd.type = 'ip'
handleChildren(module, ipd, output)
del ipd
def handle(self, tcp):
self.packet_timestamps.append(nids.get_pkt_ts())
# Established
if tcp.nids_state == nids.NIDS_JUST_EST:
# save the timestamp we just added. :P
self.packet_timestamps = [self.packet_timestamps[-1]]
tcp.client.collect = 1
tcp.server.collect = 1
# Data received
elif tcp.nids_state == nids.NIDS_DATA:
tcp.discard(0)
# End
elif tcp.nids_state in self.END_STATES:
self.emit(tcp)
self.packet_timestamps = []
def capture(self, kind, tcp, channel):
key = tuple([tcp.addr, kind])
if channel.count_new > 0:
if not self.times.has_key(key):
self.times.setdefault(key, {})
self.times[key]['begin'] = nids.get_pkt_ts()
start = channel.count - channel.count_new
payload = channel.data[start:channel.count]
self.data.setdefault(key, '')
self.data[key] += payload
if config.http.response_check_keepalive \
or config.sniffer.introspect_messages:
payload = self.data[key]
if self.is_message_complete(payload):
self.artifact_ready(tcp.addr, kind, key)
elif self.data.has_key(key):
if self.TRACE:
self.dump_data(kind, key)
self.artifact_ready(tcp.addr, kind, key)
def handleudp(addr, payload, pkt):
showmatch = False
addrkey = addr
((src, sport), (dst, dport)) = addr
count = len(payload)
start = 0
end = count
data = payload
if len(configopts['ctsregexes']) > 0 or len(
configopts['ctsfuzzpatterns']) > 0 or len(
configopts['ctsyararules']) > 0:
inspectcts = True
else:
inspectcts = False
if len(configopts['stcregexes']) > 0 or len(
configopts['stcfuzzpatterns']) > 0 or len(
configopts['stcyararules']) > 0:
inspectstc = True
else:
inspectstc = False
keya = "%s:%s" % (src, sport)
keyb = "%s:%s" % (dst, dport)
key = None
if keya in openudpflows:
key = "%s:%s" % (src, sport)
keydst = "%s:%s" % (dst, dport)
direction = configopts['ctsdirectionstring']
directionflag = configopts['ctsdirectionflag']
elif keyb in openudpflows:
key = "%s:%s" % (dst, dport)
keydst = "%s:%s" % (src, sport)
direction = configopts['stcdirectionstring']
directionflag = configopts['stcdirectionflag']
if key in openudpflows and openudpflows[key]['keydst'] == keydst:
openudpflows[key]['totdatasize'] += count
else:
if configopts['verbose'] and configopts['verboselevel'] >= 1:
doinfo(
'[IP#%d.UDP#%d] %s:%s - %s:%s remains untracked { IP tracking missed this flow }'
% (openudpflows[key]['ipct'], openudpflows[key]['id'], src,
sport, dst, dport))
regexes = []
fuzzpatterns = []
yararuleobjects = []
timestamp = datetime.datetime.fromtimestamp(
nids.get_pkt_ts()).strftime('%H:%M:%S | %Y/%m/%d')
if direction == configopts['ctsdirectionstring']:
openudpflows[key]['ctsdatasize'] += count
if 'regex' in configopts['inspectionmodes']:
for regex in configopts['ctsregexes']:
regexes.append(regex)
if 'fuzzy' in configopts['inspectionmodes']:
for fuzzpattern in configopts['ctsfuzzpatterns']:
fuzzpatterns.append(fuzzpattern)
if 'yara' in configopts['inspectionmodes']:
for yararuleobj in configopts['ctsyararules']:
yararuleobjects.append(yararuleobj)
elif direction == configopts['stcdirectionstring']:
openudpflows[key]['stcdatasize'] += count
if 'regex' in configopts['inspectionmodes']:
for regex in configopts['stcregexes']:
regexes.append(regex)
if 'fuzzy' in configopts['inspectionmodes']:
for fuzzpattern in configopts['stcfuzzpatterns']:
fuzzpatterns.append(fuzzpattern)
if 'yara' in configopts['inspectionmodes']:
for yararuleobj in configopts['stcyararules']:
yararuleobjects.append(yararuleobj)
if configopts['verbose'] and configopts['verboselevel'] >= 1:
doinfo(
'[IP#%d.UDP#%d] %s %s %s [%dB] { TRACKED: %d } { CTS: %dB, STC: %dB, TOT: %dB }'
% (openudpflows[key]['ipct'],
openudpflows[key]['id'], key, directionflag, keydst, count,
len(openudpflows), openudpflows[key]['ctsdatasize'],
openudpflows[key]['stcdatasize'],
openudpflows[key]['totdatasize']))
if not configopts['linemode']:
if configopts['udpdone']:
if configopts['tcpdone']:
if configopts['verbose'] and configopts['verboselevel'] >= 1:
doinfo(
'Done inspecting max packets (%d) and max streams (%d), \
preparing for exit' %
(configopts['maxinsppackets'],
configopts['maxinspstreams']))
exitwithstats()
else:
if configopts['verbose'] and configopts['verboselevel'] >= 1:
doinfo(
'Ignoring packet %s:%s %s %s:%s { inspudppacketct: %d == maxinsppackets: %d }'
%
(src, sport, dst, dport, configopts['inspudppacketct'],
configopts['maxinsppackets']))
return
configopts['inspudppacketct'] += 1
if configopts['linemode']:
matchstats['addr'] = addrkey
matchstats['start'] = start
matchstats['end'] = end
matchstats['matchsize'] = matchstats['end'] - matchstats['start']
matchstats['direction'] = direction
matchstats['directionflag'] = directionflag
if configopts['verbose'] and configopts['verboselevel'] >= 1:
doinfo(
'[IP#%d.UDP#%d] Skipping inspection as linemode is enabled.' %
(openudpflows[key]['ipct'], openudpflows[key]['id']))
showudpmatches(data[matchstats['start']:matchstats['end']])
if configopts['writepcap']:
markmatchedippackets(addrkey)
return
if configopts['maxinsppackets'] != 0 and configopts[
'inspudppacketct'] >= configopts['maxinsppackets']:
configopts['udpdone'] = True
if configopts['offset'] > 0 and configopts['offset'] < count:
offset = configopts['offset']
else:
offset = 0
if configopts['depth'] > 0 and configopts['depth'] <= (count - offset):
depth = configopts['depth'] + offset
else:
depth = count
inspdata = data[offset:depth]
inspdatalen = len(inspdata)
if configopts['verbose'] and configopts['verboselevel'] >= 1:
doinfo('[IP#%d.UDP#%d] Initiating inspection on %s[%d:%d] - %dB' %
(openudpflows[key]['ipct'], openudpflows[key]['id'], direction,
offset, depth, inspdatalen))
matched = inspect('UDP', inspdata, inspdatalen, regexes, fuzzpatterns,
yararuleobjects, addrkey, direction, directionflag)
if matched:
openudpflows[key]['matches'] += 1
if configopts['writepcap']:
markmatchedippackets(addrkey)
if configopts['writepcapfast']:
if addrkey in ippacketsdict.keys(
) and ippacketsdict[addrkey]['proto'] == 'UDP':
ippacketsdict[addrkey]['matched'] = True
ippacketsdict[addrkey]['id'] = configopts['packetct']
ippacketsdict[addrkey]['matchedid'] = len(
ippacketsdict[addrkey].keys()) - configopts['ipmetavars']
else:
((sip, sp), (dip, dp)) = addrkey
newaddrkey = ((dip, dp), (sip, sp))
if newaddrkey in ippacketsdict.keys(
) and ippacketsdict[newaddrkey]['proto'] == 'UDP':
ippacketsdict[newaddrkey]['matched'] = True
ippacketsdict[newaddrkey]['id'] = configopts['packetct']
ippacketsdict[newaddrkey]['matchedid'] = len(
ippacketsdict[newaddrkey].keys(
)) - configopts['ipmetavars']
matchstats['start'] += offset
matchstats['end'] += offset
matchstats['direction'] = direction
matchstats['directionflag'] = directionflag
if configopts['udpmatches'] == 0:
configopts['shortestmatch']['packet'] = matchstats['matchsize']
configopts['shortestmatch']['packetid'] = configopts['packetct']
configopts['longestmatch']['packet'] = matchstats['matchsize']
configopts['longestmatch']['packetid'] = configopts['packetct']
else:
if matchstats['matchsize'] <= configopts['shortestmatch']['packet']:
configopts['shortestmatch']['packet'] = matchstats['matchsize']
configopts['shortestmatch']['packetid'] = configopts[
'packetct']
if matchstats['matchsize'] >= configopts['longestmatch']['packet']:
configopts['longestmatch']['packet'] = matchstats['matchsize']
configopts['longestmatch']['packetid'] = configopts['packetct']
configopts['udpmatches'] += 1
matchstats['addr'] = addrkey
showudpmatches(data[matchstats['start']:matchstats['end']])
del openudpflows[key]
def __handleIPPackets(self, ip):
# iptype
# protocol type: tcp/udp/icmp/igmp/igrp/gre/esp/ah
ts = nids.get_pkt_ts()
try:
iphdr = struct.unpack('!BBHHHBBH4s4s', ip[:20])
ipproto = iphdr[6]
ipsrc = socket.inet_ntoa(iphdr[8])
ipdst = socket.inet_ntoa(iphdr[9])
ipihl = iphdr[0] & 0xF
ipihl *= 4 # ip header size
# ipversion = iphdr[0] >> 4
# iptos = iphdr[1]
# iptotallen = iphdr[2]
# ipid = iphdr[3]
# ipttl = iphdr[5]
if ipproto == 6 or ipproto == 17:
tcpudphdr = struct.unpack('!HH', ip[ipihl:ipihl + 4])
portsrc = tcpudphdr[0]
portdst = tcpudphdr[1]
len_of_ip = len(ip)
# ip lookup
try:
src_ip_geo = ip_lookup(ipsrc)
except:
src_ip_geo = ""
try:
dst_ip_geo = ip_lookup(ipdst)
except:
dst_ip_geo = ""
# port lookup
p = PortServiceMap()
src_port_service = p.lookup(portsrc)
dst_port_service = p.lookup(portdst)
# special is a in/out network commucation
if ipsrc == self.local_ip:
if ipdst == self.local_ip:
direct = 0 # loop
else:
direct = 1 # out
elif ipdst == self.local_ip:
direct = 2 # in
else:
direct = 3 # other
# tcp
if ipproto == 6:
tcphdr = struct.unpack('!LLBBHHH',
ip[ipihl + 4:ipihl + 20])
tcpseq = tcphdr[0]
tcpack = tcphdr[1]
tcpoffset = tcphdr[2] >> 4
tcphl = tcpoffset * 4 # tcp header size
tcpflags = tcphdr[3]
tcpwindow = tcphdr[4]
tcpflagsstr = convert(tcpflags)
tcpflagsstr = ",".join(tcpflagsstr)
len_of_data = len_of_ip - ipihl - tcphl
# data=ip[ipihl+tcphl:]
result = direct, \
ts, \
ipproto, ipsrc, ipdst, \
portsrc, portdst, \
len_of_ip, \
src_ip_geo, dst_ip_geo, \
src_port_service, dst_port_service, tcpseq, tcpack, tcpflagsstr, tcpwindow, \
len_of_data
else:
len_of_data = len_of_ip - ipihl - 8 # UDP Header:8
# data=ip[ipihl+8:]
result = direct, \
ts, \
ipproto, ipsrc, ipdst, \
portsrc, portdst, \
len_of_ip, \
src_ip_geo, dst_ip_geo, \
src_port_service, dst_port_service, \
len_of_data
self.__outputIP(result)
except Exception as e:
logging.error(e)
def handleudp(addr, payload, pkt):
showmatch = False
addrkey = addr
((src, sport), (dst, dport)) = addr
count = len(payload)
start = 0
end = count
data = payload
if len(configopts['ctsregexes']) > 0 or len(configopts['ctsfuzzpatterns']) > 0 or len(configopts['ctsyararules']) > 0:
inspectcts = True
else:
inspectcts = False
if len(configopts['stcregexes']) > 0 or len(configopts['stcfuzzpatterns']) > 0 or len(configopts['stcyararules']) > 0:
inspectstc = True
else:
inspectstc = False
keya = "%s:%s" % (src, sport)
keyb = "%s:%s" % (dst, dport)
key = None
if keya in openudpflows:
key = "%s:%s" % (src, sport)
keydst = "%s:%s" % (dst, dport)
direction = configopts['ctsdirectionstring']
directionflag = configopts['ctsdirectionflag']
elif keyb in openudpflows:
key = "%s:%s" % (dst, dport)
keydst = "%s:%s" % (src, sport)
direction = configopts['stcdirectionstring']
directionflag = configopts['stcdirectionflag']
if key in openudpflows and openudpflows[key]['keydst'] == keydst:
openudpflows[key]['totdatasize'] += count
else:
if configopts['verbose'] and configopts['verboselevel'] >= 1:
doinfo('[IP#%d.UDP#%d] %s:%s - %s:%s remains untracked { IP tracking missed this flow }' % (
openudpflows[key]['ipct'],
openudpflows[key]['id'],
src,
sport,
dst,
dport))
regexes = []
fuzzpatterns = []
yararuleobjects = []
timestamp = datetime.datetime.fromtimestamp(nids.get_pkt_ts()).strftime('%H:%M:%S | %Y/%m/%d')
if direction == configopts['ctsdirectionstring']:
openudpflows[key]['ctsdatasize'] += count
if 'regex' in configopts['inspectionmodes']:
for regex in configopts['ctsregexes']:
regexes.append(regex)
if 'fuzzy' in configopts['inspectionmodes']:
for fuzzpattern in configopts['ctsfuzzpatterns']:
fuzzpatterns.append(fuzzpattern)
if 'yara' in configopts['inspectionmodes']:
for yararuleobj in configopts['ctsyararules']:
yararuleobjects.append(yararuleobj)
elif direction == configopts['stcdirectionstring']:
openudpflows[key]['stcdatasize'] += count
if 'regex' in configopts['inspectionmodes']:
for regex in configopts['stcregexes']:
regexes.append(regex)
if 'fuzzy' in configopts['inspectionmodes']:
for fuzzpattern in configopts['stcfuzzpatterns']:
fuzzpatterns.append(fuzzpattern)
if 'yara' in configopts['inspectionmodes']:
for yararuleobj in configopts['stcyararules']:
yararuleobjects.append(yararuleobj)
if configopts['verbose'] and configopts['verboselevel'] >= 1:
doinfo('[IP#%d.UDP#%d] %s %s %s [%dB] { TRACKED: %d } { CTS: %dB, STC: %dB, TOT: %dB }' % (
openudpflows[key]['ipct'],
openudpflows[key]['id'],
key,
directionflag,
keydst,
count,
len(openudpflows),
openudpflows[key]['ctsdatasize'],
openudpflows[key]['stcdatasize'],
openudpflows[key]['totdatasize']))
if not configopts['linemode']:
if configopts['udpdone']:
if configopts['tcpdone']:
if configopts['verbose'] and configopts['verboselevel'] >= 1:
doinfo('Done inspecting max packets (%d) and max streams (%d), \
preparing for exit' % (
configopts['maxinsppackets'],
configopts['maxinspstreams']))
exitwithstats()
else:
if configopts['verbose'] and configopts['verboselevel'] >= 1:
doinfo('Ignoring packet %s:%s %s %s:%s { inspudppacketct: %d == maxinsppackets: %d }' % (
src,
sport,
dst,
dport,
configopts['inspudppacketct'],
configopts['maxinsppackets']))
return
configopts['inspudppacketct'] += 1
if configopts['linemode']:
matchstats['addr'] = addrkey
matchstats['start'] = start
matchstats['end'] = end
matchstats['matchsize'] = matchstats['end'] - matchstats['start']
matchstats['direction'] = direction
matchstats['directionflag'] = directionflag
if configopts['verbose'] and configopts['verboselevel'] >= 1:
doinfo('[IP#%d.UDP#%d] Skipping inspection as linemode is enabled.' % (
openudpflows[key]['ipct'],
openudpflows[key]['id']))
showudpmatches(data[matchstats['start']:matchstats['end']])
if configopts['writepcap']:
markmatchedippackets(addrkey)
return
if configopts['maxinsppackets'] != 0 and configopts['inspudppacketct'] >= configopts['maxinsppackets']:
configopts['udpdone'] = True
if configopts['offset'] > 0 and configopts['offset'] < count:
offset = configopts['offset']
else:
offset = 0
if configopts['depth'] > 0 and configopts['depth'] <= (count - offset):
depth = configopts['depth'] + offset
else:
depth = count
inspdata = data[offset:depth]
inspdatalen = len(inspdata)
if configopts['verbose'] and configopts['verboselevel'] >= 1:
doinfo('[IP#%d.UDP#%d] Initiating inspection on %s[%d:%d] - %dB' % (
openudpflows[key]['ipct'],
openudpflows[key]['id'],
direction,
offset,
depth,
inspdatalen))
matched = inspect('UDP', inspdata, inspdatalen, regexes, fuzzpatterns, yararuleobjects, addrkey, direction, directionflag)
if matched:
openudpflows[key]['matches'] += 1
if configopts['writepcap']:
markmatchedippackets(addrkey)
if configopts['writepcapfast']:
if addrkey in ippacketsdict.keys() and ippacketsdict[addrkey]['proto'] == 'UDP':
ippacketsdict[addrkey]['matched'] = True
ippacketsdict[addrkey]['id'] = configopts['packetct']
ippacketsdict[addrkey]['matchedid'] = len(ippacketsdict[addrkey].keys()) - configopts['ipmetavars']
else:
((sip, sp), (dip, dp)) = addrkey
newaddrkey = ((dip, dp), (sip, sp))
if newaddrkey in ippacketsdict.keys() and ippacketsdict[newaddrkey]['proto'] == 'UDP':
ippacketsdict[newaddrkey]['matched'] = True
ippacketsdict[newaddrkey]['id'] = configopts['packetct']
ippacketsdict[newaddrkey]['matchedid'] = len(ippacketsdict[newaddrkey].keys()) - configopts['ipmetavars']
matchstats['start'] += offset
matchstats['end'] += offset
matchstats['direction'] = direction
matchstats['directionflag'] = directionflag
if configopts['udpmatches'] == 0:
configopts['shortestmatch']['packet'] = matchstats['matchsize']
configopts['shortestmatch']['packetid'] = configopts['packetct']
configopts['longestmatch']['packet'] = matchstats['matchsize']
configopts['longestmatch']['packetid'] = configopts['packetct']
else:
if matchstats['matchsize'] <= configopts['shortestmatch']['packet']:
configopts['shortestmatch']['packet'] = matchstats['matchsize']
configopts['shortestmatch']['packetid'] = configopts['packetct']
if matchstats['matchsize'] >= configopts['longestmatch']['packet']:
configopts['longestmatch']['packet'] = matchstats['matchsize']
configopts['longestmatch']['packetid'] = configopts['packetct']
configopts['udpmatches'] += 1
matchstats['addr'] = addrkey
showudpmatches(data[matchstats['start']:matchstats['end']])
del openudpflows[key]
def gettimestamp():
sometime = int(nids.get_pkt_ts())
return sometime
def udp_handler(self, addrs, payload, pkt):
try:
(src, sport), (dst, dport) = addrs
if sport == 53 or dport == 53:
d = dns.DNS(payload)
id = d.get_transaction_id()
flags = d.get_flags()
if flags & dns.DNSFlags.QR_RESPONSE:
mode = 'response'
else:
mode = 'query'
res = {
'ts': nids.get_pkt_ts(),
'mode': mode,
'id': id,
'questions': [],
'answers': [],
'authoritatives': [],
'additionals': []
}
def add_values(key, keys, values):
info = {}
values = list(values)
values.reverse()
values = tuple(values)
for value in values:
info[keys.pop()] = value
res[key].append(info)
qdcount = d.get_qdcount()
if qdcount > 0:
questions = d.get_questions()
questions.reverse()
while questions:
add_values('questions', ['qname', 'qtype', 'qclass'], questions.pop())
ancount = d.get_ancount()
if ancount > 0:
answers = d.get_answers()
answers.reverse()
while answers:
add_values('answers', ['qname', 'qtype', 'qclass', 'qttl', 'qrdata'], answers.pop())
nscount = d.get_nscount()
if nscount > 0:
authoritatives = d.get_authoritatives()
authoritatives.reverse()
while authoritatives:
add_values('authoritatives', ['qname', 'qtype', 'qclass', 'qttl', 'qrdata'], authoritatives.pop())
arcount = d.get_arcount()
if arcount > 0:
additionals = d.get_additionals()
additionals.reverse()
while additionals:
add_values('additionals', ['qname', 'qtype', 'qclass', 'qttl', 'qrdata'], additionals.pop())
print >> self.output, json.dumps(res)
except Exception, e:
print >> sys.stderr, 'Exception', e
traceback.print_exc(file=sys.stderr)
def handle_tcp_stream(tcp):
global DEBUG
# print "tcps -", str(tcp.addr), " state:", tcp.nids_state
if tcp.nids_state == nids.NIDS_JUST_EST:
# new tcp flow
((srcip, sport), (dstip, dport)) = tcp.addr
h = (srcip, sport, dstip, dport)
#(req_start, req_stop, resp_start, resp_stop)
ts[h] = [nids.get_pkt_ts(), 0, 0 ,0]
requestcounter[h] = 0
requestdata[h] = ''
responsedata[h] = ''
if DEBUG: print "Reconstructing TCP flow:", tcp.addr
tcp.client.collect = 1 # collects server -> client data
tcp.server.collect = 1 # collects client -> server data
elif tcp.nids_state == nids.NIDS_DATA:
# keep all of the stream's new data
tcp.discard(0)
((srcip, sport), (dstip, dport)) = tcp.addr
h = (srcip, sport, dstip, dport)
if requestdata.has_key(h):
client2server_data = tcp.server.data[tcp.server.count-tcp.server.count_new:tcp.server.count]
server2client_data = tcp.client.data[tcp.client.count-tcp.client.count_new:tcp.client.count]
#this if statement is necessary to ensure proper ordering of request/response pairs in the output
if is_http_request(client2server_data):
if len(requestdata[h]) > 0:
if DEBUG: print "Added request/response..."
k = FlowHeader(ts[h][0], ts[h][1], ts[h][2], ts[h][3], h[0], h[1], h[2], h[3])
http_req[k] = add_reconstructed_flow(h)
ts[h] = [nids.get_pkt_ts(), 0, 0 ,0]
if len(client2server_data) > 0:
#sets the start timestamp for request
if(requestdata[h] == ''):
ts[h][0] = nids.get_pkt_ts()
requestdata[h] = requestdata[h] + client2server_data
#sets the end timestamp for request
ts[h][1] = nids.get_pkt_ts()
if len(server2client_data) > 0:
#sets the start timestamp for response
if(responsedata[h] == ''):
ts[h][2] = nids.get_pkt_ts()
responsedata[h] = responsedata[h] + server2client_data
#sets the end timestamp for response
ts[h][3] = nids.get_pkt_ts()
elif tcp.nids_state in NIDS_END_STATES:
((srcip, sport), (dstip, dport)) = tcp.addr
if DEBUG: print "End of flow:", tcp.addr
h = (srcip, sport, dstip, dport)
if requestdata.has_key(h) and is_http_request(requestdata[h]) and is_http_response(responsedata[h]):
k = FlowHeader(ts[h][0], ts[h][1], ts[h][2], ts[h][3], h[0], h[1], h[2], h[3])
http_req[k] = add_reconstructed_flow(h)
else:
if DEBUG:
print "Failed to add flow"
print str(h)
print "has_key? " + str(requestdata.has_key(h))
print "is_http_request? " + str(is_http_request(requestdata[h]))
print "is_http_response? " + str(is_http_response(responsedata[h]))
del ts[h]
del requestdata[h]
del responsedata[h]
del requestcounter[h]
def handleTcpStream(tcp):
global data_flow, ts, contains_flag, done, start_time, inx
if tcp.nids_state == nids.NIDS_JUST_EST:
tcp.client.collect = 1
tcp.server.collect = 1
data_flow[tcp.addr] = []
start_time[tcp.addr] = int(float(nids.get_pkt_ts()) * 1000)
contains_flag[tcp.addr] = False
elif tcp.nids_state == nids.NIDS_DATA:
actor = tcp.client if tcp.client.count_new > 0 else tcp.server
cnt = actor.count_new
data = actor.data[:cnt]
printable_data = ''.join([i if i in string.printable else '\\x{:02x}'.format(ord(i)) for i in data])
name = "c" if actor is tcp.client else "s"
last_flow = (data_flow[tcp.addr] or [None])[-1]
#this is from server, and last one is from server. Just concatenate data
if last_flow and last_flow["from"] == name:
data_flow[tcp.addr][-1]["data"] += printable_data
data_flow[tcp.addr][-1]["hex"] += data.encode("hex")
else:
data_flow[tcp.addr].append(
{"from": name,
"data": printable_data,
"hex": data.encode("hex"),
"time": int(float(nids.get_pkt_ts()) * 1000)
}
)
#only if this we don't know if this flow contains a flag
if not contains_flag[tcp.addr] and containsFlag(data):
contains_flag[tcp.addr] = True
tcp.discard(actor.count_new)
elif tcp.nids_state in end_states:
((src, sport), (dst, dport)) = tcp.addr
done += 1
if done % 100 == 0: print(done)
if len(data_flow[tcp.addr]) == 0:
return
ts = int(float(nids.get_pkt_ts()) * 1000)
current_data = data_flow[tcp.addr]
if len(current_data) > MAX_SIZE: # check if each flow is less than 16 MB (mongodb document limit)
current_data = current_data[:MAX_SIZE] + "<truncated>"
flow = {"inx": inx,
"filename": filename,
"src_ip": src,
"src_port": sport,
"dst_ip": dst,
"dst_port": dport,
"time": start_time[tcp.addr],
"duration": (ts - start_time[tcp.addr]),
"contains_flag": contains_flag[tcp.addr],
"starred": 0,
"flow": current_data
}
flows_to_import.append(flow)
del data_flow[tcp.addr]
def handletcp(tcp):
id = 0
showmatch = False
addrkey = tcp.addr
((src, sport), (dst, dport)) = tcp.addr
if not configopts['linemode']:
if configopts['tcpdone']:
if configopts['udpdone']:
if configopts['verbose'] and configopts['verboselevel'] >= 1:
if addrkey in opentcpflows: id = opentcpflows[addrkey]['id']
doinfo('[IP#%d.TCP#%d] Done inspecting max packets (%d) and max streams (%d), preparing for exit' % (
opentcpflows[addrkey]['ipct'],
opentcpflows[addrkey]['id'],
configopts['maxinsppackets'],
configopts['maxinspstreams']))
exitwithstats()
else:
if configopts['verbose'] and configopts['verboselevel'] >= 1:
if addrkey in opentcpflows: id = opentcpflows[addrkey]['id']
doinfo('[IP#%d.TCP#%d] Ignoring stream %s:%s %s %s:%s { insptcppacketct: %d == maxinspstreams: %d }' % (
opentcpflows[addrkey]['ipct'],
opentcpflows[addrkey]['id'],
src,
sport,
dst,
dport,
configopts['insptcppacketct'],
configopts['maxinspstreams']))
return
regexes = []
fuzzpatterns = []
yararuleobjects = []
timestamp = datetime.datetime.fromtimestamp(nids.get_pkt_ts()).strftime('%H:%M:%S | %Y/%m/%d')
endstates = [ nids.NIDS_CLOSE, nids.NIDS_TIMED_OUT, nids.NIDS_RESET ]
inspectcts = False
inspectstc = False
if len(configopts['ctsregexes']) > 0 or len(configopts['ctsfuzzpatterns']) > 0 or len(configopts['ctsyararules']) > 0:
inspectcts = True
if len(configopts['stcregexes']) > 0 or len(configopts['stcfuzzpatterns']) > 0 or len(configopts['stcyararules']) > 0:
inspectstc = True
if tcp.nids_state == nids.NIDS_JUST_EST:
if addrkey not in opentcpflows:
tcp.server.collect = 0
tcp.client.collect = 0
if configopts['verbose'] and configopts['verboselevel'] >= 1:
doinfo('[IP#%d.TCP#%d] %s:%s - %s:%s remains untracked { IP tracking missed this flow }' % (
opentcpflows[addrkey]['ipct'],
opentcpflows[addrkey]['id'],
src,
sport,
dst,
dport))
else:
configopts['insptcpstreamct'] += 1
if configopts['verbose'] and configopts['verboselevel'] >= 1:
doinfo('[IP#%d.TCP#%d] %s:%s - %s:%s [NEW] { TRACKED: %d }' % (
opentcpflows[addrkey]['ipct'],
opentcpflows[addrkey]['id'],
src,
sport,
dst,
dport,
len(opentcpflows)))
if configopts['linemode'] or 'shellcode' in configopts['inspectionmodes']:
tcp.server.collect = 1
tcp.client.collect = 1
if configopts['verbose'] and configopts['verboselevel'] >= 1:
doinfo('[IP#%d.TCP#%d] Enabled both CTS and STC data collection for %s:%s - %s:%s' % (
opentcpflows[addrkey]['ipct'],
opentcpflows[addrkey]['id'],
src,
sport,
dst,
dport))
else:
if inspectcts or 'shellcode' in configopts['inspectionmodes']:
tcp.server.collect = 1
if configopts['verbose'] and configopts['verboselevel'] >= 1:
doinfo('[IP#%d.TCP#%d] Enabled CTS data collection for %s:%s - %s:%s' % (
opentcpflows[addrkey]['ipct'],
opentcpflows[addrkey]['id'],
src,
sport,
dst,
dport))
if inspectstc or 'shellcode' in configopts['inspectionmodes']:
tcp.client.collect = 1
if configopts['verbose'] and configopts['verboselevel'] >= 1:
doinfo('[IP#%d.TCP#%d] Enabled STC data collection for %s:%s - %s:%s' % (
opentcpflows[addrkey]['ipct'],
opentcpflows[addrkey]['id'],
src,
sport,
dst,
dport))
if tcp.nids_state == nids.NIDS_DATA:
tcp.discard(0)
configopts['insptcppacketct'] += 1
if tcp.server.count_new > 0:
direction = configopts['ctsdirectionstring']
directionflag = configopts['ctsdirectionflag']
count = tcp.server.count
newcount = tcp.server.count_new
start = tcp.server.count - tcp.server.count_new
end = tcp.server.count
opentcpflows[addrkey]['totdatasize'] += tcp.server.count_new
if configopts['offset'] > 0 and configopts['offset'] < count:
offset = configopts['offset']
else:
offset = 0
if configopts['depth'] > 0 and configopts['depth'] <= (count - offset):
depth = configopts['depth'] + offset
else:
depth = count
offset += opentcpflows[addrkey]['multimatchskipoffset']
configopts['inspoffset'] = offset
inspdata = tcp.server.data[offset:depth]
inspdatalen = len(inspdata)
if 'regex' in configopts['inspectionmodes']:
for regex in configopts['ctsregexes'].keys():
regexes.append(regex)
if 'fuzzy' in configopts['inspectionmodes']:
for fuzzpattern in configopts['ctsfuzzpatterns']:
fuzzpatterns.append(fuzzpattern)
if 'yara' in configopts['inspectionmodes']:
for yararuleobj in configopts['ctsyararules']:
yararuleobjects.append(yararuleobj)
if tcp.client.count_new > 0:
direction = configopts['stcdirectionstring']
directionflag = configopts['stcdirectionflag']
count = tcp.client.count
newcount = tcp.client.count_new
start = tcp.client.count - tcp.client.count_new
end = tcp.client.count
opentcpflows[addrkey]['totdatasize'] += tcp.client.count_new
if configopts['offset'] > 0 and configopts['offset'] < count:
offset = configopts['offset']
else:
offset = 0
offset += opentcpflows[addrkey]['multimatchskipoffset']
configopts['inspoffset'] = offset
if configopts['depth'] > 0 and configopts['depth'] < (count - offset):
depth = offset + configopts['depth']
else:
depth = count
inspdata = tcp.client.data[offset:depth]
inspdatalen = len(inspdata)
if 'regex' in configopts['inspectionmodes']:
for regex in configopts['stcregexes'].keys():
regexes.append(regex)
if 'fuzzy' in configopts['inspectionmodes']:
for fuzzpattern in configopts['stcfuzzpatterns']:
fuzzpatterns.append(fuzzpattern)
if 'yara' in configopts['inspectionmodes']:
for yararuleobj in configopts['stcyararules']:
yararuleobjects.append(yararuleobj)
if configopts['verbose'] and configopts['verboselevel'] >= 1:
doinfo('[IP#%d.TCP#%d] %s:%s %s %s:%s [%dB] { CTS: %d, STC: %d, TOT: %d }' % (
opentcpflows[addrkey]['ipct'],
opentcpflows[addrkey]['id'],
src,
sport,
directionflag,
dst,
dport,
newcount,
tcp.server.count,
tcp.client.count,
opentcpflows[addrkey]['totdatasize']))
if configopts['linemode']:
matchstats['addr'] = addrkey
matchstats['start'] = start
matchstats['end'] = end
matchstats['matchsize'] = matchstats['end'] - matchstats['start']
matchstats['direction'] = direction
matchstats['directionflag'] = directionflag
donorm('[IP#%d.TCP#%d] Skipping inspection as linemode is enabled.' % (opentcpflows[addrkey]['ipct'], opentcpflows[addrkey]['id']))
showtcpmatches(inspdata[matchstats['start']:matchstats['end']])
if configopts['writepcap']:
markmatchedippackets(addrkey)
return
if configopts['maxinspstreams'] != 0 and configopts['insptcppacketct'] >= configopts['maxinspstreams']:
configopts['tcpdone'] = True
if configopts['verbose'] and configopts['verboselevel'] >= 1:
doinfo('[IP#%d.TCP#%d] Initiating inspection on %s[%d:%d] - %dB' % (
opentcpflows[addrkey]['ipct'],
opentcpflows[addrkey]['id'],
direction,
offset,
depth,
inspdatalen))
matched = inspect('TCP', inspdata, inspdatalen, regexes, fuzzpatterns, yararuleobjects, addrkey, direction, directionflag)
if matched:
if configopts['killtcp']: tcp.kill
if configopts['writepcap']:
markmatchedippackets(addrkey)
elif configopts['writepcapfast']:
if addrkey in ippacketsdict.keys() and ippacketsdict[addrkey]['proto'] == 'TCP':
ippacketsdict[addrkey]['matched'] = True
ippacketsdict[addrkey]['id'] = opentcpflows[addrkey]['id']
ippacketsdict[addrkey]['matchedid'] = len(ippacketsdict[addrkey].keys()) - configopts['ipmetavars']
else:
((sip, sp), (dip, dp)) = addrkey
newaddrkey = ((dip, dp), (sip, sp))
if newaddrkey in ippacketsdict.keys() and ippacketsdict[newaddrkey]['proto'] == 'TCP':
ippacketsdict[newaddrkey]['matched'] = True
ippacketsdict[newaddrkey]['id'] = opentcpflows[newaddrkey]['id']
ippacketsdict[newaddrkey]['matchedid'] = len(ippacketsdict[newaddrkey].keys()) - configopts['ipmetavars']
if direction == configopts['ctsdirectionstring']:
matchstats['direction'] = configopts['ctsdirectionstring']
matchstats['directionflag'] = configopts['ctsdirectionflag']
elif direction == 'STC':
matchstats['direction'] = configopts['stcdirectionstring']
matchstats['directionflag'] = configopts['stcdirectionflag']
if configopts['tcpmatches'] == 0:
configopts['shortestmatch']['stream'] = matchstats['matchsize']
configopts['shortestmatch']['streamid'] = opentcpflows[addrkey]['id']
configopts['longestmatch']['stream'] = matchstats['matchsize']
configopts['longestmatch']['streamid'] = opentcpflows[addrkey]['id']
else:
if matchstats['matchsize'] <= configopts['shortestmatch']['stream']:
configopts['shortestmatch']['stream'] = matchstats['matchsize']
configopts['shortestmatch']['streamid'] = opentcpflows[addrkey]['id']
if matchstats['matchsize'] >= configopts['longestmatch']['stream']:
configopts['longestmatch']['stream'] = matchstats['matchsize']
configopts['longestmatch']['streamid'] = opentcpflows[addrkey]['id']
configopts['tcpmatches'] += 1
matchstats['addr'] = addrkey
showtcpmatches(inspdata[matchstats['start']:matchstats['end']])
if not configopts['tcpmultimatch']:
tcp.server.collect = 0
tcp.client.collect = 0
if configopts['verbose'] and configopts['verboselevel'] >= 1:
doinfo('[IP#%d.TCP#%d] %s:%s - %s:%s not being tracked any further { tcpmultimatch: %s }' % (
opentcpflows[addrkey]['ipct'],
opentcpflows[addrkey]['id'],
src,
sport,
dst,
dport,
configopts['tcpmultimatch']))
del opentcpflows[addrkey]
else:
if direction == configopts['ctsdirectionstring']:
opentcpflows[addrkey]['ctspacketlendict'].clear()
elif direction == configopts['stcdirectionstring']:
opentcpflows[addrkey]['stcpacketlendict'].clear()
opentcpflows[addrkey]['multimatchskipoffset'] += matchstats['end']
if configopts['verbose'] and configopts['verboselevel'] >= 1:
doinfo('[IP#%d.TCP#%d] Marked %dB to be skipped for further %s inspection' % (
opentcpflows[addrkey]['ipct'],
opentcpflows[addrkey]['id'],
opentcpflows[addrkey]['multimatchskipoffset'],
direction))
else:
opentcpflows[addrkey]['previnspbufsize'] = inspdatalen
if tcp.nids_state in endstates:
if addrkey in opentcpflows:
ipct = opentcpflows[addrkey]['ipct']
id = opentcpflows[addrkey]['id']
del opentcpflows[addrkey]
if configopts['verbose'] and configopts['verboselevel'] >= 1:
if tcp.nids_state == nids.NIDS_CLOSE: state = 'FIN'
elif tcp.nids_state == nids.NIDS_TIMED_OUT: state = 'TIMED_OUT'
elif tcp.nids_state == nids.NIDS_RESET: state = 'RST'
else: state = 'UNKNOWN'
doinfo('[IP#%d.TCP#%d] %s:%s - %s:%s [%s] { TRACKED: %d }' % (
ipct,
id,
src,
sport,
dst,
dport,
state,
len(opentcpflows)))
def finalize_http_flow_header(header):
for i in range(len(header)):
if header[i] == 0:
header[i] = nids.get_pkt_ts()
def udp_callback(self, addrs, payload, pkt):
(saddr, sport), (daddr, dport) = addrs
if self.udp_debug:
print "UDP Packet -> {}".format(nids.get_pkt_ts())
def handleTcpStream(self, tcp):
end_states = (nids.NIDS_CLOSE, nids.NIDS_TIMEOUT, nids.NIDS_RESET)
logging.debug('tcps - {0} state: {1} timestamp: {2}'.format(str(tcp.addr),tcp.nids_state,nids.get_pkt_ts() * 1000))
if tcp.nids_state == nids.NIDS_JUST_EST:
# new to us, but do we care?
((src, sport), (dst, dport)) = tcp.addr
#if dport in self.ports:
logging.info('collecting: {}'.format(str(tcp.addr)))
tcp.client.collect = 1
tcp.server.collect = 1
elif tcp.nids_state == nids.NIDS_DATA:
tcp.discard(0)
# keep all of the stream's new data
#informs nids how many bytes in the stream to discard
#((src, sport), (dst, dport)) = tcp.addr
#serverData = tcp.server.data[:tcp.server.count]
#clientData = tcp.client.data[:tcp.client.count]
#envelopeRegex = '<soap.*:envelope.*<.*MultiSpeakMsgHeader.*<soap.*:envelope>'
#envelopeRegex2 = '</.+:[Ee]nvelope'
#if serverData is None or clientData is None:
# tcp.discard(0)
#else:
# if "Expect: 100-continue" not in serverData:
# tcp.discard(0)
# else:
# if (re.search(envelopeRegex,serverData,re.S | re.IGNORECASE) and re.search(envelopeRegex2,serverData,re.S | re.IGNORECASE) and re.search(envelopeRegex,clientData,re.S | re.IGNORECASE) and re.search(envelopeRegex2,clientData,re.S | re.IGNORECASE)):
# tcpaddr = ((dst,dport),(src,sport))
# logging.debug( "count_new: {}".format(tcp.server.count_new))
# logging.debug( "offset server: {}".format(tcp.server.offset))
# self.process_ipframe(serverData,tcp.addr,self.timestamp)
#
# logging.debug( "count_new: {}".format(tcp.server.count_new))
# logging.debug( "offset client: {}".format(tcp.client.offset))
# tcpaddr = ((dst,dport),(src,sport)) #flip it around to match our point of view (since this is the client
# self.process_ipframe(clientData,tcpaddr,self.timestamp)
# tcp.discard(tcp.server.count + tcp.client.count)
# else:
# tcp.discard(0)
elif tcp.nids_state in end_states:
((src,sport),(dst,dport)) = tcp.addr
serverData = tcp.server.data[:tcp.server.count]
clientData = tcp.client.data[:tcp.client.count]
#logging.debug("serverData: {0}".format( serverData))
#logging.debug("clientData: {0}".format(clientData))
self.timestamp = nids.get_pkt_ts() * 1000
#Add the MultiSpeakMsgHeader since we observed way too many false positives during
#the virtual field test
envelopeRegex = '<soap.*:envelope.*<.*MultiSpeakMsgHeader.*<soap.*:envelope>'
logging.info("Serv Count: {0} Client Count {1} newc: {2} news: {3}".format(tcp.server.count,tcp.client.count,tcp.client.count_new,tcp.server.count_new))
#Match even if there is a newline since we've observed some payloads with the newline
#print("server is ", tcp.server.data[:tcp.server.count], "client is ", tcp.client.data[:tcp.client.count], "count new is ", tcp.server.count_new)
if serverData is not None:
serverData = serverData.replace("\n","")
if (re.search(envelopeRegex,serverData,re.S | re.IGNORECASE)): #and tcp.server.count_new > 0):
logging.info('full message found in tcp server data')
payload = tcp.server.data[:tcp.server.count]
#tcpaddr = ((dst,dport),(src,sport))
logging.debug( "count_new: {}".format(tcp.server.count_new))
logging.debug( "offset server: {}".format(tcp.server.offset))
self.process_ipframe(payload,tcp.addr,self.timestamp)
tcp.discard(tcp.server.count)
elif "multispeak" in serverData.lower():
logging.warning("multispeak serverData but envelope failed: {}".format(serverData))
if clientData is not None:
clientData = clientData.replace("\n","")
if (re.search(envelopeRegex,clientData, re.S | re.IGNORECASE)):
logging.info('full message found in tcp client data')
tcpaddr = ((dst,dport),(src,sport)) #flip it around to match our point of view (since this is the client
payload = tcp.client.data[:tcp.client.count]
logging.debug("count_new client: {}".format(tcp.client.count_new))
logging.debug( "offset client: {}".format(tcp.client.offset))
self.process_ipframe(payload,tcpaddr,self.timestamp) #modified tcpaddr
tcp.discard(tcp.client.count)
elif "multispeak" in clientData.lower():
logging.warning("multispeak clientData but envelope failed: {}".format(clientData))
logging.debug( "addr: {}".format(tcp.addr))
logging.debug( "To server:")
logging.debug( "bytes {}".format(str(tcp.server.count)))
logging.debug( "To client:")
logging.debug( "bytes: {}".format(str(tcp.client.count)))
def udp_callback(self, addrs, payload, pkt):
(saddr, sport), (daddr, dport) = addrs
if self.udp_debug:
print "UDP Packet -> {}".format(nids.get_pkt_ts())
def nidsCallback(self, tcp):
if tcp.nids_state == nids.NIDS_JUST_EST:
((src, sport), (dst, dport)) = tcp.addr
if src == "127.0.0.1":
return
print tcp.addr
tcp.client.collect = 1
tcp.server.collect = 1
elif tcp.nids_state == nids.NIDS_DATA:
tcp.discard(0)
elif tcp.nids_state in end_states:
((src, sport), (dst, dport)) = tcp.addr
if (len(tcp.server.data[:tcp.server.count]) > 0 and len(tcp.client.data[:tcp.client.count]) > 0):
self.insertPacketIntoDatabase(src, sport, tcp.client.data[:tcp.client.count], dst, dport, tcp.server.data[:tcp.server.count], 'tcp', nids.get_pkt_ts() * 1000)
elif (len(tcp.server.data[:tcp.server.count]) > 0 and len(tcp.client.data[:tcp.client.count])) == 0:
self.insertPacketIntoDatabase(src, sport, None, dst, dport, tcp.server.data[:tcp.server.count], 'tcp', nids.get_pkt_ts() * 1000)
elif (len(tcp.server.data[:tcp.server.count]) == 0 and len(tcp.client.data[:tcp.client.count]) > 0):
self.insertPacketIntoDatabase(src, sport, tcp.client.data[:tcp.client.count], dst, dport, None, 'tcp', nids.get_pkt_ts() * 1000)
else:
self.insertPacketIntoDatabase(src, sport, None, dst, dport, None, 'tcp', nids.get_pkt_ts() * 1000)
def __handleIPPackets(self, ip):
# iptype
# protocol type: tcp/udp/icmp/igmp/igrp/gre/esp/ah
ts = nids.get_pkt_ts()
try:
iphdr = struct.unpack('!BBHHHBBH4s4s', ip[:20])
ipproto = iphdr[6]
ipsrc = socket.inet_ntoa(iphdr[8])
ipdst = socket.inet_ntoa(iphdr[9])
ipihl = iphdr[0] & 0xF
ipihl *= 4
if ipproto == 6 or ipproto == 17:
tcpudphdr = struct.unpack('!HH', ip[ipihl:ipihl + 4])
portsrc = tcpudphdr[0]
portdst = tcpudphdr[1]
len_of_ip = len(ip)
# ip lookup
try:
src_ip_geo = ip_lookup(ipsrc)
except:
src_ip_geo = ""
try:
dst_ip_geo = ip_lookup(ipdst)
except:
dst_ip_geo = ""
# port lookup
p = PortServiceMap()
src_port_service = p.lookup(portsrc)
dst_port_service = p.lookup(portdst)
# special is a in/out network commucation
if ipsrc == self.local_ip:
if ipdst == self.local_ip:
direct = 0 # loop
else:
direct = 1 # out
elif ipdst == self.local_ip:
direct = 2 # in
else:
direct = 3 # other
result = direct, \
ts, \
ipproto, ipsrc, ipdst, \
portsrc, portdst, \
len_of_ip, \
src_ip_geo, dst_ip_geo, \
src_port_service, dst_port_service
self.__outputIP(result)
except Exception as e:
logging.error(e)