def get_flowset_id_length_data(self, data, count):
total_len = len(data)
flow_data = data[SIZE_OF_HEADER_9:]
length = [0]
do = False
for flowset_num in range(0, count):
#log.debug("Flowset_num : %s", flowset_num)
start = length[flowset_num] + 0
end = length[flowset_num] + 2
flowset_id_temp = socket.ntohs(
struct.unpack('H', flow_data[start:end])[0])
self.flowset_id.append(flowset_id_temp)
length_temp = socket.ntohs(
struct.unpack(
'H', flow_data[length[flowset_num] +
2:length[flowset_num] + 4])[0])
length.append(length_temp)
#log.debug("length :%s", length)
if length_temp != 0:
#log.debug("Not appending the 0 length flowset")
self.template_header_list.append(
(flowset_id_temp, length_temp))
flow_data_list_temp = flow_data[length[flowset_num] +
0:length[flowset_num] +
length_temp]
self.flow_data_list.append(flow_data_list_temp)
checked_data_length = 0
for i in length:
checked_data_length += i
#log.debug("checked len :%s", checked_data_length + 20)
if (checked_data_length + 20) > total_len:
#log.debug("checked len is greater :%s", checked_data_length)
do = True
#log.debug("doing break no 1")
del length[-1]
del self.flowset_id[-1]
del self.template_header_list[-1]
del self.flow_data_list[-1]
break
elif (checked_data_length + 20) == total_len:
break
else:
continue
#log.debug("doing break no 2")
break
del length[0]
self.length.append(length)
def test_header():
version = get_netflow_packet_version(sample_v5[0:2])
count = socket.ntohs(struct.unpack('H', sample_v5[2:4])[0])
current_unix_sec = (struct.unpack('I', sample_v5[8:12])[0])
eq_(version, 5, "test version")
eq_(count, 1, "test count records")
eq_(current_unix_sec, 1026403152, "test current unix sec")
def proc_resp_get_neighborlist(self, dat):
"""Callback for message RESPONSE_GET_NEIGHBORLIST
List of neighbor node info (the first one is that of the connecting core) is queued rather than message itself.
This method must not be overridden.
Args:
dat (dict): received message
"""
if KeyType.neighbor_list not in dat:
self.queue.put(None)
return
neighbor_list = dat[KeyType.neighbor_list]
results = []
count = int.from_bytes(neighbor_list[:4], 'big')
for i in range(count):
base = 4 + i * (32 + 4 + 16 + 2 + 1 + 8)
node_id = neighbor_list[base:base + 32]
ipv4 = socket.inet_ntop(socket.AF_INET,
neighbor_list[base + 32:base + 36])
ipv6 = socket.inet_ntop(socket.AF_INET6,
neighbor_list[base + 36:base + 52])
port = socket.ntohs(
int.from_bytes(neighbor_list[base + 52:base + 54], 'big'))
domain0 = True if neighbor_list[base + 54] == 0x01 else False
updated_at = neighbor_list[base + 55:base + 63]
results.append([node_id, ipv4, ipv6, port, domain0])
self.queue.put(results)
def proc_resp_get_neighborlist(self, dat):
"""
Return node info
:param dat:
:return: list of node info (the first one is that of the connecting core)
"""
if KeyType.neighbor_list not in dat:
self.queue.put(None)
return
neighbor_list = dat[KeyType.neighbor_list]
results = []
count = int.from_bytes(neighbor_list[:4], 'big')
for i in range(count):
base = 4 + i * (32 + 4 + 16 + 2 + 1 + 8)
node_id = neighbor_list[base:base + 32]
ipv4 = socket.inet_ntop(socket.AF_INET,
neighbor_list[base + 32:base + 36])
ipv6 = socket.inet_ntop(socket.AF_INET6,
neighbor_list[base + 36:base + 52])
port = socket.ntohs(
int.from_bytes(neighbor_list[base + 52:base + 54], 'big'))
domain0 = True if neighbor_list[base + 54] == 0x01 else False
updated_at = neighbor_list[base + 55:base + 63]
results.append([node_id, ipv4, ipv6, port, domain0])
self.queue.put(results)
def parse_template_data(version, ip):
TEMPLATE = TEMPLATE_CACHE
DATA = DATA_CACHE
if version == 10:
TEMPLATE = IPFIX_TEMPLATE_CACHE
DATA = IPFIX_DATA_CACHE
template_info = TEMPLATE.get(ip)
data_info = DATA.get(ip)
#log.debug("temlate : %s and data: %s", template_info, data_info)
if not template_info:
log.info("No any template flowset found till date.")
elif not data_info:
log.info("No any data flowset found till date.")
else:
data_dict = []
for template_id, temp_fields in template_info.iteritems():
field_cnt = len(temp_fields)
data_record_list = data_info.get(template_id)
if not data_record_list:
log.info("Corresponding data for template_id:%s not found.",
template_id)
continue
for record in data_record_list:
record_len, single_data = record
field_dict = {} #dict of field values
i = 0
for field in xrange(field_cnt):
field_type_temp, temp_len = temp_fields[field]
start = i
end = start + temp_len
i = i + temp_len
if temp_len == 1:
val = ord(single_data[start])
elif temp_len == 2:
val = socket.ntohs(
struct.unpack('H', single_data[start:end])[0])
elif temp_len == 4:
val = socket.ntohl(
struct.unpack('I', single_data[start:end])[0])
elif temp_len == 16:
val = socket.inet_ntop(socket.AF_INET6,
single_data[start:end])
field_dict[field_type_temp] = val
_raw_data = single_data
data_dict.append(({template_id: field_dict}, _raw_data))
#log.debug("netflow_v9; data dict: %s", data_dict)
#log.debug("netflow_v9; length data dict: %s", len(data_dict))
if template_id in data_info:
del DATA[ip][template_id]
#log.debug("second : %s", DATA)
return data_dict
def get_flowset_id_length_data(self, data, count):
total_len = len(data)
skip_byte = outself.SIZE_OF_HEADER_9
if self.version == 10:
skip_byte = outself.SIZE_OF_HEADER_IPFIX
flow_data = data[skip_byte:]
length = [0]
offset = 0
#log.debug("count is : %s", count)
for flowset_num in xrange(count):
flowset_id_temp = socket.ntohs(
struct.unpack('H', flow_data[offset:offset + 2])[0])
self.flowset_id.append(flowset_id_temp)
length_temp = socket.ntohs(
struct.unpack('H',
flow_data[offset + 2:offset + 4])[0])
length.append(length_temp)
# log.debug("flowset_id_temp: %s", flowset_id_temp)
# log.debug("flowset_length: %s", length_temp)
if length_temp != 0:
flow_data_list_temp = flow_data[offset + 0:offset +
length_temp]
self.template_header_list.append(
(flowset_id_temp, length_temp,
flow_data_list_temp))
offset += length_temp
if (offset + skip_byte) > total_len:
# print "here :", offset + skip_byte
del length[-1]
del self.flowset_id[-1]
del self.template_header_list[-1]
break
elif (offset + skip_byte) == total_len:
break
else:
continue
del length[0]
self.length.append(length)
def get_template_field_value(self, template_template_list):
'''
template_dict = list of (key,val) ==> {template_id:[(field_type, length)]}
'''
template_dict = {}
for template in template_template_list:
temp_id = template[0]
field_cnt = template[1]
field_list = [] # dict of feild_types:length
i = 0
for field in xrange(field_cnt):
field_type_temp = socket.ntohs(
struct.unpack('H', template[2][4 + i:6 + i])[0])
field_length_temp = socket.ntohs(
struct.unpack('H', template[2][6 + i:8 + i])[0])
field_list.append((field_type_temp, field_length_temp))
i = i + 4
template_dict[temp_id] = field_list
return template_dict
def get_flowset_id_length_data(self, data, count):
total_len = len(data)
skip_byte = SIZE_OF_HEADER_9
if self.version == 10:
skip_byte = SIZE_OF_HEADER_IPFIX
flow_data = data[skip_byte:]
length = [0]
for flowset_num in xrange(count):
start = length[flowset_num] + 0
end = length[flowset_num] + 2
flowset_id_temp = socket.ntohs(
struct.unpack('H', flow_data[start:end])[0])
self.flowset_id.append(flowset_id_temp)
length_temp = socket.ntohs(
struct.unpack(
'H', flow_data[length[flowset_num] +
2:length[flowset_num] + 4])[0])
length.append(length_temp)
if length_temp != 0:
flow_data_list_temp = flow_data[length[flowset_num] +
0:length[flowset_num] +
length_temp]
self.template_header_list.append(
(flowset_id_temp, length_temp, flow_data_list_temp))
checked_data_length = sum(length)
if (checked_data_length + skip_byte) > total_len:
del length[-1]
del self.flowset_id[-1]
del self.template_header_list[-1]
break
elif (checked_data_length + skip_byte) == total_len:
break
else:
continue
del length[0]
self.length.append(length)
def test_header_v9():
version = get_netflow_packet_version(sample_v9[0:2])
count = socket.ntohs(struct.unpack('H', sample_v9[2:4])[0])
current_unix_sec = (struct.unpack('I', sample_v9[8:12])[0])
#Other information in packet header v9
sys_uptime_ms = (struct.unpack('!L', sample_v9[4:8])[0])
unix_secs = (struct.unpack('!L', sample_v9[8:12])[0])
package_sequence = (struct.unpack('!L', sample_v9[12:16])[0])
source_id = (struct.unpack('!L', sample_v9[16:20])[0])
eq_(version, 9, "test version")
eq_(count, 2, "test count records")
eq_(current_unix_sec, 0, "test current unix sec")
eq_(sys_uptime_ms, 29134919, "test sys_uptime")
eq_(unix_secs, 0, "test unix_secs")
eq_(package_sequence, 111, "test package_sequene")
eq_(source_id, 0, "test source_id")
def test__netflow9():
data = sample_v9
count = socket.ntohs(struct.unpack('H', sample_v9[2:4])[0])
ip = '127.0.0.1'
expire_time = 600
sid = "netflow|127.0.0.1"
result = _netflow9(data, count, expire_time, sid, netflow_out, device_name,
col_type, ip, collected_at)
data_dict = {
300: {
1: 64,
2: 1,
4: 17,
5: 1,
6: 0,
7: 4242,
8: 167772162,
9: 32,
10: 3,
11: 80,
12: 167772163,
13: 31,
14: 5,
15: 0,
16: 2,
17: 3,
21: 29134919,
22: 29074919
}
}
data_data = '\n\x00\x00\x02\n\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x05\x00\x00\x00\x01\x00\x00\x00@\x10\x92\x00P\x00\x11\x01\x00\x03\x00\x02 \x1f\x01\xbc\x90G\x01\xbb\xa5\xe7\x00\x00\x00'
#_packet_type_b = 'ASwAEgAIAAQADAAEAA8ABAAKAAQADgAEAAIABAABAAQABwACAAsAAgAGAAEABAABAAUAAQARAAIAEAACAAkAAQANAAEAFQAEABYABA==\n'
expected = [(data_dict, data_data)]
print result, expected
eq_(result, expected)
def handle_udp_data(self, data, **config):
"""
"""
try:
self.VERSION = self.get_netflow_packet_version(data[0:2])
count = socket.ntohs(struct.unpack('H', data[2:4])[0])
current_unix_sec = socket.ntohl(struct.unpack('I', data[8:12])[0])
if ((self.VERSION == 1) or (self.VERSION == 5)
or (self.VERSION == 6) or (self.VERSION == 7)):
if self.VERSION == 1:
log.info("version 1 unpacking...")
self.netflow1.unpack(data)
self.netflowdata = self.netflow1.data
elif self.VERSION == 5:
log.info("version 5 unpacking...")
self.netflow5.unpack(data)
self.netflowdata = self.netflow5.data
elif self.VERSION == 6:
log.info("version 6 unpacking...")
self.netflow6.unpack(data)
self.netflowdata = self.netflow6.data
elif self.VERSION == 7:
log.info("version 7 unpacking...")
self.netflow7.unpack(data)
self.netflowdata = self.netflow7.data
for netflow_record in self.netflowdata:
try:
try:
parsed_msg_dict = self.parse_record(netflow_record)
except Exception as e:
log.error("Could not parse the given record. %s",
repr(e))
parsed_msg_dict["_p__raw_msg_b"] = binascii.b2a_base64(
str(netflow_record))
self.prepare_event(parsed_msg_dict, "version",
self.VERSION, TYPE_NUM)
self.prepare_event(parsed_msg_dict, "current_unix_sec",
current_unix_sec, TYPE_NUM)
self.handle_data(parsed_msg_dict, config)
except Exception as e:
log.error(
"Error in constructing message, Necessary field not supplied in Netflow"
)
log.error(repr(e))
# log.warn(traceback.print_exc())
elif (self.VERSION == 9):
log.info("version 9 unpacking...")
try:
'''
get the v9 data in dict
'''
#str_data = str(data)
#log.debug("v9 str data: %s", str(str_data))
# Other information in packet header
sys_uptime_ms = socket.ntohl(
struct.unpack('I', data[4:8])[0])
unix_secs = socket.ntohl(struct.unpack('I', data[8:12])[0])
package_sequence = socket.ntohl(
struct.unpack('I', data[12:16])[0])
source_id = socket.ntohl(
struct.unpack('I', data[16:20])[0])
msg_dict = {}
msg_dict['version'] = self.VERSION
msg_dict['sys_uptime_ms'] = sys_uptime_ms
msg_dict['unix_secs'] = unix_secs
msg_dict['package_sequence'] = package_sequence
msg_dict['source_id'] = source_id
msg_dict['flowset_count'] = count
netflow_data_list_tuple = self.netflow_parser(
self.VERSION, data, count, self.expire_time, config,
msg_dict)
#log.debug("this is the returned aaaaaaa netflow_tuple: %s", netflow_data_list_tuple)
if netflow_data_list_tuple is not None and len(
netflow_data_list_tuple) != 0:
for data_dict, _p__raw_msg_b in netflow_data_list_tuple:
#log.debug("Testing data dict: %s", data_dict)
# parse record
for (k, v) in data_dict.iteritems():
try:
parsed_msg_dict = self.msgfill_parsed_record(
v)
except Exception as e:
log.error(
"Could not msgfill the parsed v9 record; %s",
repr(e))
parsed_msg_dict[
"_p__raw_msg_b"] = binascii.b2a_base64(
str(_p__raw_msg_b))
parsed_msg_dict['template_id'] = k
#self.prepare_event(parsed_msg_dict, "template_id", k, TYPE_NUM)
parsed_msg_dict.update(msg_dict)
self.prepare_msgfilling(
parsed_msg_dict, TYPE_NUM,
'template_id version sys_uptime_ms package_sequence source_id unix_secs flowset_count'
)
self.handle_data(parsed_msg_dict, config)
except Exception as e:
parsed_msg_dict = {}
parsed_msg_dict["_p__raw_msg_b"] = binascii.b2a_base64(
str(data))
parsed_msg_dict['version'] = self.VERSION
self.prepare_msgfilling(parsed_msg_dict, TYPE_NUM,
'version')
self.handle_data(parsed_msg_dict, config)
log.warn(
"Error in constructing v9 message, Necessary field not supplied in Netflow; %s",
repr(e))
# log.warn(traceback.print_exc())
elif (self.VERSION == 10):
log.info("version 10 unpacking...")
try:
'''
get the ipfix data in dict
'''
# Other information in packet header
sys_uptime_ms = socket.ntohl(
struct.unpack('I', data[4:8])[0])
package_sequence = socket.ntohl(
struct.unpack('I', data[8:12])[0])
source_id = socket.ntohl(
struct.unpack('I', data[12:16])[0])
msg_dict = {}
msg_dict['version'] = self.VERSION
msg_dict['sys_uptime_ms'] = sys_uptime_ms
msg_dict['package_sequence'] = package_sequence
msg_dict['source_id'] = source_id
msg_dict['flowset_length'] = count
netflow_data_list_tuple = self.netflow_parser(
self.VERSION, data, count, self.expire_time, config,
msg_dict)
if netflow_data_list_tuple is not None and len(
netflow_data_list_tuple) != 0:
for data_dict, _p__raw_msg_b in netflow_data_list_tuple:
#log.debug("Testing data dict: %s", data_dict)
# parse record
for (k, v) in data_dict.iteritems():
try:
parsed_msg_dict = self.msgfill_parsed_record(
v)
except Exception as e:
log.error(
"Could not msgfill the parsed ipfix record; %s",
repr(e))
parsed_msg_dict[
'_p__raw_msg_b'] = binascii.b2a_base64(
str(_p__raw_msg_b))
parsed_msg_dict['template_id'] = k
parsed_msg_dict.update(msg_dict)
self.prepare_msgfilling(
parsed_msg_dict, TYPE_NUM,
'template_id version flowset_length sys_uptime_ms package_sequence source_id'
)
self.handle_data(parsed_msg_dict, config)
except Exception as e:
parsed_msg_dict = {}
parsed_msg_dict['_p__raw_msg_b'] = binascii.b2a_base64(
str(data))
parsed_msg_dict['version'] = self.VERSION
self.prepare_msgfilling(parsed_msg_dict, TYPE_NUM,
'version')
self.handle_data(parsed_msg_dict, config)
log.warn(
"Error in constructing IPFIX message, Necessary field not supplied; %s",
repr(e))
# log.warn(traceback.print_exc())
else:
log.error("Not the correct version type.")
except Exception as e:
log.error("Incorrect Netflow data format, %s", repr(e))
def template_data_analyzer(self, config, msg_dict):
'''
template_template_list = (300, 18, 'XXX')
template_data_dict = {300: 'XXXX'}
'''
#log.debug("Analyzing whether it is template or not and setting flag")
for header in self.template_header_list:
each_data = header[2]
if header[
0] == outself.FLOWSET_ID_V9_OPTIONS_TEMPLATE or header[
0] == outself.FLOWSET_ID_V10_OPTIONS_TEMPLATE:
#log.debug("Inside ; this is v10 option template set")
self.option_template_flag = True
if header[1] > 6:
do = True
next_count = 0
while do:
this_flowset_length = socket.ntohs(
struct.unpack(
'H', each_data[2 + next_count:4 +
next_count])[0])
template_id = socket.ntohs(
struct.unpack(
'H', each_data[4 + next_count:6 +
next_count])[0])
if header[
0] == outself.FLOWSET_ID_V9_OPTIONS_TEMPLATE:
option_scope_length = socket.ntohs(
struct.unpack(
'H', each_data[6 + next_count:8 +
next_count])[0])
option_field_length = socket.ntohs(
struct.unpack(
'H', each_data[8 + next_count:10 +
next_count])[0])
total_field_count = int(
(option_scope_length +
option_field_length) / 4)
elif header[
0] == outself.FLOWSET_ID_V10_OPTIONS_TEMPLATE:
total_field_count = socket.ntohs(
struct.unpack(
'H', each_data[6 + next_count:8 +
next_count])[0])
scope_field_count = socket.ntohs(
struct.unpack(
'H', each_data[8 + next_count:10 +
next_count])[0])
present_flag = False
for i in xrange(
len(self.template_template_list)):
# replace the template if template_id is
# repeated
if template_id == self.template_template_list[
i][0]:
present_flag = True
self.template_template_list[i] = (
template_id, total_field_count,
each_data[6:])
break
if not present_flag:
self.template_template_list.append(
(template_id, total_field_count,
each_data[6:]))
# removing flowset_id and length bytes
self.template_data_dict[
template_id] = each_data[6:]
#log.debug("template_id: %s and total_field_count: %s", template_id, total_field_count)
skip_next_count = False
new_next_count = 0
if (self.template_template_list[-1][1] * 4 +
6 + 4) > (header[1]):
log.info(
"Invalid field count vs template data length; header_1: %s: temp_lsit:%s",
header[1],
self.template_template_list[-1])
del self.template_template_list[-1]
break
elif (self.template_template_list[-1][1] * 4 +
6 + 4) < (header[1]):
skip_next_count = True
skip_count = header[1] - 6
if skip_next_count:
next_count += skip_count
else:
next_count += (total_field_count * 2 *
2) + 6
if (next_count + 6) >= header[1]:
do = False
elif header[
0] == outself.FLOWSET_ID_V9_DATA_TEMPLATE or header[
0] == outself.FLOWSET_ID_V10_DATA_TEMPLATE:
'''
do the task of templates here
'''
#log.debug("Inside ; this is template set")
self.template_flag = True
if header[1] > 4:
do = True
next_count = 0
while do:
template_id = socket.ntohs(
struct.unpack(
'H', each_data[4 + next_count:6 +
next_count])[0])
field_count = socket.ntohs(
struct.unpack(
'H', each_data[6 + next_count:8 +
next_count])[0])
present_flag = False
for i in xrange(
len(self.template_template_list)):
# replace the template if template_id is
# repeated
if template_id == self.template_template_list[
i][0]:
present_flag = True
self.template_template_list[i] = (
template_id, field_count,
each_data[4:])
break
if not present_flag:
self.template_template_list.append(
(template_id, field_count,
each_data[4:]))
# removing flowset_id and length bytes
self.template_data_dict[
template_id] = each_data[4:]
#self.template_template_list.append((2344, 4567, "\x00\x12\00"))
#log.debug("template_id: %s and field_count: %s", template_id, field_count)
skip_next_count = False
new_next_count = 0
if (self.template_template_list[-1][1] * 4 +
4 + 4) > (header[1]):
log.info(
"Invalid field count vs template data length; header_1: %s: temp_lsit:%s",
header[1],
self.template_template_list[-1])
del self.template_template_list[-1]
break
elif (self.template_template_list[-1][1] * 4 +
4 + 4) < (header[1]):
skip_next_count = True
skip_count = header[1] - 4
if skip_next_count:
next_count += skip_count
else:
next_count += (field_count * 2 * 2) + 4
if (next_count + 4) >= header[1]:
do = False
else:
'''
do the task of data here
'''
template_id = header[0]
data_length = socket.ntohs(
struct.unpack('H', each_data[2:4])[0])
self.data_header_list.append(
(template_id, data_length, each_data[4:]))
#log.debug("Data header list is : %s", self.data_header_list)
if self.template_flag:
self._handle_template(config,
msg_dict,
option_template=False)
if self.option_template_flag:
self._handle_template(config,
msg_dict,
option_template=True)
def template_data_analyzer(self, sid, netflow_out, device_name, col_type,
device_ip, collected_at):
'''
template_template_list = (300, 18, 'XXX')
template_data_dict = {300: 'XXXX'}
'''
#log.debug("Analyzing whether it is template or not and setting flag")
set_id = 0
if self.version == 10:
set_id = 2
for header in self.template_header_list:
each_data = header[2]
if header[0] == set_id:
'''
do the task of templates here
'''
#log.debug("Inside ; this is template set")
self.template_flag = True
if header[1] > 4:
do = True
next_count = 0
while do:
template_id = socket.ntohs(
struct.unpack(
'H',
each_data[4 + next_count:6 + next_count])[0])
field_count = socket.ntohs(
struct.unpack(
'H',
each_data[6 + next_count:8 + next_count])[0])
present_flag = False
for i in xrange(len(self.template_template_list)):
if template_id == self.template_template_list[i][
0]: #replace the template if template_id is repeated
present_flag = True
self.template_template_list[i] = (
template_id, field_count, each_data[4:])
break
if not present_flag:
self.template_template_list.append(
(template_id, field_count, each_data[4:]))
self.template_data_dict[template_id] = each_data[
4:] #removing flowset_id and length bytes
#self.template_template_list.append((2344, 4567, "\x00\x12\00"))
#log.debug("template_id: %s and field_count: %s", template_id, field_count)
if (self.template_template_list[-1][1] * 4 + 4 +
4) != (header[1]):
log.info(
"Invalid field count vs template data length; header_1: %s: temp_lsit:%s",
header[1], self.template_template_list[-1])
del self.template_template_list[-1]
break
next_count += (field_count * 2 * 2) + 4
if (next_count + 4) >= header[1]:
do = False
else:
'''
do the task of data here
'''
template_id = header[0]
data_length = socket.ntohs(
struct.unpack('H', each_data[2:4])[0])
self.data_header_list.append(
(template_id, data_length, each_data[4:]))
if self.template_flag:
self._handle_template(sid, netflow_out, device_name, col_type,
device_ip, collected_at)
def work(id, que, config, netflow_out, col_type, collected_at, expire_time,
start, netflow1, netflow5, netflow6, netflow7, benchmark_file):
global counter
while True: #not que.qsize() == 0:
log.warn("speed: %s %s" % (time.time() - start, counter))
benchmark_file.write("%d task, total time: %s, counter: %s\n" %
(id, time.time() - start, counter))
#log.warn("%d task:" % id)
counter += 1
#data, addr = sock.recvfrom(9216)
addr = ('::ffff:192.168.2.4', 62826, 0, 0)
data = que.get()
if not data:
break
#continue
#log.debug('udp collector; from ip=%s, got msg=%s;', addr, data)
ip = inet.get_ip(addr)
config_ip = config_reader.get_config_ip(ip, config)
if not config_ip:
continue
sid = _get_sid(config_ip, config)
device_name = config['client_map'][config_ip]["device_name"]
try:
version = get_netflow_packet_version(data[0:2])
count = socket.ntohs(struct.unpack('H', data[2:4])[0])
current_unix_sec = socket.ntohl(struct.unpack('I', data[8:12])[0])
global VERSION
global netflowdata
if ((version == 1) or (version == 5) or (version == 6)
or (version == 7)):
if version == 1:
log.info("version 1 unpacking...")
VERSION = 1
netflow1.unpack(data)
netflowdata = netflow1.data
elif version == 5:
log.info("version 5 unpacking...")
VERSION = 5
netflow5.unpack(data)
netflowdata = netflow5.data
elif version == 6:
log.info("version 6 unpacking...")
VERSION = 6
netflow6.unpack(data)
netflowdata = netflow6.data
elif version == 7:
log.info("version 7 unpacking...")
VERSION = 7
netflow7.unpack(data)
netflowdata = netflow7.data
if not netflowdata:
continue
for netflow_record in netflowdata:
try:
try:
parsed_msg_dict = parse_record(netflow_record)
except Exception, e:
log.error("Could not parse the given record. %s",
repr(e))
parsed_msg_dict['_p__raw_msg_b'] = binascii.b2a_base64(
str(netflow_record))
parsed_msg_dict['version'] = VERSION
parsed_msg_dict['current_unix_sec'] = current_unix_sec
msgfilling.add_types(parsed_msg_dict, '_type_num',
'version current_unix_sec')
_handle_data(parsed_msg_dict, sid, netflow_out,
device_name, col_type, ip, collected_at)
except Exception, e:
log.error(
"Error in constructing message, Necessary field not supplied in Netflow"
)
log.error(repr(e))
except Exception, err:
new_netflow_v9[device_ip] = _New_Netflow_v9(data, count)
log.warn("Cannot create the template and data lists and dict : %s",
repr(err))
#traceback.print_exc()
return "Warning"
log.debug("***************After after updating lists********************")
new_netflow_v9[device_ip].update_template_data_list_dict()
#log.debug("Final template_template_list: %s", new_netflow_v9[device_ip].template_template_list)
#log.debug("Final total data_header_list: %s", new_netflow_v9[device_ip].data_header_list)
flow_data = data[SIZE_OF_HEADER_9:]
#flowset header
flowset_id = socket.ntohs(struct.unpack('H', flow_data[0:2])[0])
length1 = socket.ntohs(struct.unpack('H', flow_data[2:4])[0])
'''
template_dict = list of (key,val) ==> {template_id:[(field_type, length)]}
'''
template_template_list = new_netflow_v9[device_ip].template_template_list
template_data_dict = new_netflow_v9[device_ip].template_data_dict
data_header_list = new_netflow_v9[device_ip].data_header_list
data_data_dict = new_netflow_v9[device_ip].data_data_dict
temp_data_header_list = data_header_list
#log.debug("previous_data :%s :%s\n ", (data_header_list), (previous_data_header_list))
for item in previous_data_header_list:
if item in data_header_list:
def get_raw_template_dict_raw_data_dict(self, sid, netflow_out,
device_name, col_type, device_ip,
collected_at):
log.debug(
".. inside separate the data and template flowsets and store with template_id in the ;;tempalate_data_dict ---> raw template flowset ;;data_data_dict --------> raw data flowset ..."
)
for header_num in range(0, len(self.template_header_list)):
if self.template_header_list[header_num][0] == 0:
'''
do the task of templates here
'''
#log.debug("Inside ; this is template set")
if self.template_header_list[header_num][1] > 4:
#log.debug("Template Diagnonsis; template length: %s", self.template_header_list[header_num][1])
do = True
next_count = 0
while do:
template_id = socket.ntohs(
struct.unpack(
'H', self.flow_data_list[header_num]
[4 + next_count:6 + next_count])[0])
field_count = socket.ntohs(
struct.unpack(
'H', self.flow_data_list[header_num]
[6 + next_count:8 + next_count])[0])
for i in range(len(self.template_template_list)):
if template_id == self.template_template_list[i][
0]: #replace the template if template_id is repeated
self.template_template_list[i] = (template_id,
field_count)
self.template_template_list.append(
(template_id, field_count))
self.template_data_dict[
template_id] = self.flow_data_list[header_num][
4:] #removing flowset_id and length
#self.template_template_list.append((2344, 4567))
#self.template_data_dict[2344] = "\x00\x12\00"
if (self.template_template_list[-1][1] * 4 + 4 + 4
) != (self.template_header_list[header_num][1]):
log.debug(
"Here is true error; Invalid field count vs template data length"
)
del self.template_template_list[-1]
#del self.template_data_dict[2344]
break
'''
log.debug("inside template_id: %s", template_id)
log.debug("inside field_count: %s", field_count)
log.debug("inside template_template_list: %s", self.template_template_list)
log.debug("inside template_data_dict: %s", self.template_data_dict)
'''
next_count += (field_count * 2 * 2) + 4
if (next_count +
4) >= self.template_header_list[header_num][1]:
do = False
else:
log.info(
"Length Problem;;;Deleteing template header with incorrect lengths: %s",
self.template_header_list[header_num])
#del self.template_header_list[header_num]
else:
'''
do the task of data here
'''
#log.debug("Inside;; this is data set")
template_id = self.template_header_list[header_num][0]
data_length = socket.ntohs(
struct.unpack('H',
self.flow_data_list[header_num][2:4])[0])
replaced = False
self.data_header_list.append(
(template_id, data_length,
self.flow_data_list[header_num][4:]))
'''
log.debug("inside data template_id: %s", template_id)
log.debug("inside data dataflow length: %s", data_length)
log.debug("inside self.data_header_list: %s", self.data_header_list)
log.debug("inside self.data_data_dict: %s", self.data_data_dict)
'''
#handle template data
self.update_template_data_list_dict()
for (k, v) in self.template_data_dict.iteritems():
_raw_msg_b = binascii.b2a_base64(self.template_data_dict[k])
event = dict(
_p___raw_msg_b=_raw_msg_b,
packet_type="template",
template_id=k,
)
msgfilling.add_types(event, '_type_str', 'packet_type')
msgfilling.add_types(event, '_type_num', 'template_id')
_handle_data(event, sid, netflow_out, device_name, col_type,
device_ip, collected_at)
class TestNewNetflowv9(unittest.TestCase):
sample = sample_v9
count = socket.ntohs(struct.unpack('H', sample_v9[2:4])[0])
def test_init(self):
new_netflow = _New_Netflow_v9(self.sample, self.count)
eq_(new_netflow.data, self.sample)
eq_(new_netflow.count, self.count)
eq_(new_netflow._raw_data, '')
eq_(new_netflow.length, list())
eq_(new_netflow.flowset_id, list()) #lsit of flowset id
eq_(new_netflow.template_header_list,
list()) #contains flowset_id and length
eq_(new_netflow.flow_data_list, list()) #contains raw data
eq_(new_netflow.template_template_list,
list()) #contains template_id and field count
eq_(new_netflow.data_header_list,
list()) #contains template_id and length of the data
eq_(new_netflow.template_data_dict,
dict()) #contains templateflow data as template_id:data
eq_(new_netflow.data_data_dict,
list()) #contains dataflow data as template_id:data
eq_(new_netflow.unparsed_raw_data, list())
eq_(new_netflow.now, int(time.time()))
assert_not_equal(new_netflow.data, sample_v5)
assert_not_equal(new_netflow.count, 1)
assert_not_equal(new_netflow.unparsed_raw_data, sample_v9)
assert_not_equal(new_netflow.now, int(time.time()) - 1)
def test_get_flowset_id_length_data(self):
new_netflow = _New_Netflow_v9(self.sample, self.count)
eq_(new_netflow.get_flowset_id_length_data(self.sample, self.count),
None, "test get flowset id")
eq_(new_netflow.length, [[80, 56]])
eq_(new_netflow.flowset_id, [0, 300], "test get flowset id")
eq_(new_netflow.template_header_list, [(0, 80), (300, 56)])
eq_(new_netflow.flow_data_list, [
'\x00\x00\x00P\x01,\x00\x12\x00\x08\x00\x04\x00\x0c\x00\x04\x00\x0f\x00\x04\x00\n\x00\x04\x00\x0e\x00\x04\x00\x02\x00\x04\x00\x01\x00\x04\x00\x07\x00\x02\x00\x0b\x00\x02\x00\x06\x00\x01\x00\x04\x00\x01\x00\x05\x00\x01\x00\x11\x00\x02\x00\x10\x00\x02\x00\t\x00\x01\x00\r\x00\x01\x00\x15\x00\x04\x00\x16\x00\x04',
'\x01,\x008\n\x00\x00\x02\n\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x05\x00\x00\x00\x01\x00\x00\x00@\x10\x92\x00P\x00\x11\x01\x00\x03\x00\x02 \x1f\x01\xbc\x90G\x01\xbb\xa5\xe7\x00\x00\x00'
])
assert_not_equal(new_netflow.length, list())
assert_not_equal(new_netflow.flowset_id, list())
def test_get_raw_template_dict_raw_data_dict(self):
new_netflow = _New_Netflow_v9(self.sample, self.count)
eq_(new_netflow.get_flowset_id_length_data(self.sample, self.count),
None, "test get flowset id")
eq_(
new_netflow.get_raw_template_dict_raw_data_dict(
sid, netflow_out, device_name, col_type, ip, collected_at),
None, "get_raw_data")
eq_(new_netflow.template_template_list, [(300, 18)])
expected_data_header_list = [(
300, 56,
'\n\x00\x00\x02\n\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x05\x00\x00\x00\x01\x00\x00\x00@\x10\x92\x00P\x00\x11\x01\x00\x03\x00\x02 \x1f\x01\xbc\x90G\x01\xbb\xa5\xe7\x00\x00\x00'
)]
eq_(new_netflow.data_header_list, expected_data_header_list)
assert_not_equal(new_netflow.template_template_list, dict())
assert_not_equal(new_netflow.template_template_list, list())
assert_not_equal(new_netflow.template_template_list, list((300, 0)))
def test_update_template_data_list_dict(self):
new_netflow = _New_Netflow_v9(self.sample, self.count)
eq_(new_netflow.get_flowset_id_length_data(self.sample, self.count),
None, "test get flowset id")
eq_(new_netflow.update_template_data_list_dict(), None,
"test update template data list")
eq_(new_netflow.length, [[80, 56]])
eq_(new_netflow.flowset_id, [0, 300])
eq_(new_netflow.template_header_list, [(0, 80), (300, 56)])
eq_(new_netflow.flow_data_list, [
'\x00\x00\x00P\x01,\x00\x12\x00\x08\x00\x04\x00\x0c\x00\x04\x00\x0f\x00\x04\x00\n\x00\x04\x00\x0e\x00\x04\x00\x02\x00\x04\x00\x01\x00\x04\x00\x07\x00\x02\x00\x0b\x00\x02\x00\x06\x00\x01\x00\x04\x00\x01\x00\x05\x00\x01\x00\x11\x00\x02\x00\x10\x00\x02\x00\t\x00\x01\x00\r\x00\x01\x00\x15\x00\x04\x00\x16\x00\x04',
'\x01,\x008\n\x00\x00\x02\n\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x05\x00\x00\x00\x01\x00\x00\x00@\x10\x92\x00P\x00\x11\x01\x00\x03\x00\x02 \x1f\x01\xbc\x90G\x01\xbb\xa5\xe7\x00\x00\x00'
])
assert_not_equal(new_netflow.length, list())
assert_not_equal(new_netflow.flowset_id, list())
def test_clean_unparsed_raw_data(self):
new_netflow = _New_Netflow_v9(self.sample, self.count)
eq_(new_netflow.get_flowset_id_length_data(self.sample, self.count),
None, "test get flowset id")
eq_(
new_netflow.get_raw_template_dict_raw_data_dict(
sid, netflow_out, device_name, col_type, ip, collected_at),
None, "test update template data list")
eq_(new_netflow.update_template_data_list_dict(), None,
"test update template data list")
eq_(new_netflow.unparsed_raw_data, list())
assert_not_equal(new_netflow.flowset_id, list(self.sample))
assert_not_equal(new_netflow.flowset_id, dict())
def main():
config = _parse_args()
log_level = config['core']['log_level']
port = config['port']
expire_time = config['expire_time']
col_type = config['col_type']
collected_at = config["loginspect_name"]
zmq_context = zmq.Context()
netflow_out = wiring.Wire('collector_out',
zmq_context=zmq_context,
conf_path=config.get('wiring_conf_path') or None)
sock = start_udp_server(port)
while True:
data, addr = sock.recvfrom(9216)
if not data:
continue
log.debug('udp collector; from ip=%s, got msg=%s;', addr, data)
ip = inet.get_ip(addr)
config_ip = config_reader.get_config_ip(ip, config)
if not config_ip:
continue
try:
version = get_netflow_packet_version(data[0:2])
count = socket.ntohs(struct.unpack('H', data[2:4])[0])
current_unix_sec = (struct.unpack('I', data[8:12])[0])
log.debug("Version: %s", version)
log.debug("Count of no. of records: %s", count)
log.debug("Count of no. of seconds since 0000 UTC 1970: %s",
current_unix_sec)
netflow1 = netflow.Netflow1()
netflow5 = netflow.Netflow5()
netflow6 = netflow.Netflow6()
netflow7 = netflow.Netflow7()
global VERSION
global netflowdata
if ((version == 1) or (version == 5) or (version == 6)
or (version == 7)):
if version == 1:
log.info("version 1 unpacking...")
VERSION = 1
netflow1.unpack(data)
netflowdata = netflow1.data
elif version == 5:
log.info("version 5 unpacking...")
VERSION = 5
netflow5.unpack(data)
netflowdata = netflow5.data
elif version == 6:
log.info("version 6 unpacking...")
VERSION = 6
netflow6.unpack(data)
netflowdata = netflow6.data
elif version == 7:
log.info("version 7 unpacking...")
VERSION = 7
netflow7.unpack(data)
netflowdata = netflow7.data
i = 1
if not netflowdata:
continue
for netflow_record in netflowdata:
try:
i = i + 1
try:
parsed_msg_dict = parse_record(netflow_record)
except Exception, e:
log.error("Could not parse the given record. %s",
repr(e))
parsed_msg_dict['_p__raw_msg_b'] = binascii.b2a_base64(
str(netflow_record))
parsed_msg_dict['version'] = VERSION
parsed_msg_dict['current_unix_sec'] = current_unix_sec
msgfilling.add_types(parsed_msg_dict, '_type_num',
'version current_unix_sec')
sid = _get_sid(config_ip, config)
device_name = config['client_map'][config_ip][
"device_name"]
log.debug("device: %s", device_name)
log.debug("descrete ip: %s", ip)
try:
_handle_data(parsed_msg_dict, sid, netflow_out,
device_name, col_type, ip,
collected_at)
except Exception, e:
log.error("Device name not found. %s", repr(e))
except Exception, e:
log.error(
"Error in constructing message, Necessary field not supplied in Netflow"
)
log.error(repr(e))
