def get_flowset_id_length_data(self, data, count): total_len = len(data) flow_data = data[SIZE_OF_HEADER_9:] length = [0] do = False for flowset_num in range(0, count): #log.debug("Flowset_num : %s", flowset_num) start = length[flowset_num] + 0 end = length[flowset_num] + 2 flowset_id_temp = socket.ntohs( struct.unpack('H', flow_data[start:end])[0]) self.flowset_id.append(flowset_id_temp) length_temp = socket.ntohs( struct.unpack( 'H', flow_data[length[flowset_num] + 2:length[flowset_num] + 4])[0]) length.append(length_temp) #log.debug("length :%s", length) if length_temp != 0: #log.debug("Not appending the 0 length flowset") self.template_header_list.append( (flowset_id_temp, length_temp)) flow_data_list_temp = flow_data[length[flowset_num] + 0:length[flowset_num] + length_temp] self.flow_data_list.append(flow_data_list_temp) checked_data_length = 0 for i in length: checked_data_length += i #log.debug("checked len :%s", checked_data_length + 20) if (checked_data_length + 20) > total_len: #log.debug("checked len is greater :%s", checked_data_length) do = True #log.debug("doing break no 1") del length[-1] del self.flowset_id[-1] del self.template_header_list[-1] del self.flow_data_list[-1] break elif (checked_data_length + 20) == total_len: break else: continue #log.debug("doing break no 2") break del length[0] self.length.append(length)
def test_header(): version = get_netflow_packet_version(sample_v5[0:2]) count = socket.ntohs(struct.unpack('H', sample_v5[2:4])[0]) current_unix_sec = (struct.unpack('I', sample_v5[8:12])[0]) eq_(version, 5, "test version") eq_(count, 1, "test count records") eq_(current_unix_sec, 1026403152, "test current unix sec")
def proc_resp_get_neighborlist(self, dat): """Callback for message RESPONSE_GET_NEIGHBORLIST List of neighbor node info (the first one is that of the connecting core) is queued rather than message itself. This method must not be overridden. Args: dat (dict): received message """ if KeyType.neighbor_list not in dat: self.queue.put(None) return neighbor_list = dat[KeyType.neighbor_list] results = [] count = int.from_bytes(neighbor_list[:4], 'big') for i in range(count): base = 4 + i * (32 + 4 + 16 + 2 + 1 + 8) node_id = neighbor_list[base:base + 32] ipv4 = socket.inet_ntop(socket.AF_INET, neighbor_list[base + 32:base + 36]) ipv6 = socket.inet_ntop(socket.AF_INET6, neighbor_list[base + 36:base + 52]) port = socket.ntohs( int.from_bytes(neighbor_list[base + 52:base + 54], 'big')) domain0 = True if neighbor_list[base + 54] == 0x01 else False updated_at = neighbor_list[base + 55:base + 63] results.append([node_id, ipv4, ipv6, port, domain0]) self.queue.put(results)
def proc_resp_get_neighborlist(self, dat): """ Return node info :param dat: :return: list of node info (the first one is that of the connecting core) """ if KeyType.neighbor_list not in dat: self.queue.put(None) return neighbor_list = dat[KeyType.neighbor_list] results = [] count = int.from_bytes(neighbor_list[:4], 'big') for i in range(count): base = 4 + i * (32 + 4 + 16 + 2 + 1 + 8) node_id = neighbor_list[base:base + 32] ipv4 = socket.inet_ntop(socket.AF_INET, neighbor_list[base + 32:base + 36]) ipv6 = socket.inet_ntop(socket.AF_INET6, neighbor_list[base + 36:base + 52]) port = socket.ntohs( int.from_bytes(neighbor_list[base + 52:base + 54], 'big')) domain0 = True if neighbor_list[base + 54] == 0x01 else False updated_at = neighbor_list[base + 55:base + 63] results.append([node_id, ipv4, ipv6, port, domain0]) self.queue.put(results)
def parse_template_data(version, ip): TEMPLATE = TEMPLATE_CACHE DATA = DATA_CACHE if version == 10: TEMPLATE = IPFIX_TEMPLATE_CACHE DATA = IPFIX_DATA_CACHE template_info = TEMPLATE.get(ip) data_info = DATA.get(ip) #log.debug("temlate : %s and data: %s", template_info, data_info) if not template_info: log.info("No any template flowset found till date.") elif not data_info: log.info("No any data flowset found till date.") else: data_dict = [] for template_id, temp_fields in template_info.iteritems(): field_cnt = len(temp_fields) data_record_list = data_info.get(template_id) if not data_record_list: log.info("Corresponding data for template_id:%s not found.", template_id) continue for record in data_record_list: record_len, single_data = record field_dict = {} #dict of field values i = 0 for field in xrange(field_cnt): field_type_temp, temp_len = temp_fields[field] start = i end = start + temp_len i = i + temp_len if temp_len == 1: val = ord(single_data[start]) elif temp_len == 2: val = socket.ntohs( struct.unpack('H', single_data[start:end])[0]) elif temp_len == 4: val = socket.ntohl( struct.unpack('I', single_data[start:end])[0]) elif temp_len == 16: val = socket.inet_ntop(socket.AF_INET6, single_data[start:end]) field_dict[field_type_temp] = val _raw_data = single_data data_dict.append(({template_id: field_dict}, _raw_data)) #log.debug("netflow_v9; data dict: %s", data_dict) #log.debug("netflow_v9; length data dict: %s", len(data_dict)) if template_id in data_info: del DATA[ip][template_id] #log.debug("second : %s", DATA) return data_dict
def get_flowset_id_length_data(self, data, count): total_len = len(data) skip_byte = outself.SIZE_OF_HEADER_9 if self.version == 10: skip_byte = outself.SIZE_OF_HEADER_IPFIX flow_data = data[skip_byte:] length = [0] offset = 0 #log.debug("count is : %s", count) for flowset_num in xrange(count): flowset_id_temp = socket.ntohs( struct.unpack('H', flow_data[offset:offset + 2])[0]) self.flowset_id.append(flowset_id_temp) length_temp = socket.ntohs( struct.unpack('H', flow_data[offset + 2:offset + 4])[0]) length.append(length_temp) # log.debug("flowset_id_temp: %s", flowset_id_temp) # log.debug("flowset_length: %s", length_temp) if length_temp != 0: flow_data_list_temp = flow_data[offset + 0:offset + length_temp] self.template_header_list.append( (flowset_id_temp, length_temp, flow_data_list_temp)) offset += length_temp if (offset + skip_byte) > total_len: # print "here :", offset + skip_byte del length[-1] del self.flowset_id[-1] del self.template_header_list[-1] break elif (offset + skip_byte) == total_len: break else: continue del length[0] self.length.append(length)
def get_template_field_value(self, template_template_list): ''' template_dict = list of (key,val) ==> {template_id:[(field_type, length)]} ''' template_dict = {} for template in template_template_list: temp_id = template[0] field_cnt = template[1] field_list = [] # dict of feild_types:length i = 0 for field in xrange(field_cnt): field_type_temp = socket.ntohs( struct.unpack('H', template[2][4 + i:6 + i])[0]) field_length_temp = socket.ntohs( struct.unpack('H', template[2][6 + i:8 + i])[0]) field_list.append((field_type_temp, field_length_temp)) i = i + 4 template_dict[temp_id] = field_list return template_dict
def get_flowset_id_length_data(self, data, count): total_len = len(data) skip_byte = SIZE_OF_HEADER_9 if self.version == 10: skip_byte = SIZE_OF_HEADER_IPFIX flow_data = data[skip_byte:] length = [0] for flowset_num in xrange(count): start = length[flowset_num] + 0 end = length[flowset_num] + 2 flowset_id_temp = socket.ntohs( struct.unpack('H', flow_data[start:end])[0]) self.flowset_id.append(flowset_id_temp) length_temp = socket.ntohs( struct.unpack( 'H', flow_data[length[flowset_num] + 2:length[flowset_num] + 4])[0]) length.append(length_temp) if length_temp != 0: flow_data_list_temp = flow_data[length[flowset_num] + 0:length[flowset_num] + length_temp] self.template_header_list.append( (flowset_id_temp, length_temp, flow_data_list_temp)) checked_data_length = sum(length) if (checked_data_length + skip_byte) > total_len: del length[-1] del self.flowset_id[-1] del self.template_header_list[-1] break elif (checked_data_length + skip_byte) == total_len: break else: continue del length[0] self.length.append(length)
def test_header_v9(): version = get_netflow_packet_version(sample_v9[0:2]) count = socket.ntohs(struct.unpack('H', sample_v9[2:4])[0]) current_unix_sec = (struct.unpack('I', sample_v9[8:12])[0]) #Other information in packet header v9 sys_uptime_ms = (struct.unpack('!L', sample_v9[4:8])[0]) unix_secs = (struct.unpack('!L', sample_v9[8:12])[0]) package_sequence = (struct.unpack('!L', sample_v9[12:16])[0]) source_id = (struct.unpack('!L', sample_v9[16:20])[0]) eq_(version, 9, "test version") eq_(count, 2, "test count records") eq_(current_unix_sec, 0, "test current unix sec") eq_(sys_uptime_ms, 29134919, "test sys_uptime") eq_(unix_secs, 0, "test unix_secs") eq_(package_sequence, 111, "test package_sequene") eq_(source_id, 0, "test source_id")
def test__netflow9(): data = sample_v9 count = socket.ntohs(struct.unpack('H', sample_v9[2:4])[0]) ip = '127.0.0.1' expire_time = 600 sid = "netflow|127.0.0.1" result = _netflow9(data, count, expire_time, sid, netflow_out, device_name, col_type, ip, collected_at) data_dict = { 300: { 1: 64, 2: 1, 4: 17, 5: 1, 6: 0, 7: 4242, 8: 167772162, 9: 32, 10: 3, 11: 80, 12: 167772163, 13: 31, 14: 5, 15: 0, 16: 2, 17: 3, 21: 29134919, 22: 29074919 } } data_data = '\n\x00\x00\x02\n\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x05\x00\x00\x00\x01\x00\x00\x00@\x10\x92\x00P\x00\x11\x01\x00\x03\x00\x02 \x1f\x01\xbc\x90G\x01\xbb\xa5\xe7\x00\x00\x00' #_packet_type_b = 'ASwAEgAIAAQADAAEAA8ABAAKAAQADgAEAAIABAABAAQABwACAAsAAgAGAAEABAABAAUAAQARAAIAEAACAAkAAQANAAEAFQAEABYABA==\n' expected = [(data_dict, data_data)] print result, expected eq_(result, expected)
def handle_udp_data(self, data, **config): """ """ try: self.VERSION = self.get_netflow_packet_version(data[0:2]) count = socket.ntohs(struct.unpack('H', data[2:4])[0]) current_unix_sec = socket.ntohl(struct.unpack('I', data[8:12])[0]) if ((self.VERSION == 1) or (self.VERSION == 5) or (self.VERSION == 6) or (self.VERSION == 7)): if self.VERSION == 1: log.info("version 1 unpacking...") self.netflow1.unpack(data) self.netflowdata = self.netflow1.data elif self.VERSION == 5: log.info("version 5 unpacking...") self.netflow5.unpack(data) self.netflowdata = self.netflow5.data elif self.VERSION == 6: log.info("version 6 unpacking...") self.netflow6.unpack(data) self.netflowdata = self.netflow6.data elif self.VERSION == 7: log.info("version 7 unpacking...") self.netflow7.unpack(data) self.netflowdata = self.netflow7.data for netflow_record in self.netflowdata: try: try: parsed_msg_dict = self.parse_record(netflow_record) except Exception as e: log.error("Could not parse the given record. %s", repr(e)) parsed_msg_dict["_p__raw_msg_b"] = binascii.b2a_base64( str(netflow_record)) self.prepare_event(parsed_msg_dict, "version", self.VERSION, TYPE_NUM) self.prepare_event(parsed_msg_dict, "current_unix_sec", current_unix_sec, TYPE_NUM) self.handle_data(parsed_msg_dict, config) except Exception as e: log.error( "Error in constructing message, Necessary field not supplied in Netflow" ) log.error(repr(e)) # log.warn(traceback.print_exc()) elif (self.VERSION == 9): log.info("version 9 unpacking...") try: ''' get the v9 data in dict ''' #str_data = str(data) #log.debug("v9 str data: %s", str(str_data)) # Other information in packet header sys_uptime_ms = socket.ntohl( struct.unpack('I', data[4:8])[0]) unix_secs = socket.ntohl(struct.unpack('I', data[8:12])[0]) package_sequence = socket.ntohl( struct.unpack('I', data[12:16])[0]) source_id = socket.ntohl( struct.unpack('I', data[16:20])[0]) msg_dict = {} msg_dict['version'] = self.VERSION msg_dict['sys_uptime_ms'] = sys_uptime_ms msg_dict['unix_secs'] = unix_secs msg_dict['package_sequence'] = package_sequence msg_dict['source_id'] = source_id msg_dict['flowset_count'] = count netflow_data_list_tuple = self.netflow_parser( self.VERSION, data, count, self.expire_time, config, msg_dict) #log.debug("this is the returned aaaaaaa netflow_tuple: %s", netflow_data_list_tuple) if netflow_data_list_tuple is not None and len( netflow_data_list_tuple) != 0: for data_dict, _p__raw_msg_b in netflow_data_list_tuple: #log.debug("Testing data dict: %s", data_dict) # parse record for (k, v) in data_dict.iteritems(): try: parsed_msg_dict = self.msgfill_parsed_record( v) except Exception as e: log.error( "Could not msgfill the parsed v9 record; %s", repr(e)) parsed_msg_dict[ "_p__raw_msg_b"] = binascii.b2a_base64( str(_p__raw_msg_b)) parsed_msg_dict['template_id'] = k #self.prepare_event(parsed_msg_dict, "template_id", k, TYPE_NUM) parsed_msg_dict.update(msg_dict) self.prepare_msgfilling( parsed_msg_dict, TYPE_NUM, 'template_id version sys_uptime_ms package_sequence source_id unix_secs flowset_count' ) self.handle_data(parsed_msg_dict, config) except Exception as e: parsed_msg_dict = {} parsed_msg_dict["_p__raw_msg_b"] = binascii.b2a_base64( str(data)) parsed_msg_dict['version'] = self.VERSION self.prepare_msgfilling(parsed_msg_dict, TYPE_NUM, 'version') self.handle_data(parsed_msg_dict, config) log.warn( "Error in constructing v9 message, Necessary field not supplied in Netflow; %s", repr(e)) # log.warn(traceback.print_exc()) elif (self.VERSION == 10): log.info("version 10 unpacking...") try: ''' get the ipfix data in dict ''' # Other information in packet header sys_uptime_ms = socket.ntohl( struct.unpack('I', data[4:8])[0]) package_sequence = socket.ntohl( struct.unpack('I', data[8:12])[0]) source_id = socket.ntohl( struct.unpack('I', data[12:16])[0]) msg_dict = {} msg_dict['version'] = self.VERSION msg_dict['sys_uptime_ms'] = sys_uptime_ms msg_dict['package_sequence'] = package_sequence msg_dict['source_id'] = source_id msg_dict['flowset_length'] = count netflow_data_list_tuple = self.netflow_parser( self.VERSION, data, count, self.expire_time, config, msg_dict) if netflow_data_list_tuple is not None and len( netflow_data_list_tuple) != 0: for data_dict, _p__raw_msg_b in netflow_data_list_tuple: #log.debug("Testing data dict: %s", data_dict) # parse record for (k, v) in data_dict.iteritems(): try: parsed_msg_dict = self.msgfill_parsed_record( v) except Exception as e: log.error( "Could not msgfill the parsed ipfix record; %s", repr(e)) parsed_msg_dict[ '_p__raw_msg_b'] = binascii.b2a_base64( str(_p__raw_msg_b)) parsed_msg_dict['template_id'] = k parsed_msg_dict.update(msg_dict) self.prepare_msgfilling( parsed_msg_dict, TYPE_NUM, 'template_id version flowset_length sys_uptime_ms package_sequence source_id' ) self.handle_data(parsed_msg_dict, config) except Exception as e: parsed_msg_dict = {} parsed_msg_dict['_p__raw_msg_b'] = binascii.b2a_base64( str(data)) parsed_msg_dict['version'] = self.VERSION self.prepare_msgfilling(parsed_msg_dict, TYPE_NUM, 'version') self.handle_data(parsed_msg_dict, config) log.warn( "Error in constructing IPFIX message, Necessary field not supplied; %s", repr(e)) # log.warn(traceback.print_exc()) else: log.error("Not the correct version type.") except Exception as e: log.error("Incorrect Netflow data format, %s", repr(e))
def template_data_analyzer(self, config, msg_dict): ''' template_template_list = (300, 18, 'XXX') template_data_dict = {300: 'XXXX'} ''' #log.debug("Analyzing whether it is template or not and setting flag") for header in self.template_header_list: each_data = header[2] if header[ 0] == outself.FLOWSET_ID_V9_OPTIONS_TEMPLATE or header[ 0] == outself.FLOWSET_ID_V10_OPTIONS_TEMPLATE: #log.debug("Inside ; this is v10 option template set") self.option_template_flag = True if header[1] > 6: do = True next_count = 0 while do: this_flowset_length = socket.ntohs( struct.unpack( 'H', each_data[2 + next_count:4 + next_count])[0]) template_id = socket.ntohs( struct.unpack( 'H', each_data[4 + next_count:6 + next_count])[0]) if header[ 0] == outself.FLOWSET_ID_V9_OPTIONS_TEMPLATE: option_scope_length = socket.ntohs( struct.unpack( 'H', each_data[6 + next_count:8 + next_count])[0]) option_field_length = socket.ntohs( struct.unpack( 'H', each_data[8 + next_count:10 + next_count])[0]) total_field_count = int( (option_scope_length + option_field_length) / 4) elif header[ 0] == outself.FLOWSET_ID_V10_OPTIONS_TEMPLATE: total_field_count = socket.ntohs( struct.unpack( 'H', each_data[6 + next_count:8 + next_count])[0]) scope_field_count = socket.ntohs( struct.unpack( 'H', each_data[8 + next_count:10 + next_count])[0]) present_flag = False for i in xrange( len(self.template_template_list)): # replace the template if template_id is # repeated if template_id == self.template_template_list[ i][0]: present_flag = True self.template_template_list[i] = ( template_id, total_field_count, each_data[6:]) break if not present_flag: self.template_template_list.append( (template_id, total_field_count, each_data[6:])) # removing flowset_id and length bytes self.template_data_dict[ template_id] = each_data[6:] #log.debug("template_id: %s and total_field_count: %s", template_id, total_field_count) skip_next_count = False new_next_count = 0 if (self.template_template_list[-1][1] * 4 + 6 + 4) > (header[1]): log.info( "Invalid field count vs template data length; header_1: %s: temp_lsit:%s", header[1], self.template_template_list[-1]) del self.template_template_list[-1] break elif (self.template_template_list[-1][1] * 4 + 6 + 4) < (header[1]): skip_next_count = True skip_count = header[1] - 6 if skip_next_count: next_count += skip_count else: next_count += (total_field_count * 2 * 2) + 6 if (next_count + 6) >= header[1]: do = False elif header[ 0] == outself.FLOWSET_ID_V9_DATA_TEMPLATE or header[ 0] == outself.FLOWSET_ID_V10_DATA_TEMPLATE: ''' do the task of templates here ''' #log.debug("Inside ; this is template set") self.template_flag = True if header[1] > 4: do = True next_count = 0 while do: template_id = socket.ntohs( struct.unpack( 'H', each_data[4 + next_count:6 + next_count])[0]) field_count = socket.ntohs( struct.unpack( 'H', each_data[6 + next_count:8 + next_count])[0]) present_flag = False for i in xrange( len(self.template_template_list)): # replace the template if template_id is # repeated if template_id == self.template_template_list[ i][0]: present_flag = True self.template_template_list[i] = ( template_id, field_count, each_data[4:]) break if not present_flag: self.template_template_list.append( (template_id, field_count, each_data[4:])) # removing flowset_id and length bytes self.template_data_dict[ template_id] = each_data[4:] #self.template_template_list.append((2344, 4567, "\x00\x12\00")) #log.debug("template_id: %s and field_count: %s", template_id, field_count) skip_next_count = False new_next_count = 0 if (self.template_template_list[-1][1] * 4 + 4 + 4) > (header[1]): log.info( "Invalid field count vs template data length; header_1: %s: temp_lsit:%s", header[1], self.template_template_list[-1]) del self.template_template_list[-1] break elif (self.template_template_list[-1][1] * 4 + 4 + 4) < (header[1]): skip_next_count = True skip_count = header[1] - 4 if skip_next_count: next_count += skip_count else: next_count += (field_count * 2 * 2) + 4 if (next_count + 4) >= header[1]: do = False else: ''' do the task of data here ''' template_id = header[0] data_length = socket.ntohs( struct.unpack('H', each_data[2:4])[0]) self.data_header_list.append( (template_id, data_length, each_data[4:])) #log.debug("Data header list is : %s", self.data_header_list) if self.template_flag: self._handle_template(config, msg_dict, option_template=False) if self.option_template_flag: self._handle_template(config, msg_dict, option_template=True)
def template_data_analyzer(self, sid, netflow_out, device_name, col_type, device_ip, collected_at): ''' template_template_list = (300, 18, 'XXX') template_data_dict = {300: 'XXXX'} ''' #log.debug("Analyzing whether it is template or not and setting flag") set_id = 0 if self.version == 10: set_id = 2 for header in self.template_header_list: each_data = header[2] if header[0] == set_id: ''' do the task of templates here ''' #log.debug("Inside ; this is template set") self.template_flag = True if header[1] > 4: do = True next_count = 0 while do: template_id = socket.ntohs( struct.unpack( 'H', each_data[4 + next_count:6 + next_count])[0]) field_count = socket.ntohs( struct.unpack( 'H', each_data[6 + next_count:8 + next_count])[0]) present_flag = False for i in xrange(len(self.template_template_list)): if template_id == self.template_template_list[i][ 0]: #replace the template if template_id is repeated present_flag = True self.template_template_list[i] = ( template_id, field_count, each_data[4:]) break if not present_flag: self.template_template_list.append( (template_id, field_count, each_data[4:])) self.template_data_dict[template_id] = each_data[ 4:] #removing flowset_id and length bytes #self.template_template_list.append((2344, 4567, "\x00\x12\00")) #log.debug("template_id: %s and field_count: %s", template_id, field_count) if (self.template_template_list[-1][1] * 4 + 4 + 4) != (header[1]): log.info( "Invalid field count vs template data length; header_1: %s: temp_lsit:%s", header[1], self.template_template_list[-1]) del self.template_template_list[-1] break next_count += (field_count * 2 * 2) + 4 if (next_count + 4) >= header[1]: do = False else: ''' do the task of data here ''' template_id = header[0] data_length = socket.ntohs( struct.unpack('H', each_data[2:4])[0]) self.data_header_list.append( (template_id, data_length, each_data[4:])) if self.template_flag: self._handle_template(sid, netflow_out, device_name, col_type, device_ip, collected_at)
def work(id, que, config, netflow_out, col_type, collected_at, expire_time, start, netflow1, netflow5, netflow6, netflow7, benchmark_file): global counter while True: #not que.qsize() == 0: log.warn("speed: %s %s" % (time.time() - start, counter)) benchmark_file.write("%d task, total time: %s, counter: %s\n" % (id, time.time() - start, counter)) #log.warn("%d task:" % id) counter += 1 #data, addr = sock.recvfrom(9216) addr = ('::ffff:192.168.2.4', 62826, 0, 0) data = que.get() if not data: break #continue #log.debug('udp collector; from ip=%s, got msg=%s;', addr, data) ip = inet.get_ip(addr) config_ip = config_reader.get_config_ip(ip, config) if not config_ip: continue sid = _get_sid(config_ip, config) device_name = config['client_map'][config_ip]["device_name"] try: version = get_netflow_packet_version(data[0:2]) count = socket.ntohs(struct.unpack('H', data[2:4])[0]) current_unix_sec = socket.ntohl(struct.unpack('I', data[8:12])[0]) global VERSION global netflowdata if ((version == 1) or (version == 5) or (version == 6) or (version == 7)): if version == 1: log.info("version 1 unpacking...") VERSION = 1 netflow1.unpack(data) netflowdata = netflow1.data elif version == 5: log.info("version 5 unpacking...") VERSION = 5 netflow5.unpack(data) netflowdata = netflow5.data elif version == 6: log.info("version 6 unpacking...") VERSION = 6 netflow6.unpack(data) netflowdata = netflow6.data elif version == 7: log.info("version 7 unpacking...") VERSION = 7 netflow7.unpack(data) netflowdata = netflow7.data if not netflowdata: continue for netflow_record in netflowdata: try: try: parsed_msg_dict = parse_record(netflow_record) except Exception, e: log.error("Could not parse the given record. %s", repr(e)) parsed_msg_dict['_p__raw_msg_b'] = binascii.b2a_base64( str(netflow_record)) parsed_msg_dict['version'] = VERSION parsed_msg_dict['current_unix_sec'] = current_unix_sec msgfilling.add_types(parsed_msg_dict, '_type_num', 'version current_unix_sec') _handle_data(parsed_msg_dict, sid, netflow_out, device_name, col_type, ip, collected_at) except Exception, e: log.error( "Error in constructing message, Necessary field not supplied in Netflow" ) log.error(repr(e))
except Exception, err: new_netflow_v9[device_ip] = _New_Netflow_v9(data, count) log.warn("Cannot create the template and data lists and dict : %s", repr(err)) #traceback.print_exc() return "Warning" log.debug("***************After after updating lists********************") new_netflow_v9[device_ip].update_template_data_list_dict() #log.debug("Final template_template_list: %s", new_netflow_v9[device_ip].template_template_list) #log.debug("Final total data_header_list: %s", new_netflow_v9[device_ip].data_header_list) flow_data = data[SIZE_OF_HEADER_9:] #flowset header flowset_id = socket.ntohs(struct.unpack('H', flow_data[0:2])[0]) length1 = socket.ntohs(struct.unpack('H', flow_data[2:4])[0]) ''' template_dict = list of (key,val) ==> {template_id:[(field_type, length)]} ''' template_template_list = new_netflow_v9[device_ip].template_template_list template_data_dict = new_netflow_v9[device_ip].template_data_dict data_header_list = new_netflow_v9[device_ip].data_header_list data_data_dict = new_netflow_v9[device_ip].data_data_dict temp_data_header_list = data_header_list #log.debug("previous_data :%s :%s\n ", (data_header_list), (previous_data_header_list)) for item in previous_data_header_list: if item in data_header_list:
def get_raw_template_dict_raw_data_dict(self, sid, netflow_out, device_name, col_type, device_ip, collected_at): log.debug( ".. inside separate the data and template flowsets and store with template_id in the ;;tempalate_data_dict ---> raw template flowset ;;data_data_dict --------> raw data flowset ..." ) for header_num in range(0, len(self.template_header_list)): if self.template_header_list[header_num][0] == 0: ''' do the task of templates here ''' #log.debug("Inside ; this is template set") if self.template_header_list[header_num][1] > 4: #log.debug("Template Diagnonsis; template length: %s", self.template_header_list[header_num][1]) do = True next_count = 0 while do: template_id = socket.ntohs( struct.unpack( 'H', self.flow_data_list[header_num] [4 + next_count:6 + next_count])[0]) field_count = socket.ntohs( struct.unpack( 'H', self.flow_data_list[header_num] [6 + next_count:8 + next_count])[0]) for i in range(len(self.template_template_list)): if template_id == self.template_template_list[i][ 0]: #replace the template if template_id is repeated self.template_template_list[i] = (template_id, field_count) self.template_template_list.append( (template_id, field_count)) self.template_data_dict[ template_id] = self.flow_data_list[header_num][ 4:] #removing flowset_id and length #self.template_template_list.append((2344, 4567)) #self.template_data_dict[2344] = "\x00\x12\00" if (self.template_template_list[-1][1] * 4 + 4 + 4 ) != (self.template_header_list[header_num][1]): log.debug( "Here is true error; Invalid field count vs template data length" ) del self.template_template_list[-1] #del self.template_data_dict[2344] break ''' log.debug("inside template_id: %s", template_id) log.debug("inside field_count: %s", field_count) log.debug("inside template_template_list: %s", self.template_template_list) log.debug("inside template_data_dict: %s", self.template_data_dict) ''' next_count += (field_count * 2 * 2) + 4 if (next_count + 4) >= self.template_header_list[header_num][1]: do = False else: log.info( "Length Problem;;;Deleteing template header with incorrect lengths: %s", self.template_header_list[header_num]) #del self.template_header_list[header_num] else: ''' do the task of data here ''' #log.debug("Inside;; this is data set") template_id = self.template_header_list[header_num][0] data_length = socket.ntohs( struct.unpack('H', self.flow_data_list[header_num][2:4])[0]) replaced = False self.data_header_list.append( (template_id, data_length, self.flow_data_list[header_num][4:])) ''' log.debug("inside data template_id: %s", template_id) log.debug("inside data dataflow length: %s", data_length) log.debug("inside self.data_header_list: %s", self.data_header_list) log.debug("inside self.data_data_dict: %s", self.data_data_dict) ''' #handle template data self.update_template_data_list_dict() for (k, v) in self.template_data_dict.iteritems(): _raw_msg_b = binascii.b2a_base64(self.template_data_dict[k]) event = dict( _p___raw_msg_b=_raw_msg_b, packet_type="template", template_id=k, ) msgfilling.add_types(event, '_type_str', 'packet_type') msgfilling.add_types(event, '_type_num', 'template_id') _handle_data(event, sid, netflow_out, device_name, col_type, device_ip, collected_at)
class TestNewNetflowv9(unittest.TestCase): sample = sample_v9 count = socket.ntohs(struct.unpack('H', sample_v9[2:4])[0]) def test_init(self): new_netflow = _New_Netflow_v9(self.sample, self.count) eq_(new_netflow.data, self.sample) eq_(new_netflow.count, self.count) eq_(new_netflow._raw_data, '') eq_(new_netflow.length, list()) eq_(new_netflow.flowset_id, list()) #lsit of flowset id eq_(new_netflow.template_header_list, list()) #contains flowset_id and length eq_(new_netflow.flow_data_list, list()) #contains raw data eq_(new_netflow.template_template_list, list()) #contains template_id and field count eq_(new_netflow.data_header_list, list()) #contains template_id and length of the data eq_(new_netflow.template_data_dict, dict()) #contains templateflow data as template_id:data eq_(new_netflow.data_data_dict, list()) #contains dataflow data as template_id:data eq_(new_netflow.unparsed_raw_data, list()) eq_(new_netflow.now, int(time.time())) assert_not_equal(new_netflow.data, sample_v5) assert_not_equal(new_netflow.count, 1) assert_not_equal(new_netflow.unparsed_raw_data, sample_v9) assert_not_equal(new_netflow.now, int(time.time()) - 1) def test_get_flowset_id_length_data(self): new_netflow = _New_Netflow_v9(self.sample, self.count) eq_(new_netflow.get_flowset_id_length_data(self.sample, self.count), None, "test get flowset id") eq_(new_netflow.length, [[80, 56]]) eq_(new_netflow.flowset_id, [0, 300], "test get flowset id") eq_(new_netflow.template_header_list, [(0, 80), (300, 56)]) eq_(new_netflow.flow_data_list, [ '\x00\x00\x00P\x01,\x00\x12\x00\x08\x00\x04\x00\x0c\x00\x04\x00\x0f\x00\x04\x00\n\x00\x04\x00\x0e\x00\x04\x00\x02\x00\x04\x00\x01\x00\x04\x00\x07\x00\x02\x00\x0b\x00\x02\x00\x06\x00\x01\x00\x04\x00\x01\x00\x05\x00\x01\x00\x11\x00\x02\x00\x10\x00\x02\x00\t\x00\x01\x00\r\x00\x01\x00\x15\x00\x04\x00\x16\x00\x04', '\x01,\x008\n\x00\x00\x02\n\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x05\x00\x00\x00\x01\x00\x00\x00@\x10\x92\x00P\x00\x11\x01\x00\x03\x00\x02 \x1f\x01\xbc\x90G\x01\xbb\xa5\xe7\x00\x00\x00' ]) assert_not_equal(new_netflow.length, list()) assert_not_equal(new_netflow.flowset_id, list()) def test_get_raw_template_dict_raw_data_dict(self): new_netflow = _New_Netflow_v9(self.sample, self.count) eq_(new_netflow.get_flowset_id_length_data(self.sample, self.count), None, "test get flowset id") eq_( new_netflow.get_raw_template_dict_raw_data_dict( sid, netflow_out, device_name, col_type, ip, collected_at), None, "get_raw_data") eq_(new_netflow.template_template_list, [(300, 18)]) expected_data_header_list = [( 300, 56, '\n\x00\x00\x02\n\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x05\x00\x00\x00\x01\x00\x00\x00@\x10\x92\x00P\x00\x11\x01\x00\x03\x00\x02 \x1f\x01\xbc\x90G\x01\xbb\xa5\xe7\x00\x00\x00' )] eq_(new_netflow.data_header_list, expected_data_header_list) assert_not_equal(new_netflow.template_template_list, dict()) assert_not_equal(new_netflow.template_template_list, list()) assert_not_equal(new_netflow.template_template_list, list((300, 0))) def test_update_template_data_list_dict(self): new_netflow = _New_Netflow_v9(self.sample, self.count) eq_(new_netflow.get_flowset_id_length_data(self.sample, self.count), None, "test get flowset id") eq_(new_netflow.update_template_data_list_dict(), None, "test update template data list") eq_(new_netflow.length, [[80, 56]]) eq_(new_netflow.flowset_id, [0, 300]) eq_(new_netflow.template_header_list, [(0, 80), (300, 56)]) eq_(new_netflow.flow_data_list, [ '\x00\x00\x00P\x01,\x00\x12\x00\x08\x00\x04\x00\x0c\x00\x04\x00\x0f\x00\x04\x00\n\x00\x04\x00\x0e\x00\x04\x00\x02\x00\x04\x00\x01\x00\x04\x00\x07\x00\x02\x00\x0b\x00\x02\x00\x06\x00\x01\x00\x04\x00\x01\x00\x05\x00\x01\x00\x11\x00\x02\x00\x10\x00\x02\x00\t\x00\x01\x00\r\x00\x01\x00\x15\x00\x04\x00\x16\x00\x04', '\x01,\x008\n\x00\x00\x02\n\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x05\x00\x00\x00\x01\x00\x00\x00@\x10\x92\x00P\x00\x11\x01\x00\x03\x00\x02 \x1f\x01\xbc\x90G\x01\xbb\xa5\xe7\x00\x00\x00' ]) assert_not_equal(new_netflow.length, list()) assert_not_equal(new_netflow.flowset_id, list()) def test_clean_unparsed_raw_data(self): new_netflow = _New_Netflow_v9(self.sample, self.count) eq_(new_netflow.get_flowset_id_length_data(self.sample, self.count), None, "test get flowset id") eq_( new_netflow.get_raw_template_dict_raw_data_dict( sid, netflow_out, device_name, col_type, ip, collected_at), None, "test update template data list") eq_(new_netflow.update_template_data_list_dict(), None, "test update template data list") eq_(new_netflow.unparsed_raw_data, list()) assert_not_equal(new_netflow.flowset_id, list(self.sample)) assert_not_equal(new_netflow.flowset_id, dict())
def main(): config = _parse_args() log_level = config['core']['log_level'] port = config['port'] expire_time = config['expire_time'] col_type = config['col_type'] collected_at = config["loginspect_name"] zmq_context = zmq.Context() netflow_out = wiring.Wire('collector_out', zmq_context=zmq_context, conf_path=config.get('wiring_conf_path') or None) sock = start_udp_server(port) while True: data, addr = sock.recvfrom(9216) if not data: continue log.debug('udp collector; from ip=%s, got msg=%s;', addr, data) ip = inet.get_ip(addr) config_ip = config_reader.get_config_ip(ip, config) if not config_ip: continue try: version = get_netflow_packet_version(data[0:2]) count = socket.ntohs(struct.unpack('H', data[2:4])[0]) current_unix_sec = (struct.unpack('I', data[8:12])[0]) log.debug("Version: %s", version) log.debug("Count of no. of records: %s", count) log.debug("Count of no. of seconds since 0000 UTC 1970: %s", current_unix_sec) netflow1 = netflow.Netflow1() netflow5 = netflow.Netflow5() netflow6 = netflow.Netflow6() netflow7 = netflow.Netflow7() global VERSION global netflowdata if ((version == 1) or (version == 5) or (version == 6) or (version == 7)): if version == 1: log.info("version 1 unpacking...") VERSION = 1 netflow1.unpack(data) netflowdata = netflow1.data elif version == 5: log.info("version 5 unpacking...") VERSION = 5 netflow5.unpack(data) netflowdata = netflow5.data elif version == 6: log.info("version 6 unpacking...") VERSION = 6 netflow6.unpack(data) netflowdata = netflow6.data elif version == 7: log.info("version 7 unpacking...") VERSION = 7 netflow7.unpack(data) netflowdata = netflow7.data i = 1 if not netflowdata: continue for netflow_record in netflowdata: try: i = i + 1 try: parsed_msg_dict = parse_record(netflow_record) except Exception, e: log.error("Could not parse the given record. %s", repr(e)) parsed_msg_dict['_p__raw_msg_b'] = binascii.b2a_base64( str(netflow_record)) parsed_msg_dict['version'] = VERSION parsed_msg_dict['current_unix_sec'] = current_unix_sec msgfilling.add_types(parsed_msg_dict, '_type_num', 'version current_unix_sec') sid = _get_sid(config_ip, config) device_name = config['client_map'][config_ip][ "device_name"] log.debug("device: %s", device_name) log.debug("descrete ip: %s", ip) try: _handle_data(parsed_msg_dict, sid, netflow_out, device_name, col_type, ip, collected_at) except Exception, e: log.error("Device name not found. %s", repr(e)) except Exception, e: log.error( "Error in constructing message, Necessary field not supplied in Netflow" ) log.error(repr(e))