def parse_template_data(version, ip): TEMPLATE = TEMPLATE_CACHE DATA = DATA_CACHE if version == 10: TEMPLATE = IPFIX_TEMPLATE_CACHE DATA = IPFIX_DATA_CACHE template_info = TEMPLATE.get(ip) data_info = DATA.get(ip) #log.debug("temlate : %s and data: %s", template_info, data_info) if not template_info: log.info("No any template flowset found till date.") elif not data_info: log.info("No any data flowset found till date.") else: data_dict = [] for template_id, temp_fields in template_info.iteritems(): field_cnt = len(temp_fields) data_record_list = data_info.get(template_id) if not data_record_list: log.info("Corresponding data for template_id:%s not found.", template_id) continue for record in data_record_list: record_len, single_data = record field_dict = {} #dict of field values i = 0 for field in xrange(field_cnt): field_type_temp, temp_len = temp_fields[field] start = i end = start + temp_len i = i + temp_len if temp_len == 1: val = ord(single_data[start]) elif temp_len == 2: val = socket.ntohs( struct.unpack('H', single_data[start:end])[0]) elif temp_len == 4: val = socket.ntohl( struct.unpack('I', single_data[start:end])[0]) elif temp_len == 16: val = socket.inet_ntop(socket.AF_INET6, single_data[start:end]) field_dict[field_type_temp] = val _raw_data = single_data data_dict.append(({template_id: field_dict}, _raw_data)) #log.debug("netflow_v9; data dict: %s", data_dict) #log.debug("netflow_v9; length data dict: %s", len(data_dict)) if template_id in data_info: del DATA[ip][template_id] #log.debug("second : %s", DATA) return data_dict
def handle_socket(sock, addr): # logger.debug("handle_socket serve sock %s and addr %s" % (str(sock), str(addr))) peer_id_string = "" peer_id = MPTN_MAX_ID try: peer_id_string = sock.recv(MPTN_ID_LEN) if peer_id_string == "": sock.close() peer_id = socket.ntohl(struct.unpack("!L", peer_id_string)[0]) if peer_id != MPTN_MAX_ID: ConnectionManager.init().add_peer( addr, Peer(socket=sock, id=peer_id, address=addr)) except Exception as e: logger.error( "handle_socket peer_id_string:%s peer_id:%s error=%s\n%s" % (str(e), peer_id_string, str( ID_TO_STRING(peer_id)), traceback.format_exc())) socket_recv(sock, addr, peer_id)
def get_data128(raw_data): global data data = raw_data skip_bytes(data, 16) return socket.ntohl(struct.unpack('I', raw_data[12:16])[0])
def peek_data128(raw_data): return socket.ntohl(struct.unpack('I', raw_data[12:16])[0])
def get_data64(raw_data): global data data = raw_data[8:] return socket.ntohl(struct.unpack('L', raw_data[0:8])[0])
def get_data32(raw_data): global data data = raw_data[4:] return socket.ntohl(struct.unpack('I', raw_data[0:4])[0])
def peek_data32(raw_data): return socket.ntohl(struct.unpack('I', raw_data[0:4])[0])
) log.error(repr(e)) elif (version == 9): log.info("version 9 unpacking...") VERSION = 9 try: ''' get the v9 data in dict ''' #str_data = str(data) #log.debug("v9 str data: %s", str(str_data)) #Other information in packet header sys_uptime_ms = socket.ntohl( struct.unpack('I', data[4:8])[0]) unix_secs = socket.ntohl(struct.unpack('I', data[8:12])[0]) package_sequence = socket.ntohl( struct.unpack('I', data[12:16])[0]) source_id = socket.ntohl( struct.unpack('I', data[16:20])[0]) msg_dict = {} msg_dict['version'] = VERSION msg_dict['sys_uptime_ms'] = sys_uptime_ms msg_dict['unix_secs'] = unix_secs msg_dict['package_sequence'] = package_sequence msg_dict['source_id'] = source_id msgfilling.add_types( msg_dict, '_type_num', 'version sys_uptime_ms unix_secs package_sequence source_id'
def get_data128(self): raw_data = self.data self.skip_bytes(16) return socket.ntohl(struct.unpack('I',raw_data[12:16])[0])
def peek_data128(self): return socket.ntohl(struct.unpack('I',self.data[12:16])[0])
def get_data64(self): raw_data = self.data self.data = raw_data[8:] return socket.ntohl(struct.unpack('L',raw_data[0:8])[0])
def get_data32(self): raw_data = self.data self.data = raw_data[4:] return socket.ntohl(struct.unpack('I',raw_data[0:4])[0])
def peek_data32(self): return socket.ntohl(struct.unpack('I',self.data[0:4])[0])
def work(id, que, config, netflow_out, col_type, collected_at, expire_time, start, netflow1, netflow5, netflow6, netflow7, benchmark_file): global counter while True: #not que.qsize() == 0: log.warn("speed: %s %s" % (time.time() - start, counter)) benchmark_file.write("%d task, total time: %s, counter: %s\n" % (id, time.time() - start, counter)) #log.warn("%d task:" % id) counter += 1 #data, addr = sock.recvfrom(9216) addr = ('::ffff:192.168.2.4', 62826, 0, 0) data = que.get() if not data: break #continue #log.debug('udp collector; from ip=%s, got msg=%s;', addr, data) ip = inet.get_ip(addr) config_ip = config_reader.get_config_ip(ip, config) if not config_ip: continue sid = _get_sid(config_ip, config) device_name = config['client_map'][config_ip]["device_name"] try: version = get_netflow_packet_version(data[0:2]) count = socket.ntohs(struct.unpack('H', data[2:4])[0]) current_unix_sec = socket.ntohl(struct.unpack('I', data[8:12])[0]) global VERSION global netflowdata if ((version == 1) or (version == 5) or (version == 6) or (version == 7)): if version == 1: log.info("version 1 unpacking...") VERSION = 1 netflow1.unpack(data) netflowdata = netflow1.data elif version == 5: log.info("version 5 unpacking...") VERSION = 5 netflow5.unpack(data) netflowdata = netflow5.data elif version == 6: log.info("version 6 unpacking...") VERSION = 6 netflow6.unpack(data) netflowdata = netflow6.data elif version == 7: log.info("version 7 unpacking...") VERSION = 7 netflow7.unpack(data) netflowdata = netflow7.data if not netflowdata: continue for netflow_record in netflowdata: try: try: parsed_msg_dict = parse_record(netflow_record) except Exception, e: log.error("Could not parse the given record. %s", repr(e)) parsed_msg_dict['_p__raw_msg_b'] = binascii.b2a_base64( str(netflow_record)) parsed_msg_dict['version'] = VERSION parsed_msg_dict['current_unix_sec'] = current_unix_sec msgfilling.add_types(parsed_msg_dict, '_type_num', 'version current_unix_sec') _handle_data(parsed_msg_dict, sid, netflow_out, device_name, col_type, ip, collected_at) except Exception, e: log.error( "Error in constructing message, Necessary field not supplied in Netflow" ) log.error(repr(e))
def socket_recv(sock, addr, peer_id): while True: nonce = "" size_string = "" size = 0 message = "" try: nonce = sock.recv(MPTN_TCP_NONCE_SIZE) if nonce == "": logger.error("handle_socket closed. nonce. addr=%s" % (str(addr))) logger.debug("handle_socket closed. connections before %s" % str(ConnectionManager.init())) ConnectionManager.init().remove_peer(addr) logger.debug("handle_socket closed. connections after %s" % str(ConnectionManager.init())) return size_string = sock.recv(MPTN_TCP_PACKET_SIZE) if size_string == "": logger.error("handle_socket closed. size. addr=%s, nonce=%s" % (str(addr), str(map(ord, nonce)))) logger.debug("handle_socket closed. connections before %s" % str(ConnectionManager.init())) ConnectionManager.init().remove_peer(addr) logger.debug("handle_socket closed. connections after %s" % str(ConnectionManager.init())) return size = socket.ntohl(struct.unpack("!L", size_string)[0]) while len(message) < size: part_msg = sock.recv(size - len(message)) if part_msg == "": logger.error( "handle_socket closed. message. addr=%s, nonce=%s, size_string=%s, size=%d, message=\n%s" % (str(addr), str(map( ord, nonce)), str(map(ord, size_string)), size, str(formatted_print(split_packet_to_list(message))))) logger.debug( "handle_socket closed. connections before %s" % str(ConnectionManager.init())) ConnectionManager.init().remove_peer(addr) logger.debug("handle_socket closed. connections after %s" % str(ConnectionManager.init())) return message += part_msg # logger.debug("handle_socket receive message from addr %s" % str(addr)) context = Context(peer_id, addr, ONLY_FROM_TCP_SERVER, sock, nonce) process_message_handler(context, message) # process_message_handler modify context.id if peer_id == MPTN_MAX_ID and context.id != MPTN_MAX_ID: ConnectionManager.init().add_peer( addr, Peer(socket=sock, id=context.id, address=addr)) peer_id = context.id logger.debug( "handle_socket update context id %s 0x%X from addr %s" % (ID_TO_STRING(context.id), context.id, str(addr))) except Exception as e: logger.error( "handle_socket addr=%s, nonce=%s, size_string=%s, size=%d, message is \n%s\nerror=%s\n%s" % (str(addr), str(map(ord, nonce)), str(map(ord, size_string)), size, str(formatted_print(split_packet_to_list(message))), str(e), traceback.format_exc())) logger.debug("handle_socket error before %s" % str(ConnectionManager.init())) ConnectionManager.init().remove_peer(addr) logger.debug("handle_socket error after %s" % str(ConnectionManager.init())) return gevent.sleep(0)
def ip_to_int(self, ip): return struct.unpack( 'I', struct.pack('I', (socket.ntohl( struct.unpack('I', socket.inet_aton(ip))[0]))))[0]
def handle_udp_data(self, data, **config): """ """ try: self.VERSION = self.get_netflow_packet_version(data[0:2]) count = socket.ntohs(struct.unpack('H', data[2:4])[0]) current_unix_sec = socket.ntohl(struct.unpack('I', data[8:12])[0]) if ((self.VERSION == 1) or (self.VERSION == 5) or (self.VERSION == 6) or (self.VERSION == 7)): if self.VERSION == 1: log.info("version 1 unpacking...") self.netflow1.unpack(data) self.netflowdata = self.netflow1.data elif self.VERSION == 5: log.info("version 5 unpacking...") self.netflow5.unpack(data) self.netflowdata = self.netflow5.data elif self.VERSION == 6: log.info("version 6 unpacking...") self.netflow6.unpack(data) self.netflowdata = self.netflow6.data elif self.VERSION == 7: log.info("version 7 unpacking...") self.netflow7.unpack(data) self.netflowdata = self.netflow7.data for netflow_record in self.netflowdata: try: try: parsed_msg_dict = self.parse_record(netflow_record) except Exception as e: log.error("Could not parse the given record. %s", repr(e)) parsed_msg_dict["_p__raw_msg_b"] = binascii.b2a_base64( str(netflow_record)) self.prepare_event(parsed_msg_dict, "version", self.VERSION, TYPE_NUM) self.prepare_event(parsed_msg_dict, "current_unix_sec", current_unix_sec, TYPE_NUM) self.handle_data(parsed_msg_dict, config) except Exception as e: log.error( "Error in constructing message, Necessary field not supplied in Netflow" ) log.error(repr(e)) # log.warn(traceback.print_exc()) elif (self.VERSION == 9): log.info("version 9 unpacking...") try: ''' get the v9 data in dict ''' #str_data = str(data) #log.debug("v9 str data: %s", str(str_data)) # Other information in packet header sys_uptime_ms = socket.ntohl( struct.unpack('I', data[4:8])[0]) unix_secs = socket.ntohl(struct.unpack('I', data[8:12])[0]) package_sequence = socket.ntohl( struct.unpack('I', data[12:16])[0]) source_id = socket.ntohl( struct.unpack('I', data[16:20])[0]) msg_dict = {} msg_dict['version'] = self.VERSION msg_dict['sys_uptime_ms'] = sys_uptime_ms msg_dict['unix_secs'] = unix_secs msg_dict['package_sequence'] = package_sequence msg_dict['source_id'] = source_id msg_dict['flowset_count'] = count netflow_data_list_tuple = self.netflow_parser( self.VERSION, data, count, self.expire_time, config, msg_dict) #log.debug("this is the returned aaaaaaa netflow_tuple: %s", netflow_data_list_tuple) if netflow_data_list_tuple is not None and len( netflow_data_list_tuple) != 0: for data_dict, _p__raw_msg_b in netflow_data_list_tuple: #log.debug("Testing data dict: %s", data_dict) # parse record for (k, v) in data_dict.iteritems(): try: parsed_msg_dict = self.msgfill_parsed_record( v) except Exception as e: log.error( "Could not msgfill the parsed v9 record; %s", repr(e)) parsed_msg_dict[ "_p__raw_msg_b"] = binascii.b2a_base64( str(_p__raw_msg_b)) parsed_msg_dict['template_id'] = k #self.prepare_event(parsed_msg_dict, "template_id", k, TYPE_NUM) parsed_msg_dict.update(msg_dict) self.prepare_msgfilling( parsed_msg_dict, TYPE_NUM, 'template_id version sys_uptime_ms package_sequence source_id unix_secs flowset_count' ) self.handle_data(parsed_msg_dict, config) except Exception as e: parsed_msg_dict = {} parsed_msg_dict["_p__raw_msg_b"] = binascii.b2a_base64( str(data)) parsed_msg_dict['version'] = self.VERSION self.prepare_msgfilling(parsed_msg_dict, TYPE_NUM, 'version') self.handle_data(parsed_msg_dict, config) log.warn( "Error in constructing v9 message, Necessary field not supplied in Netflow; %s", repr(e)) # log.warn(traceback.print_exc()) elif (self.VERSION == 10): log.info("version 10 unpacking...") try: ''' get the ipfix data in dict ''' # Other information in packet header sys_uptime_ms = socket.ntohl( struct.unpack('I', data[4:8])[0]) package_sequence = socket.ntohl( struct.unpack('I', data[8:12])[0]) source_id = socket.ntohl( struct.unpack('I', data[12:16])[0]) msg_dict = {} msg_dict['version'] = self.VERSION msg_dict['sys_uptime_ms'] = sys_uptime_ms msg_dict['package_sequence'] = package_sequence msg_dict['source_id'] = source_id msg_dict['flowset_length'] = count netflow_data_list_tuple = self.netflow_parser( self.VERSION, data, count, self.expire_time, config, msg_dict) if netflow_data_list_tuple is not None and len( netflow_data_list_tuple) != 0: for data_dict, _p__raw_msg_b in netflow_data_list_tuple: #log.debug("Testing data dict: %s", data_dict) # parse record for (k, v) in data_dict.iteritems(): try: parsed_msg_dict = self.msgfill_parsed_record( v) except Exception as e: log.error( "Could not msgfill the parsed ipfix record; %s", repr(e)) parsed_msg_dict[ '_p__raw_msg_b'] = binascii.b2a_base64( str(_p__raw_msg_b)) parsed_msg_dict['template_id'] = k parsed_msg_dict.update(msg_dict) self.prepare_msgfilling( parsed_msg_dict, TYPE_NUM, 'template_id version flowset_length sys_uptime_ms package_sequence source_id' ) self.handle_data(parsed_msg_dict, config) except Exception as e: parsed_msg_dict = {} parsed_msg_dict['_p__raw_msg_b'] = binascii.b2a_base64( str(data)) parsed_msg_dict['version'] = self.VERSION self.prepare_msgfilling(parsed_msg_dict, TYPE_NUM, 'version') self.handle_data(parsed_msg_dict, config) log.warn( "Error in constructing IPFIX message, Necessary field not supplied; %s", repr(e)) # log.warn(traceback.print_exc()) else: log.error("Not the correct version type.") except Exception as e: log.error("Incorrect Netflow data format, %s", repr(e))
def main(): config = _parse_args() port = config['port'] expire_time = config['expire_time'] col_type = config['col_type'] zmq_context = zmq.Context() netflow_out = wiring.Wire('collector_out', zmq_context=zmq_context, conf_path=config.get('wiring_conf_path') or None) sock = start_udp_server(port) netflow1 = netflow.Netflow1() netflow5 = netflow.Netflow5() netflow6 = netflow.Netflow6() netflow7 = netflow.Netflow7() while True: collected_at = config["loginspect_name"] data, addr = sock.recvfrom(9216) if not data: continue #log.debug('udp collector; from ip=%s, got msg=%s;', addr, data) ip = inet.get_ip(addr) config_ip = config_reader.get_config_ip(ip, config) if not config_ip: continue sid = _get_sid(config_ip, config) device_name = config['client_map'][config_ip]["device_name"] try: version = get_netflow_packet_version(data[0:2]) count = socket.ntohs(struct.unpack('H', data[2:4])[0]) current_unix_sec = socket.ntohl(struct.unpack('I', data[8:12])[0]) global VERSION global netflowdata if ((version == 1) or (version == 5) or (version == 6) or (version == 7)): if version == 1: log.info("version 1 unpacking...") VERSION = 1 netflow1.unpack(data) netflowdata = netflow1.data elif version == 5: log.info("version 5 unpacking...") VERSION = 5 netflow5.unpack(data) netflowdata = netflow5.data elif version == 6: log.info("version 6 unpacking...") VERSION = 6 netflow6.unpack(data) netflowdata = netflow6.data elif version == 7: log.info("version 7 unpacking...") VERSION = 7 netflow7.unpack(data) netflowdata = netflow7.data if not netflowdata: continue for netflow_record in netflowdata: try: try: parsed_msg_dict = parse_record(netflow_record) except Exception, e: log.error("Could not parse the given record. %s", repr(e)) parsed_msg_dict['_p__raw_msg_b'] = binascii.b2a_base64( str(netflow_record)) parsed_msg_dict['version'] = VERSION parsed_msg_dict['current_unix_sec'] = current_unix_sec msgfilling.add_types(parsed_msg_dict, '_type_num', 'version current_unix_sec') _handle_data(parsed_msg_dict, sid, netflow_out, device_name, col_type, ip, collected_at) except Exception, e: log.error( "Error in constructing message, Necessary field not supplied in Netflow" ) log.error(repr(e))