def parse_template_data(version, ip):
TEMPLATE = TEMPLATE_CACHE
DATA = DATA_CACHE
if version == 10:
TEMPLATE = IPFIX_TEMPLATE_CACHE
DATA = IPFIX_DATA_CACHE
template_info = TEMPLATE.get(ip)
data_info = DATA.get(ip)
#log.debug("temlate : %s and data: %s", template_info, data_info)
if not template_info:
log.info("No any template flowset found till date.")
elif not data_info:
log.info("No any data flowset found till date.")
else:
data_dict = []
for template_id, temp_fields in template_info.iteritems():
field_cnt = len(temp_fields)
data_record_list = data_info.get(template_id)
if not data_record_list:
log.info("Corresponding data for template_id:%s not found.",
template_id)
continue
for record in data_record_list:
record_len, single_data = record
field_dict = {} #dict of field values
i = 0
for field in xrange(field_cnt):
field_type_temp, temp_len = temp_fields[field]
start = i
end = start + temp_len
i = i + temp_len
if temp_len == 1:
val = ord(single_data[start])
elif temp_len == 2:
val = socket.ntohs(
struct.unpack('H', single_data[start:end])[0])
elif temp_len == 4:
val = socket.ntohl(
struct.unpack('I', single_data[start:end])[0])
elif temp_len == 16:
val = socket.inet_ntop(socket.AF_INET6,
single_data[start:end])
field_dict[field_type_temp] = val
_raw_data = single_data
data_dict.append(({template_id: field_dict}, _raw_data))
#log.debug("netflow_v9; data dict: %s", data_dict)
#log.debug("netflow_v9; length data dict: %s", len(data_dict))
if template_id in data_info:
del DATA[ip][template_id]
#log.debug("second : %s", DATA)
return data_dict
def handle_socket(sock, addr):
# logger.debug("handle_socket serve sock %s and addr %s" % (str(sock), str(addr)))
peer_id_string = ""
peer_id = MPTN_MAX_ID
try:
peer_id_string = sock.recv(MPTN_ID_LEN)
if peer_id_string == "":
sock.close()
peer_id = socket.ntohl(struct.unpack("!L", peer_id_string)[0])
if peer_id != MPTN_MAX_ID:
ConnectionManager.init().add_peer(
addr, Peer(socket=sock, id=peer_id, address=addr))
except Exception as e:
logger.error(
"handle_socket peer_id_string:%s peer_id:%s error=%s\n%s" %
(str(e), peer_id_string, str(
ID_TO_STRING(peer_id)), traceback.format_exc()))
socket_recv(sock, addr, peer_id)
def get_data128(raw_data):
global data
data = raw_data
skip_bytes(data, 16)
return socket.ntohl(struct.unpack('I', raw_data[12:16])[0])
def peek_data128(raw_data):
return socket.ntohl(struct.unpack('I', raw_data[12:16])[0])
def get_data64(raw_data):
global data
data = raw_data[8:]
return socket.ntohl(struct.unpack('L', raw_data[0:8])[0])
def get_data32(raw_data):
global data
data = raw_data[4:]
return socket.ntohl(struct.unpack('I', raw_data[0:4])[0])
def peek_data32(raw_data):
return socket.ntohl(struct.unpack('I', raw_data[0:4])[0])
)
log.error(repr(e))
elif (version == 9):
log.info("version 9 unpacking...")
VERSION = 9
try:
'''
get the v9 data in dict
'''
#str_data = str(data)
#log.debug("v9 str data: %s", str(str_data))
#Other information in packet header
sys_uptime_ms = socket.ntohl(
struct.unpack('I', data[4:8])[0])
unix_secs = socket.ntohl(struct.unpack('I', data[8:12])[0])
package_sequence = socket.ntohl(
struct.unpack('I', data[12:16])[0])
source_id = socket.ntohl(
struct.unpack('I', data[16:20])[0])
msg_dict = {}
msg_dict['version'] = VERSION
msg_dict['sys_uptime_ms'] = sys_uptime_ms
msg_dict['unix_secs'] = unix_secs
msg_dict['package_sequence'] = package_sequence
msg_dict['source_id'] = source_id
msgfilling.add_types(
msg_dict, '_type_num',
'version sys_uptime_ms unix_secs package_sequence source_id'
def get_data128(self):
raw_data = self.data
self.skip_bytes(16)
return socket.ntohl(struct.unpack('I',raw_data[12:16])[0])
def peek_data128(self):
return socket.ntohl(struct.unpack('I',self.data[12:16])[0])
def get_data64(self):
raw_data = self.data
self.data = raw_data[8:]
return socket.ntohl(struct.unpack('L',raw_data[0:8])[0])
def get_data32(self):
raw_data = self.data
self.data = raw_data[4:]
return socket.ntohl(struct.unpack('I',raw_data[0:4])[0])
def peek_data32(self):
return socket.ntohl(struct.unpack('I',self.data[0:4])[0])
def work(id, que, config, netflow_out, col_type, collected_at, expire_time,
start, netflow1, netflow5, netflow6, netflow7, benchmark_file):
global counter
while True: #not que.qsize() == 0:
log.warn("speed: %s %s" % (time.time() - start, counter))
benchmark_file.write("%d task, total time: %s, counter: %s\n" %
(id, time.time() - start, counter))
#log.warn("%d task:" % id)
counter += 1
#data, addr = sock.recvfrom(9216)
addr = ('::ffff:192.168.2.4', 62826, 0, 0)
data = que.get()
if not data:
break
#continue
#log.debug('udp collector; from ip=%s, got msg=%s;', addr, data)
ip = inet.get_ip(addr)
config_ip = config_reader.get_config_ip(ip, config)
if not config_ip:
continue
sid = _get_sid(config_ip, config)
device_name = config['client_map'][config_ip]["device_name"]
try:
version = get_netflow_packet_version(data[0:2])
count = socket.ntohs(struct.unpack('H', data[2:4])[0])
current_unix_sec = socket.ntohl(struct.unpack('I', data[8:12])[0])
global VERSION
global netflowdata
if ((version == 1) or (version == 5) or (version == 6)
or (version == 7)):
if version == 1:
log.info("version 1 unpacking...")
VERSION = 1
netflow1.unpack(data)
netflowdata = netflow1.data
elif version == 5:
log.info("version 5 unpacking...")
VERSION = 5
netflow5.unpack(data)
netflowdata = netflow5.data
elif version == 6:
log.info("version 6 unpacking...")
VERSION = 6
netflow6.unpack(data)
netflowdata = netflow6.data
elif version == 7:
log.info("version 7 unpacking...")
VERSION = 7
netflow7.unpack(data)
netflowdata = netflow7.data
if not netflowdata:
continue
for netflow_record in netflowdata:
try:
try:
parsed_msg_dict = parse_record(netflow_record)
except Exception, e:
log.error("Could not parse the given record. %s",
repr(e))
parsed_msg_dict['_p__raw_msg_b'] = binascii.b2a_base64(
str(netflow_record))
parsed_msg_dict['version'] = VERSION
parsed_msg_dict['current_unix_sec'] = current_unix_sec
msgfilling.add_types(parsed_msg_dict, '_type_num',
'version current_unix_sec')
_handle_data(parsed_msg_dict, sid, netflow_out,
device_name, col_type, ip, collected_at)
except Exception, e:
log.error(
"Error in constructing message, Necessary field not supplied in Netflow"
)
log.error(repr(e))
def socket_recv(sock, addr, peer_id):
while True:
nonce = ""
size_string = ""
size = 0
message = ""
try:
nonce = sock.recv(MPTN_TCP_NONCE_SIZE)
if nonce == "":
logger.error("handle_socket closed. nonce. addr=%s" %
(str(addr)))
logger.debug("handle_socket closed. connections before %s" %
str(ConnectionManager.init()))
ConnectionManager.init().remove_peer(addr)
logger.debug("handle_socket closed. connections after %s" %
str(ConnectionManager.init()))
return
size_string = sock.recv(MPTN_TCP_PACKET_SIZE)
if size_string == "":
logger.error("handle_socket closed. size. addr=%s, nonce=%s" %
(str(addr), str(map(ord, nonce))))
logger.debug("handle_socket closed. connections before %s" %
str(ConnectionManager.init()))
ConnectionManager.init().remove_peer(addr)
logger.debug("handle_socket closed. connections after %s" %
str(ConnectionManager.init()))
return
size = socket.ntohl(struct.unpack("!L", size_string)[0])
while len(message) < size:
part_msg = sock.recv(size - len(message))
if part_msg == "":
logger.error(
"handle_socket closed. message. addr=%s, nonce=%s, size_string=%s, size=%d, message=\n%s"
%
(str(addr), str(map(
ord, nonce)), str(map(ord, size_string)), size,
str(formatted_print(split_packet_to_list(message)))))
logger.debug(
"handle_socket closed. connections before %s" %
str(ConnectionManager.init()))
ConnectionManager.init().remove_peer(addr)
logger.debug("handle_socket closed. connections after %s" %
str(ConnectionManager.init()))
return
message += part_msg
# logger.debug("handle_socket receive message from addr %s" % str(addr))
context = Context(peer_id, addr, ONLY_FROM_TCP_SERVER, sock, nonce)
process_message_handler(context, message)
# process_message_handler modify context.id
if peer_id == MPTN_MAX_ID and context.id != MPTN_MAX_ID:
ConnectionManager.init().add_peer(
addr, Peer(socket=sock, id=context.id, address=addr))
peer_id = context.id
logger.debug(
"handle_socket update context id %s 0x%X from addr %s" %
(ID_TO_STRING(context.id), context.id, str(addr)))
except Exception as e:
logger.error(
"handle_socket addr=%s, nonce=%s, size_string=%s, size=%d, message is \n%s\nerror=%s\n%s"
% (str(addr), str(map(ord, nonce)), str(map(ord, size_string)),
size, str(formatted_print(split_packet_to_list(message))),
str(e), traceback.format_exc()))
logger.debug("handle_socket error before %s" %
str(ConnectionManager.init()))
ConnectionManager.init().remove_peer(addr)
logger.debug("handle_socket error after %s" %
str(ConnectionManager.init()))
return
gevent.sleep(0)
def ip_to_int(self, ip):
return struct.unpack(
'I',
struct.pack('I', (socket.ntohl(
struct.unpack('I', socket.inet_aton(ip))[0]))))[0]
def handle_udp_data(self, data, **config):
"""
"""
try:
self.VERSION = self.get_netflow_packet_version(data[0:2])
count = socket.ntohs(struct.unpack('H', data[2:4])[0])
current_unix_sec = socket.ntohl(struct.unpack('I', data[8:12])[0])
if ((self.VERSION == 1) or (self.VERSION == 5)
or (self.VERSION == 6) or (self.VERSION == 7)):
if self.VERSION == 1:
log.info("version 1 unpacking...")
self.netflow1.unpack(data)
self.netflowdata = self.netflow1.data
elif self.VERSION == 5:
log.info("version 5 unpacking...")
self.netflow5.unpack(data)
self.netflowdata = self.netflow5.data
elif self.VERSION == 6:
log.info("version 6 unpacking...")
self.netflow6.unpack(data)
self.netflowdata = self.netflow6.data
elif self.VERSION == 7:
log.info("version 7 unpacking...")
self.netflow7.unpack(data)
self.netflowdata = self.netflow7.data
for netflow_record in self.netflowdata:
try:
try:
parsed_msg_dict = self.parse_record(netflow_record)
except Exception as e:
log.error("Could not parse the given record. %s",
repr(e))
parsed_msg_dict["_p__raw_msg_b"] = binascii.b2a_base64(
str(netflow_record))
self.prepare_event(parsed_msg_dict, "version",
self.VERSION, TYPE_NUM)
self.prepare_event(parsed_msg_dict, "current_unix_sec",
current_unix_sec, TYPE_NUM)
self.handle_data(parsed_msg_dict, config)
except Exception as e:
log.error(
"Error in constructing message, Necessary field not supplied in Netflow"
)
log.error(repr(e))
# log.warn(traceback.print_exc())
elif (self.VERSION == 9):
log.info("version 9 unpacking...")
try:
'''
get the v9 data in dict
'''
#str_data = str(data)
#log.debug("v9 str data: %s", str(str_data))
# Other information in packet header
sys_uptime_ms = socket.ntohl(
struct.unpack('I', data[4:8])[0])
unix_secs = socket.ntohl(struct.unpack('I', data[8:12])[0])
package_sequence = socket.ntohl(
struct.unpack('I', data[12:16])[0])
source_id = socket.ntohl(
struct.unpack('I', data[16:20])[0])
msg_dict = {}
msg_dict['version'] = self.VERSION
msg_dict['sys_uptime_ms'] = sys_uptime_ms
msg_dict['unix_secs'] = unix_secs
msg_dict['package_sequence'] = package_sequence
msg_dict['source_id'] = source_id
msg_dict['flowset_count'] = count
netflow_data_list_tuple = self.netflow_parser(
self.VERSION, data, count, self.expire_time, config,
msg_dict)
#log.debug("this is the returned aaaaaaa netflow_tuple: %s", netflow_data_list_tuple)
if netflow_data_list_tuple is not None and len(
netflow_data_list_tuple) != 0:
for data_dict, _p__raw_msg_b in netflow_data_list_tuple:
#log.debug("Testing data dict: %s", data_dict)
# parse record
for (k, v) in data_dict.iteritems():
try:
parsed_msg_dict = self.msgfill_parsed_record(
v)
except Exception as e:
log.error(
"Could not msgfill the parsed v9 record; %s",
repr(e))
parsed_msg_dict[
"_p__raw_msg_b"] = binascii.b2a_base64(
str(_p__raw_msg_b))
parsed_msg_dict['template_id'] = k
#self.prepare_event(parsed_msg_dict, "template_id", k, TYPE_NUM)
parsed_msg_dict.update(msg_dict)
self.prepare_msgfilling(
parsed_msg_dict, TYPE_NUM,
'template_id version sys_uptime_ms package_sequence source_id unix_secs flowset_count'
)
self.handle_data(parsed_msg_dict, config)
except Exception as e:
parsed_msg_dict = {}
parsed_msg_dict["_p__raw_msg_b"] = binascii.b2a_base64(
str(data))
parsed_msg_dict['version'] = self.VERSION
self.prepare_msgfilling(parsed_msg_dict, TYPE_NUM,
'version')
self.handle_data(parsed_msg_dict, config)
log.warn(
"Error in constructing v9 message, Necessary field not supplied in Netflow; %s",
repr(e))
# log.warn(traceback.print_exc())
elif (self.VERSION == 10):
log.info("version 10 unpacking...")
try:
'''
get the ipfix data in dict
'''
# Other information in packet header
sys_uptime_ms = socket.ntohl(
struct.unpack('I', data[4:8])[0])
package_sequence = socket.ntohl(
struct.unpack('I', data[8:12])[0])
source_id = socket.ntohl(
struct.unpack('I', data[12:16])[0])
msg_dict = {}
msg_dict['version'] = self.VERSION
msg_dict['sys_uptime_ms'] = sys_uptime_ms
msg_dict['package_sequence'] = package_sequence
msg_dict['source_id'] = source_id
msg_dict['flowset_length'] = count
netflow_data_list_tuple = self.netflow_parser(
self.VERSION, data, count, self.expire_time, config,
msg_dict)
if netflow_data_list_tuple is not None and len(
netflow_data_list_tuple) != 0:
for data_dict, _p__raw_msg_b in netflow_data_list_tuple:
#log.debug("Testing data dict: %s", data_dict)
# parse record
for (k, v) in data_dict.iteritems():
try:
parsed_msg_dict = self.msgfill_parsed_record(
v)
except Exception as e:
log.error(
"Could not msgfill the parsed ipfix record; %s",
repr(e))
parsed_msg_dict[
'_p__raw_msg_b'] = binascii.b2a_base64(
str(_p__raw_msg_b))
parsed_msg_dict['template_id'] = k
parsed_msg_dict.update(msg_dict)
self.prepare_msgfilling(
parsed_msg_dict, TYPE_NUM,
'template_id version flowset_length sys_uptime_ms package_sequence source_id'
)
self.handle_data(parsed_msg_dict, config)
except Exception as e:
parsed_msg_dict = {}
parsed_msg_dict['_p__raw_msg_b'] = binascii.b2a_base64(
str(data))
parsed_msg_dict['version'] = self.VERSION
self.prepare_msgfilling(parsed_msg_dict, TYPE_NUM,
'version')
self.handle_data(parsed_msg_dict, config)
log.warn(
"Error in constructing IPFIX message, Necessary field not supplied; %s",
repr(e))
# log.warn(traceback.print_exc())
else:
log.error("Not the correct version type.")
except Exception as e:
log.error("Incorrect Netflow data format, %s", repr(e))
def main():
config = _parse_args()
port = config['port']
expire_time = config['expire_time']
col_type = config['col_type']
zmq_context = zmq.Context()
netflow_out = wiring.Wire('collector_out',
zmq_context=zmq_context,
conf_path=config.get('wiring_conf_path') or None)
sock = start_udp_server(port)
netflow1 = netflow.Netflow1()
netflow5 = netflow.Netflow5()
netflow6 = netflow.Netflow6()
netflow7 = netflow.Netflow7()
while True:
collected_at = config["loginspect_name"]
data, addr = sock.recvfrom(9216)
if not data:
continue
#log.debug('udp collector; from ip=%s, got msg=%s;', addr, data)
ip = inet.get_ip(addr)
config_ip = config_reader.get_config_ip(ip, config)
if not config_ip:
continue
sid = _get_sid(config_ip, config)
device_name = config['client_map'][config_ip]["device_name"]
try:
version = get_netflow_packet_version(data[0:2])
count = socket.ntohs(struct.unpack('H', data[2:4])[0])
current_unix_sec = socket.ntohl(struct.unpack('I', data[8:12])[0])
global VERSION
global netflowdata
if ((version == 1) or (version == 5) or (version == 6)
or (version == 7)):
if version == 1:
log.info("version 1 unpacking...")
VERSION = 1
netflow1.unpack(data)
netflowdata = netflow1.data
elif version == 5:
log.info("version 5 unpacking...")
VERSION = 5
netflow5.unpack(data)
netflowdata = netflow5.data
elif version == 6:
log.info("version 6 unpacking...")
VERSION = 6
netflow6.unpack(data)
netflowdata = netflow6.data
elif version == 7:
log.info("version 7 unpacking...")
VERSION = 7
netflow7.unpack(data)
netflowdata = netflow7.data
if not netflowdata:
continue
for netflow_record in netflowdata:
try:
try:
parsed_msg_dict = parse_record(netflow_record)
except Exception, e:
log.error("Could not parse the given record. %s",
repr(e))
parsed_msg_dict['_p__raw_msg_b'] = binascii.b2a_base64(
str(netflow_record))
parsed_msg_dict['version'] = VERSION
parsed_msg_dict['current_unix_sec'] = current_unix_sec
msgfilling.add_types(parsed_msg_dict, '_type_num',
'version current_unix_sec')
_handle_data(parsed_msg_dict, sid, netflow_out,
device_name, col_type, ip, collected_at)
except Exception, e:
log.error(
"Error in constructing message, Necessary field not supplied in Netflow"
)
log.error(repr(e))
