def _process_file_and_save_json(inputfile):
to_set, filename, sensorname = _get_data_structures(inputfile)
if _RED.sismember("FILES", filename):
return f'Filename {inputfile} was already imported ... skip ...\n'
proc = subprocess.Popen(_CMD.format(inputfile), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
first_packet = {"type": potiron.TYPE_SOURCE, "sensorname": sensorname, "filename": filename}
first_packet.update(_FIRST_PACKET)
allpackets = [first_packet]
lastday = day_from_filename(filename)
_RED.sadd(f"{sensorname}_DAYS", lastday)
packet_id = 0
for line in proc.stdout.readlines():
packet = _create_packet(line)
packet['timestamp'] = _set_json_timestamp(packet['timestamp'])
allpackets.append(_create_json_packet(packet, packet_id))
day, time = packet.pop('timestamp').split(' ')
timestamp = f'{day}_{time}'
day = day.replace('-', '')
if day != lastday:
_RED.sadd(f"{sensorname}_DAYS", day)
lastday = day
ports = "_".join([f"{port}{packet.pop(value)}" for port, value in zip(('src', 'dst'), ('sport', 'dport'))])
key = f"{sensorname}_{ports}_{timestamp}"
to_set[key] = {isn_type: value for isn_type, value in packet.items()}
packet_id += 1
p = _RED.pipeline()
for key, item in to_set.items():
p.hmset(key, item)
p.execute()
proc.wait()
potiron.store_packet(_ROOTDIR, filename, json.dumps(allpackets))
_RED.sadd("FILES", filename)
return f'ISN Data from {filename} parsed and stored in json format.'
def _process_file(inputfile):
to_set, filename, sensorname = _get_data_structures(inputfile)
if _RED.sismember("FILES", filename):
return f'Filename {inputfile} was already imported ... skip ...\n'
proc = subprocess.Popen(_CMD.format(inputfile),
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
lastday = day_from_filename(filename)
_RED.sadd(f"{sensorname}_DAYS", lastday)
for line in proc.stdout.readlines():
packet = _create_packet(line)
timestamp = _set_json_timestamp(packet.pop('timestamp'))
day, time = timestamp.split(' ')
timestamp = f"{day}_{time}"
day = day.replace('-', '')
if day != lastday:
_RED.sadd(f"{sensorname}_DAYS", day)
lastday = day
ports = "_".join([
f"{port}{packet.pop(value)}"
for port, value in zip(('src', 'dst'), ('sport', 'dport'))
])
key = f"{sensorname}_{ports}_{timestamp}"
to_set[key] = {isn_type: value for isn_type, value in packet.items()}
p = _RED.pipeline()
for key, item in to_set.items():
p.hmset(key, item)
p.execute()
proc.wait()
_RED.sadd("FILES", filename)
return f'ISN Data from {filename} parsed.'
def _process_file(inputfile):
to_set, filename, sensorname = _get_data_structures(inputfile)
if _RED.sismember("FILES", filename):
return f'Filename {inputfile} was already imported ... skip ...\n'
proc = subprocess.Popen(_CMD.format(inputfile), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
lastday = day_from_filename(filename)
_RED.sadd(f"{sensorname}_DAYS", lastday)
for line in proc.stdout.readlines():
packet = _create_packet(line)
timestamp = _set_json_timestamp(packet.pop('timestamp'))
day, time = timestamp.split(' ')
timestamp = f"{day}_{time}"
day = day.replace('-', '')
if day != lastday:
_RED.sadd(f"{sensorname}_DAYS", day)
lastday = day
ports = "_".join([f"{port}{packet.pop(value)}" for port, value in zip(('src', 'dst'), ('sport', 'dport'))])
key = f"{sensorname}_{ports}_{timestamp}"
to_set[key] = {isn_type: value for isn_type, value in packet.items()}
p = _RED.pipeline()
for key, item in to_set.items():
p.hmset(key, item)
p.execute()
proc.wait()
_RED.sadd("FILES", filename)
return f'ISN Data from {filename} parsed.'
def _process_file(inputfile):
to_set = {}
to_incr, filename, sensorname = _get_data_structures(inputfile)
if _RED.sismember("FILES", filename):
return f'Filename {inputfile} was already imported ... skip ...\n'
proc = subprocess.Popen(_CMD.format(inputfile),
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
lastday = day_from_filename(filename)
_RED.sadd(f"{sensorname}_DAYS", lastday)
count_key = f"{sensorname}_{lastday}_count"
for line in proc.stdout.readlines():
packet = _create_packet(line)
timestamp = _set_json_timestamp(packet.pop('timestamp'))
day, time = timestamp.split(' ')
timestamp = f"{day}_{time}"
day = day.replace('-', '')
if day != lastday:
_RED.sadd(f"{sensorname}_DAYS", day)
count_key = f"{sensorname}_{day}_count"
lastday = day
if packet['opcode'] == '1':
keyname = f"{sensorname}_{packet['ipdst']}_{timestamp}"
values = [packet[value] for value in ('ethsrc', 'ipsrc', 'arpsrc')]
to_set[keyname] = {
key: value
for key, value in zip(('req_src_mac', 'req_src_ip',
'req_src_arp_mac'), values)
}
to_incr[count_key]['request'] += 1
timestamp_key = timestamp
else:
keyname = f"{sensorname}_{packet['ipsrc']}_{timestamp_key}"
values = [
packet[value]
for value in ('ipdst', 'ethsrc', 'ethdst', 'arpsrc', 'arpdst')
]
keys = ('rep_dst_ip', 'rep_src_mac', 'rep_dst_mac',
'rep_src_arp_mac', 'rep_dst_arp_mac')
to_set[keyname] = {key: value for key, value in zip(keys, values)}
to_set[keyname]['rep_timestamp'] = timestamp
to_incr[count_key]['reply'] += 1
p = _RED.pipeline()
for key, values in to_set.items():
p.hmset(key, values)
for key, values in to_incr.items():
for value, amount in values.items():
p.zincrby(key, amount, value)
p.execute()
proc.wait()
_RED.sadd("FILES", filename)
return f"Layer2 data from {filename} parsed."
def _process_file_and_save_json(inputfile):
to_set, filename, sensorname = _get_data_structures(inputfile)
if _RED.sismember("FILES", filename):
return f'Filename {inputfile} was already imported ... skip ...\n'
proc = subprocess.Popen(_CMD.format(inputfile),
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
first_packet = {
"type": potiron.TYPE_SOURCE,
"sensorname": sensorname,
"filename": filename
}
first_packet.update(_FIRST_PACKET)
allpackets = [first_packet]
lastday = day_from_filename(filename)
_RED.sadd(f"{sensorname}_DAYS", lastday)
packet_id = 0
for line in proc.stdout.readlines():
packet = _create_packet(line)
packet['timestamp'] = _set_json_timestamp(packet['timestamp'])
allpackets.append(_create_json_packet(packet, packet_id))
day, time = packet.pop('timestamp').split(' ')
timestamp = f'{day}_{time}'
day = day.replace('-', '')
if day != lastday:
_RED.sadd(f"{sensorname}_DAYS", day)
lastday = day
ports = "_".join([
f"{port}{packet.pop(value)}"
for port, value in zip(('src', 'dst'), ('sport', 'dport'))
])
key = f"{sensorname}_{ports}_{timestamp}"
to_set[key] = {isn_type: value for isn_type, value in packet.items()}
packet_id += 1
p = _RED.pipeline()
for key, item in to_set.items():
p.hmset(key, item)
p.execute()
proc.wait()
potiron.store_packet(_ROOTDIR, filename, json.dumps(allpackets))
_RED.sadd("FILES", filename)
return f'ISN Data from {filename} parsed and stored in json format.'
def _store_standard_data(allpackets, sensorname, filename):
to_incr = defaultdict(lambda: defaultdict(int))
lastday = day_from_filename(filename)
_RED.sadd(f"{sensorname}_DAYS", lastday)
for packet in allpackets:
redis_key, day = _KEY_FUNCTION(packet, sensorname)
if day != lastday:
_RED.sadd(f"{sensorname}_DAYS", day)
lastday = day
for field in _JSON_FIELDS:
to_incr[f"{redis_key}:{field}"][packet[field]] += 1
p = _RED.pipeline()
for key, values in to_incr.items():
for value, amount in values.items():
p.zincrby(key, amount, value)
p.execute()
_RED.sadd("FILES", filename)
return f"Data from {filename} parsed from JSON file."
def _store_standard_data(allpackets, sensorname, filename):
to_incr = defaultdict(lambda: defaultdict(int))
lastday = day_from_filename(filename)
_RED.sadd(f"{sensorname}_DAYS", lastday)
for packet in allpackets:
redis_key, day = _KEY_FUNCTION(packet, sensorname)
if day != lastday:
_RED.sadd(f"{sensorname}_DAYS", day)
lastday = day
for field in _JSON_FIELDS:
to_incr[f"{redis_key}:{field}"][packet[field]] += 1
p = _RED.pipeline()
for key, values in to_incr.items():
for value, amount in values.items():
p.zincrby(key, amount, value)
p.execute()
_RED.sadd("FILES", filename)
return f"Data from {filename} parsed from JSON file."
def _store_layer2_data(allpackets, sensorname, filename):
to_set = {}
to_incr = defaultdict(lambda: defaultdict(int))
lastday = day_from_filename(filename)
_RED.sadd(f"{sensorname}_DAYS", lastday)
count_key = f"{sensorname}_{lastday}_count"
for packet in allpackets:
day, time = packet.pop('timestamp').split(' ')
timestamp = f"{day}_{time}"
day = day.replace('-', '')
if day != lastday:
_RED.sadd(f"{sensorname}_DAYS", day)
count_key = f"{sensorname}_{day}_count"
lastday = day
if packet['opcode'] == '1':
keyname = f"{sensorname}_{packet['ipdst']}_{timestamp}"
values = [packet[value] for value in ('ethsrc', 'ipsrc', 'arpsrc')]
to_set[keyname] = {
key: value
for key, value in zip(('req_src_mac', 'req_src_ip',
'req_src_arp_mac'), values)
}
to_incr[count_key]['request'] += 1
timestamp_key = timestamp
else:
keyname = f"{sensorname}_{packet['ipsrc']}_{timestamp_key}"
values = [
packet[value]
for value in ('ipdst', 'ethsrc', 'ethdst', 'arpsrc', 'arpdst')
]
keys = ('rep_dst_ip', 'rep_src_mac', 'rep_dst_mac',
'rep_src_arp_mac', 'rep_dst_arp_mac')
to_set[keyname] = {key: value for key, value in zip(keys, values)}
to_set[keyname]['rep_timestamp'] = timestamp
to_incr[count_key]['reply'] += 1
p = _RED.pipeline()
for key, values in to_set.items():
p.hmset(key, values)
for key, values in to_incr.items():
for value, amount in values.items():
p.zincrby(key, amount, value)
p.execute()
_RED.sadd("FILES", filename)
return f"Layer2 data from {filename} parsed from JSON file."
def _store_isn_data(allpackets, sensorname, filename):
to_set = {}
lastday = day_from_filename(filename)
_RED.sadd(f"{sensorname}_DAYS", lastday)
for packet in allpackets:
day, time = packet.pop('timestamp').split(' ')
timestamp = f"{day}_{time}"
day = day.replace('-', '')
if day != lastday:
_RED.sadd(f"{sensorname}_DAYS", day)
lastday = day
ports = "_".join([f"{port}{packet.pop(value)}" for port, value in zip(('src', 'dst'), ('sport', 'dport'))])
key = f"{sensorname}_{ports}_{timestamp}"
to_set[key] = {isn_type: packet[isn_type] for isn_type in _isn_fields}
p = _RED.pipeline()
for key, item in to_set.items():
p.hmset(key, item)
p.execute()
_RED.sadd("FILES", filename)
return f"ISN data from {filename} parsed from JSON file."
def _store_isn_data(allpackets, sensorname, filename):
to_set = {}
lastday = day_from_filename(filename)
_RED.sadd(f"{sensorname}_DAYS", lastday)
for packet in allpackets:
day, time = packet.pop('timestamp').split(' ')
timestamp = f"{day}_{time}"
day = day.replace('-', '')
if day != lastday:
_RED.sadd(f"{sensorname}_DAYS", day)
lastday = day
ports = "_".join([
f"{port}{packet.pop(value)}"
for port, value in zip(('src', 'dst'), ('sport', 'dport'))
])
key = f"{sensorname}_{ports}_{timestamp}"
to_set[key] = {isn_type: packet[isn_type] for isn_type in _isn_fields}
p = _RED.pipeline()
for key, item in to_set.items():
p.hmset(key, item)
p.execute()
_RED.sadd("FILES", filename)
return f"ISN data from {filename} parsed from JSON file."
def _store_layer2_data(allpackets, sensorname, filename):
to_set = {}
to_incr = defaultdict(lambda: defaultdict(int))
lastday = day_from_filename(filename)
_RED.sadd(f"{sensorname}_DAYS", lastday)
count_key = f"{sensorname}_{lastday}_count"
for packet in allpackets:
day, time = packet.pop('timestamp').split(' ')
timestamp = f"{day}_{time}"
day = day.replace('-', '')
if day != lastday:
_RED.sadd(f"{sensorname}_DAYS", day)
count_key = f"{sensorname}_{day}_count"
lastday = day
if packet['opcode'] == '1':
keyname = f"{sensorname}_{packet['ipdst']}_{timestamp}"
values = [packet[value] for value in ('ethsrc', 'ipsrc', 'arpsrc')]
to_set[keyname] = {key: value for key, value in zip(('req_src_mac', 'req_src_ip', 'req_src_arp_mac'), values)}
to_incr[count_key]['request'] += 1
timestamp_key = timestamp
else:
keyname = f"{sensorname}_{packet['ipsrc']}_{timestamp_key}"
values = [packet[value] for value in ('ipdst', 'ethsrc', 'ethdst', 'arpsrc', 'arpdst')]
keys = ('rep_dst_ip', 'rep_src_mac', 'rep_dst_mac', 'rep_src_arp_mac', 'rep_dst_arp_mac')
to_set[keyname] = {key: value for key, value in zip(keys, values)}
to_set[keyname]['rep_timestamp'] = timestamp
to_incr[count_key]['reply'] += 1
p = _RED.pipeline()
for key, values in to_set.items():
p.hmset(key, values)
for key, values in to_incr.items():
for value, amount in values.items():
p.zincrby(key, amount, value)
p.execute()
_RED.sadd("FILES", filename)
return f"Layer2 data from {filename} parsed from JSON file."
def _process_file_and_save_json(inputfile):
to_set = {}
to_incr, filename, sensorname = _get_data_structures(inputfile)
if _RED.sismember("FILES", filename):
return f'Filename {inputfile} was already imported ... skip ...\n'
proc = subprocess.Popen(_CMD.format(inputfile),
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
first_packet = {
"type": potiron.TYPE_SOURCE,
"sensorname": sensorname,
"filename": filename
}
first_packet.update(_FIRST_PACKET)
allpackets = [first_packet]
lastday = day_from_filename(filename)
_RED.sadd(f"{sensorname}_DAYS", lastday)
count_key = f"{sensorname}_{lastday}_count"
packet_id = 0
for line in proc.stdout.readlines():
packet = _create_packet(line)
packet['timestamp'] = _set_json_timestamp(packet['timestamp'])
allpackets.append(_create_json_packet(packet, packet_id))
day, time = packet.pop('timestamp').split(' ')
timestamp = f"{day}_{time}"
day = day.replace('-', '')
if day != lastday:
_RED.sadd(f"{sensorname}_DAYS", day)
count_key = f"{sensorname}_{day}_count"
lastdady = day
if packet['opcode'] == '1':
keyname = f"{sensorname}_{packet['ipdst']}_{timestamp}"
values = [packet[value] for value in ('ethsrc', 'ipsrc', 'arpsrc')]
to_set[keyname] = {
key: value
for key, value in zip(('req_src_mac', 'req_src_ip',
'req_src_arp_mac'), values)
}
to_incr[count_key]['request'] += 1
timestamp_key = timestamp
else:
keyname = f"{sensorname}_{packet['ipsrc']}_{timestamp_key}"
values = [
packet[value]
for value in ('ipdst', 'ethsrc', 'ethdst', 'arpsrc', 'arpdst')
]
keys = ('rep_dst_ip', 'rep_src_mac', 'rep_dst_mac',
'rep_src_arp_mac', 'rep_dst_arp_mac')
to_set[keyname] = {key: value for key, value in zip(keys, values)}
to_set[keyname]['rep_timestamp'] = timestamp
to_incr[count_key]['reply'] += 1
packet_id += 1
p = _RED.pipeline()
for key, values in to_set.items():
p.hmset(key, values)
for key, values in to_incr.items():
for value, amount in values.items():
p.zincrby(key, amount, value)
p.execute()
proc.wait()
potiron.store_packet(_ROOTDIR, filename, json.dumps(allpackets))
_RED.sadd("FILES", filename)
return f"Layer2 data from {filename} parsed and stored in json format."