def _process_file_and_save_json(inputfile): to_set, filename, sensorname = _get_data_structures(inputfile) if _RED.sismember("FILES", filename): return f'Filename {inputfile} was already imported ... skip ...\n' proc = subprocess.Popen(_CMD.format(inputfile), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) first_packet = {"type": potiron.TYPE_SOURCE, "sensorname": sensorname, "filename": filename} first_packet.update(_FIRST_PACKET) allpackets = [first_packet] lastday = day_from_filename(filename) _RED.sadd(f"{sensorname}_DAYS", lastday) packet_id = 0 for line in proc.stdout.readlines(): packet = _create_packet(line) packet['timestamp'] = _set_json_timestamp(packet['timestamp']) allpackets.append(_create_json_packet(packet, packet_id)) day, time = packet.pop('timestamp').split(' ') timestamp = f'{day}_{time}' day = day.replace('-', '') if day != lastday: _RED.sadd(f"{sensorname}_DAYS", day) lastday = day ports = "_".join([f"{port}{packet.pop(value)}" for port, value in zip(('src', 'dst'), ('sport', 'dport'))]) key = f"{sensorname}_{ports}_{timestamp}" to_set[key] = {isn_type: value for isn_type, value in packet.items()} packet_id += 1 p = _RED.pipeline() for key, item in to_set.items(): p.hmset(key, item) p.execute() proc.wait() potiron.store_packet(_ROOTDIR, filename, json.dumps(allpackets)) _RED.sadd("FILES", filename) return f'ISN Data from {filename} parsed and stored in json format.'
def _process_file(inputfile): to_set, filename, sensorname = _get_data_structures(inputfile) if _RED.sismember("FILES", filename): return f'Filename {inputfile} was already imported ... skip ...\n' proc = subprocess.Popen(_CMD.format(inputfile), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) lastday = day_from_filename(filename) _RED.sadd(f"{sensorname}_DAYS", lastday) for line in proc.stdout.readlines(): packet = _create_packet(line) timestamp = _set_json_timestamp(packet.pop('timestamp')) day, time = timestamp.split(' ') timestamp = f"{day}_{time}" day = day.replace('-', '') if day != lastday: _RED.sadd(f"{sensorname}_DAYS", day) lastday = day ports = "_".join([ f"{port}{packet.pop(value)}" for port, value in zip(('src', 'dst'), ('sport', 'dport')) ]) key = f"{sensorname}_{ports}_{timestamp}" to_set[key] = {isn_type: value for isn_type, value in packet.items()} p = _RED.pipeline() for key, item in to_set.items(): p.hmset(key, item) p.execute() proc.wait() _RED.sadd("FILES", filename) return f'ISN Data from {filename} parsed.'
def _process_file(inputfile): to_set, filename, sensorname = _get_data_structures(inputfile) if _RED.sismember("FILES", filename): return f'Filename {inputfile} was already imported ... skip ...\n' proc = subprocess.Popen(_CMD.format(inputfile), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) lastday = day_from_filename(filename) _RED.sadd(f"{sensorname}_DAYS", lastday) for line in proc.stdout.readlines(): packet = _create_packet(line) timestamp = _set_json_timestamp(packet.pop('timestamp')) day, time = timestamp.split(' ') timestamp = f"{day}_{time}" day = day.replace('-', '') if day != lastday: _RED.sadd(f"{sensorname}_DAYS", day) lastday = day ports = "_".join([f"{port}{packet.pop(value)}" for port, value in zip(('src', 'dst'), ('sport', 'dport'))]) key = f"{sensorname}_{ports}_{timestamp}" to_set[key] = {isn_type: value for isn_type, value in packet.items()} p = _RED.pipeline() for key, item in to_set.items(): p.hmset(key, item) p.execute() proc.wait() _RED.sadd("FILES", filename) return f'ISN Data from {filename} parsed.'
def _process_file(inputfile): to_set = {} to_incr, filename, sensorname = _get_data_structures(inputfile) if _RED.sismember("FILES", filename): return f'Filename {inputfile} was already imported ... skip ...\n' proc = subprocess.Popen(_CMD.format(inputfile), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) lastday = day_from_filename(filename) _RED.sadd(f"{sensorname}_DAYS", lastday) count_key = f"{sensorname}_{lastday}_count" for line in proc.stdout.readlines(): packet = _create_packet(line) timestamp = _set_json_timestamp(packet.pop('timestamp')) day, time = timestamp.split(' ') timestamp = f"{day}_{time}" day = day.replace('-', '') if day != lastday: _RED.sadd(f"{sensorname}_DAYS", day) count_key = f"{sensorname}_{day}_count" lastday = day if packet['opcode'] == '1': keyname = f"{sensorname}_{packet['ipdst']}_{timestamp}" values = [packet[value] for value in ('ethsrc', 'ipsrc', 'arpsrc')] to_set[keyname] = { key: value for key, value in zip(('req_src_mac', 'req_src_ip', 'req_src_arp_mac'), values) } to_incr[count_key]['request'] += 1 timestamp_key = timestamp else: keyname = f"{sensorname}_{packet['ipsrc']}_{timestamp_key}" values = [ packet[value] for value in ('ipdst', 'ethsrc', 'ethdst', 'arpsrc', 'arpdst') ] keys = ('rep_dst_ip', 'rep_src_mac', 'rep_dst_mac', 'rep_src_arp_mac', 'rep_dst_arp_mac') to_set[keyname] = {key: value for key, value in zip(keys, values)} to_set[keyname]['rep_timestamp'] = timestamp to_incr[count_key]['reply'] += 1 p = _RED.pipeline() for key, values in to_set.items(): p.hmset(key, values) for key, values in to_incr.items(): for value, amount in values.items(): p.zincrby(key, amount, value) p.execute() proc.wait() _RED.sadd("FILES", filename) return f"Layer2 data from {filename} parsed."
def _process_file_and_save_json(inputfile): to_set, filename, sensorname = _get_data_structures(inputfile) if _RED.sismember("FILES", filename): return f'Filename {inputfile} was already imported ... skip ...\n' proc = subprocess.Popen(_CMD.format(inputfile), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) first_packet = { "type": potiron.TYPE_SOURCE, "sensorname": sensorname, "filename": filename } first_packet.update(_FIRST_PACKET) allpackets = [first_packet] lastday = day_from_filename(filename) _RED.sadd(f"{sensorname}_DAYS", lastday) packet_id = 0 for line in proc.stdout.readlines(): packet = _create_packet(line) packet['timestamp'] = _set_json_timestamp(packet['timestamp']) allpackets.append(_create_json_packet(packet, packet_id)) day, time = packet.pop('timestamp').split(' ') timestamp = f'{day}_{time}' day = day.replace('-', '') if day != lastday: _RED.sadd(f"{sensorname}_DAYS", day) lastday = day ports = "_".join([ f"{port}{packet.pop(value)}" for port, value in zip(('src', 'dst'), ('sport', 'dport')) ]) key = f"{sensorname}_{ports}_{timestamp}" to_set[key] = {isn_type: value for isn_type, value in packet.items()} packet_id += 1 p = _RED.pipeline() for key, item in to_set.items(): p.hmset(key, item) p.execute() proc.wait() potiron.store_packet(_ROOTDIR, filename, json.dumps(allpackets)) _RED.sadd("FILES", filename) return f'ISN Data from {filename} parsed and stored in json format.'
def _store_standard_data(allpackets, sensorname, filename): to_incr = defaultdict(lambda: defaultdict(int)) lastday = day_from_filename(filename) _RED.sadd(f"{sensorname}_DAYS", lastday) for packet in allpackets: redis_key, day = _KEY_FUNCTION(packet, sensorname) if day != lastday: _RED.sadd(f"{sensorname}_DAYS", day) lastday = day for field in _JSON_FIELDS: to_incr[f"{redis_key}:{field}"][packet[field]] += 1 p = _RED.pipeline() for key, values in to_incr.items(): for value, amount in values.items(): p.zincrby(key, amount, value) p.execute() _RED.sadd("FILES", filename) return f"Data from {filename} parsed from JSON file."
def _store_layer2_data(allpackets, sensorname, filename): to_set = {} to_incr = defaultdict(lambda: defaultdict(int)) lastday = day_from_filename(filename) _RED.sadd(f"{sensorname}_DAYS", lastday) count_key = f"{sensorname}_{lastday}_count" for packet in allpackets: day, time = packet.pop('timestamp').split(' ') timestamp = f"{day}_{time}" day = day.replace('-', '') if day != lastday: _RED.sadd(f"{sensorname}_DAYS", day) count_key = f"{sensorname}_{day}_count" lastday = day if packet['opcode'] == '1': keyname = f"{sensorname}_{packet['ipdst']}_{timestamp}" values = [packet[value] for value in ('ethsrc', 'ipsrc', 'arpsrc')] to_set[keyname] = { key: value for key, value in zip(('req_src_mac', 'req_src_ip', 'req_src_arp_mac'), values) } to_incr[count_key]['request'] += 1 timestamp_key = timestamp else: keyname = f"{sensorname}_{packet['ipsrc']}_{timestamp_key}" values = [ packet[value] for value in ('ipdst', 'ethsrc', 'ethdst', 'arpsrc', 'arpdst') ] keys = ('rep_dst_ip', 'rep_src_mac', 'rep_dst_mac', 'rep_src_arp_mac', 'rep_dst_arp_mac') to_set[keyname] = {key: value for key, value in zip(keys, values)} to_set[keyname]['rep_timestamp'] = timestamp to_incr[count_key]['reply'] += 1 p = _RED.pipeline() for key, values in to_set.items(): p.hmset(key, values) for key, values in to_incr.items(): for value, amount in values.items(): p.zincrby(key, amount, value) p.execute() _RED.sadd("FILES", filename) return f"Layer2 data from {filename} parsed from JSON file."
def _store_isn_data(allpackets, sensorname, filename): to_set = {} lastday = day_from_filename(filename) _RED.sadd(f"{sensorname}_DAYS", lastday) for packet in allpackets: day, time = packet.pop('timestamp').split(' ') timestamp = f"{day}_{time}" day = day.replace('-', '') if day != lastday: _RED.sadd(f"{sensorname}_DAYS", day) lastday = day ports = "_".join([f"{port}{packet.pop(value)}" for port, value in zip(('src', 'dst'), ('sport', 'dport'))]) key = f"{sensorname}_{ports}_{timestamp}" to_set[key] = {isn_type: packet[isn_type] for isn_type in _isn_fields} p = _RED.pipeline() for key, item in to_set.items(): p.hmset(key, item) p.execute() _RED.sadd("FILES", filename) return f"ISN data from {filename} parsed from JSON file."
def _store_isn_data(allpackets, sensorname, filename): to_set = {} lastday = day_from_filename(filename) _RED.sadd(f"{sensorname}_DAYS", lastday) for packet in allpackets: day, time = packet.pop('timestamp').split(' ') timestamp = f"{day}_{time}" day = day.replace('-', '') if day != lastday: _RED.sadd(f"{sensorname}_DAYS", day) lastday = day ports = "_".join([ f"{port}{packet.pop(value)}" for port, value in zip(('src', 'dst'), ('sport', 'dport')) ]) key = f"{sensorname}_{ports}_{timestamp}" to_set[key] = {isn_type: packet[isn_type] for isn_type in _isn_fields} p = _RED.pipeline() for key, item in to_set.items(): p.hmset(key, item) p.execute() _RED.sadd("FILES", filename) return f"ISN data from {filename} parsed from JSON file."
def _store_layer2_data(allpackets, sensorname, filename): to_set = {} to_incr = defaultdict(lambda: defaultdict(int)) lastday = day_from_filename(filename) _RED.sadd(f"{sensorname}_DAYS", lastday) count_key = f"{sensorname}_{lastday}_count" for packet in allpackets: day, time = packet.pop('timestamp').split(' ') timestamp = f"{day}_{time}" day = day.replace('-', '') if day != lastday: _RED.sadd(f"{sensorname}_DAYS", day) count_key = f"{sensorname}_{day}_count" lastday = day if packet['opcode'] == '1': keyname = f"{sensorname}_{packet['ipdst']}_{timestamp}" values = [packet[value] for value in ('ethsrc', 'ipsrc', 'arpsrc')] to_set[keyname] = {key: value for key, value in zip(('req_src_mac', 'req_src_ip', 'req_src_arp_mac'), values)} to_incr[count_key]['request'] += 1 timestamp_key = timestamp else: keyname = f"{sensorname}_{packet['ipsrc']}_{timestamp_key}" values = [packet[value] for value in ('ipdst', 'ethsrc', 'ethdst', 'arpsrc', 'arpdst')] keys = ('rep_dst_ip', 'rep_src_mac', 'rep_dst_mac', 'rep_src_arp_mac', 'rep_dst_arp_mac') to_set[keyname] = {key: value for key, value in zip(keys, values)} to_set[keyname]['rep_timestamp'] = timestamp to_incr[count_key]['reply'] += 1 p = _RED.pipeline() for key, values in to_set.items(): p.hmset(key, values) for key, values in to_incr.items(): for value, amount in values.items(): p.zincrby(key, amount, value) p.execute() _RED.sadd("FILES", filename) return f"Layer2 data from {filename} parsed from JSON file."
def _process_file_and_save_json(inputfile): to_set = {} to_incr, filename, sensorname = _get_data_structures(inputfile) if _RED.sismember("FILES", filename): return f'Filename {inputfile} was already imported ... skip ...\n' proc = subprocess.Popen(_CMD.format(inputfile), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) first_packet = { "type": potiron.TYPE_SOURCE, "sensorname": sensorname, "filename": filename } first_packet.update(_FIRST_PACKET) allpackets = [first_packet] lastday = day_from_filename(filename) _RED.sadd(f"{sensorname}_DAYS", lastday) count_key = f"{sensorname}_{lastday}_count" packet_id = 0 for line in proc.stdout.readlines(): packet = _create_packet(line) packet['timestamp'] = _set_json_timestamp(packet['timestamp']) allpackets.append(_create_json_packet(packet, packet_id)) day, time = packet.pop('timestamp').split(' ') timestamp = f"{day}_{time}" day = day.replace('-', '') if day != lastday: _RED.sadd(f"{sensorname}_DAYS", day) count_key = f"{sensorname}_{day}_count" lastdady = day if packet['opcode'] == '1': keyname = f"{sensorname}_{packet['ipdst']}_{timestamp}" values = [packet[value] for value in ('ethsrc', 'ipsrc', 'arpsrc')] to_set[keyname] = { key: value for key, value in zip(('req_src_mac', 'req_src_ip', 'req_src_arp_mac'), values) } to_incr[count_key]['request'] += 1 timestamp_key = timestamp else: keyname = f"{sensorname}_{packet['ipsrc']}_{timestamp_key}" values = [ packet[value] for value in ('ipdst', 'ethsrc', 'ethdst', 'arpsrc', 'arpdst') ] keys = ('rep_dst_ip', 'rep_src_mac', 'rep_dst_mac', 'rep_src_arp_mac', 'rep_dst_arp_mac') to_set[keyname] = {key: value for key, value in zip(keys, values)} to_set[keyname]['rep_timestamp'] = timestamp to_incr[count_key]['reply'] += 1 packet_id += 1 p = _RED.pipeline() for key, values in to_set.items(): p.hmset(key, values) for key, values in to_incr.items(): for value, amount in values.items(): p.zincrby(key, amount, value) p.execute() proc.wait() potiron.store_packet(_ROOTDIR, filename, json.dumps(allpackets)) _RED.sadd("FILES", filename) return f"Layer2 data from {filename} parsed and stored in json format."