for f in args.tsharkfilter:
tsharkfilter += "{} ".format(f)
bpf += " && {}".format(tsharkfilter[:-1])
b_redis = args.redis
disable_json = args.disable_json
if disable_json:
b_redis = True
rootdir = None
else:
if args.outputdir is None:
sys.stderr.write("You should specify an output directory.\n")
sys.exit(1)
else:
rootdir = args.outputdir[0]
potiron.create_dirs(rootdir, inputfile)
if os.path.isdir(rootdir) is False:
sys.stderr.write("The root directory is not a directory\n")
sys.exit(1)
if b_redis:
if args.unix is None:
sys.stderr.write('A Unix socket must be specified.\n')
sys.exit(1)
usocket = args.unix[0]
red = redis.Redis(unix_socket_path=usocket)
ck = args.combined_keys
process_file(rootdir, inputfile, fieldfilter, b_redis, disable_json, ck)
def process_file(rootdir, filename):
if not potiron.check_program("ipsumdump"):
raise OSError("The program ipsumpdump is not installed")
# FIXME Put in config file
if rootdir is not None:
potiron.create_dirs(rootdir, filename)
packet = {}
sensorname = potiron.derive_sensor_name(filename)
allpackets = []
# Describe the source
allpackets.append({"type": potiron.TYPE_SOURCE, "sensorname": sensorname,
"filename": os.path.basename(filename)})
# Each packet as a incremental numeric id
# A packet is identified with its sensorname filename and packet id for
# further aggregation with meta data.
# Assumption: Each program process the pcap file the same way?
packet_id = 0
proc = subprocess.Popen(["ipsumdump", "--no-headers", "--quiet", "--timestamp",
"--length", "--protocol", "--ip-src", "--ip-dst", "--ip-opt",
"--ip-ttl", "--ip-tos", "--sport", "--dport", "--tcp-seq", "--tcp-ack",
"--icmp-code", "--icmp-type", "-f", potiron.bpfilter, "-r", filename],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
for line in proc.stdout.readlines():
packet_id = packet_id + 1
line = line[:-1].decode()
timestamp, length, protocol, ipsrc, ipdst, ipop, ipttl, iptos, sport, dport, tcpseq, tcpack, icmpcode, icmptype = line.split(' ')
ilength = -1
iipttl = -1
iiptos = -1
isport = -1
idport = -1
itcpseq = -1
itcpack = -1
iicmpcode = 255
iicmptype = 255
try:
ilength = int(length)
except ValueError:
pass
try:
iipttl = int(ipttl)
except ValueError:
pass
try:
iiptos = int(iptos)
except ValueError:
pass
try:
isport = int(sport)
except ValueError:
pass
try:
idport = int(dport)
except ValueError:
pass
try:
itcpseq = int(tcpseq)
except ValueError:
pass
try:
itcpack = int(tcpack)
except ValueError:
pass
try:
iicmpcode = int(icmpcode)
except ValueError:
pass
try:
iicmptype = int(icmptype)
except ValueError:
pass
if ipsrc == '-':
ipsrc = None
if ipdst == '-':
ipdst = None
# Convert timestamp
a, b = timestamp.split('.')
dobj = datetime.datetime.fromtimestamp(float(a))
stime = dobj.strftime("%Y-%m-%d %H:%M:%S")
stime = stime + "." + b
packet = {'timestamp': stime,
'length': ilength,
'protocol': numerize_proto(protocol),
'ipsrc': ipsrc,
'ipdst': ipdst,
'ipop': ipop,
'ipttl': iipttl,
'iptos': iiptos,
'sport': isport,
'dport': idport,
'tcpseq': itcpseq,
'tcpack': itcpack,
'icmpcode': iicmpcode,
'icmptype': iicmptype,
'packet_id': packet_id,
'type': potiron.TYPE_PACKET,
'state': potiron.STATE_NOT_ANNOATE
}
# FIXME might consume a lot of memory
allpackets.append(packet)
# FIXME Implement polling because wait can last forever
proc.wait()
if proc.returncode != 0:
errmsg = "".join(proc.stderr.readlines())
raise OSError("ipsumdump failed. Return code {}. {}".format(proc.returncode, errmsg))
potiron.store_packet(rootdir, filename, json.dumps(allpackets))
bpf += " && {}".format(tsharkfilter)
else:
tsharkfilter = ""
for f in args.tsharkfilter:
tsharkfilter += "{} ".format(f)
bpf += " && {}".format(tsharkfilter[:-1])
# Check if file was already imported
fn = os.path.basename(filename)
if red.sismember("FILES", fn):
sys.stderr.write(
'[INFO] Filename {} was already imported ... skip ...\n'.format(
fn))
sys.exit(0)
red.sadd("FILES", fn)
if not args.reverse:
potiron.create_reverse_global_dicts(red)
potiron.infomsg("Created global reverse annotation dictionaries")
sys.exit(0)
if args.outputdir is None:
sys.stderr.write('An output directory must be specified\n')
else:
outputdir = args.outputdir[0]
potiron.create_dirs(outputdir, filename)
if os.path.isdir(outputdir) is False:
sys.stderr.write("The root directory is not a directory\n")
sys.exit(1)
process_file(outputdir, filename)
if os.path.exists(args.input[0]) is False:
sys.stderr.write("The filename {} was not found\n".format(args.input[0]))
sys.exit(1)
inputfile = args.input[0]
if args.unix is None:
sys.stderr.write('A Unix socket must be specified.\n')
sys.exit(1)
usocket = args.unix[0]
red = redis.Redis(unix_socket_path=usocket)
if args.outputdir is None:
sys.stderr.write("You should specify an output directory.\n")
sys.exit(1)
else:
rootdir = args.outputdir[0]
potiron.create_dirs(rootdir, inputfile)
if os.path.isdir(rootdir) is False:
sys.stderr.write("The root directory is not a directory\n")
sys.exit(1)
# Check if file was already imported
fn = os.path.basename(inputfile)
fn = '{}.json'.format(fn.split('.')[0])
if red.sismember("FILES", fn):
sys.stderr.write('[INFO] Filename {} was already imported ... skip ...\n'.format(fn))
sys.exit(0)
red.sadd("FILES", fn)
process_file(rootdir, inputfile)
def process_file(rootdir, filename):
if not potiron.check_program("ipsumdump"):
raise OSError("The program ipsumpdump is not installed")
# FIXME Put in config file
if rootdir is not None:
potiron.create_dirs(rootdir, filename)
packet = {}
sensorname = potiron.derive_sensor_name(filename)
allpackets = []
# Describe the source
allpackets.append({
"type": potiron.TYPE_SOURCE,
"sensorname": sensorname,
"filename": os.path.basename(filename)
})
# Each packet as a incremental numeric id
# A packet is identified with its sensorname filename and packet id for
# further aggregation with meta data.
# Assumption: Each program process the pcap file the same way?
packet_id = 0
proc = subprocess.Popen([
"ipsumdump", "--no-headers", "--quiet", "--timestamp", "--length",
"--protocol", "--ip-src", "--ip-dst", "--ip-opt", "--ip-ttl",
"--ip-tos", "--sport", "--dport", "--tcp-seq", "--tcp-ack",
"--icmp-code", "--icmp-type", "-f", potiron.bpfilter, "-r", filename
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
for line in proc.stdout.readlines():
packet_id = packet_id + 1
line = line[:-1].decode()
timestamp, length, protocol, ipsrc, ipdst, ipop, ipttl, iptos, sport, dport, tcpseq, tcpack, icmpcode, icmptype = line.split(
' ')
ilength = -1
iipttl = -1
iiptos = -1
isport = -1
idport = -1
itcpseq = -1
itcpack = -1
iicmpcode = 255
iicmptype = 255
try:
ilength = int(length)
except ValueError:
pass
try:
iipttl = int(ipttl)
except ValueError:
pass
try:
iiptos = int(iptos)
except ValueError:
pass
try:
isport = int(sport)
except ValueError:
pass
try:
idport = int(dport)
except ValueError:
pass
try:
itcpseq = int(tcpseq)
except ValueError:
pass
try:
itcpack = int(tcpack)
except ValueError:
pass
try:
iicmpcode = int(icmpcode)
except ValueError:
pass
try:
iicmptype = int(icmptype)
except ValueError:
pass
if ipsrc == '-':
ipsrc = None
if ipdst == '-':
ipdst = None
# Convert timestamp
a, b = timestamp.split('.')
dobj = datetime.datetime.fromtimestamp(float(a))
stime = dobj.strftime("%Y-%m-%d %H:%M:%S")
stime = stime + "." + b
packet = {
'timestamp': stime,
'length': ilength,
'protocol': numerize_proto(protocol),
'ipsrc': ipsrc,
'ipdst': ipdst,
'ipop': ipop,
'ipttl': iipttl,
'iptos': iiptos,
'sport': isport,
'dport': idport,
'tcpseq': itcpseq,
'tcpack': itcpack,
'icmpcode': iicmpcode,
'icmptype': iicmptype,
'packet_id': packet_id,
'type': potiron.TYPE_PACKET,
'state': potiron.STATE_NOT_ANNOATE
}
# FIXME might consume a lot of memory
allpackets.append(packet)
# FIXME Implement polling because wait can last forever
proc.wait()
if proc.returncode != 0:
errmsg = "".join(proc.stderr.readlines())
raise OSError("ipsumdump failed. Return code {}. {}".format(
proc.returncode, errmsg))
potiron.store_packet(rootdir, filename, json.dumps(allpackets))
red.sadd('BPF', bpf)
if not args.reverse:
potiron.create_reverse_global_dicts(red)
potiron.infomsg("Created global reverse annotation dictionaries")
sys.exit(0)
disable_json = args.disable_json
if disable_json:
outputdir = None
else:
if args.outputdir is None:
sys.stderr.write('An output directory must be specified\n')
sys.exit(1)
else:
outputdir = args.outputdir[0]
potiron.create_dirs(outputdir, filename)
if os.path.isdir(outputdir) is False:
sys.stderr.write("The root directory is not a directory\n")
sys.exit(1)
# Check if file was already imported
fn = os.path.basename(filename)
fn = '{}.json'.format(fn.split('.')[0])
if red.sismember("FILES", fn):
sys.stderr.write('[INFO] Filename {} was already imported ... skip ...\n'.format(fn))
sys.exit(0)
red.sadd("FILES", fn)
process_file(outputdir, filename)