def get_next_flow(self):
if self.prevFlow:
ret = self.prevFlow
self.prevFlow = None
return ret
line = self.input_file.readline()
if not line:
return None
# we only support the default format at the moment.
# TODO: parse the header, get the appropriate names and
# field separators ...
if line.startswith('#'):
return self.get_next_flow()
fields = line.split()
srcFlow = {}
srcFlow[common.COL_FIRST_SWITCHED] = float(fields[0])
srcFlow[common.COL_SRC_IP] = ip2int(fields[2])
srcFlow[common.COL_SRC_PORT] = int(fields[3])
srcFlow[common.COL_DST_IP] = ip2int(fields[4])
srcFlow[common.COL_DST_PORT] = int(fields[5])
srcFlow[common.COL_PROTO] = common.getValueFromProto(fields[6])
if fields[8] == '-':
srcFlow[common.COL_LAST_SWITCHED] = float(fields[0])
else:
srcFlow[common.COL_LAST_SWITCHED] = float(fields[0]) + float(
fields[8])
srcFlow[common.COL_PKTS] = int(fields[15])
srcFlow[common.COL_BYTES] = int(fields[16])
dstFlow = {}
dstFlow[common.COL_FIRST_SWITCHED] = float(fields[0])
dstFlow[common.COL_SRC_IP] = ip2int(fields[4])
dstFlow[common.COL_SRC_PORT] = int(fields[5])
dstFlow[common.COL_DST_IP] = ip2int(fields[2])
dstFlow[common.COL_DST_PORT] = int(fields[3])
dstFlow[common.COL_PROTO] = common.getValueFromProto(fields[6])
if fields[8] == '-':
dstFlow[common.COL_LAST_SWITCHED] = float(fields[0])
else:
dstFlow[common.COL_LAST_SWITCHED] = float(fields[0]) + float(
fields[8])
dstFlow[common.COL_PKTS] = int(fields[17])
dstFlow[common.COL_BYTES] = int(fields[18])
# return srcFlow now, return dstflow at the next call to get_next_flow()
self.prevFlow = dstFlow
return srcFlow
def get_next_flow(self):
if self.prevFlow:
ret = self.prevFlow
self.prevFlow = None
return ret
line = self.input_file.readline()
if not line:
return None
# we only support the default format at the moment.
# TODO: parse the header, get the appropriate names and
# field separators ...
if line.startswith("#"):
return self.get_next_flow()
fields = line.split()
srcFlow = {}
srcFlow[common.COL_FIRST_SWITCHED] = float(fields[0])
srcFlow[common.COL_SRC_IP] = ip2int(fields[2])
srcFlow[common.COL_SRC_PORT] = int(fields[3])
srcFlow[common.COL_DST_IP] = ip2int(fields[4])
srcFlow[common.COL_DST_PORT] = int(fields[5])
srcFlow[common.COL_PROTO] = common.getValueFromProto(fields[6])
if fields[8] == "-":
srcFlow[common.COL_LAST_SWITCHED] = float(fields[0])
else:
srcFlow[common.COL_LAST_SWITCHED] = float(fields[0]) + float(fields[8])
srcFlow[common.COL_PKTS] = int(fields[15])
srcFlow[common.COL_BYTES] = int(fields[16])
dstFlow = {}
dstFlow[common.COL_FIRST_SWITCHED] = float(fields[0])
dstFlow[common.COL_SRC_IP] = ip2int(fields[4])
dstFlow[common.COL_SRC_PORT] = int(fields[5])
dstFlow[common.COL_DST_IP] = ip2int(fields[2])
dstFlow[common.COL_DST_PORT] = int(fields[3])
dstFlow[common.COL_PROTO] = common.getValueFromProto(fields[6])
if fields[8] == "-":
dstFlow[common.COL_LAST_SWITCHED] = float(fields[0])
else:
dstFlow[common.COL_LAST_SWITCHED] = float(fields[0]) + float(fields[8])
dstFlow[common.COL_PKTS] = int(fields[17])
dstFlow[common.COL_BYTES] = int(fields[18])
# return srcFlow now, return dstflow at the next call to get_next_flow()
self.prevFlow = dstFlow
return srcFlow
def get_next_flow(self):
if self.prevFlow:
ret = self.prevFlow
self.prevFlow = None
return ret
flow = self.c.fetchone()
if flow == None:
if len(self.tables) == 0:
return None
table = self.tables.pop()
print "Importing table ", table, "..."
self.c.execute("SELECT * FROM " + table + " ORDER BY stime ASC")
return self.get_next_flow()
obj = dict()
revObj = dict()
firstSwitched = None
for j, col in enumerate(self.c.description):
print col[0]
if col[0] in self.columnmap:
fieldName = self.columnmap[col[0]]
print "fieldName: ", fieldName
# argus tables contain reverse flow information
# create a revObj which matches the reverse flow
if fieldName == common.COL_SRC_IP:
ip = ip2int(flow[j])
obj[common.COL_SRC_IP] = ip
revObj[common.COL_DST_IP] = ip
if fieldName == common.COL_DST_IP:
ip = ip2int(flow[j])
obj[common.COL_DST_IP] = ip
revObj[common.COL_SRC_IP] = ip
if fieldName == common.COL_SRC_PORT:
obj[common.COL_SRC_PORT] = flow[j]
revObj[common.COL_DST_PORT] = flow[j]
if fieldName == common.COL_DST_PORT:
obj[common.COL_DST_PORT] = flow[j]
revObj[common.COL_SRC_PORT] = flow[j]
if fieldName == common.COL_PROTO:
obj[common.COL_PROTO] = common.getValueFromProto(flow[j])
revObj[common.COL_PROTO] = common.getValueFromProto(flow[j])
elif col[0] in self.srcDirectionMap:
fieldName = self.srcDirectionMap[col[0]]
obj[fieldName] = flow[j]
elif col[0] in self.dstDirectionMap:
fieldName = self.dstDirectionMap[col[0]]
revObj[fieldName] = flow[j]
elif col[0] == "stime":
firstSwitched = float(flow[j])
obj[common.COL_FIRST_SWITCHED] = firstSwitched
revObj[common.COL_FIRST_SWITCHED] = firstSwitched
elif col[0] == "dur":
obj[common.COL_LAST_SWITCHED] = firstSwitched + float(flow[j])
revObj[common.COL_LAST_SWITCHED] = firstSwitched + float(flow[j])
# check if the reverse flow start time is > 0. if it is zero, this means that we have a
# one way flow (e.g. coming from a scan) that should not be imported into the database
if common.COL_FIRST_SWITCHED in revObj and revObj[common.COL_FIRST_SWITCHED] > 0:
self.prevFlow = revObj
print obj
return obj
def extract_mongo_query_params():
# construct query
limit = 0
if "limit" in request.GET:
try:
limit = int(request.GET["limit"])
except ValueError:
raise HTTPError(output="Param limit has to be an integer.")
if limit < 0:
limit = 0
fields = None
if "fields" in request.GET:
fields = request.GET["fields"].strip()
fields = map(lambda v: v.strip(), fields.split(","))
sort = None
if "sort" in request.GET:
sort = request.GET["sort"].strip()
sort = map(lambda v: v.strip(), sort.split(","))
for i in range(0, len(sort)):
field = sort[i].split(" ")
order = 1
if field[-1].lower() == "asc":
field.pop()
elif field[-1].lower() == "desc":
order = -1
field.pop()
field = " ".join(field)
sort[i] = (field, order)
count = False
if "count" in request.GET:
count = True
# get query params
start_bucket = 0
if "start_bucket" in request.GET:
try:
start_bucket = int(request.GET["start_bucket"])
except ValueError:
raise HTTPError(output="Param start_bucket has to be an integer.")
if start_bucket < 0:
start_bucket = 0
end_bucket = sys.maxint
if "end_bucket" in request.GET:
try:
end_bucket = int(request.GET["end_bucket"])
except ValueError:
raise HTTPError(output="Param end_bucket has to be an integer.")
if end_bucket < 0:
end_bucket = 0
# the bucket resolution to query (number of buckets)
resolution = 1
if "resolution" in request.GET:
try:
resolution = int(request.GET["resolution"])
except ValueError:
raise HTTPError(output="Param resolution has to be an integer.")
if resolution < 1:
resolution = 1
# or set the bucket size directly
bucket_size = None
if "bucket_size" in request.GET:
try:
bucket_size = int(request.GET["bucket_size"])
except ValueError:
raise HTTPError(output="Param bucket_size has to be an integer.")
if bucket_size not in config.flow_bucket_sizes:
raise HTTPError(output="This bucket size is not available.")
# biflow aggregation
# This simply removes the difference between srcIP and dstIP
# (The smaller ip will always be the srcIP)
biflow = False
if "biflow" in request.GET:
biflow = True
# protocol filter
include_protos = []
if "include_protos" in request.GET:
include_protos = request.GET["include_protos"].strip()
include_protos = map(lambda v: common.getValueFromProto(v.strip()), include_protos.split(","))
exclude_protos = []
if "exclude_protos" in request.GET:
exclude_protos = request.GET["exclude_protos"].strip()
exclude_protos = map(lambda v: common.getValueFromProto(v.strip()), exclude_protos.split(","))
# port filter
include_ports = []
if "include_ports" in request.GET:
include_ports = request.GET["include_ports"].strip()
try:
include_ports = map(lambda v: int(v.strip()), include_ports.split(","))
except ValueError:
raise HTTPError(output="Ports have to be integers.")
exclude_ports = []
if "exclude_ports" in request.GET:
exclude_ports = request.GET["exclude_ports"].strip()
try:
exclude_ports = map(lambda v: int(v.strip()), exclude_ports.split(","))
except ValueError:
raise HTTPError(output="Ports have to be integers.")
# ip filter
include_ips = []
if "include_ips" in request.GET:
include_ips = request.GET["include_ips"].strip()
include_ips = map(lambda v: int(v.strip()), include_ips.split(","))
exclude_ips = []
if "exclude_ips" in request.GET:
exclude_ips = request.GET["exclude_ips"].strip()
exclude_ips = map(lambda v: int(v.strip()), exclude_ips.split(","))
# get buckets and aggregate
if bucket_size == None:
bucket_size = db.getBucketSize(start_bucket, end_bucket, resolution)
# only stated fields will be available, all others will be aggregated toghether
# filter for known aggregation values
#if fields != None:
# fields = [v for v in fields if v in config.flow_aggr_values]
black_others = False
if "black_others" in request.GET:
black_others = True
aggregate = []
if "aggregate" in request.GET:
aggregate = request.GET["aggregate"].strip()
aggregate = map(lambda v: v.strip(), aggregate.split(","))
result = {}
result["fields"] = fields
result["sort"] = sort
result["limit"] = limit
result["count"] = count
result["start_bucket"] = start_bucket
result["end_bucket"] = end_bucket
result["resolution"] = resolution
result["bucket_size"] = bucket_size
result["biflow"] = biflow
result["include_ports"] = include_ports
result["exclude_ports"] = exclude_ports
result["include_ips"] = include_ips
result["exclude_ips"] = exclude_ips
result["include_protos"] = include_protos
result["exclude_protos"] = exclude_protos
result["batch_size"] = 1000
result["aggregate"] = aggregate
result["black_others"] = black_others
return result
def extract_mongo_query_params():
# construct query
limit = 0
if "limit" in request.GET:
try:
limit = int(request.GET["limit"])
except ValueError:
raise HTTPError(output="Param limit has to be an integer.")
if limit < 0:
limit = 0
fields = None
if "fields" in request.GET:
fields = request.GET["fields"].strip()
fields = map(lambda v: v.strip(), fields.split(","))
sort = None
if "sort" in request.GET:
sort = request.GET["sort"].strip()
sort = map(lambda v: v.strip(), sort.split(","))
for i in range(0, len(sort)):
field = sort[i].split(" ")
order = 1
if field[-1].lower() == "asc":
field.pop()
elif field[-1].lower() == "desc":
order = -1
field.pop()
field = " ".join(field)
sort[i] = (field, order)
count = False
if "count" in request.GET:
count = True
# get query params
start_bucket = 0
if "start_bucket" in request.GET:
try:
start_bucket = int(request.GET["start_bucket"])
except ValueError:
raise HTTPError(output="Param start_bucket has to be an integer.")
if start_bucket < 0:
start_bucket = 0
end_bucket = sys.maxint
if "end_bucket" in request.GET:
try:
end_bucket = int(request.GET["end_bucket"])
except ValueError:
raise HTTPError(output="Param end_bucket has to be an integer.")
if end_bucket < 0:
end_bucket = 0
# the bucket resolution to query (number of buckets)
resolution = 1
if "resolution" in request.GET:
try:
resolution = int(request.GET["resolution"])
except ValueError:
raise HTTPError(output="Param resolution has to be an integer.")
if resolution < 1:
resolution = 1
# or set the bucket size directly
bucket_size = None
if "bucket_size" in request.GET:
try:
bucket_size = int(request.GET["bucket_size"])
except ValueError:
raise HTTPError(output="Param bucket_size has to be an integer.")
if bucket_size not in config.flow_bucket_sizes:
raise HTTPError(output="This bucket size is not available.")
# biflow aggregation
# This simply removes the difference between srcIP and dstIP
# (The smaller ip will always be the srcIP)
biflow = False
if "biflow" in request.GET:
biflow = True
# protocol filter
include_protos = []
if "include_protos" in request.GET:
include_protos = request.GET["include_protos"].strip()
include_protos = map(lambda v: common.getValueFromProto(v.strip()),
include_protos.split(","))
exclude_protos = []
if "exclude_protos" in request.GET:
exclude_protos = request.GET["exclude_protos"].strip()
exclude_protos = map(lambda v: common.getValueFromProto(v.strip()),
exclude_protos.split(","))
# port filter
include_ports = []
if "include_ports" in request.GET:
include_ports = request.GET["include_ports"].strip()
try:
include_ports = map(lambda v: int(v.strip()),
include_ports.split(","))
except ValueError:
raise HTTPError(output="Ports have to be integers.")
exclude_ports = []
if "exclude_ports" in request.GET:
exclude_ports = request.GET["exclude_ports"].strip()
try:
exclude_ports = map(lambda v: int(v.strip()),
exclude_ports.split(","))
except ValueError:
raise HTTPError(output="Ports have to be integers.")
# ip filter
include_ips = []
if "include_ips" in request.GET:
include_ips = request.GET["include_ips"].strip()
include_ips = map(lambda v: int(v.strip()), include_ips.split(","))
exclude_ips = []
if "exclude_ips" in request.GET:
exclude_ips = request.GET["exclude_ips"].strip()
exclude_ips = map(lambda v: int(v.strip()), exclude_ips.split(","))
# get buckets and aggregate
if bucket_size == None:
bucket_size = db.getBucketSize(start_bucket, end_bucket, resolution)
# only stated fields will be available, all others will be aggregated toghether
# filter for known aggregation values
#if fields != None:
# fields = [v for v in fields if v in config.flow_aggr_values]
black_others = False
if "black_others" in request.GET:
black_others = True
aggregate = []
if "aggregate" in request.GET:
aggregate = request.GET["aggregate"].strip()
aggregate = map(lambda v: v.strip(), aggregate.split(","))
result = {}
result["fields"] = fields
print "Fields: " + str(fields)
result["sort"] = sort
result["limit"] = limit
result["count"] = count
result["start_bucket"] = start_bucket
result["end_bucket"] = end_bucket
result["resolution"] = resolution
result["bucket_size"] = bucket_size
result["biflow"] = biflow
result["include_ports"] = include_ports
result["exclude_ports"] = exclude_ports
result["include_ips"] = include_ips
result["exclude_ips"] = exclude_ips
result["include_protos"] = include_protos
result["exclude_protos"] = exclude_protos
result["batch_size"] = 1000
result["aggregate"] = aggregate
result["black_others"] = black_others
return result
def get_next_flow(self):
if self.prevFlow:
ret = self.prevFlow
self.prevFlow = None
return ret
flow = self.c.fetchone()
if flow == None:
if len(self.tables) == 0:
return None
table = self.tables.pop()
print "Importing table ", table, "..."
self.c.execute("SELECT * FROM " + table + " ORDER BY stime ASC")
return self.get_next_flow()
obj = dict()
revObj = dict()
firstSwitched = None
for j, col in enumerate(self.c.description):
print col[0]
if col[0] in self.columnmap:
fieldName = self.columnmap[col[0]]
print "fieldName: ", fieldName
# argus tables contain reverse flow information
# create a revObj which matches the reverse flow
if fieldName == common.COL_SRC_IP:
ip = ip2int(flow[j])
obj[common.COL_SRC_IP] = ip
revObj[common.COL_DST_IP] = ip
if fieldName == common.COL_DST_IP:
ip = ip2int(flow[j])
obj[common.COL_DST_IP] = ip
revObj[common.COL_SRC_IP] = ip
if fieldName == common.COL_SRC_PORT:
obj[common.COL_SRC_PORT] = flow[j]
revObj[common.COL_DST_PORT] = flow[j]
if fieldName == common.COL_DST_PORT:
obj[common.COL_DST_PORT] = flow[j]
revObj[common.COL_SRC_PORT] = flow[j]
if fieldName == common.COL_PROTO:
obj[common.COL_PROTO] = common.getValueFromProto(flow[j])
revObj[common.COL_PROTO] = common.getValueFromProto(
flow[j])
elif col[0] in self.srcDirectionMap:
fieldName = self.srcDirectionMap[col[0]]
obj[fieldName] = flow[j]
elif col[0] in self.dstDirectionMap:
fieldName = self.dstDirectionMap[col[0]]
revObj[fieldName] = flow[j]
elif col[0] == "stime":
firstSwitched = float(flow[j])
obj[common.COL_FIRST_SWITCHED] = firstSwitched
revObj[common.COL_FIRST_SWITCHED] = firstSwitched
elif col[0] == "dur":
obj[common.COL_LAST_SWITCHED] = firstSwitched + float(flow[j])
revObj[common.COL_LAST_SWITCHED] = firstSwitched + float(
flow[j])
# check if the reverse flow start time is > 0. if it is zero, this means that we have a
# one way flow (e.g. coming from a scan) that should not be imported into the database
if common.COL_FIRST_SWITCHED in revObj and revObj[
common.COL_FIRST_SWITCHED] > 0:
self.prevFlow = revObj
print obj
return obj
