def get_next_flow(self): if self.prevFlow: ret = self.prevFlow self.prevFlow = None return ret line = self.input_file.readline() if not line: return None # we only support the default format at the moment. # TODO: parse the header, get the appropriate names and # field separators ... if line.startswith('#'): return self.get_next_flow() fields = line.split() srcFlow = {} srcFlow[common.COL_FIRST_SWITCHED] = float(fields[0]) srcFlow[common.COL_SRC_IP] = ip2int(fields[2]) srcFlow[common.COL_SRC_PORT] = int(fields[3]) srcFlow[common.COL_DST_IP] = ip2int(fields[4]) srcFlow[common.COL_DST_PORT] = int(fields[5]) srcFlow[common.COL_PROTO] = common.getValueFromProto(fields[6]) if fields[8] == '-': srcFlow[common.COL_LAST_SWITCHED] = float(fields[0]) else: srcFlow[common.COL_LAST_SWITCHED] = float(fields[0]) + float( fields[8]) srcFlow[common.COL_PKTS] = int(fields[15]) srcFlow[common.COL_BYTES] = int(fields[16]) dstFlow = {} dstFlow[common.COL_FIRST_SWITCHED] = float(fields[0]) dstFlow[common.COL_SRC_IP] = ip2int(fields[4]) dstFlow[common.COL_SRC_PORT] = int(fields[5]) dstFlow[common.COL_DST_IP] = ip2int(fields[2]) dstFlow[common.COL_DST_PORT] = int(fields[3]) dstFlow[common.COL_PROTO] = common.getValueFromProto(fields[6]) if fields[8] == '-': dstFlow[common.COL_LAST_SWITCHED] = float(fields[0]) else: dstFlow[common.COL_LAST_SWITCHED] = float(fields[0]) + float( fields[8]) dstFlow[common.COL_PKTS] = int(fields[17]) dstFlow[common.COL_BYTES] = int(fields[18]) # return srcFlow now, return dstflow at the next call to get_next_flow() self.prevFlow = dstFlow return srcFlow
def get_next_flow(self): if self.prevFlow: ret = self.prevFlow self.prevFlow = None return ret line = self.input_file.readline() if not line: return None # we only support the default format at the moment. # TODO: parse the header, get the appropriate names and # field separators ... if line.startswith("#"): return self.get_next_flow() fields = line.split() srcFlow = {} srcFlow[common.COL_FIRST_SWITCHED] = float(fields[0]) srcFlow[common.COL_SRC_IP] = ip2int(fields[2]) srcFlow[common.COL_SRC_PORT] = int(fields[3]) srcFlow[common.COL_DST_IP] = ip2int(fields[4]) srcFlow[common.COL_DST_PORT] = int(fields[5]) srcFlow[common.COL_PROTO] = common.getValueFromProto(fields[6]) if fields[8] == "-": srcFlow[common.COL_LAST_SWITCHED] = float(fields[0]) else: srcFlow[common.COL_LAST_SWITCHED] = float(fields[0]) + float(fields[8]) srcFlow[common.COL_PKTS] = int(fields[15]) srcFlow[common.COL_BYTES] = int(fields[16]) dstFlow = {} dstFlow[common.COL_FIRST_SWITCHED] = float(fields[0]) dstFlow[common.COL_SRC_IP] = ip2int(fields[4]) dstFlow[common.COL_SRC_PORT] = int(fields[5]) dstFlow[common.COL_DST_IP] = ip2int(fields[2]) dstFlow[common.COL_DST_PORT] = int(fields[3]) dstFlow[common.COL_PROTO] = common.getValueFromProto(fields[6]) if fields[8] == "-": dstFlow[common.COL_LAST_SWITCHED] = float(fields[0]) else: dstFlow[common.COL_LAST_SWITCHED] = float(fields[0]) + float(fields[8]) dstFlow[common.COL_PKTS] = int(fields[17]) dstFlow[common.COL_BYTES] = int(fields[18]) # return srcFlow now, return dstflow at the next call to get_next_flow() self.prevFlow = dstFlow return srcFlow
def get_next_flow(self): if self.prevFlow: ret = self.prevFlow self.prevFlow = None return ret flow = self.c.fetchone() if flow == None: if len(self.tables) == 0: return None table = self.tables.pop() print "Importing table ", table, "..." self.c.execute("SELECT * FROM " + table + " ORDER BY stime ASC") return self.get_next_flow() obj = dict() revObj = dict() firstSwitched = None for j, col in enumerate(self.c.description): print col[0] if col[0] in self.columnmap: fieldName = self.columnmap[col[0]] print "fieldName: ", fieldName # argus tables contain reverse flow information # create a revObj which matches the reverse flow if fieldName == common.COL_SRC_IP: ip = ip2int(flow[j]) obj[common.COL_SRC_IP] = ip revObj[common.COL_DST_IP] = ip if fieldName == common.COL_DST_IP: ip = ip2int(flow[j]) obj[common.COL_DST_IP] = ip revObj[common.COL_SRC_IP] = ip if fieldName == common.COL_SRC_PORT: obj[common.COL_SRC_PORT] = flow[j] revObj[common.COL_DST_PORT] = flow[j] if fieldName == common.COL_DST_PORT: obj[common.COL_DST_PORT] = flow[j] revObj[common.COL_SRC_PORT] = flow[j] if fieldName == common.COL_PROTO: obj[common.COL_PROTO] = common.getValueFromProto(flow[j]) revObj[common.COL_PROTO] = common.getValueFromProto(flow[j]) elif col[0] in self.srcDirectionMap: fieldName = self.srcDirectionMap[col[0]] obj[fieldName] = flow[j] elif col[0] in self.dstDirectionMap: fieldName = self.dstDirectionMap[col[0]] revObj[fieldName] = flow[j] elif col[0] == "stime": firstSwitched = float(flow[j]) obj[common.COL_FIRST_SWITCHED] = firstSwitched revObj[common.COL_FIRST_SWITCHED] = firstSwitched elif col[0] == "dur": obj[common.COL_LAST_SWITCHED] = firstSwitched + float(flow[j]) revObj[common.COL_LAST_SWITCHED] = firstSwitched + float(flow[j]) # check if the reverse flow start time is > 0. if it is zero, this means that we have a # one way flow (e.g. coming from a scan) that should not be imported into the database if common.COL_FIRST_SWITCHED in revObj and revObj[common.COL_FIRST_SWITCHED] > 0: self.prevFlow = revObj print obj return obj
def extract_mongo_query_params(): # construct query limit = 0 if "limit" in request.GET: try: limit = int(request.GET["limit"]) except ValueError: raise HTTPError(output="Param limit has to be an integer.") if limit < 0: limit = 0 fields = None if "fields" in request.GET: fields = request.GET["fields"].strip() fields = map(lambda v: v.strip(), fields.split(",")) sort = None if "sort" in request.GET: sort = request.GET["sort"].strip() sort = map(lambda v: v.strip(), sort.split(",")) for i in range(0, len(sort)): field = sort[i].split(" ") order = 1 if field[-1].lower() == "asc": field.pop() elif field[-1].lower() == "desc": order = -1 field.pop() field = " ".join(field) sort[i] = (field, order) count = False if "count" in request.GET: count = True # get query params start_bucket = 0 if "start_bucket" in request.GET: try: start_bucket = int(request.GET["start_bucket"]) except ValueError: raise HTTPError(output="Param start_bucket has to be an integer.") if start_bucket < 0: start_bucket = 0 end_bucket = sys.maxint if "end_bucket" in request.GET: try: end_bucket = int(request.GET["end_bucket"]) except ValueError: raise HTTPError(output="Param end_bucket has to be an integer.") if end_bucket < 0: end_bucket = 0 # the bucket resolution to query (number of buckets) resolution = 1 if "resolution" in request.GET: try: resolution = int(request.GET["resolution"]) except ValueError: raise HTTPError(output="Param resolution has to be an integer.") if resolution < 1: resolution = 1 # or set the bucket size directly bucket_size = None if "bucket_size" in request.GET: try: bucket_size = int(request.GET["bucket_size"]) except ValueError: raise HTTPError(output="Param bucket_size has to be an integer.") if bucket_size not in config.flow_bucket_sizes: raise HTTPError(output="This bucket size is not available.") # biflow aggregation # This simply removes the difference between srcIP and dstIP # (The smaller ip will always be the srcIP) biflow = False if "biflow" in request.GET: biflow = True # protocol filter include_protos = [] if "include_protos" in request.GET: include_protos = request.GET["include_protos"].strip() include_protos = map(lambda v: common.getValueFromProto(v.strip()), include_protos.split(",")) exclude_protos = [] if "exclude_protos" in request.GET: exclude_protos = request.GET["exclude_protos"].strip() exclude_protos = map(lambda v: common.getValueFromProto(v.strip()), exclude_protos.split(",")) # port filter include_ports = [] if "include_ports" in request.GET: include_ports = request.GET["include_ports"].strip() try: include_ports = map(lambda v: int(v.strip()), include_ports.split(",")) except ValueError: raise HTTPError(output="Ports have to be integers.") exclude_ports = [] if "exclude_ports" in request.GET: exclude_ports = request.GET["exclude_ports"].strip() try: exclude_ports = map(lambda v: int(v.strip()), exclude_ports.split(",")) except ValueError: raise HTTPError(output="Ports have to be integers.") # ip filter include_ips = [] if "include_ips" in request.GET: include_ips = request.GET["include_ips"].strip() include_ips = map(lambda v: int(v.strip()), include_ips.split(",")) exclude_ips = [] if "exclude_ips" in request.GET: exclude_ips = request.GET["exclude_ips"].strip() exclude_ips = map(lambda v: int(v.strip()), exclude_ips.split(",")) # get buckets and aggregate if bucket_size == None: bucket_size = db.getBucketSize(start_bucket, end_bucket, resolution) # only stated fields will be available, all others will be aggregated toghether # filter for known aggregation values #if fields != None: # fields = [v for v in fields if v in config.flow_aggr_values] black_others = False if "black_others" in request.GET: black_others = True aggregate = [] if "aggregate" in request.GET: aggregate = request.GET["aggregate"].strip() aggregate = map(lambda v: v.strip(), aggregate.split(",")) result = {} result["fields"] = fields result["sort"] = sort result["limit"] = limit result["count"] = count result["start_bucket"] = start_bucket result["end_bucket"] = end_bucket result["resolution"] = resolution result["bucket_size"] = bucket_size result["biflow"] = biflow result["include_ports"] = include_ports result["exclude_ports"] = exclude_ports result["include_ips"] = include_ips result["exclude_ips"] = exclude_ips result["include_protos"] = include_protos result["exclude_protos"] = exclude_protos result["batch_size"] = 1000 result["aggregate"] = aggregate result["black_others"] = black_others return result
def extract_mongo_query_params(): # construct query limit = 0 if "limit" in request.GET: try: limit = int(request.GET["limit"]) except ValueError: raise HTTPError(output="Param limit has to be an integer.") if limit < 0: limit = 0 fields = None if "fields" in request.GET: fields = request.GET["fields"].strip() fields = map(lambda v: v.strip(), fields.split(",")) sort = None if "sort" in request.GET: sort = request.GET["sort"].strip() sort = map(lambda v: v.strip(), sort.split(",")) for i in range(0, len(sort)): field = sort[i].split(" ") order = 1 if field[-1].lower() == "asc": field.pop() elif field[-1].lower() == "desc": order = -1 field.pop() field = " ".join(field) sort[i] = (field, order) count = False if "count" in request.GET: count = True # get query params start_bucket = 0 if "start_bucket" in request.GET: try: start_bucket = int(request.GET["start_bucket"]) except ValueError: raise HTTPError(output="Param start_bucket has to be an integer.") if start_bucket < 0: start_bucket = 0 end_bucket = sys.maxint if "end_bucket" in request.GET: try: end_bucket = int(request.GET["end_bucket"]) except ValueError: raise HTTPError(output="Param end_bucket has to be an integer.") if end_bucket < 0: end_bucket = 0 # the bucket resolution to query (number of buckets) resolution = 1 if "resolution" in request.GET: try: resolution = int(request.GET["resolution"]) except ValueError: raise HTTPError(output="Param resolution has to be an integer.") if resolution < 1: resolution = 1 # or set the bucket size directly bucket_size = None if "bucket_size" in request.GET: try: bucket_size = int(request.GET["bucket_size"]) except ValueError: raise HTTPError(output="Param bucket_size has to be an integer.") if bucket_size not in config.flow_bucket_sizes: raise HTTPError(output="This bucket size is not available.") # biflow aggregation # This simply removes the difference between srcIP and dstIP # (The smaller ip will always be the srcIP) biflow = False if "biflow" in request.GET: biflow = True # protocol filter include_protos = [] if "include_protos" in request.GET: include_protos = request.GET["include_protos"].strip() include_protos = map(lambda v: common.getValueFromProto(v.strip()), include_protos.split(",")) exclude_protos = [] if "exclude_protos" in request.GET: exclude_protos = request.GET["exclude_protos"].strip() exclude_protos = map(lambda v: common.getValueFromProto(v.strip()), exclude_protos.split(",")) # port filter include_ports = [] if "include_ports" in request.GET: include_ports = request.GET["include_ports"].strip() try: include_ports = map(lambda v: int(v.strip()), include_ports.split(",")) except ValueError: raise HTTPError(output="Ports have to be integers.") exclude_ports = [] if "exclude_ports" in request.GET: exclude_ports = request.GET["exclude_ports"].strip() try: exclude_ports = map(lambda v: int(v.strip()), exclude_ports.split(",")) except ValueError: raise HTTPError(output="Ports have to be integers.") # ip filter include_ips = [] if "include_ips" in request.GET: include_ips = request.GET["include_ips"].strip() include_ips = map(lambda v: int(v.strip()), include_ips.split(",")) exclude_ips = [] if "exclude_ips" in request.GET: exclude_ips = request.GET["exclude_ips"].strip() exclude_ips = map(lambda v: int(v.strip()), exclude_ips.split(",")) # get buckets and aggregate if bucket_size == None: bucket_size = db.getBucketSize(start_bucket, end_bucket, resolution) # only stated fields will be available, all others will be aggregated toghether # filter for known aggregation values #if fields != None: # fields = [v for v in fields if v in config.flow_aggr_values] black_others = False if "black_others" in request.GET: black_others = True aggregate = [] if "aggregate" in request.GET: aggregate = request.GET["aggregate"].strip() aggregate = map(lambda v: v.strip(), aggregate.split(",")) result = {} result["fields"] = fields print "Fields: " + str(fields) result["sort"] = sort result["limit"] = limit result["count"] = count result["start_bucket"] = start_bucket result["end_bucket"] = end_bucket result["resolution"] = resolution result["bucket_size"] = bucket_size result["biflow"] = biflow result["include_ports"] = include_ports result["exclude_ports"] = exclude_ports result["include_ips"] = include_ips result["exclude_ips"] = exclude_ips result["include_protos"] = include_protos result["exclude_protos"] = exclude_protos result["batch_size"] = 1000 result["aggregate"] = aggregate result["black_others"] = black_others return result
def get_next_flow(self): if self.prevFlow: ret = self.prevFlow self.prevFlow = None return ret flow = self.c.fetchone() if flow == None: if len(self.tables) == 0: return None table = self.tables.pop() print "Importing table ", table, "..." self.c.execute("SELECT * FROM " + table + " ORDER BY stime ASC") return self.get_next_flow() obj = dict() revObj = dict() firstSwitched = None for j, col in enumerate(self.c.description): print col[0] if col[0] in self.columnmap: fieldName = self.columnmap[col[0]] print "fieldName: ", fieldName # argus tables contain reverse flow information # create a revObj which matches the reverse flow if fieldName == common.COL_SRC_IP: ip = ip2int(flow[j]) obj[common.COL_SRC_IP] = ip revObj[common.COL_DST_IP] = ip if fieldName == common.COL_DST_IP: ip = ip2int(flow[j]) obj[common.COL_DST_IP] = ip revObj[common.COL_SRC_IP] = ip if fieldName == common.COL_SRC_PORT: obj[common.COL_SRC_PORT] = flow[j] revObj[common.COL_DST_PORT] = flow[j] if fieldName == common.COL_DST_PORT: obj[common.COL_DST_PORT] = flow[j] revObj[common.COL_SRC_PORT] = flow[j] if fieldName == common.COL_PROTO: obj[common.COL_PROTO] = common.getValueFromProto(flow[j]) revObj[common.COL_PROTO] = common.getValueFromProto( flow[j]) elif col[0] in self.srcDirectionMap: fieldName = self.srcDirectionMap[col[0]] obj[fieldName] = flow[j] elif col[0] in self.dstDirectionMap: fieldName = self.dstDirectionMap[col[0]] revObj[fieldName] = flow[j] elif col[0] == "stime": firstSwitched = float(flow[j]) obj[common.COL_FIRST_SWITCHED] = firstSwitched revObj[common.COL_FIRST_SWITCHED] = firstSwitched elif col[0] == "dur": obj[common.COL_LAST_SWITCHED] = firstSwitched + float(flow[j]) revObj[common.COL_LAST_SWITCHED] = firstSwitched + float( flow[j]) # check if the reverse flow start time is > 0. if it is zero, this means that we have a # one way flow (e.g. coming from a scan) that should not be imported into the database if common.COL_FIRST_SWITCHED in revObj and revObj[ common.COL_FIRST_SWITCHED] > 0: self.prevFlow = revObj print obj return obj