def calculateFlowsAndPlot(elevation, rain, resampleF):
# plot input rasters
plotRaster(elevation.getData(), "Original elevation (m)")
plotRaster(rain.getData(), "Rainfall")
resampledElevations = elevation.createWithIncreasedCellsize(resampleF)
################# step 1 find and plot the intial network #######
fr = flow.FlowRaster(resampledElevations)
plotFlowNetwork(resampledElevations,
fr,
"Network structure - before lakes",
plotLakes=False)
################Step 2 ######################################
plotExtractedData(fr, flow.FlowExtractor(),
"River flow rates - constant rain")
################# step 3 #######################################
#handle variable rainfall
fr.addRainfall(rain.getData())
plotExtractedData(fr, flow.FlowExtractor(),
"River flow rates - variable rainfall")
############# step 4 and step 5 #######################################
#handle lakes
fr.calculateLakes()
plotFlowNetwork(resampledElevations, fr,
"Network structure (i.e. watersheds) - with lakes")
plotExtractedData(fr, flow.LakeDepthExtractor(), "Lake depth")
plotExtractedData(fr, flow.FlowExtractor(),
"River flow rates - variable rainfall")
maxflow, maxnode = fr.getMaxFlow()
print("Maximum Flow is " + str(maxflow) + " mm per year at " +
str(maxnode))
def removeUseless(net, addLengths=False, ucFlow=None, removeReals=False):
'''
Will remove sites from the network with only two neighbors (1 up 1 down)
Will then merge the two flows together into one flow, keeping the length
of 1 of the deleted flows. (This is done to resolve MultiLineString) entries in
geoJSON files.
net [Network]: Network to operate on
Returns [None]
Notes: Do NOT run this method if you want to take loops into account as this
method will break the runtime if loops are present!
'''
i = 0
sf = ucFlow
while i in range(len(net.siteTable)):
sit = net.siteTable[i]
if sit.isReal and not removeReals:
i += 1
continue
cs = sit.connectedSites()
if len(cs) == 2 and cs[0][2].reachCode == cs[1][2].reachCode:
# This site is deletable
coni0 = cs[0]
coni1 = cs[1]
if addLengths:
newLen = coni0[2].length + coni1[2].length
else:
assert (coni0[2].length == coni1[2].length)
newLen = coni0[2].length
fl2Add = None
if coni0[1] == DOWNSTREAM_CON:
# coni0 is downstream of deletable site ('sit')
# coni1 is upstream
fl2Add = Flow(coni0[2].id, coni1[0], coni0[0], newLen,
coni0[2].reachCode)
else:
# coni0 is upstream of 'sit'
# coni1 is downstream
fl2Add = Flow(coni1[2].id, coni0[0], coni1[0], newLen,
coni1[2].reachCode)
if not sf is None:
if sf.downstreamSite == sit or sf.upstreamSite == sit:
sf = fl2Add
net.removeInvolvedFlows(sit)
coni0[0].removeInvolvedFlows(sit)
coni1[0].removeInvolvedFlows(sit)
net.siteTable.remove(sit)
coni0[0].flowsCon.append(fl2Add)
coni1[0].flowsCon.append(fl2Add)
net.flowTable.append(fl2Add)
if i > 0:
i -= 1
else:
i += 1
return sf
def calculateFlowsAndPlot(elevation, rain, resampleF):
"""Calculates all the flows and plots them
Input Parameter:
elevation – a Raster class object containing elevation
rain – a Raster class object containing rainfall
resampleF – an Integer
"""
# plot input rasters
plotRaster(elevation.getData(), "Original elevation (m)") #plot elevation
plotRaster(rain.getData(), "Rainfall") #plot rainfall
resampledElevations = elevation.createWithIncreasedCellsize(resampleF)
################# step 1 find and plot the intial network #######
fr = Flow.FlowRaster(resampledElevations) #create FlowRaster
plotFlowNetwork(fr,
fr,
"Task 1: Network structure - before lakes",
plotLakes=False) #plot flow raster
################Step 2 ######################################
plotExtractedData(fr, Flow.FlowExtractor(1),
"Task 2: River flow rates - constant rain")
################# step 3 #######################################
#handle variable rainfall
fr.addRainfall(rain.getData())
plotExtractedData(fr, Flow.FlowExtractor(),
"Task 3: River flow rates - variable rainfall")
############# step 4 and step 5 #######################################
# handle lakes
fr.calculateLakes()
plotFlowNetwork(
fr, fr, "Task 4: Network structure (i.e. watersheds) - with lakes")
plotExtractedData(fr, Flow.LakeDepthExtractor(), "Task 4: Lake depth")
plotExtractedData(fr, Flow.FlowExtractor(),
"Task 4: River flow rates - variable rainfall")
#TESTING
#this line tests if total raster outflow is equal to total rainfall on the raster
assert round(fr.getTotalFlow(), 2) == round(fr.getTotalRainfall(), 2)
############# step 5 #######################################
maxflow = fr.getMaximumFlow()
print("Task 5: Maximum Flow: {} mm, at FlowNode object: {}".format(
round(maxflow[0], 3), maxflow[1]))
def anomaly_detection(number):
flows = {}
with open('flows.csv', 'r') as csv_file:
reader = csv.reader(csv_file)
dictionary = dict(reader)
for key, value in dictionary.items():
values = value.split(',')
# print(values)
flow = Flow.flow(values[0], values[1], int(values[2]),
int(values[3]), float(values[4]))
flow.strConstructor(float(values[5]), float(values[6]),
float(values[7]), float(values[8]),
float(values[9]), float(values[10]))
flows[key] = flow
matrix = np.zeros([0, 6])
for i, j in flows.items():
temp = np.array([
j.byte_total_avg, j.packet_total_avg, j.avg_bps, j.avg_pps,
j.avg_time_diff, j.std_time_diff
])
matrix = np.vstack([matrix, temp])
#
#get Hash Ids
temp1 = np.array(list(flows.keys()))
#Standarization Anomaly Detection
start_time = time.time()
scaler = abs(StandardScaler().fit_transform(matrix))
#sum rows
temp2 = scaler.sum(axis=1)
scaler = np.insert(scaler, 0, temp1, axis=1)
scaler = np.insert(scaler, 7, temp2, axis=1)
scaler = scaler[scaler[:, 7].argsort()[::-1]]
#TODO add std check if the values are little or similar
print("Standarization indicates the Hash ID:", scaler[:number, 0])
print("Produced Standarziation Anomaly Detection in: %s seconds ---" %
(time.time() - start_time))
#Spearman Correlation Anomaly Detection
start_time = time.time()
_, p = spearmanr(matrix, axis=1)
temp2 = p.sum(axis=1)
output = np.vstack([temp1, temp2])
output = np.transpose(output)
output = output[output[:, 1].argsort()[::-1]]
print("Pearson Correlation indicates the Hash ID:", output[:number, 0])
print("Produced Pearson Correlation Anomaly Detection in: %s seconds ---" %
(time.time() - start_time))
#Relative Entropy
start_time = time.time()
temp2 = abs(
scale(entropy(np.transpose(matrix)), with_mean=True, with_std=True))
output = np.vstack([temp1, temp2])
output = np.transpose(output)
output = output[output[:, 1].argsort()[::-1]]
print("Relative Entropy indicates the Hash ID:", output[:number, 0])
print("Relative Entropy Anomaly Detection in: %s seconds ---" %
(time.time() - start_time))
def plotFlowNetwork(originalRaster, flowRaster, title="", plotLakes=True):
"""Plots a flow network
Input Parameter:
originalRaster – a Raster object
flowRaster – a FlowRaster object
title – plot title, a string
plotLake – binary variable stating if lakes should be plotted
True if lakes should be plotted
False if lakes should be ignored
"""
print("\n\n{}".format(title))
mp.imshow(originalRaster.extractValues(Flow.ElevationExtractor()))
mp.colorbar()
colouri = -1
colours = [
"black", "red", "magenta", "yellow", "green", "cyan", "white",
"orange", "grey", "brown"
]
for i in range(flowRaster.getRows()):
for j in range(flowRaster.getCols()):
node = flowRaster._data[i, j]
if (node.getPitFlag()): # dealing with a pit
mp.scatter(node.get_x(), node.get_y(), color="red")
colouri += 1
plotstreams(node, colours[colouri % len(colours)])
if (plotLakes and node.getLakeDepth() >
0): #if lakedepth is zero, it is not a lake
mp.scatter(node.get_x(), node.get_y(), color="blue")
mp.show()
def runGenerateSuperFlows(flow_data_dir, super_flow_data_dir, flowgap):
#TIMEGAP IN SECONDS
csvfiles = getCSVFiles(flow_data_dir)
#print csvfiles
flowdata = []
for filename in csvfiles:
inputfile = open(filename)
data = [line.strip() for line in inputfile]
inputfile.close()
for eachline in data:
fields = eachline.split(',')
flowdata.append(SuperFlow.SuperFlow(fields))
print '\tNo. of flows to be processed: ' + str(len(flowdata))
flowdata = Flow.combineFlows(flowdata, flowgap)
print '\tSuperflows (Flows with flowgap = ' + str(
flowgap) + ' sec) : ' + str(len(flowdata))
outfile = open(super_flow_data_dir + str(flowgap) + '.csv', 'w')
to_write = []
for flow in flowdata:
to_write.append(
socket.inet_ntoa(flow.ip1) + ',' + socket.inet_ntoa(flow.ip2) +
',' + str(flow.getNoOfPackets()) + ',' + str(flow.getNoOfBytes()) +
',' + '%.6f' % flow.getInterArrivaltime() + ',' +
'%.6f' % flow.getStart() + ',' + '%.6f' % flow.getEnd() + ',' +
'%.6f' % flow.getDurationInSeconds())
outfile.write("\n".join(to_write))
outfile.close()
def initRobotFromCfg(runtimeID):
done = False
index = 1
robots = []
thisRobot = None
srcDict = CFGReader.ReadConfig("swarm.cfg", "src")
dstDict = CFGReader.ReadConfig("swarm.cfg", "dst")
if not srcDict:
sys.exit("No [src] found in swarm.cfg")
if not dstDict:
sys.exit("No [dst] found in swarm.cfg")
srcFID = srcDict["fid"]
srcLoc = Location(float(srcDict["x"]), float(srcDict["y"]),
float(srcDict["z"]))
srcNode = Robot("-1", srcLoc, srcFID, None, 6666, 6666, isSrc=True)
robots.append(srcNode)
dstFID = dstDict["fid"]
dstLoc = Location(float(dstDict["x"]), float(dstDict["y"]),
float(dstDict["z"]))
dstNode = Robot("-2", dstLoc, dstFID, None, 6666, 6666, isDest=True)
robots.append(dstNode)
while done == False:
section = "R" + str(index)
robotDict = CFGReader.ReadConfig("swarm.cfg", section)
if not robotDict:
done = True
break
rID = robotDict['rid']
fID = robotDict['fid']
x = float(robotDict['x'])
y = float(robotDict['y'])
z = float(robotDict['z'])
loc = Location(x, y, z)
newRobot = Robot(rID, loc, fID, None, 6666, 6666)
robots.append(newRobot)
if rID == runtimeID:
thisRobot = newRobot
index += 1
if (thisRobot == None):
sys.exit("No [R%s] found in swarm.cfg" % runtimeID)
thisRobot.all_robots["-1"] = srcLoc
thisRobot.all_robots["-2"] = dstLoc
currFlow = Flow.Flow(thisRobot.fid, robots, srcNode, dstNode)
thisRobot.setFlowAndGraph(currFlow, robots)
thisRobot.establishConnections()
return robots
def calculateFlowsAndPlot(elevation, rain, resampleF):
# plot input rasters
plotRaster(elevation.getData(), "Original elevation (m)")
plotRaster(rain.getData(), "Rainfall")
resampledElevations = elevation.createWithIncreasedCellsize(resampleF)
################# step 1 find and plot the intial network #######
'''
1. From Raster import Raster
2. Execute full code
'''
fr = flow.FlowRaster(resampledElevations)
plotFlowNetwork(elevation,
fr,
"Network structure - before lakes",
plotLakes=False)
################Step 2 ######################################
'''
Calculate flow volume
Resursively call each up-node and add one each time
Solution:
def getValue(self, node):
return node.numUpnodes()
'''
plotExtractedData(fr, flow.FlowExtractor(),
"River flow rates - constant rain")
################# step 3 #######################################
#handle variable rainfall
'''
addRainfall does not replace any values but adds
the rainfall values on top of the current _data attribute of every node
'''
fr.addRainfall(rain.getData())
plotExtractedData(fr,
flow.FlowExtractor(),
"River flow rates - variable rainfall",
isRainFall=True)
def calculateFlowsAndPlot(elevation, rain, resampleF):
"""
The main function to execute the project. Processes elevation and rain
data, producing a series of plots of elevations, flow networks, and lake
depths.
"""
# plot input rasters
plotRaster(elevation.getData(), "Original elevation (m)")
plotRaster(rain.getData(), "Rainfall")
resampledElevations = elevation.createWithIncreasedCellsize(resampleF)
################# step 1 find and plot the intial network #######
fr = flow.FlowRaster(resampledElevations)
plotFlowNetwork(elevation,
fr,
"Network structure - before lakes",
plotLakes=False)
################Step 2 ######################################
plotExtractedData(fr, flow.FlowExtractor(),
"River flow rates - constant rain")
################# step 3 #######################################
#handle variable rainfall
fr.addRainfall(rain.getData())
plotExtractedData(fr, flow.FlowExtractor(),
"River flow rates - variable rainfall")
############# step 4 and step 5 #######################################
# handle lakes
fr.calculateLakes()
plotFlowNetwork(elevation, fr,
"Network structure (i.e. watersheds) - with lakes")
plotExtractedData(fr, flow.LakeDepthExtractor(), "Lake depth")
plotExtractedData(fr, flow.FlowExtractor(),
"River flow rates - variable rainfall")
def calculateFlowsAndPlot(elevation,
rain,
resampleF,
tasks=['1', '2', '3', '4', '5']):
"""
The main function to execute the project. Processes elevation and rain
data, producing a series of plots of elevations, flow networks, and lake
depths.
"""
# plot input rasters
plotRaster(elevation.getData(), "Original elevation (m)")
plotRaster(rain.getData(), "Rainfall")
resampledElevations = elevation.createWithIncreasedCellsize(resampleF)
################# step 1 find and plot the intial network #######
fr = flow.FlowRaster(resampledElevations)
plotFlowNetwork(elevation,
fr,
"Network structure - before lakes",
plotLakes=False)
################Step 2 ######################################
plotExtractedData(fr, flow.FlowExtractor(),
"River flow rates - constant rain")
################# step 3 #######################################
#handle variable rainfall
fr.addRainfall(rain.getData())
plotExtractedData(fr, flow.FlowExtractor(),
"River flow rates - variable rainfall")
############# step 4 and step 5 #######################################
# handle lakes
fr.calculateLakes()
plotFlowNetwork(elevation, fr,
"Network structure (i.e. watersheds) - with lakes")
plotExtractedData(fr, flow.LakeDepthExtractor(), "Lake depth")
plotExtractedData(fr, flow.FlowExtractor(),
"River flow rates - variable rainfall")
############# Answer questions for step 5 #############################
values = fr.extractValues(flow.FlowExtractor())
# Max
maxflow = np.max(values)
# Row
rowMax = np.where(values == maxflow)[0][0]
# Col
colMax = np.where(values == maxflow)[1][0]
# Print message
print("Maximum flow of\n\n{} found in node [{},{}]".format(
maxflow, rowMax, colMax))
def SetFlows(self, flowNums, flowInfoList, startTime):
self.flowNums = flowNums
self.jobStartTime = startTime
self.jobFinishTime = startTime
self.flows = [None]
# add each flow into flow list
for i in range(flowNums):
f = Flow()
f.startId = flowInfoList[i][0]
f.endId = flowInfoList[i][1]
f.remainSize = flowInfoList[i][2]
f.startTime = startTime
f.finishTime = startTime
self.flows.append(f)
self.curFlowId = 0
def fetchFlow(self):
fid = Helper.ReadConfig("model/flow.cfg", "Flow Information")['id']
sid = Helper.ReadConfig("model/flow.cfg", "Flow Information")['src']
s_loc = Location(
Helper.ReadConfig("model/location_e.cfg", sid)['x'],
Helper.ReadConfig("model/location_e.cfg", sid)['y'],
Helper.ReadConfig("model/location_e.cfg", sid)['z'])
src = Host.Host(sid, s_loc)
did = Helper.ReadConfig("model/flow.cfg", "Flow Information")['dest']
d_loc = Location(
Helper.ReadConfig("model/location_e.cfg", did)['x'],
Helper.ReadConfig("model/location_e.cfg", did)['y'],
Helper.ReadConfig("model/location_e.cfg", did)['z'])
dest = Host.Host(did, d_loc)
flow = Flow.Flow(fid, src, dest)
self.setFlow(flow)
def SetFlows(self, flowNums, flowInfoList, startTime):
self.flowNums = flowNums
self.jobStartTime = startTime
self.jobFinishTime = startTime
self.flows = [None]
# add each flow into flow list
for i in range(flowNums):
f = Flow()
f.startId = flowInfoList[i][0]
f.endId = flowInfoList[i][1]
f.remainSize = flowInfoList[i][2]
f.startTime = startTime
f.finishTime = startTime
self.flows.append(f)
self.curFlowId = 0
def SetFlows(self, flowNums, flowInfoList, startTime):
"""
flowInfList contain <flowNums> element and each element is formed as:
(startId, endId, flowSize)
"""
self.flowNums = flowNums
self.jobStartTime = startTime
self.jobFinishTime = startTime
self.flows = [None]
# add each flow into flow list
for i in range(flowNums):
f = Flow()
f.startId = flowInfoList[i][0]
f.endId = flowInfoList[i][1]
f.remainSize = flowInfoList[i][2]
f.startTime = startTime
f.finishTime = startTime
self.flows.append(f)
self.curFlowId = 0
def SetFlows(self, flowNums, flowInfoList, startTime):
"""
flowInfList contain <flowNums> element and each element is formed as:
(startId, endId, flowSize)
"""
self.flowNums = flowNums
self.jobStartTime = startTime
self.jobFinishTime = startTime
self.flows = [None]
# add each flow into flow list
for i in range(flowNums):
f = Flow()
f.startId = flowInfoList[i][0]
f.endId = flowInfoList[i][1]
f.remainSize = flowInfoList[i][2]
f.startTime = startTime
f.finishTime = startTime
self.flows.append(f)
self.curFlowId = 0
csvfiles = getCSVFiles(FLOWDATADIR)
print csvfiles
flowdata = []
for filename in csvfiles:
inputfile = open(filename)
data = [line.strip() for line in inputfile]
inputfile.close()
for eachline in data:
fields = eachline.split(',')
flowdata.append(SuperFlow.SuperFlow(fields))
print 'No. of flows to be processed: ' + str(len(flowdata))
while starttime <= endtime:
flowdata = Flow.combineFlows(flowdata, starttime)
print 'No. of flows with timegap = ' + str(starttime) + 'sec : ' + str(len(flowdata))
outfile = open(SUPERFLOWDATADIR + str(starttime) + '.csv', 'w')
for flow in flowdata:
outfile.write(
socket.inet_ntoa(flow.ip1) + ',' +
socket.inet_ntoa(flow.ip2) + ',' +
str(flow.prot) + ',' +
str(flow.getNoOfPackets()) + ',' +
str(flow.getNoOfBytes()) + ',' +
'%.6f'%flow.getInterArrivaltime() + ',' +
'%.6f'%flow.getStart() + ',' +
'%.6f'%flow.getEnd() + ',' +
'%.6f'%flow.getDurationInSeconds() + '\n')
outfile.close()
def ProcessPcap(filename, output_folder="", flow_length=10,
label=None, category=None, Timeout=30, ActivityTimeout=5,
TorDetect=False, TorPickle=None,
output_sizes_stat=False, pickle_flows=False,
filter_packets_eq_less_than=-1):
df_sizes = pandas.DataFrame()
list_sizes = []
try:
input_pcap = PacketCapture.PacketCapture(filename)
except (OSError, FileNotFoundError, FileExistsError) as e:
print(e)
print("Error while accessing file %s." % (filename))
exit(1)
Flows = []
time.sleep(1)
if (TorDetect):
tornodes = NodeGuard.NodeGuard()
if(TorPickle):
tornodes.loadNodesFromPickle(TorPickle)
else:
tornodes.loadNodes()
tor_traffic = detectTorIP(input_pcap.streamslist, tornodes)
for flow in tor_traffic:
subflows = detectFlows(input_pcap.streams[(flow[0], flow[1], flow[2], flow[3])], Timeout=Timeout)
# set up desttination (Tor EntryNode) and source
flow_dest = flow[4]
flow_source = None
# from tcpdump point of view dest is Tor node
if flow_dest == flow[2]:
flow_source = flow[0]
else:
# then tcpdump saw Tor node as source and we don't like it..
flow_source = flow[2]
for f in subflows:
# process flows with at least 10 packets
if len(f) > flow_length:
ff = Flow.Flow(FlowTimeout=Timeout, ActivityTimeout=ActivityTimeout, \
ipsrc=flow_source, ipdst=flow_dest)
ff.loadPackets(f)
if (filter_packets_eq_less_than != -1):
ff.filter_packets_eq_less_than(filter_packets_eq_less_than)
Flows.append(ff)
if output_sizes_stat:
list_sizes.append(ff.packets_size_to_pandas())
# No TOR detection
else:
for flow in input_pcap.streams:
subflows = detectFlows(input_pcap.streams[flow], Timeout=Timeout)
for f in subflows:
if len(f) > flow_length:
ff = Flow.Flow(FlowTimeout=Timeout, ActivityTimeout=ActivityTimeout)
ff.loadPackets(f)
if (filter_packets_eq_less_than != -1):
ff.filter_packets_eq_less_than(filter_packets_eq_less_than)
Flows.append(ff)
if output_sizes_stat:
list_sizes.append(ff.packets_size_to_pandas())
print("Flows extracted: %d" %(len(Flows)) )
name = os.path.basename(filename)
exportflowsToCsv(Flows, output_folder + name + "_flows_" + str(Timeout) + "_" + \
str(ActivityTimeout) + ".csv", label=label, category=category)
if output_sizes_stat and len(list_sizes) > 0:
df_sizes = reduce(partial(pandas.DataFrame.add, fill_value=0), list_sizes)
df_sizes.to_csv(output_folder + name + "_flows_" + str(Timeout) + "_" + str(ActivityTimeout) + "_size_stats.csv", sep=",", encoding='utf-8', index=True)
if pickle_flows:
with open(output_folder + name + "_flows_" + str(Timeout) + "_" + str(ActivityTimeout) + ".pkl", 'wb' ) as f:
pickle.dump(Flows, f, 0)
return Flows
I = sourcepanel.solveGeometricIntegrals(XC, YC, panels, phi, beta, XB, YB)
np.fill_diagonal(I, np.pi)
b = -2 * np.pi * Vinf * np.cos(beta)
lamda = linalg.solve(I, b)
# grid for flow computations
nxx, nyy = 100, 100
# lon and lat rep
x_flow = np.linspace(minlon, maxlon, nxx)
y_flow = np.linspace(minlat, maxlat, nyy)
X, Y = np.meshgrid(x_flow, y_flow)
# sink located at Trondheim
lamda_trond = 0.1 # sink strength
trond_flow = Flow.Velocity(lamda_trond, trond_lon, trond_lat)
vx_sink, vy_sink = trond_flow.sink(X, Y)
vx = np.zeros((nxx, nyy))
vy = np.zeros((nxx, nyy))
# compute velocites at every grid cell (XP, YP)
for i in range(nxx):
for j in range(nyy):
XGeom, YGeom = sourcepoint.solvePointGeometry(X[i, j], Y[i, j], panels,
phi, XB, YB)
if m.is_land(X[i, j], Y[i, j]) == True:
vx[i, j] = 0
vy[i, j] = 0
else:
vx[i, j] = Vinf * np.cos(AoA) + np.dot(
def generate(self, samplesQuantity, obstaclesProb=0.3, dataAugmentation=True):
"""Generate a list of given quantity of samples."""
quantitySamplesGenerated = 0
# samples = np.empty(shape=(samplesQuantity,), dtype=object)
samples = np.empty(shape=(samplesQuantity, 1, self.size, self.size), dtype=float)
labels = np.empty(shape=(samplesQuantity,), dtype="byte")
# for _ in range(worldsQuantity):
# while quantitySamplesGenerated < samplesQuantity:
def getValidRandomGoal(world):
while True:
goal = world.cells[randint(0, len(world.cells)-1)]
if goal.reachable:
return goal
worldsQuantity = 0
while True:
# print("new world")
# generate an area with obstacles
world = GridWorld.GridWorld(self.size, self.size)
obstaclesQuantity = math.floor(obstaclesProb * world.getLength())
world.addRandomObstacles(obstaclesQuantity)
if worldsQuantity == 0:
print("Example of world:")
print(world.__str__(" "))
# for goal in world.cells:
# with getValidRandomGoal(world) as goal:
goal = getValidRandomGoal(world)
worldsQuantity += 1
if True:
# if quantitySamplesGenerated >= samplesQuantity:
# break
if goal.reachable: # for each possible goal location
solution = Flow.solve(world, goal) # solve the world
goalInSolution = solution.get(goal.col, goal.line)
print(Flow.enigmaAsStr(solution, goalInSolution, " "))
a = 0/0
for startCell in solution.cells: # create a sample for each reachable cell
if startCell.reachable and startCell != goalInSolution:
for line in range(solution.height):
for col in range(solution.width):
cell = solution.get(line, col)
if cell == goalInSolution:
# print("goal in solution", cell)
samples[quantitySamplesGenerated, 0, col, line] = Dataset.GreyScales.goal
elif cell == startCell:
# print("start in solution", startCell)
samples[quantitySamplesGenerated, 0, col, line] = Dataset.GreyScales.start
elif cell.reachable:
samples[quantitySamplesGenerated, 0, col, line] = Dataset.GreyScales.reachable
else:
samples[quantitySamplesGenerated, 0, col, line] = Dataset.GreyScales.obstacle
labels[quantitySamplesGenerated] = startCell.direction.value
# print("Added:", Flow.enigmaAsStr(solution, goalInSolution))
# print(samples[quantitySamplesGenerated, 0], " go ", Flow.Direction(labels[quantitySamplesGenerated]).name)
quantitySamplesGenerated += 1
if quantitySamplesGenerated == samplesQuantity:
return samples, labels
if quantitySamplesGenerated % math.ceil(samplesQuantity / 20) == 0:
print("\t Generating:", math.ceil((quantitySamplesGenerated/samplesQuantity)*100), "%. So far,", worldsQuantity, "worlds.")
# Set out: # Source -> start point # Sink -> end point # doublet -> island (none - traversable area) # Set up grid nx, ny = 30, 30 x = np.linspace(minlon, maxlon, nx) y = np.linspace(minlat, maxlat, ny) X, Y = np.meshgrid(x, y) # Create flow types source_str = 200 x_source = 9.31318 y_source = 63.604 flow0 = Flow.Velocity(source_str, x_source, y_source) u_source, v_source = flow0.source(X, Y) sink_str = 1e3 x_sink = 9.48137 y_sink = 63.7375 flow1 = Flow.Velocity(sink_str, x_sink, y_sink) u_sink, v_sink = flow1.sink(X, Y) # Trying to represent source and doublets as islands. Use correct strengths in for loop source_strs = [100, 200, 300, 400, 500, 750] doublet_strs = [0.01, 0.1, 0.2, 0.3, 0.5, 1] gs = gridspec.GridSpec(nrows=3, ncols=2) fig, ax = plt.subplots(3, 2, figsize=(10, 10), sharex=True, sharey=True)
def main():
print_detail = False
print_json = True
for iti in range(23):
if iti < 10:
continue
for jti in range(2):
if jti == 0:
hour = iti
minute = 0
shour = str(hour)
ehour = str(hour)
sminute = '00'
eminute = '30'
start = str(hour) + ':00:00'
end = str(hour) + ':30:00'
start_time = datetime.datetime(2017, 11, 16, hour, 0, 0)
stop_time = datetime.datetime(2017, 11, 16, hour, 30, 0)
start_time_server = datetime.datetime(2017, 11, 16, hour - 1,
58, 0)
stop_time_server = datetime.datetime(2017, 11, 16, hour, 28, 0)
else:
hour = iti
minute = 30
shour = str(hour)
ehour = str(hour + 1)
sminute = '30'
eminute = '00'
start = str(hour) + ':30:00'
end = str(hour + 1) + ':00:00'
start_time = datetime.datetime(2017, 11, 16, hour, 30, 0)
stop_time = datetime.datetime(2017, 11, 16, hour + 1, 0, 0)
start_time_server = datetime.datetime(2017, 11, 16, hour, 28,
0)
stop_time_server = datetime.datetime(2017, 11, 16, hour, 58, 0)
fout_string = 'out.' + start
#d = pynfdump.Dumper('/data2/datasource/',profile='16/',sources=['nfcapd.201711161000','nfcapd.201711161005'])
d = pynfdump.Dumper()
# ponce = 0
# d.set_where(start=None,end=None,filename='/data2/datasource/16/nfcapd.201711161000')
# dstring = '/data2/datasource/16/'
# for i in range(6):
# nstr = str(i*5)
# if len(nstr) < 2:
# nstr = '0' + nstr
# dstring += 'nfcapd.2017111610' + nstr + ':'
#
# dstring = dstring[:-1]
dfiles = '/data2/datasource/16/nfcapd.20171116' + shour + sminute + ':nfcapd.20171116' + ehour + eminute
d.set_where(start=None, end=None, dirfiles=dfiles)
# d.set_where(start=None,end=None,filename='/data2/datasource/16/nfcapd.201711161000')
records = d.search('proto icmp and host 166.111.8.241')
fin = open('/data2/datasource/ICMP/time/' + start + '.txt', 'r')
ip_dict = {}
agg_dict = {}
print dfiles, start
for line in fin.readlines():
items = line.split('#')
ip = items[0].strip()
string = items[1].split(',')
if ip not in ip_dict:
ip_dict[ip] = {
'IN': [],
'OUT': [],
'ICMP': [string[1], string[2], string[3].strip()]
}
agg_dict[ip]= {\
'IN':[],\
'OUT':[],\
'ICMP':[string[1],string[2],string[3].strip()],\
'time_itv':-3,\
'flow_time_error_in':[0,0],\
'flow_time_error_out':[0,0],\
'first_time_error':False,\
'last_time_error':False,\
'itvf':-1,\
'itvl':-1}
# -3: initialized but not assigned; -2 host unreachable -1: first_time_error(first time out is later than in) and last_time_error
fin.close()
for r in records:
first = r['first']
last = r['last']
msec_first = r['msec_first']
msec_last = r['msec_last']
srcip = str(r['srcip'])
dstip = str(r['dstip'])
srcport = r['srcport']
dstport = r['dstport']
packets = r['packets']
tbytes = r['bytes']
srcip_prefix = srcip.split('.')[0] + '.' + srcip.split('.')[1]
dstip_prefix = dstip.split('.')[0] + '.' + dstip.split('.')[1]
# first_time = first + datetime.timedelta(microseconds = msec_first)
# last_time = last + datetime.timedelta(microseconds = msec_last)
if dstip in ip_dict:
key = dstip
flag = 'OUT'
elif srcip in ip_dict:
key = srcip
flag = 'IN'
else:
continue
# prtstr = srcip + ' => ' + dstip + ':' + str(dstport) + ' '+ first_time.strftime("%Y-%m-%d %H:%M:%S.%f") + ' ' + last_time.strftime("%Y-%m-%d %H:%M:%S.%f")+ ' ' + str(packets) + ' ' + str(tbytes)
ip_dict[key][flag].append(
Flow.IcmpFlow(srcip, srcport, dstip, dstport, first,
msec_first, last, msec_last, packets, tbytes,
flag))
for key in ip_dict:
if len(ip_dict[key]['IN']) == 0:
agg_dict[key]['time_itv'] = -2
continue
if len(ip_dict[key]['OUT']) == 0:
agg_dict[key]['time_itv'] = -2
continue
#Here to gathering the flows
ip = key
srcip, srcport = ip_dict[key]['OUT'][0].get_srcip()
dstip, dstport = ip_dict[key]['OUT'][0].get_dstip()
lgin = len(ip_dict[key]['IN'])
lgout = len(ip_dict[key]['OUT'])
real_flow_dict = {'IN': [], 'OUT': []}
real_flow_list = []
real_flow_dict['IN'] = [
Flow.FlowGroup(srcip, srcport, dstip, dstport)
]
real_flow_dict['OUT'] = [
Flow.FlowGroup(srcip, srcport, dstip, dstport)
]
i = 0
for j in range(lgout):
agg_dict[key]['flow_time_error_out'][1] += 1
if ip_dict[key]['OUT'][j].get_first_time(
) > ip_dict[key]['OUT'][j].get_last_time():
agg_dict[key]['flow_time_error_out'][0] += 1
if real_flow_dict['OUT'][i].add_flow(
ip_dict[key]['OUT'][j]):
pass
else:
i += 1
real_flow_dict['OUT'].append(
Flow.FlowGroup(srcip, srcport, dstip, dstport))
real_flow_dict['OUT'][i].add_flow(
ip_dict[key]['OUT'][j])
# except TypeError:
# print key,ip_dict[key]
# print j,ip_dict[key]['OUT'][j]
# exit()
# for item in ip_dict[key]:
# print item.print_string()
# print
# for item in real_flow_dict[key]:
# print item.display_string():
# exit()
i = 0
for j in range(lgin):
agg_dict[key]['flow_time_error_in'][1] += 1
if ip_dict[key]['IN'][j].get_first_time(
) > ip_dict[key]['IN'][j].get_last_time():
agg_dict[key]['flow_time_error_in'][0] += 1
if real_flow_dict['IN'][i].add_flow(ip_dict[key]['IN'][j]):
pass
else:
i += 1
real_flow_dict['IN'].append(
Flow.FlowGroup(srcip, srcport, dstip, dstport))
real_flow_dict['IN'][i].add_flow(ip_dict[key]['IN'][j])
for item in real_flow_dict['OUT']:
agg_dict[key]['OUT'].append(item)
for item in real_flow_dict['IN']:
agg_dict[key]['IN'].append(item)
lenin = len(real_flow_dict['IN'])
for item in real_flow_dict['OUT']:
for i in range(lenin):
if datetime.timedelta(0, -10, 0) < item.get_first_time(
) - real_flow_dict['IN'][i].get_first_time(
) < datetime.timedelta(0, 10, 0):
real_flow_list.append({
'IN': real_flow_dict['IN'][i],
'OUT': item
})
break
if i == range(lenin):
pass
for item in real_flow_list:
if agg_dict[key]['time_itv'] in [-2, -4]:
break
elif agg_dict[key]['time_itv'] != -3:
if item['OUT'].get_first_time() > stop_time_server:
break
else:
agg_dict[key]['time_itv'] = -4
time_itvf = item['IN'].get_first_time(
) - item['OUT'].get_first_time()
time_itvl = item['IN'].get_last_time(
) - item['OUT'].get_last_time()
dtzero = datetime.timedelta(0, 0, 0)
if time_itvf > dtzero:
agg_dict[key][
'itvf'] = time_itvf.seconds * 1000 + time_itvf.microseconds / 1000
else:
agg_dict[key]['itvf'] = -1
if time_itvl > dtzero:
agg_dict[key][
'itvl'] = time_itvl.seconds * 1000 + time_itvl.microseconds / 1000
else:
agg_dict[key]['itvl'] = -1
if time_itvf < dtzero:
agg_dict[key]['first_time_error'] = True
if time_itvl < dtzero:
agg_dict[key]['time_itv'] = -1
agg_dict[key]['last_time_error'] = True
else:
agg_dict[key]['time_itv'] = time_itvl
else:
agg_dict[key]['time_itv'] = time_itvf
if time_itvl < dtzero:
agg_dict[key]['last_time_error'] = True
elif time_itvf > datetime.timedelta(0, 0, 100000):
if time_itvl < datetime.timedelta(0, 0, 100000):
agg_dict[key]['time_itv'] = time_itvl
elif datetime.timedelta(
0, 0, -50000
) < time_itvl - time_itvf < datetime.timedelta(
0, 0, 50000):
agg_dict[key]['time_itv'] = (time_itvf +
time_itvl) / 2
if print_detail:
p_detail(ip_dict, agg_dict, fout_string)
if print_json:
p_json(ip_dict, agg_dict, fout_string)
second = int(hms[2])
msec_last = int(l.split('.')[1][:-3])
last = datetime.datetime(year,month,day,hour,minite,second)
packets = int(r[4])
tbytes = int(r[5])
if dstip in ip_dict:
flag = 'OUT'
elif srcip in ip_dict:
flag = 'IN'
else:
continue
ip_dict[key][flag].append(Flow.IcmpFlow(srcip,srcport,dstip,dstport,first,msec_first,last,msec_last,packets,tbytes,flag))
for key in ip_dict:
if len(ip_dict[key]['IN']) == 0:
agg_dict[key]['time_itv'] = -2
continue
if len(ip_dict[key]['OUT']) == 0:
agg_dict[key]['time_itv'] = -2
continue
#Here to gathering the flows
ip = key
srcip, srcport = ip_dict[key]['OUT'][0].get_srcip()
dstip, dstport = ip_dict[key]['OUT'][0].get_dstip()
lgin = len(ip_dict[key]['IN'])
lgout = len(ip_dict[key]['OUT'])
real_flow_dict = {'IN':[],'OUT':[]}
def training():
counterIncoherentFlows = 0
timer = config['QueryTime']
#Retrieve thresholds from previous run.
retrieveThresholds()
try:
while True:
r = requests.get(
'http://127.0.0.1:2401/tennison/ipfix/query/ipfix-threshold-app'
)
logging.info('Quering IPfix messages')
# Debug comments
# pprint(r.json())
# Pass to ordered dict, noticed that keeps order in iterations
data = OrderedDict(r.json())
for _, j in data.items():
try:
# IPs
destinationIPv4 = j['destinationIPv4Address']
sourceIPv4Address = j['sourceIPv4Address']
# Ports
destinationTransportPort = j['destinationTransportPort']
sourceTransportPort = j['sourceTransportPort']
# Time
try:
flowStartMilliseconds = (mktime(datetime.datetime.strptime(j['flowStartMilliseconds'], "%Y-%m-%dT%H:%M:%S.%f").timetuple())) +\
(datetime.datetime.strptime(j['flowStartMilliseconds'], "%Y-%m-%dT%H:%M:%S.%f").microsecond/1e6)
except ValueError:
flowStartMilliseconds = (mktime(
datetime.datetime.strptime(
j['flowStartMilliseconds'],
"%Y-%m-%dT%H:%M:%S").timetuple()))
try:
flowEndMilliseconds = (mktime(datetime.datetime.strptime(j['flowEndMilliseconds'],
"%Y-%m-%dT%H:%M:%S.%f").timetuple())) + \
(datetime.datetime.strptime(j['flowEndMilliseconds'], "%Y-%m-%dT%H:%M:%S.%f").microsecond / 1e6)
except ValueError:
flowEndMilliseconds = (mktime(
datetime.datetime.strptime(
j['flowEndMilliseconds'],
"%Y-%m-%dT%H:%M:%S").timetuple()))
# Bytes count
octetDeltaCount = j['octetDeltaCount']
# Packet count
packetDeltaCount = j['packetDeltaCount']
# Type of IPfix
subType = j['subtype']
# Count the variety of IPfix Subtypes
if j['subtype'] in countTypes:
countTypes[j['subtype']] += 1
else:
countTypes[j['subtype']] = 1
flow = Flow.flow(destinationIPv4, sourceIPv4Address,
destinationTransportPort,
sourceTransportPort,
flowStartMilliseconds,
octetDeltaCount) #,
global flows
if flow.__hash__() in flows:
flowPriori = flows.get(flow.__hash__())
flowPriori.update(flowEndMilliseconds, octetDeltaCount,
packetDeltaCount, subType)
flows[flow.__hash__()] = flowPriori
else:
flow.update(flowEndMilliseconds, octetDeltaCount,
packetDeltaCount, subType)
flows[flow.__hash__()] = flow
except KeyError:
counterIncoherentFlows += 1
continue
sleep(timer)
if config['type'] == 0:
updateThresholds()
elif config['type'] == 1:
updateThresholdsLLT()
else:
print("Please define an ML type")
except KeyboardInterrupt:
logging.info("The total number of flows are:" + str(len(flows)))
#TODO fix if needed
# with open('flows.csv', 'w') as csv_file:
# writer = csv.writer(csv_file, lineterminator='\n')
# for key, value in flows.items():
# writer.writerow([key, str(value)])
lst = []
#print ("Printing Json of flows")
for k, v in flows.items():
d = {}
d['FlowID'] = k
d['values'] = v.__dict__
lst.append(d)
flowfile = os.path.dirname(os.path.realpath(__file__)) + "/flows.json"
with open(flowfile, mode='w') as feedsjson:
json.dump(lst, feedsjson, indent=4)
#TODO fix this to string
logging.info("The ipfix subTypes are: ")
logging.info(json.dumps(countTypes))
logging.info(
"The number of Ipfix which miss values (Incoherent) is : " +
str(counterIncoherentFlows))
print("Written to file and finished now!")
params.printHelp()
exit()
elif len(sys.argv) != 2:
print("[E] One input parameters in json format in required")
params.printHelp()
exit()
# load parameters
params.load(sys.argv[1])
print("[I] parameters = %s" % (params))
db = MagicalDB.MagicalDB(params) # The flow database
db.parse() #parsing the input files
flow = Flow.Flow(db)
flow.run()
#Workaround for Pyinstaller. ref:https://github.com/pyinstaller/pyinstaller/issues/2820
if 0:
import UserList
import UserString
import UserDict
import itertools
import collections
import future.backports.misc
import commands
import base64
import __buildin__
import math
import reprlib
csvfiles = getCSVFiles(FLOWDATADIR)
print csvfiles
flowdata = []
for filename in csvfiles:
inputfile = open(filename)
data = [line.strip() for line in inputfile]
inputfile.close()
for eachline in data:
fields = eachline.split(',')
flowdata.append(SuperFlow.SuperFlow(fields))
print 'No. of flows to be processed: ' + str(len(flowdata))
while starttime <= endtime:
flowdata = Flow.combineFlows(flowdata, starttime)
print 'No. of flows with timegap = ' + str(starttime) + 'sec : ' + str(
len(flowdata))
outfile = open(SUPERFLOWDATADIR + str(starttime) + '.csv', 'w')
for flow in flowdata:
outfile.write(
socket.inet_ntoa(flow.ip1) + ',' + socket.inet_ntoa(flow.ip2) +
',' + str(flow.getNoOfPackets()) + ',' + str(flow.getNoOfBytes()) +
',' + '%.6f' % flow.getInterArrivaltime() + ',' +
'%.6f' % flow.getStart() + ',' + '%.6f' % flow.getEnd() + ',' +
'%.6f' % flow.getDurationInSeconds() + '\n')
outfile.close()
starttime += increment
#rand.seed(123456789)
rand.seed(rseed)
# Initialize storage lists
nodes = [] # List of nodes (initially empty)
vessels = [] # List of vessel segments (intially empty)
node_degree = [] # List of nodal degree (initially empty)
BCs = [] # List of boundary conditions (intially empty)
bifur_nodes = [] # List of bifurcation nodes (intially empty)
bifur = [] # List of bifurcation state (initially empty)
# Create the network
Geom.create_network(nodes, vessels, node_degree, bifur_nodes, bifur, BCs)
# Solve for flow
Flow.solve_for_flow(nodes, vessels, node_degree, BCs, bifur_nodes, bifur)
#Output.print_vessels(vessels)
# Initial polarity realignment
Network.realign_polarity(vessels)
#Output.print_cells(vessels)
# Print out the time step
print("Time step 0...".format(0, Nt))
# Initialize the ouput file
out = Output.OutFile(out_filename, nodes, vessels, node_degree, BCs, bifur_nodes, bifur)
# Step through time
def AddFlow(self, attri, sz):
f = Flow(attri, sz, self.IdxCnt)
self.FlowList.append(f)
self.IdxCnt += 1
pass
def waterfall(dump, dumpl, limit, fmt, cstats, stats):
# init modules
src = ArffReader.ArffReader(sys.stdin)
dst = sys.stdout
cs = Cascade()
st = Stats()
# setup proto filter
if hasattr(P, "skip"):
check = lambda gt: gt not in P.skip
elif hasattr(P, "select"):
check = lambda gt: gt in P.select
else:
check = lambda: True
# arff header?
if fmt == "arff":
src.printh(nodata=True, dst=dst)
dst.write("%% Multilevel Traffic Classifier: Waterfall approach\n")
dst.write("% waterfall_module: classifier that contributed the answer\n")
dst.write("% waterfall_proto: the identified protocol\n")
dst.write("@attribute waterfall_module string\n")
dst.write("@attribute waterfall_proto string\n\n")
dst.write("@data\n")
# go!
for d in src:
# check if gt is enabled
gt = d[P.gtcol]
if not check(gt): continue
# parse and classify
f = Flow(src, d, gt)
show = cs.classify(f, dump, dumpl)
# print it to the user?
if not show:
continue
if limit:
if f.isunk():
if "unk" not in limit: continue
elif "ans" not in limit:
if f.isok() and "ok" not in limit: continue
if f.iserr() and "err" not in limit: continue
# print and count
f.write(fmt, dst)
st.count(f)
# cascade stats?
if cstats:
if fmt != "none": print("")
print("Cascade statistics")
print("==================")
print(cs.get_stats())
# perf stats?
if stats:
if cstats: print("")
print("Performance statistics")
print("======================")
print(st.get_cm())
def search(ip_list):
middle_print = [
False, False, False, False
] #Prior one is middlefile, latter one is middlefile without those without IN or OUT
middle_process = True
if middle_print[0]:
fout0 = open('middlefile_total.txt', 'w')
if middle_print[1]:
fout1 = open('middlefile_only_O', 'w')
if middle_print[2]:
fout2 = open('middlefile_not_equal', 'w')
if middle_print[3]:
fout3 = open('middlefile_equal', 'w')
logger = logging.getLogger()
hdlr = logging.StreamHandler()
logger.addHandler(hdlr)
logger.setLevel(logging.WARNING)
InnerIP_list = [
'166.111', '59.66', '101.5', '101.6', '183.172', '183.173', '118.229',
'202.38', '202.112'
]
ipdict_dict = {}
datain = pynfdump.Dumper()
datain.set_where(
start=None,
end=None,
filename='/lrjapps/netflowdata/datasource/flowdata/nfcapd.201705171800'
)
query = ""
for ip in ip_list:
query = query + ' or host ' + ip
query = 'proto icmp and (' + query[4:] + ')'
logger.debug(query)
records = datain.search(query)
for r in records:
af = r['af']
first = r['first']
last = r['last']
msec_first = r['msec_first']
msec_last = r['msec_last']
proto = r['prot']
srcip = str(r['srcip'])
srcport = r['srcport']
dstip = str(r['dstip'])
dstport = r['dstport']
srcas = r['srcas']
dstas = r['dstas']
packets = r['packets']
tbytes = r['bytes']
srcip_prefix = srcip.split('.')[0] + '.' + srcip.split('.')[1]
dstip_prefix = dstip.split('.')[0] + '.' + dstip.split('.')[1]
if srcip in ip_list and dstip_prefix in InnerIP_list:
flag = 'IN'
elif dstip in ip_list and srcip_prefix in InnerIP_list:
flag = 'OUT'
elif srcip not in ip_list and dstip not in ip_list:
logging.error(srcip + ' ' + dscip +
' UNEXCEPTED source/destination address')
continue
else: #This means srcip or dstip is in ip_list, but dstip/srcip is located out of campus
continue
if flag == 'IN':
key = (dstip, srcip) #Tsinghua IP is [0] and target IP is [1]
else:
key = (srcip, dstip)
if key not in ipdict_dict:
ipdict_dict[key] = {'IN': [], 'OUT': []}
if flag == 'IN':
ipdict_dict[key]['IN'].append(
Flow.IcmpFlow(srcip, srcport, dstip, dstport, first,
msec_first, last, msec_last, packets, tbytes,
flag))
else:
ipdict_dict[key]['OUT'].append(
Flow.IcmpFlow(srcip, srcport, dstip, dstport, first,
msec_first, last, msec_last, packets, tbytes,
flag))
if middle_print[0]:
for key in ipdict_dict:
prtln = key[0] + ' => ' + key[1] + '\n' + '\tIN:\n'
fout0.write(prtln)
for item in ipdict_dict[key]['IN']:
fout0.write('\t\t' + item.display_string() + '\n')
fout0.write('\tOUT:\n')
for item in ipdict_dict[key]['OUT']:
fout0.write('\t\t' + item.display_string() + '\n')
fout0.write('\n')
fout0.close()
if middle_print[1]:
for key in ipdict_dict:
if len(ipdict_dict[key]['IN']) == 0:
prtln = key[0] + ' => ' + key[1] + '\n' + '\tOUT:\n'
fg = 0
for item in ipdict_dict[key]['OUT']:
if item.display_dict()['type'] == 'REQUEST_OUT':
if fg == 0:
fout1.write(prtln)
fg = 1
fout1.write('\t\t' + item.display_string() + '\n')
fout1.write('\n')
fout1.close()
if middle_print[2]:
for key in ipdict_dict:
if len(ipdict_dict[key]['IN']) == len(
ipdict_dict[key]['OUT']) or len(
ipdict_dict[key]['IN']) == 0:
continue
prtln = key[0] + ' => ' + key[1] + '\n' + '\tIN:\n'
fout2.write(prtln)
for item in ipdict_dict[key]['IN']:
fout2.write('\t\t' + item.display_string() + '\n')
fout2.write('\tOUT:\n')
for item in ipdict_dict[key]['OUT']:
fout2.write('\t\t' + item.display_string() + '\n')
fout2.write('\n')
fout2.close()
if middle_print[3]:
for key in ipdict_dict:
if len(ipdict_dict[key]['IN']) != len(ipdict_dict[key]['OUT']):
continue
prtln = key[0] + ' => ' + key[1] + '\n' + '\tIN:\n'
fout3.write(prtln)
for item in ipdict_dict[key]['IN']:
fout3.write('\t\t' + item.display_string() + '\n')
fout3.write('\tOUT:\n')
for item in ipdict_dict[key]['OUT']:
fout3.write('\t\t' + item.display_string() + '\n')
fout3.write('\n')
fout3.close()
if middle_process:
middle_process_f(ipdict_dict)
def bks(limit, fmt, cstats, stats, train, test, profile):
src = ArffReader.ArffReader(sys.stdin)
dst = sys.stdout
cs = BKS(profile)
st = Stats()
# setup proto filter
if hasattr(P, "skip"):
check = lambda gt: gt not in P.skip
elif hasattr(P, "select"):
check = lambda gt: gt in P.select
else:
check = lambda: True
# arff header?
if fmt == "arff":
src.printh(nodata=True, dst=dst)
dst.write("%% Multilevel Traffic Classifier: BKS approach\n")
dst.write("% bks_proto: the identified protocol\n")
dst.write("@attribute bks_proto string\n\n")
dst.write("@data\n")
# load a model?
if test:
cs.train_load(test)
# go!
for d in src:
# check if gt is enabled
gt = d[P.gtcol]
if not check(gt): continue
# parse and pass to classifiers
f = Flow(src, d, gt)
s = cs.ask(f)
if train:
cs.train(s, f.gt)
elif test:
cs.classify(f, s)
# print it to the user?
if limit:
if f.isunk():
if "unk" not in limit: continue
elif "ans" not in limit:
if f.isok() and "ok" not in limit: continue
if f.iserr() and "err" not in limit: continue
# print and count
f.write(fmt, dst)
st.count(f)
# finish training
if train:
cs.train_finish()
cs.train_store(train)
# finish profiling
if profile:
cs.profile_store(profile)
# algo stats?
if cstats:
print("BKS statistics")
print("==============")
print(cs.get_stats())
# stats?
if stats:
if train:
cs.train_show()
else:
print("Performance statistics")
print("======================")
print(st.get_cm())
def test_1(self):
samples, labels = self.generator.generate(10)
print("example\n", samples[1], Flow.Direction(labels[1]).name)
pass
def __init__(self, jsonFile):
self.params = Params.Params()
self.params.load(jsonFile)
self.db = MagicalDB.MagicalDB(self.params) # The flow database
self.db.parse() #parsing the input files
self.flow = Flow.Flow(self.db)
