def getTrace(visit_file, direction=None):
with open(visit_file, 'r') as f:
sample = json.load(f)
Config.hostname.update(sample[u'ip_to_name'])
webId, traceId = sample['visit_log'][u'current_url'], sample['visit_log']['visit_id']
trace = Trace(traceId, webId)
for tcp_conn in sample['tcp_connections']:
connection_id = tcp_conn['connection_id']
TCP = TCPConnection(connection_id, webId, hostip=sample['visit_log'][u'host_ip'])
for pkt in tcp_conn['packets']:
pkt_time, pkt_size = pkt[0], abs(pkt[1])
if pkt_size == 0:
continue
pkt_dir = Packet.Outgoing if pkt[1] < 0 else Packet.Incoming
if direction is None or direction == pkt_dir:
TCP.addPacket(Packet(pkt_time, pkt_size, pkt_dir))
TCP._packets.sort(key=lambda x: x.getTime())
trace.addTcpCon(TCP)
return trace
def loadTraces(self) :
tracesPaths=os.listdir(self.directoryPath)
for tracePath in tracesPaths :
path=os.path.join(self.directoryPath,tracePath)
t=Trace(path)
t.loadTrace()
self.traces.append(t)
def WriteClasses():
for c in CharClass.classes:
Trace.Write(str(c.name), -10)
Trace.Write(': ')
c.WriteSet()
Trace.WriteLine()
Trace.WriteLine()
def applyCountermeasure(trace):
# bsize = random.choice(range(2, 10, 1))
bsize = 6
rand = 0
newTrace = Trace(trace.getId())
# rand = random.choice(range(8,256,8))
count = 0
curDir = 0
for packet in trace.getPackets():
direction = packet.getDirection()
length = packet.getLength()
newlength = min(length + rand, Packet.MTU)
time = packet.getTime()
newPacket = Packet(direction,
time,
newlength )
newTrace.addPacket(newPacket)
if (((direction == 0 and curDir == 0) | (direction == 1 and curDir == 1)) and count < bsize):
count +=1
elif (((direction == 0 and curDir == 0) | (direction == 1 and curDir == 1)) and count == bsize):
BurstSize6.makeDummy(newTrace, time)
count = 0
else:
count = 1
if(curDir):
curDir = 0
else:
curDir = 1
return newTrace
def Trace(self):
self.tracepop = Trace(df=self.model._df)
try:
self.tracepop.Trace()
self.tracepop.show()
except (KeyError):
self.ErrorEvent()
def loadTraces(self):
tracesPaths = os.listdir(self.directoryPath)
for tracePath in tracesPaths:
path = os.path.join(self.directoryPath, tracePath)
t = Trace(path)
t.loadTrace()
self.traces.append(t)
def applyCountermeasure(trace):
packets = []
for packet in trace.getPackets():
if packet.getDirection() == Packet.UP:
x = 1 # outgoing
else:
x = -1 # incoming
packets.append([packet.getTime(), x])
list2 = []
parameters = [""]
Tamaraw.Anoa(packets, list2, parameters)
list2 = sorted(list2, key=lambda list2: list2[0])
list3 = []
Tamaraw.AnoaPad(list2, list3, 100, 0)
newTrace = Trace(trace.getId())
for item in list3:
pDirection = Packet.UP
if (item[1] < 0):
pDirection = Packet.DOWN
newTrace.addPacket(Packet(direction=pDirection, time=item[0], length=abs(item[1])))
return newTrace
def getTraceHerrmann(webpageId, traceIndex):
if config.DATA_SOURCE == 1:
datasourceId = 4
elif config.DATA_SOURCE == 2:
datasourceId = 5
key = '.'.join([
'Webpage', 'H',
str(datasourceId),
str(webpageId),
str(traceIndex)
])
trace = mc.get(key)
if ENABLE_CACHE and trace:
trace = cPickle.loads(trace)
else:
connection = MySQLdb.connect(host=config.MYSQL_HOST,
user=config.MYSQL_USER,
passwd=config.MYSQL_PASSWD,
db=config.MYSQL_DB)
cursor = connection.cursor()
command = """SELECT packets.trace_id,
packets.size,
ROUND(packets.abstime*1000)
FROM (SELECT id
FROM traces
WHERE site_id = (SELECT id
FROM sites
WHERE dataset_id = """ + str(
datasourceId) + """
ORDER BY id
LIMIT """ + str(
webpageId) + """,1)
ORDER BY id
LIMIT """ + str(
traceIndex) + """,1) traces,
packets
WHERE traces.id = packets.trace_id
ORDER BY packets.trace_id, packets.abstime"""
cursor.execute(command)
data = cursor.fetchall()
trace = Trace(webpageId)
for item in data:
direction = Packet.UP
if int(item[1]) > 0:
direction = Packet.DOWN
time = item[2]
length = int(math.fabs(item[1]))
trace.addPacket(Packet(direction, time, length))
connection.close()
mc.set(key, cPickle.dumps(trace,
protocol=cPickle.HIGHEST_PROTOCOL))
return trace
def applyCountermeasure(trace):
newTrace = Trace(trace.getId())
for packet in trace.getPackets():
newPacket = Packet(packet.getDirection(), packet.getTime(),
PadRoundLinear.calcLength(packet.getLength()))
newTrace.addPacket(newPacket)
return newTrace
def run_trace_morph_test(self, src, dst, exp, **kwargs):
t1_src = src if isinstance(src, Trace) else Trace.create_from_array(1, src)
t1_dst = dst if isinstance(dst, Trace) else Trace.create_from_array(1, dst)
cm = SmartMorphing()
for k, v in kwargs.items():
cm.set_param(k, v)
cm.dst_trace = t1_dst
t2 = cm.apply_to_trace(t1_src)
self.assertTraceEqual(t2, exp)
def applyCountermeasure(trace):
newTrace = Trace(trace.getId())
for packet in trace.getPackets():
newPacket = Packet(
packet.getDirection(), packet.getTime(), PadRoundExponential.calcLength(packet.getLength())
)
newTrace.addPacket(newPacket)
return newTrace
def applyCountermeasure(trace):
newTrace = Trace(trace.getId())
for packet in trace.getPackets():
newPacket = Packet( packet.getDirection(),
packet.getTime(),
MiceElephants.calcLength(packet.getLength()) )
newTrace.addPacket( newPacket )
return newTrace
def applyCountermeasure(trace):
newTrace = Trace(trace.getId())
# pad all packets to the MTU
for packet in trace.getPackets():
newPacket = Packet( packet.getDirection(),
packet.getTime(),
Packet.MTU )
newTrace.addPacket( newPacket )
return newTrace
def applyCountermeasure(trace):
newTrace = Trace(trace.getId())
for packet in trace.getPackets():
rand = random.choice(range(8, 256, 8))
length = min(packet.getLength() + rand, Packet.MTU)
newPacket = Packet(packet.getDirection(), packet.getTime(), length)
newTrace.addPacket(newPacket)
return newTrace
def applyCountermeasure(trace):
newTrace = Trace(trace.getId())
for packet in trace.getPackets():
length = Packet.MTU
if Packet.MTU - packet.getLength() > 0:
length = packet.getLength() + random.choice(range(0, Packet.MTU - packet.getLength(), 8))
newPacket = Packet(packet.getDirection(), packet.getTime(), length)
newTrace.addPacket(newPacket)
return newTrace
def readEsorics16TorFile( webpageId, traceIndex ):
if webpageId < 100: # 100 and more is nonMon
file = os.path.join(config.PCAP_ROOT, str(webpageId)+"-"+str(traceIndex))
else:
file = os.path.join(config.PCAP_ROOT, str(webpageId-100)) # as the nonMon id starts from 100 and the file names are 0, 1, 2, 3, ...
fileList = Utils.readFile(file)
trace = Trace(webpageId)
'''
<time-stamp>\t<directed-packet-length>
1392354075.204044 -565
1392354075.20464 565
1392354075.709575 -565
1392354075.956634 -565
1392354075.981538 565
1392354076.192185 565
1392354076.36483 -565
1392354076.647188 -565
1392354076.685665 -1448
1392354076.685685 -1448
...
'''
startTime = 0
for i in range(0,len(fileList)):
pArray = fileList[i].split("\t")
cTime = pArray[0]
if startTime == 0:
startTime = float(cTime) # first packet's time as a reference
cDirection = int(pArray[1])
pDirection = Packet.UP
if (cDirection < 0):
pDirection = Packet.DOWN
# as in the pcapparser.py
# delta = int(round(((ts - start) * 1000),0))
#pTime = int(round(((float(cTime) - startTime) * 1000),0))
pTime = int(round(((float(cTime) - startTime) ), 0))
pLength = abs(int(pArray[1]))
pkt = Packet(pDirection, pTime, pLength)
pkt.timeStr = cTime
trace.addPacket(pkt)
return trace
def applyCountermeasure(trace):
newTrace = Trace(trace.getId())
count = 0
curDir = 0
rand = random.choice(range(8,256,8))
count = 0
for packet in trace.getPackets():
direction = packet.getDirection()
length = packet.getLength()
newlength = min(length + rand, Packet.MTU)
time = packet.getTime()
newPacket = Packet(direction,
time,
newlength )
if ((direction == 1 and curDir == 1)):
count +=1
newTrace.addPacket(newPacket)
elif (direction == 0 and curDir == 0):
count = 0
newTrace.addPacket(newPacket)
elif (direction == 1 and curDir == 0):
curDir = 1
count = 1
newTrace.addPacket(newPacket)
elif (direction == 0 and curDir == 1):
BurstExpon.makeDummy(newTrace, length, time, curDir, count, rand)
newTrace.addPacket(newPacket)
curDir = 0
count = 0
return newTrace
def run_trace_morph_test(self, src, dst, exp, **kwargs):
t1_src = src if isinstance(src, Trace) else Trace.create_from_array(
1, src)
t1_dst = dst if isinstance(dst, Trace) else Trace.create_from_array(
1, dst)
cm = SmartMorphing()
for k, v in kwargs.items():
cm.set_param(k, v)
cm.dst_trace = t1_dst
t2 = cm.apply_to_trace(t1_src)
self.assertTraceEqual(t2, exp)
def applyCountermeasure(trace):
newTrace = Trace(trace.getId())
for packet in trace.getPackets():
length = Packet.MTU
if Packet.MTU - packet.getLength() > 0:
length = packet.getLength() + random.choice(
range(0, Packet.MTU - packet.getLength(), 8))
newPacket = Packet(packet.getDirection(), packet.getTime(), length)
newTrace.addPacket(newPacket)
return newTrace
def getTraceHerrmann( webpageId, traceIndex ):
if config.DATA_SOURCE == 1:
datasourceId = 4
elif config.DATA_SOURCE == 2:
datasourceId = 5
key = '.'.join(['Webpage',
'H',
str(datasourceId),
str(webpageId),
str(traceIndex)])
trace = mc.get(key)
if ENABLE_CACHE and trace:
trace = cPickle.loads(trace)
else:
connection = MySQLdb.connect(host=config.MYSQL_HOST,
user=config.MYSQL_USER,
passwd=config.MYSQL_PASSWD,
db=config.MYSQL_DB )
cursor = connection.cursor()
command = """SELECT packets.trace_id,
packets.size,
ROUND(packets.abstime*1000)
FROM (SELECT id
FROM traces
WHERE site_id = (SELECT id
FROM sites
WHERE dataset_id = """+str(datasourceId)+"""
ORDER BY id
LIMIT """+str(webpageId)+""",1)
ORDER BY id
LIMIT """+str(traceIndex)+""",1) traces,
packets
WHERE traces.id = packets.trace_id
ORDER BY packets.trace_id, packets.abstime"""
cursor.execute( command )
data = cursor.fetchall()
trace = Trace(webpageId)
for item in data:
direction = Packet.UP
if int(item[1])>0:
direction = Packet.DOWN
time = item[2]
length = int(math.fabs(item[1]))
trace.addPacket( Packet( direction, time, length ) )
connection.close()
mc.set(key,cPickle.dumps(trace,protocol=cPickle.HIGHEST_PROTOCOL))
return trace
def applyCountermeasure(trace):
rand = random.choice(range(8,512,8))
newTrace = Trace(trace.getId())
for packet in trace.getPackets():
length = min( packet.getLength()+rand, Packet.MTU )
newPacket = Packet( packet.getDirection(),
packet.getTime(),
length )
newTrace.addPacket( newPacket )
return newTrace
def is_good(self, filename):
t = Trace(self.raw_path, filename)
if t.white_perc > self.white_perc_thr:
print('too crowded: ', t.white_perc)
return False
t.compute_inertia()
if t.eccentricity_from_inertia < self.eccentricity_thr:
print('too spherical ', t.eccentricity_from_inertia)
return False
self.interesting_count += 1
return True
def createTrace(self):
'''Fit for the position and width of the trace.'''
self.speak("populating the trace parameters")
tracefilename = self.directory + 'trace_{0}.npy'.format(self.name)
skyfilename = self.directory + 'skyMask_{0}.npy'.format(self.name)
# define the trace object (will either load saved, or remake)
self.trace = Trace(self)
self.images['RoughLSF'] = np.exp(
-0.5 * ((self.s - self.trace.traceCenter(self.w)) /
self.trace.tracefitwidth)**2)
def get_records():
trace = request.args.get('trace') == "true"
domain = request.args.get('domain')
if domain is None:
return 'Domain is missing', 400
domain = domain.split("/")[0]
try:
get_cache()
response: tp.Optional[IpRecord] = None
if not trace:
response = domain_cache[domain]
if response is not None and response.expires_at < time.time():
response = None
if response is not None:
print(f"Using cache for {domain}")
return jsonify(
domain=domain,
ips=response.ips,
ttl=max(response.expires_at - time.time(), 0),
)
data_trace = Trace()
response = Dns.find_recursive(domain, Dns.ROOT_SERVERS_DNS, data_trace)
except Exception as e:
print(e)
response = None
if response is None:
return jsonify(error='Domain is not resolved',
trace=data_trace.compose()), 404
domain_cache[domain] = response
if trace:
return jsonify(domain=domain,
ips=response.ips,
ttl=max(response.expires_at - time.time(), 0),
trace=data_trace.compose(),
exact_match=any(
[d.startswith(domain) for d, _ in response.ips]))
else:
return jsonify(domain=domain,
ips=response.ips,
ttl=max(response.expires_at - time.time(), 0),
exact_match=any(
[d.startswith(domain) for d, _ in response.ips]))
def WriteStatistics( ):
Trace.WriteLine( )
Trace.WriteLine( 'Statistics:' )
Trace.WriteLine( '-----------' )
Trace.WriteLine( )
Trace.WriteLine( str(len( Symbol.terminals )) + ' terminals' )
Trace.WriteLine( str(len( Symbol.terminals ) + len( Symbol.pragmas ) + len( Symbol.nonterminals )) + ' symbols' )
Trace.WriteLine( str(len(Node.nodes)) + ' nodes' )
Trace.WriteLine( str(len(ParserGen.symSet)) + ' sets' )
Trace.WriteLine( )
def encrypt(self, plaintext):
# input plaintext -> np.array
self.trace = Trace(self.b, self.noise)
self.plain_state = plaintext.reshape(4, 4)
self.__add_round_key(self.plain_state, self.round_keys[:4])
for i in range(1, 10):
self.__round_encrypt(self.plain_state,
self.round_keys[4 * i:4 * (i + 1)])
self.__sub_bytes(self.plain_state)
self.__shift_rows(self.plain_state)
self.__add_round_key(self.plain_state, self.round_keys[40:])
return self.plain_state.flatten(), self.trace
def applyCountermeasure( trace, metadata ):
[targetDistributionBi,
targetDistributionUp,
targetDistributionDown] = metadata
newTrace = Trace(trace.getId())
# primary sampling
timeCursor = 0
for packet in trace.getPackets():
timeCursor = packet.getTime()
targetDistribution = targetDistributionDown
if packet.getDirection()==Packet.UP:
targetDistribution = targetDistributionUp
packets = DirectTargetSampling.morphPacket( packet, targetDistribution )
for newPacket in packets:
newTrace.addPacket( newPacket )
# secondary sampling
while True:
l1Distance = newTrace.calcL1Distance( targetDistributionBi )
if l1Distance <= DirectTargetSampling.L1_THRESHHOLD:
break
timeCursor += 10
newDirection, newLen = newTrace.getMostSkewedDimension( targetDistributionBi )
packet = Packet( newDirection, timeCursor, newLen )
newTrace.addPacket( packet )
return newTrace
def readfile( month, day, hour, webpageId ):
strId = '.'.join([str(month), str(day), str(hour), str(webpageId)])
trace = Trace(webpageId)
start = 0
absPath = __constructAbsolutePath( month, day, hour, webpageId )
if absPath:
pcapReader = dpkt.pcap.Reader( file( absPath, "rb") )
for ts, buf in pcapReader:
eth = dpkt.ethernet.Ethernet(buf)
ip = eth.data
tcp = ip.data
if start==0: start = ts
direction = Packet.UP
if (tcp.sport==22):
direction = Packet.DOWN
delta = int(round(((ts - start) * 1000),0))
length = ip.len + Packet.HEADER_ETHERNET
trace.addPacket( Packet(direction, delta, length ) )
return trace
def traces(self):
"yields all traceroutes of query"
if self.queryResult == None:
return None
for raw_trace in self.queryResult['queries']:
yield Trace(raw_trace)
def test_send_1(self):
t1 = Trace.create_from_array(1, self.TRACE_1)
Folklore.FIXED_PACKET_LEN = 1000
Folklore.TIMER_CLOCK_SPEED = 10
Folklore.MILLISECONDS_TO_RUN = 200
t2 = Folklore.applyCountermeasure(t1)
expected_trace = [
(1, 0, 1000), (0, 0, 1000), # - u1
(1, 10, 1000), (0, 10, 1000), # - -
(1, 20, 1000), (0, 20, 1000), # d1 -
(1, 30, 1000), (0, 30, 1000), # - u2
(1, 40, 1000), (0, 40, 1000), # d2 -
(1, 50, 1000), (0, 50, 1000), # d2 u3
(1, 60, 1000), (0, 60, 1000), # d3 -
(1, 70, 1000), (0, 70, 1000), # d4 u4
(1, 80, 1000), (0, 80, 1000), # d4 -
(1, 90, 1000), (0, 90, 1000), # d5 -
(1, 100, 1000), (0, 100, 1000), # d5 -
(1, 110, 1000), (0, 110, 1000), # d6 -
(1, 120, 1000), (0, 120, 1000), # - -
(1, 130, 1000), (0, 130, 1000), # - -
(1, 140, 1000), (0, 140, 1000), # - -
(1, 150, 1000), (0, 150, 1000), # - -
(1, 160, 1000), (0, 160, 1000), # - -
(1, 170, 1000), (0, 170, 1000), # - -
(1, 180, 1000), (0, 180, 1000), # - -
(1, 190, 1000), (0, 190, 1000), # - -
(1, 200, 1000), (0, 200, 1000), # - -
]
self.assertTraceEqual(t2, expected_trace)
def readWangTorFile( webpageId, traceIndex ):
if webpageId < 100: # 100 and more is nonMon
if config.DATA_SOURCE == 5:
file = os.path.join(config.PCAP_ROOT, str(webpageId)+"-"+str(traceIndex))
elif config.DATA_SOURCE == 9:
file = os.path.join(config.PCAP_ROOT, str(webpageId) + "-" + str(traceIndex)) + ".cell"
else:
if config.DATA_SOURCE == 5:
file = os.path.join(config.PCAP_ROOT, str(webpageId-100)) # as the nonMon id starts from 100 and the file names are 0, 1, 2, 3, ...
elif config.DATA_SOURCE == 9:
file = os.path.join(config.PCAP_ROOT, str(webpageId-100)) + ".cell" # as the nonMon id starts from 100 and the file names are 0, 1, 2, 3, ...
fileList = Utils.readFile(file)
trace = Trace(webpageId)
'''
0.0 1 cell
0.0 1
0.116133928299 1
0.499715805054 -1
0.499715805054 -1
...
'''
for i in range(1,len(fileList)):
cellArray = fileList[i].split("\t")
cTime = cellArray[0]
cDirection = int(cellArray[1])
pDirection = Packet.UP
if (cDirection==-1):
pDirection = Packet.DOWN
# as in the pcapparser.py
# delta = int(round(((ts - start) * 1000),0))
pTime = int(round((float(cTime) * 1000),0))
#pTime = int(round((float(cTime) * 10000), 0))
#pTime = int(round((float(cTime) * 1000000), 0))
pLength = abs(int(cellArray[1])) # sizes are only 1 and -1
trace.addPacket(Packet(pDirection, pTime, pLength))
return trace
def WriteSet(self):
i = 0
mx = max(self.set) + 1
while i < mx:
while i < mx and (i not in self.set):
i += 1
if i == mx:
break
j = i
while i < mx and (i in self.set):
i += 1
if j < (i - 1):
Trace.Write(
str(CharClass.Ch(j)) + ".." + str(CharClass.Ch(i - 1)) +
" ")
else:
Trace.Write(str(CharClass.Ch(j) + " "))
def readWangTorFileOld( webpageId, traceIndex ):
if webpageId < 100:
file = os.path.join(config.PCAP_ROOT, str(webpageId)+"-"+str(traceIndex))
else:
file = os.path.join(config.PCAP_ROOT, str(webpageId-100)) # as the nonMon id starts from 100 and the file names are 0, 1, 2, 3, ...
fileList = Utils.readFile(file)
trace = Trace(webpageId)
'''
0.0 1 cell
0.0 1
0.116133928299 1
0.499715805054 -1
0.499715805054 -1
...
'''
prevcTime = currcTime = fileList[0].split("\t")[0] # previous and current cell time (0.0 in the example above)
prevcDirection = currcDirection = int(fileList[0].split("\t")[1]) # previous and current cell direction
#cLength = 512 # cell length is always 512 bytes in Tor
cellCtr = 1
for i in range(1,len(fileList)):
cellArray = fileList[i].split("\t")
cTime = cellArray[0]
cDirection = cellArray[1]
currcTime = cTime
currcDirection = int(cDirection)
if currcTime != prevcTime:
#pLength = cellCtr * cLength
Datastore.addPacketsFromCells( trace, prevcDirection, prevcTime, cellCtr )
prevcDirection = currcDirection
prevcTime = currcTime
cellCtr = 1
continue
elif currcDirection != prevcDirection:
#pLength = cellCtr * cLength
Datastore.addPacketsFromCells( trace, prevcDirection, prevcTime, cellCtr )
prevcDirection = currcDirection
prevcTime = currcTime
cellCtr = 1
continue
else: # same time, same direciton
cellCtr = cellCtr + 1
prevcDirection = currcDirection
prevcTime = currcTime
# for the last cell
Datastore.addPacketsFromCells( trace, prevcDirection, prevcTime, cellCtr )
return trace
def readfile(month, day, hour, webpageId):
strId = '.'.join([str(month), str(day), str(hour), str(webpageId)])
trace = Trace(webpageId)
start = 0
absPath = __constructAbsolutePath(month, day, hour, webpageId)
# testing
#print absPath
if absPath:
pcapReader = dpkt.pcap.Reader(file(absPath, "rb"))
for ts, buf in pcapReader:
eth = dpkt.ethernet.Ethernet(buf)
ip = eth.data
tcp = ip.data
#dpkt.Packet.__flags__
if start == 0: start = ts
direction = Packet.UP
if (tcp.sport == 22):
direction = Packet.DOWN
if (config.DATA_SOURCE == 3):
if (tcp.sport == 9001 or tcp.sport == 443):
direction = Packet.DOWN
if (config.DATA_SOURCE == 4 or config.DATA_SOURCE == 41
or config.DATA_SOURCE == 42):
if (tcp.sport == 8080 or tcp.sport == 443):
direction = Packet.DOWN
#testing
#origTimeDiff = ts - start
#origTimeDiff = (ts - start) * 1000
#origTimeDiff = round(((ts - start) * 1000),0)
#origTimeDiff = int(round(((ts - start) * 1000),0))
#print origTimeDiff
delta = int(round(((ts - start) * 1000), 0))
length = ip.len + Packet.HEADER_ETHERNET
'''
if (config.DATA_SOURCE==3): # overcoming the packet size greater than 1500
while True:
if length > 1500: # MTU
excludedLength = random.randint(595, 1500)
trace.addPacket( Packet(direction, delta, excludedLength ) )
length = length - excludedLength
else:
trace.addPacket( Packet(direction, delta, length ) )
break
else:
trace.addPacket( Packet(direction, delta, length ) )
'''
trace.addPacket(Packet(direction, delta, length))
return trace
def applyCountermeasure(trace, metadata):
[
targetDistributionBi, targetDistributionUp, targetDistributionDown,
srcDistributionUp, srcDistributionDown, morphingMatrixUp,
morphingMatrixDown
] = metadata
newTrace = Trace(trace.getId())
# primary sampling
timeCursor = 0
for packet in trace.getPackets():
timeCursor = packet.getLength()
index = (packet.getLength() - Packet.HEADER_LENGTH) / 8
targetDistribution = None
morphingColumn = None
if packet.getDirection() == Packet.UP:
if morphingMatrixUp:
morphingColumn = morphingMatrixUp[:, index]
else:
targetDistribution = targetDistributionUp
targetDistributionSecondary = targetDistributionUp
else:
if morphingMatrixDown:
morphingColumn = morphingMatrixDown[:, index]
else:
targetDistribution = targetDistributionDown
targetDistributionSecondary = targetDistributionDown
if morphingColumn:
targetDistribution = {}
for i in range(len(morphingColumn)):
key = str(packet.getDirection()) + '-' + str(
i * 8 + Packet.HEADER_LENGTH)
targetDistribution[key] = morphingColumn[i]
packets = WrightStyleMorphing.morphPacket(
packet, targetDistribution, targetDistributionSecondary)
for newPacket in packets:
newTrace.addPacket(newPacket)
# secondary sampling
while True:
l1Distance = newTrace.calcL1Distance(targetDistributionBi)
if l1Distance <= WrightStyleMorphing.L1_THRESHHOLD:
break
timeCursor += 10
newDirection, newLen = newTrace.getMostSkewedDimension(
targetDistributionBi)
packet = Packet(newDirection, timeCursor, newLen)
newTrace.addPacket(packet)
return newTrace
def __init__(self, foldername):
"""
Initialize Driver with a foldername.
"""
self._id = int(os.path.basename(foldername))
self._traces = []
files = [f for f in os.listdir(foldername) if f.endswith(".csv")]
for filename in files:
self._traces.append(Trace(os.path.join(foldername, filename)))
def Join_Particles(particles=[],
start_ID=1,
folder='./',
overlap_thr=0.5,
autotrigger=True,
eccentricity_thr=10,
verbose=True):
ID = start_ID
for filename in tqdm(os.listdir(folder)):
if not filename.startswith('mean_'):
continue
# e.g filename = mean_280519-video7_000-007_opened_cc02.png
prefix, dot, name = str(filename).partition(
'-') # = mean_280519, -, video7_000-007_opened_cc02.png
segment, dot, suffix1 = name.partition(
'-') # = video7_000, -, 007_opened_cc02.png
frame, dot, suffix2 = suffix1.partition(
'_') # = 007, _, opened_cc02.png
frameID = int(frame)
previous_name = segment + ('-%03d' % (frameID - 1))
t = Trace(folder, filename)
if autotrigger:
if not Trigger(
t, eccentricity_thr=eccentricity_thr, verbose=verbose):
continue
found = False
for p in particles:
if p.traces[-1].frame_name == previous_name:
diff_matrix = np.abs(p.traces[-1].matrix - t.matrix)
# compute white_number for diff_matrix
n_whites = 0
for x in diff_matrix:
for y in x:
if y > 0:
n_whites += 1
overlap = 1.0 - n_whites / (p.traces[-1].white_number +
t.white_number)
if verbose:
print('\n' + previous_name + '-' + t.frame_name +
': overlap = ' + str(overlap))
if overlap >= overlap_thr:
found = True
p.traces.append(t)
break
if not found:
particles.append(Particle(folder, ID, [t]))
ID += 1
return particles, ID - start_ID
def __init__(self, filename):
self.filename = filename
raw = Trace.from_file(self.filename)
self.traces = {'raw': raw}
self.last_trace = raw
self.time_units = "min"
self.intensity_metric = "Absorbance"
self.intensity_units = "mAU"
def __init__(self, foldername):
"""
Initialize Driver with a foldername.
"""
self._id = int(os.path.basename(foldername))
self._traces = []
file_num = len(os.listdir(foldername))
files = ['%d.csv' % (i) for i in range(1, file_num+1)]
for filename in files:
self._traces.append(Trace(os.path.join(foldername, filename)))
def get_trace(cls,
trace_id=None,
site_id=None,
dataset=2,
limit=1,
multi=False):
if cls.conn is None:
cls.conn = MySQLdb.connect(host=config.MYSQL_HOST,
user=config.MYSQL_USER,
passwd=config.MYSQL_PASSWD,
db=config.MYSQL_DB)
cur = cls.conn.cursor()
if trace_id is None:
cur.execute(
'SELECT id FROM traces where site_id=%s ORDER BY RAND() LIMIT {}'
.format(limit), [site_id])
if limit == 1:
trace_ids = [cur.fetchone()[0]]
else:
trace_ids = [r[0] for r in cur.fetchall()]
else:
trace_ids = [trace_id]
# print('SEL-TRACE', trace_ids)
traces = []
for trace_id in trace_ids:
cur.execute(
'SELECT size, ROUND(abstime*1000) FROM packets WHERE trace_id=%s ORDER BY abstime',
[trace_id])
data = cur.fetchall()
trace = Trace(trace_id, webpage=site_id)
for item in data:
direction = Packet.UP
if int(item[0]) > 0:
direction = Packet.DOWN
time = item[1]
length = int(math.fabs(item[0]))
trace.addPacket(Packet(direction, time, length))
traces.append(trace)
if limit == 1 and not multi:
return traces[0] if traces else None
return traces
def applyCountermeasure( trace, metadata ):
[targetDistributionBi,
targetDistributionUp,
targetDistributionDown,
srcDistributionUp,
srcDistributionDown,
morphingMatrixUp,
morphingMatrixDown] = metadata
newTrace = Trace(trace.getId())
# primary sampling
timeCursor = 0
for packet in trace.getPackets():
timeCursor = packet.getLength()
index = (packet.getLength()-Packet.HEADER_LENGTH)/8
targetDistribution = None
morphingColumn = None
if packet.getDirection()==Packet.UP:
if morphingMatrixUp:
morphingColumn = morphingMatrixUp[:,index]
else:
targetDistribution = targetDistributionUp
targetDistributionSecondary = targetDistributionUp
else:
if morphingMatrixDown:
morphingColumn = morphingMatrixDown[:,index]
else:
targetDistribution = targetDistributionDown
targetDistributionSecondary = targetDistributionDown
if morphingColumn:
targetDistribution = {}
for i in range(len(morphingColumn)):
key = str(packet.getDirection())+'-'+str( i*8 + Packet.HEADER_LENGTH )
targetDistribution[key] = morphingColumn[i]
packets = WrightStyleMorphing.morphPacket( packet, targetDistribution, targetDistributionSecondary )
for newPacket in packets:
newTrace.addPacket( newPacket )
# secondary sampling
while True:
l1Distance = newTrace.calcL1Distance( targetDistributionBi )
if l1Distance <= WrightStyleMorphing.L1_THRESHHOLD:
break
timeCursor += 10
newDirection, newLen = newTrace.getMostSkewedDimension( targetDistributionBi )
packet = Packet( newDirection, timeCursor, newLen )
newTrace.addPacket( packet )
return newTrace
def test_tamaraw_1(self):
t1 = Trace.create_from_array(1, self.TRACE_2)
cm = Tamaraw()
cm.set_param('RUN_PADDING', 4)
t2 = cm.apply_to_trace(t1)
expected_trace = [
(1, 0, 750),
(0, 0, 750),
(1, 6, 750),
(1, 12, 750),
(1, 18, 750),
(0, 20, 750), # u1
(1, 24, 750), # d1
(1, 30, 750), # d2-1
(1, 36, 750), # d2-2
(0, 40, 750), # u2
(1, 42, 750),
(0, 60, 750),
]
self.assertTraceEqual(t2, expected_trace)
def generate_trace(self): trace_obj = Trace(self.src_f.name, self.bin_fn, self.trace_fn) trace_obj.run()
def build_new_trace(self):
self.new_trace = Trace(self.trace.getId())
class CounterMeasure(object):
DEFAULT_PARAMS = {}
def __init__(self, params=None):
self.trace = None
self.new_trace = None
self.metadata = None
self.model = None
self.params = copy.deepcopy(self.DEFAULT_PARAMS)
self.new_trace_size = 0
if params:
self.params.update(params)
def set_param(self, parameter, value):
self.params[parameter] = value
self.update_params()
def update_params(self):
pass
def train(self, src_page=None, target_page=None):
pass
def apply(self):
""" Create a new trace by applying the countermeasure to the original trace
The original trace is available in self.trace
The modified trace should be saved in self.new_trace or returned
"""
raise NotImplementedError
def build_new_trace(self):
self.new_trace = Trace(self.trace.getId())
def add_packet(self, packet):
# print packet.get_details()
t = packet.getTime()
self.new_trace_size += packet.size
ind = len(self.new_trace.packets)
while 0 < ind and t < self.new_trace.packets[ind - 1].getTime():
ind -= 1
self.new_trace.add_packet(packet, index=ind)
def get_new_trace_time(self):
return self.new_trace.packets[-1].time if self.new_trace.packets else 0
def get_new_trace_size(self):
return self.new_trace_size
def apply_to_trace(self, trace):
self.trace = trace
self.build_new_trace()
t2 = self.apply()
if t2 is None:
t2 = self.new_trace
return t2
@classmethod
def initialize(cls):
pass
@classmethod
def get_name(cls):
return 'Generic Countermeasure'
def applyCountermeasure(trace):
bsize = random.choice(range(3, 4, 1))
dsize = random.choice(range(3, 4, 1))
newTrace = Trace(trace.getId())
rand = random.choice(range(8,256,8))
count = 0
curDir = 0
for packet in trace.getPackets():
direction = packet.getDirection()
length = packet.getLength()
newlength = min(length + rand, Packet.MTU)
time = packet.getTime()
newPacket = Packet(direction,
time,
newlength )
if ((direction == 1 and curDir == 1) and count < bsize):
count +=1
newTrace.addPacket(newPacket)
elif ((direction == 1 and curDir == 1) and count == bsize):
BurstClient2Server.makeDummy(newTrace, length, time, curDir, dsize)
newTrace.addPacket(newPacket)
count = 1
elif ((direction == 1 and curDir == 0)):
newTrace.addPacket(newPacket)
curDir = 1
count = 1
elif ((direction == 0 and curDir == 1)):
newTrace.addPacket(newPacket)
curDir = 0
count = 1
elif (direction == 0 and curDir == 0):
newTrace.addPacket(newPacket)
count = 1
return newTrace
def applyCountermeasure(trace):
bsize = random.choice(range(2,10,1))
newTrace = Trace(trace.getId())
count = 0
curDir = 0
rand = random.choice(range(8,256,8))
for packet in trace.getPackets():
direction = packet.getDirection()
length = packet.getLength()
newlength = min( packet.getLength()+rand, Packet.MTU )
time = packet.getTime()
newPacket = Packet(direction,
time,
newlength )
if (direction == 1 and curDir == 1 and (count < bsize)):
count +=1
newTrace.addPacket(newPacket)
elif (direction == 1 and curDir == 1 and (count == bsize)):
BurstPadAllRand.sendDummyAck(newTrace, time)
newTrace.addPacket(newPacket)
bsize = random.choice(range(2,10,1))
count = 1
elif (direction == 0 and curDir == 0):
count = 0
newTrace.addPacket(newPacket)
elif (direction == 1 and curDir == 0):
curDir = 1
count = 1
newTrace.addPacket(newPacket)
elif (direction == 0 and curDir == 1):
BurstPadAllRand.makeDummy(newTrace, length, time, bsize, count, rand)
newTrace.addPacket(newPacket)
curDir = 0
count = 0
bsize = random.choice(range(2,10,1))
return newTrace
def applyCountermeasure(trace):
bsize = random.choice(range(3000, 15000, 8))
rand = random.choice(range(8,256,8))
newTrace = Trace(trace.getId())
count = 0
curDir = 0
for packet in trace.getPackets():
direction = packet.getDirection()
length = packet.getLength()
newlength = min(length + rand, Packet.MTU)
time = packet.getTime()
newPacket = Packet(direction,
time,
newlength )
if (direction == 1 and curDir == 1 and (count+newlength < bsize)):
count +=newlength
newTrace.addPacket(newPacket)
elif (direction == 1 and curDir == 1 and (count+newlength >= bsize)):
pad = bsize - count
packetFirstHalf = Packet(1, time, pad)
newTrace.addPacket(packetFirstHalf)
BurstByteLevelRand.sendDummyAck(newTrace, time)
count = newlength-pad
finishPacket = Packet(1, time, count)
newTrace.addPacket(finishPacket)
bsize = random.choice(range(3000, 15000, 8))
elif (direction == 0 and curDir == 0):
count = 0
newTrace.addPacket(newPacket)
elif (direction == 1 and curDir == 0):
curDir = 1
count = newlength
newTrace.addPacket(newPacket)
elif (direction == 0 and curDir == 1):
BurstByteLevelRand.makeDummy(newTrace, length, time, bsize, count, rand)
newTrace.addPacket(newPacket)
curDir = 0
count = 0
bsize = random.choice(range(3000, 15000, 8))
return newTrace
def applyCountermeasure(trace):
# print "dir\tlength\ttime"
bsize = random.choice(range(2, 10, 1))
newTrace = Trace(trace.getId())
count = 0
curDir = 0
count = 0
for packet in trace.getPackets():
direction = packet.getDirection()
length = packet.getLength()
newlength = BurstFixedRand.calcLength(length)
time = packet.getTime()
# print direction, "\t", length, "\t", time
newPacket = Packet(direction,
time,
newlength )
newTrace.addPacket(newPacket)
if (((direction == 0 and curDir == 0) | (direction == 1 and curDir == 1)) and count < bsize):
count +=1
elif (((direction == 0 and curDir == 0) | (direction == 1 and curDir == 1)) and count == bsize):
BurstFixedRand.makeDummy(newTrace, length, time, curDir)
count = 0
else:
count = 1
if(curDir):
curDir = 0
else:
curDir = 1
# for i in range(count, 3):
# Burst.makeDummy(newTrace, length, time, curDir)
# count = 1
# elif (count < 7):
# if(curDir):
# curDir = 0
# else:
# curDir = 1
# for i in range(count, 7):
# Burst.makeDummy(newTrace, length, time, curDir)
# count = 1
# elif (count < ):
# if(curDir):
# curDir = 0
# else:
# curDir = 1
# for i in range(count, 12):
# Burst.makeDummy(newTrace, length, time, curDir)
# count = 1
# else:
# if(curDir):
# curDir = 0
# else:
# curDir = 1
# for i in range(count, 25):
# Burst.makeDummy(newTrace, length, time, curDir)
# count = 1
# newTrace.addPacket(newPacket)
return newTrace
def test_readfile(self):
actualTrace = pcapparser.readfile( month=3, day=14, hour=22, webpageId=8 )
expectedTrace = Trace(8)
expectedTrace.addPacket( Packet( Packet.UP , 0 , 148 ) )
expectedTrace.addPacket( Packet( Packet.DOWN, 0 , 100 ) )
expectedTrace.addPacket( Packet( Packet.UP , 0 , 52 ) )
expectedTrace.addPacket( Packet( Packet.UP , 3 , 500 ) )
expectedTrace.addPacket( Packet( Packet.DOWN, 18 , 244 ) )
expectedTrace.addPacket( Packet( Packet.UP , 35 , 436 ) )
expectedTrace.addPacket( Packet( Packet.DOWN, 75 , 52 ) )
expectedTrace.addPacket( Packet( Packet.DOWN, 118, 292 ) )
expectedTrace.addPacket( Packet( Packet.UP , 158, 52 ) )
