def alignMessages(self, values):
self.args = []
for (data, tags) in values:
message = RawMessage(data=data)
for pos, tag in list(tags.items()):
message.addSemanticTag(pos, tag)
self.args.append(WrapperMessage(message, "Virtual symbol"))
def __init__(self, receptionTimeout=None):
super(EmptySymbol, self).__init__(fields=None,
name="Empty Symbol",
messages=[RawMessage()])
super(EmptySymbol, self).__init__(fields=None,
name="Empty Symbol",
messages=[RawMessage()])
if receptionTimeout is None:
receptionTimeout = EmptySymbol.defaultReceptionTimeout()
self.receptionTimeout = receptionTimeout
def message(self, message):
if message is None:
message = RawMessage()
self.__message = message
self.__message = message
def calcHexDist(hexA, hexB):
from netzob.Model.Vocabulary.Messages.RawMessage import RawMessage
from nemere.inference.analyzers import Value
from nemere.inference.segments import MessageSegment
from nemere.inference.templates import DistanceCalculator
bytedata = [bytes.fromhex(hexA),bytes.fromhex(hexB)]
messages = [RawMessage(bd) for bd in bytedata]
analyzers = [Value(message) for message in messages]
segments = [MessageSegment(analyzer, 0, len(analyzer.message.data)) for analyzer in analyzers]
dc = DistanceCalculator(segments)
return dc.pairDistance(*segments)
def readSymbols(self, timeout=EmptySymbol.defaultReceptionTimeout()):
"""Read from the abstraction layer a flow and abstract it with one or more consecutive symbols
The timeout parameter represents the amount of time (in millisecond) above which
no reception of a message triggers the reception of an :class:`netzob.Model.Vocabulary.EmptySymbol.EmptySymbol`. If timeout set to None
or to a negative value means it always wait for the reception of a message.
:keyword timeout: the time above which no reception of message triggers the reception of an :class:`netzob.Model.Vocabulary.EmptySymbol.EmptySymbol`
:type timeout: :class:`int`
:raise TypeError if the parameter is not valid and Exception if an error occurs.
"""
self._logger.debug("Reading data from communication channel...")
data = self.channel.read(timeout=timeout)
self._logger.debug("Received : {}".format(repr(data)))
symbols = []
# if we read some bytes, we try to abstract them
if len(data) > 0:
try:
symbols_and_data = self.flow_parser.parseFlow(
RawMessage(data), self.symbols)
for (symbol, alignment) in symbols_and_data:
symbols.append(symbol)
except Exception as e:
self._logger.error(e)
if len(symbols) > 0:
self.memory = self.flow_parser.memory
self.specializer.memory = self.memory
else:
symbols.append(EmptySymbol())
if len(symbols) == 0 and len(data) > 0:
msg = RawMessage(data)
symbols.append(UnknownSymbol(message=msg))
return (symbols, data)
def abstract(data, fields):
"""Search in the fields/symbols the first one that can abstract the data.
>>> from netzob.all import *
>>> messages = ["{0}, what's up in {1} ?".format(pseudo, city) for pseudo in ['netzob', 'zoby'] for city in ['Paris', 'Berlin']]
>>> f1a = Field("netzob")
>>> f2a = Field(", what's up in ")
>>> f3a = Field(Alt(["Paris", "Berlin"]))
>>> f4a = Field(" ?")
>>> s1 = Symbol([f1a, f2a, f3a, f4a], name="Symbol-netzob")
>>> f1b = Field("zoby")
>>> f2b = Field(", what's up in ")
>>> f3b = Field(Alt(["Paris", "Berlin"]))
>>> f4b = Field(" ?")
>>> s2 = Symbol([f1b, f2b, f3b, f4b], name="Symbol-zoby")
>>> for m in messages:
... abstractedSymbol = AbstractField.abstract(m, [s1, s2])
... print(abstractedSymbol.name)
Symbol-netzob
Symbol-netzob
Symbol-zoby
Symbol-zoby
:parameter data: the data that should be abstracted in symbol
:type data: :class:`str`
:parameter fields: a list of fields/symbols targeted during the abstraction process
:type fields: :class:`list` of :class:`netzob.Model.Vocabulary.AbstractField`
:return: a field/symbol
:rtype: :class:`netzob.Model.Vocabulary.AbstractField`
:raises: :class:`netzob.Model.Vocabulary.AbstractField.AbstractionException` if an error occurs while abstracting the data
"""
from netzob.Common.Utils.DataAlignment.DataAlignment import DataAlignment
for field in fields:
try:
DataAlignment.align([data], field, encoded=False)
return field
except:
pass
from netzob.Model.Vocabulary.UnknownSymbol import UnknownSymbol
from netzob.Model.Vocabulary.Messages.RawMessage import RawMessage
unknown_symbol = UnknownSymbol(RawMessage(data))
logging.error(
"Impossible to abstract the message in one of the specified symbols, we create an unknown symbol for it: '%s'",
unknown_symbol)
return unknown_symbol
def readSymbol(self, timeout=EmptySymbol.defaultReceptionTimeout()):
"""Read from the abstraction layer a message and abstract it
into a message.
The timeout parameter represents the amount of time (in millisecond) above which
no reception of a message triggers the reception of an :class:`netzob.Model.Vocabulary.EmptySymbol.EmptySymbol`. If timeout set to None
or to a negative value means it always wait for the reception of a message.
:keyword timeout: the time above which no reception of message triggers the reception of an :class:`netzob.Model.Vocabulary.EmptySymbol.EmptySymbol`
:type timeout: :class:`int`
:raise TypeError if the parameter is not valid and Exception if an error occurs.
"""
self._logger.debug("Reading data from communication channel...")
data = self.channel.read(timeout=timeout)
self._logger.debug("Received : {}".format(repr(data)))
symbol = None
# if we read some bytes, we try to abstract them
if len(data) > 0:
for potential in self.symbols:
try:
self.parser.parseMessage(RawMessage(data), potential)
symbol = potential
self.memory = self.parser.memory
self.specializer.memory = self.memory
break
except Exception:
symbol = None
if symbol is None and len(data) > 0:
msg = RawMessage(data)
symbol = UnknownSymbol(message=msg)
elif symbol is None and len(data) == 0:
symbol = EmptySymbol()
return (symbol, data)
def create_message(self,
cid,
data,
date=None,
source=None,
destination=None):
capture = self.__captures[str(cid)]
message = RawMessage(data=data,
date=date,
source=source,
destination=destination)
capture.messages.append(message)
self.__messages[str(message.id)] = message
return message
def generateTestSegments():
from netzob.Model.Vocabulary.Messages.RawMessage import RawMessage
from nemere.inference.analyzers import Value
from nemere.inference.segments import MessageSegment
bytedata = [
bytes([1, 2, 3, 4]),
bytes([2, 3, 4]),
bytes([1, 3, 4]),
bytes([2, 4]),
bytes([2, 3]),
bytes([20, 30, 37, 50, 69, 2, 30]),
bytes([37, 5, 69]),
bytes([0, 0, 0, 0]),
bytes([3, 2, 3, 4])
]
messages = [RawMessage(bd) for bd in bytedata]
analyzers = [Value(message) for message in messages]
segments = [MessageSegment(analyzer, 0, len(analyzer.message.data)) for analyzer in analyzers]
return segments
def clusterClusters():
"""
alternative idea of merging clusters by clustering them:
does not improve merging - perhaps the similarity matrix is not good enough?!
:return:
"""
# ClusterClusterer
clusterclusterer = ClusterClusterer(tyl.alignedClusters, dc)
# clusterDists = clusterclusterer.calcClusterDistances()
mergeEps, mergeMpts = clusterclusterer.autoconfigureDBSCAN()
cluclu, labels, mergeclusterer = clusterclusterer.clusterMessageTypesDBSCAN(mergeEps, min_samples=2)
clusterClustersNoiseless = {k: v for k, v in cluclu.items() if k > -1}
mergedClusters = ClusterClusterer.mergeClusteredClusters(clusterClustersNoiseless, tyl.messageObjClusters)
ClusterClusterer.printShouldMerge(list(clusterClustersNoiseless.values()), splitClusterReport.precisionRecallList)
mergedObjClusters = {lab: [comparator.messages[element[0].message] for element in segseq]
for lab, segseq in mergedClusters.items()}
inferenceParams.postProcess += "split+mergedAlt-{}-eps={:.2f}-min_samples={}".format(
type(mergeclusterer).__name__, mergeclusterer.eps, mergeclusterer.min_samples)
clusteredClusterReport = IndividualClusterReport(groundtruth, filechecker)
clusteredClusterReport.write(mergedObjClusters, inferenceParams.dict)
from netzob.Model.Vocabulary.Messages.RawMessage import RawMessage
from nemere.visualization.distancesPlotter import DistancesPlotter
typedClusterDummys = list()
for clun in clusterclusterer.clusterOrder:
clusta = None
for stats in clusteredClusterReport.precisionRecallList:
if stats is not None and stats[0] == clun:
clusta = stats[1] if stats[2] == 1.0 else "({})".format(stats[1])
break
msgdum = RawMessage(messageType=clusta)
typedClusterDummys.append(msgdum)
dipl = DistancesPlotter(specimens, "cluster-clustering-" + inferenceParams.plotTitle, False)
dipl.plotManifoldDistances(typedClusterDummys, clusterclusterer.distances, labels)
dipl.writeOrShowFigure(filechecker.reportFullPath)
def packetHandler(self, packet: Packet):
epoch = packet.time
l1Payload = bytes(packet)
if len(l1Payload) == 0:
return
# Build the RawMessage
rawMessage = RawMessage(l1Payload, epoch, source=None, destination=None)
if isinstance(packet, RadioTap):
# lift layer to Dot11 if there is a RadioTap dummy frame
packet = packet.payload
if self.importLayer == 2:
(l2Proto, l2SrcAddr, l2DstAddr, l2Payload) = self.__decodeLayer2(packet)
if len(l2Payload) == 0:
return
# Build the L2NetworkMessage
l2Message = L2NetworkMessage(l2Payload, epoch, l2Proto, l2SrcAddr, l2DstAddr)
self._messages.add(l2Message)
self._rawmessages.add(rawMessage)
else:
# Use Netzob's PCAPImporter if layer 2 is not WLAN
raise NetzobImportException("PCAP", "Unsupported import layer. Currently only handles layer 2.",
PCAPImporter.INVALID_LAYER2)
def abstract(data, fields):
"""Search in the fields/symbols the first one that can abstract the data.
>>> from netzob.all import *
>>> messages = ["{0}, what's up in {1} ?".format(pseudo, city) for pseudo in ['netzob', 'zoby'] for city in ['Paris', 'Berlin']]
>>> f1a = Field(name="name", domain="netzob")
>>> f2a = Field(name="question", domain=", what's up in ")
>>> f3a = Field(name="city", domain=Alt(["Paris", "Berlin"]))
>>> f4a = Field(name="mark", domain=" ?")
>>> s1 = Symbol([f1a, f2a, f3a, f4a], name="Symbol-netzob")
>>> f1b = Field(name="name", domain="zoby")
>>> f2b = Field(name="question", domain=", what's up in ")
>>> f3b = Field(name="city", domain=Alt(["Paris", "Berlin"]))
>>> f4b = Field(name="mark", domain=" ?")
>>> s2 = Symbol([f1b, f2b, f3b, f4b], name="Symbol-zoby")
>>> for m in messages:
... (abstractedSymbol, structured_data) = AbstractField.abstract(m, [s1, s2])
... print(structured_data)
... print(abstractedSymbol.name)
OrderedDict([('name', b'netzob'), ('question', b", what's up in "), ('city', b'Paris'), ('mark', b' ?')])
Symbol-netzob
OrderedDict([('name', b'netzob'), ('question', b", what's up in "), ('city', b'Berlin'), ('mark', b' ?')])
Symbol-netzob
OrderedDict([('name', b'zoby'), ('question', b", what's up in "), ('city', b'Paris'), ('mark', b' ?')])
Symbol-zoby
OrderedDict([('name', b'zoby'), ('question', b", what's up in "), ('city', b'Berlin'), ('mark', b' ?')])
Symbol-zoby
:parameter data: the data that should be abstracted in symbol
:type data: :class:`str`
:parameter fields: a list of fields/symbols targeted during the abstraction process
:type fields: :class:`list` of :class:`netzob.Model.Vocabulary.AbstractField`
:return: a field/symbol and the structured received message
:rtype: a tuple (:class:`netzob.Model.Vocabulary.AbstractField`, dict)
:raises: :class:`netzob.Model.Vocabulary.AbstractField.AbstractionException` if an error occurs while abstracting the data
"""
from netzob.Common.Utils.DataAlignment.DataAlignment import DataAlignment
for field in fields:
try:
# Try to align/parse the data with the current field
alignedData = DataAlignment.align([data], field, encoded=False)
# If it matches, we build a dict that contains, for each field, the associated value that was present in the message
structured_data = OrderedDict()
for fields_value in alignedData:
for i, field_value in enumerate(fields_value):
structured_data[alignedData.headers[i]] = field_value
return (field, structured_data)
except:
pass
from netzob.Model.Vocabulary.UnknownSymbol import UnknownSymbol
from netzob.Model.Vocabulary.Messages.RawMessage import RawMessage
unknown_symbol = UnknownSymbol(RawMessage(data))
structured_data = OrderedDict()
logging.error(
"Impossible to abstract the message in one of the specified symbols, we create an unknown symbol for it: '%s'",
unknown_symbol)
return (unknown_symbol, structured_data)
def __packetHandler(self, header, payload):
"""Internal callback executed on each packet when parsing the pcap"""
(secs, usecs) = header.getts()
epoch = secs + (usecs / 1000000.0)
self._logger.debug('ImportLayer = ' + str(self.importLayer))
if self.importLayer == 1:
if len(payload) == 0:
return
# Build the RawMessage
rawMessage = RawMessage(payload,
epoch,
source=None,
destination=None)
self.messages.add(rawMessage)
elif self.importLayer == 2:
try:
(l2Proto, l2SrcAddr, l2DstAddr, l2Payload,
etherType) = self.__decodeLayer2(header, payload)
except NetzobImportException as e:
self._logger.warn(
"An error occured while decoding layer2 of a packet: {0}".
format(e))
return
if len(l2Payload) == 0:
return
# Build the L2NetworkMessage
l2Message = L2NetworkMessage(l2Payload, epoch, l2Proto, l2SrcAddr,
l2DstAddr)
self.messages.add(l2Message)
elif self.importLayer == 3:
try:
(l2Proto, l2SrcAddr, l2DstAddr, l2Payload,
etherType) = self.__decodeLayer2(header, payload)
(l3Proto, l3SrcAddr, l3DstAddr, l3Payload,
ipProtocolNum) = self.__decodeLayer3(etherType, l2Payload)
except NetzobImportException as e:
self._logger.warn(
"An error occured while decoding layer2 and layer3 of a packet: {0}"
.format(e))
return
if len(l3Payload) == 0:
return
# Build the L3NetworkMessage
l3Message = L3NetworkMessage(l3Payload, epoch, l2Proto, l2SrcAddr,
l2DstAddr, l3Proto, l3SrcAddr,
l3DstAddr)
self.messages.add(l3Message)
elif self.importLayer == 4:
try:
(l2Proto, l2SrcAddr, l2DstAddr, l2Payload,
etherType) = self.__decodeLayer2(header, payload)
(l3Proto, l3SrcAddr, l3DstAddr, l3Payload,
ipProtocolNum) = self.__decodeLayer3(etherType, l2Payload)
(l4Proto, l4SrcPort, l4DstPort,
l4Payload) = self.__decodeLayer4(ipProtocolNum, l3Payload)
except NetzobImportException as e:
self._logger.warn(
"An error occured while decoding layer2, layer3 or layer4 of a packet: {0}"
.format(e))
return
if len(l4Payload) == 0:
return
# Build the L4NetworkMessage
l4Message = L4NetworkMessage(l4Payload, epoch, l2Proto, l2SrcAddr,
l2DstAddr, l3Proto, l3SrcAddr,
l3DstAddr, l4Proto, l4SrcPort,
l4DstPort)
self.messages.add(l4Message)
else:
try:
(l2Proto, l2SrcAddr, l2DstAddr, l2Payload,
etherType) = self.__decodeLayer2(header, payload)
(l3Proto, l3SrcAddr, l3DstAddr, l3Payload,
ipProtocolNum) = self.__decodeLayer3(etherType, l2Payload)
(l4Proto, l4SrcPort, l4DstPort,
l4Payload) = self.__decodeLayer4(ipProtocolNum, l3Payload)
except NetzobImportException as e:
self._logger.warn(
"An error occured while decoding layer2, layer3, layer4 or layer5 of a packet: {0}"
.format(e))
return
if len(l4Payload) == 0:
return
l5Message = L4NetworkMessage(l4Payload, epoch, l2Proto, l2SrcAddr,
l2DstAddr, l3Proto, l3SrcAddr,
l3DstAddr, l4Proto, l4SrcPort,
l4DstPort)
self.messages.add(l5Message)
"merged-{}-{}-eps={:.2f}-min_samples={}".format(
tokenizer,
type(mergeclusterer).__name__, mergeclusterer.eps,
mergeclusterer.min_samples), comparator)
from netzob.Model.Vocabulary.Messages.RawMessage import RawMessage
from visualization.distancesPlotter import DistancesPlotter
typedClusterDummys = list()
for clunu in clusterclusterer.clusterOrder:
clusta = None
for stats in clusterStats:
if stats is not None and stats[0] == clunu:
clusta = stats[1] if stats[2] == 1.0 else "({})".format(
stats[1])
break
msgdum = RawMessage(messageType=clusta)
typedClusterDummys.append(msgdum)
dp = DistancesPlotter(specimens, "cluster-clustering-" + plotTitle,
False)
dp.plotManifoldDistances(typedClusterDummys,
clusterclusterer.distances, labels)
dp.writeOrShowFigure()
# # overwrite existing variables
# # # # # # # # # # # # # # # # # # # # # # # #
# messageClusters = mergedClusters
#
# # align clusters that have been merged
# mergedAligned = dict()
# for cLabel, clusterMerges in messageClusters.items(): # type: Union[int, str], List[Tuple[MessageSegment]]