def printResultMatrix(matchprecision):
"""
Prettyprint the content of a dict of symbols and their quality parameters.
:param matchprecision: a list of tuples of symbols/formats and their
(qualitymetric, fieldcount, exactf, nearf, uospecific)
:type matchprecision: list of tuples: [ (netzob.Symbol.name, formatname, float, int, int, int, int) ]
:rtype: None
"""
# qualmatrix = [[] for i in range(0,6)]
qualmatrix = [ [""], ["Quality"], ["Under-/Over-Specified"],
["Near Field Matches"], ["Exact Field Matches"], ["Field Count"] ]
for symbol, mformat, quality, fieldcount, exactf, nearf, uospecific in matchprecision:
qualmatrix[0].append(symbol + "/" + mformat)
qualmatrix[1].append(round(quality,3))
qualmatrix[2].append(uospecific)
qualmatrix[3].append(nearf)
qualmatrix[4].append(exactf)
qualmatrix[5].append(fieldcount)
# print "{:s}: {:f}".format(symbol.name, quality)
# print("Field Under- (pos.)/Over- (neg.) Specification: {:d}".format(uospecific))
# print("Near Field Matches: {:d}".format(nearf))
# print("Exact Field Matches: {:d}".format(exactf))
# print("Field Count: {:d}".format(fieldcount))
ml = MatrixList()
ml.headers = qualmatrix[0]
ml.extend(qualmatrix[1:])
print(ml)
def execute(self, data):
"""Execute the parallel alignment on the specified list of data
:param data: the list of data that will be aligned
:type data: a :class:`list` of data to align
:return: a list of aligned data sorted in order to respect the provided order of data.
:rtype: a :class:`netzob.Common.Utils.MatrixList.MatrixList`
"""
# Create a list of data removed from duplicate entry
noDuplicateData = list(set(data))
# Async result host just aligned data under form [(data, alignedData)]
self.asyncResult = OrderedDict()
# Measure start time
start = time.time()
# Create a pool of 'nbThead' threads (process)
pool = multiprocessing.Pool(self.nbThread)
# Execute Data Alignment
pool.map_async(_executeDataAlignment,
list(
zip(noDuplicateData,
[self.field] * len(noDuplicateData),
[self.encoded] * len(noDuplicateData),
[self.styled] * len(noDuplicateData))),
callback=self.__collectResults_cb)
# Waits all alignment tasks finish
pool.close()
pool.join()
# Measure end time
end = time.time()
# create a Matrix List based on aligned data and requested data
result = MatrixList()
if len(data) > 0:
result.headers = self.asyncResult[data[0]].headers
for d in data:
if d not in list(self.asyncResult.keys()):
raise Exception(
"At least one data ({0}) has not been successfully computed by the alignment"
.format(repr(d)))
result.extend(self.asyncResult[d])
# check the number of computed alignment
if len(result) != len(data):
raise Exception(
"There are not the same number of alignment ({0}) than the number of data ({1})"
.format(len(result), len(data)))
self._logger.debug(
"Alignment of {0} data took {1}s with {2} threads.".format(
len(data), end - start, self.nbThread))
return result
def printMatrix(lines: Iterable[Iterable], headers: Iterable=None):
ml = MatrixList()
if headers:
ml.headers = headers
strlines = [ [ "{:0.3f}".format(cell) if isinstance(cell, float) else str(cell) for cell in row] for row in lines ]
ml.extend(strlines)
print(ml)
def execute(self, data):
"""Execute the parallel alignment on the specified list of data
:param data: the list of data that will be aligned
:type data: a :class:`list` of data to align
:return: a list of aligned data sorted in order to respect the provided order of data.
:rtype: a :class:`netzob.Common.Utils.MatrixList.MatrixList`
"""
# Create a list of data removed from duplicate entry
noDuplicateData = list(set(data))
# Async result host just aligned data under form [(data, alignedData)]
self.asyncResult = OrderedDict()
# Measure start time
start = time.time()
# Create a pool of 'nbThead' threads (process)
pool = multiprocessing.Pool(self.nbThread)
# Execute Data Alignment
pool.map_async(
_executeDataAlignment,
list(
zip(noDuplicateData, [self.field] * len(noDuplicateData),
[self.encoded] * len(noDuplicateData), [self.styled] * len(
noDuplicateData))),
callback=self.__collectResults_cb)
# Waits all alignment tasks finish
pool.close()
pool.join()
# Measure end time
end = time.time()
# create a Matrix List based on aligned data and requested data
result = MatrixList()
if len(data) > 0:
result.headers = self.asyncResult[data[0]].headers
for d in data:
if d not in list(self.asyncResult.keys()):
raise Exception(
"At least one data ({0}) has not been successfully computed by the alignment".
format(repr(d)))
result.extend(self.asyncResult[d])
# check the number of computed alignment
if len(result) != len(data):
raise Exception(
"There are not the same number of alignment ({0}) than the number of data ({1})".
format(len(result), len(data)))
self._logger.debug("Alignment of {0} data took {1}s with {2} threads.".
format(len(data), end - start, self.nbThread))
return result
def execute(self):
"""Execute the alignment of data following specified field
"""
if self.data is None:
raise TypeError("Data cannot be None")
if self.field is None:
raise TypeError("Field cannot be None")
# Aligned messages are stored in a MatrixList for better display
result = MatrixList()
# We retrieve all the leaf fields of the root of the provided field
rootLeafFields = self.__root.getLeafFields(depth=self.depth)
# if self.__root != self.field:
# targetedFieldLeafFields = self.field.getLeafFields(depth=self.depth)
# else:
targetedFieldLeafFields = rootLeafFields
result.headers = [str(field.name) for field in targetedFieldLeafFields]
from netzob.Model.Vocabulary.Domain.Parser.MessageParser import MessageParser
for d in self.data:
mp = MessageParser()
# alignedMsg = mp.parseRaw(TypeConverter.convert(d, HexaString, Raw), targetedFieldLeafFields)
alignedMsg = next(mp.parseRaw(d, targetedFieldLeafFields))
alignedEncodedMsg = []
for ifield, currentField in enumerate(targetedFieldLeafFields):
# now we apply encoding and mathematic functions
fieldValue = alignedMsg[ifield]
if self.encoded and len(
list(currentField.encodingFunctions.values())) > 0:
for encodingFunction in list(
currentField.encodingFunctions.values()):
fieldValue = encodingFunction.encode(fieldValue)
else:
fieldValue = TypeConverter.convert(fieldValue, BitArray,
Raw)
if currentField in self.field.getLeafFields(depth=self.depth):
alignedEncodedMsg.append(fieldValue)
result.append(alignedEncodedMsg)
return result
def execute(self):
"""Execute the alignment of data following specified field
"""
if self.data is None:
raise TypeError("Data cannot be None")
if self.field is None:
raise TypeError("Field cannot be None")
# Aligned messages are stored in a MatrixList for better display
result = MatrixList()
# We retrieve all the leaf fields of the root of the provided field
rootLeafFields = self.__root._getLeafFields(depth=self.depth)
# if self.__root != self.field:
# targetedFieldLeafFields = self.field._getLeafFields(depth=self.depth)
# else:
targetedFieldLeafFields = rootLeafFields
result.headers = [str(field.name) for field in targetedFieldLeafFields]
from netzob.Common.Models.Vocabulary.Domain.Parser.MessageParser import MessageParser
for d in self.data:
mp = MessageParser()
# alignedMsg = mp.parseRaw(TypeConverter.convert(d, HexaString, Raw), targetedFieldLeafFields)
alignedMsg = mp.parseRaw(d, targetedFieldLeafFields)
alignedEncodedMsg = []
for ifield, currentField in enumerate(targetedFieldLeafFields):
# now we apply encoding and mathematic functions
fieldValue = alignedMsg[ifield]
if self.encoded and len(currentField.encodingFunctions.values()) > 0:
for encodingFunction in currentField.encodingFunctions.values():
fieldValue = encodingFunction.encode(fieldValue)
else:
fieldValue = TypeConverter.convert(fieldValue, BitArray, Raw)
if currentField in self.field._getLeafFields(depth=self.depth):
alignedEncodedMsg.append(fieldValue)
result.append(alignedEncodedMsg)
return result
def printMinMax(self,
formatmatchmetrics: Dict[Tuple[int, AbstractMessage], FormatMatchScore]):
"""
Print the Format Match Score min/max per threshold.
:param formatmatchmetrics: Dict[Threshold, Message], FormatMatchScore]
"""
mmm = self.minMaxMean(formatmatchmetrics)
qualmatrix = [["Thresh"], ["min"], ["max"], ["mean"]]
for th, (minft, maxft, meanft) in mmm.items():
qualmatrix[0].append(str(th))
qualmatrix[1].append("{:03.3f}".format(minft))
qualmatrix[2].append("{:03.3f}".format(maxft))
qualmatrix[3].append("{:03.3f}".format(meanft))
ml = MatrixList()
ml.headers = qualmatrix[0]
ml.extend(qualmatrix[1:])
print('Overall format matching score statistics:')
print(ml)
