def annotateFieldTypes(
analyzerType: type,
analysisArgs: Union[Tuple, None],
comparator,
unit=MessageAnalyzer.U_BYTE) -> List[Tuple[TypedSegment]]:
"""
:return: list of lists of segments that are annotated with their field type.
"""
segmentedMessages = [
segmentsFromLabels(
MessageAnalyzer.findExistingAnalysis(analyzerType, unit, l4msg,
analysisArgs),
comparator.dissections[rmsg])
for l4msg, rmsg in comparator.messages.items()
]
return segmentedMessages
def messageSegmentation(self) -> List[MessageSegment]:
"""
Segment message by determining local extrema of sigma-s-gauss-filtered sliding n-byte-mean bit-congruence.
>>> from netzob.Model.Vocabulary.Messages.L4NetworkMessage import L4NetworkMessage
>>> tstmsg = '19040aec0000027b000012850a6400c8d23d06a2535ed71ed23d09faa4673315d23d09faa1766325d23d09faa17b4b10'
>>> l4m = L4NetworkMessage(bytes.fromhex(tstmsg))
>>> hbg = HorizonBitcongruenceGauss(l4m)
>>> hbg.setAnalysisParams()
>>> hbg.analyze()
>>> spm = hbg.messageSegmentation()
>>> print(b''.join([seg.bytes for seg in spm]).hex() == spm[0].message.data.hex())
True
:return: Segmentation of this message based on this analyzer's type.
"""
if not self.values:
if not self._analysisArgs:
raise ValueError('No values or analysis parameters set.')
self.analyze()
bcd = MessageAnalyzer.findExistingAnalysis(BitCongruenceDelta,
MessageAnalyzer.U_BYTE,
self.message)
# all local minima
bclmins = self.pinpointMinima()
# local maxima, if bc[e] < bc[e+1] or bc[e] > 2*s2mbc[e] for all e in cadidate indices
bclmaxs = self.pinpointMaxima()
bcdmaxs = [
e for e in bclmaxs
if bcd.values[e + 1] > bcd.values[e] or bcd.values[e] > 2 *
self.bitcongruences[e]
]
minmax = bclmins
for bdm in bcdmaxs: # only keep bcdmaxs if not in scope if min
if bdm + 1 not in minmax and bdm - 1 not in minmax:
minmax.append(bdm)
# starts of plateaus of bit congruences
bcplats = MessageAnalyzer.plateouStart(
self.bitcongruences)[0] # bcd.values
for bps in bcplats: # only keep platoustarts if not in scope if min or max
if bps + 1 not in minmax and bps - 1 not in minmax:
minmax.append(bps)
# # separate nan-values
# nansep = MessageAnalyzer.separateNaNs(self.values)
relevantPositions = list(sorted(minmax))
# get candidates to cut segments from message
cutCandidates = [0] + [int(b) for b in relevantPositions if not numpy.isnan(b)] \
+ [len(self._message.data)] # add the message end
# cut only where a segment is of a length larger than 1
cutPositions = [0] + [
right for left, right in zip(cutCandidates[:-1], cutCandidates[1:])
if right - left > 1
]
# cutPositions = list(sorted(cutPositions + nansep[0]))
# add the end of the message if its not already there
if cutPositions[-1] != cutCandidates[-1]:
cutPositions[-1] = cutCandidates[-1]
segments = list()
for lmaxCurr, lmaxNext in zip(cutPositions[:-1], cutPositions[1:]):
segments.append(MessageSegment(self, lmaxCurr,
lmaxNext - lmaxCurr))
return segments
def setAnalysisParams(self, analysisMethod: Type[MessageAnalyzer],
*analysisArgs):
self._am = MessageAnalyzer.findExistingAnalysis(
analysisMethod, MessageAnalyzer.U_BYTE, self._message,
analysisArgs)
def cacheAndLoadDC(pcapfilename: str, analysisTitle: str, tokenizer: str, debug: bool,
analyzerType: type, analysisArgs: Tuple=None, sigma: float=None, filterTrivial=False,
refinementCallback:Union[Callable, None] = refinements,
disableCache=False) \
-> Tuple[SpecimenLoader, MessageComparator, List[Tuple[MessageSegment]], DistanceCalculator,
float, float]:
"""
cache or load the DistanceCalculator to or from the filesystem
:param filterTrivial: Filter out **one-byte** segments and such just consisting of **zeros**.
:param disableCache: When experimenting with distances manipulation, deactivate caching!
:return:
"""
pcapbasename = os.path.basename(pcapfilename)
# if refinementCallback == pcaMocoRefinements:
# sigma = pcamocoSigmapertrace[pcapbasename] if not sigma and pcapbasename in pcamocoSigmapertrace else \
# 0.9 if not sigma else sigma
# else:
sigma = sigmapertrace[pcapbasename] if not sigma and pcapbasename in sigmapertrace else \
0.9 if not sigma else sigma
pcapName = os.path.splitext(pcapbasename)[0]
# noinspection PyUnboundLocalVariable
tokenparm = tokenizer if tokenizer != "nemesys" else \
"{}{:.0f}".format(tokenizer, sigma * 10)
dccachefn = os.path.join(
cacheFolder, 'cache-dc-{}-{}-{}-{}-{}.{}'.format(
analysisTitle, tokenparm, "filtered" if filterTrivial else "all",
refinementCallback.__name__
if refinementCallback is not None else "raw", pcapName, 'ddc'))
# dccachefn = 'cache-dc-{}-{}-{}.{}'.format(analysisTitle, tokenizer, pcapName, 'dc')
if disableCache or not os.path.exists(dccachefn):
# dissect and label messages
print("Load messages from {}...".format(pcapName))
specimens = SpecimenLoader(pcapfilename, 2, True)
comparator = MessageComparator(specimens, 2, True, debug=debug)
print("Segmenting messages...", end=' ')
segmentationTime = time.time()
# select tokenizer by command line parameter
if tokenizer == "tshark":
# 1. segment messages according to true fields from the labels
segmentedMessages = annotateFieldTypes(analyzerType, analysisArgs,
comparator)
elif tokenizer == "4bytesfixed":
# 2. segment messages into fixed size chunks for testing
segmentedMessages = segmentsFixed(4, comparator, analyzerType,
analysisArgs)
elif tokenizer == "nemesys":
# 3. segment messages by NEMESYS
segmentsPerMsg = bcDeltaGaussMessageSegmentation(specimens, sigma)
# get analyzer requested by analyzerType/analysisArgs
segmentedMessages = [[
MessageSegment(
MessageAnalyzer.findExistingAnalysis(
analyzerType, MessageAnalyzer.U_BYTE, seg.message,
analysisArgs), seg.offset, seg.length) for seg in msg
] for msg in segmentsPerMsg]
if refinementCallback is not None:
if refinementCallback.__code__.co_argcount > 1:
# assume the second argument is expected to be a distance calculator
chainedSegments = list(
chain.from_iterable(segmentedMessages))
print("Refinement: Calculate distance for {} segments...".
format(len(chainedSegments)))
if len(chainedSegments)**2 > MemmapDC.maxMemMatrix:
refinementDC = MemmapDC(chainedSegments)
else:
refinementDC = DelegatingDC(chainedSegments)
segmentedMessages = refinementCallback(
segmentedMessages, refinementDC)
else:
segmentedMessages = refinementCallback(segmentedMessages)
# segments = list(chain.from_iterable(segmentedMessages))
segmentationTime = time.time() - segmentationTime
print("done.")
if filterTrivial:
# noinspection PyUnboundLocalVariable
chainedSegments = [
seg for seg in chain.from_iterable(segmentedMessages)
if seg.length > 1 and set(seg.values) != {0}
]
else:
# noinspection PyUnboundLocalVariable
chainedSegments = list(chain.from_iterable(segmentedMessages))
print("Calculate distance for {} segments...".format(
len(chainedSegments)))
# dc = DistanceCalculator(chainedSegments, reliefFactor=0.33) # Pairwise similarity of segments: dc.distanceMatrix
dist_calc_segmentsTime = time.time()
if len(chainedSegments)**2 > MemmapDC.maxMemMatrix:
dc = MemmapDC(chainedSegments)
else:
dc = DelegatingDC(chainedSegments)
assert chainedSegments == dc.rawSegments
dist_calc_segmentsTime = time.time() - dist_calc_segmentsTime
try:
with open(dccachefn, 'wb') as f:
pickle.dump((segmentedMessages, comparator, dc), f,
pickle.HIGHEST_PROTOCOL)
except MemoryError as e:
print("DC could not be cached due to a MemoryError. Removing",
dccachefn, "and continuing.")
os.remove(dccachefn)
else:
print("Load distances from cache file {}".format(dccachefn))
with open(dccachefn, 'rb') as f:
segmentedMessages, comparator, dc = pickle.load(f)
if not (isinstance(comparator, MessageComparator)
and isinstance(dc, DistanceCalculator)):
print('Loading of cached distances failed.')
exit(10)
specimens = comparator.specimens
# chainedSegments = list(chain.from_iterable(segmentedMessages))
segmentationTime, dist_calc_segmentsTime = None, None
return specimens, comparator, segmentedMessages, dc, segmentationTime, dist_calc_segmentsTime