def nemetylRefinements( segmentsPerMsg: Sequence[Sequence[MessageSegment]] ) -> List[List[MessageSegment]]: """ Refine the segmentation using specific improvements for the feature: Inflections of gauss-filtered bit-congruence deltas. :param segmentsPerMsg: a list of one list of segments per message. :return: refined segments in list per message """ import inference.formatRefinement as refine print("Refine segmentation (nemetyl refinements)...") refinedPerMsg = list() for msg in segmentsPerMsg: # merge consecutive segments of printable-char values (\t, \n, \r, >= 0x20 and <= 0x7e) into one text field. charsMerged = refine.MergeConsecutiveChars(msg).merge() charSplited = refine.ResplitConsecutiveChars(charsMerged).split() refinedPerMsg.append(charSplited) # for tests use test_segment-refinements.py moco = refine.CropDistinct.countCommonValues(refinedPerMsg) newstuff = list() for msg in refinedPerMsg: croppedMsg = refine.CropDistinct(msg, moco).split() charmerged = refine.CumulativeCharMerger(croppedMsg).merge() splitfixed = refine.SplitFixed(charmerged).split(0, 1) newstuff.append(splitfixed) return newstuff
def charRefinements( segmentsPerMsg: Sequence[Sequence[MessageSegment]] ) -> List[Sequence[MessageSegment]]: """ Refine the segmentation using specific improvements for the feature: Inflections of gauss-filtered bit-congruence deltas. Copy of inference.segmentHandler.refinements without * frequency reinforced segments (CropDistinct) and * splitting of first segment (SplitFixed) :param segmentsPerMsg: a list of one list of segments per message. :return: refined segments in list per message """ import inference.formatRefinement as refine print("Refine segmentation (char refinements)...") refinedPerMsg = list() for msg in segmentsPerMsg: # merge consecutive segments of printable-char values (\t, \n, \r, >= 0x20 and <= 0x7e) into one text field. charsMerged = refine.MergeConsecutiveChars(msg).merge() charSplited = refine.ResplitConsecutiveChars(charsMerged).split() refinedPerMsg.append(charSplited) # for tests use test_segment-refinements.py newstuff = list() for msg in refinedPerMsg: charmerged = refine.CumulativeCharMerger(msg).merge() newstuff.append(charmerged) return newstuff
def refinements(segmentsPerMsg: List[List[MessageSegment]]): """ Refine the segmentation using specific improvements for the feature: Inflections of gauss-filtered bit-congruence deltas. :param segmentsPerMsg: a list of one list of segments per message. :return: refined segments in on list per message """ import inference.formatRefinement as refine print("Refine segmentation...") refinedPerMsg = [ # merge consecutive segments of printable-char values (\t, \n, \r, >= 0x20 and <= 0x7e) into one text field. refine.ResplitConsecutiveChars( refine.MergeConsecutiveChars(m).merge() ).split() for m in segmentsPerMsg] return refinedPerMsg
def originalRefinements( segmentsPerMsg: Sequence[Sequence[MessageSegment]] ) -> List[List[MessageSegment]]: """ Refine the segmentation according to the WOOT2018 paper method using specific improvements for the feature: Inflections of gauss-filtered bit-congruence deltas. :param segmentsPerMsg: a list of one list of segments per message. :return: refined segments in list per message """ import inference.formatRefinement as refine print("Refine segmentation (WOOT18 refinements)...") refinedPerMsg = list() for msg in segmentsPerMsg: # merge consecutive segments of printable-char values (\t, \n, \r, >= 0x20 and <= 0x7e) into one text field. charsMerged = refine.MergeConsecutiveChars(msg).merge() charSplited = refine.ResplitConsecutiveChars(charsMerged).split() refinedPerMsg.append(charSplited) return refinedPerMsg