def nemetylRefinements(
segmentsPerMsg: Sequence[Sequence[MessageSegment]]
) -> List[List[MessageSegment]]:
"""
Refine the segmentation using specific improvements for the feature:
Inflections of gauss-filtered bit-congruence deltas.
:param segmentsPerMsg: a list of one list of segments per message.
:return: refined segments in list per message
"""
import inference.formatRefinement as refine
print("Refine segmentation (nemetyl refinements)...")
refinedPerMsg = list()
for msg in segmentsPerMsg:
# merge consecutive segments of printable-char values (\t, \n, \r, >= 0x20 and <= 0x7e) into one text field.
charsMerged = refine.MergeConsecutiveChars(msg).merge()
charSplited = refine.ResplitConsecutiveChars(charsMerged).split()
refinedPerMsg.append(charSplited)
# for tests use test_segment-refinements.py
moco = refine.CropDistinct.countCommonValues(refinedPerMsg)
newstuff = list()
for msg in refinedPerMsg:
croppedMsg = refine.CropDistinct(msg, moco).split()
charmerged = refine.CumulativeCharMerger(croppedMsg).merge()
splitfixed = refine.SplitFixed(charmerged).split(0, 1)
newstuff.append(splitfixed)
return newstuff
def charRefinements(
segmentsPerMsg: Sequence[Sequence[MessageSegment]]
) -> List[Sequence[MessageSegment]]:
"""
Refine the segmentation using specific improvements for the feature:
Inflections of gauss-filtered bit-congruence deltas.
Copy of inference.segmentHandler.refinements without
* frequency reinforced segments (CropDistinct) and
* splitting of first segment (SplitFixed)
:param segmentsPerMsg: a list of one list of segments per message.
:return: refined segments in list per message
"""
import inference.formatRefinement as refine
print("Refine segmentation (char refinements)...")
refinedPerMsg = list()
for msg in segmentsPerMsg:
# merge consecutive segments of printable-char values (\t, \n, \r, >= 0x20 and <= 0x7e) into one text field.
charsMerged = refine.MergeConsecutiveChars(msg).merge()
charSplited = refine.ResplitConsecutiveChars(charsMerged).split()
refinedPerMsg.append(charSplited)
# for tests use test_segment-refinements.py
newstuff = list()
for msg in refinedPerMsg:
charmerged = refine.CumulativeCharMerger(msg).merge()
newstuff.append(charmerged)
return newstuff
def refinements(segmentsPerMsg: List[List[MessageSegment]]):
"""
Refine the segmentation using specific improvements for the feature:
Inflections of gauss-filtered bit-congruence deltas.
:param segmentsPerMsg: a list of one list of segments per message.
:return: refined segments in on list per message
"""
import inference.formatRefinement as refine
print("Refine segmentation...")
refinedPerMsg = [
# merge consecutive segments of printable-char values (\t, \n, \r, >= 0x20 and <= 0x7e) into one text field.
refine.ResplitConsecutiveChars(
refine.MergeConsecutiveChars(m).merge()
).split()
for m in segmentsPerMsg]
return refinedPerMsg
def originalRefinements(
segmentsPerMsg: Sequence[Sequence[MessageSegment]]
) -> List[List[MessageSegment]]:
"""
Refine the segmentation according to the WOOT2018 paper method using specific improvements for the feature:
Inflections of gauss-filtered bit-congruence deltas.
:param segmentsPerMsg: a list of one list of segments per message.
:return: refined segments in list per message
"""
import inference.formatRefinement as refine
print("Refine segmentation (WOOT18 refinements)...")
refinedPerMsg = list()
for msg in segmentsPerMsg:
# merge consecutive segments of printable-char values (\t, \n, \r, >= 0x20 and <= 0x7e) into one text field.
charsMerged = refine.MergeConsecutiveChars(msg).merge()
charSplited = refine.ResplitConsecutiveChars(charsMerged).split()
refinedPerMsg.append(charSplited)
return refinedPerMsg