def test11get_match_element_match_context_input_validation(self):
"""Check if an exception is raised, when other classes than MatchContext are used in get_match_element."""
model_element = Base64StringModelElement(self.id_)
data = b"VGhpcyBpcyBzb21lIHN0cmluZyB0byBiZSBlbmNvZGVkLg=="
model_element.get_match_element(self.path, DummyMatchContext(data))
model_element.get_match_element(self.path, MatchContext(data))
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, MatchElement(self.path, data, None, None))
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, data)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, data.decode())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, 123)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, 123.22)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, True)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, None)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, [])
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, {"key": MatchContext(data)})
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, set())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, ())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, model_element)
def test8get_match_element_no_match(self):
"""Parse not matching substring from MatchContext and check if the MatchContext was not changed."""
data = b"!Hello World"
match_context = DummyMatchContext(data)
base64_dme = Base64StringModelElement("s0")
match_element = base64_dme.get_match_element("base64", match_context)
self.assertIsNone(match_element, None)
self.assertEqual(match_context.match_data, data)
def test9get_match_element_unicode_exception(self):
"""Parse a Base64 string which can not be decoded as UTF-8, so it has to be returned base64 encoded."""
# ² encoded with ISO-8859-1
base64_string = b"sg=="
match_context = DummyMatchContext(base64_string)
base64_dme = Base64StringModelElement(self.id_)
match_element = base64_dme.get_match_element(self.path, match_context)
self.compare_match_results(base64_string, match_element, match_context,
self.id_, self.path, base64_string,
base64_string, None)
def test5get_match_element_valid_match_string_without_padding(self):
"""Parse matching substring without padding from MatchContext and check if the MatchContext was updated with all base64 data."""
string = b"This is some string to be encoded without the padding character =."
base64_string = b"VGhpcyBpcyBzb21lIHN0cmluZyB0byBiZSBlbmNvZGVkIHdpdGhvdXQgdGhlIHBhZGRpbmcgY2hhcmFjdGVyID0u"
match_context = DummyMatchContext(base64_string)
base64_dme = Base64StringModelElement(self.id_)
match_element = base64_dme.get_match_element(self.path, match_context)
self.compare_match_results(base64_string, match_element, match_context,
self.id_, self.path, base64_string, string,
None)
def test4get_match_element_valid_match_string_with_one_byte_padding(self):
"""Parse matching substring with padding from MatchContext and check if the MatchContext was updated with all base64 data."""
string = b"This is some encoded strin"
base64_string = b"VGhpcyBpcyBzb21lIGVuY29kZWQgc3RyaW4="
match_context = DummyMatchContext(base64_string)
base64_dme = Base64StringModelElement(self.id_)
match_element = base64_dme.get_match_element(self.path, match_context)
self.compare_match_results(base64_string, match_element, match_context,
self.id_, self.path, base64_string, string,
None)
def test3get_match_element_valid_match_string_with_padding(self):
"""Parse matching substring with padding from MatchContext and check if the MatchContext was updated with all base64 data."""
string = b"This is some string to be encoded."
base64_string = b"VGhpcyBpcyBzb21lIHN0cmluZyB0byBiZSBlbmNvZGVkLg=="
match_context = DummyMatchContext(base64_string)
base64_dme = Base64StringModelElement(self.id_)
match_element = base64_dme.get_match_element(self.path, match_context)
self.compare_match_results(base64_string, match_element, match_context,
self.id_, self.path, base64_string, string,
None)
def test7get_match_element_valid_match_string_with_partial_length(self):
"""Parse matching substring out of the MatchContext and check if the MatchContext was updated with all base64 data."""
string = b"This is some encoded strin"
base64_string = b"VGhpcyBpcyBzb21lIGVuY29kZWQgc3RyaW4="
data = base64_string + b"\nContent: Public Key"
match_context = DummyMatchContext(data)
base64_dme = Base64StringModelElement(self.id_)
match_element = base64_dme.get_match_element(self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, base64_string, string,
None)
def test6get_match_element_valid_match_string_without_exact_length(self):
"""Parse matching substring without exact length (divisible by 4) and check if the MatchContext was updated with all base64 data."""
string = b"This is some encoded strin"
base64_string = b"VGhpcyBpcyBzb21lIGVuY29kZWQgc3RyaW4"
match_context = DummyMatchContext(base64_string)
base64_dme = Base64StringModelElement(self.id_)
match_element = base64_dme.get_match_element(self.path, match_context)
self.compare_match_results(base64_string, match_element, match_context,
self.id_, self.path,
base64_string[:-(len(base64_string) % 4)],
string[:-2], None)
class Base64StringModelElementTest(unittest.TestCase):
base64_string_model_element = Base64StringModelElement('base64')
match_base64 = 'match/base64'
def test1_legit_base64_strings(self):
"""In this test case some legit base64 strings are matched with the ModelElement."""
# string with padding
string = b'This is some string to be encoded.'
base64_string = base64.b64encode(string)
match_context = MatchContext(base64_string)
match_element = self.base64_string_model_element.get_match_element('match', match_context)
self.assertEqual(match_element.get_path(), self.match_base64)
self.assertEqual(match_element.get_match_string(), base64_string)
self.assertEqual(match_element.get_match_object(), string)
self.assertEqual(match_element.get_children(), None)
self.assertEqual(match_context.match_data, b'')
# string without padding
string = b'This is some string to be encoded without the padding character =.'
base64_string = base64.b64encode(string)
match_context = MatchContext(base64_string)
match_element = self.base64_string_model_element.get_match_element('match', match_context)
self.assertEqual(match_element.get_path(), self.match_base64)
self.assertEqual(match_element.get_match_string(), base64_string)
self.assertEqual(match_element.get_match_object(), string)
self.assertEqual(match_element.get_children(), None)
self.assertEqual(match_context.match_data, b'')
def test2_base64_string_with_wrong_characters(self):
"""In this test case some base64 strings with not allowed characters are matched with the ModelElement. Also the padding checks
of base64 strings is tested. """
# string with padding
string = b'This is some string to be encoded.'
string_after_padding = b'This+string+is+not+encoded+any+more+'
base64_string = base64.b64encode(string)
match_context = MatchContext(base64_string + string_after_padding + b'!')
match_element = self.base64_string_model_element.get_match_element('match', match_context)
self.assertEqual(match_element.get_path(), self.match_base64)
self.assertEqual(match_element.get_match_string(), base64_string)
self.assertEqual(match_element.get_match_object(), string)
self.assertEqual(match_element.get_children(), None)
self.assertEqual(match_context.match_data, string_after_padding + b'!')
# string without padding
string = b'This is some string to be encoded without the padding character =.'
base64_string = base64.b64encode(string)
match_context = MatchContext(base64_string + string_after_padding + b'!')
match_element = self.base64_string_model_element.get_match_element('match', match_context)
self.assertEqual(match_element.get_path(), self.match_base64)
self.assertEqual(match_element.get_match_string(), base64_string + string_after_padding)
self.assertEqual(match_element.get_match_object(), string + base64.b64decode(string_after_padding))
self.assertEqual(match_element.get_children(), None)
self.assertEqual(match_context.match_data, b'!')
def test8get_match_element_no_match(self):
"""Parse not matching substring from MatchContext and check if the MatchContext was not changed."""
base64_dme = Base64StringModelElement(self.id_)
data = b"!Hello World"
match_context = DummyMatchContext(data)
match_element = base64_dme.get_match_element(self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b"\x90\x90Hello World"
match_context = DummyMatchContext(data)
match_element = base64_dme.get_match_element(self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
def test9get_match_element_unicode_exception(self):
"""Parse a Base64 string which can not be decoded as UTF-8."""
# ² encoded with ISO-8859-1
base64_string = b'sg=='
match_context = DummyMatchContext(base64_string)
base64_dme = Base64StringModelElement("s0")
match_element = base64_dme.get_match_element("base64", match_context)
self.assertEqual(match_element.path, "base64/s0")
self.assertEqual(match_element.match_string, base64_string)
self.assertEqual(match_element.match_object, base64_string)
self.assertIsNone(match_element.children, None)
self.assertEqual(match_context.match_string, base64_string)
def test5get_match_element_valid_match_string_without_padding(self):
"""Parse matching substring without padding from MatchContext and check if the MatchContext was updated accordingly."""
string = b'This is some string to be encoded without the padding character =.'
base64_string = b'VGhpcyBpcyBzb21lIHN0cmluZyB0byBiZSBlbmNvZGVkIHdpdGhvdXQgdGhlIHBhZGRpbmcgY2hhcmFjdGVyID0u'
match_context = DummyMatchContext(base64_string)
base64_dme = Base64StringModelElement("s0")
match_element = base64_dme.get_match_element("base64", match_context)
self.assertEqual(match_element.path, "base64/s0")
self.assertEqual(match_element.match_string, base64_string)
self.assertEqual(match_element.match_object, string)
self.assertIsNone(match_element.children, None)
self.assertEqual(match_context.match_string, base64_string)
def test6get_match_element_valid_match_string_without_exact_length(self):
"""Parse matching substring without exact length (divisible by 4) and check if the MatchContext was updated accordingly."""
string = b'This is some encoded strin'
base64_string = b'VGhpcyBpcyBzb21lIGVuY29kZWQgc3RyaW4'
match_context = DummyMatchContext(base64_string)
base64_dme = Base64StringModelElement("s0")
match_element = base64_dme.get_match_element("base64", match_context)
self.assertEqual(match_element.path, "base64/s0")
self.assertEqual(match_element.match_string, base64_string[:-(len(base64_string) % 4)])
self.assertEqual(match_element.match_object, string[:-2])
self.assertIsNone(match_element.children, None)
self.assertEqual(match_context.match_string, base64_string[:-(len(base64_string) % 4)])
def test4get_match_element_valid_match_string_with_one_byte_padding(self):
"""Parse matching substring with padding from MatchContext and check if the MatchContext was updated accordingly."""
string = b'This is some encoded strin'
base64_string = b'VGhpcyBpcyBzb21lIGVuY29kZWQgc3RyaW4='
match_context = DummyMatchContext(base64_string)
base64_dme = Base64StringModelElement("s0")
match_element = base64_dme.get_match_element("base64", match_context)
self.assertEqual(match_element.path, "base64/s0")
self.assertEqual(match_element.match_string, base64_string)
self.assertEqual(match_element.match_object, string)
self.assertIsNone(match_element.children, None)
self.assertEqual(match_context.match_string, base64_string)
def test3get_match_element_valid_match_string_with_padding(self):
"""Parse matching substring with padding from MatchContext and check if the MatchContext was updated accordingly."""
string = b'This is some string to be encoded.'
base64_string = b'VGhpcyBpcyBzb21lIHN0cmluZyB0byBiZSBlbmNvZGVkLg=='
match_context = DummyMatchContext(base64_string)
base64_dme = Base64StringModelElement("s0")
match_element = base64_dme.get_match_element("base64", match_context)
self.assertEqual(match_element.path, "base64/s0")
self.assertEqual(match_element.match_string, base64_string)
self.assertEqual(match_element.match_object, string)
self.assertIsNone(match_element.children, None)
self.assertEqual(match_context.match_string, base64_string)
def test7get_match_element_valid_match_string_with_partial_length(self):
"""Parse matching substring out of the MatchContext and check if the MatchContext was updated accordingly."""
string = b'This is some encoded strin'
base64_string = b'VGhpcyBpcyBzb21lIGVuY29kZWQgc3RyaW4='
data = base64_string + b'\nContent: Public Key'
match_context = DummyMatchContext(data)
base64_dme = Base64StringModelElement("s0")
match_element = base64_dme.get_match_element("base64", match_context)
self.assertEqual(match_element.path, "base64/s0")
self.assertEqual(match_element.match_string, base64_string)
self.assertEqual(match_element.match_object, string)
self.assertIsNone(match_element.children, None)
self.assertEqual(match_context.match_string, base64_string)
def build_analysis_pipeline(analysis_context):
"""
Define the function to create pipeline for parsing the log data.
It has also to define an AtomizerFactory to instruct aminer how to process incoming data streams to create log atoms from them.
"""
date_format_string = b'%Y-%m-%d %H:%M:%S'
cron = b' cron['
# Build the parsing model:
service_children_disk_report = [
FixedDataModelElement('Space', b' Current Disk Data is: Filesystem Type Size Used Avail Use%'),
DelimitedDataModelElement('Data', b'%'), AnyByteDataModelElement('Rest')]
service_children_login_details = [
FixedDataModelElement('User/LoginDetails', b'User '), DelimitedDataModelElement('Username', b' '),
FixedWordlistDataModelElement('Status', [b' logged in', b' logged out']),
OptionalMatchModelElement('PastTime', SequenceModelElement('Time', [
FixedDataModelElement('Blank', b' '), DecimalIntegerValueModelElement('Minutes'),
FixedDataModelElement('Ago', b' minutes ago.')]))]
service_children_cron_job = [
DateTimeModelElement('DTM', date_format_string), FixedDataModelElement('UNameSpace1', b' '),
DelimitedDataModelElement('UName', b' '), FixedDataModelElement('UNameSpace2', b' '), DelimitedDataModelElement('User', b' '),
FixedDataModelElement('Cron', cron), DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Details', b']: Job `cron.daily` started.')]
service_children_random_time = [FixedDataModelElement('Space', b'Random: '), DecimalIntegerValueModelElement('Random')]
service_children_sensors = [SequenceModelElement('CPUTemp', [
FixedDataModelElement('FixedTemp', b'CPU Temp: '), DecimalIntegerValueModelElement('Temp'),
FixedDataModelElement('Degrees', b'\xc2\xb0C')]), FixedDataModelElement('Space1', b', '), SequenceModelElement('CPUWorkload', [
FixedDataModelElement('FixedWorkload', b'CPU Workload: '), DecimalIntegerValueModelElement('Workload'),
FixedDataModelElement('Percent', b'%')]), FixedDataModelElement('Space2', b', '),
DateTimeModelElement('DTM', date_format_string)]
service_children_user_ip_address = [
FixedDataModelElement('User/UserIPAddress', b'User '), DelimitedDataModelElement('Username', b' '),
FixedDataModelElement('Action', b' changed IP address to '), IpAddressDataModelElement('IP')]
service_children_cron_job_announcement = [
DateTimeModelElement('DTM', date_format_string), FixedDataModelElement('Space', b' '),
DelimitedDataModelElement('UName', b' '), FixedDataModelElement('Cron', cron), DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Run', b']: Will run job `'),
FixedWordlistDataModelElement('CronType', [b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']),
FixedDataModelElement('StartTime', b'\' in 5 min.')]
service_children_cron_job_execution = [
DateTimeModelElement('DTM', date_format_string), FixedDataModelElement('Space1', b' '),
DelimitedDataModelElement('UName', b' '), FixedDataModelElement('Cron', cron), DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Job', b']: Job `'),
FixedWordlistDataModelElement('CronType', [b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']),
FixedDataModelElement('Started', b'\' started')]
service_children_audit = [SequenceModelElement('path', [
FixedDataModelElement('type', b'type=PATH '), FixedDataModelElement('msg_audit', b'msg=audit('),
DelimitedDataModelElement('msg', b':'), FixedDataModelElement('placeholder', b':'), DecimalIntegerValueModelElement('id'),
FixedDataModelElement('item_string', b'): item='), DecimalIntegerValueModelElement('item'),
FixedDataModelElement('name_string', b' name="'), DelimitedDataModelElement('name', b'"'),
FixedDataModelElement('inode_string', b'" inode='), DecimalIntegerValueModelElement('inode'),
FixedDataModelElement('dev_string', b' dev='), DelimitedDataModelElement('dev', b' '),
FixedDataModelElement('mode_string', b' mode='),
DecimalIntegerValueModelElement('mode', value_pad_type=DecimalIntegerValueModelElement.PAD_TYPE_ZERO),
FixedDataModelElement('ouid_string', b' ouid='), DecimalIntegerValueModelElement('ouid'),
FixedDataModelElement('ogid_string', b' ogid='), DecimalIntegerValueModelElement('ogid'),
FixedDataModelElement('rdev_string', b' rdev='), DelimitedDataModelElement('rdev', b' '),
FixedDataModelElement('nametype_string', b' nametype='), FixedWordlistDataModelElement('nametype', [b'NORMAL', b'ERROR'])]),
SequenceModelElement('syscall', [
FixedDataModelElement('type', b'type=SYSCALL '), FixedDataModelElement('msg_audit', b'msg=audit('),
DelimitedDataModelElement('msg', b':'), FixedDataModelElement('placeholder', b':'), DecimalIntegerValueModelElement('id'),
FixedDataModelElement('arch_string', b'): arch='), DelimitedDataModelElement('arch', b' '),
FixedDataModelElement('syscall_string', b' syscall='), DecimalIntegerValueModelElement('syscall'),
FixedDataModelElement('success_string', b' success='), FixedWordlistDataModelElement('success', [b'yes', b'no']),
FixedDataModelElement('exit_string', b' exit='), DecimalIntegerValueModelElement('exit'),
AnyByteDataModelElement('remainding_data')])]
service_children_parsing_model_element = [
DateTimeModelElement('DateTimeModelElement', b'Current DateTime: %d.%m.%Y %H:%M:%S'),
DecimalFloatValueModelElement('DecimalFloatValueModelElement', value_sign_type='optional'),
DecimalIntegerValueModelElement('DecimalIntegerValueModelElement', value_sign_type='optional', value_pad_type='blank'),
SequenceModelElement('se', [
DelimitedDataModelElement('DelimitedDataModelElement', b';'), FixedDataModelElement('FixedDataModelElement', b';')])]
# ElementValueBranchModelElement
fixed_data_me1 = FixedDataModelElement("fixed1", b'match ')
fixed_data_me2 = FixedDataModelElement("fixed2", b'fixed String')
fixed_wordlist_data_model_element = FixedWordlistDataModelElement("wordlist", [b'data: ', b'string: '])
decimal_integer_value_model_element = DecimalIntegerValueModelElement("decimal")
service_children_parsing_model_element.append(
ElementValueBranchModelElement('ElementValueBranchModelElement', FirstMatchModelElement("first", [
SequenceModelElement("seq1", [fixed_data_me1, fixed_wordlist_data_model_element]),
SequenceModelElement("seq2", [fixed_data_me1, fixed_wordlist_data_model_element, fixed_data_me2])]), "wordlist",
{0: decimal_integer_value_model_element, 1: fixed_data_me2}))
service_children_parsing_model_element.append(HexStringModelElement('HexStringModelElement'))
service_children_parsing_model_element.append(SequenceModelElement('se2', [
FixedDataModelElement('FixedDataModelElement', b'Gateway IP-Address: '), IpAddressDataModelElement('IpAddressDataModelElement')]))
import locale
loc = locale.getlocale()
if loc == (None, None):
loc = ('en_US', 'utf8')
service_children_parsing_model_element.append(
MultiLocaleDateTimeModelElement('MultiLocaleDateTimeModelElement', [(b'%b %d %Y', None, '%s.%s' % loc)]))
service_children_parsing_model_element.append(
RepeatedElementDataModelElement('RepeatedElementDataModelElement', SequenceModelElement('SequenceModelElement', [
FixedDataModelElement('FixedDataModelElement', b'[drawn number]: '),
DecimalIntegerValueModelElement('DecimalIntegerValueModelElement')]), 1))
service_children_parsing_model_element.append(VariableByteDataModelElement('VariableByteDataModelElement', b'-@#'))
service_children_parsing_model_element.append(SequenceModelElement('se', [
WhiteSpaceLimitedDataModelElement('WhiteSpaceLimitedDataModelElement'), FixedDataModelElement('fixed', b' ')]))
# The Base64StringModelElement must be just before the AnyByteDataModelElement to avoid unexpected Matches.
service_children_parsing_model_element.append(Base64StringModelElement('Base64StringModelElement'))
# The OptionalMatchModelElement must be paired with a FirstMatchModelElement because it accepts all data and thus no data gets
# to the AnyByteDataModelElement. The AnyByteDataModelElement must be last, because all bytes are accepted.
service_children_parsing_model_element.append(
OptionalMatchModelElement('/', FirstMatchModelElement('FirstMatchModelElement//optional', [
FixedDataModelElement('FixedDataModelElement', b'The-searched-element-was-found!'), SequenceModelElement('se', [
FixedDataModelElement('FixedDME', b'Any:'), AnyByteDataModelElement('AnyByteDataModelElement')])])))
alphabet = b'ghijkl'
service_children_ecd = []
for _, char in enumerate(alphabet):
char = bytes([char])
service_children_ecd.append(FixedDataModelElement(char.decode(), char))
parsing_model = FirstMatchModelElement('model', [
SequenceModelElement('CronAnnouncement', service_children_cron_job_announcement),
SequenceModelElement('CronExecution', service_children_cron_job_execution),
SequenceModelElement('DailyCron', service_children_cron_job), SequenceModelElement('DiskReport', service_children_disk_report),
SequenceModelElement('LoginDetails', service_children_login_details), DecimalIntegerValueModelElement('Random'),
SequenceModelElement('RandomTime', service_children_random_time), SequenceModelElement('Sensors', service_children_sensors),
SequenceModelElement('IPAddresses', service_children_user_ip_address), FirstMatchModelElement('type', service_children_audit),
FirstMatchModelElement('ECD', service_children_ecd), FirstMatchModelElement('ParsingME', service_children_parsing_model_element)])
# Some generic imports.
from aminer.analysis import AtomFilters
# Create all global handler lists here and append the real handlers later on.
# Use this filter to distribute all atoms to the analysis handlers.
atom_filter = AtomFilters.SubhandlerFilter(None)
from aminer.analysis.TimestampCorrectionFilters import SimpleMonotonicTimestampAdjust
simple_monotonic_timestamp_adjust = SimpleMonotonicTimestampAdjust([atom_filter])
analysis_context.register_component(simple_monotonic_timestamp_adjust, component_name="SimpleMonotonicTimestampAdjust")
from aminer.events.StreamPrinterEventHandler import StreamPrinterEventHandler
from aminer.events.JsonConverterHandler import JsonConverterHandler
stream_printer_event_handler = StreamPrinterEventHandler(analysis_context)
json_converter_handler = JsonConverterHandler([stream_printer_event_handler], analysis_context)
anomaly_event_handlers = [json_converter_handler]
# Now define the AtomizerFactory using the model. A simple line based one is usually sufficient.
from aminer.input.SimpleByteStreamLineAtomizerFactory import SimpleByteStreamLineAtomizerFactory
analysis_context.atomizer_factory = SimpleByteStreamLineAtomizerFactory(parsing_model, [simple_monotonic_timestamp_adjust],
anomaly_event_handlers)
# Just report all unparsed atoms to the event handlers.
from aminer.analysis.UnparsedAtomHandlers import SimpleUnparsedAtomHandler, VerboseUnparsedAtomHandler
simple_unparsed_atom_handler = SimpleUnparsedAtomHandler(anomaly_event_handlers)
atom_filter.add_handler(simple_unparsed_atom_handler, stop_when_handled_flag=False)
analysis_context.register_component(simple_unparsed_atom_handler, component_name="SimpleUnparsedHandler")
verbose_unparsed_atom_handler = VerboseUnparsedAtomHandler(anomaly_event_handlers, parsing_model)
atom_filter.add_handler(verbose_unparsed_atom_handler, stop_when_handled_flag=True)
analysis_context.register_component(verbose_unparsed_atom_handler, component_name="VerboseUnparsedHandler")
from aminer.analysis.TimestampsUnsortedDetector import TimestampsUnsortedDetector
timestamps_unsorted_detector = TimestampsUnsortedDetector(analysis_context.aminer_config, anomaly_event_handlers)
atom_filter.add_handler(timestamps_unsorted_detector)
analysis_context.register_component(timestamps_unsorted_detector, component_name="TimestampsUnsortedDetector")
from aminer.analysis import Rules
from aminer.analysis.AllowlistViolationDetector import AllowlistViolationDetector
allowlist_rules = [
Rules.OrMatchRule([
Rules.AndMatchRule([
Rules.PathExistsMatchRule('/model/LoginDetails/PastTime/Time/Minutes'),
Rules.NegationMatchRule(Rules.ValueMatchRule('/model/LoginDetails/Username', b'root')),
Rules.DebugMatchRule(debug_match_result=True)]),
Rules.AndMatchRule([
Rules.NegationMatchRule(Rules.PathExistsMatchRule('/model/LoginDetails/PastTime/Time/Minutes')),
Rules.PathExistsMatchRule('/model/LoginDetails'),
Rules.DebugMatchRule(debug_match_result=True)]),
Rules.NegationMatchRule(Rules.PathExistsMatchRule('/model/LoginDetails'))])]
# This rule list should trigger, when the line does not look like: User root (logged in, logged out)
# or User 'username' (logged in, logged out) x minutes ago.
allowlist_violation_detector = AllowlistViolationDetector(analysis_context.aminer_config, allowlist_rules, anomaly_event_handlers,
output_log_line=True)
analysis_context.register_component(allowlist_violation_detector, component_name="Allowlist")
atom_filter.add_handler(allowlist_violation_detector)
from aminer.analysis.ParserCount import ParserCount
parser_count = ParserCount(analysis_context.aminer_config, None, anomaly_event_handlers, 10)
analysis_context.register_component(parser_count, component_name="ParserCount")
atom_filter.add_handler(parser_count)
from aminer.analysis.EventTypeDetector import EventTypeDetector
etd = EventTypeDetector(analysis_context.aminer_config, anomaly_event_handlers)
analysis_context.register_component(etd, component_name="EventTypeDetector")
atom_filter.add_handler(etd)
from aminer.analysis.VariableTypeDetector import VariableTypeDetector
vtd = VariableTypeDetector(analysis_context.aminer_config, anomaly_event_handlers, etd, silence_output_except_indicator=False,
output_log_line=False, ignore_list=["/model/RandomTime"])
analysis_context.register_component(vtd, component_name="VariableTypeDetector")
atom_filter.add_handler(vtd)
from aminer.analysis.VariableCorrelationDetector import VariableCorrelationDetector
vtd = VariableCorrelationDetector(analysis_context.aminer_config, anomaly_event_handlers, etd, disc_div_thres=0.5,
ignore_list=["/model/RandomTime"])
analysis_context.register_component(vtd, component_name="VariableCorrelationDetector")
atom_filter.add_handler(vtd)
from aminer.analysis.EventCorrelationDetector import EventCorrelationDetector
ecd = EventCorrelationDetector(analysis_context.aminer_config, anomaly_event_handlers, check_rules_flag=True,
hypothesis_max_delta_time=1.0)
analysis_context.register_component(ecd, component_name="EventCorrelationDetector")
atom_filter.add_handler(ecd)
from aminer.analysis.EventFrequencyDetector import EventFrequencyDetector
efd = EventFrequencyDetector(analysis_context.aminer_config, anomaly_event_handlers, window_size=0.1)
analysis_context.register_component(efd, component_name="EventFrequencyDetector")
atom_filter.add_handler(efd)
from aminer.analysis.EventSequenceDetector import EventSequenceDetector
esd = EventSequenceDetector(analysis_context.aminer_config, anomaly_event_handlers, ['/model/ParsingME'], ignore_list=[
'/model/ECD/g', '/model/ECD/h', '/model/ECD/i', '/model/ECD/j', '/model/ECD/k', '/model/ECD/l', '/model/Random',
'/model/RandomTime', '/model/DailyCron'])
analysis_context.register_component(esd, component_name="EventSequenceDetector")
atom_filter.add_handler(esd)
from aminer.analysis.MatchFilter import MatchFilter
match_filter = MatchFilter(analysis_context.aminer_config, ['/model/Random'], anomaly_event_handlers, target_value_list=[
1, 10, 100], output_log_line=True)
analysis_context.register_component(match_filter, component_name="MatchFilter")
atom_filter.add_handler(match_filter)
from aminer.analysis.NewMatchPathDetector import NewMatchPathDetector
new_match_path_detector = NewMatchPathDetector(analysis_context.aminer_config, anomaly_event_handlers, auto_include_flag=True,
output_log_line=True)
analysis_context.register_component(new_match_path_detector, component_name="NewMatchPath")
atom_filter.add_handler(new_match_path_detector)
def tuple_transformation_function(match_value_list):
"""Only allow output of the EnhancedNewMatchPathValueComboDetector after every 10th element."""
extra_data = enhanced_new_match_path_value_combo_detector.known_values_dict.get(tuple(match_value_list))
if extra_data is not None:
mod = 10
if (extra_data[2] + 1) % mod == 0:
enhanced_new_match_path_value_combo_detector.auto_include_flag = False
else:
enhanced_new_match_path_value_combo_detector.auto_include_flag = True
return match_value_list
from aminer.analysis.EnhancedNewMatchPathValueComboDetector import EnhancedNewMatchPathValueComboDetector
enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector(analysis_context.aminer_config, [
'/model/DailyCron/UName', '/model/DailyCron/JobNumber'], anomaly_event_handlers, auto_include_flag=True,
tuple_transformation_function=tuple_transformation_function, output_log_line=True)
analysis_context.register_component(enhanced_new_match_path_value_combo_detector, component_name="EnhancedNewValueCombo")
atom_filter.add_handler(enhanced_new_match_path_value_combo_detector)
import re
ip_match_action = Rules.EventGenerationMatchAction(
"Analysis.Rules.IPv4InRFC1918MatchRule", "Private IP address occurred!", anomaly_event_handlers)
vdmt = Rules.ValueDependentModuloTimeMatchRule(None, 3, ["/model/ECD/j", "/model/ECD/k", "/model/ECD/l"], {b"e": [0, 2.95]}, [0, 3])
mt = Rules.ModuloTimeMatchRule(None, 3, 0, 3, None)
time_allowlist_rules = [
Rules.AndMatchRule([
Rules.ParallelMatchRule([
Rules.ValueDependentDelegatedMatchRule([
'/model/ECD/g', '/model/ECD/h', '/model/ECD/i', '/model/ECD/j', '/model/ECD/k', '/model/ECD/l'], {
(b"a",): mt, (b"b",): mt, (b"c",): mt, (b"d",): vdmt, (b"e",): vdmt, (b"f",): vdmt, None: mt}, mt),
Rules.IPv4InRFC1918MatchRule("/model/ParsingME/se2/IpAddressDataModelElement", ip_match_action),
Rules.DebugHistoryMatchRule(debug_match_result=True)
]),
# IP addresses 8.8.8.8, 8.8.4.4 and 10.0.0.0 - 10.255.255.255 are not allowed
Rules.NegationMatchRule(Rules.ValueListMatchRule("/model/ParsingME/se2/IpAddressDataModelElement", [134744072, 134743044])),
Rules.NegationMatchRule(Rules.ValueRangeMatchRule("/model/ParsingME/se2/IpAddressDataModelElement", 167772160, 184549375)),
Rules.NegationMatchRule(Rules.StringRegexMatchRule("/model/type/syscall/success", re.compile(b"^no$")))
])
]
time_allowlist_violation_detector = AllowlistViolationDetector(
analysis_context.aminer_config, time_allowlist_rules, anomaly_event_handlers, output_log_line=True)
analysis_context.register_component(time_allowlist_violation_detector, component_name="TimeAllowlist")
atom_filter.add_handler(time_allowlist_violation_detector)
from aminer.analysis.HistogramAnalysis import HistogramAnalysis, LinearNumericBinDefinition, ModuloTimeBinDefinition, \
PathDependentHistogramAnalysis
modulo_time_bin_definition = ModuloTimeBinDefinition(86400, 3600, 0, 1, 24, True)
linear_numeric_bin_definition = LinearNumericBinDefinition(50, 5, 20, True)
histogram_analysis = HistogramAnalysis(analysis_context.aminer_config, [
('/model/RandomTime/Random', modulo_time_bin_definition), ('/model/Random', linear_numeric_bin_definition)], 10,
anomaly_event_handlers, output_log_line=True)
analysis_context.register_component(histogram_analysis, component_name="HistogramAnalysis")
atom_filter.add_handler(histogram_analysis)
path_dependent_histogram_analysis = PathDependentHistogramAnalysis(
analysis_context.aminer_config, '/model/RandomTime', modulo_time_bin_definition, 10, anomaly_event_handlers, output_log_line=True)
analysis_context.register_component(path_dependent_histogram_analysis, component_name="PathDependentHistogramAnalysis")
atom_filter.add_handler(path_dependent_histogram_analysis)
from aminer.analysis.MatchValueAverageChangeDetector import MatchValueAverageChangeDetector
match_value_average_change_detector = MatchValueAverageChangeDetector(analysis_context.aminer_config, anomaly_event_handlers, None, [
'/model/Random'], 100, 10, output_log_line=True)
analysis_context.register_component(match_value_average_change_detector, component_name="MatchValueAverageChange")
atom_filter.add_handler(match_value_average_change_detector)
import sys
from aminer.analysis.MatchValueStreamWriter import MatchValueStreamWriter
match_value_stream_writer = MatchValueStreamWriter(
sys.stdout, ['/model/Sensors/CPUTemp', '/model/Sensors/CPUWorkload', '/model/Sensors/DTM'], b';', b'')
analysis_context.register_component(match_value_stream_writer, component_name="MatchValueStreamWriter")
atom_filter.add_handler(match_value_stream_writer)
from aminer.analysis.NewMatchPathValueComboDetector import NewMatchPathValueComboDetector
new_match_path_value_combo_detector = NewMatchPathValueComboDetector(
analysis_context.aminer_config, ['/model/IPAddresses/Username', '/model/IPAddresses/IP'],
anomaly_event_handlers, output_log_line=True)
analysis_context.register_component(new_match_path_value_combo_detector, component_name="NewMatchPathValueCombo")
atom_filter.add_handler(new_match_path_value_combo_detector)
from aminer.analysis.NewMatchIdValueComboDetector import NewMatchIdValueComboDetector
new_match_id_value_combo_detector = NewMatchIdValueComboDetector(analysis_context.aminer_config, [
'/model/type/path/name', '/model/type/syscall/syscall'], anomaly_event_handlers, id_path_list=[
'/model/type/path/id', '/model/type/syscall/id'], min_allowed_time_diff=5, auto_include_flag=True, allow_missing_values_flag=True,
output_log_line=True)
analysis_context.register_component(new_match_id_value_combo_detector, component_name="NewMatchIdValueComboDetector")
atom_filter.add_handler(new_match_id_value_combo_detector)
from aminer.analysis.NewMatchPathValueDetector import NewMatchPathValueDetector
new_match_path_value_detector = NewMatchPathValueDetector(analysis_context.aminer_config, [
'/model/DailyCron/JobNumber', '/model/IPAddresses/Username'], anomaly_event_handlers, auto_include_flag=True, output_log_line=True)
analysis_context.register_component(new_match_path_value_detector, component_name="NewMatchPathValue")
atom_filter.add_handler(new_match_path_value_detector)
from aminer.analysis.MissingMatchPathValueDetector import MissingMatchPathValueDetector
missing_match_path_value_detector = MissingMatchPathValueDetector(
analysis_context.aminer_config, ['/model/DiskReport/Space'], anomaly_event_handlers, auto_include_flag=True, default_interval=2,
realert_interval=5, output_log_line=True)
analysis_context.register_component(missing_match_path_value_detector, component_name="MissingMatch")
atom_filter.add_handler(missing_match_path_value_detector)
from aminer.analysis.TimeCorrelationDetector import TimeCorrelationDetector
time_correlation_detector = TimeCorrelationDetector(
analysis_context.aminer_config, anomaly_event_handlers, 2, min_rule_attributes=1, max_rule_attributes=5,
record_count_before_event=10000, output_log_line=True)
analysis_context.register_component(time_correlation_detector, component_name="TimeCorrelationDetector")
atom_filter.add_handler(time_correlation_detector)
from aminer.analysis.TimeCorrelationViolationDetector import TimeCorrelationViolationDetector, CorrelationRule, EventClassSelector
cron_job_announcement = CorrelationRule('CronJobAnnouncement', 5, 6, max_artefacts_a_for_single_b=1, artefact_match_parameters=[
('/model/CronAnnouncement/JobNumber', '/model/CronExecution/JobNumber')])
a_class_selector = EventClassSelector('Announcement', [cron_job_announcement], None)
b_class_selector = EventClassSelector('Execution', None, [cron_job_announcement])
rules = [Rules.PathExistsMatchRule('/model/CronAnnouncement/Run', a_class_selector),
Rules.PathExistsMatchRule('/model/CronExecution/Job', b_class_selector)]
time_correlation_violation_detector = TimeCorrelationViolationDetector(analysis_context.aminer_config, rules, anomaly_event_handlers,
output_log_line=True)
analysis_context.register_component(time_correlation_violation_detector, component_name="TimeCorrelationViolationDetector")
atom_filter.add_handler(time_correlation_violation_detector)
def build_analysis_pipeline(analysis_context):
"""
Define the function to create pipeline for parsing the log data.
It has also to define an AtomizerFactory to instruct aminer how to process incoming data streams to create log atoms from them.
"""
date_format_string = b'%Y-%m-%d %H:%M:%S'
cron = b' cron['
# Build the parsing model:
service_children_disk_report = [
FixedDataModelElement(
'Space',
b' Current Disk Data is: Filesystem Type Size Used Avail Use%'
),
DelimitedDataModelElement('Data', b'%'),
AnyByteDataModelElement('Rest')
]
service_children_login_details = [
FixedDataModelElement('User', b'User '),
DelimitedDataModelElement('Username', b' '),
FixedWordlistDataModelElement('Status',
[b' logged in', b' logged out']),
OptionalMatchModelElement(
'PastTime',
SequenceModelElement('Time', [
FixedDataModelElement('Blank', b' '),
DecimalIntegerValueModelElement('Minutes'),
FixedDataModelElement('Ago', b' minutes ago.')
]))
]
service_children_cron_job = [
DateTimeModelElement('DTM', date_format_string),
FixedDataModelElement('UNameSpace1', b' '),
DelimitedDataModelElement('UName', b' '),
FixedDataModelElement('UNameSpace2', b' '),
DelimitedDataModelElement('User', b' '),
FixedDataModelElement('Cron', cron),
DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Details', b']: Job `cron.daily` started.')
]
service_children_random_time = [
FixedDataModelElement('Space', b'Random: '),
DecimalIntegerValueModelElement('Random')
]
service_children_sensors = [
SequenceModelElement('CPUTemp', [
FixedDataModelElement('FixedTemp', b'CPU Temp: '),
DecimalIntegerValueModelElement('Temp'),
FixedDataModelElement('Degrees', b'\xc2\xb0C')
]),
FixedDataModelElement('Space1', b', '),
SequenceModelElement('CPUWorkload', [
FixedDataModelElement('Fixed Workload', b'CPU Workload: '),
DecimalIntegerValueModelElement('Workload'),
FixedDataModelElement('Percent', b'%')
]),
FixedDataModelElement('Space2', b', '),
DateTimeModelElement('DTM', date_format_string)
]
service_children_user_ip_address = [
FixedDataModelElement('User', b'User '),
DelimitedDataModelElement('Username', b' '),
FixedDataModelElement('Action', b' changed IP address to '),
IpAddressDataModelElement('IP')
]
service_children_cron_job_announcement = [
DateTimeModelElement('DTM', date_format_string),
FixedDataModelElement('Space', b' '),
DelimitedDataModelElement('UName', b' '),
FixedDataModelElement('Cron', cron),
DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Run', b']: Will run job `'),
FixedWordlistDataModelElement(
'CronType',
[b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']),
FixedDataModelElement('Start Time', b'\' in 5 min.')
]
service_children_cron_job_execution = [
DateTimeModelElement('DTM', date_format_string),
FixedDataModelElement('Space1', b' '),
DelimitedDataModelElement('UName', b' '),
FixedDataModelElement('Cron', cron),
DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Job', b']: Job `'),
FixedWordlistDataModelElement(
'CronType',
[b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']),
FixedDataModelElement('Started', b'\' started')
]
service_children_parsing_model_element = [
DateTimeModelElement('DateTimeModelElement',
b'Current DateTime: %d.%m.%Y %H:%M:%S'),
DecimalFloatValueModelElement('DecimalFloatValueModelElement',
value_sign_type='optional'),
DecimalIntegerValueModelElement('DecimalIntegerValueModelElement',
value_sign_type='optional',
value_pad_type='blank'),
SequenceModelElement('', [
DelimitedDataModelElement('DelimitedDataModelElement', b';'),
FixedDataModelElement('FixedDataModelElement', b';')
])
]
# ElementValueBranchModelElement
fixed_data_me1 = FixedDataModelElement("fixed1", b'match ')
fixed_data_me2 = FixedDataModelElement("fixed2", b'fixed String')
fixed_wordlist_data_model_element = FixedWordlistDataModelElement(
"wordlist", [b'data: ', b'string: '])
decimal_integer_value_model_element = DecimalIntegerValueModelElement(
"decimal")
service_children_parsing_model_element.append(
ElementValueBranchModelElement(
'ElementValueBranchModelElement',
FirstMatchModelElement("first", [
SequenceModelElement(
"seq1",
[fixed_data_me1, fixed_wordlist_data_model_element]),
SequenceModelElement("seq2", [
fixed_data_me1, fixed_wordlist_data_model_element,
fixed_data_me2
])
]), "wordlist", {
0: decimal_integer_value_model_element,
1: fixed_data_me2
}))
service_children_parsing_model_element.append(
HexStringModelElement('HexStringModelElement'))
service_children_parsing_model_element.append(
SequenceModelElement('', [
FixedDataModelElement('FixedDataModelElement',
b'Gateway IP-Address: '),
IpAddressDataModelElement('IpAddressDataModelElement')
]))
service_children_parsing_model_element.append(
MultiLocaleDateTimeModelElement('MultiLocaleDateTimeModelElement',
[(b'%b %d %Y', "de_AT.utf8", None)]))
service_children_parsing_model_element.append(
RepeatedElementDataModelElement(
'RepeatedElementDataModelElement',
SequenceModelElement('SequenceModelElement', [
FixedDataModelElement('FixedDataModelElement',
b'drawn number: '),
DecimalIntegerValueModelElement(
'DecimalIntegerValueModelElement')
]), 1))
service_children_parsing_model_element.append(
VariableByteDataModelElement('VariableByteDataModelElement', b'-@#'))
service_children_parsing_model_element.append(
SequenceModelElement('', [
WhiteSpaceLimitedDataModelElement(
'WhiteSpaceLimitedDataModelElement'),
FixedDataModelElement('', b' ')
]))
# The Base64StringModelElement must be just before the AnyByteDataModelElement to avoid unexpected Matches.
service_children_parsing_model_element.append(
Base64StringModelElement('Base64StringModelElement'))
# The OptionalMatchModelElement must be paired with a FirstMatchModelElement because it accepts all data and thus no data gets
# to the AnyByteDataModelElement. The AnyByteDataModelElement must be last, because all bytes are accepted.
service_children_parsing_model_element.append(
OptionalMatchModelElement(
'OptionalMatchModelElement',
FirstMatchModelElement('FirstMatchModelElement', [
FixedDataModelElement('FixedDataModelElement',
b'The-searched-element-was-found!'),
AnyByteDataModelElement('AnyByteDataModelElement')
])))
parsing_model = FirstMatchModelElement('model', [
SequenceModelElement('CronAnnouncement',
service_children_cron_job_announcement),
SequenceModelElement('CronExecution',
service_children_cron_job_execution),
SequenceModelElement('DailyCron', service_children_cron_job),
SequenceModelElement('DiskReport', service_children_disk_report),
SequenceModelElement('LoginDetails', service_children_login_details),
DecimalIntegerValueModelElement('Random'),
SequenceModelElement('RandomTime', service_children_random_time),
SequenceModelElement('Sensors', service_children_sensors),
SequenceModelElement('IPAddresses', service_children_user_ip_address),
FirstMatchModelElement('ParsingME',
service_children_parsing_model_element)
])
# Some generic imports.
from aminer.analysis import AtomFilters
# Create all global handler lists here and append the real handlers
# later on.
# Use this filter to distribute all atoms to the analysis handlers.
atom_filter = AtomFilters.SubhandlerFilter(None)
from aminer.analysis.TimestampCorrectionFilters import SimpleMonotonicTimestampAdjust
simple_monotonic_timestamp_adjust = SimpleMonotonicTimestampAdjust(
[atom_filter])
analysis_context.register_component(
simple_monotonic_timestamp_adjust,
component_name="SimpleMonotonicTimestampAdjust")
from aminer.events.StreamPrinterEventHandler import StreamPrinterEventHandler
stream_printer_event_handler = StreamPrinterEventHandler(analysis_context)
from aminer.events.SyslogWriterEventHandler import SyslogWriterEventHandler
syslog_event_handler = SyslogWriterEventHandler(analysis_context)
from aminer.events.DefaultMailNotificationEventHandler import DefaultMailNotificationEventHandler
if DefaultMailNotificationEventHandler.CONFIG_KEY_MAIL_TARGET_ADDRESS in analysis_context.aminer_config.config_properties:
mail_notification_handler = DefaultMailNotificationEventHandler(
analysis_context)
analysis_context.register_component(mail_notification_handler,
component_name="MailHandler")
anomaly_event_handlers = [
stream_printer_event_handler, syslog_event_handler,
mail_notification_handler
]
# Now define the AtomizerFactory using the model. A simple line based one is usually sufficient.
from aminer.input.SimpleByteStreamLineAtomizerFactory import SimpleByteStreamLineAtomizerFactory
analysis_context.atomizer_factory = SimpleByteStreamLineAtomizerFactory(
parsing_model, [simple_monotonic_timestamp_adjust],
anomaly_event_handlers)
# Just report all unparsed atoms to the event handlers.
from aminer.analysis.UnparsedAtomHandlers import SimpleUnparsedAtomHandler
simple_unparsed_atom_handler = SimpleUnparsedAtomHandler(
anomaly_event_handlers)
atom_filter.add_handler(simple_unparsed_atom_handler,
stop_when_handled_flag=True)
analysis_context.register_component(simple_unparsed_atom_handler,
component_name="UnparsedHandler")
from aminer.analysis.TimestampsUnsortedDetector import TimestampsUnsortedDetector
timestamps_unsorted_detector = TimestampsUnsortedDetector(
analysis_context.aminer_config, anomaly_event_handlers)
atom_filter.add_handler(timestamps_unsorted_detector)
analysis_context.register_component(
timestamps_unsorted_detector,
component_name="TimestampsUnsortedDetector")
from aminer.analysis import Rules
from aminer.analysis.AllowlistViolationDetector import AllowlistViolationDetector
allowlist_rules = [
Rules.OrMatchRule([
Rules.AndMatchRule([
Rules.PathExistsMatchRule(
'/model/LoginDetails/PastTime/Time/Minutes'),
Rules.NegationMatchRule(
Rules.ValueMatchRule('/model/LoginDetails/Username',
b'root'))
]),
Rules.AndMatchRule([
Rules.NegationMatchRule(
Rules.PathExistsMatchRule(
'/model/LoginDetails/PastTime/Time/Minutes')),
Rules.PathExistsMatchRule('/model/LoginDetails')
]),
Rules.NegationMatchRule(
Rules.PathExistsMatchRule('/model/LoginDetails'))
])
]
# This rule list should trigger, when the line does not look like: User root (logged in, logged out)
# or User 'username' (logged in, logged out) x minutes ago.
allowlist_violation_detector = AllowlistViolationDetector(
analysis_context.aminer_config, allowlist_rules,
anomaly_event_handlers)
analysis_context.register_component(allowlist_violation_detector,
component_name="Allowlist")
atom_filter.add_handler(allowlist_violation_detector)
from aminer.analysis.NewMatchPathDetector import NewMatchPathDetector
new_match_path_detector = NewMatchPathDetector(
analysis_context.aminer_config,
anomaly_event_handlers,
auto_include_flag=True)
analysis_context.register_component(new_match_path_detector,
component_name="NewMatchPath")
atom_filter.add_handler(new_match_path_detector)
def tuple_transformation_function(match_value_list):
"""Only allow output of the EnhancedNewMatchPathValueComboDetector after every 10000th element."""
extra_data = enhanced_new_match_path_value_combo_detector.known_values_dict.get(
tuple(match_value_list))
if extra_data is not None:
mod = 10000
if (extra_data[2] + 1) % mod == 0:
enhanced_new_match_path_value_combo_detector.auto_include_flag = False
else:
enhanced_new_match_path_value_combo_detector.auto_include_flag = True
return match_value_list
from aminer.analysis.EnhancedNewMatchPathValueComboDetector import EnhancedNewMatchPathValueComboDetector
enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector(
analysis_context.aminer_config,
['/model/DailyCron/UName', '/model/DailyCron/JobNumber'],
anomaly_event_handlers,
auto_include_flag=True,
tuple_transformation_function=tuple_transformation_function)
analysis_context.register_component(
enhanced_new_match_path_value_combo_detector,
component_name="EnhancedNewValueCombo")
atom_filter.add_handler(enhanced_new_match_path_value_combo_detector)
from aminer.analysis.HistogramAnalysis import HistogramAnalysis, LinearNumericBinDefinition, ModuloTimeBinDefinition, \
PathDependentHistogramAnalysis
modulo_time_bin_definition = ModuloTimeBinDefinition(
86400, 3600, 0, 1, 24, True)
linear_numeric_bin_definition = LinearNumericBinDefinition(50, 5, 20, True)
histogram_analysis = HistogramAnalysis(
analysis_context.aminer_config,
[('/model/RandomTime/Random', modulo_time_bin_definition),
('/model/Random', linear_numeric_bin_definition)], 10,
anomaly_event_handlers)
analysis_context.register_component(histogram_analysis,
component_name="HistogramAnalysis")
atom_filter.add_handler(histogram_analysis)
path_dependent_histogram_analysis = PathDependentHistogramAnalysis(
analysis_context.aminer_config, '/model/RandomTime',
modulo_time_bin_definition, 10, anomaly_event_handlers)
analysis_context.register_component(
path_dependent_histogram_analysis,
component_name="PathDependentHistogramAnalysis")
atom_filter.add_handler(path_dependent_histogram_analysis)
from aminer.analysis.MatchValueAverageChangeDetector import MatchValueAverageChangeDetector
match_value_average_change_detector = MatchValueAverageChangeDetector(
analysis_context.aminer_config, anomaly_event_handlers, None,
['/model/Random'], 100, 10)
analysis_context.register_component(
match_value_average_change_detector,
component_name="MatchValueAverageChange")
atom_filter.add_handler(match_value_average_change_detector)
import sys
from aminer.analysis.MatchValueStreamWriter import MatchValueStreamWriter
match_value_stream_writer = MatchValueStreamWriter(sys.stdout, [
'/model/Sensors/CPUTemp', '/model/Sensors/CPUWorkload',
'/model/Sensors/DTM'
], b';', b'')
analysis_context.register_component(
match_value_stream_writer, component_name="MatchValueStreamWriter")
atom_filter.add_handler(match_value_stream_writer)
from aminer.analysis.NewMatchPathValueComboDetector import NewMatchPathValueComboDetector
new_match_path_value_combo_detector = NewMatchPathValueComboDetector(
analysis_context.aminer_config,
['/model/IPAddresses/Username', '/model/IPAddresses/IP'],
anomaly_event_handlers,
auto_include_flag=True)
analysis_context.register_component(
new_match_path_value_combo_detector,
component_name="NewMatchPathValueCombo")
atom_filter.add_handler(new_match_path_value_combo_detector)
from aminer.analysis.NewMatchPathValueDetector import NewMatchPathValueDetector
new_match_path_value_detector = NewMatchPathValueDetector(
analysis_context.aminer_config,
['/model/DailyCron/JobNumber', '/model/IPAddresses/Username'],
anomaly_event_handlers,
auto_include_flag=True)
analysis_context.register_component(new_match_path_value_detector,
component_name="NewMatchPathValue")
atom_filter.add_handler(new_match_path_value_detector)
from aminer.analysis.MissingMatchPathValueDetector import MissingMatchPathValueDetector
missing_match_path_value_detector = MissingMatchPathValueDetector(
analysis_context.aminer_config, ['/model/DiskReport/Space'],
anomaly_event_handlers,
auto_include_flag=True,
default_interval=2,
realert_interval=5)
analysis_context.register_component(missing_match_path_value_detector,
component_name="MissingMatch")
atom_filter.add_handler(missing_match_path_value_detector)
from aminer.analysis.TimeCorrelationDetector import TimeCorrelationDetector
time_correlation_detector = TimeCorrelationDetector(
analysis_context.aminer_config,
anomaly_event_handlers,
2,
min_rule_attributes=1,
max_rule_attributes=5,
record_count_before_event=70000,
output_log_line=True)
analysis_context.register_component(
time_correlation_detector, component_name="TimeCorrelationDetector")
atom_filter.add_handler(time_correlation_detector)
from aminer.analysis.TimeCorrelationViolationDetector import TimeCorrelationViolationDetector, CorrelationRule, EventClassSelector
cron_job_announcement = CorrelationRule(
'CronJobAnnouncement',
5,
6,
max_artefacts_a_for_single_b=1,
artefact_match_parameters=[('/model/CronAnnouncement/JobNumber',
'/model/CronExecution/JobNumber')])
a_class_selector = EventClassSelector('Announcement',
[cron_job_announcement], None)
b_class_selector = EventClassSelector('Execution', None,
[cron_job_announcement])
rules = [
Rules.PathExistsMatchRule('/model/CronAnnouncement/Run',
a_class_selector),
Rules.PathExistsMatchRule('/model/CronExecution/Job', b_class_selector)
]
time_correlation_violation_detector = TimeCorrelationViolationDetector(
analysis_context.aminer_config, rules, anomaly_event_handlers)
analysis_context.register_component(
time_correlation_violation_detector,
component_name="TimeCorrelationViolationDetector")
atom_filter.add_handler(time_correlation_violation_detector)
class Base64StringModelElementTest(TestBase):
"""Unittests for the Base64StringModelElement."""
base64_string_model_element = Base64StringModelElement('base64')
match_base64 = 'match/base64'
def test1get_id(self):
"""Test if get_id works properly."""
base64_dme = Base64StringModelElement("s0")
self.assertEqual(base64_dme.get_id(), "s0")
def test2get_child_elements(self):
"""Test if get_child_elements returns None."""
base64_dme = Base64StringModelElement("s0")
self.assertEqual(base64_dme.get_child_elements(), None)
def test3get_match_element_valid_match_string_with_padding(self):
"""Parse matching substring with padding from MatchContext and check if the MatchContext was updated accordingly."""
string = b'This is some string to be encoded.'
base64_string = b'VGhpcyBpcyBzb21lIHN0cmluZyB0byBiZSBlbmNvZGVkLg=='
match_context = DummyMatchContext(base64_string)
base64_dme = Base64StringModelElement("s0")
match_element = base64_dme.get_match_element("base64", match_context)
self.assertEqual(match_element.path, "base64/s0")
self.assertEqual(match_element.match_string, base64_string)
self.assertEqual(match_element.match_object, string)
self.assertIsNone(match_element.children, None)
self.assertEqual(match_context.match_string, base64_string)
def test4get_match_element_valid_match_string_with_one_byte_padding(self):
"""Parse matching substring with padding from MatchContext and check if the MatchContext was updated accordingly."""
string = b'This is some encoded strin'
base64_string = b'VGhpcyBpcyBzb21lIGVuY29kZWQgc3RyaW4='
match_context = DummyMatchContext(base64_string)
base64_dme = Base64StringModelElement("s0")
match_element = base64_dme.get_match_element("base64", match_context)
self.assertEqual(match_element.path, "base64/s0")
self.assertEqual(match_element.match_string, base64_string)
self.assertEqual(match_element.match_object, string)
self.assertIsNone(match_element.children, None)
self.assertEqual(match_context.match_string, base64_string)
def test5get_match_element_valid_match_string_without_padding(self):
"""Parse matching substring without padding from MatchContext and check if the MatchContext was updated accordingly."""
string = b'This is some string to be encoded without the padding character =.'
base64_string = b'VGhpcyBpcyBzb21lIHN0cmluZyB0byBiZSBlbmNvZGVkIHdpdGhvdXQgdGhlIHBhZGRpbmcgY2hhcmFjdGVyID0u'
match_context = DummyMatchContext(base64_string)
base64_dme = Base64StringModelElement("s0")
match_element = base64_dme.get_match_element("base64", match_context)
self.assertEqual(match_element.path, "base64/s0")
self.assertEqual(match_element.match_string, base64_string)
self.assertEqual(match_element.match_object, string)
self.assertIsNone(match_element.children, None)
self.assertEqual(match_context.match_string, base64_string)
def test6get_match_element_valid_match_string_without_exact_length(self):
"""Parse matching substring without exact length (divisible by 4) and check if the MatchContext was updated accordingly."""
string = b'This is some encoded strin'
base64_string = b'VGhpcyBpcyBzb21lIGVuY29kZWQgc3RyaW4'
match_context = DummyMatchContext(base64_string)
base64_dme = Base64StringModelElement("s0")
match_element = base64_dme.get_match_element("base64", match_context)
self.assertEqual(match_element.path, "base64/s0")
self.assertEqual(match_element.match_string, base64_string[:-(len(base64_string) % 4)])
self.assertEqual(match_element.match_object, string[:-2])
self.assertIsNone(match_element.children, None)
self.assertEqual(match_context.match_string, base64_string[:-(len(base64_string) % 4)])
def test7get_match_element_valid_match_string_with_partial_length(self):
"""Parse matching substring out of the MatchContext and check if the MatchContext was updated accordingly."""
string = b'This is some encoded strin'
base64_string = b'VGhpcyBpcyBzb21lIGVuY29kZWQgc3RyaW4='
data = base64_string + b'\nContent: Public Key'
match_context = DummyMatchContext(data)
base64_dme = Base64StringModelElement("s0")
match_element = base64_dme.get_match_element("base64", match_context)
self.assertEqual(match_element.path, "base64/s0")
self.assertEqual(match_element.match_string, base64_string)
self.assertEqual(match_element.match_object, string)
self.assertIsNone(match_element.children, None)
self.assertEqual(match_context.match_string, base64_string)
def test8get_match_element_no_match(self):
"""Parse not matching substring from MatchContext and check if the MatchContext was not changed."""
data = b"!Hello World"
match_context = DummyMatchContext(data)
base64_dme = Base64StringModelElement("s0")
match_element = base64_dme.get_match_element("base64", match_context)
self.assertIsNone(match_element, None)
self.assertEqual(match_context.match_data, data)
def test9get_match_element_unicode_exception(self):
"""Parse a Base64 string which can not be decoded as UTF-8."""
# ² encoded with ISO-8859-1
base64_string = b'sg=='
match_context = DummyMatchContext(base64_string)
base64_dme = Base64StringModelElement("s0")
match_element = base64_dme.get_match_element("base64", match_context)
self.assertEqual(match_element.path, "base64/s0")
self.assertEqual(match_element.match_string, base64_string)
self.assertEqual(match_element.match_object, base64_string)
self.assertIsNone(match_element.children, None)
self.assertEqual(match_context.match_string, base64_string)
def test10path_id_input_validation(self):
"""Check if element_id is validated."""
# empty element_id
path_id = ""
self.assertRaises(ValueError, Base64StringModelElement, path_id)
# bytes element_id is not allowed
path_id = b"path"
self.assertRaises(TypeError, Base64StringModelElement, path_id)
# integer element_id is not allowed
path_id = 123
self.assertRaises(TypeError, Base64StringModelElement, path_id)
# float element_id is not allowed
path_id = 123.22
self.assertRaises(TypeError, Base64StringModelElement, path_id)
# dict element_id is not allowed
path_id = {"id": "path"}
self.assertRaises(TypeError, Base64StringModelElement, path_id)
# list element_id is not allowed
path_id = ["path"]
self.assertRaises(TypeError, Base64StringModelElement, path_id)
def test11performance(self): # skipcq: PYL-R0201
"""Test the performance of the implementation. Comment this test out in normal cases."""
import_setup = """
import copy
from unit.TestBase import DummyMatchContext
from aminer.parsing.Base64StringModelElement import Base64StringModelElement
times = 100000
"""
string100_setup = """
# b'ASCII stands for American Standard Code for Information Interchange. Computers can only understand.'
base64_string = b'QVNDSUkgc3RhbmRzIGZvciBBbWVyaWNhbiBTdGFuZGFyZCBDb2RlIGZvciBJbmZvcm1hdGlvbiBJbnRlcmNoYW5nZS4gQ29tcHV0ZXJzIGNhb' \
b'iBvbmx5IHVuZGVyc3RhbmQu'
"""
string4096_setup = """
# b"ASCII stands for American Standard Code for Information Interchange. Computers can only understand numbers, so an ASCII code " \
# b"is the numerical representation of a character such as 'a' or '@' or an action of some sort. ASCII was developed a long time " \
# b"ago and now the non-printing characters are rarely used for their original purpose. Below is the ASCII character table and " \
# b"this includes descriptions of the first 32 non-printing characters. ASCII was actually designed for use with teletypes and " \
# b"so the descriptions are somewhat obscure. If someone says they want your CV however in ASCII format, all this means is they " \
# b"want 'plain' text with no formatting such as tabs, bold or underscoring - the raw format that any computer can understand. " \
# b"This is usually so they can easily import the file into their own applications without issues. Notepad.exe creates ASCII " \
# b"text, or in MS Word you can save a file as 'text only'ASCII stands for American Standard Code for Information Interchange. " \
# b"Computers can only understand numbers, so an ASCII code is the numerical representation of a character such as 'a' or '@' " \
# b"or an action of some sort. ASCII was developed a long time ago and now the non-printing characters are rarely used for their " \
# b"original purpose. Below is the ASCII character table and this includes descriptions of the first 32 non-printing characters. " \
# b"ASCII was actually designed for use with teletypes and so the descriptions are somewhat obscure. If someone says they want " \
# b"your CV however in ASCII format, all this means is they want 'plain' text with no formatting such as tabs, bold or " \
# b"underscoring - the raw format that any computer can understand. This is usually so they can easily import the file into " \
# b"their own applications without issues. Notepad.exe creates ASCII text, or in MS Word you can save a file as 'text only'" \
# b"ASCII stands for American Standard Code for Information Interchange. Computers can only understand numbers, so an ASCII " \
# b"code is the numerical representation of a character such as 'a' or '@' or an action of some sort. ASCII was developed a " \
# b"long time ago and now the non-printing characters are rarely used for their original purpose. Below is the ASCII " \
# b"character table and this includes descriptions of the first 32 non-printing characters. ASCII was actually designed for " \
# b"use with teletypes and so the descriptions are somewhat obscure. If someone says they want your CV however in ASCII format, " \
# b"all this means is they want 'plain' text with no formatting such as tabs, bold or underscoring - the raw format that any " \
# b"computer can understand. This is usually so they can easily import the file into their own applications without issues. " \
# b"Notepad.exe creates ASCII text, or in MS Word you can save a file as 'text only'ASCII stands for American Standard Code for " \
# b"Information Interchange. Computers can only understand numbers, so an ASCII code is the numerical representation of a " \
# b"character such as 'a' or '@' or an action of some sort. ASCII was developed a long time ago and now the non-printing " \
# b"characters are rarely used for their original purpose. Below is the ASCII character table and this includes descriptions " \
# b"of the first 32 non-printing characters. ASCII was actually designed for use with teletypes and so the descriptions are " \
# b"somewhat obscure. If someone says they want your CV however in ASCII format, all this means is they want 'plain' text with " \
# b"no formatting such as tabs, bold or underscoring - the raw format that any computer can understand. This is usually so they " \
# b"can easily import the file into their own applications without issues. Notepad.exe creates ASCII text, or in MS Word you " \
# b"can save a file as 'text only'ASCII stands for American Standard Code for Information Interchange. Computers can only " \
# b"understand numbers, so an ASCII code is the numerical representation of a character such as 'a' or '@' or an action of " \
# b"some sort. ASCII was developed a long time ago and now the non-printing characters are rarely used for their original " \
# b"purpose. Below is the ASCII character table and this includes descriptions of the first 32 non-prin"
base64_string = b"QVNDSUkgc3RhbmRzIGZvciBBbWVyaWNhbiBTdGFuZGFyZCBDb2RlIGZvciBJbmZvcm1hdGlvbiBJbnRlcmNoYW5nZS4gQ29tcHV0ZXJzIGNhbiBvbmx5IHV" \
b"uZGVyc3RhbmQgbnVtYmVycywgc28gYW4gQVNDSUkgY29kZSBpcyB0aGUgbnVtZXJpY2FsIHJlcHJlc2VudGF0aW9uIG9mIGEgY2hhcmFjdGVyIHN1Y2ggYX" \
b"MgJ2EnIG9yICdAJyBvciBhbiBhY3Rpb24gb2Ygc29tZSBzb3J0LiBBU0NJSSB3YXMgZGV2ZWxvcGVkIGEgbG9uZyB0aW1lIGFnbyBhbmQgbm93IHRoZSBub" \
b"24tcHJpbnRpbmcgY2hhcmFjdGVycyBhcmUgcmFyZWx5IHVzZWQgZm9yIHRoZWlyIG9yaWdpbmFsIHB1cnBvc2UuIEJlbG93IGlzIHRoZSBBU0NJSSBjaGFy" \
b"YWN0ZXIgdGFibGUgYW5kIHRoaXMgaW5jbHVkZXMgZGVzY3JpcHRpb25zIG9mIHRoZSBmaXJzdCAzMiBub24tcHJpbnRpbmcgY2hhcmFjdGVycy4gQVNDSUk" \
b"gd2FzIGFjdHVhbGx5IGRlc2lnbmVkIGZvciB1c2Ugd2l0aCB0ZWxldHlwZXMgYW5kIHNvIHRoZSBkZXNjcmlwdGlvbnMgYXJlIHNvbWV3aGF0IG9ic2N1cm" \
b"UuIElmIHNvbWVvbmUgc2F5cyB0aGV5IHdhbnQgeW91ciBDViBob3dldmVyIGluIEFTQ0lJIGZvcm1hdCwgYWxsIHRoaXMgbWVhbnMgaXMgdGhleSB3YW50I" \
b"CdwbGFpbicgdGV4dCB3aXRoIG5vIGZvcm1hdHRpbmcgc3VjaCBhcyB0YWJzLCBib2xkIG9yIHVuZGVyc2NvcmluZyAtIHRoZSByYXcgZm9ybWF0IHRoYXQg" \
b"YW55IGNvbXB1dGVyIGNhbiB1bmRlcnN0YW5kLiBUaGlzIGlzIHVzdWFsbHkgc28gdGhleSBjYW4gZWFzaWx5IGltcG9ydCB0aGUgZmlsZSBpbnRvIHRoZWl" \
b"yIG93biBhcHBsaWNhdGlvbnMgd2l0aG91dCBpc3N1ZXMuIE5vdGVwYWQuZXhlIGNyZWF0ZXMgQVNDSUkgdGV4dCwgb3IgaW4gTVMgV29yZCB5b3UgY2FuIH" \
b"NhdmUgYSBmaWxlIGFzICd0ZXh0IG9ubHknQVNDSUkgc3RhbmRzIGZvciBBbWVyaWNhbiBTdGFuZGFyZCBDb2RlIGZvciBJbmZvcm1hdGlvbiBJbnRlcmNoY" \
b"W5nZS4gQ29tcHV0ZXJzIGNhbiBvbmx5IHVuZGVyc3RhbmQgbnVtYmVycywgc28gYW4gQVNDSUkgY29kZSBpcyB0aGUgbnVtZXJpY2FsIHJlcHJlc2VudGF0" \
b"aW9uIG9mIGEgY2hhcmFjdGVyIHN1Y2ggYXMgJ2EnIG9yICdAJyBvciBhbiBhY3Rpb24gb2Ygc29tZSBzb3J0LiBBU0NJSSB3YXMgZGV2ZWxvcGVkIGEgbG9" \
b"uZyB0aW1lIGFnbyBhbmQgbm93IHRoZSBub24tcHJpbnRpbmcgY2hhcmFjdGVycyBhcmUgcmFyZWx5IHVzZWQgZm9yIHRoZWlyIG9yaWdpbmFsIHB1cnBvc2" \
b"UuIEJlbG93IGlzIHRoZSBBU0NJSSBjaGFyYWN0ZXIgdGFibGUgYW5kIHRoaXMgaW5jbHVkZXMgZGVzY3JpcHRpb25zIG9mIHRoZSBmaXJzdCAzMiBub24tc" \
b"HJpbnRpbmcgY2hhcmFjdGVycy4gQVNDSUkgd2FzIGFjdHVhbGx5IGRlc2lnbmVkIGZvciB1c2Ugd2l0aCB0ZWxldHlwZXMgYW5kIHNvIHRoZSBkZXNjcmlw" \
b"dGlvbnMgYXJlIHNvbWV3aGF0IG9ic2N1cmUuIElmIHNvbWVvbmUgc2F5cyB0aGV5IHdhbnQgeW91ciBDViBob3dldmVyIGluIEFTQ0lJIGZvcm1hdCwgYWx" \
b"sIHRoaXMgbWVhbnMgaXMgdGhleSB3YW50ICdwbGFpbicgdGV4dCB3aXRoIG5vIGZvcm1hdHRpbmcgc3VjaCBhcyB0YWJzLCBib2xkIG9yIHVuZGVyc2Nvcm" \
b"luZyAtIHRoZSByYXcgZm9ybWF0IHRoYXQgYW55IGNvbXB1dGVyIGNhbiB1bmRlcnN0YW5kLiBUaGlzIGlzIHVzdWFsbHkgc28gdGhleSBjYW4gZWFzaWx5I" \
b"GltcG9ydCB0aGUgZmlsZSBpbnRvIHRoZWlyIG93biBhcHBsaWNhdGlvbnMgd2l0aG91dCBpc3N1ZXMuIE5vdGVwYWQuZXhlIGNyZWF0ZXMgQVNDSUkgdGV4" \
b"dCwgb3IgaW4gTVMgV29yZCB5b3UgY2FuIHNhdmUgYSBmaWxlIGFzICd0ZXh0IG9ubHknQVNDSUkgc3RhbmRzIGZvciBBbWVyaWNhbiBTdGFuZGFyZCBDb2R" \
b"lIGZvciBJbmZvcm1hdGlvbiBJbnRlcmNoYW5nZS4gQ29tcHV0ZXJzIGNhbiBvbmx5IHVuZGVyc3RhbmQgbnVtYmVycywgc28gYW4gQVNDSUkgY29kZSBpcy" \
b"B0aGUgbnVtZXJpY2FsIHJlcHJlc2VudGF0aW9uIG9mIGEgY2hhcmFjdGVyIHN1Y2ggYXMgJ2EnIG9yICdAJyBvciBhbiBhY3Rpb24gb2Ygc29tZSBzb3J0L" \
b"iBBU0NJSSB3YXMgZGV2ZWxvcGVkIGEgbG9uZyB0aW1lIGFnbyBhbmQgbm93IHRoZSBub24tcHJpbnRpbmcgY2hhcmFjdGVycyBhcmUgcmFyZWx5IHVzZWQg" \
b"Zm9yIHRoZWlyIG9yaWdpbmFsIHB1cnBvc2UuIEJlbG93IGlzIHRoZSBBU0NJSSBjaGFyYWN0ZXIgdGFibGUgYW5kIHRoaXMgaW5jbHVkZXMgZGVzY3JpcHR" \
b"pb25zIG9mIHRoZSBmaXJzdCAzMiBub24tcHJpbnRpbmcgY2hhcmFjdGVycy4gQVNDSUkgd2FzIGFjdHVhbGx5IGRlc2lnbmVkIGZvciB1c2Ugd2l0aCB0ZW" \
b"xldHlwZXMgYW5kIHNvIHRoZSBkZXNjcmlwdGlvbnMgYXJlIHNvbWV3aGF0IG9ic2N1cmUuIElmIHNvbWVvbmUgc2F5cyB0aGV5IHdhbnQgeW91ciBDViBob" \
b"3dldmVyIGluIEFTQ0lJIGZvcm1hdCwgYWxsIHRoaXMgbWVhbnMgaXMgdGhleSB3YW50ICdwbGFpbicgdGV4dCB3aXRoIG5vIGZvcm1hdHRpbmcgc3VjaCBh" \
b"cyB0YWJzLCBib2xkIG9yIHVuZGVyc2NvcmluZyAtIHRoZSByYXcgZm9ybWF0IHRoYXQgYW55IGNvbXB1dGVyIGNhbiB1bmRlcnN0YW5kLiBUaGlzIGlzIHV" \
b"zdWFsbHkgc28gdGhleSBjYW4gZWFzaWx5IGltcG9ydCB0aGUgZmlsZSBpbnRvIHRoZWlyIG93biBhcHBsaWNhdGlvbnMgd2l0aG91dCBpc3N1ZXMuIE5vdG" \
b"VwYWQuZXhlIGNyZWF0ZXMgQVNDSUkgdGV4dCwgb3IgaW4gTVMgV29yZCB5b3UgY2FuIHNhdmUgYSBmaWxlIGFzICd0ZXh0IG9ubHknQVNDSUkgc3RhbmRzI" \
b"GZvciBBbWVyaWNhbiBTdGFuZGFyZCBDb2RlIGZvciBJbmZvcm1hdGlvbiBJbnRlcmNoYW5nZS4gQ29tcHV0ZXJzIGNhbiBvbmx5IHVuZGVyc3RhbmQgbnVt" \
b"YmVycywgc28gYW4gQVNDSUkgY29kZSBpcyB0aGUgbnVtZXJpY2FsIHJlcHJlc2VudGF0aW9uIG9mIGEgY2hhcmFjdGVyIHN1Y2ggYXMgJ2EnIG9yICdAJyB" \
b"vciBhbiBhY3Rpb24gb2Ygc29tZSBzb3J0LiBBU0NJSSB3YXMgZGV2ZWxvcGVkIGEgbG9uZyB0aW1lIGFnbyBhbmQgbm93IHRoZSBub24tcHJpbnRpbmcgY2" \
b"hhcmFjdGVycyBhcmUgcmFyZWx5IHVzZWQgZm9yIHRoZWlyIG9yaWdpbmFsIHB1cnBvc2UuIEJlbG93IGlzIHRoZSBBU0NJSSBjaGFyYWN0ZXIgdGFibGUgY" \
b"W5kIHRoaXMgaW5jbHVkZXMgZGVzY3JpcHRpb25zIG9mIHRoZSBmaXJzdCAzMiBub24tcHJpbnRpbmcgY2hhcmFjdGVycy4gQVNDSUkgd2FzIGFjdHVhbGx5" \
b"IGRlc2lnbmVkIGZvciB1c2Ugd2l0aCB0ZWxldHlwZXMgYW5kIHNvIHRoZSBkZXNjcmlwdGlvbnMgYXJlIHNvbWV3aGF0IG9ic2N1cmUuIElmIHNvbWVvbmU" \
b"gc2F5cyB0aGV5IHdhbnQgeW91ciBDViBob3dldmVyIGluIEFTQ0lJIGZvcm1hdCwgYWxsIHRoaXMgbWVhbnMgaXMgdGhleSB3YW50ICdwbGFpbicgdGV4dC" \
b"B3aXRoIG5vIGZvcm1hdHRpbmcgc3VjaCBhcyB0YWJzLCBib2xkIG9yIHVuZGVyc2NvcmluZyAtIHRoZSByYXcgZm9ybWF0IHRoYXQgYW55IGNvbXB1dGVyI" \
b"GNhbiB1bmRlcnN0YW5kLiBUaGlzIGlzIHVzdWFsbHkgc28gdGhleSBjYW4gZWFzaWx5IGltcG9ydCB0aGUgZmlsZSBpbnRvIHRoZWlyIG93biBhcHBsaWNh" \
b"dGlvbnMgd2l0aG91dCBpc3N1ZXMuIE5vdGVwYWQuZXhlIGNyZWF0ZXMgQVNDSUkgdGV4dCwgb3IgaW4gTVMgV29yZCB5b3UgY2FuIHNhdmUgYSBmaWxlIGF" \
b"zICd0ZXh0IG9ubHknQVNDSUkgc3RhbmRzIGZvciBBbWVyaWNhbiBTdGFuZGFyZCBDb2RlIGZvciBJbmZvcm1hdGlvbiBJbnRlcmNoYW5nZS4gQ29tcHV0ZX" \
b"JzIGNhbiBvbmx5IHVuZGVyc3RhbmQgbnVtYmVycywgc28gYW4gQVNDSUkgY29kZSBpcyB0aGUgbnVtZXJpY2FsIHJlcHJlc2VudGF0aW9uIG9mIGEgY2hhc" \
b"mFjdGVyIHN1Y2ggYXMgJ2EnIG9yICdAJyBvciBhbiBhY3Rpb24gb2Ygc29tZSBzb3J0LiBBU0NJSSB3YXMgZGV2ZWxvcGVkIGEgbG9uZyB0aW1lIGFnbyBh" \
b"bmQgbm93IHRoZSBub24tcHJpbnRpbmcgY2hhcmFjdGVycyBhcmUgcmFyZWx5IHVzZWQgZm9yIHRoZWlyIG9yaWdpbmFsIHB1cnBvc2UuIEJlbG93IGlzIHR" \
b"oZSBBU0NJSSBjaGFyYWN0ZXIgdGFibGUgYW5kIHRoaXMgaW5jbHVkZXMgZGVzY3JpcHRpb25zIG9mIHRoZSBmaXJzdCAzMiBub24tcHJpbg=="
"""
end_setup = """
dummy_match_context = DummyMatchContext(base64_string)
dummy_match_context_list = [copy.deepcopy(dummy_match_context) for _ in range(times)]
base64_dme = Base64StringModelElement("s0")
def run():
match_context = dummy_match_context_list.pop(0)
base64_dme.get_match_element("base64", match_context)
"""
_setup100 = import_setup + string100_setup + end_setup
_setup4096 = import_setup + string4096_setup + end_setup
def test2get_child_elements(self):
"""Test if get_child_elements returns None."""
base64_dme = Base64StringModelElement("s0")
self.assertEqual(base64_dme.get_child_elements(), None)
def test1get_id(self):
"""Test if get_id works properly."""
base64_dme = Base64StringModelElement("s0")
self.assertEqual(base64_dme.get_id(), "s0")
