def test13mandatory_none_padding_exponent(self):
match_context = MatchContext(b'+25537.21e10 uid=2')
decimal_float_value_me = DecimalFloatValueModelElement(
None, DecimalFloatValueModelElement.SIGN_TYPE_MANDATORY,
DecimalFloatValueModelElement.PAD_TYPE_NONE,
DecimalFloatValueModelElement.EXP_TYPE_MANDATORY)
match_element = decimal_float_value_me.get_match_element(
None, match_context)
self.assertNotEqual(match_element, None,
self.match_element_should_exist)
self.assertEqual(match_element.get_match_string(), b'+25537.21e10',
self.match_element_unexpected_result)
self.assertEqual(match_element.get_match_object(),
25537.21 * math.pow(10, 10),
self.match_element_unexpected_value)
match_context = MatchContext(b'+25537.21e uid=2')
decimal_float_value_me = DecimalFloatValueModelElement(
None, DecimalFloatValueModelElement.SIGN_TYPE_MANDATORY,
DecimalFloatValueModelElement.PAD_TYPE_NONE,
DecimalFloatValueModelElement.EXP_TYPE_MANDATORY)
match_element = decimal_float_value_me.get_match_element(
None, match_context)
self.assertEqual(match_element, None,
self.match_element_should_not_exist)
def test11value_pad_type_input_validation(self):
"""Check if value_pad_type is validated."""
DecimalFloatValueModelElement(self.id_, value_pad_type="none")
DecimalFloatValueModelElement(self.id_, value_pad_type="zero")
DecimalFloatValueModelElement(self.id_, value_pad_type="blank")
self.assertRaises(ValueError,
DecimalFloatValueModelElement,
self.id_,
value_pad_type="None")
self.assertRaises(TypeError,
DecimalFloatValueModelElement,
self.id_,
value_pad_type=None)
self.assertRaises(TypeError,
DecimalFloatValueModelElement,
self.id_,
value_pad_type=b"none")
self.assertRaises(TypeError,
DecimalFloatValueModelElement,
self.id_,
value_pad_type=True)
self.assertRaises(TypeError,
DecimalFloatValueModelElement,
self.id_,
value_pad_type=123)
self.assertRaises(TypeError,
DecimalFloatValueModelElement,
self.id_,
value_pad_type=123.22)
self.assertRaises(TypeError,
DecimalFloatValueModelElement,
self.id_,
value_pad_type={"value_sign_type": "none"})
self.assertRaises(TypeError,
DecimalFloatValueModelElement,
self.id_,
value_pad_type=["none"])
self.assertRaises(TypeError,
DecimalFloatValueModelElement,
self.id_,
value_pad_type=[])
self.assertRaises(TypeError,
DecimalFloatValueModelElement,
self.id_,
value_pad_type=())
self.assertRaises(TypeError,
DecimalFloatValueModelElement,
self.id_,
value_pad_type=set())
def test12exponent_type_input_validation(self):
"""Check if exponent_type is validated."""
DecimalFloatValueModelElement(self.id_, exponent_type="none")
DecimalFloatValueModelElement(self.id_, exponent_type="optional")
DecimalFloatValueModelElement(self.id_, exponent_type="mandatory")
self.assertRaises(ValueError,
DecimalFloatValueModelElement,
self.id_,
exponent_type="None")
self.assertRaises(TypeError,
DecimalFloatValueModelElement,
self.id_,
exponent_type=None)
self.assertRaises(TypeError,
DecimalFloatValueModelElement,
self.id_,
exponent_type=b"none")
self.assertRaises(TypeError,
DecimalFloatValueModelElement,
self.id_,
exponent_type=True)
self.assertRaises(TypeError,
DecimalFloatValueModelElement,
self.id_,
exponent_type=123)
self.assertRaises(TypeError,
DecimalFloatValueModelElement,
self.id_,
exponent_type=123.22)
self.assertRaises(TypeError,
DecimalFloatValueModelElement,
self.id_,
exponent_type={"value_sign_type": "none"})
self.assertRaises(TypeError,
DecimalFloatValueModelElement,
self.id_,
exponent_type=["none"])
self.assertRaises(TypeError,
DecimalFloatValueModelElement,
self.id_,
exponent_type=[])
self.assertRaises(TypeError,
DecimalFloatValueModelElement,
self.id_,
exponent_type=())
self.assertRaises(TypeError,
DecimalFloatValueModelElement,
self.id_,
exponent_type=set())
def test7positive_number_zero_padding(self):
"""In this testcase the positive Integer equivalence class in combination with the zero padding, which represents the padding
equivalence class, is tested."""
match_context = MatchContext(b'00025537.21 uid=2')
decimal_float_value_me = DecimalFloatValueModelElement(
None, DecimalFloatValueModelElement.SIGN_TYPE_NONE,
DecimalFloatValueModelElement.PAD_TYPE_ZERO,
DecimalFloatValueModelElement.EXP_TYPE_NONE)
match_element = decimal_float_value_me.get_match_element(
None, match_context)
self.assertNotEqual(match_element, None,
self.match_element_should_exist)
self.assertEqual(match_element.get_match_string(), b'00025537.21',
self.match_element_unexpected_result)
self.assertEqual(match_element.get_match_object(), 25537.21,
self.match_element_unexpected_value)
match_context = MatchContext(self.positive_string)
match_element = decimal_float_value_me.get_match_element(
None, match_context)
self.assertNotEqual(match_element, None,
self.match_element_should_exist)
self.assertEqual(match_element.get_match_string(), b'25537.21',
self.match_element_unexpected_result)
self.assertEqual(match_element.get_match_object(), 25537.21,
self.match_element_unexpected_value)
def test5sign_type_mandatory_none_padding(self):
"""This testcase represents the equivalence class of all numbers with a mandatory sign in combination with no padding. It unit
the correctness of the Path usage for all integers with a mandatory sign without padding."""
match_context = MatchContext(self.negative_string)
decimal_float_value_me = DecimalFloatValueModelElement(
None, DecimalFloatValueModelElement.SIGN_TYPE_MANDATORY,
DecimalFloatValueModelElement.PAD_TYPE_NONE,
DecimalFloatValueModelElement.EXP_TYPE_NONE)
match_element = decimal_float_value_me.get_match_element(
None, match_context)
self.assertNotEqual(match_element, None,
self.match_element_should_exist)
self.assertEqual(match_element.get_match_string(),
self.negative_number,
self.match_element_unexpected_result)
self.assertEqual(match_element.get_match_object(), -25537.21,
self.match_element_unexpected_value)
match_context = MatchContext(b'+25537 uid=2')
match_element = decimal_float_value_me.get_match_element(
None, match_context)
self.assertNotEqual(match_element, None,
self.match_element_should_exist)
self.assertEqual(match_element.get_match_string(), b'+25537',
self.match_element_unexpected_result)
self.assertEqual(match_element.get_match_object(), 25537,
self.match_element_unexpected_value)
def get_systemd_model():
"""Return the parsing model for messages directly from systemd."""
type_children = [
FixedDataModelElement('apt-daily-start',
b'Starting Daily apt activities...'),
FixedDataModelElement('apt-daily-started',
b'Started Daily apt activities.'),
SequenceModelElement('apt-daily-timer', [
FixedDataModelElement('s0', b'apt-daily.timer: Adding '),
OptionalMatchModelElement(
'hopt',
SequenceModelElement('hblock', [
DecimalIntegerValueModelElement('hours'),
FixedDataModelElement('s1', b'h ')
])),
DecimalIntegerValueModelElement('minutes'),
FixedDataModelElement('s2', b'min '),
DecimalFloatValueModelElement('seconds'),
FixedDataModelElement('s3', b's random time.')
]),
FixedDataModelElement('tmp-file-cleanup',
b'Starting Cleanup of Temporary Directories...'),
FixedDataModelElement('tmp-file-cleanup-started',
b'Started Cleanup of Temporary Directories.')
]
model = SequenceModelElement('systemd', [
FixedDataModelElement('sname', b'systemd['),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s0', b']: '),
FirstMatchModelElement('msg', type_children)
])
return model
def test13get_match_element_match_context_input_validation(self):
"""Check if an exception is raised, when other classes than MatchContext are used in get_match_element."""
model_element = DecimalFloatValueModelElement(self.id_)
data = b"123.22"
model_element.get_match_element(self.path, DummyMatchContext(data))
model_element.get_match_element(self.path, MatchContext(data))
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, MatchElement(self.path, data, None, None))
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, data)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, data.decode())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, True)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, 123)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, 123.22)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, None)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, [])
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, {"key": MatchContext(data)})
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, set())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, ())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, model_element)
def test9negative_number_blank_padding(self):
"""In this testcase the negative Integer equivalence class in combination with the blank character padding, which represents the
padding equivalence class, is tested."""
match_context = MatchContext(b'- 25537.21 uid=2')
decimal_float_value_me = DecimalFloatValueModelElement(
None, DecimalFloatValueModelElement.SIGN_TYPE_OPTIONAL,
DecimalFloatValueModelElement.PAD_TYPE_BLANK,
DecimalFloatValueModelElement.EXP_TYPE_NONE)
match_element = decimal_float_value_me.get_match_element(
None, match_context)
self.assertNotEqual(match_element, None,
self.match_element_should_exist)
self.assertEqual(match_element.get_match_string(), b'- 25537.21',
self.match_element_unexpected_result)
self.assertEqual(match_element.get_match_object(), -25537.21,
self.match_element_unexpected_value)
match_context = MatchContext(self.negative_string)
match_element = decimal_float_value_me.get_match_element(
None, match_context)
self.assertNotEqual(match_element, None,
self.match_element_should_exist)
self.assertEqual(match_element.get_match_string(),
self.negative_number,
self.match_element_unexpected_result)
self.assertEqual(match_element.get_match_object(), -25537.21,
self.match_element_unexpected_value)
def test13mandatory_none_padding_exponent(self):
"""In this testcase the mandatory sign equivalence class in combination with the None padding and exponent type mandatory."""
match_context = MatchContext(b'+25537.21e10 uid=2')
decimal_float_value_me = DecimalFloatValueModelElement(
None, DecimalFloatValueModelElement.SIGN_TYPE_MANDATORY, DecimalFloatValueModelElement.PAD_TYPE_NONE,
DecimalFloatValueModelElement.EXP_TYPE_MANDATORY)
match_element = decimal_float_value_me.get_match_element(None, match_context)
self.assertNotEqual(match_element, None, self.match_element_should_exist)
self.assertEqual(match_element.get_match_string(), b'+25537.21e10', self.match_element_unexpected_result)
self.assertEqual(match_element.get_match_object(), 25537.21 * math.pow(10, 10), self.match_element_unexpected_value)
match_context = MatchContext(b'+25537.21e uid=2')
decimal_float_value_me = DecimalFloatValueModelElement(
None, DecimalFloatValueModelElement.SIGN_TYPE_MANDATORY, DecimalFloatValueModelElement.PAD_TYPE_NONE,
DecimalFloatValueModelElement.EXP_TYPE_MANDATORY)
match_element = decimal_float_value_me.get_match_element(None, match_context)
self.assertEqual(match_element, None, self.match_element_should_not_exist)
def test8positive_number_zero_padding_no_match(self):
"""In this testcase the positive Integer equivalence class in combination with the zero padding is tested with no match expected."""
match_context = MatchContext(b' 00025537.21 uid=2')
decimal_float_value_me = DecimalFloatValueModelElement(
None, DecimalFloatValueModelElement.SIGN_TYPE_NONE, DecimalFloatValueModelElement.PAD_TYPE_ZERO,
DecimalFloatValueModelElement.EXP_TYPE_NONE)
match_element = decimal_float_value_me.get_match_element(None, match_context)
self.assertEqual(match_element, None, self.match_element_should_not_exist)
def test6sign_type_mandatory_none_padding_no_match(self):
"""
This testcase represents the equivalence class of all numbers with a mandatory sign in combination with no padding.
It tests the correctness of the Path usage for all integers with a mandatory sign without padding, when no match is found.
"""
match_context = MatchContext(self.positive_string)
decimal_float_value_me = DecimalFloatValueModelElement(
None, DecimalFloatValueModelElement.SIGN_TYPE_MANDATORY, DecimalFloatValueModelElement.PAD_TYPE_NONE,
DecimalFloatValueModelElement.EXP_TYPE_NONE)
match_element = decimal_float_value_me.get_match_element(None, match_context)
self.assertEqual(match_element, None, self.match_element_should_not_exist)
def test4negative_number_none_padding_no_match(self):
"""
This testcase represents the equivalence class of negative numbers in combination with no padding.
It tests the correctness of the path usage for all negative integers without padding, when no match is found.
"""
match_context = MatchContext(b'- 25537.21 uid=2')
decimal_float_value_me = DecimalFloatValueModelElement(
None, DecimalFloatValueModelElement.SIGN_TYPE_NONE, DecimalFloatValueModelElement.PAD_TYPE_NONE,
DecimalFloatValueModelElement.EXP_TYPE_NONE)
match_element = decimal_float_value_me.get_match_element(None, match_context)
self.assertEqual(match_element, None, self.match_element_should_not_exist)
def test10negative_number_blank_padding_no_match(self):
"""
In this testcase the negative Integer equivalence class in combination with the blank character padding is tested.
No match expected.
"""
match_context = MatchContext(b' -25537 uid=2')
decimal_float_value_me = DecimalFloatValueModelElement(
None, DecimalFloatValueModelElement.SIGN_TYPE_OPTIONAL, DecimalFloatValueModelElement.PAD_TYPE_BLANK,
DecimalFloatValueModelElement.EXP_TYPE_NONE)
match_element = decimal_float_value_me.get_match_element(None, match_context)
self.assertEqual(match_element, None, self.match_element_should_not_exist)
def get_systemd_model():
"""Return the parsing model for messages directly from systemd."""
type_children = [
FixedDataModelElement("apt-daily-start", b"Starting Daily apt upgrade and clean activities..."),
FixedDataModelElement("apt-daily-started", b"Started Daily apt upgrade and clean activities."),
FixedDataModelElement("apt-daily-finished", b"Finished Daily apt upgrade and clean activities."),
SequenceModelElement("service-succeeded", [
DelimitedDataModelElement("service", b" "),
FixedDataModelElement("s0", b" Succeeded.")
]),
FixedDataModelElement("clean-php", b"Finished Clean php session files."),
FixedDataModelElement("finished-logrotate", b"Finished Rotate log files."),
FixedDataModelElement("finished-man-db-daily", b"Finished Daily man-db regeneration."),
FixedDataModelElement("finished-ubuntu-advantages", b"Finished Ubuntu Advantage APT and MOTD Messages."),
FixedDataModelElement("finished-refresh", b"Finished Refresh fwupd metadata and update motd."),
FixedDataModelElement("finished-daily-apt", b"Finished Daily apt download activities."),
SequenceModelElement("apt-daily-timer", [
FixedDataModelElement("s0", b"apt-daily.timer: Adding "),
OptionalMatchModelElement("hopt", SequenceModelElement("hblock", [
DecimalIntegerValueModelElement("hours"),
FixedDataModelElement("s1", b"h ")
])),
DecimalIntegerValueModelElement("minutes"),
FixedDataModelElement("s2", b"min "),
DecimalFloatValueModelElement("seconds"),
FixedDataModelElement("s3", b"s random time.")
]),
FixedDataModelElement("tmp-file-cleanup", b"Starting Cleanup of Temporary Directories..."),
FixedDataModelElement("tmp-file-cleanup-started", b"Started Cleanup of Temporary Directories."),
SequenceModelElement("killing-process", [
DelimitedDataModelElement("service", b":"),
FixedDataModelElement("s0", b": Killing process "),
DecimalIntegerValueModelElement("pid"),
FixedDataModelElement("s1", b" (update-notifier) with signal SIGKILL.")
]),
SequenceModelElement("starting", [
FixedDataModelElement("s0", b"Starting "),
DelimitedDataModelElement("service", b"."),
FixedDataModelElement("s1", b"...")
]),
SequenceModelElement("started", [
FixedDataModelElement("s0", b"Started "),
DelimitedDataModelElement("service", b".", consume_delimiter=True)
]),
FixedDataModelElement("reloading", b"Reloading.")
]
model = SequenceModelElement("systemd", [
FixedDataModelElement("sname", b"systemd["),
DecimalIntegerValueModelElement("pid"),
FixedDataModelElement("s0", b"]: "),
FirstMatchModelElement("msg", type_children)
])
return model
def get_model():
"""Get the model."""
interface_name_model = VariableByteDataModelElement('interface', b'0123456789abcdefghijklmnopqrstuvwxyz.')
type_children = [
SequenceModelElement('exit', [
FixedDataModelElement('s0', b'ntpd exiting on signal '),
DecimalIntegerValueModelElement('signal')
]),
SequenceModelElement('listen-drop', [
FixedDataModelElement('s0', b'Listen and drop on '),
DecimalIntegerValueModelElement('fd'),
FixedDataModelElement('s1', b' '),
interface_name_model,
FixedDataModelElement('s2', b' '),
FirstMatchModelElement('address', [
IpAddressDataModelElement('ipv4'),
DelimitedDataModelElement('ipv6', b' ')
]),
FixedDataModelElement('s3', b' UDP 123')
]),
SequenceModelElement('listen-normal', [
FixedDataModelElement('s0', b'Listen normally on '),
DecimalIntegerValueModelElement('fd'),
FixedDataModelElement('s1', b' '),
interface_name_model,
FixedDataModelElement('s2', b' '),
IpAddressDataModelElement('ip'),
FirstMatchModelElement('msg', [
FixedDataModelElement('port-new', b':123'),
FixedDataModelElement('port-old', b' UDP 123')
])
]),
SequenceModelElement('listen-routing', [
FixedDataModelElement('s0', b'Listening on routing socket on fd #'),
DecimalIntegerValueModelElement('fd'),
FixedDataModelElement('s1', b' for interface updates')
]),
FixedDataModelElement('new-interfaces', b'new interface(s) found: waking up resolver'),
FixedDataModelElement('ntp-io', b'ntp_io: estimated max descriptors: 1024, initial socket boundary: 16'),
FixedDataModelElement('peers-refreshed', b'peers refreshed'),
SequenceModelElement('precision', [
FixedDataModelElement('s0', b'proto: precision = '),
DecimalFloatValueModelElement('precision'),
FixedDataModelElement('s1', b' usec')])]
model = SequenceModelElement('ntpd', [
FixedDataModelElement('sname', b'ntpd['),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s0', b']: '),
FirstMatchModelElement('msg', type_children)
])
return model
def test3get_match_element_default_values(self):
"""Test valid float values with default values of value_sign_type, value_pad_type and exponent_type."""
decimal_float_value_me = DecimalFloatValueModelElement(
self.id_, DecimalFloatValueModelElement.SIGN_TYPE_NONE,
DecimalFloatValueModelElement.PAD_TYPE_NONE,
DecimalFloatValueModelElement.EXP_TYPE_NONE)
data = b"22.25 some string."
value = b"22.25"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, 22.25, None)
data = b"0.25 some string."
value = b"0.25"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, 0.25, None)
data = b"22 some string."
value = b"22"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, 22, None)
data = b"22.12.2021 some string."
value = b"22.12"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, 22.12, None)
data = b"22. some string"
value = b"22."
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, 22.0, None)
data = b"0 some string"
value = b"0"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, 0, None)
def test3negative_number_none_padding(self):
"""
This testcase represents the equivalence class of negative numbers in combination with no padding.
It tests the correctness of the path usage for all negative integers without padding.
"""
match_context = MatchContext(self.negative_string)
decimal_float_value_me = DecimalFloatValueModelElement(
None, DecimalFloatValueModelElement.SIGN_TYPE_OPTIONAL, DecimalFloatValueModelElement.PAD_TYPE_NONE,
DecimalFloatValueModelElement.EXP_TYPE_NONE)
match_element = decimal_float_value_me.get_match_element(None, match_context)
self.assertNotEqual(match_element, None, self.match_element_should_exist)
self.assertEqual(match_element.get_match_string(), self.negative_number, self.match_element_unexpected_result)
self.assertEqual(match_element.get_match_object(), -25537.21, self.match_element_unexpected_value)
def test12mandatory_zero_padding_no_match(self):
"""In this testcase the mandatory sign equivalence class in combination with the zero padding, which represents the padding
equivalence class, is tested with no match expected."""
match_context = MatchContext(b'00025537.21 uid=2')
decimal_float_value_me = DecimalFloatValueModelElement(
None, DecimalFloatValueModelElement.SIGN_TYPE_MANDATORY,
DecimalFloatValueModelElement.PAD_TYPE_ZERO,
DecimalFloatValueModelElement.EXP_TYPE_NONE)
match_element = decimal_float_value_me.get_match_element(
None, match_context)
self.assertEqual(match_element, None,
self.match_element_should_not_exist)
match_context = MatchContext(self.positive_string)
match_element = decimal_float_value_me.get_match_element(
None, match_context)
self.assertEqual(match_element, None,
self.match_element_should_not_exist)
def test6get_match_element_optional_zero_values_no_match(self):
"""Test not matching values with default values of value_sign_type, value_pad_type and exponent_type."""
decimal_float_value_me = DecimalFloatValueModelElement(
self.id_, DecimalFloatValueModelElement.SIGN_TYPE_OPTIONAL,
DecimalFloatValueModelElement.PAD_TYPE_ZERO,
DecimalFloatValueModelElement.EXP_TYPE_OPTIONAL)
data = b"+22.25"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b"22,25"
value = b"22"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, 22, None)
data = b".25"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b" 25"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b" 25"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b"e+10"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
def test11mandatory_zero_padding(self):
"""In this testcase the mandatory sign equivalence class in combination with the zero padding is tested."""
match_context = MatchContext(b'+00025537.21 uid=2')
decimal_float_value_me = DecimalFloatValueModelElement(
None, DecimalFloatValueModelElement.SIGN_TYPE_MANDATORY, DecimalFloatValueModelElement.PAD_TYPE_ZERO,
DecimalFloatValueModelElement.EXP_TYPE_NONE)
match_element = decimal_float_value_me.get_match_element(None, match_context)
self.assertNotEqual(match_element, None, self.match_element_should_exist)
self.assertEqual(match_element.get_match_string(), b'+00025537.21', self.match_element_unexpected_result)
self.assertEqual(match_element.get_match_object(), 25537.21, self.match_element_unexpected_value)
match_context = MatchContext(b'-00025537.21 uid=2')
match_element = decimal_float_value_me.get_match_element(None, match_context)
self.assertNotEqual(match_element, None, self.match_element_should_exist)
self.assertEqual(match_element.get_match_string(), b'-00025537.21', self.match_element_unexpected_result)
self.assertEqual(match_element.get_match_object(), -25537.21, self.match_element_unexpected_value)
match_context = MatchContext(b'+25537.21 uid=2')
match_element = decimal_float_value_me.get_match_element(None, match_context)
self.assertNotEqual(match_element, None, self.match_element_should_exist)
self.assertEqual(match_element.get_match_string(), b'+25537.21', self.match_element_unexpected_result)
self.assertEqual(match_element.get_match_object(), 25537.21, self.match_element_unexpected_value)
def build_analysis_pipeline(analysis_context):
"""
Define the function to create pipeline for parsing the log data.
It has also to define an AtomizerFactory to instruct aminer how to process incoming data streams to create log atoms from them.
"""
date_format_string = b'%Y-%m-%d %H:%M:%S'
cron = b' cron['
# Build the parsing model:
service_children_disk_report = [
FixedDataModelElement('Space', b' Current Disk Data is: Filesystem Type Size Used Avail Use%'),
DelimitedDataModelElement('Data', b'%'), AnyByteDataModelElement('Rest')]
service_children_login_details = [
FixedDataModelElement('User/LoginDetails', b'User '), DelimitedDataModelElement('Username', b' '),
FixedWordlistDataModelElement('Status', [b' logged in', b' logged out']),
OptionalMatchModelElement('PastTime', SequenceModelElement('Time', [
FixedDataModelElement('Blank', b' '), DecimalIntegerValueModelElement('Minutes'),
FixedDataModelElement('Ago', b' minutes ago.')]))]
service_children_cron_job = [
DateTimeModelElement('DTM', date_format_string), FixedDataModelElement('UNameSpace1', b' '),
DelimitedDataModelElement('UName', b' '), FixedDataModelElement('UNameSpace2', b' '), DelimitedDataModelElement('User', b' '),
FixedDataModelElement('Cron', cron), DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Details', b']: Job `cron.daily` started.')]
service_children_random_time = [FixedDataModelElement('Space', b'Random: '), DecimalIntegerValueModelElement('Random')]
service_children_sensors = [SequenceModelElement('CPUTemp', [
FixedDataModelElement('FixedTemp', b'CPU Temp: '), DecimalIntegerValueModelElement('Temp'),
FixedDataModelElement('Degrees', b'\xc2\xb0C')]), FixedDataModelElement('Space1', b', '), SequenceModelElement('CPUWorkload', [
FixedDataModelElement('FixedWorkload', b'CPU Workload: '), DecimalIntegerValueModelElement('Workload'),
FixedDataModelElement('Percent', b'%')]), FixedDataModelElement('Space2', b', '),
DateTimeModelElement('DTM', date_format_string)]
service_children_user_ip_address = [
FixedDataModelElement('User/UserIPAddress', b'User '), DelimitedDataModelElement('Username', b' '),
FixedDataModelElement('Action', b' changed IP address to '), IpAddressDataModelElement('IP')]
service_children_cron_job_announcement = [
DateTimeModelElement('DTM', date_format_string), FixedDataModelElement('Space', b' '),
DelimitedDataModelElement('UName', b' '), FixedDataModelElement('Cron', cron), DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Run', b']: Will run job `'),
FixedWordlistDataModelElement('CronType', [b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']),
FixedDataModelElement('StartTime', b'\' in 5 min.')]
service_children_cron_job_execution = [
DateTimeModelElement('DTM', date_format_string), FixedDataModelElement('Space1', b' '),
DelimitedDataModelElement('UName', b' '), FixedDataModelElement('Cron', cron), DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Job', b']: Job `'),
FixedWordlistDataModelElement('CronType', [b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']),
FixedDataModelElement('Started', b'\' started')]
service_children_audit = [SequenceModelElement('path', [
FixedDataModelElement('type', b'type=PATH '), FixedDataModelElement('msg_audit', b'msg=audit('),
DelimitedDataModelElement('msg', b':'), FixedDataModelElement('placeholder', b':'), DecimalIntegerValueModelElement('id'),
FixedDataModelElement('item_string', b'): item='), DecimalIntegerValueModelElement('item'),
FixedDataModelElement('name_string', b' name="'), DelimitedDataModelElement('name', b'"'),
FixedDataModelElement('inode_string', b'" inode='), DecimalIntegerValueModelElement('inode'),
FixedDataModelElement('dev_string', b' dev='), DelimitedDataModelElement('dev', b' '),
FixedDataModelElement('mode_string', b' mode='),
DecimalIntegerValueModelElement('mode', value_pad_type=DecimalIntegerValueModelElement.PAD_TYPE_ZERO),
FixedDataModelElement('ouid_string', b' ouid='), DecimalIntegerValueModelElement('ouid'),
FixedDataModelElement('ogid_string', b' ogid='), DecimalIntegerValueModelElement('ogid'),
FixedDataModelElement('rdev_string', b' rdev='), DelimitedDataModelElement('rdev', b' '),
FixedDataModelElement('nametype_string', b' nametype='), FixedWordlistDataModelElement('nametype', [b'NORMAL', b'ERROR'])]),
SequenceModelElement('syscall', [
FixedDataModelElement('type', b'type=SYSCALL '), FixedDataModelElement('msg_audit', b'msg=audit('),
DelimitedDataModelElement('msg', b':'), FixedDataModelElement('placeholder', b':'), DecimalIntegerValueModelElement('id'),
FixedDataModelElement('arch_string', b'): arch='), DelimitedDataModelElement('arch', b' '),
FixedDataModelElement('syscall_string', b' syscall='), DecimalIntegerValueModelElement('syscall'),
FixedDataModelElement('success_string', b' success='), FixedWordlistDataModelElement('success', [b'yes', b'no']),
FixedDataModelElement('exit_string', b' exit='), DecimalIntegerValueModelElement('exit'),
AnyByteDataModelElement('remainding_data')])]
service_children_parsing_model_element = [
DateTimeModelElement('DateTimeModelElement', b'Current DateTime: %d.%m.%Y %H:%M:%S'),
DecimalFloatValueModelElement('DecimalFloatValueModelElement', value_sign_type='optional'),
DecimalIntegerValueModelElement('DecimalIntegerValueModelElement', value_sign_type='optional', value_pad_type='blank'),
SequenceModelElement('se', [
DelimitedDataModelElement('DelimitedDataModelElement', b';'), FixedDataModelElement('FixedDataModelElement', b';')])]
# ElementValueBranchModelElement
fixed_data_me1 = FixedDataModelElement("fixed1", b'match ')
fixed_data_me2 = FixedDataModelElement("fixed2", b'fixed String')
fixed_wordlist_data_model_element = FixedWordlistDataModelElement("wordlist", [b'data: ', b'string: '])
decimal_integer_value_model_element = DecimalIntegerValueModelElement("decimal")
service_children_parsing_model_element.append(
ElementValueBranchModelElement('ElementValueBranchModelElement', FirstMatchModelElement("first", [
SequenceModelElement("seq1", [fixed_data_me1, fixed_wordlist_data_model_element]),
SequenceModelElement("seq2", [fixed_data_me1, fixed_wordlist_data_model_element, fixed_data_me2])]), "wordlist",
{0: decimal_integer_value_model_element, 1: fixed_data_me2}))
service_children_parsing_model_element.append(HexStringModelElement('HexStringModelElement'))
service_children_parsing_model_element.append(SequenceModelElement('se2', [
FixedDataModelElement('FixedDataModelElement', b'Gateway IP-Address: '), IpAddressDataModelElement('IpAddressDataModelElement')]))
import locale
loc = locale.getlocale()
if loc == (None, None):
loc = ('en_US', 'utf8')
service_children_parsing_model_element.append(
MultiLocaleDateTimeModelElement('MultiLocaleDateTimeModelElement', [(b'%b %d %Y', None, '%s.%s' % loc)]))
service_children_parsing_model_element.append(
RepeatedElementDataModelElement('RepeatedElementDataModelElement', SequenceModelElement('SequenceModelElement', [
FixedDataModelElement('FixedDataModelElement', b'[drawn number]: '),
DecimalIntegerValueModelElement('DecimalIntegerValueModelElement')]), 1))
service_children_parsing_model_element.append(VariableByteDataModelElement('VariableByteDataModelElement', b'-@#'))
service_children_parsing_model_element.append(SequenceModelElement('se', [
WhiteSpaceLimitedDataModelElement('WhiteSpaceLimitedDataModelElement'), FixedDataModelElement('fixed', b' ')]))
# The Base64StringModelElement must be just before the AnyByteDataModelElement to avoid unexpected Matches.
service_children_parsing_model_element.append(Base64StringModelElement('Base64StringModelElement'))
# The OptionalMatchModelElement must be paired with a FirstMatchModelElement because it accepts all data and thus no data gets
# to the AnyByteDataModelElement. The AnyByteDataModelElement must be last, because all bytes are accepted.
service_children_parsing_model_element.append(
OptionalMatchModelElement('/', FirstMatchModelElement('FirstMatchModelElement//optional', [
FixedDataModelElement('FixedDataModelElement', b'The-searched-element-was-found!'), SequenceModelElement('se', [
FixedDataModelElement('FixedDME', b'Any:'), AnyByteDataModelElement('AnyByteDataModelElement')])])))
alphabet = b'ghijkl'
service_children_ecd = []
for _, char in enumerate(alphabet):
char = bytes([char])
service_children_ecd.append(FixedDataModelElement(char.decode(), char))
parsing_model = FirstMatchModelElement('model', [
SequenceModelElement('CronAnnouncement', service_children_cron_job_announcement),
SequenceModelElement('CronExecution', service_children_cron_job_execution),
SequenceModelElement('DailyCron', service_children_cron_job), SequenceModelElement('DiskReport', service_children_disk_report),
SequenceModelElement('LoginDetails', service_children_login_details), DecimalIntegerValueModelElement('Random'),
SequenceModelElement('RandomTime', service_children_random_time), SequenceModelElement('Sensors', service_children_sensors),
SequenceModelElement('IPAddresses', service_children_user_ip_address), FirstMatchModelElement('type', service_children_audit),
FirstMatchModelElement('ECD', service_children_ecd), FirstMatchModelElement('ParsingME', service_children_parsing_model_element)])
# Some generic imports.
from aminer.analysis import AtomFilters
# Create all global handler lists here and append the real handlers later on.
# Use this filter to distribute all atoms to the analysis handlers.
atom_filter = AtomFilters.SubhandlerFilter(None)
from aminer.analysis.TimestampCorrectionFilters import SimpleMonotonicTimestampAdjust
simple_monotonic_timestamp_adjust = SimpleMonotonicTimestampAdjust([atom_filter])
analysis_context.register_component(simple_monotonic_timestamp_adjust, component_name="SimpleMonotonicTimestampAdjust")
from aminer.events.StreamPrinterEventHandler import StreamPrinterEventHandler
from aminer.events.JsonConverterHandler import JsonConverterHandler
stream_printer_event_handler = StreamPrinterEventHandler(analysis_context)
json_converter_handler = JsonConverterHandler([stream_printer_event_handler], analysis_context)
anomaly_event_handlers = [json_converter_handler]
# Now define the AtomizerFactory using the model. A simple line based one is usually sufficient.
from aminer.input.SimpleByteStreamLineAtomizerFactory import SimpleByteStreamLineAtomizerFactory
analysis_context.atomizer_factory = SimpleByteStreamLineAtomizerFactory(parsing_model, [simple_monotonic_timestamp_adjust],
anomaly_event_handlers)
# Just report all unparsed atoms to the event handlers.
from aminer.analysis.UnparsedAtomHandlers import SimpleUnparsedAtomHandler, VerboseUnparsedAtomHandler
simple_unparsed_atom_handler = SimpleUnparsedAtomHandler(anomaly_event_handlers)
atom_filter.add_handler(simple_unparsed_atom_handler, stop_when_handled_flag=False)
analysis_context.register_component(simple_unparsed_atom_handler, component_name="SimpleUnparsedHandler")
verbose_unparsed_atom_handler = VerboseUnparsedAtomHandler(anomaly_event_handlers, parsing_model)
atom_filter.add_handler(verbose_unparsed_atom_handler, stop_when_handled_flag=True)
analysis_context.register_component(verbose_unparsed_atom_handler, component_name="VerboseUnparsedHandler")
from aminer.analysis.TimestampsUnsortedDetector import TimestampsUnsortedDetector
timestamps_unsorted_detector = TimestampsUnsortedDetector(analysis_context.aminer_config, anomaly_event_handlers)
atom_filter.add_handler(timestamps_unsorted_detector)
analysis_context.register_component(timestamps_unsorted_detector, component_name="TimestampsUnsortedDetector")
from aminer.analysis import Rules
from aminer.analysis.AllowlistViolationDetector import AllowlistViolationDetector
allowlist_rules = [
Rules.OrMatchRule([
Rules.AndMatchRule([
Rules.PathExistsMatchRule('/model/LoginDetails/PastTime/Time/Minutes'),
Rules.NegationMatchRule(Rules.ValueMatchRule('/model/LoginDetails/Username', b'root')),
Rules.DebugMatchRule(debug_match_result=True)]),
Rules.AndMatchRule([
Rules.NegationMatchRule(Rules.PathExistsMatchRule('/model/LoginDetails/PastTime/Time/Minutes')),
Rules.PathExistsMatchRule('/model/LoginDetails'),
Rules.DebugMatchRule(debug_match_result=True)]),
Rules.NegationMatchRule(Rules.PathExistsMatchRule('/model/LoginDetails'))])]
# This rule list should trigger, when the line does not look like: User root (logged in, logged out)
# or User 'username' (logged in, logged out) x minutes ago.
allowlist_violation_detector = AllowlistViolationDetector(analysis_context.aminer_config, allowlist_rules, anomaly_event_handlers,
output_log_line=True)
analysis_context.register_component(allowlist_violation_detector, component_name="Allowlist")
atom_filter.add_handler(allowlist_violation_detector)
from aminer.analysis.ParserCount import ParserCount
parser_count = ParserCount(analysis_context.aminer_config, None, anomaly_event_handlers, 10)
analysis_context.register_component(parser_count, component_name="ParserCount")
atom_filter.add_handler(parser_count)
from aminer.analysis.EventTypeDetector import EventTypeDetector
etd = EventTypeDetector(analysis_context.aminer_config, anomaly_event_handlers)
analysis_context.register_component(etd, component_name="EventTypeDetector")
atom_filter.add_handler(etd)
from aminer.analysis.VariableTypeDetector import VariableTypeDetector
vtd = VariableTypeDetector(analysis_context.aminer_config, anomaly_event_handlers, etd, silence_output_except_indicator=False,
output_log_line=False, ignore_list=["/model/RandomTime"])
analysis_context.register_component(vtd, component_name="VariableTypeDetector")
atom_filter.add_handler(vtd)
from aminer.analysis.VariableCorrelationDetector import VariableCorrelationDetector
vtd = VariableCorrelationDetector(analysis_context.aminer_config, anomaly_event_handlers, etd, disc_div_thres=0.5,
ignore_list=["/model/RandomTime"])
analysis_context.register_component(vtd, component_name="VariableCorrelationDetector")
atom_filter.add_handler(vtd)
from aminer.analysis.EventCorrelationDetector import EventCorrelationDetector
ecd = EventCorrelationDetector(analysis_context.aminer_config, anomaly_event_handlers, check_rules_flag=True,
hypothesis_max_delta_time=1.0)
analysis_context.register_component(ecd, component_name="EventCorrelationDetector")
atom_filter.add_handler(ecd)
from aminer.analysis.EventFrequencyDetector import EventFrequencyDetector
efd = EventFrequencyDetector(analysis_context.aminer_config, anomaly_event_handlers, window_size=0.1)
analysis_context.register_component(efd, component_name="EventFrequencyDetector")
atom_filter.add_handler(efd)
from aminer.analysis.EventSequenceDetector import EventSequenceDetector
esd = EventSequenceDetector(analysis_context.aminer_config, anomaly_event_handlers, ['/model/ParsingME'], ignore_list=[
'/model/ECD/g', '/model/ECD/h', '/model/ECD/i', '/model/ECD/j', '/model/ECD/k', '/model/ECD/l', '/model/Random',
'/model/RandomTime', '/model/DailyCron'])
analysis_context.register_component(esd, component_name="EventSequenceDetector")
atom_filter.add_handler(esd)
from aminer.analysis.MatchFilter import MatchFilter
match_filter = MatchFilter(analysis_context.aminer_config, ['/model/Random'], anomaly_event_handlers, target_value_list=[
1, 10, 100], output_log_line=True)
analysis_context.register_component(match_filter, component_name="MatchFilter")
atom_filter.add_handler(match_filter)
from aminer.analysis.NewMatchPathDetector import NewMatchPathDetector
new_match_path_detector = NewMatchPathDetector(analysis_context.aminer_config, anomaly_event_handlers, auto_include_flag=True,
output_log_line=True)
analysis_context.register_component(new_match_path_detector, component_name="NewMatchPath")
atom_filter.add_handler(new_match_path_detector)
def tuple_transformation_function(match_value_list):
"""Only allow output of the EnhancedNewMatchPathValueComboDetector after every 10th element."""
extra_data = enhanced_new_match_path_value_combo_detector.known_values_dict.get(tuple(match_value_list))
if extra_data is not None:
mod = 10
if (extra_data[2] + 1) % mod == 0:
enhanced_new_match_path_value_combo_detector.auto_include_flag = False
else:
enhanced_new_match_path_value_combo_detector.auto_include_flag = True
return match_value_list
from aminer.analysis.EnhancedNewMatchPathValueComboDetector import EnhancedNewMatchPathValueComboDetector
enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector(analysis_context.aminer_config, [
'/model/DailyCron/UName', '/model/DailyCron/JobNumber'], anomaly_event_handlers, auto_include_flag=True,
tuple_transformation_function=tuple_transformation_function, output_log_line=True)
analysis_context.register_component(enhanced_new_match_path_value_combo_detector, component_name="EnhancedNewValueCombo")
atom_filter.add_handler(enhanced_new_match_path_value_combo_detector)
import re
ip_match_action = Rules.EventGenerationMatchAction(
"Analysis.Rules.IPv4InRFC1918MatchRule", "Private IP address occurred!", anomaly_event_handlers)
vdmt = Rules.ValueDependentModuloTimeMatchRule(None, 3, ["/model/ECD/j", "/model/ECD/k", "/model/ECD/l"], {b"e": [0, 2.95]}, [0, 3])
mt = Rules.ModuloTimeMatchRule(None, 3, 0, 3, None)
time_allowlist_rules = [
Rules.AndMatchRule([
Rules.ParallelMatchRule([
Rules.ValueDependentDelegatedMatchRule([
'/model/ECD/g', '/model/ECD/h', '/model/ECD/i', '/model/ECD/j', '/model/ECD/k', '/model/ECD/l'], {
(b"a",): mt, (b"b",): mt, (b"c",): mt, (b"d",): vdmt, (b"e",): vdmt, (b"f",): vdmt, None: mt}, mt),
Rules.IPv4InRFC1918MatchRule("/model/ParsingME/se2/IpAddressDataModelElement", ip_match_action),
Rules.DebugHistoryMatchRule(debug_match_result=True)
]),
# IP addresses 8.8.8.8, 8.8.4.4 and 10.0.0.0 - 10.255.255.255 are not allowed
Rules.NegationMatchRule(Rules.ValueListMatchRule("/model/ParsingME/se2/IpAddressDataModelElement", [134744072, 134743044])),
Rules.NegationMatchRule(Rules.ValueRangeMatchRule("/model/ParsingME/se2/IpAddressDataModelElement", 167772160, 184549375)),
Rules.NegationMatchRule(Rules.StringRegexMatchRule("/model/type/syscall/success", re.compile(b"^no$")))
])
]
time_allowlist_violation_detector = AllowlistViolationDetector(
analysis_context.aminer_config, time_allowlist_rules, anomaly_event_handlers, output_log_line=True)
analysis_context.register_component(time_allowlist_violation_detector, component_name="TimeAllowlist")
atom_filter.add_handler(time_allowlist_violation_detector)
from aminer.analysis.HistogramAnalysis import HistogramAnalysis, LinearNumericBinDefinition, ModuloTimeBinDefinition, \
PathDependentHistogramAnalysis
modulo_time_bin_definition = ModuloTimeBinDefinition(86400, 3600, 0, 1, 24, True)
linear_numeric_bin_definition = LinearNumericBinDefinition(50, 5, 20, True)
histogram_analysis = HistogramAnalysis(analysis_context.aminer_config, [
('/model/RandomTime/Random', modulo_time_bin_definition), ('/model/Random', linear_numeric_bin_definition)], 10,
anomaly_event_handlers, output_log_line=True)
analysis_context.register_component(histogram_analysis, component_name="HistogramAnalysis")
atom_filter.add_handler(histogram_analysis)
path_dependent_histogram_analysis = PathDependentHistogramAnalysis(
analysis_context.aminer_config, '/model/RandomTime', modulo_time_bin_definition, 10, anomaly_event_handlers, output_log_line=True)
analysis_context.register_component(path_dependent_histogram_analysis, component_name="PathDependentHistogramAnalysis")
atom_filter.add_handler(path_dependent_histogram_analysis)
from aminer.analysis.MatchValueAverageChangeDetector import MatchValueAverageChangeDetector
match_value_average_change_detector = MatchValueAverageChangeDetector(analysis_context.aminer_config, anomaly_event_handlers, None, [
'/model/Random'], 100, 10, output_log_line=True)
analysis_context.register_component(match_value_average_change_detector, component_name="MatchValueAverageChange")
atom_filter.add_handler(match_value_average_change_detector)
import sys
from aminer.analysis.MatchValueStreamWriter import MatchValueStreamWriter
match_value_stream_writer = MatchValueStreamWriter(
sys.stdout, ['/model/Sensors/CPUTemp', '/model/Sensors/CPUWorkload', '/model/Sensors/DTM'], b';', b'')
analysis_context.register_component(match_value_stream_writer, component_name="MatchValueStreamWriter")
atom_filter.add_handler(match_value_stream_writer)
from aminer.analysis.NewMatchPathValueComboDetector import NewMatchPathValueComboDetector
new_match_path_value_combo_detector = NewMatchPathValueComboDetector(
analysis_context.aminer_config, ['/model/IPAddresses/Username', '/model/IPAddresses/IP'],
anomaly_event_handlers, output_log_line=True)
analysis_context.register_component(new_match_path_value_combo_detector, component_name="NewMatchPathValueCombo")
atom_filter.add_handler(new_match_path_value_combo_detector)
from aminer.analysis.NewMatchIdValueComboDetector import NewMatchIdValueComboDetector
new_match_id_value_combo_detector = NewMatchIdValueComboDetector(analysis_context.aminer_config, [
'/model/type/path/name', '/model/type/syscall/syscall'], anomaly_event_handlers, id_path_list=[
'/model/type/path/id', '/model/type/syscall/id'], min_allowed_time_diff=5, auto_include_flag=True, allow_missing_values_flag=True,
output_log_line=True)
analysis_context.register_component(new_match_id_value_combo_detector, component_name="NewMatchIdValueComboDetector")
atom_filter.add_handler(new_match_id_value_combo_detector)
from aminer.analysis.NewMatchPathValueDetector import NewMatchPathValueDetector
new_match_path_value_detector = NewMatchPathValueDetector(analysis_context.aminer_config, [
'/model/DailyCron/JobNumber', '/model/IPAddresses/Username'], anomaly_event_handlers, auto_include_flag=True, output_log_line=True)
analysis_context.register_component(new_match_path_value_detector, component_name="NewMatchPathValue")
atom_filter.add_handler(new_match_path_value_detector)
from aminer.analysis.MissingMatchPathValueDetector import MissingMatchPathValueDetector
missing_match_path_value_detector = MissingMatchPathValueDetector(
analysis_context.aminer_config, ['/model/DiskReport/Space'], anomaly_event_handlers, auto_include_flag=True, default_interval=2,
realert_interval=5, output_log_line=True)
analysis_context.register_component(missing_match_path_value_detector, component_name="MissingMatch")
atom_filter.add_handler(missing_match_path_value_detector)
from aminer.analysis.TimeCorrelationDetector import TimeCorrelationDetector
time_correlation_detector = TimeCorrelationDetector(
analysis_context.aminer_config, anomaly_event_handlers, 2, min_rule_attributes=1, max_rule_attributes=5,
record_count_before_event=10000, output_log_line=True)
analysis_context.register_component(time_correlation_detector, component_name="TimeCorrelationDetector")
atom_filter.add_handler(time_correlation_detector)
from aminer.analysis.TimeCorrelationViolationDetector import TimeCorrelationViolationDetector, CorrelationRule, EventClassSelector
cron_job_announcement = CorrelationRule('CronJobAnnouncement', 5, 6, max_artefacts_a_for_single_b=1, artefact_match_parameters=[
('/model/CronAnnouncement/JobNumber', '/model/CronExecution/JobNumber')])
a_class_selector = EventClassSelector('Announcement', [cron_job_announcement], None)
b_class_selector = EventClassSelector('Execution', None, [cron_job_announcement])
rules = [Rules.PathExistsMatchRule('/model/CronAnnouncement/Run', a_class_selector),
Rules.PathExistsMatchRule('/model/CronExecution/Job', b_class_selector)]
time_correlation_violation_detector = TimeCorrelationViolationDetector(analysis_context.aminer_config, rules, anomaly_event_handlers,
output_log_line=True)
analysis_context.register_component(time_correlation_violation_detector, component_name="TimeCorrelationViolationDetector")
atom_filter.add_handler(time_correlation_violation_detector)
def build_analysis_pipeline(analysis_context):
"""
Define the function to create pipeline for parsing the log data.
It has also to define an AtomizerFactory to instruct aminer how to process incoming data streams to create log atoms from them.
"""
date_format_string = b'%Y-%m-%d %H:%M:%S'
cron = b' cron['
# Build the parsing model:
service_children_disk_report = [
FixedDataModelElement(
'Space',
b' Current Disk Data is: Filesystem Type Size Used Avail Use%'
),
DelimitedDataModelElement('Data', b'%'),
AnyByteDataModelElement('Rest')
]
service_children_login_details = [
FixedDataModelElement('User', b'User '),
DelimitedDataModelElement('Username', b' '),
FixedWordlistDataModelElement('Status',
[b' logged in', b' logged out']),
OptionalMatchModelElement(
'PastTime',
SequenceModelElement('Time', [
FixedDataModelElement('Blank', b' '),
DecimalIntegerValueModelElement('Minutes'),
FixedDataModelElement('Ago', b' minutes ago.')
]))
]
service_children_cron_job = [
DateTimeModelElement('DTM', date_format_string),
FixedDataModelElement('UNameSpace1', b' '),
DelimitedDataModelElement('UName', b' '),
FixedDataModelElement('UNameSpace2', b' '),
DelimitedDataModelElement('User', b' '),
FixedDataModelElement('Cron', cron),
DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Details', b']: Job `cron.daily` started.')
]
service_children_random_time = [
FixedDataModelElement('Space', b'Random: '),
DecimalIntegerValueModelElement('Random')
]
service_children_sensors = [
SequenceModelElement('CPUTemp', [
FixedDataModelElement('FixedTemp', b'CPU Temp: '),
DecimalIntegerValueModelElement('Temp'),
FixedDataModelElement('Degrees', b'\xc2\xb0C')
]),
FixedDataModelElement('Space1', b', '),
SequenceModelElement('CPUWorkload', [
FixedDataModelElement('Fixed Workload', b'CPU Workload: '),
DecimalIntegerValueModelElement('Workload'),
FixedDataModelElement('Percent', b'%')
]),
FixedDataModelElement('Space2', b', '),
DateTimeModelElement('DTM', date_format_string)
]
service_children_user_ip_address = [
FixedDataModelElement('User', b'User '),
DelimitedDataModelElement('Username', b' '),
FixedDataModelElement('Action', b' changed IP address to '),
IpAddressDataModelElement('IP')
]
service_children_cron_job_announcement = [
DateTimeModelElement('DTM', date_format_string),
FixedDataModelElement('Space', b' '),
DelimitedDataModelElement('UName', b' '),
FixedDataModelElement('Cron', cron),
DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Run', b']: Will run job `'),
FixedWordlistDataModelElement(
'CronType',
[b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']),
FixedDataModelElement('Start Time', b'\' in 5 min.')
]
service_children_cron_job_execution = [
DateTimeModelElement('DTM', date_format_string),
FixedDataModelElement('Space1', b' '),
DelimitedDataModelElement('UName', b' '),
FixedDataModelElement('Cron', cron),
DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Job', b']: Job `'),
FixedWordlistDataModelElement(
'CronType',
[b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']),
FixedDataModelElement('Started', b'\' started')
]
service_children_parsing_model_element = [
DateTimeModelElement('DateTimeModelElement',
b'Current DateTime: %d.%m.%Y %H:%M:%S'),
DecimalFloatValueModelElement('DecimalFloatValueModelElement',
value_sign_type='optional'),
DecimalIntegerValueModelElement('DecimalIntegerValueModelElement',
value_sign_type='optional',
value_pad_type='blank'),
SequenceModelElement('', [
DelimitedDataModelElement('DelimitedDataModelElement', b';'),
FixedDataModelElement('FixedDataModelElement', b';')
])
]
# ElementValueBranchModelElement
fixed_data_me1 = FixedDataModelElement("fixed1", b'match ')
fixed_data_me2 = FixedDataModelElement("fixed2", b'fixed String')
fixed_wordlist_data_model_element = FixedWordlistDataModelElement(
"wordlist", [b'data: ', b'string: '])
decimal_integer_value_model_element = DecimalIntegerValueModelElement(
"decimal")
service_children_parsing_model_element.append(
ElementValueBranchModelElement(
'ElementValueBranchModelElement',
FirstMatchModelElement("first", [
SequenceModelElement(
"seq1",
[fixed_data_me1, fixed_wordlist_data_model_element]),
SequenceModelElement("seq2", [
fixed_data_me1, fixed_wordlist_data_model_element,
fixed_data_me2
])
]), "wordlist", {
0: decimal_integer_value_model_element,
1: fixed_data_me2
}))
service_children_parsing_model_element.append(
HexStringModelElement('HexStringModelElement'))
service_children_parsing_model_element.append(
SequenceModelElement('', [
FixedDataModelElement('FixedDataModelElement',
b'Gateway IP-Address: '),
IpAddressDataModelElement('IpAddressDataModelElement')
]))
service_children_parsing_model_element.append(
MultiLocaleDateTimeModelElement('MultiLocaleDateTimeModelElement',
[(b'%b %d %Y', "de_AT.utf8", None)]))
service_children_parsing_model_element.append(
RepeatedElementDataModelElement(
'RepeatedElementDataModelElement',
SequenceModelElement('SequenceModelElement', [
FixedDataModelElement('FixedDataModelElement',
b'drawn number: '),
DecimalIntegerValueModelElement(
'DecimalIntegerValueModelElement')
]), 1))
service_children_parsing_model_element.append(
VariableByteDataModelElement('VariableByteDataModelElement', b'-@#'))
service_children_parsing_model_element.append(
SequenceModelElement('', [
WhiteSpaceLimitedDataModelElement(
'WhiteSpaceLimitedDataModelElement'),
FixedDataModelElement('', b' ')
]))
# The Base64StringModelElement must be just before the AnyByteDataModelElement to avoid unexpected Matches.
service_children_parsing_model_element.append(
Base64StringModelElement('Base64StringModelElement'))
# The OptionalMatchModelElement must be paired with a FirstMatchModelElement because it accepts all data and thus no data gets
# to the AnyByteDataModelElement. The AnyByteDataModelElement must be last, because all bytes are accepted.
service_children_parsing_model_element.append(
OptionalMatchModelElement(
'OptionalMatchModelElement',
FirstMatchModelElement('FirstMatchModelElement', [
FixedDataModelElement('FixedDataModelElement',
b'The-searched-element-was-found!'),
AnyByteDataModelElement('AnyByteDataModelElement')
])))
parsing_model = FirstMatchModelElement('model', [
SequenceModelElement('CronAnnouncement',
service_children_cron_job_announcement),
SequenceModelElement('CronExecution',
service_children_cron_job_execution),
SequenceModelElement('DailyCron', service_children_cron_job),
SequenceModelElement('DiskReport', service_children_disk_report),
SequenceModelElement('LoginDetails', service_children_login_details),
DecimalIntegerValueModelElement('Random'),
SequenceModelElement('RandomTime', service_children_random_time),
SequenceModelElement('Sensors', service_children_sensors),
SequenceModelElement('IPAddresses', service_children_user_ip_address),
FirstMatchModelElement('ParsingME',
service_children_parsing_model_element)
])
# Some generic imports.
from aminer.analysis import AtomFilters
# Create all global handler lists here and append the real handlers
# later on.
# Use this filter to distribute all atoms to the analysis handlers.
atom_filter = AtomFilters.SubhandlerFilter(None)
from aminer.analysis.TimestampCorrectionFilters import SimpleMonotonicTimestampAdjust
simple_monotonic_timestamp_adjust = SimpleMonotonicTimestampAdjust(
[atom_filter])
analysis_context.register_component(
simple_monotonic_timestamp_adjust,
component_name="SimpleMonotonicTimestampAdjust")
from aminer.events.StreamPrinterEventHandler import StreamPrinterEventHandler
stream_printer_event_handler = StreamPrinterEventHandler(analysis_context)
from aminer.events.SyslogWriterEventHandler import SyslogWriterEventHandler
syslog_event_handler = SyslogWriterEventHandler(analysis_context)
from aminer.events.DefaultMailNotificationEventHandler import DefaultMailNotificationEventHandler
if DefaultMailNotificationEventHandler.CONFIG_KEY_MAIL_TARGET_ADDRESS in analysis_context.aminer_config.config_properties:
mail_notification_handler = DefaultMailNotificationEventHandler(
analysis_context)
analysis_context.register_component(mail_notification_handler,
component_name="MailHandler")
anomaly_event_handlers = [
stream_printer_event_handler, syslog_event_handler,
mail_notification_handler
]
# Now define the AtomizerFactory using the model. A simple line based one is usually sufficient.
from aminer.input.SimpleByteStreamLineAtomizerFactory import SimpleByteStreamLineAtomizerFactory
analysis_context.atomizer_factory = SimpleByteStreamLineAtomizerFactory(
parsing_model, [simple_monotonic_timestamp_adjust],
anomaly_event_handlers)
# Just report all unparsed atoms to the event handlers.
from aminer.analysis.UnparsedAtomHandlers import SimpleUnparsedAtomHandler
simple_unparsed_atom_handler = SimpleUnparsedAtomHandler(
anomaly_event_handlers)
atom_filter.add_handler(simple_unparsed_atom_handler,
stop_when_handled_flag=True)
analysis_context.register_component(simple_unparsed_atom_handler,
component_name="UnparsedHandler")
from aminer.analysis.TimestampsUnsortedDetector import TimestampsUnsortedDetector
timestamps_unsorted_detector = TimestampsUnsortedDetector(
analysis_context.aminer_config, anomaly_event_handlers)
atom_filter.add_handler(timestamps_unsorted_detector)
analysis_context.register_component(
timestamps_unsorted_detector,
component_name="TimestampsUnsortedDetector")
from aminer.analysis import Rules
from aminer.analysis.AllowlistViolationDetector import AllowlistViolationDetector
allowlist_rules = [
Rules.OrMatchRule([
Rules.AndMatchRule([
Rules.PathExistsMatchRule(
'/model/LoginDetails/PastTime/Time/Minutes'),
Rules.NegationMatchRule(
Rules.ValueMatchRule('/model/LoginDetails/Username',
b'root'))
]),
Rules.AndMatchRule([
Rules.NegationMatchRule(
Rules.PathExistsMatchRule(
'/model/LoginDetails/PastTime/Time/Minutes')),
Rules.PathExistsMatchRule('/model/LoginDetails')
]),
Rules.NegationMatchRule(
Rules.PathExistsMatchRule('/model/LoginDetails'))
])
]
# This rule list should trigger, when the line does not look like: User root (logged in, logged out)
# or User 'username' (logged in, logged out) x minutes ago.
allowlist_violation_detector = AllowlistViolationDetector(
analysis_context.aminer_config, allowlist_rules,
anomaly_event_handlers)
analysis_context.register_component(allowlist_violation_detector,
component_name="Allowlist")
atom_filter.add_handler(allowlist_violation_detector)
from aminer.analysis.NewMatchPathDetector import NewMatchPathDetector
new_match_path_detector = NewMatchPathDetector(
analysis_context.aminer_config,
anomaly_event_handlers,
auto_include_flag=True)
analysis_context.register_component(new_match_path_detector,
component_name="NewMatchPath")
atom_filter.add_handler(new_match_path_detector)
def tuple_transformation_function(match_value_list):
"""Only allow output of the EnhancedNewMatchPathValueComboDetector after every 10000th element."""
extra_data = enhanced_new_match_path_value_combo_detector.known_values_dict.get(
tuple(match_value_list))
if extra_data is not None:
mod = 10000
if (extra_data[2] + 1) % mod == 0:
enhanced_new_match_path_value_combo_detector.auto_include_flag = False
else:
enhanced_new_match_path_value_combo_detector.auto_include_flag = True
return match_value_list
from aminer.analysis.EnhancedNewMatchPathValueComboDetector import EnhancedNewMatchPathValueComboDetector
enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector(
analysis_context.aminer_config,
['/model/DailyCron/UName', '/model/DailyCron/JobNumber'],
anomaly_event_handlers,
auto_include_flag=True,
tuple_transformation_function=tuple_transformation_function)
analysis_context.register_component(
enhanced_new_match_path_value_combo_detector,
component_name="EnhancedNewValueCombo")
atom_filter.add_handler(enhanced_new_match_path_value_combo_detector)
from aminer.analysis.HistogramAnalysis import HistogramAnalysis, LinearNumericBinDefinition, ModuloTimeBinDefinition, \
PathDependentHistogramAnalysis
modulo_time_bin_definition = ModuloTimeBinDefinition(
86400, 3600, 0, 1, 24, True)
linear_numeric_bin_definition = LinearNumericBinDefinition(50, 5, 20, True)
histogram_analysis = HistogramAnalysis(
analysis_context.aminer_config,
[('/model/RandomTime/Random', modulo_time_bin_definition),
('/model/Random', linear_numeric_bin_definition)], 10,
anomaly_event_handlers)
analysis_context.register_component(histogram_analysis,
component_name="HistogramAnalysis")
atom_filter.add_handler(histogram_analysis)
path_dependent_histogram_analysis = PathDependentHistogramAnalysis(
analysis_context.aminer_config, '/model/RandomTime',
modulo_time_bin_definition, 10, anomaly_event_handlers)
analysis_context.register_component(
path_dependent_histogram_analysis,
component_name="PathDependentHistogramAnalysis")
atom_filter.add_handler(path_dependent_histogram_analysis)
from aminer.analysis.MatchValueAverageChangeDetector import MatchValueAverageChangeDetector
match_value_average_change_detector = MatchValueAverageChangeDetector(
analysis_context.aminer_config, anomaly_event_handlers, None,
['/model/Random'], 100, 10)
analysis_context.register_component(
match_value_average_change_detector,
component_name="MatchValueAverageChange")
atom_filter.add_handler(match_value_average_change_detector)
import sys
from aminer.analysis.MatchValueStreamWriter import MatchValueStreamWriter
match_value_stream_writer = MatchValueStreamWriter(sys.stdout, [
'/model/Sensors/CPUTemp', '/model/Sensors/CPUWorkload',
'/model/Sensors/DTM'
], b';', b'')
analysis_context.register_component(
match_value_stream_writer, component_name="MatchValueStreamWriter")
atom_filter.add_handler(match_value_stream_writer)
from aminer.analysis.NewMatchPathValueComboDetector import NewMatchPathValueComboDetector
new_match_path_value_combo_detector = NewMatchPathValueComboDetector(
analysis_context.aminer_config,
['/model/IPAddresses/Username', '/model/IPAddresses/IP'],
anomaly_event_handlers,
auto_include_flag=True)
analysis_context.register_component(
new_match_path_value_combo_detector,
component_name="NewMatchPathValueCombo")
atom_filter.add_handler(new_match_path_value_combo_detector)
from aminer.analysis.NewMatchPathValueDetector import NewMatchPathValueDetector
new_match_path_value_detector = NewMatchPathValueDetector(
analysis_context.aminer_config,
['/model/DailyCron/JobNumber', '/model/IPAddresses/Username'],
anomaly_event_handlers,
auto_include_flag=True)
analysis_context.register_component(new_match_path_value_detector,
component_name="NewMatchPathValue")
atom_filter.add_handler(new_match_path_value_detector)
from aminer.analysis.MissingMatchPathValueDetector import MissingMatchPathValueDetector
missing_match_path_value_detector = MissingMatchPathValueDetector(
analysis_context.aminer_config, ['/model/DiskReport/Space'],
anomaly_event_handlers,
auto_include_flag=True,
default_interval=2,
realert_interval=5)
analysis_context.register_component(missing_match_path_value_detector,
component_name="MissingMatch")
atom_filter.add_handler(missing_match_path_value_detector)
from aminer.analysis.TimeCorrelationDetector import TimeCorrelationDetector
time_correlation_detector = TimeCorrelationDetector(
analysis_context.aminer_config,
anomaly_event_handlers,
2,
min_rule_attributes=1,
max_rule_attributes=5,
record_count_before_event=70000,
output_log_line=True)
analysis_context.register_component(
time_correlation_detector, component_name="TimeCorrelationDetector")
atom_filter.add_handler(time_correlation_detector)
from aminer.analysis.TimeCorrelationViolationDetector import TimeCorrelationViolationDetector, CorrelationRule, EventClassSelector
cron_job_announcement = CorrelationRule(
'CronJobAnnouncement',
5,
6,
max_artefacts_a_for_single_b=1,
artefact_match_parameters=[('/model/CronAnnouncement/JobNumber',
'/model/CronExecution/JobNumber')])
a_class_selector = EventClassSelector('Announcement',
[cron_job_announcement], None)
b_class_selector = EventClassSelector('Execution', None,
[cron_job_announcement])
rules = [
Rules.PathExistsMatchRule('/model/CronAnnouncement/Run',
a_class_selector),
Rules.PathExistsMatchRule('/model/CronExecution/Job', b_class_selector)
]
time_correlation_violation_detector = TimeCorrelationViolationDetector(
analysis_context.aminer_config, rules, anomaly_event_handlers)
analysis_context.register_component(
time_correlation_violation_detector,
component_name="TimeCorrelationViolationDetector")
atom_filter.add_handler(time_correlation_violation_detector)
def test4get_match_element_default_values_no_match(self):
"""Test not matching values with default values of value_sign_type, value_pad_type and exponent_type."""
decimal_float_value_me = DecimalFloatValueModelElement(
self.id_, DecimalFloatValueModelElement.SIGN_TYPE_NONE,
DecimalFloatValueModelElement.PAD_TYPE_NONE,
DecimalFloatValueModelElement.EXP_TYPE_NONE)
data = b"+22.25"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b"-22.25"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b"22,25"
value = b"22"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, 22, None)
data = b".25"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b"025"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b"0025"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b" 25"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b" 25"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b"1e-5"
value = b"1"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, 1, None)
data = b"e+10"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b"1e+0"
value = b"1"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, 1, None)
data = b"00"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
def test10get_match_element_float_exponents(self):
"""
Parse float values with exponents.
The principle of only testing dummy classes can not be applied here, as the functionality between the JsonModelElement and
DecimalFloatValueModelElement must be tested directly.
"""
json_model_element = JsonModelElement(
self.id_, {
"a":
DecimalFloatValueModelElement(
self.id_,
exponent_type=DecimalFloatValueModelElement.
EXP_TYPE_OPTIONAL),
"b":
DecimalFloatValueModelElement(
self.id_,
exponent_type=DecimalFloatValueModelElement.
EXP_TYPE_OPTIONAL)
})
def format_float(val):
"""
This function formats the float-value and parses the sign and the exponent
"""
exp = None
if "e" in val:
exp = "e"
elif "E" in val:
exp = "E"
if "+" in val:
sign = "+"
else:
sign = "-"
if exp is not None:
pos_point = val.find(exp)
if "." in val:
pos_point = val.find(".")
if len(val) - val.find(sign) <= 2:
result = format(float(val),
"1.%dE" % (val.find(exp) - pos_point))[:-2]
result += format(float(val),
"1.%dE" % (val.find(exp) - pos_point))[-1]
return result
return format(float(val),
"1.%dE" % (val.find(exp) - pos_point))
return float(val)
data = b'{"a": 111.1, "b": 111.1}'
value = json.loads(data, parse_float=format_float)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
data = b'{"a": 1E-01, "b": 111.1}'
value = json.loads(data, parse_float=format_float)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
data = b'{"a": 111.1, "b": 1E-1}'
value = json.loads(data, parse_float=format_float)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
data = b'{"a": 1E-1, "b": 1E-1}'
value = json.loads(data, parse_float=format_float)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
def test8get_match_element_mandatory_blank_values_no_match(self):
"""Test not matching values with default values of value_sign_type, value_pad_type and exponent_type."""
decimal_float_value_me = DecimalFloatValueModelElement(
self.id_, DecimalFloatValueModelElement.SIGN_TYPE_MANDATORY,
DecimalFloatValueModelElement.PAD_TYPE_BLANK,
DecimalFloatValueModelElement.EXP_TYPE_MANDATORY)
data = b"22.25"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b"+ 22.25"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b"- 22.25"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b"+22,25"
value = b"+22"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, 22, None)
data = b"22,25"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b"22.12.2021 some string."
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b".25"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b" +25"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b" -25"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b"025"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b"0025"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b"e+10"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b"00"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
def test7get_match_element_mandatory_blank_values(self):
"""Test valid float values with "mandatory" or "blank" values of value_sign_type, value_pad_type and exponent_type."""
decimal_float_value_me = DecimalFloatValueModelElement(
self.id_, DecimalFloatValueModelElement.SIGN_TYPE_MANDATORY,
DecimalFloatValueModelElement.PAD_TYPE_BLANK,
DecimalFloatValueModelElement.EXP_TYPE_MANDATORY)
data = b"+22.25 some string."
value = b"+22.25"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, 22.25, None)
data = b"-22.25 some string."
value = b"-22.25"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, -22.25, None)
data = b"+0.25 some string."
value = b"+0.25"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, 0.25, None)
data = b"+22 some string."
value = b"+22"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, 22, None)
data = b"+22. some string"
value = b"+22."
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, 22.0, None)
data = b"+ 25 some string"
value = b"+ 25"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, 25, None)
data = b"- 25 some string"
value = b"- 25"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, -25, None)
data = b"+1e-5 some string"
value = b"+1e-5"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, 1e-5, None)
data = b"+1e+0 some string"
value = b"+1e+0"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, 1, None)
data = b"+ 1e+0 some string"
value = b"+ 1e+0"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, 1, None)
data = b"+0 some string"
value = b"+0"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, 0, None)
def test2get_child_elements(self):
"""Test if get_child_elements returns None."""
decimal_float_me = DecimalFloatValueModelElement(self.id_)
self.assertEqual(decimal_float_me.get_child_elements(), None)
def test5get_match_element_optional_zero_values(self):
"""Test valid float values with "optional" or "zero" values of value_sign_type, value_pad_type and exponent_type."""
decimal_float_value_me = DecimalFloatValueModelElement(
self.id_, DecimalFloatValueModelElement.SIGN_TYPE_OPTIONAL,
DecimalFloatValueModelElement.PAD_TYPE_ZERO,
DecimalFloatValueModelElement.EXP_TYPE_OPTIONAL)
data = b"22.25 some string."
value = b"22.25"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, 22.25, None)
data = b"-22.25 some string."
value = b"-22.25"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, -22.25, None)
data = b"0.25 some string."
value = b"0.25"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, 0.25, None)
data = b"22 some string."
value = b"22"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, 22, None)
data = b"22.12.2021 some string."
value = b"22.12"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, 22.12, None)
data = b"22. some string"
value = b"22."
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, 22.0, None)
data = b"025 some string"
value = b"025"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, 25, None)
data = b"0025 some string"
value = b"0025"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, 25, None)
data = b"0025.22 some string"
value = b"0025.22"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, 25.22, None)
data = b"1e-5 some string"
value = b"1e-5"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, 1e-5, None)
data = b"1e+0 some string"
value = b"1e+0"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, 1, None)
data = b"0 some string"
value = b"0"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, 0, None)
data = b"00 some string"
value = b"00"
match_context = DummyMatchContext(data)
match_element = decimal_float_value_me.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, 0, None)
def test1get_id(self):
"""Test if get_id works properly."""
decimal_float_me = DecimalFloatValueModelElement(self.id_)
self.assertEqual(decimal_float_me.get_id(), self.id_)
def get_model():
"""Get the model."""
interface_name_model = VariableByteDataModelElement(
"interface", b"0123456789abcdefghijklmnopqrstuvwxyz.")
dtme = DateTimeModelElement("expire-date", b"%Y-%m-%dT%H:%M:%SZ")
type_children = [
SequenceModelElement("exit", [
FixedDataModelElement("s0", b"ntpd exiting on signal "),
DecimalIntegerValueModelElement("signal")
]),
SequenceModelElement("listen-drop", [
FixedDataModelElement("s0", b"Listen and drop on "),
DecimalIntegerValueModelElement("fd"),
FixedDataModelElement("s1", b" "), interface_name_model,
FixedDataModelElement("s2", b" "),
FirstMatchModelElement("address", [
IpAddressDataModelElement("ipv4"),
DelimitedDataModelElement("ipv6", b" "),
FixedDataModelElement("ipv6_missing", b"[::]")
]),
FirstMatchModelElement("udp", [
FixedDataModelElement("s3", b" UDP 123"),
FixedDataModelElement("s3", b":123")
])
]),
SequenceModelElement("listen-normal", [
FixedDataModelElement("s0", b"Listen normally on "),
DecimalIntegerValueModelElement("fd"),
FixedDataModelElement("s1", b" "), interface_name_model,
FixedDataModelElement("s2", b" "),
IpAddressDataModelElement("ip"),
FirstMatchModelElement("msg", [
FixedDataModelElement("port-new", b":123"),
FixedDataModelElement("port-old", b" UDP 123")
])
]),
SequenceModelElement("listen-routing", [
FixedDataModelElement("s0",
b"Listening on routing socket on fd #"),
DecimalIntegerValueModelElement("fd"),
FixedDataModelElement("s1", b" for interface updates")
]),
SequenceModelElement("soliciting-pool", [
FixedDataModelElement("s0", b"Soliciting pool server "),
IpAddressDataModelElement("pool-server-ip")
]),
SequenceModelElement("starting", [
FixedDataModelElement("s0", b"ntpd "),
DelimitedDataModelElement("version", b" "),
FixedDataModelElement("s1", b" (1): Starting")
]),
SequenceModelElement("no-root", [
FixedDataModelElement("s0", b"must be run as root, not uid "),
DecimalIntegerValueModelElement("uid")
]),
SequenceModelElement("leapsecond-file", [
FixedDataModelElement("s0", b"leapsecond file ('"),
DelimitedDataModelElement("file", b"'"),
FixedDataModelElement("s1", b"'): "),
FirstMatchModelElement("first", [
FixedDataModelElement("msg", b"good hash signature"),
SequenceModelElement("seq", [
FixedDataModelElement("s2", b"loaded, expire="), dtme,
FixedDataModelElement("s3", b" last="), dtme,
FixedDataModelElement("s4", b" ofs="),
DecimalIntegerValueModelElement("ofs")
])
])
]),
FixedDataModelElement(
"unable-to-bind",
b"unable to bind to wildcard address :: - another process may be running - EXITING"
),
FixedDataModelElement("new-interfaces",
b"new interface(s) found: waking up resolver"),
FixedDataModelElement(
"ntp-io",
b"ntp_io: estimated max descriptors: 1024, initial socket boundary: 16"
),
FixedDataModelElement("peers-refreshed", b"peers refreshed"),
FixedDataModelElement("log-file", b"logging to file /var/log/ntplog"),
FixedDataModelElement("command-line", b"Command line: ntpd"),
SequenceModelElement("precision", [
FixedDataModelElement("s0", b"proto: precision = "),
DecimalFloatValueModelElement("precision"),
FixedDataModelElement("s1", b" usec ("),
DecimalIntegerValueModelElement(
"usec",
value_sign_type=DecimalIntegerValueModelElement.
SIGN_TYPE_OPTIONAL),
FixedDataModelElement("s2", b")")
])
]
model = SequenceModelElement("ntpd", [
FixedDataModelElement("sname", b"ntpd["),
DecimalIntegerValueModelElement("pid"),
FixedDataModelElement("s0", b"]: "),
FirstMatchModelElement("msg", type_children)
])
return model