def _prepare_user_parser(self, line_id):
"""
:type pattern_match: PatternMatch
"""
pattern_match = self.pattern_assistant.get_pattern_match(line_id)
teacher_parser = self._parsers[line_id]
pattern_type = self.pattern_assistant.TYPE
return UserParserIntent(
pattern_type, teacher_parser.name, pattern_match.pattern, teacher_parser.log_type,
pattern_match.primary_key, pattern_match.param_groups,
teacher_parser.line.line_content, teacher_parser.line.offset,
teacher_parser.line.line_source, line_id
) # yapf: disable
def test_default_user_parser(self):
user_rule = self.teacher.get_rule()
effect_parser = user_rule.parsers[self.effect_id]
wanted_effect_parser = UserParserIntent(
'regex_assistant',
'error_occurred_in_reading',
r'([0-9]{4}-[0-9]{1,2}-[0-9]{1,2} [0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}) Error occurred in reading test$',
None,
[1],
{
1: ParamGroup(
content='2015-12-03 12:11:00',
converter=ConverterType.TO_DATE
)
},
self.effect_front_input.line_content,
self.effect_front_input.offset,
self.effect_front_input.line_source,
self.effect_id
) # yapf: disable
assert wanted_effect_parser == effect_parser
def create_sample_rule():
identical_constr = "identical"
different_constr = "different"
hetero_constr = "hetero"
to_date = "date"
to_string = "string"
to_int = "int"
sample_line1 = "2016-04-12 23:54:45 Connection error occurred on comp1. Host name: host1"
sample_line2 = "2016-04-12 23:54:40 Data migration from comp1 to comp2 failed. Host name: host2"
sample_line3 = "2016-04-12 23:54:43 Data is missing at comp2. Loss = 150 GB. Host name: host2"
regex1 = "^(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d) Connection error occurred on (.*)\. Host name: (.*)$"
regex2 = "^(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d) Data migration from (.*) to (.*) failed\. Host name: (.*)$"
regex3 = "^(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d) Data is missing at (.*)\. Loss = (.*) GB\. Host name: (.*)$"
# yapf: disable
groups1 = {
1: LineParamGroup("2016-04-12 23:54:45", to_date),
2: LineParamGroup("comp1", to_string),
3: LineParamGroup("host1", to_string)
}
groups2 = {
1: LineParamGroup("2016-04-12 23:54:40", to_date),
2: LineParamGroup("comp2", to_string),
3: LineParamGroup("host2", to_string)
}
groups3 = {
1: LineParamGroup("2016-04-12 23:54:43", to_date),
2: LineParamGroup("comp2", to_string),
3: LineParamGroup("150", to_int),
4: LineParamGroup("host2", to_string)
}
# resource location is temporary a string, because there is no ResourceLocation class yet.
# TODO: Change following resource locations to ResourceLocation objects
resource_location1 = "serwer1"
resource_location2 = "serwer2"
resource_location3 = "serwer3"
regex_type = AssistantType.REGEX
parser_intent1 = UserParserIntent(
regex_type, "connectionerror", regex1, "hydra", [1], groups1, sample_line1, 18,
resource_location1
)
parser_intent2 = UserParserIntent(
regex_type, "datamigration", regex2, "hydra", [1], groups2, sample_line2, 9,
resource_location2
)
parser_intent3 = UserParserIntent(
regex_type, "lostdata", regex3, "filesystem", [1], groups3, sample_line3, 1994,
resource_location3
)
# yapf: enable
parsers = {0: parser_intent1, 1: parser_intent2, 2: parser_intent3}
effect_id = 2
constraint1 = UserConstraintIntent(identical_constr, [[0, 2], [1, 2]])
constraint2 = UserConstraintIntent(identical_constr, [[1, 3], [2, 2]])
constraint3 = UserConstraintIntent(different_constr, [[1, 2], [1, 3]])
constraint4 = UserConstraintIntent(hetero_constr, [[0, 3], [1, 4], [2, 4]],
{"different": 1})
constraints = [constraint1, constraint2, constraint3, constraint4]
return UserRuleIntent(effect_id, parsers, constraints)
def setUpClass(cls):
SettingsFactorySelector.WHYLOG_DIR = TestPaths.WHYLOG_DIR
cls.sample_line1 = "(2016-04-12 23:54:45) Connection error occurred on comp1. Host name: host1"
cls.sample_line2 = "(2016-04-12 23:54:40) Data migration from comp1 to comp2 failed. Host name: host2"
cls.sample_line3 = "(2016-04-12 23:54:43) Data is missing at comp2. Loss = (.*) GB. Host name: host2"
cls.regex1 = "^(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d) Connection error occurred on (.*)\. Host name: (.*)$"
cls.regex2 = "^(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d) Data migration from (.*) to (.*) failed\. Host name: (.*)$"
cls.regex3 = "^(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d) Data is missing at (.*)\. Loss = (.*) GB\. Host name: (.*)$"
regex_type = AssistantType.REGEX
# yapf: disable
cls.groups1 = {
1: ParamGroup("2016-04-12 23:54:45", to_date),
2: ParamGroup("comp1", to_string),
3: ParamGroup("host1", to_string)
}
cls.groups2 = {
1: ParamGroup("2016-04-12 23:54:40", to_date),
2: ParamGroup("comp2", to_string),
3: ParamGroup("host2", to_string)
}
cls.groups3 = {
1: ParamGroup("2016-04-12 23:54:43", to_date),
2: ParamGroup("comp2", to_string),
3: ParamGroup("150", to_int),
4: ParamGroup("host2", to_string)
}
cls.parser_intent1 = UserParserIntent(
regex_type,
"connectionerror",
cls.regex1,
"default",
[1],
cls.groups1,
cls.sample_line1,
line_offset=None,
line_resource_location=None
)
cls.parser_intent2 = UserParserIntent(
regex_type,
"datamigration",
cls.regex2,
"default",
[1],
cls.groups2,
cls.sample_line2,
line_offset=None,
line_resource_location=None
)
cls.parser_intent3 = UserParserIntent(
regex_type,
"lostdata",
cls.regex3,
"default",
[1],
cls.groups3,
cls.sample_line3,
line_offset=None,
line_resource_location=None
)
parsers = {0: cls.parser_intent1, 1: cls.parser_intent2, 2: cls.parser_intent3}
effect_id = 2
constraint1 = UserConstraintIntent(identical_constr, [[0, 2], [1, 2]])
constraint2 = UserConstraintIntent(identical_constr, [[1, 3], [2, 2]])
constraint3 = UserConstraintIntent(different_constr, [[1, 2], [1, 3]])
constraint4 = UserConstraintIntent(
hetero_constr, [[0, 3], [1, 4], [2, 4]], {
"different": 1
}
)
# yapf: enable
constraints = [constraint1, constraint2, constraint3, constraint4]
cls.user_intent = UserRuleIntent(effect_id, parsers, constraints)
path_config = ['whylog', 'tests', 'tests_config', 'test_files', '.whylog', 'config.yaml']
path = os.path.join(*path_config)
cls.config = SettingsFactorySelector.load_settings(path)['config']
def setUpClass(cls):
cls.connection_error_line = "2015-12-03 12:08:09 Connection error occurred on alfa36. Host name: 2"
cls.data_migration_line = "2015-12-03 12:10:10 Data migration from alfa36 to alfa21 failed. Host name: 2"
cls.lost_data_line = "2015-12-03 12:11:00 Data is missing at alfa21. Loss = 567.02 GB. Host name: 101"
cls.root_cause_line = "root cause"
cls.data_missing_line = "2015-12-03 12:11:00 Data is missing"
cls.data_missing_at_line = "2015-12-03 12:11:00 Data is missing at alfa21"
cls.dummy_line = "dummy regex"
regex1 = "^(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d) Connection error occurred on (.*)\. Host name: (.*)$"
regex2 = "^(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d) Data migration from (.*) to (.*) failed\. Host name: (.*)$"
regex3 = "^(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d) Data is missing at (.*)\. Loss = (.*) GB\. Host name: (.*)$"
regex4 = "^root cause$"
regex5 = "^(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d) Data is missing"
regex6 = "^(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d) Data is missing at (.*)$"
regex7 = "^dummy regex"
date_group = LineParamGroup(None, to_date)
string_group = LineParamGroup(None, to_string)
float_group = LineParamGroup(None, to_float)
regex_type = AssistantType.REGEX
# yapf: disable
parser_intent1 = UserParserIntent(
regex_type, "connectionerror", regex1, "hydra", [1], {
1: date_group,
2: string_group,
3: string_group
}, cls.connection_error_line, 1, "serwer1"
)
parser_intent2 = UserParserIntent(
regex_type, "datamigration", regex2, "hydra", [1], {
1: date_group,
2: string_group,
3: string_group,
4: string_group
}, cls.data_migration_line, 2, "serwer2"
)
parser_intent3 = UserParserIntent(
regex_type, "lostdata", regex3, "filesystem", [1], {
1: date_group,
2: string_group,
3: float_group,
4: string_group
}, cls.lost_data_line, 3, "serwer3"
)
parser_intent4 = UserParserIntent(
regex_type, "rootcause", regex4, "filesystem", [], {}, cls.root_cause_line, 4, "serwer4"
)
parser_intent5 = UserParserIntent(
regex_type, "date", regex5, "filesystem", [1], {1: date_group}, cls.data_missing_line,
5, "serwer5"
)
parser_intent6 = UserParserIntent(
regex_type, "onlymissdata", regex6, "filesystem", [1], {
1: date_group,
2: string_group
}, cls.data_missing_at_line, 6, "serwer6"
)
parser_intent7 = UserParserIntent(
regex_type, "dummy", regex7, "filesystem", [], {}, cls.dummy_line, 7, "serwer7"
)
# yapf: enable
cls.connection_error = RegexParserFactory.create_from_intent(
parser_intent1)
cls.data_migration = RegexParserFactory.create_from_intent(
parser_intent2)
cls.lost_data = RegexParserFactory.create_from_intent(parser_intent3)
cls.root_cause = RegexParserFactory.create_from_intent(parser_intent4)
cls.lost_data_date = RegexParserFactory.create_from_intent(
parser_intent5)
cls.lost_data_suffix = RegexParserFactory.create_from_intent(
parser_intent6)
cls.dummy_parser = RegexParserFactory.create_from_intent(
parser_intent7)
cls.no_lost_data_parser_list = cls.get_no_lost_data_parser_list()