def setUpClass(cls):
SettingsFactorySelector.WHYLOG_DIR = TestPaths.WHYLOG_DIR
cls.config = SettingsFactorySelector.get_settings()['config']
cls.whylog_dir = SettingsFactorySelector._attach_whylog_dir(os.getcwd())
cause1_regex = '^(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d) cause1 transaction number: (\d+) Host: (\w)$'
cause1_line = '2016-04-12 23:39:43 cause1 transaction number: 10101 Host: db_host'
convertions = {1: 'date'}
cls.cause1 = RegexParser("cause1", cause1_line, cause1_regex, [1], 'database', convertions)
cause2_regex = '^(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d) cause2 moved resource id: (\d+) Host: (\w)$'
cause2_line = '2016-04-12 23:40:43 cause2 moved resource id: 1234 Host: apache_host'
convertions = {1: 'date'}
cls.cause2 = RegexParser("cause2", cause2_line, cause2_regex, [1], 'apache', convertions)
effect_regex = '^(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d) effect internal server error Host: (\w)$'
effect_line = '2016-04-12 23:54:43 effect internal server error Host: apache_host'
convertions = {1: 'date'}
cls.effect = RegexParser("effect", effect_line, effect_regex, [1], 'apache', convertions)
line_source = LineSource('localhost', 'node_1.log')
cls.effect_time = datetime(2016, 4, 12, 23, 54, 43)
effect_line = '2016-04-12 23:54:43 effect internal server error Host: apache_host'
cls.effect_clues = {
'effect': Clue((cls.effect_time, 'apache_host'), effect_line, 40, line_source)
}
cls.earliest_date = datetime(1, 1, 1, 1, 1, 1)
cls.ten_second_earlier = datetime(2016, 4, 12, 23, 54, 33)
cls.one_hundred_second_earlier = datetime(2016, 4, 12, 23, 53, 3)
cls.ten_second_later = datetime(2016, 4, 12, 23, 54, 53)
def find_config_in_parent_dir(cls, path):
YamlSettingsFactory.create_new_settings_dir(
path, SettingsFactorySelector.WHYLOG_DIR,
SettingsFactorySelector.SETTINGS_FILE)
config = SettingsFactorySelector.get_settings()['config']
expected_path = SettingsFactorySelector._attach_whylog_dir(path)
cls.validate_created_config(config, expected_path)
def test_log_type_rename(self):
whylog_dir = SettingsFactorySelector._attach_whylog_dir(os.getcwd())
config = SettingsFactorySelector.get_settings()['config']
super_parser = RegexSuperParser('^(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d).*', [1], {1: 'date'})
matcher = WildCardFilenameMatcher('localhost', '/temp/*.log', 'default', super_parser)
log_type = LogType('default', [matcher])
config.add_log_type(log_type)
config.add_rule(self.user_intent)
parsers_name = ['connectionerror', 'datamigration', 'lostdata']
assert 'default' in config._log_types.keys()
assert 'default' in config._parsers_grouped_by_log_type.keys()
assert sorted(config._parsers.keys()) == parsers_name
assert sorted(parser.name for parser in config._parsers_grouped_by_log_type['default']) == \
parsers_name
assert sorted(log_type.name for log_type in config.get_all_log_types()) == ['default']
config.rename_log_type('default', 'test_log_type')
self._check_log_type_renaming(config, parsers_name)
config = SettingsFactorySelector.get_settings()['config']
self._check_log_type_renaming(config, parsers_name)
shutil.rmtree(whylog_dir)
def setUpClass(cls):
SettingsFactorySelector.WHYLOG_DIR = TestPaths.WHYLOG_DIR
cls.config = SettingsFactorySelector.get_settings()['config']
cls.whylog_dir = SettingsFactorySelector._attach_whylog_dir(os.getcwd())
cause1_regex = '^(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d) cause1 transaction number: (\d+) Host: (\w)$'
cause1_line = '2016-04-12 23:39:43 cause1 transaction number: 10101 Host: db_host'
convertions = {1: 'date'}
cls.cause1 = RegexParser("cause1", cause1_line, cause1_regex, [1], 'database', convertions)
cause2_regex = '^(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d) cause2 moved resource id: (\d+) Host: (\w)$'
cause2_line = '2016-04-12 23:40:43 cause2 moved resource id: 1234 Host: apache_host'
convertions = {1: 'date'}
cls.cause2 = RegexParser("cause2", cause2_line, cause2_regex, [1], 'apache', convertions)
effect_regex = '^(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d) effect internal server error Host: (\w)$'
effect_line = '2016-04-12 23:54:43 effect internal server error Host: apache_host'
convertions = {1: 'date'}
cls.effect = RegexParser("effect", effect_line, effect_regex, [1], 'apache', convertions)
line_source = LineSource('localhost', 'node_1.log')
cls.effect_time = datetime(2016, 4, 12, 23, 54, 43)
effect_line = '2016-04-12 23:54:43 effect internal server error Host: apache_host'
cls.effect_clues = {
'effect': Clue((cls.effect_time, 'apache_host'), effect_line, 40, line_source)
}
cls.earliest_date = datetime.min
cls.ten_second_earlier = datetime(2016, 4, 12, 23, 54, 33)
cls.one_hundred_second_earlier = datetime(2016, 4, 12, 23, 53, 3)
cls.ten_second_later = datetime(2016, 4, 12, 23, 54, 53)
def find_config_in_parent_dir(cls, path):
YamlSettingsFactory.create_new_settings_dir(
path, SettingsFactorySelector.WHYLOG_DIR, SettingsFactorySelector.SETTINGS_FILE
)
config = SettingsFactorySelector.get_settings()['config']
expected_path = SettingsFactorySelector._attach_whylog_dir(path)
cls.validate_created_config(config, expected_path)
def test_find_config_in_parent_dir(self):
path = os.getcwd()
self.find_config_in_parent_dir(path)
shutil.rmtree(SettingsFactorySelector._attach_whylog_dir(path))
for _ in six.moves.range(2):
path, _ = os.path.split(path)
self.find_config_in_parent_dir(path)
shutil.rmtree(SettingsFactorySelector._attach_whylog_dir(path))
def test_find_config_in_parent_dir(self):
path = os.getcwd()
self.find_config_in_parent_dir(path)
shutil.rmtree(SettingsFactorySelector._attach_whylog_dir(path))
for i in six.moves.range(2):
path, _ = os.path.split(path)
self.find_config_in_parent_dir(path)
shutil.rmtree(SettingsFactorySelector._attach_whylog_dir(path))
def test_add_new_rule_to_empty_config(self):
config = SettingsFactorySelector.get_settings()['config']
whylog_dir = SettingsFactorySelector._attach_whylog_dir(os.getcwd())
config.add_rule(self.user_intent)
self.check_loaded_config(config, whylog_dir)
config = SettingsFactorySelector.get_settings()['config']
self.check_loaded_config(config, whylog_dir)
shutil.rmtree(whylog_dir)
def whylog_factory():
"""
This function returns LogReader object and teachers_generator.
"""
settings = SettingsFactorySelector.get_settings()
pattern_assistant = settings['assistant']()
config = settings['config']
return LogReader(config), partial(Teacher, config, pattern_assistant), config
def whylog_factory():
"""
This function returns LogReader object and teachers_generator.
"""
settings = SettingsFactorySelector.get_settings()
pattern_assistant = settings['assistant']()
config = settings['config']
return LogReader(config), partial(Teacher, config,
pattern_assistant), config
def test_add_log_type(self):
SettingsFactorySelector.WHYLOG_DIR = TestPaths.WHYLOG_DIR
config = SettingsFactorySelector.get_settings()['config']
whylog_dir = SettingsFactorySelector._attach_whylog_dir(os.getcwd())
super_parser = RegexSuperParser('^(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d).*', [1], {1: 'date'})
matcher = WildCardFilenameMatcher('localhost', 'node_1.log', 'test_log_type', super_parser)
default_log_type = LogType('test_log_type', [matcher])
config.add_log_type(default_log_type)
config = SettingsFactorySelector.get_settings()['config']
assert len(config._log_types) == 2
log_type = config._log_types['test_log_type']
assert log_type.name == 'test_log_type'
assert len(log_type.filename_matchers) == 1
matcher = log_type.filename_matchers[0]
assert matcher.host_pattern == 'localhost'
assert matcher.path_pattern == 'node_1.log'
assert matcher.log_type_name == 'test_log_type'
assert matcher.super_parser == super_parser
shutil.rmtree(whylog_dir)
def test_add_log_type(self):
SettingsFactorySelector.WHYLOG_DIR = TestPaths.WHYLOG_DIR
config = SettingsFactorySelector.get_settings()['config']
whylog_dir = SettingsFactorySelector._attach_whylog_dir(os.getcwd())
super_parser = RegexSuperParser(
'^(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d).*', [1], {1: 'date'})
matcher = WildCardFilenameMatcher('localhost', 'node_1.log',
'test_log_type', super_parser)
default_log_type = LogType('test_log_type', [matcher])
config.add_log_type(default_log_type)
config = SettingsFactorySelector.get_settings()['config']
assert len(config._log_types) == 1
log_type = config._log_types['test_log_type']
assert log_type.name == 'test_log_type'
assert len(log_type.filename_matchers) == 1
matcher = log_type.filename_matchers[0]
assert matcher.host_pattern == 'localhost'
assert matcher.path_pattern == 'node_1.log'
assert matcher.log_type_name == 'test_log_type'
assert matcher.super_parser == super_parser
shutil.rmtree(whylog_dir)
def test_find_config_in_home_directory(self):
self.find_config_in_parent_dir(SettingsFactorySelector.HOME_DIR)
shutil.rmtree(
SettingsFactorySelector._attach_whylog_dir(
SettingsFactorySelector.HOME_DIR))
def test_find_config_in_home_directory(self):
self.find_config_in_parent_dir(SettingsFactorySelector.HOME_DIR)
shutil.rmtree(SettingsFactorySelector._attach_whylog_dir(SettingsFactorySelector.HOME_DIR))
def tearDownClass(cls):
# remove .test_directory if test test_add_new_rule_to_empty_config failed
test_whylog_dir = SettingsFactorySelector._attach_whylog_dir(os.getcwd())
if os.path.isdir(test_whylog_dir):
shutil.rmtree(test_whylog_dir)
def tearDownClass(cls):
whylog_dir = SettingsFactorySelector._attach_whylog_dir(os.getcwd())
if os.path.isdir(whylog_dir):
shutil.rmtree(whylog_dir)
def tearDownClass(cls):
whylog_dir = SettingsFactorySelector._attach_whylog_dir(os.getcwd())
if os.path.isdir(whylog_dir):
shutil.rmtree(whylog_dir)
def setUpClass(cls):
SettingsFactorySelector.WHYLOG_DIR = TestPaths.WHYLOG_DIR
cls.sample_line1 = "(2016-04-12 23:54:45) Connection error occurred on comp1. Host name: host1"
cls.sample_line2 = "(2016-04-12 23:54:40) Data migration from comp1 to comp2 failed. Host name: host2"
cls.sample_line3 = "(2016-04-12 23:54:43) Data is missing at comp2. Loss = (.*) GB. Host name: host2"
cls.regex1 = "^(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d) Connection error occurred on (.*)\. Host name: (.*)$"
cls.regex2 = "^(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d) Data migration from (.*) to (.*) failed\. Host name: (.*)$"
cls.regex3 = "^(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d) Data is missing at (.*)\. Loss = (.*) GB\. Host name: (.*)$"
regex_type = AssistantType.REGEX
# yapf: disable
cls.groups1 = {
1: ParamGroup("2016-04-12 23:54:45", to_date),
2: ParamGroup("comp1", to_string),
3: ParamGroup("host1", to_string)
}
cls.groups2 = {
1: ParamGroup("2016-04-12 23:54:40", to_date),
2: ParamGroup("comp2", to_string),
3: ParamGroup("host2", to_string)
}
cls.groups3 = {
1: ParamGroup("2016-04-12 23:54:43", to_date),
2: ParamGroup("comp2", to_string),
3: ParamGroup("150", to_int),
4: ParamGroup("host2", to_string)
}
cls.parser_intent1 = UserParserIntent(
regex_type,
"connectionerror",
cls.regex1,
"default",
[1],
cls.groups1,
cls.sample_line1,
line_offset=None,
line_resource_location=None
)
cls.parser_intent2 = UserParserIntent(
regex_type,
"datamigration",
cls.regex2,
"default",
[1],
cls.groups2,
cls.sample_line2,
line_offset=None,
line_resource_location=None
)
cls.parser_intent3 = UserParserIntent(
regex_type,
"lostdata",
cls.regex3,
"default",
[1],
cls.groups3,
cls.sample_line3,
line_offset=None,
line_resource_location=None
)
parsers = {0: cls.parser_intent1, 1: cls.parser_intent2, 2: cls.parser_intent3}
effect_id = 2
constraint1 = UserConstraintIntent(identical_constr, [[0, 2], [1, 2]])
constraint2 = UserConstraintIntent(identical_constr, [[1, 3], [2, 2]])
constraint3 = UserConstraintIntent(different_constr, [[1, 2], [1, 3]])
constraint4 = UserConstraintIntent(
hetero_constr, [[0, 3], [1, 4], [2, 4]], {
"different": 1
}
)
# yapf: enable
constraints = [constraint1, constraint2, constraint3, constraint4]
cls.user_intent = UserRuleIntent(effect_id, parsers, constraints)
path_config = ['whylog', 'tests', 'tests_config', 'test_files', '.whylog', 'config.yaml']
path = os.path.join(*path_config)
cls.config = SettingsFactorySelector.load_settings(path)['config']