示例#1
0
    def setUpClass(cls):
        SettingsFactorySelector.WHYLOG_DIR = TestPaths.WHYLOG_DIR
        cls.config = SettingsFactorySelector.get_settings()['config']
        cls.whylog_dir = SettingsFactorySelector._attach_whylog_dir(os.getcwd())

        cause1_regex = '^(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d) cause1 transaction number: (\d+) Host: (\w)$'
        cause1_line = '2016-04-12 23:39:43 cause1 transaction number: 10101 Host: db_host'
        convertions = {1: 'date'}
        cls.cause1 = RegexParser("cause1", cause1_line, cause1_regex, [1], 'database', convertions)

        cause2_regex = '^(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d) cause2 moved resource id: (\d+) Host: (\w)$'
        cause2_line = '2016-04-12 23:40:43 cause2 moved resource id: 1234 Host: apache_host'
        convertions = {1: 'date'}
        cls.cause2 = RegexParser("cause2", cause2_line, cause2_regex, [1], 'apache', convertions)

        effect_regex = '^(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d) effect internal server error Host: (\w)$'
        effect_line = '2016-04-12 23:54:43 effect internal server error Host: apache_host'
        convertions = {1: 'date'}
        cls.effect = RegexParser("effect", effect_line, effect_regex, [1], 'apache', convertions)

        line_source = LineSource('localhost', 'node_1.log')
        cls.effect_time = datetime(2016, 4, 12, 23, 54, 43)
        effect_line = '2016-04-12 23:54:43 effect internal server error Host: apache_host'
        cls.effect_clues = {
            'effect': Clue((cls.effect_time, 'apache_host'), effect_line, 40, line_source)
        }

        cls.earliest_date = datetime(1, 1, 1, 1, 1, 1)
        cls.ten_second_earlier = datetime(2016, 4, 12, 23, 54, 33)
        cls.one_hundred_second_earlier = datetime(2016, 4, 12, 23, 53, 3)
        cls.ten_second_later = datetime(2016, 4, 12, 23, 54, 53)
示例#2
0
 def find_config_in_parent_dir(cls, path):
     YamlSettingsFactory.create_new_settings_dir(
         path, SettingsFactorySelector.WHYLOG_DIR,
         SettingsFactorySelector.SETTINGS_FILE)
     config = SettingsFactorySelector.get_settings()['config']
     expected_path = SettingsFactorySelector._attach_whylog_dir(path)
     cls.validate_created_config(config, expected_path)
示例#3
0
文件: __init__.py 项目: wlixin/whylog
    def test_log_type_rename(self):
        whylog_dir = SettingsFactorySelector._attach_whylog_dir(os.getcwd())
        config = SettingsFactorySelector.get_settings()['config']

        super_parser = RegexSuperParser('^(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d).*', [1], {1: 'date'})
        matcher = WildCardFilenameMatcher('localhost', '/temp/*.log', 'default', super_parser)
        log_type = LogType('default', [matcher])
        config.add_log_type(log_type)
        config.add_rule(self.user_intent)

        parsers_name = ['connectionerror', 'datamigration', 'lostdata']
        assert 'default' in config._log_types.keys()
        assert 'default' in config._parsers_grouped_by_log_type.keys()
        assert sorted(config._parsers.keys()) == parsers_name
        assert sorted(parser.name for parser in config._parsers_grouped_by_log_type['default']) == \
               parsers_name
        assert sorted(log_type.name for log_type in config.get_all_log_types()) == ['default']

        config.rename_log_type('default', 'test_log_type')
        self._check_log_type_renaming(config, parsers_name)

        config = SettingsFactorySelector.get_settings()['config']
        self._check_log_type_renaming(config, parsers_name)

        shutil.rmtree(whylog_dir)
    def setUpClass(cls):
        SettingsFactorySelector.WHYLOG_DIR = TestPaths.WHYLOG_DIR
        cls.config = SettingsFactorySelector.get_settings()['config']
        cls.whylog_dir = SettingsFactorySelector._attach_whylog_dir(os.getcwd())

        cause1_regex = '^(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d) cause1 transaction number: (\d+) Host: (\w)$'
        cause1_line = '2016-04-12 23:39:43 cause1 transaction number: 10101 Host: db_host'
        convertions = {1: 'date'}
        cls.cause1 = RegexParser("cause1", cause1_line, cause1_regex, [1], 'database', convertions)

        cause2_regex = '^(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d) cause2 moved resource id: (\d+) Host: (\w)$'
        cause2_line = '2016-04-12 23:40:43 cause2 moved resource id: 1234 Host: apache_host'
        convertions = {1: 'date'}
        cls.cause2 = RegexParser("cause2", cause2_line, cause2_regex, [1], 'apache', convertions)

        effect_regex = '^(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d) effect internal server error Host: (\w)$'
        effect_line = '2016-04-12 23:54:43 effect internal server error Host: apache_host'
        convertions = {1: 'date'}
        cls.effect = RegexParser("effect", effect_line, effect_regex, [1], 'apache', convertions)

        line_source = LineSource('localhost', 'node_1.log')
        cls.effect_time = datetime(2016, 4, 12, 23, 54, 43)
        effect_line = '2016-04-12 23:54:43 effect internal server error Host: apache_host'
        cls.effect_clues = {
            'effect': Clue((cls.effect_time, 'apache_host'), effect_line, 40, line_source)
        }

        cls.earliest_date = datetime.min
        cls.ten_second_earlier = datetime(2016, 4, 12, 23, 54, 33)
        cls.one_hundred_second_earlier = datetime(2016, 4, 12, 23, 53, 3)
        cls.ten_second_later = datetime(2016, 4, 12, 23, 54, 53)
示例#5
0
 def find_config_in_parent_dir(cls, path):
     YamlSettingsFactory.create_new_settings_dir(
         path, SettingsFactorySelector.WHYLOG_DIR, SettingsFactorySelector.SETTINGS_FILE
     )
     config = SettingsFactorySelector.get_settings()['config']
     expected_path = SettingsFactorySelector._attach_whylog_dir(path)
     cls.validate_created_config(config, expected_path)
示例#6
0
 def test_find_config_in_parent_dir(self):
     path = os.getcwd()
     self.find_config_in_parent_dir(path)
     shutil.rmtree(SettingsFactorySelector._attach_whylog_dir(path))
     for _ in six.moves.range(2):
         path, _ = os.path.split(path)
         self.find_config_in_parent_dir(path)
         shutil.rmtree(SettingsFactorySelector._attach_whylog_dir(path))
示例#7
0
 def test_find_config_in_parent_dir(self):
     path = os.getcwd()
     self.find_config_in_parent_dir(path)
     shutil.rmtree(SettingsFactorySelector._attach_whylog_dir(path))
     for i in six.moves.range(2):
         path, _ = os.path.split(path)
         self.find_config_in_parent_dir(path)
         shutil.rmtree(SettingsFactorySelector._attach_whylog_dir(path))
示例#8
0
文件: __init__.py 项目: wlixin/whylog
    def test_add_new_rule_to_empty_config(self):
        config = SettingsFactorySelector.get_settings()['config']
        whylog_dir = SettingsFactorySelector._attach_whylog_dir(os.getcwd())
        config.add_rule(self.user_intent)
        self.check_loaded_config(config, whylog_dir)

        config = SettingsFactorySelector.get_settings()['config']
        self.check_loaded_config(config, whylog_dir)
        shutil.rmtree(whylog_dir)
示例#9
0
def whylog_factory():
    """
    This function returns LogReader object and teachers_generator.
    """
    settings = SettingsFactorySelector.get_settings()
    pattern_assistant = settings['assistant']()
    config = settings['config']
    return LogReader(config), partial(Teacher, config, pattern_assistant), config
示例#10
0
def whylog_factory():
    """
    This function returns LogReader object and teachers_generator.
    """
    settings = SettingsFactorySelector.get_settings()
    pattern_assistant = settings['assistant']()
    config = settings['config']
    return LogReader(config), partial(Teacher, config,
                                      pattern_assistant), config
示例#11
0
    def test_add_log_type(self):
        SettingsFactorySelector.WHYLOG_DIR = TestPaths.WHYLOG_DIR
        config = SettingsFactorySelector.get_settings()['config']
        whylog_dir = SettingsFactorySelector._attach_whylog_dir(os.getcwd())

        super_parser = RegexSuperParser('^(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d).*', [1], {1: 'date'})
        matcher = WildCardFilenameMatcher('localhost', 'node_1.log', 'test_log_type', super_parser)
        default_log_type = LogType('test_log_type', [matcher])
        config.add_log_type(default_log_type)

        config = SettingsFactorySelector.get_settings()['config']
        assert len(config._log_types) == 2
        log_type = config._log_types['test_log_type']
        assert log_type.name == 'test_log_type'
        assert len(log_type.filename_matchers) == 1
        matcher = log_type.filename_matchers[0]
        assert matcher.host_pattern == 'localhost'
        assert matcher.path_pattern == 'node_1.log'
        assert matcher.log_type_name == 'test_log_type'
        assert matcher.super_parser == super_parser

        shutil.rmtree(whylog_dir)
示例#12
0
    def test_add_log_type(self):
        SettingsFactorySelector.WHYLOG_DIR = TestPaths.WHYLOG_DIR
        config = SettingsFactorySelector.get_settings()['config']
        whylog_dir = SettingsFactorySelector._attach_whylog_dir(os.getcwd())

        super_parser = RegexSuperParser(
            '^(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d).*', [1], {1: 'date'})
        matcher = WildCardFilenameMatcher('localhost', 'node_1.log',
                                          'test_log_type', super_parser)
        default_log_type = LogType('test_log_type', [matcher])
        config.add_log_type(default_log_type)

        config = SettingsFactorySelector.get_settings()['config']
        assert len(config._log_types) == 1
        log_type = config._log_types['test_log_type']
        assert log_type.name == 'test_log_type'
        assert len(log_type.filename_matchers) == 1
        matcher = log_type.filename_matchers[0]
        assert matcher.host_pattern == 'localhost'
        assert matcher.path_pattern == 'node_1.log'
        assert matcher.log_type_name == 'test_log_type'
        assert matcher.super_parser == super_parser

        shutil.rmtree(whylog_dir)
示例#13
0
 def test_find_config_in_home_directory(self):
     self.find_config_in_parent_dir(SettingsFactorySelector.HOME_DIR)
     shutil.rmtree(
         SettingsFactorySelector._attach_whylog_dir(
             SettingsFactorySelector.HOME_DIR))
示例#14
0
 def test_find_config_in_home_directory(self):
     self.find_config_in_parent_dir(SettingsFactorySelector.HOME_DIR)
     shutil.rmtree(SettingsFactorySelector._attach_whylog_dir(SettingsFactorySelector.HOME_DIR))
示例#15
0
文件: __init__.py 项目: wlixin/whylog
 def tearDownClass(cls):
     # remove .test_directory if test test_add_new_rule_to_empty_config failed
     test_whylog_dir = SettingsFactorySelector._attach_whylog_dir(os.getcwd())
     if os.path.isdir(test_whylog_dir):
         shutil.rmtree(test_whylog_dir)
示例#16
0
 def tearDownClass(cls):
     whylog_dir = SettingsFactorySelector._attach_whylog_dir(os.getcwd())
     if os.path.isdir(whylog_dir):
         shutil.rmtree(whylog_dir)
示例#17
0
 def tearDownClass(cls):
     whylog_dir = SettingsFactorySelector._attach_whylog_dir(os.getcwd())
     if os.path.isdir(whylog_dir):
         shutil.rmtree(whylog_dir)
示例#18
0
文件: __init__.py 项目: wlixin/whylog
    def setUpClass(cls):
        SettingsFactorySelector.WHYLOG_DIR = TestPaths.WHYLOG_DIR

        cls.sample_line1 = "(2016-04-12 23:54:45) Connection error occurred on comp1. Host name: host1"
        cls.sample_line2 = "(2016-04-12 23:54:40) Data migration from comp1 to comp2 failed. Host name: host2"
        cls.sample_line3 = "(2016-04-12 23:54:43) Data is missing at comp2. Loss = (.*) GB. Host name: host2"

        cls.regex1 = "^(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d) Connection error occurred on (.*)\. Host name: (.*)$"
        cls.regex2 = "^(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d) Data migration from (.*) to (.*) failed\. Host name: (.*)$"
        cls.regex3 = "^(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d) Data is missing at (.*)\. Loss = (.*) GB\. Host name: (.*)$"

        regex_type = AssistantType.REGEX

        # yapf: disable
        cls.groups1 = {
            1: ParamGroup("2016-04-12 23:54:45", to_date),
            2: ParamGroup("comp1", to_string),
            3: ParamGroup("host1", to_string)
        }
        cls.groups2 = {
            1: ParamGroup("2016-04-12 23:54:40", to_date),
            2: ParamGroup("comp2", to_string),
            3: ParamGroup("host2", to_string)
        }
        cls.groups3 = {
            1: ParamGroup("2016-04-12 23:54:43", to_date),
            2: ParamGroup("comp2", to_string),
            3: ParamGroup("150", to_int),
            4: ParamGroup("host2", to_string)
        }

        cls.parser_intent1 = UserParserIntent(
            regex_type,
            "connectionerror",
            cls.regex1,
            "default",
            [1],
            cls.groups1,
            cls.sample_line1,
            line_offset=None,
            line_resource_location=None
        )
        cls.parser_intent2 = UserParserIntent(
            regex_type,
            "datamigration",
            cls.regex2,
            "default",
            [1],
            cls.groups2,
            cls.sample_line2,
            line_offset=None,
            line_resource_location=None
        )
        cls.parser_intent3 = UserParserIntent(
            regex_type,
            "lostdata",
            cls.regex3,
            "default",
            [1],
            cls.groups3,
            cls.sample_line3,
            line_offset=None,
            line_resource_location=None
        )

        parsers = {0: cls.parser_intent1, 1: cls.parser_intent2, 2: cls.parser_intent3}
        effect_id = 2

        constraint1 = UserConstraintIntent(identical_constr, [[0, 2], [1, 2]])
        constraint2 = UserConstraintIntent(identical_constr, [[1, 3], [2, 2]])
        constraint3 = UserConstraintIntent(different_constr, [[1, 2], [1, 3]])
        constraint4 = UserConstraintIntent(
            hetero_constr, [[0, 3], [1, 4], [2, 4]], {
                "different": 1
            }
        )
        #  yapf: enable

        constraints = [constraint1, constraint2, constraint3, constraint4]

        cls.user_intent = UserRuleIntent(effect_id, parsers, constraints)

        path_config = ['whylog', 'tests', 'tests_config', 'test_files', '.whylog', 'config.yaml']
        path = os.path.join(*path_config)
        cls.config = SettingsFactorySelector.load_settings(path)['config']