Пример #1
0
 def test_categorize_general_rule_two_tags(self):
     rules = [('tag', '', 'error'),
              ('tag2', '', 'fail')]
     tags = {'tag', 'tag2'}
     self.assertEqual(categorize('proj', rules, '.. error .. fail'), ({'tag', 'tag2'}, tags))
     self.assertEqual(categorize('proj', rules, '.. fail ..'), ({'tag2'}, tags))
     self.assertEqual(categorize('proj', rules, '.. error ..'), ({'tag'}, tags))
     self.assertEqual(categorize('proj', rules, '.. ok ..'), (set(), tags))
Пример #2
0
 def test_categorize_full_line_regexp(self):
     rules = [('tag2', 'proj', '^error$')]
     self.assertEqual(categorize('proj', rules, 'error'),
                      ({'tag2'}, {'tag2'}))
     self.assertEqual(categorize('proj', rules, '\nerror\n'),
                      ({'tag2'}, {'tag2'}))
     self.assertEqual(categorize('proj', rules, 'xerror'),
                      (set(), {'tag2'}))
     self.assertEqual(categorize('proj', rules, '\nerrorx\n'),
                      (set(), {'tag2'}))
Пример #3
0
 def test_categorize_general_rule_two_tags(self):
     rules = [('tag', '', 'error'), ('tag2', '', 'fail')]
     tags = {'tag', 'tag2'}
     self.assertEqual(categorize('proj', rules, '.. error .. fail'),
                      ({'tag', 'tag2'}, tags))
     self.assertEqual(categorize('proj', rules, '.. fail ..'),
                      ({'tag2'}, tags))
     self.assertEqual(categorize('proj', rules, '.. error ..'),
                      ({'tag'}, tags))
     self.assertEqual(categorize('proj', rules, '.. ok ..'), (set(), tags))
Пример #4
0
def _categorize_step_logs(job):
    """
    Args:
        job (Job): The Job to categorize logs for.
    Returns:
        Dict[UUID, Set[str]]: Mapping from JobStep ID to the categories observed for its logs.
    """
    tags_by_step = defaultdict(set)
    rules = _get_rules()
    if rules:
        for ls in _get_failing_log_sources(job):
            logdata = _get_log_data(ls)
            tags, applicable = categorize.categorize(job.project.slug, rules, logdata)
            tags_by_step[ls.step_id].update(tags)
            _incr("failing-log-processed")
            if not tags and applicable:
                _incr("failing-log-uncategorized")
                logger.warning("Uncategorized log", extra={
                    # Supplying the 'data' this way makes it available in log handlers
                    # like Sentry while keeping the warnings grouped together.
                    # See https://github.com/getsentry/raven-python/blob/master/docs/integrations/logging.rst#usage
                    # for Sentry's interpretation.
                    'data': {
                        'logsource.id': ls.id.hex,
                        'log.url': ls.get_url(),
                    }
                })
            else:
                for tag in tags:
                    _incr("failing-log-category-{}".format(tag))
    return tags_by_step
Пример #5
0
def _categorize_step_logs(job):
    """
    Args:
        job (Job): The Job to categorize logs for.
    Returns:
        Dict[UUID, Set[str]]: Mapping from JobStep ID to the categories observed for its logs.
    """
    tags_by_step = defaultdict(set)
    rules = _get_rules()
    if rules:
        for ls in _get_failing_log_sources(job):
            logdata = _get_log_data(ls)
            tags, applicable = categorize.categorize(job.project.slug, rules, logdata)
            tags_by_step[ls.step_id].update(tags)
            _incr("failing-log-processed")
            if not tags and applicable:
                _incr("failing-log-uncategorized")
                logger.warning("Uncategorized log", extra={
                    # Supplying the 'data' this way makes it available in log handlers
                    # like Sentry while keeping the warnings grouped together.
                    # See https://github.com/getsentry/raven-python/blob/master/docs/integrations/logging.rst#usage
                    # for Sentry's interpretation.
                    'data': {
                        'logsource.id': ls.id.hex,
                        'log.url': ls.get_url(),
                    }
                })
            else:
                for tag in tags:
                    _incr("failing-log-category-{}".format(tag))
    return tags_by_step
Пример #6
0
def job_finished_handler(job_id, **kwargs):
    job = Job.query.get(job_id)
    if job is None:
        return

    rules = _get_rules()
    if not rules:
        return

    for ls in _get_failing_log_sources(job):
        logdata = _get_log_data(ls)
        tags, applicable = categorize.categorize(job.project.slug, rules, logdata)
        _incr("failing-log-processed")
        if not tags and applicable:
            _incr("failing-log-uncategorized")
            logger.warning("Uncategorized log", extra={
                # Supplying the 'data' this way makes it available in log handlers
                # like Sentry while keeping the warnings grouped together.
                # See https://github.com/getsentry/raven-python/blob/master/docs/integrations/logging.rst#usage
                # for Sentry's interpretation.
                'data': {
                    'logsource.id': ls.id.hex,
                    'log.url': _log_uri(ls),
                }
            })
        else:
            for tag in tags:
                _incr("failing-log-category-{}".format(tag))
def _categorize_step_logs(job):
    """
    Args:
        job (Job): The Job to categorize logs for.
    Returns:
        Dict[UUID, Set[str]]: Mapping from JobStep ID to the categories observed for its logs.
    """
    tags_by_step = defaultdict(set)
    rules = _get_rules()
    if rules:
        for ls in _get_failing_log_sources(job):
            logdata = _get_log_data(ls)
            tags, applicable = categorize.categorize(job.project.slug, rules,
                                                     logdata)
            tags_by_step[ls.step_id].update(tags)
            _incr("failing-log-processed")
            if not tags and applicable:
                _incr("failing-log-uncategorized")
            else:
                for tag in tags:
                    _incr("failing-log-category-{}".format(tag))
    return tags_by_step
Пример #8
0
 def test_categorize_match_newline(self):
     rules = [('atag', 'aproj', 'line1.*line2')]
     self.assertEqual(categorize('aproj', rules, 'line1\n\nline2'), ({'atag'}, {'atag'}))
Пример #9
0
 def test_categorize_full_line_regexp_cr_lf(self):
     rules = [('tag', 'proj', '^error$')]
     self.assertEqual(categorize('proj', rules, '\r\nerror\r\n'), ({'tag'}, {'tag'}))
Пример #10
0
 def test_categorize_full_line_regexp(self):
     rules = [('tag2', 'proj', '^error$')]
     self.assertEqual(categorize('proj', rules, 'error'), ({'tag2'}, {'tag2'}))
     self.assertEqual(categorize('proj', rules, '\nerror\n'), ({'tag2'}, {'tag2'}))
     self.assertEqual(categorize('proj', rules, 'xerror'), (set(), {'tag2'}))
     self.assertEqual(categorize('proj', rules, '\nerrorx\n'), (set(), {'tag2'}))
Пример #11
0
 def test_categorize_project_rule(self):
     rules = [('tag2', 'proj', 'error')]
     self.assertEqual(categorize('proj', rules, '.. error ..'), ({'tag2'}, {'tag2'}))
     self.assertEqual(categorize('proj2', rules, '.. error ..'), (set(), set()))
Пример #12
0
 def test_categorize_general_rule(self):
     rules = [('tag', '', 'error')]
     self.assertEqual(categorize('proj', rules, '.. error ..'), ({'tag'}, {'tag'}))
     self.assertEqual(categorize('proj', rules, '.. Error ..'), (set(), {'tag'}))
Пример #13
0
 def test_categorize_match_newline(self):
     rules = [('atag', 'aproj', 'line1.*line2')]
     self.assertEqual(categorize('aproj', rules, 'line1\n\nline2'),
                      ({'atag'}, {'atag'}))
Пример #14
0
 def test_categorize_full_line_regexp_cr_lf(self):
     rules = [('tag', 'proj', '^error$')]
     self.assertEqual(categorize('proj', rules, '\r\nerror\r\n'),
                      ({'tag'}, {'tag'}))
Пример #15
0
 def test_categorize_project_rule(self):
     rules = [('tag2', 'proj', 'error')]
     self.assertEqual(categorize('proj', rules, '.. error ..'),
                      ({'tag2'}, {'tag2'}))
     self.assertEqual(categorize('proj2', rules, '.. error ..'),
                      (set(), set()))
Пример #16
0
 def test_categorize_general_rule(self):
     rules = [('tag', '', 'error')]
     self.assertEqual(categorize('proj', rules, '.. error ..'),
                      ({'tag'}, {'tag'}))
     self.assertEqual(categorize('proj', rules, '.. Error ..'),
                      (set(), {'tag'}))