def test_categorize_general_rule_two_tags(self):
rules = [('tag', '', 'error'),
('tag2', '', 'fail')]
tags = {'tag', 'tag2'}
self.assertEqual(categorize('proj', rules, '.. error .. fail'), ({'tag', 'tag2'}, tags))
self.assertEqual(categorize('proj', rules, '.. fail ..'), ({'tag2'}, tags))
self.assertEqual(categorize('proj', rules, '.. error ..'), ({'tag'}, tags))
self.assertEqual(categorize('proj', rules, '.. ok ..'), (set(), tags))
def test_categorize_full_line_regexp(self):
rules = [('tag2', 'proj', '^error$')]
self.assertEqual(categorize('proj', rules, 'error'),
({'tag2'}, {'tag2'}))
self.assertEqual(categorize('proj', rules, '\nerror\n'),
({'tag2'}, {'tag2'}))
self.assertEqual(categorize('proj', rules, 'xerror'),
(set(), {'tag2'}))
self.assertEqual(categorize('proj', rules, '\nerrorx\n'),
(set(), {'tag2'}))
def test_categorize_general_rule_two_tags(self):
rules = [('tag', '', 'error'), ('tag2', '', 'fail')]
tags = {'tag', 'tag2'}
self.assertEqual(categorize('proj', rules, '.. error .. fail'),
({'tag', 'tag2'}, tags))
self.assertEqual(categorize('proj', rules, '.. fail ..'),
({'tag2'}, tags))
self.assertEqual(categorize('proj', rules, '.. error ..'),
({'tag'}, tags))
self.assertEqual(categorize('proj', rules, '.. ok ..'), (set(), tags))
def _categorize_step_logs(job):
"""
Args:
job (Job): The Job to categorize logs for.
Returns:
Dict[UUID, Set[str]]: Mapping from JobStep ID to the categories observed for its logs.
"""
tags_by_step = defaultdict(set)
rules = _get_rules()
if rules:
for ls in _get_failing_log_sources(job):
logdata = _get_log_data(ls)
tags, applicable = categorize.categorize(job.project.slug, rules, logdata)
tags_by_step[ls.step_id].update(tags)
_incr("failing-log-processed")
if not tags and applicable:
_incr("failing-log-uncategorized")
logger.warning("Uncategorized log", extra={
# Supplying the 'data' this way makes it available in log handlers
# like Sentry while keeping the warnings grouped together.
# See https://github.com/getsentry/raven-python/blob/master/docs/integrations/logging.rst#usage
# for Sentry's interpretation.
'data': {
'logsource.id': ls.id.hex,
'log.url': ls.get_url(),
}
})
else:
for tag in tags:
_incr("failing-log-category-{}".format(tag))
return tags_by_step
def _categorize_step_logs(job):
"""
Args:
job (Job): The Job to categorize logs for.
Returns:
Dict[UUID, Set[str]]: Mapping from JobStep ID to the categories observed for its logs.
"""
tags_by_step = defaultdict(set)
rules = _get_rules()
if rules:
for ls in _get_failing_log_sources(job):
logdata = _get_log_data(ls)
tags, applicable = categorize.categorize(job.project.slug, rules, logdata)
tags_by_step[ls.step_id].update(tags)
_incr("failing-log-processed")
if not tags and applicable:
_incr("failing-log-uncategorized")
logger.warning("Uncategorized log", extra={
# Supplying the 'data' this way makes it available in log handlers
# like Sentry while keeping the warnings grouped together.
# See https://github.com/getsentry/raven-python/blob/master/docs/integrations/logging.rst#usage
# for Sentry's interpretation.
'data': {
'logsource.id': ls.id.hex,
'log.url': ls.get_url(),
}
})
else:
for tag in tags:
_incr("failing-log-category-{}".format(tag))
return tags_by_step
def job_finished_handler(job_id, **kwargs):
job = Job.query.get(job_id)
if job is None:
return
rules = _get_rules()
if not rules:
return
for ls in _get_failing_log_sources(job):
logdata = _get_log_data(ls)
tags, applicable = categorize.categorize(job.project.slug, rules, logdata)
_incr("failing-log-processed")
if not tags and applicable:
_incr("failing-log-uncategorized")
logger.warning("Uncategorized log", extra={
# Supplying the 'data' this way makes it available in log handlers
# like Sentry while keeping the warnings grouped together.
# See https://github.com/getsentry/raven-python/blob/master/docs/integrations/logging.rst#usage
# for Sentry's interpretation.
'data': {
'logsource.id': ls.id.hex,
'log.url': _log_uri(ls),
}
})
else:
for tag in tags:
_incr("failing-log-category-{}".format(tag))
def _categorize_step_logs(job):
"""
Args:
job (Job): The Job to categorize logs for.
Returns:
Dict[UUID, Set[str]]: Mapping from JobStep ID to the categories observed for its logs.
"""
tags_by_step = defaultdict(set)
rules = _get_rules()
if rules:
for ls in _get_failing_log_sources(job):
logdata = _get_log_data(ls)
tags, applicable = categorize.categorize(job.project.slug, rules,
logdata)
tags_by_step[ls.step_id].update(tags)
_incr("failing-log-processed")
if not tags and applicable:
_incr("failing-log-uncategorized")
else:
for tag in tags:
_incr("failing-log-category-{}".format(tag))
return tags_by_step
def test_categorize_match_newline(self):
rules = [('atag', 'aproj', 'line1.*line2')]
self.assertEqual(categorize('aproj', rules, 'line1\n\nline2'), ({'atag'}, {'atag'}))
def test_categorize_full_line_regexp_cr_lf(self):
rules = [('tag', 'proj', '^error$')]
self.assertEqual(categorize('proj', rules, '\r\nerror\r\n'), ({'tag'}, {'tag'}))
def test_categorize_full_line_regexp(self):
rules = [('tag2', 'proj', '^error$')]
self.assertEqual(categorize('proj', rules, 'error'), ({'tag2'}, {'tag2'}))
self.assertEqual(categorize('proj', rules, '\nerror\n'), ({'tag2'}, {'tag2'}))
self.assertEqual(categorize('proj', rules, 'xerror'), (set(), {'tag2'}))
self.assertEqual(categorize('proj', rules, '\nerrorx\n'), (set(), {'tag2'}))
def test_categorize_project_rule(self):
rules = [('tag2', 'proj', 'error')]
self.assertEqual(categorize('proj', rules, '.. error ..'), ({'tag2'}, {'tag2'}))
self.assertEqual(categorize('proj2', rules, '.. error ..'), (set(), set()))
def test_categorize_general_rule(self):
rules = [('tag', '', 'error')]
self.assertEqual(categorize('proj', rules, '.. error ..'), ({'tag'}, {'tag'}))
self.assertEqual(categorize('proj', rules, '.. Error ..'), (set(), {'tag'}))
def test_categorize_match_newline(self):
rules = [('atag', 'aproj', 'line1.*line2')]
self.assertEqual(categorize('aproj', rules, 'line1\n\nline2'),
({'atag'}, {'atag'}))
def test_categorize_full_line_regexp_cr_lf(self):
rules = [('tag', 'proj', '^error$')]
self.assertEqual(categorize('proj', rules, '\r\nerror\r\n'),
({'tag'}, {'tag'}))
def test_categorize_project_rule(self):
rules = [('tag2', 'proj', 'error')]
self.assertEqual(categorize('proj', rules, '.. error ..'),
({'tag2'}, {'tag2'}))
self.assertEqual(categorize('proj2', rules, '.. error ..'),
(set(), set()))
def test_categorize_general_rule(self):
rules = [('tag', '', 'error')]
self.assertEqual(categorize('proj', rules, '.. error ..'),
({'tag'}, {'tag'}))
self.assertEqual(categorize('proj', rules, '.. Error ..'),
(set(), {'tag'}))