def test_categorize_general_rule_two_tags(self): rules = [('tag', '', 'error'), ('tag2', '', 'fail')] tags = {'tag', 'tag2'} self.assertEqual(categorize('proj', rules, '.. error .. fail'), ({'tag', 'tag2'}, tags)) self.assertEqual(categorize('proj', rules, '.. fail ..'), ({'tag2'}, tags)) self.assertEqual(categorize('proj', rules, '.. error ..'), ({'tag'}, tags)) self.assertEqual(categorize('proj', rules, '.. ok ..'), (set(), tags))
def test_categorize_full_line_regexp(self): rules = [('tag2', 'proj', '^error$')] self.assertEqual(categorize('proj', rules, 'error'), ({'tag2'}, {'tag2'})) self.assertEqual(categorize('proj', rules, '\nerror\n'), ({'tag2'}, {'tag2'})) self.assertEqual(categorize('proj', rules, 'xerror'), (set(), {'tag2'})) self.assertEqual(categorize('proj', rules, '\nerrorx\n'), (set(), {'tag2'}))
def _categorize_step_logs(job): """ Args: job (Job): The Job to categorize logs for. Returns: Dict[UUID, Set[str]]: Mapping from JobStep ID to the categories observed for its logs. """ tags_by_step = defaultdict(set) rules = _get_rules() if rules: for ls in _get_failing_log_sources(job): logdata = _get_log_data(ls) tags, applicable = categorize.categorize(job.project.slug, rules, logdata) tags_by_step[ls.step_id].update(tags) _incr("failing-log-processed") if not tags and applicable: _incr("failing-log-uncategorized") logger.warning("Uncategorized log", extra={ # Supplying the 'data' this way makes it available in log handlers # like Sentry while keeping the warnings grouped together. # See https://github.com/getsentry/raven-python/blob/master/docs/integrations/logging.rst#usage # for Sentry's interpretation. 'data': { 'logsource.id': ls.id.hex, 'log.url': ls.get_url(), } }) else: for tag in tags: _incr("failing-log-category-{}".format(tag)) return tags_by_step
def job_finished_handler(job_id, **kwargs): job = Job.query.get(job_id) if job is None: return rules = _get_rules() if not rules: return for ls in _get_failing_log_sources(job): logdata = _get_log_data(ls) tags, applicable = categorize.categorize(job.project.slug, rules, logdata) _incr("failing-log-processed") if not tags and applicable: _incr("failing-log-uncategorized") logger.warning("Uncategorized log", extra={ # Supplying the 'data' this way makes it available in log handlers # like Sentry while keeping the warnings grouped together. # See https://github.com/getsentry/raven-python/blob/master/docs/integrations/logging.rst#usage # for Sentry's interpretation. 'data': { 'logsource.id': ls.id.hex, 'log.url': _log_uri(ls), } }) else: for tag in tags: _incr("failing-log-category-{}".format(tag))
def _categorize_step_logs(job): """ Args: job (Job): The Job to categorize logs for. Returns: Dict[UUID, Set[str]]: Mapping from JobStep ID to the categories observed for its logs. """ tags_by_step = defaultdict(set) rules = _get_rules() if rules: for ls in _get_failing_log_sources(job): logdata = _get_log_data(ls) tags, applicable = categorize.categorize(job.project.slug, rules, logdata) tags_by_step[ls.step_id].update(tags) _incr("failing-log-processed") if not tags and applicable: _incr("failing-log-uncategorized") else: for tag in tags: _incr("failing-log-category-{}".format(tag)) return tags_by_step
def test_categorize_match_newline(self): rules = [('atag', 'aproj', 'line1.*line2')] self.assertEqual(categorize('aproj', rules, 'line1\n\nline2'), ({'atag'}, {'atag'}))
def test_categorize_full_line_regexp_cr_lf(self): rules = [('tag', 'proj', '^error$')] self.assertEqual(categorize('proj', rules, '\r\nerror\r\n'), ({'tag'}, {'tag'}))
def test_categorize_project_rule(self): rules = [('tag2', 'proj', 'error')] self.assertEqual(categorize('proj', rules, '.. error ..'), ({'tag2'}, {'tag2'})) self.assertEqual(categorize('proj2', rules, '.. error ..'), (set(), set()))
def test_categorize_general_rule(self): rules = [('tag', '', 'error')] self.assertEqual(categorize('proj', rules, '.. error ..'), ({'tag'}, {'tag'})) self.assertEqual(categorize('proj', rules, '.. Error ..'), (set(), {'tag'}))