def convert_db_to_json(build_db_data, gatekeeper_config, f):
"""Converts build_db to a format suitable for JSON encoding and writes it."""
# Remove all but the last finished build.
for builders in build_db_data.masters.values():
for builder in builders:
unfinished = [(k, v) for k, v in builders[builder].iteritems()
if not v.finished]
finished = [(k, v) for k, v in builders[builder].iteritems()
if v.finished]
builders[builder] = dict(unfinished)
if finished:
max_finished = max(finished, key=lambda x: x[0])
builders[builder][max_finished[0]] = max_finished[1]
build_db = gen_db(masters=build_db_data.masters, aux=build_db_data.aux)
# Output the gatekeeper sections we're operating with, so a human reading the
# file can debug issues. This is discarded by the parser in get_build_db.
used_sections = set([])
for masters in build_db_data.masters.values():
for builder in masters.values():
used_sections |= set(t for b in builder.values() for t in b.triggered)
for master in gatekeeper_config.values():
for section in master:
section_hash = gatekeeper_ng_config.gatekeeper_section_hash(section)
if section_hash in used_sections:
build_db.sections[section_hash] = section
json.dump(build_db.asJson(), f, cls=gatekeeper_ng_config.SetEncoder,
sort_keys=True)
def check_builds(master_builds, master_jsons, gatekeeper_config):
"""Given a gatekeeper configuration, see which builds have failed."""
succeeded_builds = []
failed_builds = []
# Sort by buildnumber, highest first.
sorted_builds = sorted(master_builds, key=lambda x: x[3], reverse=True)
successful_builder_steps = defaultdict(lambda: defaultdict(set))
current_builds_successful = True
for build_json, master_url, builder, buildnum in sorted_builds:
gatekeeper_sections = gatekeeper_config.get(master_url, [])
for gatekeeper_section in gatekeeper_sections:
section_hash = gatekeeper_ng_config.gatekeeper_section_hash(
gatekeeper_section)
gatekeeper = get_builder_section(gatekeeper_section,
build_json['builderName'])
if not gatekeeper:
succeeded_builds.append((master_url, builder, buildnum))
continue
steps = build_json['steps']
excluded_steps = set(gatekeeper.get('excluded_steps', []))
forgiving_optional = (
set(gatekeeper.get('forgiving_optional', [])) - excluded_steps)
closing_optional = ((set(gatekeeper.get('closing_optional', []))
| forgiving_optional) - excluded_steps)
tree_notify = set(gatekeeper.get('tree_notify', []))
sheriff_classes = set(gatekeeper.get('sheriff_classes', []))
status_template = gatekeeper.get(
'status_template',
gatekeeper_ng_config.DEFAULTS['status_template'])
subject_template = gatekeeper.get(
'subject_template',
gatekeeper_ng_config.DEFAULTS['subject_template'])
finished = [s for s in steps if s.get('isFinished')]
close_tree = gatekeeper.get('close_tree', True)
respect_build_status = gatekeeper.get('respect_build_status',
False)
# We ignore EXCEPTION and RETRY here since those are usually
# infrastructure-related instead of actual test errors.
successful_steps = set(
s['name'] for s in finished
if s.get('results', [FAILURE])[0] != FAILURE)
successful_builder_steps[master_url][builder].update(
successful_steps)
finished_steps = set(s['name'] for s in finished)
if '*' in forgiving_optional:
forgiving_optional = (finished_steps - excluded_steps)
if '*' in closing_optional:
closing_optional = (finished_steps - excluded_steps)
failed_steps = finished_steps - successful_steps
unsatisfied_steps = failed_steps & closing_optional
# Build is not yet finished, don't penalize on unstarted/unfinished steps.
if build_json.get('results', None) is None:
unsatisfied_steps &= finished_steps
# If the entire build failed.
if (not unsatisfied_steps and 'results' in build_json
and build_json['results'] == FAILURE
and respect_build_status):
unsatisfied_steps.add('[overall build status]')
buildbot_url = master_jsons[master_url]['project']['buildbotURL']
project_name = master_jsons[master_url]['project']['title']
if unsatisfied_steps:
failed_builds.append(({
'base_url': buildbot_url,
'build': build_json,
'close_tree': close_tree,
'forgiving_steps': forgiving_optional,
'project_name': project_name,
'sheriff_classes': sheriff_classes,
'subject_template': subject_template,
'status_template': status_template,
'tree_notify': tree_notify,
'unsatisfied': unsatisfied_steps,
}, master_url, builder, buildnum, section_hash))
# If there is a failing step that a newer builder hasn't succeeded on,
# don't open the tree.
still_failing_steps = (
unsatisfied_steps -
successful_builder_steps[master_url][builder])
if still_failing_steps:
logging.debug('%s failed on %s, not yet resolved.',
','.join(still_failing_steps),
generate_build_url(failed_builds[-1][0]))
current_builds_successful = False
else:
succeeded_builds.append((master_url, builder, buildnum))
return (list(reversed(failed_builds)), list(reversed(succeeded_builds)),
successful_builder_steps, current_builds_successful)
def check_builds(master_builds, master_jsons, gatekeeper_config):
"""Given a gatekeeper configuration, see which builds have failed."""
failed_builds = []
for build_json, master_url, builder, buildnum in master_builds:
gatekeeper_sections = gatekeeper_config.get(master_url, [])
for gatekeeper_section in gatekeeper_sections:
section_hash = gatekeeper_ng_config.gatekeeper_section_hash(
gatekeeper_section)
if build_json['builderName'] in gatekeeper_section:
gatekeeper = gatekeeper_section[build_json['builderName']]
elif '*' in gatekeeper_section:
gatekeeper = gatekeeper_section['*']
else:
continue
# Check if the buildername is in the excluded builder list and disable if
# so.
if build_json['builderName'] in gatekeeper.get('excluded_builders', []):
continue
steps = build_json['steps']
excluded_steps = set(gatekeeper.get('excluded_steps', []))
forgiving = set(gatekeeper.get('forgiving_steps', [])) - excluded_steps
forgiving_optional = (
set(gatekeeper.get('forgiving_optional', [])) - excluded_steps)
closing_steps = (
set(gatekeeper.get('closing_steps', [])) | forgiving) - excluded_steps
closing_optional = (
(set(gatekeeper.get('closing_optional', [])) | forgiving_optional) -
excluded_steps
)
tree_notify = set(gatekeeper.get('tree_notify', []))
sheriff_classes = set(gatekeeper.get('sheriff_classes', []))
subject_template = gatekeeper.get('subject_template',
gatekeeper_ng_config.DEFAULTS[
'subject_template'])
finished = [s for s in steps if s.get('isFinished')]
close_tree = gatekeeper.get('close_tree', True)
respect_build_status = gatekeeper.get('respect_build_status', False)
successful_steps = set(s['name'] for s in finished
if (s.get('results', [FAILURE])[0] == SUCCESS or
s.get('results', [FAILURE])[0] == WARNINGS))
finished_steps = set(s['name'] for s in finished)
if '*' in forgiving_optional:
forgiving_optional = (finished_steps - excluded_steps)
if '*' in closing_optional:
closing_optional = (finished_steps - excluded_steps)
unsatisfied_steps = closing_steps - successful_steps
failed_steps = finished_steps - successful_steps
failed_optional_steps = failed_steps & closing_optional
unsatisfied_steps |= failed_optional_steps
# Build is not yet finished, don't penalize on unstarted/unfinished steps.
if build_json.get('results', None) is None:
unsatisfied_steps &= finished_steps
# If the entire build failed.
if (not unsatisfied_steps and 'results' in build_json and
build_json['results'] != SUCCESS and respect_build_status):
unsatisfied_steps.add('[overall build status]')
buildbot_url = master_jsons[master_url]['project']['buildbotURL']
project_name = master_jsons[master_url]['project']['title']
if unsatisfied_steps:
failed_builds.append(({'base_url': buildbot_url,
'build': build_json,
'close_tree': close_tree,
'forgiving_steps': (
forgiving | forgiving_optional),
'project_name': project_name,
'sheriff_classes': sheriff_classes,
'subject_template': subject_template,
'tree_notify': tree_notify,
'unsatisfied': unsatisfied_steps,
},
master_url,
builder,
buildnum,
section_hash))
return failed_builds
def check_builds(master_builds, master_jsons, gatekeeper_config):
"""Given a gatekeeper configuration, see which builds have failed."""
failed_builds = []
for build_json, master_url, builder, buildnum in master_builds:
gatekeeper_sections = gatekeeper_config.get(master_url, [])
for gatekeeper_section in gatekeeper_sections:
section_hash = gatekeeper_ng_config.gatekeeper_section_hash(
gatekeeper_section)
if build_json['builderName'] in gatekeeper_section:
gatekeeper = gatekeeper_section[build_json['builderName']]
elif '*' in gatekeeper_section:
gatekeeper = gatekeeper_section['*']
else:
continue
# Check if the buildername is in the excluded builder list and disable if
# so.
if build_json['builderName'] in gatekeeper.get(
'excluded_builders', []):
continue
steps = build_json['steps']
excluded_steps = set(gatekeeper.get('excluded_steps', []))
forgiving = set(gatekeeper.get('forgiving_steps',
[])) - excluded_steps
forgiving_optional = (
set(gatekeeper.get('forgiving_optional', [])) - excluded_steps)
closing_steps = (set(gatekeeper.get('closing_steps', []))
| forgiving) - excluded_steps
closing_optional = ((set(gatekeeper.get('closing_optional', []))
| forgiving_optional) - excluded_steps)
tree_notify = set(gatekeeper.get('tree_notify', []))
sheriff_classes = set(gatekeeper.get('sheriff_classes', []))
subject_template = gatekeeper.get(
'subject_template',
gatekeeper_ng_config.DEFAULTS['subject_template'])
finished = [s for s in steps if s.get('isFinished')]
close_tree = gatekeeper.get('close_tree', True)
respect_build_status = gatekeeper.get('respect_build_status',
False)
successful_steps = set(
s['name'] for s in finished
if (s.get('results', [FAILURE])[0] == SUCCESS
or s.get('results', [FAILURE])[0] == WARNINGS))
finished_steps = set(s['name'] for s in finished)
if '*' in forgiving_optional:
forgiving_optional = (finished_steps - excluded_steps)
if '*' in closing_optional:
closing_optional = (finished_steps - excluded_steps)
unsatisfied_steps = closing_steps - successful_steps
failed_steps = finished_steps - successful_steps
failed_optional_steps = failed_steps & closing_optional
unsatisfied_steps |= failed_optional_steps
# Build is not yet finished, don't penalize on unstarted/unfinished steps.
if build_json.get('results', None) is None:
unsatisfied_steps &= finished_steps
# If the entire build failed.
if (not unsatisfied_steps and 'results' in build_json
and build_json['results'] != SUCCESS
and respect_build_status):
unsatisfied_steps.add('[overall build status]')
buildbot_url = master_jsons[master_url]['project']['buildbotURL']
project_name = master_jsons[master_url]['project']['title']
if unsatisfied_steps:
failed_builds.append(({
'base_url':
buildbot_url,
'build':
build_json,
'close_tree':
close_tree,
'forgiving_steps': (forgiving | forgiving_optional),
'project_name':
project_name,
'sheriff_classes':
sheriff_classes,
'subject_template':
subject_template,
'tree_notify':
tree_notify,
'unsatisfied':
unsatisfied_steps,
}, master_url, builder, buildnum, section_hash))
return failed_builds
def check_builds(master_builds, master_jsons, gatekeeper_config):
"""Given a gatekeeper configuration, see which builds have failed."""
succeeded_builds = []
failed_builds = []
# Sort by buildnumber, highest first.
sorted_builds = sorted(master_builds, key=lambda x: x[3], reverse=True)
successful_builder_steps = defaultdict(lambda: defaultdict(set))
current_builds_successful = True
for build_json, master_url, builder, buildnum in sorted_builds:
gatekeeper_sections = gatekeeper_config.get(master_url, [])
for gatekeeper_section in gatekeeper_sections:
section_hash = gatekeeper_ng_config.gatekeeper_section_hash(
gatekeeper_section)
gatekeeper = get_builder_section(
gatekeeper_section, build_json['builderName'])
if not gatekeeper:
succeeded_builds.append((master_url, builder, buildnum))
continue
steps = build_json['steps']
excluded_steps = set(gatekeeper.get('excluded_steps', []))
forgiving = set(gatekeeper.get('forgiving_steps', [])) - excluded_steps
forgiving_optional = (
set(gatekeeper.get('forgiving_optional', [])) - excluded_steps)
closing_steps = (
set(gatekeeper.get('closing_steps', [])) | forgiving) - excluded_steps
closing_optional = (
(set(gatekeeper.get('closing_optional', [])) | forgiving_optional) -
excluded_steps
)
tree_notify = set(gatekeeper.get('tree_notify', []))
sheriff_classes = set(gatekeeper.get('sheriff_classes', []))
status_template = gatekeeper.get(
'status_template', gatekeeper_ng_config.DEFAULTS['status_template'])
subject_template = gatekeeper.get(
'subject_template', gatekeeper_ng_config.DEFAULTS[
'subject_template'])
finished = [s for s in steps if s.get('isFinished')]
close_tree = gatekeeper.get('close_tree', True)
respect_build_status = gatekeeper.get('respect_build_status', False)
# We ignore EXCEPTION and RETRY here since those are usually
# infrastructure-related instead of actual test errors.
successful_steps = set(s['name'] for s in finished
if s.get('results', [FAILURE])[0] != FAILURE)
successful_builder_steps[master_url][builder].update(successful_steps)
finished_steps = set(s['name'] for s in finished)
if '*' in forgiving_optional:
forgiving_optional = (finished_steps - excluded_steps)
if '*' in closing_optional:
closing_optional = (finished_steps - excluded_steps)
unsatisfied_steps = closing_steps - successful_steps
failed_steps = finished_steps - successful_steps
failed_optional_steps = failed_steps & closing_optional
unsatisfied_steps |= failed_optional_steps
# Build is not yet finished, don't penalize on unstarted/unfinished steps.
if build_json.get('results', None) is None:
unsatisfied_steps &= finished_steps
# If the entire build failed.
if (not unsatisfied_steps and 'results' in build_json and
build_json['results'] == FAILURE and respect_build_status):
unsatisfied_steps.add('[overall build status]')
buildbot_url = master_jsons[master_url]['project']['buildbotURL']
project_name = master_jsons[master_url]['project']['title']
if unsatisfied_steps:
failed_builds.append(({'base_url': buildbot_url,
'build': build_json,
'close_tree': close_tree,
'forgiving_steps': (
forgiving | forgiving_optional),
'project_name': project_name,
'sheriff_classes': sheriff_classes,
'subject_template': subject_template,
'status_template': status_template,
'tree_notify': tree_notify,
'unsatisfied': unsatisfied_steps,
},
master_url,
builder,
buildnum,
section_hash))
# If there is a failing step that a newer builder hasn't succeeded on,
# don't open the tree.
still_failing_steps = (
unsatisfied_steps - successful_builder_steps[master_url][builder])
if still_failing_steps and close_tree:
logging.debug('%s failed on %s, not yet resolved.',
','.join(still_failing_steps),
generate_build_url(failed_builds[-1][0]))
current_builds_successful = False
else:
succeeded_builds.append((master_url, builder, buildnum))
return (list(reversed(failed_builds)), list(reversed(succeeded_builds)),
successful_builder_steps, current_builds_successful)
