def main(args):
parser = argparse.ArgumentParser()
parser.add_argument('--use-cache', action='store_true')
parser.add_argument('--builders', action='store_true')
parser.add_argument('--master-filter', action='store')
parser.add_argument('--noise-threshold', action='store', type=int, default=2)
parser.add_argument('--build-limit', action='store', type=int, default=100)
parser.add_argument('--show-pass', action='store_true')
args = parser.parse_args(args)
if args.use_cache:
requests_cache.install_cache('failure_stats')
gatekeeper_config = gatekeeper_ng_config.load_gatekeeper_config(CONFIG_PATH)
all_outcomes = collections.Counter()
for master_url, master_config in gatekeeper_config.items():
master_outcomes = collections.Counter()
master_name = urlparse.urlparse(master_url).path.split('/')[-1]
if args.master_filter:
if args.master_filter not in master_name:
continue
if args.builders:
print
print master_name
common_config = master_config[0].get('*', {})
excluded_builders = common_config.get('excluded_builders', set())
builder_names = fetch_builder_names(master_name)
builder_names = sorted(set(builder_names) - excluded_builders)
for builder_name in builder_names:
outcomes = collections.Counter()
recent_builds = builds_for_builder(master_name, builder_name)
for build in recent_builds[:args.build_limit]:
if build.get('results', 0) == 0:
continue
failing_results = [step['results'][1] for step in build['steps']
if step['results'][0]]
failure_name = pretty_failure_name(failing_results)
outcomes[failure_name] += 1
if args.builders:
fail_strings = ['{:2} {:<30}'.format(*reversed(tup)) for tup in outcomes.most_common(3) if tup[1] >= args.noise_threshold]
if not outcomes.most_common(3):
if args.show_pass:
print '%30s : PASS' % builder_name
elif outcomes.most_common(1)[0][1] < args.noise_threshold:
print '%30s : noise' % builder_name
else:
print '%30s : %s' % (builder_name, ' | '.join(fail_strings))
master_outcomes += outcomes
print_worst_failures(master_name, master_outcomes)
all_outcomes += master_outcomes
if not args.master_filter:
print_worst_failures('total for all %s masters' % CONFIG_PATH, all_outcomes)
def main(args):
parser = argparse.ArgumentParser()
parser.add_argument('data_url', action='store', nargs='*')
parser.add_argument('--use-cache', action='store_true')
parser.add_argument('--master-filter', action='store')
args = parser.parse_args(args)
if not args.data_url:
log.warn("No /data url passed, won't do anything")
if args.use_cache:
requests_cache.install_cache('failure_stats')
else:
requests_cache.install_cache(backend='memory')
gatekeeper = gatekeeper_ng_config.load_gatekeeper_config(CONFIG_PATH)
master_urls = fetch_master_urls(gatekeeper, args)
start_time = datetime.datetime.now()
latest_revisions = {}
cache = buildbot.BuildCache(CACHE_PATH)
alerts = []
for master_url in master_urls:
master_json = buildbot.fetch_master_json(master_url)
master_alerts = alert_builder.alerts_for_master(cache, master_url, master_json)
alerts.extend(master_alerts)
# FIXME: This doesn't really belong here. garden-o-matic wants
# this data and we happen to have the builder json cached at
# this point so it's cheap to compute.
revisions = buildbot.latest_revisions_for_master(cache, master_url, master_json)
latest_revisions.update(revisions)
print "Fetch took: %s" % (datetime.datetime.now() - start_time)
alerts = apply_gatekeeper_rules(alerts, gatekeeper)
alerts = analysis.assign_keys(alerts)
reason_groups = analysis.group_by_reason(alerts)
range_groups = analysis.merge_by_range(reason_groups)
data = { 'content': json.dumps({
'alerts': alerts,
'reason_groups': reason_groups,
'range_groups': range_groups,
'latest_revisions': latest_revisions,
})}
for url in args.data_url:
log.info('POST %s alerts to %s' % (len(alerts), url))
requests.post(url, data=data)
def main():
prog_desc = 'Parses the build_db and outputs to stdout.'
usage = '%prog [options]'
parser = optparse.OptionParser(usage=(usage + '\n\n' + prog_desc))
parser.add_option('--json', default=os.path.join(DATA_DIR, 'gatekeeper.json'),
help='location of gatekeeper configuration file')
parser.add_option('--build-db', default='build_db.json',
help='records the build status information for builders')
options, _ = parser.parse_args()
build_db = get_build_db(options.build_db)
gatekeeper_config = gatekeeper_ng_config.load_gatekeeper_config(options.json)
convert_db_to_json(build_db, gatekeeper_config, sys.stdout)
print
return 0
def main(argv):
args = get_args(argv)
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO)
gatekeeper_config = gatekeeper_ng_config.load_gatekeeper_config(args.json)
if args.verify:
return 0
simulate = bool(args.simulate_master)
if args.flatten_json:
if not args.no_hashes:
gatekeeper_config = gatekeeper_ng_config.inject_hashes(
gatekeeper_config)
gatekeeper_ng_config.flatten_to_json(gatekeeper_config, sys.stdout)
print
return 0
if args.set_status and not simulate:
args.password = get_pwd(args.password_file)
masters = defaultdict(set)
for m in args.master_url:
if m.count(':') > 1:
# Master in master_url:builder,builder format.
http, mname, builderlist = m.split(':', 2)
mastername = ':'.join([http, mname])
masters[mastername].update(builderlist.split(','))
else:
# Regular master URL, just add '*'.
masters[m].add(build_scan.BUILDER_WILDCARD)
if not set(masters) <= set(gatekeeper_config):
print 'The following masters are not present in the gatekeeper config:'
for m in set(masters) - set(gatekeeper_config):
print ' ' + m
return 1
emoji = []
if args.emoji != 'None':
try:
with open(args.emoji) as f:
emoji = json.load(f)
except (IOError, ValueError) as e:
logging.warning('Could not load emoji file %s: %s', args.emoji, e)
if args.clear_build_db:
build_db = build_scan_db.gen_db()
build_scan_db.save_build_db(build_db, gatekeeper_config, args.build_db)
else:
build_db = build_scan_db.get_build_db(args.build_db)
if not simulate:
master_jsons, build_jsons = build_scan.get_updated_builds(
masters, build_db, args.parallelism, args.milo_creds)
else:
master_jsons, build_jsons = simulate_build_failure(
build_db, args.simulate_master, args.simulate_builder,
*args.simulate_step)
if args.sync_build_db:
build_scan_db.save_build_db(build_db, gatekeeper_config, args.build_db)
return 0
(failure_tuples, success_tuples, successful_builder_steps,
current_builds_successful) = check_builds(build_jsons, master_jsons,
gatekeeper_config)
# Write failure / success information back to the build_db.
propagate_build_status_back_to_db(failure_tuples, success_tuples, build_db)
# opening is an option, mostly to keep the unittests working which
# assume that any setting of status is negative.
if args.open_tree:
open_tree_if_possible(build_db, master_jsons, successful_builder_steps,
current_builds_successful, args.status_user,
args.password, args.status_url, args.set_status,
emoji, simulate)
# debounce_failures does 3 things:
# 1. Groups logging by builder
# 2. Selects out the "build" part from the failure tuple.
# 3. Rejects builds we've already warned about (and logs).
new_failures = debounce_failures(failure_tuples, current_builds_successful,
build_db)
if args.track_revisions:
# Only close the tree if it's a newer revision than before.
properties = args.revision_properties.split(',')
triggered_revisions = build_db.aux.get('triggered_revisions', {})
if not triggered_revisions or (sorted(triggered_revisions) !=
sorted(properties)):
logging.info(
'revision properties have changed from %s to %s. '
'clearing previous data.', triggered_revisions, properties)
build_db.aux['triggered_revisions'] = dict.fromkeys(properties)
new_failures = reject_old_revisions(new_failures, build_db)
close_tree_if_necessary(build_db, new_failures, args.status_user,
args.password, args.status_url, args.set_status,
args.revision_properties.split(','), simulate)
try:
notify_failures(new_failures, args.sheriff_url,
args.default_from_email, args.email_app_url,
args.email_app_secret, args.email_domain,
args.filter_domain, args.disable_domain_filter,
simulate)
finally:
if not args.skip_build_db_update and not simulate:
build_scan_db.save_build_db(build_db, gatekeeper_config,
args.build_db)
return 0
def inner_loop(args):
if not args.data_url:
logging.warn('No /data url passed, will write to builder_alerts.json')
if args.use_cache:
requests_cache.install_cache('failure_stats')
else:
requests_cache.install_cache(backend='memory')
# FIXME: gatekeeper_config should find gatekeeper.json for us.
gatekeeper_path = os.path.abspath(args.gatekeeper)
logging.debug('Processsing gatekeeper json: %s', gatekeeper_path)
gatekeeper = gatekeeper_ng_config.load_gatekeeper_config(gatekeeper_path)
gatekeeper_trees_path = os.path.abspath(args.gatekeeper_trees)
logging.debug('Processing gatekeeper trees json: %s',
gatekeeper_trees_path)
gatekeeper_trees = gatekeeper_ng_config.load_gatekeeper_tree_config(
gatekeeper_trees_path)
master_urls = gatekeeper_extras.fetch_master_urls(gatekeeper, args)
start_time = datetime.datetime.utcnow()
cache = buildbot.DiskCache(CACHE_PATH)
old_alerts = {}
if args.data_url:
try:
old_alerts_raw = requests.get(args.data_url[0]).json()
except ValueError:
logging.debug('No old alerts found.')
else:
# internal-alerts will have a redirect instead of alerts if you're
# signed in.
if 'alerts' in old_alerts_raw:
for alert in old_alerts_raw['alerts']:
master = alert['master_url']
builder = alert['builder_name']
step = alert['step_name']
reason = alert['reason']
alert_key = alert_builder.generate_alert_key(
master, builder, step, reason)
if alert_key in old_alerts:
logging.critical(
'Incorrectly overwriting an alert reason from the'
' old alert data. master: %s, builder: %s, step: %s, reason:'
' %s' % (master, builder, step, reason))
old_alerts[alert_key] = alert
latest_builder_info = {}
stale_builder_alerts = []
missing_masters = []
alerts = []
suspected_cls = []
pool = multiprocessing.Pool(processes=args.processes)
master_datas = pool.map(
SubProcess(cache, old_alerts, args.builder_filter, args.jobs),
master_urls)
pool.close()
pool.join()
for data in master_datas:
# TODO(ojan): We should put an alert in the JSON for this master so
# we can show that the master is down in the sheriff-o-matic UI.
if not data[0]:
missing_masters.extend([data[3]])
continue
alerts.extend(data[0])
latest_builder_info.update(data[1])
stale_builder_alerts.extend(data[2])
logging.info('Fetch took: %s seconds.',
(datetime.datetime.utcnow() - start_time).total_seconds())
alerts = gatekeeper_extras.apply_gatekeeper_rules(alerts, gatekeeper,
gatekeeper_trees)
stale_builder_alerts = gatekeeper_extras.apply_gatekeeper_rules(
stale_builder_alerts, gatekeeper, gatekeeper_trees)
alerts = analysis.assign_keys(alerts)
reason_groups = analysis.group_by_reason(alerts)
range_groups = analysis.merge_by_range(reason_groups)
if args.findit_api_url and alerts:
suspected_cls = query_findit(args.findit_api_url, alerts)
data = {
'alerts': alerts,
'suspected_cls': suspected_cls,
'reason_groups': reason_groups,
'range_groups': range_groups,
'latest_builder_info': latest_builder_info,
'stale_builder_alerts': stale_builder_alerts,
'missing_masters': missing_masters,
}
if not args.data_url:
with open('builder_alerts.json', 'w') as f:
f.write(json.dumps(data, indent=1))
ret = True
json_data = json.dumps(data)
logging.info('Alerts json is %s bytes uncompressed.', len(json_data))
s = cStringIO.StringIO()
with contextlib.closing(gzip.GzipFile(fileobj=s, mode='w')) as g:
g.write(json_data)
gzipped_data = s.getvalue()
for url in args.data_url:
logging.info('POST %s alerts (%s bytes compressed) to %s', len(alerts),
len(gzipped_data), url)
resp = requests.post(url,
data=gzipped_data,
headers={'content-encoding': 'gzip'})
try:
resp.raise_for_status()
except requests.HTTPError as e:
logging.error('POST to %s failed! %d %s, %s, %s', url,
resp.status_code, resp.reason, resp.content, e)
ret = False
return ret
def inner_loop(args):
old_api_endpoint = string_helpers.slash_join(args.api_endpoint_prefix,
args.old_api_path) if args.old_api_path else None
if not old_api_endpoint:
logging.warn(
'No /data url passed, will write to builder_alerts.json. JSON posted '
'to new API endpoints will be written to builder_alerts_<tree>.json '
'files.')
if args.use_cache:
requests_cache.install_cache('failure_stats')
else:
requests_cache.install_cache(backend='memory')
# FIXME: gatekeeper_config should find gatekeeper.json for us.
gatekeeper_path = os.path.abspath(args.gatekeeper)
logging.debug('Processsing gatekeeper json: %s', gatekeeper_path)
gatekeeper = gatekeeper_ng_config.load_gatekeeper_config(gatekeeper_path)
gatekeeper_trees_path = os.path.abspath(args.gatekeeper_trees)
logging.debug('Processing gatekeeper trees json: %s', gatekeeper_trees_path)
gatekeeper_trees = gatekeeper_ng_config.load_gatekeeper_tree_config(
gatekeeper_trees_path)
master_urls = gatekeeper_extras.fetch_master_urls(gatekeeper, args)
start_time = datetime.datetime.utcnow()
cache = buildbot.DiskCache(CACHE_PATH)
old_alerts = {}
if old_api_endpoint:
try:
old_alerts_raw = requests.get(old_api_endpoint).json()
except ValueError:
logging.debug('No old alerts found.')
else:
# internal-alerts will have a redirect instead of alerts if you're
# signed in.
if 'alerts' in old_alerts_raw:
for alert in old_alerts_raw['alerts']:
master = alert['master_url']
builder = alert['builder_name']
step = alert['step_name']
reason = alert['reason']
alert_key = alert_builder.generate_alert_key(
master, builder, step, reason)
if alert_key in old_alerts:
logging.critical(
'Incorrectly overwriting an alert reason from the'
' old alert data. master: %s, builder: %s, step: %s, reason:'
' %s' % (master, builder, step, reason))
old_alerts[alert_key] = alert
latest_builder_info = {}
stale_builder_alerts = []
missing_masters = []
alerts = []
suspected_cls = []
pool = multiprocessing.Pool(processes=args.processes)
master_datas = pool.map(SubProcess(cache, old_alerts, args.builder_filter,
args.jobs), master_urls)
pool.close()
pool.join()
for data in master_datas:
# TODO(ojan): We should put an alert in the JSON for this master so
# we can show that the master is down in the sheriff-o-matic UI.
if not data[0]:
missing_masters.extend([data[3]])
continue
alerts.extend(data[0])
latest_builder_info.update(data[1])
stale_builder_alerts.extend(data[2])
logging.info('Fetch took: %s seconds.',
(datetime.datetime.utcnow() - start_time).total_seconds())
alerts = gatekeeper_extras.apply_gatekeeper_rules(alerts, gatekeeper,
gatekeeper_trees)
stale_builder_alerts = gatekeeper_extras.apply_gatekeeper_rules(
stale_builder_alerts, gatekeeper, gatekeeper_trees)
alerts = analysis.assign_keys(alerts)
reason_groups = analysis.group_by_reason(alerts)
range_groups = analysis.merge_by_range(reason_groups)
if args.findit_api_url and alerts:
suspected_cls = query_findit(args.findit_api_url, alerts)
data = {
'alerts': alerts,
'suspected_cls': suspected_cls,
'reason_groups': reason_groups,
'range_groups': range_groups,
'latest_builder_info': latest_builder_info,
'stale_builder_alerts': stale_builder_alerts,
'missing_masters': missing_masters,
}
if not old_api_endpoint:
with open('builder_alerts.json', 'w') as f:
f.write(json.dumps(data, indent=1))
ret = True
json_data = json.dumps(data)
logging.info('Alerts json is %s bytes uncompressed.', len(json_data))
gzipped_data = gzipped(json_data)
if old_api_endpoint:
logging.info('POST %s alerts (%s bytes compressed) to %s',
len(alerts), len(gzipped_data), old_api_endpoint)
resp = requests.post(old_api_endpoint, data=gzipped_data,
headers={'content-encoding': 'gzip'})
try:
resp.raise_for_status()
except requests.HTTPError as e:
logging.error('POST to %s failed! %d %s, %s, %s', old_api_endpoint,
resp.status_code, resp.reason, resp.content, e)
ret = False
# Query sheriff issues and post them to the new API endpoint.
if args.crbug_service_account:
global issue_tracker_last_poll
seconds_since_last_poll = (
datetime.datetime.utcnow() - issue_tracker_last_poll).total_seconds()
if seconds_since_last_poll > ISSUE_TRACKER_POLLING_FREQUENCY_SEC:
issue_tracker_last_poll = datetime.datetime.utcnow()
issues_per_tree = crbug_issues.query(args.crbug_service_account,
args.use_monorail)
for tree, issues in issues_per_tree.iteritems():
json_data = {'alerts': issues}
gzipped_data = gzipped(json.dumps(json_data))
if args.api_endpoint_prefix:
new_api_endpoint = string_helpers.slash_join(
args.api_endpoint_prefix, 'api/v1/alerts', tree)
logging.info('POST %s alerts (%s bytes compressed) to %s',
len(issues), len(gzipped_data), new_api_endpoint)
resp = requests.post(new_api_endpoint, data=gzipped_data,
headers={'content-encoding': 'gzip'})
try:
resp.raise_for_status()
except requests.HTTPError:
logging.exception('POST to %s failed! %d %s, %s', new_api_endpoint,
resp.status_code, resp.reason, resp.content)
ret = False
else:
with open('builder_alerts_%s.json' % tree, 'w') as f:
f.write(json.dumps(json_data, indent=1))
else:
logging.error(
'--crbug-service-account was not specified, can not get crbug issues')
ret = False
return ret
def main():
options, args = get_options()
logging.basicConfig(
level=logging.DEBUG if options.verbose else logging.INFO)
gatekeeper_config = gatekeeper_ng_config.load_gatekeeper_config(
options.json)
if options.verify:
return 0
if options.flatten_json:
if not options.no_hashes:
gatekeeper_config = gatekeeper_ng_config.inject_hashes(
gatekeeper_config)
gatekeeper_ng_config.flatten_to_json(gatekeeper_config, sys.stdout)
print
return 0
if options.set_status:
options.password = get_pwd(options.password_file)
masters = set(args)
if not masters <= set(gatekeeper_config):
print 'The following masters are not present in the gatekeeper config:'
for m in masters - set(gatekeeper_config):
print ' ' + m
return 1
if options.clear_build_db:
build_db = {}
build_scan_db.save_build_db(build_db, gatekeeper_config,
options.build_db)
else:
build_db = build_scan_db.get_build_db(options.build_db)
master_jsons, build_jsons = build_scan.get_updated_builds(
masters, build_db, options.parallelism)
if options.sync_build_db:
build_scan_db.save_build_db(build_db, gatekeeper_config,
options.build_db)
return 0
failure_tuples = check_builds(build_jsons, master_jsons, gatekeeper_config)
# opening is an option, mostly to keep the unittests working which
# assume that any setting of status is negative.
if options.open_tree:
# failures are actually tuples, we only care about the build part.
failing_builds = [b[0] for b in failure_tuples]
open_tree_if_possible(failing_builds, options.status_user,
options.password, options.status_url,
options.set_status)
# debounce_failures does 3 things:
# 1. Groups logging by builder
# 2. Selects out the "build" part from the failure tuple.
# 3. Rejects builds we've already warned about (and logs).
new_failures = debounce_failures(failure_tuples, build_db)
close_tree_if_necessary(new_failures, options.status_user,
options.password, options.status_url,
options.set_status)
notify_failures(new_failures, options.sheriff_url,
options.default_from_email, options.email_app_url,
options.email_app_secret, options.email_domain,
options.filter_domain, options.disable_domain_filter)
if not options.skip_build_db_update:
build_scan_db.save_build_db(build_db, gatekeeper_config,
options.build_db)
return 0
def load_config(self):
with open(self.fname, 'w') as cfg_file:
cfg_file.write(json.dumps(self.cfg))
return gatekeeper_ng_config.load_gatekeeper_config(self.fname)
def main():
options, args = get_options()
logging.basicConfig(level=logging.DEBUG if options.verbose else logging.INFO)
gatekeeper_config = gatekeeper_ng_config.load_gatekeeper_config(options.json)
if options.verify:
return 0
if options.flatten_json:
if not options.no_hashes:
gatekeeper_config = gatekeeper_ng_config.inject_hashes(gatekeeper_config)
gatekeeper_ng_config.flatten_to_json(gatekeeper_config, sys.stdout)
print
return 0
if options.set_status:
options.password = get_pwd(options.password_file)
masters = set(args)
if not masters <= set(gatekeeper_config):
print 'The following masters are not present in the gatekeeper config:'
for m in masters - set(gatekeeper_config):
print ' ' + m
return 1
if options.clear_build_db:
build_db = {}
build_scan_db.save_build_db(build_db, gatekeeper_config,
options.build_db)
else:
build_db = build_scan_db.get_build_db(options.build_db)
master_jsons, build_jsons = build_scan.get_updated_builds(
masters, build_db, options.parallelism)
if options.sync_build_db:
build_scan_db.save_build_db(build_db, gatekeeper_config,
options.build_db)
return 0
failure_tuples = check_builds(build_jsons, master_jsons, gatekeeper_config)
# opening is an option, mostly to keep the unittests working which
# assume that any setting of status is negative.
if options.open_tree:
# failures are actually tuples, we only care about the build part.
failing_builds = [b[0] for b in failure_tuples]
open_tree_if_possible(failing_builds, options.status_user, options.password,
options.status_url, options.set_status)
# debounce_failures does 3 things:
# 1. Groups logging by builder
# 2. Selects out the "build" part from the failure tuple.
# 3. Rejects builds we've already warned about (and logs).
new_failures = debounce_failures(failure_tuples, build_db)
close_tree_if_necessary(new_failures, options.status_user, options.password,
options.status_url, options.set_status)
notify_failures(new_failures, options.sheriff_url, options.default_from_email,
options.email_app_url, options.email_app_secret,
options.email_domain, options.filter_domain,
options.disable_domain_filter)
if not options.skip_build_db_update:
build_scan_db.save_build_db(build_db, gatekeeper_config,
options.build_db)
return 0
def main():
options, args = get_options()
logging.basicConfig(level=logging.DEBUG if options.verbose else logging.INFO)
gatekeeper_config = gatekeeper_ng_config.load_gatekeeper_config(options.json)
if options.verify:
return 0
if options.flatten_json:
if not options.no_hashes:
gatekeeper_config = gatekeeper_ng_config.inject_hashes(gatekeeper_config)
gatekeeper_ng_config.flatten_to_json(gatekeeper_config, sys.stdout)
print
return 0
if options.set_status:
options.password = get_pwd(options.password_file)
masters = set(args)
if not masters <= set(gatekeeper_config):
print 'The following masters are not present in the gatekeeper config:'
for m in masters - set(gatekeeper_config):
print ' ' + m
return 1
emoji = []
if options.emoji != 'None':
try:
with open(options.emoji) as f:
emoji = json.load(f)
except (IOError, ValueError) as e:
logging.warning('Could not load emoji file %s: %s', options.emoji, e)
if options.clear_build_db:
build_db = {}
build_scan_db.save_build_db(build_db, gatekeeper_config,
options.build_db)
else:
build_db = build_scan_db.get_build_db(options.build_db)
master_jsons, build_jsons = build_scan.get_updated_builds(
masters, build_db, options.parallelism)
if options.sync_build_db:
build_scan_db.save_build_db(build_db, gatekeeper_config,
options.build_db)
return 0
(failure_tuples, success_tuples, successful_builder_steps,
current_builds_successful) = check_builds(
build_jsons, master_jsons, gatekeeper_config)
# Write failure / success information back to the build_db.
propagate_build_status_back_to_db(failure_tuples, success_tuples, build_db)
# opening is an option, mostly to keep the unittests working which
# assume that any setting of status is negative.
if options.open_tree:
open_tree_if_possible(build_db, master_jsons, successful_builder_steps,
current_builds_successful, options.status_user, options.password,
options.status_url, options.set_status, emoji)
# debounce_failures does 3 things:
# 1. Groups logging by builder
# 2. Selects out the "build" part from the failure tuple.
# 3. Rejects builds we've already warned about (and logs).
new_failures = debounce_failures(failure_tuples,
current_builds_successful, build_db)
if options.track_revisions:
# Only close the tree if it's a newer revision than before.
properties = options.revision_properties.split(',')
triggered_revisions = build_db.aux.get('triggered_revisions', {})
if not triggered_revisions or (
sorted(triggered_revisions) != sorted(properties)):
logging.info('revision properties have changed from %s to %s. '
'clearing previous data.', triggered_revisions, properties)
build_db.aux['triggered_revisions'] = dict.fromkeys(properties)
new_failures = reject_old_revisions(new_failures, build_db)
close_tree_if_necessary(new_failures, options.status_user, options.password,
options.status_url, options.set_status,
options.revision_properties.split(','))
notify_failures(new_failures, options.sheriff_url, options.default_from_email,
options.email_app_url, options.email_app_secret,
options.email_domain, options.filter_domain,
options.disable_domain_filter)
if not options.skip_build_db_update:
build_scan_db.save_build_db(build_db, gatekeeper_config,
options.build_db)
return 0
def main():
options, args = get_options()
logging.basicConfig(
level=logging.DEBUG if options.verbose else logging.INFO)
gatekeeper_config = gatekeeper_ng_config.load_gatekeeper_config(
options.json)
if options.verify:
return 0
if options.flatten_json:
if not options.no_hashes:
gatekeeper_config = gatekeeper_ng_config.inject_hashes(
gatekeeper_config)
gatekeeper_ng_config.flatten_to_json(gatekeeper_config, sys.stdout)
print
return 0
if options.set_status:
options.password = get_pwd(options.password_file)
masters = set(args)
if not masters <= set(gatekeeper_config):
print 'The following masters are not present in the gatekeeper config:'
for m in masters - set(gatekeeper_config):
print ' ' + m
return 1
emoji = []
if options.emoji != 'None':
try:
with open(options.emoji) as f:
emoji = json.load(f)
except (IOError, ValueError) as e:
logging.warning('Could not load emoji file %s: %s', options.emoji,
e)
if options.clear_build_db:
build_db = {}
build_scan_db.save_build_db(build_db, gatekeeper_config,
options.build_db)
else:
build_db = build_scan_db.get_build_db(options.build_db)
master_jsons, build_jsons = build_scan.get_updated_builds(
masters, build_db, options.parallelism)
if options.sync_build_db:
build_scan_db.save_build_db(build_db, gatekeeper_config,
options.build_db)
return 0
(failure_tuples, success_tuples, successful_builder_steps,
current_builds_successful) = check_builds(build_jsons, master_jsons,
gatekeeper_config)
# Write failure / success information back to the build_db.
propagate_build_status_back_to_db(failure_tuples, success_tuples, build_db)
# opening is an option, mostly to keep the unittests working which
# assume that any setting of status is negative.
if options.open_tree:
open_tree_if_possible(build_db, master_jsons, successful_builder_steps,
current_builds_successful, options.status_user,
options.password, options.status_url,
options.set_status, emoji)
# debounce_failures does 3 things:
# 1. Groups logging by builder
# 2. Selects out the "build" part from the failure tuple.
# 3. Rejects builds we've already warned about (and logs).
new_failures = debounce_failures(failure_tuples, current_builds_successful,
build_db)
if options.track_revisions:
# Only close the tree if it's a newer revision than before.
properties = options.revision_properties.split(',')
triggered_revisions = build_db.aux.get('triggered_revisions', {})
if not triggered_revisions or (sorted(triggered_revisions) !=
sorted(properties)):
logging.info(
'revision properties have changed from %s to %s. '
'clearing previous data.', triggered_revisions, properties)
build_db.aux['triggered_revisions'] = dict.fromkeys(properties)
new_failures = reject_old_revisions(new_failures, build_db)
close_tree_if_necessary(new_failures, options.status_user,
options.password, options.status_url,
options.set_status,
options.revision_properties.split(','))
notify_failures(new_failures, options.sheriff_url,
options.default_from_email, options.email_app_url,
options.email_app_secret, options.email_domain,
options.filter_domain, options.disable_domain_filter)
if not options.skip_build_db_update:
build_scan_db.save_build_db(build_db, gatekeeper_config,
options.build_db)
return 0
def load_config(self):
with open(self.fname, 'w') as cfg_file:
cfg_file.write(json.dumps(self.cfg))
return gatekeeper_ng_config.load_gatekeeper_config(self.fname)
def inner_loop(args):
if not args.data_url:
logging.warn('No /data url passed, will write to builder_alerts.json')
if args.use_cache:
requests_cache.install_cache('failure_stats')
else:
requests_cache.install_cache(backend='memory')
# FIXME: gatekeeper_config should find gatekeeper.json for us.
gatekeeper_path = os.path.abspath(args.gatekeeper)
logging.debug('Processsing gatekeeper json: %s', gatekeeper_path)
gatekeeper = gatekeeper_ng_config.load_gatekeeper_config(gatekeeper_path)
gatekeeper_trees_path = os.path.abspath(args.gatekeeper_trees)
logging.debug('Processing gatekeeper trees json: %s', gatekeeper_trees_path)
gatekeeper_trees = gatekeeper_ng_config.load_gatekeeper_tree_config(
gatekeeper_trees_path)
master_urls = gatekeeper_extras.fetch_master_urls(gatekeeper, args)
start_time = datetime.datetime.utcnow()
cache = buildbot.DiskCache(CACHE_PATH)
old_alerts = {}
if args.data_url:
try:
old_alerts_raw = requests.get(args.data_url[0]).json()
except ValueError:
logging.debug('No old alerts found.')
else:
# internal-alerts will have a redirect instead of alerts if you're
# signed in.
if 'alerts' in old_alerts_raw:
for alert in old_alerts_raw['alerts']:
master = alert['master_url']
builder = alert['builder_name']
step = alert['step_name']
reason = alert['reason']
alert_key = alert_builder.generate_alert_key(
master, builder, step, reason)
if alert_key in old_alerts:
logging.critical(
'Incorrectly overwriting an alert reason from the'
' old alert data. master: %s, builder: %s, step: %s, reason:'
' %s' % (master, builder, step, reason))
old_alerts[alert_key] = alert
latest_builder_info = {}
stale_builder_alerts = []
missing_masters = []
alerts = []
suspected_cls = []
pool = multiprocessing.Pool(processes=args.processes)
master_datas = pool.map(SubProcess(cache, old_alerts, args.builder_filter,
args.jobs), master_urls)
pool.close()
pool.join()
for data in master_datas:
# TODO(ojan): We should put an alert in the JSON for this master so
# we can show that the master is down in the sheriff-o-matic UI.
if not data[0]:
missing_masters.extend([data[3]])
continue
alerts.extend(data[0])
latest_builder_info.update(data[1])
stale_builder_alerts.extend(data[2])
logging.info('Fetch took: %s seconds.',
(datetime.datetime.utcnow() - start_time).total_seconds())
alerts = gatekeeper_extras.apply_gatekeeper_rules(alerts, gatekeeper,
gatekeeper_trees)
stale_builder_alerts = gatekeeper_extras.apply_gatekeeper_rules(
stale_builder_alerts, gatekeeper, gatekeeper_trees)
alerts = analysis.assign_keys(alerts)
reason_groups = analysis.group_by_reason(alerts)
range_groups = analysis.merge_by_range(reason_groups)
if args.findit_api_url and alerts:
suspected_cls = query_findit(args.findit_api_url, alerts)
data = {
'alerts': alerts,
'suspected_cls': suspected_cls,
'reason_groups': reason_groups,
'range_groups': range_groups,
'latest_builder_info': latest_builder_info,
'stale_builder_alerts': stale_builder_alerts,
'missing_masters': missing_masters,
}
if not args.data_url:
with open('builder_alerts.json', 'w') as f:
f.write(json.dumps(data, indent=1))
ret = True
json_data = json.dumps(data)
logging.info('Alerts json is %s bytes uncompressed.', len(json_data))
s = cStringIO.StringIO()
with contextlib.closing(gzip.GzipFile(fileobj=s, mode='w')) as g:
g.write(json_data)
gzipped_data = s.getvalue()
for url in args.data_url:
logging.info('POST %s alerts (%s bytes compressed) to %s',
len(alerts), len(gzipped_data), url)
resp = requests.post(url, data=gzipped_data,
headers={'content-encoding': 'gzip'})
try:
resp.raise_for_status()
except requests.HTTPError as e:
logging.error('POST to %s failed! %d %s, %s, %s',
url, resp.status_code, resp.reason, resp.content, e)
ret = False
return ret
