def main(args): parser = argparse.ArgumentParser() parser.add_argument('--use-cache', action='store_true') parser.add_argument('--builders', action='store_true') parser.add_argument('--master-filter', action='store') parser.add_argument('--noise-threshold', action='store', type=int, default=2) parser.add_argument('--build-limit', action='store', type=int, default=100) parser.add_argument('--show-pass', action='store_true') args = parser.parse_args(args) if args.use_cache: requests_cache.install_cache('failure_stats') gatekeeper_config = gatekeeper_ng_config.load_gatekeeper_config(CONFIG_PATH) all_outcomes = collections.Counter() for master_url, master_config in gatekeeper_config.items(): master_outcomes = collections.Counter() master_name = urlparse.urlparse(master_url).path.split('/')[-1] if args.master_filter: if args.master_filter not in master_name: continue if args.builders: print print master_name common_config = master_config[0].get('*', {}) excluded_builders = common_config.get('excluded_builders', set()) builder_names = fetch_builder_names(master_name) builder_names = sorted(set(builder_names) - excluded_builders) for builder_name in builder_names: outcomes = collections.Counter() recent_builds = builds_for_builder(master_name, builder_name) for build in recent_builds[:args.build_limit]: if build.get('results', 0) == 0: continue failing_results = [step['results'][1] for step in build['steps'] if step['results'][0]] failure_name = pretty_failure_name(failing_results) outcomes[failure_name] += 1 if args.builders: fail_strings = ['{:2} {:<30}'.format(*reversed(tup)) for tup in outcomes.most_common(3) if tup[1] >= args.noise_threshold] if not outcomes.most_common(3): if args.show_pass: print '%30s : PASS' % builder_name elif outcomes.most_common(1)[0][1] < args.noise_threshold: print '%30s : noise' % builder_name else: print '%30s : %s' % (builder_name, ' | '.join(fail_strings)) master_outcomes += outcomes print_worst_failures(master_name, master_outcomes) all_outcomes += master_outcomes if not args.master_filter: print_worst_failures('total for all %s masters' % CONFIG_PATH, all_outcomes)
def main(args): parser = argparse.ArgumentParser() parser.add_argument('data_url', action='store', nargs='*') parser.add_argument('--use-cache', action='store_true') parser.add_argument('--master-filter', action='store') args = parser.parse_args(args) if not args.data_url: log.warn("No /data url passed, won't do anything") if args.use_cache: requests_cache.install_cache('failure_stats') else: requests_cache.install_cache(backend='memory') gatekeeper = gatekeeper_ng_config.load_gatekeeper_config(CONFIG_PATH) master_urls = fetch_master_urls(gatekeeper, args) start_time = datetime.datetime.now() latest_revisions = {} cache = buildbot.BuildCache(CACHE_PATH) alerts = [] for master_url in master_urls: master_json = buildbot.fetch_master_json(master_url) master_alerts = alert_builder.alerts_for_master(cache, master_url, master_json) alerts.extend(master_alerts) # FIXME: This doesn't really belong here. garden-o-matic wants # this data and we happen to have the builder json cached at # this point so it's cheap to compute. revisions = buildbot.latest_revisions_for_master(cache, master_url, master_json) latest_revisions.update(revisions) print "Fetch took: %s" % (datetime.datetime.now() - start_time) alerts = apply_gatekeeper_rules(alerts, gatekeeper) alerts = analysis.assign_keys(alerts) reason_groups = analysis.group_by_reason(alerts) range_groups = analysis.merge_by_range(reason_groups) data = { 'content': json.dumps({ 'alerts': alerts, 'reason_groups': reason_groups, 'range_groups': range_groups, 'latest_revisions': latest_revisions, })} for url in args.data_url: log.info('POST %s alerts to %s' % (len(alerts), url)) requests.post(url, data=data)
def main(): prog_desc = 'Parses the build_db and outputs to stdout.' usage = '%prog [options]' parser = optparse.OptionParser(usage=(usage + '\n\n' + prog_desc)) parser.add_option('--json', default=os.path.join(DATA_DIR, 'gatekeeper.json'), help='location of gatekeeper configuration file') parser.add_option('--build-db', default='build_db.json', help='records the build status information for builders') options, _ = parser.parse_args() build_db = get_build_db(options.build_db) gatekeeper_config = gatekeeper_ng_config.load_gatekeeper_config(options.json) convert_db_to_json(build_db, gatekeeper_config, sys.stdout) print return 0
def main(argv): args = get_args(argv) logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO) gatekeeper_config = gatekeeper_ng_config.load_gatekeeper_config(args.json) if args.verify: return 0 simulate = bool(args.simulate_master) if args.flatten_json: if not args.no_hashes: gatekeeper_config = gatekeeper_ng_config.inject_hashes( gatekeeper_config) gatekeeper_ng_config.flatten_to_json(gatekeeper_config, sys.stdout) print return 0 if args.set_status and not simulate: args.password = get_pwd(args.password_file) masters = defaultdict(set) for m in args.master_url: if m.count(':') > 1: # Master in master_url:builder,builder format. http, mname, builderlist = m.split(':', 2) mastername = ':'.join([http, mname]) masters[mastername].update(builderlist.split(',')) else: # Regular master URL, just add '*'. masters[m].add(build_scan.BUILDER_WILDCARD) if not set(masters) <= set(gatekeeper_config): print 'The following masters are not present in the gatekeeper config:' for m in set(masters) - set(gatekeeper_config): print ' ' + m return 1 emoji = [] if args.emoji != 'None': try: with open(args.emoji) as f: emoji = json.load(f) except (IOError, ValueError) as e: logging.warning('Could not load emoji file %s: %s', args.emoji, e) if args.clear_build_db: build_db = build_scan_db.gen_db() build_scan_db.save_build_db(build_db, gatekeeper_config, args.build_db) else: build_db = build_scan_db.get_build_db(args.build_db) if not simulate: master_jsons, build_jsons = build_scan.get_updated_builds( masters, build_db, args.parallelism, args.milo_creds) else: master_jsons, build_jsons = simulate_build_failure( build_db, args.simulate_master, args.simulate_builder, *args.simulate_step) if args.sync_build_db: build_scan_db.save_build_db(build_db, gatekeeper_config, args.build_db) return 0 (failure_tuples, success_tuples, successful_builder_steps, current_builds_successful) = check_builds(build_jsons, master_jsons, gatekeeper_config) # Write failure / success information back to the build_db. propagate_build_status_back_to_db(failure_tuples, success_tuples, build_db) # opening is an option, mostly to keep the unittests working which # assume that any setting of status is negative. if args.open_tree: open_tree_if_possible(build_db, master_jsons, successful_builder_steps, current_builds_successful, args.status_user, args.password, args.status_url, args.set_status, emoji, simulate) # debounce_failures does 3 things: # 1. Groups logging by builder # 2. Selects out the "build" part from the failure tuple. # 3. Rejects builds we've already warned about (and logs). new_failures = debounce_failures(failure_tuples, current_builds_successful, build_db) if args.track_revisions: # Only close the tree if it's a newer revision than before. properties = args.revision_properties.split(',') triggered_revisions = build_db.aux.get('triggered_revisions', {}) if not triggered_revisions or (sorted(triggered_revisions) != sorted(properties)): logging.info( 'revision properties have changed from %s to %s. ' 'clearing previous data.', triggered_revisions, properties) build_db.aux['triggered_revisions'] = dict.fromkeys(properties) new_failures = reject_old_revisions(new_failures, build_db) close_tree_if_necessary(build_db, new_failures, args.status_user, args.password, args.status_url, args.set_status, args.revision_properties.split(','), simulate) try: notify_failures(new_failures, args.sheriff_url, args.default_from_email, args.email_app_url, args.email_app_secret, args.email_domain, args.filter_domain, args.disable_domain_filter, simulate) finally: if not args.skip_build_db_update and not simulate: build_scan_db.save_build_db(build_db, gatekeeper_config, args.build_db) return 0
def inner_loop(args): if not args.data_url: logging.warn('No /data url passed, will write to builder_alerts.json') if args.use_cache: requests_cache.install_cache('failure_stats') else: requests_cache.install_cache(backend='memory') # FIXME: gatekeeper_config should find gatekeeper.json for us. gatekeeper_path = os.path.abspath(args.gatekeeper) logging.debug('Processsing gatekeeper json: %s', gatekeeper_path) gatekeeper = gatekeeper_ng_config.load_gatekeeper_config(gatekeeper_path) gatekeeper_trees_path = os.path.abspath(args.gatekeeper_trees) logging.debug('Processing gatekeeper trees json: %s', gatekeeper_trees_path) gatekeeper_trees = gatekeeper_ng_config.load_gatekeeper_tree_config( gatekeeper_trees_path) master_urls = gatekeeper_extras.fetch_master_urls(gatekeeper, args) start_time = datetime.datetime.utcnow() cache = buildbot.DiskCache(CACHE_PATH) old_alerts = {} if args.data_url: try: old_alerts_raw = requests.get(args.data_url[0]).json() except ValueError: logging.debug('No old alerts found.') else: # internal-alerts will have a redirect instead of alerts if you're # signed in. if 'alerts' in old_alerts_raw: for alert in old_alerts_raw['alerts']: master = alert['master_url'] builder = alert['builder_name'] step = alert['step_name'] reason = alert['reason'] alert_key = alert_builder.generate_alert_key( master, builder, step, reason) if alert_key in old_alerts: logging.critical( 'Incorrectly overwriting an alert reason from the' ' old alert data. master: %s, builder: %s, step: %s, reason:' ' %s' % (master, builder, step, reason)) old_alerts[alert_key] = alert latest_builder_info = {} stale_builder_alerts = [] missing_masters = [] alerts = [] suspected_cls = [] pool = multiprocessing.Pool(processes=args.processes) master_datas = pool.map( SubProcess(cache, old_alerts, args.builder_filter, args.jobs), master_urls) pool.close() pool.join() for data in master_datas: # TODO(ojan): We should put an alert in the JSON for this master so # we can show that the master is down in the sheriff-o-matic UI. if not data[0]: missing_masters.extend([data[3]]) continue alerts.extend(data[0]) latest_builder_info.update(data[1]) stale_builder_alerts.extend(data[2]) logging.info('Fetch took: %s seconds.', (datetime.datetime.utcnow() - start_time).total_seconds()) alerts = gatekeeper_extras.apply_gatekeeper_rules(alerts, gatekeeper, gatekeeper_trees) stale_builder_alerts = gatekeeper_extras.apply_gatekeeper_rules( stale_builder_alerts, gatekeeper, gatekeeper_trees) alerts = analysis.assign_keys(alerts) reason_groups = analysis.group_by_reason(alerts) range_groups = analysis.merge_by_range(reason_groups) if args.findit_api_url and alerts: suspected_cls = query_findit(args.findit_api_url, alerts) data = { 'alerts': alerts, 'suspected_cls': suspected_cls, 'reason_groups': reason_groups, 'range_groups': range_groups, 'latest_builder_info': latest_builder_info, 'stale_builder_alerts': stale_builder_alerts, 'missing_masters': missing_masters, } if not args.data_url: with open('builder_alerts.json', 'w') as f: f.write(json.dumps(data, indent=1)) ret = True json_data = json.dumps(data) logging.info('Alerts json is %s bytes uncompressed.', len(json_data)) s = cStringIO.StringIO() with contextlib.closing(gzip.GzipFile(fileobj=s, mode='w')) as g: g.write(json_data) gzipped_data = s.getvalue() for url in args.data_url: logging.info('POST %s alerts (%s bytes compressed) to %s', len(alerts), len(gzipped_data), url) resp = requests.post(url, data=gzipped_data, headers={'content-encoding': 'gzip'}) try: resp.raise_for_status() except requests.HTTPError as e: logging.error('POST to %s failed! %d %s, %s, %s', url, resp.status_code, resp.reason, resp.content, e) ret = False return ret
def inner_loop(args): old_api_endpoint = string_helpers.slash_join(args.api_endpoint_prefix, args.old_api_path) if args.old_api_path else None if not old_api_endpoint: logging.warn( 'No /data url passed, will write to builder_alerts.json. JSON posted ' 'to new API endpoints will be written to builder_alerts_<tree>.json ' 'files.') if args.use_cache: requests_cache.install_cache('failure_stats') else: requests_cache.install_cache(backend='memory') # FIXME: gatekeeper_config should find gatekeeper.json for us. gatekeeper_path = os.path.abspath(args.gatekeeper) logging.debug('Processsing gatekeeper json: %s', gatekeeper_path) gatekeeper = gatekeeper_ng_config.load_gatekeeper_config(gatekeeper_path) gatekeeper_trees_path = os.path.abspath(args.gatekeeper_trees) logging.debug('Processing gatekeeper trees json: %s', gatekeeper_trees_path) gatekeeper_trees = gatekeeper_ng_config.load_gatekeeper_tree_config( gatekeeper_trees_path) master_urls = gatekeeper_extras.fetch_master_urls(gatekeeper, args) start_time = datetime.datetime.utcnow() cache = buildbot.DiskCache(CACHE_PATH) old_alerts = {} if old_api_endpoint: try: old_alerts_raw = requests.get(old_api_endpoint).json() except ValueError: logging.debug('No old alerts found.') else: # internal-alerts will have a redirect instead of alerts if you're # signed in. if 'alerts' in old_alerts_raw: for alert in old_alerts_raw['alerts']: master = alert['master_url'] builder = alert['builder_name'] step = alert['step_name'] reason = alert['reason'] alert_key = alert_builder.generate_alert_key( master, builder, step, reason) if alert_key in old_alerts: logging.critical( 'Incorrectly overwriting an alert reason from the' ' old alert data. master: %s, builder: %s, step: %s, reason:' ' %s' % (master, builder, step, reason)) old_alerts[alert_key] = alert latest_builder_info = {} stale_builder_alerts = [] missing_masters = [] alerts = [] suspected_cls = [] pool = multiprocessing.Pool(processes=args.processes) master_datas = pool.map(SubProcess(cache, old_alerts, args.builder_filter, args.jobs), master_urls) pool.close() pool.join() for data in master_datas: # TODO(ojan): We should put an alert in the JSON for this master so # we can show that the master is down in the sheriff-o-matic UI. if not data[0]: missing_masters.extend([data[3]]) continue alerts.extend(data[0]) latest_builder_info.update(data[1]) stale_builder_alerts.extend(data[2]) logging.info('Fetch took: %s seconds.', (datetime.datetime.utcnow() - start_time).total_seconds()) alerts = gatekeeper_extras.apply_gatekeeper_rules(alerts, gatekeeper, gatekeeper_trees) stale_builder_alerts = gatekeeper_extras.apply_gatekeeper_rules( stale_builder_alerts, gatekeeper, gatekeeper_trees) alerts = analysis.assign_keys(alerts) reason_groups = analysis.group_by_reason(alerts) range_groups = analysis.merge_by_range(reason_groups) if args.findit_api_url and alerts: suspected_cls = query_findit(args.findit_api_url, alerts) data = { 'alerts': alerts, 'suspected_cls': suspected_cls, 'reason_groups': reason_groups, 'range_groups': range_groups, 'latest_builder_info': latest_builder_info, 'stale_builder_alerts': stale_builder_alerts, 'missing_masters': missing_masters, } if not old_api_endpoint: with open('builder_alerts.json', 'w') as f: f.write(json.dumps(data, indent=1)) ret = True json_data = json.dumps(data) logging.info('Alerts json is %s bytes uncompressed.', len(json_data)) gzipped_data = gzipped(json_data) if old_api_endpoint: logging.info('POST %s alerts (%s bytes compressed) to %s', len(alerts), len(gzipped_data), old_api_endpoint) resp = requests.post(old_api_endpoint, data=gzipped_data, headers={'content-encoding': 'gzip'}) try: resp.raise_for_status() except requests.HTTPError as e: logging.error('POST to %s failed! %d %s, %s, %s', old_api_endpoint, resp.status_code, resp.reason, resp.content, e) ret = False # Query sheriff issues and post them to the new API endpoint. if args.crbug_service_account: global issue_tracker_last_poll seconds_since_last_poll = ( datetime.datetime.utcnow() - issue_tracker_last_poll).total_seconds() if seconds_since_last_poll > ISSUE_TRACKER_POLLING_FREQUENCY_SEC: issue_tracker_last_poll = datetime.datetime.utcnow() issues_per_tree = crbug_issues.query(args.crbug_service_account, args.use_monorail) for tree, issues in issues_per_tree.iteritems(): json_data = {'alerts': issues} gzipped_data = gzipped(json.dumps(json_data)) if args.api_endpoint_prefix: new_api_endpoint = string_helpers.slash_join( args.api_endpoint_prefix, 'api/v1/alerts', tree) logging.info('POST %s alerts (%s bytes compressed) to %s', len(issues), len(gzipped_data), new_api_endpoint) resp = requests.post(new_api_endpoint, data=gzipped_data, headers={'content-encoding': 'gzip'}) try: resp.raise_for_status() except requests.HTTPError: logging.exception('POST to %s failed! %d %s, %s', new_api_endpoint, resp.status_code, resp.reason, resp.content) ret = False else: with open('builder_alerts_%s.json' % tree, 'w') as f: f.write(json.dumps(json_data, indent=1)) else: logging.error( '--crbug-service-account was not specified, can not get crbug issues') ret = False return ret
def main(): options, args = get_options() logging.basicConfig( level=logging.DEBUG if options.verbose else logging.INFO) gatekeeper_config = gatekeeper_ng_config.load_gatekeeper_config( options.json) if options.verify: return 0 if options.flatten_json: if not options.no_hashes: gatekeeper_config = gatekeeper_ng_config.inject_hashes( gatekeeper_config) gatekeeper_ng_config.flatten_to_json(gatekeeper_config, sys.stdout) print return 0 if options.set_status: options.password = get_pwd(options.password_file) masters = set(args) if not masters <= set(gatekeeper_config): print 'The following masters are not present in the gatekeeper config:' for m in masters - set(gatekeeper_config): print ' ' + m return 1 if options.clear_build_db: build_db = {} build_scan_db.save_build_db(build_db, gatekeeper_config, options.build_db) else: build_db = build_scan_db.get_build_db(options.build_db) master_jsons, build_jsons = build_scan.get_updated_builds( masters, build_db, options.parallelism) if options.sync_build_db: build_scan_db.save_build_db(build_db, gatekeeper_config, options.build_db) return 0 failure_tuples = check_builds(build_jsons, master_jsons, gatekeeper_config) # opening is an option, mostly to keep the unittests working which # assume that any setting of status is negative. if options.open_tree: # failures are actually tuples, we only care about the build part. failing_builds = [b[0] for b in failure_tuples] open_tree_if_possible(failing_builds, options.status_user, options.password, options.status_url, options.set_status) # debounce_failures does 3 things: # 1. Groups logging by builder # 2. Selects out the "build" part from the failure tuple. # 3. Rejects builds we've already warned about (and logs). new_failures = debounce_failures(failure_tuples, build_db) close_tree_if_necessary(new_failures, options.status_user, options.password, options.status_url, options.set_status) notify_failures(new_failures, options.sheriff_url, options.default_from_email, options.email_app_url, options.email_app_secret, options.email_domain, options.filter_domain, options.disable_domain_filter) if not options.skip_build_db_update: build_scan_db.save_build_db(build_db, gatekeeper_config, options.build_db) return 0
def load_config(self): with open(self.fname, 'w') as cfg_file: cfg_file.write(json.dumps(self.cfg)) return gatekeeper_ng_config.load_gatekeeper_config(self.fname)
def main(): options, args = get_options() logging.basicConfig(level=logging.DEBUG if options.verbose else logging.INFO) gatekeeper_config = gatekeeper_ng_config.load_gatekeeper_config(options.json) if options.verify: return 0 if options.flatten_json: if not options.no_hashes: gatekeeper_config = gatekeeper_ng_config.inject_hashes(gatekeeper_config) gatekeeper_ng_config.flatten_to_json(gatekeeper_config, sys.stdout) print return 0 if options.set_status: options.password = get_pwd(options.password_file) masters = set(args) if not masters <= set(gatekeeper_config): print 'The following masters are not present in the gatekeeper config:' for m in masters - set(gatekeeper_config): print ' ' + m return 1 if options.clear_build_db: build_db = {} build_scan_db.save_build_db(build_db, gatekeeper_config, options.build_db) else: build_db = build_scan_db.get_build_db(options.build_db) master_jsons, build_jsons = build_scan.get_updated_builds( masters, build_db, options.parallelism) if options.sync_build_db: build_scan_db.save_build_db(build_db, gatekeeper_config, options.build_db) return 0 failure_tuples = check_builds(build_jsons, master_jsons, gatekeeper_config) # opening is an option, mostly to keep the unittests working which # assume that any setting of status is negative. if options.open_tree: # failures are actually tuples, we only care about the build part. failing_builds = [b[0] for b in failure_tuples] open_tree_if_possible(failing_builds, options.status_user, options.password, options.status_url, options.set_status) # debounce_failures does 3 things: # 1. Groups logging by builder # 2. Selects out the "build" part from the failure tuple. # 3. Rejects builds we've already warned about (and logs). new_failures = debounce_failures(failure_tuples, build_db) close_tree_if_necessary(new_failures, options.status_user, options.password, options.status_url, options.set_status) notify_failures(new_failures, options.sheriff_url, options.default_from_email, options.email_app_url, options.email_app_secret, options.email_domain, options.filter_domain, options.disable_domain_filter) if not options.skip_build_db_update: build_scan_db.save_build_db(build_db, gatekeeper_config, options.build_db) return 0
def main(): options, args = get_options() logging.basicConfig(level=logging.DEBUG if options.verbose else logging.INFO) gatekeeper_config = gatekeeper_ng_config.load_gatekeeper_config(options.json) if options.verify: return 0 if options.flatten_json: if not options.no_hashes: gatekeeper_config = gatekeeper_ng_config.inject_hashes(gatekeeper_config) gatekeeper_ng_config.flatten_to_json(gatekeeper_config, sys.stdout) print return 0 if options.set_status: options.password = get_pwd(options.password_file) masters = set(args) if not masters <= set(gatekeeper_config): print 'The following masters are not present in the gatekeeper config:' for m in masters - set(gatekeeper_config): print ' ' + m return 1 emoji = [] if options.emoji != 'None': try: with open(options.emoji) as f: emoji = json.load(f) except (IOError, ValueError) as e: logging.warning('Could not load emoji file %s: %s', options.emoji, e) if options.clear_build_db: build_db = {} build_scan_db.save_build_db(build_db, gatekeeper_config, options.build_db) else: build_db = build_scan_db.get_build_db(options.build_db) master_jsons, build_jsons = build_scan.get_updated_builds( masters, build_db, options.parallelism) if options.sync_build_db: build_scan_db.save_build_db(build_db, gatekeeper_config, options.build_db) return 0 (failure_tuples, success_tuples, successful_builder_steps, current_builds_successful) = check_builds( build_jsons, master_jsons, gatekeeper_config) # Write failure / success information back to the build_db. propagate_build_status_back_to_db(failure_tuples, success_tuples, build_db) # opening is an option, mostly to keep the unittests working which # assume that any setting of status is negative. if options.open_tree: open_tree_if_possible(build_db, master_jsons, successful_builder_steps, current_builds_successful, options.status_user, options.password, options.status_url, options.set_status, emoji) # debounce_failures does 3 things: # 1. Groups logging by builder # 2. Selects out the "build" part from the failure tuple. # 3. Rejects builds we've already warned about (and logs). new_failures = debounce_failures(failure_tuples, current_builds_successful, build_db) if options.track_revisions: # Only close the tree if it's a newer revision than before. properties = options.revision_properties.split(',') triggered_revisions = build_db.aux.get('triggered_revisions', {}) if not triggered_revisions or ( sorted(triggered_revisions) != sorted(properties)): logging.info('revision properties have changed from %s to %s. ' 'clearing previous data.', triggered_revisions, properties) build_db.aux['triggered_revisions'] = dict.fromkeys(properties) new_failures = reject_old_revisions(new_failures, build_db) close_tree_if_necessary(new_failures, options.status_user, options.password, options.status_url, options.set_status, options.revision_properties.split(',')) notify_failures(new_failures, options.sheriff_url, options.default_from_email, options.email_app_url, options.email_app_secret, options.email_domain, options.filter_domain, options.disable_domain_filter) if not options.skip_build_db_update: build_scan_db.save_build_db(build_db, gatekeeper_config, options.build_db) return 0
def main(): options, args = get_options() logging.basicConfig( level=logging.DEBUG if options.verbose else logging.INFO) gatekeeper_config = gatekeeper_ng_config.load_gatekeeper_config( options.json) if options.verify: return 0 if options.flatten_json: if not options.no_hashes: gatekeeper_config = gatekeeper_ng_config.inject_hashes( gatekeeper_config) gatekeeper_ng_config.flatten_to_json(gatekeeper_config, sys.stdout) print return 0 if options.set_status: options.password = get_pwd(options.password_file) masters = set(args) if not masters <= set(gatekeeper_config): print 'The following masters are not present in the gatekeeper config:' for m in masters - set(gatekeeper_config): print ' ' + m return 1 emoji = [] if options.emoji != 'None': try: with open(options.emoji) as f: emoji = json.load(f) except (IOError, ValueError) as e: logging.warning('Could not load emoji file %s: %s', options.emoji, e) if options.clear_build_db: build_db = {} build_scan_db.save_build_db(build_db, gatekeeper_config, options.build_db) else: build_db = build_scan_db.get_build_db(options.build_db) master_jsons, build_jsons = build_scan.get_updated_builds( masters, build_db, options.parallelism) if options.sync_build_db: build_scan_db.save_build_db(build_db, gatekeeper_config, options.build_db) return 0 (failure_tuples, success_tuples, successful_builder_steps, current_builds_successful) = check_builds(build_jsons, master_jsons, gatekeeper_config) # Write failure / success information back to the build_db. propagate_build_status_back_to_db(failure_tuples, success_tuples, build_db) # opening is an option, mostly to keep the unittests working which # assume that any setting of status is negative. if options.open_tree: open_tree_if_possible(build_db, master_jsons, successful_builder_steps, current_builds_successful, options.status_user, options.password, options.status_url, options.set_status, emoji) # debounce_failures does 3 things: # 1. Groups logging by builder # 2. Selects out the "build" part from the failure tuple. # 3. Rejects builds we've already warned about (and logs). new_failures = debounce_failures(failure_tuples, current_builds_successful, build_db) if options.track_revisions: # Only close the tree if it's a newer revision than before. properties = options.revision_properties.split(',') triggered_revisions = build_db.aux.get('triggered_revisions', {}) if not triggered_revisions or (sorted(triggered_revisions) != sorted(properties)): logging.info( 'revision properties have changed from %s to %s. ' 'clearing previous data.', triggered_revisions, properties) build_db.aux['triggered_revisions'] = dict.fromkeys(properties) new_failures = reject_old_revisions(new_failures, build_db) close_tree_if_necessary(new_failures, options.status_user, options.password, options.status_url, options.set_status, options.revision_properties.split(',')) notify_failures(new_failures, options.sheriff_url, options.default_from_email, options.email_app_url, options.email_app_secret, options.email_domain, options.filter_domain, options.disable_domain_filter) if not options.skip_build_db_update: build_scan_db.save_build_db(build_db, gatekeeper_config, options.build_db) return 0
def inner_loop(args): if not args.data_url: logging.warn('No /data url passed, will write to builder_alerts.json') if args.use_cache: requests_cache.install_cache('failure_stats') else: requests_cache.install_cache(backend='memory') # FIXME: gatekeeper_config should find gatekeeper.json for us. gatekeeper_path = os.path.abspath(args.gatekeeper) logging.debug('Processsing gatekeeper json: %s', gatekeeper_path) gatekeeper = gatekeeper_ng_config.load_gatekeeper_config(gatekeeper_path) gatekeeper_trees_path = os.path.abspath(args.gatekeeper_trees) logging.debug('Processing gatekeeper trees json: %s', gatekeeper_trees_path) gatekeeper_trees = gatekeeper_ng_config.load_gatekeeper_tree_config( gatekeeper_trees_path) master_urls = gatekeeper_extras.fetch_master_urls(gatekeeper, args) start_time = datetime.datetime.utcnow() cache = buildbot.DiskCache(CACHE_PATH) old_alerts = {} if args.data_url: try: old_alerts_raw = requests.get(args.data_url[0]).json() except ValueError: logging.debug('No old alerts found.') else: # internal-alerts will have a redirect instead of alerts if you're # signed in. if 'alerts' in old_alerts_raw: for alert in old_alerts_raw['alerts']: master = alert['master_url'] builder = alert['builder_name'] step = alert['step_name'] reason = alert['reason'] alert_key = alert_builder.generate_alert_key( master, builder, step, reason) if alert_key in old_alerts: logging.critical( 'Incorrectly overwriting an alert reason from the' ' old alert data. master: %s, builder: %s, step: %s, reason:' ' %s' % (master, builder, step, reason)) old_alerts[alert_key] = alert latest_builder_info = {} stale_builder_alerts = [] missing_masters = [] alerts = [] suspected_cls = [] pool = multiprocessing.Pool(processes=args.processes) master_datas = pool.map(SubProcess(cache, old_alerts, args.builder_filter, args.jobs), master_urls) pool.close() pool.join() for data in master_datas: # TODO(ojan): We should put an alert in the JSON for this master so # we can show that the master is down in the sheriff-o-matic UI. if not data[0]: missing_masters.extend([data[3]]) continue alerts.extend(data[0]) latest_builder_info.update(data[1]) stale_builder_alerts.extend(data[2]) logging.info('Fetch took: %s seconds.', (datetime.datetime.utcnow() - start_time).total_seconds()) alerts = gatekeeper_extras.apply_gatekeeper_rules(alerts, gatekeeper, gatekeeper_trees) stale_builder_alerts = gatekeeper_extras.apply_gatekeeper_rules( stale_builder_alerts, gatekeeper, gatekeeper_trees) alerts = analysis.assign_keys(alerts) reason_groups = analysis.group_by_reason(alerts) range_groups = analysis.merge_by_range(reason_groups) if args.findit_api_url and alerts: suspected_cls = query_findit(args.findit_api_url, alerts) data = { 'alerts': alerts, 'suspected_cls': suspected_cls, 'reason_groups': reason_groups, 'range_groups': range_groups, 'latest_builder_info': latest_builder_info, 'stale_builder_alerts': stale_builder_alerts, 'missing_masters': missing_masters, } if not args.data_url: with open('builder_alerts.json', 'w') as f: f.write(json.dumps(data, indent=1)) ret = True json_data = json.dumps(data) logging.info('Alerts json is %s bytes uncompressed.', len(json_data)) s = cStringIO.StringIO() with contextlib.closing(gzip.GzipFile(fileobj=s, mode='w')) as g: g.write(json_data) gzipped_data = s.getvalue() for url in args.data_url: logging.info('POST %s alerts (%s bytes compressed) to %s', len(alerts), len(gzipped_data), url) resp = requests.post(url, data=gzipped_data, headers={'content-encoding': 'gzip'}) try: resp.raise_for_status() except requests.HTTPError as e: logging.error('POST to %s failed! %d %s, %s, %s', url, resp.status_code, resp.reason, resp.content, e) ret = False return ret