def inner_loop(args):
if not args.data_url:
logging.warn('No /data url passed, will write to builder_alerts.json')
if args.use_cache:
requests_cache.install_cache('failure_stats')
else:
requests_cache.install_cache(backend='memory')
# FIXME: gatekeeper_config should find gatekeeper.json for us.
gatekeeper_path = os.path.abspath(args.gatekeeper)
logging.debug('Processsing gatekeeper json: %s', gatekeeper_path)
gatekeeper = gatekeeper_ng_config.load_gatekeeper_config(gatekeeper_path)
gatekeeper_trees_path = os.path.abspath(args.gatekeeper_trees)
logging.debug('Processing gatekeeper trees json: %s',
gatekeeper_trees_path)
gatekeeper_trees = gatekeeper_ng_config.load_gatekeeper_tree_config(
gatekeeper_trees_path)
master_urls = gatekeeper_extras.fetch_master_urls(gatekeeper, args)
start_time = datetime.datetime.utcnow()
cache = buildbot.DiskCache(CACHE_PATH)
old_alerts = {}
if args.data_url:
try:
old_alerts_raw = requests.get(args.data_url[0]).json()
except ValueError:
logging.debug('No old alerts found.')
else:
# internal-alerts will have a redirect instead of alerts if you're
# signed in.
if 'alerts' in old_alerts_raw:
for alert in old_alerts_raw['alerts']:
master = alert['master_url']
builder = alert['builder_name']
step = alert['step_name']
reason = alert['reason']
alert_key = alert_builder.generate_alert_key(
master, builder, step, reason)
if alert_key in old_alerts:
logging.critical(
'Incorrectly overwriting an alert reason from the'
' old alert data. master: %s, builder: %s, step: %s, reason:'
' %s' % (master, builder, step, reason))
old_alerts[alert_key] = alert
latest_builder_info = {}
stale_builder_alerts = []
missing_masters = []
alerts = []
suspected_cls = []
pool = multiprocessing.Pool(processes=args.processes)
master_datas = pool.map(
SubProcess(cache, old_alerts, args.builder_filter, args.jobs),
master_urls)
pool.close()
pool.join()
for data in master_datas:
# TODO(ojan): We should put an alert in the JSON for this master so
# we can show that the master is down in the sheriff-o-matic UI.
if not data[0]:
missing_masters.extend([data[3]])
continue
alerts.extend(data[0])
latest_builder_info.update(data[1])
stale_builder_alerts.extend(data[2])
logging.info('Fetch took: %s seconds.',
(datetime.datetime.utcnow() - start_time).total_seconds())
alerts = gatekeeper_extras.apply_gatekeeper_rules(alerts, gatekeeper,
gatekeeper_trees)
stale_builder_alerts = gatekeeper_extras.apply_gatekeeper_rules(
stale_builder_alerts, gatekeeper, gatekeeper_trees)
alerts = analysis.assign_keys(alerts)
reason_groups = analysis.group_by_reason(alerts)
range_groups = analysis.merge_by_range(reason_groups)
if args.findit_api_url and alerts:
suspected_cls = query_findit(args.findit_api_url, alerts)
data = {
'alerts': alerts,
'suspected_cls': suspected_cls,
'reason_groups': reason_groups,
'range_groups': range_groups,
'latest_builder_info': latest_builder_info,
'stale_builder_alerts': stale_builder_alerts,
'missing_masters': missing_masters,
}
if not args.data_url:
with open('builder_alerts.json', 'w') as f:
f.write(json.dumps(data, indent=1))
ret = True
json_data = json.dumps(data)
logging.info('Alerts json is %s bytes uncompressed.', len(json_data))
s = cStringIO.StringIO()
with contextlib.closing(gzip.GzipFile(fileobj=s, mode='w')) as g:
g.write(json_data)
gzipped_data = s.getvalue()
for url in args.data_url:
logging.info('POST %s alerts (%s bytes compressed) to %s', len(alerts),
len(gzipped_data), url)
resp = requests.post(url,
data=gzipped_data,
headers={'content-encoding': 'gzip'})
try:
resp.raise_for_status()
except requests.HTTPError as e:
logging.error('POST to %s failed! %d %s, %s, %s', url,
resp.status_code, resp.reason, resp.content, e)
ret = False
return ret
def inner_loop(args):
old_api_endpoint = string_helpers.slash_join(args.api_endpoint_prefix,
args.old_api_path) if args.old_api_path else None
if not old_api_endpoint:
logging.warn(
'No /data url passed, will write to builder_alerts.json. JSON posted '
'to new API endpoints will be written to builder_alerts_<tree>.json '
'files.')
if args.use_cache:
requests_cache.install_cache('failure_stats')
else:
requests_cache.install_cache(backend='memory')
# FIXME: gatekeeper_config should find gatekeeper.json for us.
gatekeeper_path = os.path.abspath(args.gatekeeper)
logging.debug('Processsing gatekeeper json: %s', gatekeeper_path)
gatekeeper = gatekeeper_ng_config.load_gatekeeper_config(gatekeeper_path)
gatekeeper_trees_path = os.path.abspath(args.gatekeeper_trees)
logging.debug('Processing gatekeeper trees json: %s', gatekeeper_trees_path)
gatekeeper_trees = gatekeeper_ng_config.load_gatekeeper_tree_config(
gatekeeper_trees_path)
master_urls = gatekeeper_extras.fetch_master_urls(gatekeeper, args)
start_time = datetime.datetime.utcnow()
cache = buildbot.DiskCache(CACHE_PATH)
old_alerts = {}
if old_api_endpoint:
try:
old_alerts_raw = requests.get(old_api_endpoint).json()
except ValueError:
logging.debug('No old alerts found.')
else:
# internal-alerts will have a redirect instead of alerts if you're
# signed in.
if 'alerts' in old_alerts_raw:
for alert in old_alerts_raw['alerts']:
master = alert['master_url']
builder = alert['builder_name']
step = alert['step_name']
reason = alert['reason']
alert_key = alert_builder.generate_alert_key(
master, builder, step, reason)
if alert_key in old_alerts:
logging.critical(
'Incorrectly overwriting an alert reason from the'
' old alert data. master: %s, builder: %s, step: %s, reason:'
' %s' % (master, builder, step, reason))
old_alerts[alert_key] = alert
latest_builder_info = {}
stale_builder_alerts = []
missing_masters = []
alerts = []
suspected_cls = []
pool = multiprocessing.Pool(processes=args.processes)
master_datas = pool.map(SubProcess(cache, old_alerts, args.builder_filter,
args.jobs), master_urls)
pool.close()
pool.join()
for data in master_datas:
# TODO(ojan): We should put an alert in the JSON for this master so
# we can show that the master is down in the sheriff-o-matic UI.
if not data[0]:
missing_masters.extend([data[3]])
continue
alerts.extend(data[0])
latest_builder_info.update(data[1])
stale_builder_alerts.extend(data[2])
logging.info('Fetch took: %s seconds.',
(datetime.datetime.utcnow() - start_time).total_seconds())
alerts = gatekeeper_extras.apply_gatekeeper_rules(alerts, gatekeeper,
gatekeeper_trees)
stale_builder_alerts = gatekeeper_extras.apply_gatekeeper_rules(
stale_builder_alerts, gatekeeper, gatekeeper_trees)
alerts = analysis.assign_keys(alerts)
reason_groups = analysis.group_by_reason(alerts)
range_groups = analysis.merge_by_range(reason_groups)
if args.findit_api_url and alerts:
suspected_cls = query_findit(args.findit_api_url, alerts)
data = {
'alerts': alerts,
'suspected_cls': suspected_cls,
'reason_groups': reason_groups,
'range_groups': range_groups,
'latest_builder_info': latest_builder_info,
'stale_builder_alerts': stale_builder_alerts,
'missing_masters': missing_masters,
}
if not old_api_endpoint:
with open('builder_alerts.json', 'w') as f:
f.write(json.dumps(data, indent=1))
ret = True
json_data = json.dumps(data)
logging.info('Alerts json is %s bytes uncompressed.', len(json_data))
gzipped_data = gzipped(json_data)
if old_api_endpoint:
logging.info('POST %s alerts (%s bytes compressed) to %s',
len(alerts), len(gzipped_data), old_api_endpoint)
resp = requests.post(old_api_endpoint, data=gzipped_data,
headers={'content-encoding': 'gzip'})
try:
resp.raise_for_status()
except requests.HTTPError as e:
logging.error('POST to %s failed! %d %s, %s, %s', old_api_endpoint,
resp.status_code, resp.reason, resp.content, e)
ret = False
# Query sheriff issues and post them to the new API endpoint.
if args.crbug_service_account:
global issue_tracker_last_poll
seconds_since_last_poll = (
datetime.datetime.utcnow() - issue_tracker_last_poll).total_seconds()
if seconds_since_last_poll > ISSUE_TRACKER_POLLING_FREQUENCY_SEC:
issue_tracker_last_poll = datetime.datetime.utcnow()
issues_per_tree = crbug_issues.query(args.crbug_service_account,
args.use_monorail)
for tree, issues in issues_per_tree.iteritems():
json_data = {'alerts': issues}
gzipped_data = gzipped(json.dumps(json_data))
if args.api_endpoint_prefix:
new_api_endpoint = string_helpers.slash_join(
args.api_endpoint_prefix, 'api/v1/alerts', tree)
logging.info('POST %s alerts (%s bytes compressed) to %s',
len(issues), len(gzipped_data), new_api_endpoint)
resp = requests.post(new_api_endpoint, data=gzipped_data,
headers={'content-encoding': 'gzip'})
try:
resp.raise_for_status()
except requests.HTTPError:
logging.exception('POST to %s failed! %d %s, %s', new_api_endpoint,
resp.status_code, resp.reason, resp.content)
ret = False
else:
with open('builder_alerts_%s.json' % tree, 'w') as f:
f.write(json.dumps(json_data, indent=1))
else:
logging.error(
'--crbug-service-account was not specified, can not get crbug issues')
ret = False
return ret
def test_apply_gatekeeper_rules(self):
gatekeeper_cfg = {
'http://build.chromium.org/p/chromium': {
'key': 'value'
}
}
gatekeeper_trees_cfg = {}
alerts = [
{
'master_url':
'http://build.chromium.org/p/project.without.config',
'builder_name': 'Linux',
'step_name': 'test_xyz'
},
{
'master_url': 'http://build.chromium.org/p/chromium',
'builder_name': 'Ignored',
'step_name': 'bot_update'
},
{
'master_url': 'http://build.chromium.org/p/chromium',
'builder_name': 'Win',
'step_name': 'bot_update'
},
{
'master_url': 'http://build.chromium.org/p/chromium',
'builder_name': 'Mac'
},
# stale master alert
{
'last_update_time': 1234,
'master_url': 'http://build.chromium.org/p/chromium',
'master_name': 'chromium',
},
# stale builder alert
{
'master_url': 'http://build.chromium.org/p/chromium',
'builder_name': 'Linux',
'state': 'offline',
'last_update_time': 1234,
'pending_builds': [],
'step': 'bot_update',
'latest_build': 1234,
}
]
filtered_alerts = gatekeeper_extras.apply_gatekeeper_rules(
alerts, gatekeeper_cfg, gatekeeper_trees_cfg)
self.assertEqual(len(filtered_alerts), 5)
self.assertIn(
{
'master_url': 'http://build.chromium.org/p/chromium',
'builder_name': 'Win',
'step_name': 'bot_update',
'would_close_tree': True,
'tree': 'test-tree'
}, filtered_alerts)
self.assertIn(
{
'master_url':
'http://build.chromium.org/p/project.without.config',
'builder_name': 'Linux',
'step_name': 'test_xyz'
}, filtered_alerts)
self.assertIn(
{
'master_url': 'http://build.chromium.org/p/chromium',
'builder_name': 'Mac',
'tree': 'test-tree'
}, filtered_alerts)
self.assertIn(
{
'last_update_time': 1234,
'master_url': 'http://build.chromium.org/p/chromium',
'master_name': 'chromium',
'tree': 'test-tree',
}, filtered_alerts)
self.assertIn(
{
'master_url': 'http://build.chromium.org/p/chromium',
'builder_name': 'Linux',
'state': 'offline',
'last_update_time': 1234,
'pending_builds': [],
'step': 'bot_update',
'latest_build': 1234,
'tree': 'test-tree',
}, filtered_alerts)
def test_apply_gatekeeper_rules(self):
gatekeeper_cfg = {'http://build.chromium.org/p/chromium': {'key': 'value'}}
gatekeeper_trees_cfg = {}
alerts = [
{'master_url': 'http://build.chromium.org/p/project.without.config',
'builder_name': 'Linux',
'step_name': 'test_xyz'},
{'master_url': 'http://build.chromium.org/p/chromium',
'builder_name': 'Ignored',
'step_name': 'bot_update'},
{'master_url': 'http://build.chromium.org/p/chromium',
'builder_name': 'Win',
'step_name': 'bot_update'},
{'master_url': 'http://build.chromium.org/p/chromium',
'builder_name': 'Mac'},
# stale master alert
{
'last_update_time': 1234,
'master_url': 'http://build.chromium.org/p/chromium',
'master_name': 'chromium',
},
# stale builder alert
{
'master_url': 'http://build.chromium.org/p/chromium',
'builder_name': 'Linux',
'state': 'offline',
'last_update_time': 1234,
'pending_builds': [],
'step': 'bot_update',
'latest_build': 1234,
}
]
filtered_alerts = gatekeeper_extras.apply_gatekeeper_rules(
alerts, gatekeeper_cfg, gatekeeper_trees_cfg)
self.assertEqual(len(filtered_alerts), 5)
self.assertIn({'master_url': 'http://build.chromium.org/p/chromium',
'builder_name': 'Win',
'step_name': 'bot_update',
'would_close_tree': True,
'tree': 'test-tree'}, filtered_alerts)
self.assertIn(
{'master_url': 'http://build.chromium.org/p/project.without.config',
'builder_name': 'Linux',
'step_name': 'test_xyz'},
filtered_alerts)
self.assertIn(
{'master_url': 'http://build.chromium.org/p/chromium',
'builder_name': 'Mac',
'tree': 'test-tree'},
filtered_alerts)
self.assertIn(
{
'last_update_time': 1234,
'master_url': 'http://build.chromium.org/p/chromium',
'master_name': 'chromium',
'tree': 'test-tree',
},
filtered_alerts)
self.assertIn(
{
'master_url': 'http://build.chromium.org/p/chromium',
'builder_name': 'Linux',
'state': 'offline',
'last_update_time': 1234,
'pending_builds': [],
'step': 'bot_update',
'latest_build': 1234,
'tree': 'test-tree',
},
filtered_alerts)
def inner_loop(args):
if not args.data_url:
logging.warn('No /data url passed, will write to builder_alerts.json')
if args.use_cache:
requests_cache.install_cache('failure_stats')
else:
requests_cache.install_cache(backend='memory')
# FIXME: gatekeeper_config should find gatekeeper.json for us.
gatekeeper_path = os.path.abspath(args.gatekeeper)
logging.debug('Processsing gatekeeper json: %s', gatekeeper_path)
gatekeeper = gatekeeper_ng_config.load_gatekeeper_config(gatekeeper_path)
gatekeeper_trees_path = os.path.abspath(args.gatekeeper_trees)
logging.debug('Processing gatekeeper trees json: %s', gatekeeper_trees_path)
gatekeeper_trees = gatekeeper_ng_config.load_gatekeeper_tree_config(
gatekeeper_trees_path)
master_urls = gatekeeper_extras.fetch_master_urls(gatekeeper, args)
start_time = datetime.datetime.utcnow()
cache = buildbot.DiskCache(CACHE_PATH)
old_alerts = {}
if args.data_url:
try:
old_alerts_raw = requests.get(args.data_url[0]).json()
except ValueError:
logging.debug('No old alerts found.')
else:
# internal-alerts will have a redirect instead of alerts if you're
# signed in.
if 'alerts' in old_alerts_raw:
for alert in old_alerts_raw['alerts']:
master = alert['master_url']
builder = alert['builder_name']
step = alert['step_name']
reason = alert['reason']
alert_key = alert_builder.generate_alert_key(
master, builder, step, reason)
if alert_key in old_alerts:
logging.critical(
'Incorrectly overwriting an alert reason from the'
' old alert data. master: %s, builder: %s, step: %s, reason:'
' %s' % (master, builder, step, reason))
old_alerts[alert_key] = alert
latest_builder_info = {}
stale_builder_alerts = []
missing_masters = []
alerts = []
suspected_cls = []
pool = multiprocessing.Pool(processes=args.processes)
master_datas = pool.map(SubProcess(cache, old_alerts, args.builder_filter,
args.jobs), master_urls)
pool.close()
pool.join()
for data in master_datas:
# TODO(ojan): We should put an alert in the JSON for this master so
# we can show that the master is down in the sheriff-o-matic UI.
if not data[0]:
missing_masters.extend([data[3]])
continue
alerts.extend(data[0])
latest_builder_info.update(data[1])
stale_builder_alerts.extend(data[2])
logging.info('Fetch took: %s seconds.',
(datetime.datetime.utcnow() - start_time).total_seconds())
alerts = gatekeeper_extras.apply_gatekeeper_rules(alerts, gatekeeper,
gatekeeper_trees)
stale_builder_alerts = gatekeeper_extras.apply_gatekeeper_rules(
stale_builder_alerts, gatekeeper, gatekeeper_trees)
alerts = analysis.assign_keys(alerts)
reason_groups = analysis.group_by_reason(alerts)
range_groups = analysis.merge_by_range(reason_groups)
if args.findit_api_url and alerts:
suspected_cls = query_findit(args.findit_api_url, alerts)
data = {
'alerts': alerts,
'suspected_cls': suspected_cls,
'reason_groups': reason_groups,
'range_groups': range_groups,
'latest_builder_info': latest_builder_info,
'stale_builder_alerts': stale_builder_alerts,
'missing_masters': missing_masters,
}
if not args.data_url:
with open('builder_alerts.json', 'w') as f:
f.write(json.dumps(data, indent=1))
ret = True
json_data = json.dumps(data)
logging.info('Alerts json is %s bytes uncompressed.', len(json_data))
s = cStringIO.StringIO()
with contextlib.closing(gzip.GzipFile(fileobj=s, mode='w')) as g:
g.write(json_data)
gzipped_data = s.getvalue()
for url in args.data_url:
logging.info('POST %s alerts (%s bytes compressed) to %s',
len(alerts), len(gzipped_data), url)
resp = requests.post(url, data=gzipped_data,
headers={'content-encoding': 'gzip'})
try:
resp.raise_for_status()
except requests.HTTPError as e:
logging.error('POST to %s failed! %d %s, %s, %s',
url, resp.status_code, resp.reason, resp.content, e)
ret = False
return ret