def test_fill_in_transition_uses_old_alerts(self):
'''Tests that fill_in_transition doesn't fetch data for old alerts'''
old_compute_transition = alert_builder.compute_transition
def mock_compute_transition(cache, alert,
recent_build_ids): # pragma: no cover
return AlertBuilderTest.k_example_failing_build, None
alert_builder.compute_transition = mock_compute_transition
try:
cache = {}
alert = {
'last_result_time': 1407827665.556039,
'latest_revisions': {
'v8': '23036',
'chromium': '288872',
'nacl': '13599',
'blink': '179989'
},
'builder_name': 'Win Release',
'last_failing_build': 4119,
'reason': 'Foo.NewFailure',
'master_url': 'https://build.chromium.org/p/chromium.lkgr',
'step_name': 'foo_tests'
}
alert_key = alert_builder.generate_alert_key(alert['master_url'],
alert['builder_name'], alert['step_name'], alert['reason'])
recent_build_ids = [4119]
old_alerts = {}
expected_alert = {
'passing_build': 23,
'failing_build': 42,
'failing_revisions': 4711,
'passing_revisions': 31337,
}
old_alerts[alert_key] = expected_alert
expected_alert.update(alert)
alerts = alert_builder.fill_in_transition(cache, alert, recent_build_ids,
old_alerts)
self.assertEqual(alerts, expected_alert)
finally:
alert_builder.compute_transition = old_compute_transition
def inner_loop(args):
if not args.data_url:
logging.warn('No /data url passed, will write to builder_alerts.json')
if args.use_cache:
requests_cache.install_cache('failure_stats')
else:
requests_cache.install_cache(backend='memory')
# FIXME: gatekeeper_config should find gatekeeper.json for us.
gatekeeper_path = os.path.abspath(args.gatekeeper)
logging.debug('Processsing gatekeeper json: %s', gatekeeper_path)
gatekeeper = gatekeeper_ng_config.load_gatekeeper_config(gatekeeper_path)
gatekeeper_trees_path = os.path.abspath(args.gatekeeper_trees)
logging.debug('Processing gatekeeper trees json: %s',
gatekeeper_trees_path)
gatekeeper_trees = gatekeeper_ng_config.load_gatekeeper_tree_config(
gatekeeper_trees_path)
master_urls = gatekeeper_extras.fetch_master_urls(gatekeeper, args)
start_time = datetime.datetime.utcnow()
cache = buildbot.DiskCache(CACHE_PATH)
old_alerts = {}
if args.data_url:
try:
old_alerts_raw = requests.get(args.data_url[0]).json()
except ValueError:
logging.debug('No old alerts found.')
else:
# internal-alerts will have a redirect instead of alerts if you're
# signed in.
if 'alerts' in old_alerts_raw:
for alert in old_alerts_raw['alerts']:
master = alert['master_url']
builder = alert['builder_name']
step = alert['step_name']
reason = alert['reason']
alert_key = alert_builder.generate_alert_key(
master, builder, step, reason)
if alert_key in old_alerts:
logging.critical(
'Incorrectly overwriting an alert reason from the'
' old alert data. master: %s, builder: %s, step: %s, reason:'
' %s' % (master, builder, step, reason))
old_alerts[alert_key] = alert
latest_builder_info = {}
stale_builder_alerts = []
missing_masters = []
alerts = []
suspected_cls = []
pool = multiprocessing.Pool(processes=args.processes)
master_datas = pool.map(
SubProcess(cache, old_alerts, args.builder_filter, args.jobs),
master_urls)
pool.close()
pool.join()
for data in master_datas:
# TODO(ojan): We should put an alert in the JSON for this master so
# we can show that the master is down in the sheriff-o-matic UI.
if not data[0]:
missing_masters.extend([data[3]])
continue
alerts.extend(data[0])
latest_builder_info.update(data[1])
stale_builder_alerts.extend(data[2])
logging.info('Fetch took: %s seconds.',
(datetime.datetime.utcnow() - start_time).total_seconds())
alerts = gatekeeper_extras.apply_gatekeeper_rules(alerts, gatekeeper,
gatekeeper_trees)
stale_builder_alerts = gatekeeper_extras.apply_gatekeeper_rules(
stale_builder_alerts, gatekeeper, gatekeeper_trees)
alerts = analysis.assign_keys(alerts)
reason_groups = analysis.group_by_reason(alerts)
range_groups = analysis.merge_by_range(reason_groups)
if args.findit_api_url and alerts:
suspected_cls = query_findit(args.findit_api_url, alerts)
data = {
'alerts': alerts,
'suspected_cls': suspected_cls,
'reason_groups': reason_groups,
'range_groups': range_groups,
'latest_builder_info': latest_builder_info,
'stale_builder_alerts': stale_builder_alerts,
'missing_masters': missing_masters,
}
if not args.data_url:
with open('builder_alerts.json', 'w') as f:
f.write(json.dumps(data, indent=1))
ret = True
json_data = json.dumps(data)
logging.info('Alerts json is %s bytes uncompressed.', len(json_data))
s = cStringIO.StringIO()
with contextlib.closing(gzip.GzipFile(fileobj=s, mode='w')) as g:
g.write(json_data)
gzipped_data = s.getvalue()
for url in args.data_url:
logging.info('POST %s alerts (%s bytes compressed) to %s', len(alerts),
len(gzipped_data), url)
resp = requests.post(url,
data=gzipped_data,
headers={'content-encoding': 'gzip'})
try:
resp.raise_for_status()
except requests.HTTPError as e:
logging.error('POST to %s failed! %d %s, %s, %s', url,
resp.status_code, resp.reason, resp.content, e)
ret = False
return ret
def inner_loop(args):
old_api_endpoint = string_helpers.slash_join(args.api_endpoint_prefix,
args.old_api_path) if args.old_api_path else None
if not old_api_endpoint:
logging.warn(
'No /data url passed, will write to builder_alerts.json. JSON posted '
'to new API endpoints will be written to builder_alerts_<tree>.json '
'files.')
if args.use_cache:
requests_cache.install_cache('failure_stats')
else:
requests_cache.install_cache(backend='memory')
# FIXME: gatekeeper_config should find gatekeeper.json for us.
gatekeeper_path = os.path.abspath(args.gatekeeper)
logging.debug('Processsing gatekeeper json: %s', gatekeeper_path)
gatekeeper = gatekeeper_ng_config.load_gatekeeper_config(gatekeeper_path)
gatekeeper_trees_path = os.path.abspath(args.gatekeeper_trees)
logging.debug('Processing gatekeeper trees json: %s', gatekeeper_trees_path)
gatekeeper_trees = gatekeeper_ng_config.load_gatekeeper_tree_config(
gatekeeper_trees_path)
master_urls = gatekeeper_extras.fetch_master_urls(gatekeeper, args)
start_time = datetime.datetime.utcnow()
cache = buildbot.DiskCache(CACHE_PATH)
old_alerts = {}
if old_api_endpoint:
try:
old_alerts_raw = requests.get(old_api_endpoint).json()
except ValueError:
logging.debug('No old alerts found.')
else:
# internal-alerts will have a redirect instead of alerts if you're
# signed in.
if 'alerts' in old_alerts_raw:
for alert in old_alerts_raw['alerts']:
master = alert['master_url']
builder = alert['builder_name']
step = alert['step_name']
reason = alert['reason']
alert_key = alert_builder.generate_alert_key(
master, builder, step, reason)
if alert_key in old_alerts:
logging.critical(
'Incorrectly overwriting an alert reason from the'
' old alert data. master: %s, builder: %s, step: %s, reason:'
' %s' % (master, builder, step, reason))
old_alerts[alert_key] = alert
latest_builder_info = {}
stale_builder_alerts = []
missing_masters = []
alerts = []
suspected_cls = []
pool = multiprocessing.Pool(processes=args.processes)
master_datas = pool.map(SubProcess(cache, old_alerts, args.builder_filter,
args.jobs), master_urls)
pool.close()
pool.join()
for data in master_datas:
# TODO(ojan): We should put an alert in the JSON for this master so
# we can show that the master is down in the sheriff-o-matic UI.
if not data[0]:
missing_masters.extend([data[3]])
continue
alerts.extend(data[0])
latest_builder_info.update(data[1])
stale_builder_alerts.extend(data[2])
logging.info('Fetch took: %s seconds.',
(datetime.datetime.utcnow() - start_time).total_seconds())
alerts = gatekeeper_extras.apply_gatekeeper_rules(alerts, gatekeeper,
gatekeeper_trees)
stale_builder_alerts = gatekeeper_extras.apply_gatekeeper_rules(
stale_builder_alerts, gatekeeper, gatekeeper_trees)
alerts = analysis.assign_keys(alerts)
reason_groups = analysis.group_by_reason(alerts)
range_groups = analysis.merge_by_range(reason_groups)
if args.findit_api_url and alerts:
suspected_cls = query_findit(args.findit_api_url, alerts)
data = {
'alerts': alerts,
'suspected_cls': suspected_cls,
'reason_groups': reason_groups,
'range_groups': range_groups,
'latest_builder_info': latest_builder_info,
'stale_builder_alerts': stale_builder_alerts,
'missing_masters': missing_masters,
}
if not old_api_endpoint:
with open('builder_alerts.json', 'w') as f:
f.write(json.dumps(data, indent=1))
ret = True
json_data = json.dumps(data)
logging.info('Alerts json is %s bytes uncompressed.', len(json_data))
gzipped_data = gzipped(json_data)
if old_api_endpoint:
logging.info('POST %s alerts (%s bytes compressed) to %s',
len(alerts), len(gzipped_data), old_api_endpoint)
resp = requests.post(old_api_endpoint, data=gzipped_data,
headers={'content-encoding': 'gzip'})
try:
resp.raise_for_status()
except requests.HTTPError as e:
logging.error('POST to %s failed! %d %s, %s, %s', old_api_endpoint,
resp.status_code, resp.reason, resp.content, e)
ret = False
# Query sheriff issues and post them to the new API endpoint.
if args.crbug_service_account:
global issue_tracker_last_poll
seconds_since_last_poll = (
datetime.datetime.utcnow() - issue_tracker_last_poll).total_seconds()
if seconds_since_last_poll > ISSUE_TRACKER_POLLING_FREQUENCY_SEC:
issue_tracker_last_poll = datetime.datetime.utcnow()
issues_per_tree = crbug_issues.query(args.crbug_service_account,
args.use_monorail)
for tree, issues in issues_per_tree.iteritems():
json_data = {'alerts': issues}
gzipped_data = gzipped(json.dumps(json_data))
if args.api_endpoint_prefix:
new_api_endpoint = string_helpers.slash_join(
args.api_endpoint_prefix, 'api/v1/alerts', tree)
logging.info('POST %s alerts (%s bytes compressed) to %s',
len(issues), len(gzipped_data), new_api_endpoint)
resp = requests.post(new_api_endpoint, data=gzipped_data,
headers={'content-encoding': 'gzip'})
try:
resp.raise_for_status()
except requests.HTTPError:
logging.exception('POST to %s failed! %d %s, %s', new_api_endpoint,
resp.status_code, resp.reason, resp.content)
ret = False
else:
with open('builder_alerts_%s.json' % tree, 'w') as f:
f.write(json.dumps(json_data, indent=1))
else:
logging.error(
'--crbug-service-account was not specified, can not get crbug issues')
ret = False
return ret
def inner_loop(args):
if not args.data_url:
logging.warn('No /data url passed, will write to builder_alerts.json')
if args.use_cache:
requests_cache.install_cache('failure_stats')
else:
requests_cache.install_cache(backend='memory')
# FIXME: gatekeeper_config should find gatekeeper.json for us.
gatekeeper_path = os.path.abspath(args.gatekeeper)
logging.debug('Processsing gatekeeper json: %s', gatekeeper_path)
gatekeeper = gatekeeper_ng_config.load_gatekeeper_config(gatekeeper_path)
gatekeeper_trees_path = os.path.abspath(args.gatekeeper_trees)
logging.debug('Processing gatekeeper trees json: %s', gatekeeper_trees_path)
gatekeeper_trees = gatekeeper_ng_config.load_gatekeeper_tree_config(
gatekeeper_trees_path)
master_urls = gatekeeper_extras.fetch_master_urls(gatekeeper, args)
start_time = datetime.datetime.utcnow()
cache = buildbot.DiskCache(CACHE_PATH)
old_alerts = {}
if args.data_url:
try:
old_alerts_raw = requests.get(args.data_url[0]).json()
except ValueError:
logging.debug('No old alerts found.')
else:
# internal-alerts will have a redirect instead of alerts if you're
# signed in.
if 'alerts' in old_alerts_raw:
for alert in old_alerts_raw['alerts']:
master = alert['master_url']
builder = alert['builder_name']
step = alert['step_name']
reason = alert['reason']
alert_key = alert_builder.generate_alert_key(
master, builder, step, reason)
if alert_key in old_alerts:
logging.critical(
'Incorrectly overwriting an alert reason from the'
' old alert data. master: %s, builder: %s, step: %s, reason:'
' %s' % (master, builder, step, reason))
old_alerts[alert_key] = alert
latest_builder_info = {}
stale_builder_alerts = []
missing_masters = []
alerts = []
suspected_cls = []
pool = multiprocessing.Pool(processes=args.processes)
master_datas = pool.map(SubProcess(cache, old_alerts, args.builder_filter,
args.jobs), master_urls)
pool.close()
pool.join()
for data in master_datas:
# TODO(ojan): We should put an alert in the JSON for this master so
# we can show that the master is down in the sheriff-o-matic UI.
if not data[0]:
missing_masters.extend([data[3]])
continue
alerts.extend(data[0])
latest_builder_info.update(data[1])
stale_builder_alerts.extend(data[2])
logging.info('Fetch took: %s seconds.',
(datetime.datetime.utcnow() - start_time).total_seconds())
alerts = gatekeeper_extras.apply_gatekeeper_rules(alerts, gatekeeper,
gatekeeper_trees)
stale_builder_alerts = gatekeeper_extras.apply_gatekeeper_rules(
stale_builder_alerts, gatekeeper, gatekeeper_trees)
alerts = analysis.assign_keys(alerts)
reason_groups = analysis.group_by_reason(alerts)
range_groups = analysis.merge_by_range(reason_groups)
if args.findit_api_url and alerts:
suspected_cls = query_findit(args.findit_api_url, alerts)
data = {
'alerts': alerts,
'suspected_cls': suspected_cls,
'reason_groups': reason_groups,
'range_groups': range_groups,
'latest_builder_info': latest_builder_info,
'stale_builder_alerts': stale_builder_alerts,
'missing_masters': missing_masters,
}
if not args.data_url:
with open('builder_alerts.json', 'w') as f:
f.write(json.dumps(data, indent=1))
ret = True
json_data = json.dumps(data)
logging.info('Alerts json is %s bytes uncompressed.', len(json_data))
s = cStringIO.StringIO()
with contextlib.closing(gzip.GzipFile(fileobj=s, mode='w')) as g:
g.write(json_data)
gzipped_data = s.getvalue()
for url in args.data_url:
logging.info('POST %s alerts (%s bytes compressed) to %s',
len(alerts), len(gzipped_data), url)
resp = requests.post(url, data=gzipped_data,
headers={'content-encoding': 'gzip'})
try:
resp.raise_for_status()
except requests.HTTPError as e:
logging.error('POST to %s failed! %d %s, %s, %s',
url, resp.status_code, resp.reason, resp.content, e)
ret = False
return ret