def test_ignore_expired_cache(self):
# When a build is in the cache but expired, we hit the network instead.
cache_key = buildbot.cache_key_for_build('foo', 'bar', 1)
self.cache.set(cache_key, {'men': 'at work'})
def _mock_key_age(cache_key):
return datetime.datetime.now() - datetime.timedelta(seconds=121)
_real_key_age = self.cache.key_age
def _mock_fetch_and_cache_build(cache, url, cache_key):
return {'I come': 'from a land down under'}
_real_fetch_and_cache_build = buildbot.fetch_and_cache_build
try:
self.cache.key_age = _mock_key_age
buildbot.fetch_and_cache_build = _mock_fetch_and_cache_build
build, source = buildbot.fetch_build_json(self.cache, 'foo', 'bar',
1)
self.assertEqual(build, {'I come': 'from a land down under'})
self.assertEqual(source, 'chrome-build-extract')
finally:
self.cache.key_age = _real_key_age
buildbot.fetch_and_cache_build = _real_fetch_and_cache_build
def alerts_from_step_failure(cache, step_failure, master_url,
builder_name): # pragma: no cover
build = (
buildbot.fetch_build_json(
cache, master_url, builder_name,
step_failure['build_number'])[0])
step = next((s for s in build['steps']
if s['name'] == step_failure['step_name']), None)
step_template = {
'master_url': master_url,
'last_result_time': step['times'][1],
'builder_name': builder_name,
'last_failing_build': step_failure['build_number'],
'step_name': step['name'],
'latest_revisions': buildbot.revisions_from_build(build),
}
alerts = []
reasons = reasons_for_failure(cache, step, build, builder_name, master_url)
if not reasons:
alert = dict(step_template)
alert['reason'] = None
alerts.append(alert)
else:
for reason in reasons:
alert = dict(step_template)
alert['reason'] = reason
alerts.append(alert)
return alerts
def test_fetch_from_cache(self):
# When a build is in the cache, we return that build from the cache.
cache_key = buildbot.cache_key_for_build('foo', 'bar', 1)
self.cache.set(cache_key, {'men': 'at work'})
build, source = buildbot.fetch_build_json(self.cache, 'foo', 'bar', 1)
self.assertEqual(build, {'men': 'at work'})
self.assertEqual(source, 'disk cache')
def test_fetch_from_cache(self):
# When a build is in the cache, we return that build from the cache.
cache_key = buildbot.cache_key_for_build('foo', 'bar', 1)
self.cache.set(cache_key, {'men': 'at work'})
build, source = buildbot.fetch_build_json(self.cache, 'foo', 'bar', 1)
self.assertEqual(build, {'men': 'at work'})
self.assertEqual(source, 'disk cache')
def test_fetch_from_cbe(self):
# If a build is available on chrome-build-extract, use that.
def _mock_fetch_and_cache_build(cache, url, cache_key):
if buildbot.CBE_BASE in url: # pragma: no branch
return {'I come': 'from a land down under'}
_real_fetch_and_cache_build = buildbot.fetch_and_cache_build
try:
buildbot.fetch_and_cache_build = _mock_fetch_and_cache_build
build, source = buildbot.fetch_build_json(self.cache, 'foo', 'bar', 1)
self.assertEqual(build, {'I come': 'from a land down under'})
self.assertEqual(source, 'chrome-build-extract')
finally:
buildbot.fetch_and_cache_build = _real_fetch_and_cache_build
def test_fetch_from_cbe(self):
# If a build is available on chrome-build-extract, use that.
def _mock_fetch_and_cache_build(cache, url, cache_key):
if buildbot.CBE_BASE in url: # pragma: no branch
return {'I come': 'from a land down under'}
_real_fetch_and_cache_build = buildbot.fetch_and_cache_build
try:
buildbot.fetch_and_cache_build = _mock_fetch_and_cache_build
build, source = buildbot.fetch_build_json(self.cache, 'foo', 'bar',
1)
self.assertEqual(build, {'I come': 'from a land down under'})
self.assertEqual(source, 'chrome-build-extract')
finally:
buildbot.fetch_and_cache_build = _real_fetch_and_cache_build
def test_fetch_from_master(self):
# If both the cache and CBE fall through, go straight to the master.
def _mock_fetch_and_cache_build(cache, url, cache_key):
if buildbot.CBE_BASE in url:
return None
if 'build.chromium.org' in url: # pragma: no branch
return {'Can you hear': 'the thunder'}
_real_fetch_and_cache_build = buildbot.fetch_and_cache_build
try:
buildbot.fetch_and_cache_build = _mock_fetch_and_cache_build
build, source = buildbot.fetch_build_json(self.cache, 'foo', 'bar', 1)
self.assertEqual(build, {'Can you hear': 'the thunder'})
self.assertEqual(source, 'master')
finally:
buildbot.fetch_and_cache_build = _real_fetch_and_cache_build
def test_fetch_from_master(self):
# If both the cache and CBE fall through, go straight to the master.
def _mock_fetch_and_cache_build(cache, url, cache_key):
if buildbot.CBE_BASE in url:
return None
if 'build.chromium.org' in url: # pragma: no branch
return {'Can you hear': 'the thunder'}
_real_fetch_and_cache_build = buildbot.fetch_and_cache_build
try:
buildbot.fetch_and_cache_build = _mock_fetch_and_cache_build
build, source = buildbot.fetch_build_json(self.cache, 'foo', 'bar',
1)
self.assertEqual(build, {'Can you hear': 'the thunder'})
self.assertEqual(source, 'master')
finally:
buildbot.fetch_and_cache_build = _real_fetch_and_cache_build
def alerts_for_builder(cache, master_url, builder_name,
recent_build_ids, old_alerts): # pragma: no cover
recent_build_ids = sorted(recent_build_ids, reverse=True)
# Limit to 100 to match our current cache-warming logic
recent_build_ids = recent_build_ids[:100]
fetch_function = lambda num: buildbot.fetch_build_json(
cache, master_url, builder_name, num)
step_failures = find_current_step_failures(fetch_function, recent_build_ids)
alerts = []
for step_failure in step_failures:
alerts += alerts_from_step_failure(cache, step_failure,
master_url, builder_name)
start_time = time.time()
filled_in_alerts = []
should_fill_in_remaining_alerts = True
for alert in alerts:
old_should_fill_in_remaining_alerts = should_fill_in_remaining_alerts
should_fill_in_remaining_alerts = (
time.time() - start_time < PER_BOT_TIME_LIMIT_FOR_COMPUTING_TRANSITIONS)
if old_should_fill_in_remaining_alerts != should_fill_in_remaining_alerts:
logging.debug(
'Filled in transitions for %d/%s alerts for %s:%s',
len(filled_in_alerts), len(alerts), master_url, builder_name)
if should_fill_in_remaining_alerts:
filled_in_alerts.append(
fill_in_transition(cache, alert, recent_build_ids, old_alerts))
else:
update_data = {
'passing_build': None,
'failing_build': None,
'failing_revisions': None,
'passing_revisions': None,
}
alert.update(update_data)
filled_in_alerts.append(alert)
return filled_in_alerts
def test_ignore_expired_cache(self):
# When a build is in the cache but expired, we hit the network instead.
cache_key = buildbot.cache_key_for_build('foo', 'bar', 1)
self.cache.set(cache_key, {'men': 'at work'})
def _mock_key_age(cache_key):
return datetime.datetime.now() - datetime.timedelta(seconds=121)
_real_key_age = self.cache.key_age
def _mock_fetch_and_cache_build(cache, url, cache_key):
return {'I come': 'from a land down under'}
_real_fetch_and_cache_build = buildbot.fetch_and_cache_build
try:
self.cache.key_age = _mock_key_age
buildbot.fetch_and_cache_build = _mock_fetch_and_cache_build
build, source = buildbot.fetch_build_json(self.cache, 'foo', 'bar', 1)
self.assertEqual(build, {'I come': 'from a land down under'})
self.assertEqual(source, 'chrome-build-extract')
finally:
self.cache.key_age = _real_key_age
buildbot.fetch_and_cache_build = _real_fetch_and_cache_build
def fetch_function(num): return buildbot.fetch_build_json(cache, master, builder, num)
