def post(self):
"""Add new isolate information.
Args:
builder_name: The name of the builder that produced the isolate.
change: The Change the isolate is for, as a JSON string.
isolate_server: The hostname of the server where the isolates are stored.
isolate_map: A JSON dict mapping the target names to the isolate hashes.
"""
# Check permissions.
if self.request.remote_addr not in utils.GetIpWhitelist():
self.response.set_status(403)
self.response.write('Permission denied')
return
# Get parameters.
parameters = (
('builder_name', str),
('change', lambda x: change_module.Change.FromDict(json.loads(x))),
('isolate_server', str),
('isolate_map', json.loads),
)
try:
# pylint: disable=unbalanced-tuple-unpacking
builder_name, change, isolate_server, isolate_map = (
self._ValidateParameters(parameters))
except (KeyError, TypeError, ValueError) as e:
self.response.set_status(400)
self.response.write(e)
return
# Put information into the datastore.
isolate_infos = {(builder_name, change, target, isolate_hash)
for target, isolate_hash in isolate_map.iteritems()}
isolate.Put(isolate_server, isolate_infos)
def testSimultaneousBuilds(self, put, get_job_status):
# Two builds started at the same time on the same Change should reuse the
# same build request.
change = change_module.Change(
(change_module.Commit('src', 'base git hash'),))
quest = find_isolate.FindIsolate('Mac Builder', 'telemetry_perf_tests')
execution_1 = quest.Start(change)
execution_2 = quest.Start(change)
# Request a build.
put.return_value = {'build': {'id': 'build_id'}}
execution_1.Poll()
execution_2.Poll()
self.assertFalse(execution_1.completed)
self.assertFalse(execution_2.completed)
self.assertEqual(put.call_count, 1)
# Check build status.
get_job_status.return_value = {'build': {'status': 'STARTED'}}
execution_1.Poll()
execution_2.Poll()
self.assertFalse(execution_1.completed)
self.assertFalse(execution_2.completed)
self.assertEqual(get_job_status.call_count, 2)
# Look up isolate hash.
isolate.Put((('Mac Builder', change,
'telemetry_perf_tests', 'isolate git hash'),))
execution_1.Poll()
execution_2.Poll()
self.assertExecutionSuccess(execution_1)
self.assertExecutionSuccess(execution_2)
def Post(self):
"""Add new isolate information.
Args:
builder_name: The name of the builder that produced the isolate.
change: The Change the isolate is for, as a JSON string.
isolate_server: The hostname of the server where the isolates are stored.
isolate_map: A JSON dict mapping the target names to the isolate hashes.
"""
# Get parameters.
parameters = (
('builder_name', str),
('change', lambda x: change_module.Change.FromDict(json.loads(x))),
('isolate_server', str),
('isolate_map', json.loads),
)
try:
# pylint: disable=unbalanced-tuple-unpacking
builder_name, change, isolate_server, isolate_map = (
self._ValidateParameters(parameters))
except (KeyError, TypeError, ValueError) as e:
self.response.set_status(400)
self.response.write(json.dumps({'error': e.message}))
return
# Put information into the datastore.
isolate_infos = [
(builder_name, change, target, isolate_server, isolate_hash)
for target, isolate_hash in isolate_map.iteritems()]
isolate.Put(isolate_infos)
# Respond to the API user.
self.response.write(json.dumps(isolate_infos))
def setUp(self):
super(_FindIsolateExecutionTest, self).setUp()
change = change_test.Change(123)
isolate.Put(
'https://isolate.server',
(('Mac Builder', change, 'telemetry_perf_tests', '7c7e90be'),))
def testSuccesfulBuilderLookupForAllBuilders(self):
builder_testers = (
('arm-builder-rel', 'health-plan-clankium-phone'),
('Android Builder', 'Android Nexus5 Perf'),
('Android arm64 Builder', 'Android Nexus5X Perf'),
('Linux Builder', 'Linux Perf'),
('Mac Builder', 'Mac Air Perf'),
('Win Builder', 'Win 7 Perf'),
('Win x64 Builder', 'Win Zenbook Perf'),
)
change = change_module.Change((change_module.Commit('src',
'git hash'), ))
isolate.Put(
(builder, change, 'telemetry_perf_tests', hex(hash(builder)))
for builder, _ in builder_testers)
for builder, tester in builder_testers:
quest = find_isolate.FindIsolate(tester, 'telemetry_perf_tests')
execution = quest.Start(change)
execution.Poll()
self.assertExecutionSuccess(execution)
self.assertEqual(execution.result_arguments,
{'isolate_hash': hex(hash(builder))})
def testPutAndGet(self):
isolate.Put(
(('Mac Builder', 'f9f2b720', 'telemetry_perf', '7c7e90be'),
('Mac Builder', 'f35be4f1', 'telemetry_perf', '38e2f262')))
isolate_hash = isolate.Get('Mac Builder', 'f9f2b720', 'telemetry_perf')
self.assertEqual(isolate_hash, '7c7e90be')
def testDeleteExpiredIsolate(self):
isolate_infos = (
('Mac Builder Perf', change_test.Change(0), 'target_name',
'https://isolate.server', '123'),
('Mac Builder Perf', change_test.Change(1), 'target_name',
'https://isolate.server', '456'),
)
isolate.Put(isolate_infos)
cur = ndb.Key(
'Isolate',
isolate._Key('Mac Builder Perf', change_test.Change(0),
'target_name')).get()
cur.created = datetime.datetime.now() - datetime.timedelta(hours=1)
cur.put()
cur = ndb.Key(
'Isolate',
isolate._Key(isolate_infos[1][0], isolate_infos[1][1],
isolate_infos[1][2])).get()
cur.created = datetime.datetime.now() - (
isolate.ISOLATE_EXPIRY_DURATION + datetime.timedelta(hours=1))
cur.put()
isolate.DeleteExpiredIsolates()
q = isolate.Isolate.query()
isolates = q.fetch()
self.assertEqual(1, len(isolates))
self.assertEqual('123', isolates[0].isolate_hash)
def testPutAndGet(self):
isolate.Put((
('Mac Builder', _CHANGE_1, 'telemetry_perf', '7c7e90be'),
('Mac Builder', _CHANGE_2, 'telemetry_perf', '38e2f262')))
isolate_hash = isolate.Get('Mac Builder', _CHANGE_1, 'telemetry_perf')
self.assertEqual(isolate_hash, '7c7e90be')
def testInitiate_FoundIsolate(self, *_):
# Seed the isolate for this change.
change = change_module.Change(
commits=[change_module.Commit('chromium', '7c7e90be')])
isolate.Put((('Mac Builder', change, 'telemetry_perf_tests',
'https://isolate.server', '7c7e90be'), ))
# Then ensure that we can find the seeded isolate for the specified
# revision.
self.assertDictEqual(
{
'find_isolate_chromium@7c7e90be': {
'bucket': 'luci.bucket',
'builder': 'Mac Builder',
'change': mock.ANY,
'isolate_hash': '7c7e90be',
'isolate_server': 'https://isolate.server',
'status': 'completed',
'target': 'telemetry_perf_tests',
},
},
task_module.Evaluate(
self.job,
event_module.Event(
type='initiate',
target_task='find_isolate_chromium@7c7e90be',
payload={}), find_isolate.Evaluator(self.job)))
def setUp(self):
super(_FindIsolateExecutionTest, self).setUp()
change = change_module.Change(
(change_module.Commit('chromium', 'f9f2b720'), ))
isolate.Put(
'https://isolate.server',
(('Mac Builder', change, 'telemetry_perf_tests', '7c7e90be'), ))
def testPutAndGet(self):
isolate_infos = (
('Mac Builder Perf', _CHANGE_1, 'telemetry_perf', '7c7e90be'),
('Mac Builder Perf', _CHANGE_2, 'telemetry_perf', '38e2f262'),
)
isolate.Put('https://isolate.server', isolate_infos)
isolate_server, isolate_hash = isolate.Get('Mac Builder Perf',
_CHANGE_1, 'telemetry_perf')
self.assertEqual(isolate_server, 'https://isolate.server')
self.assertEqual(isolate_hash, '7c7e90be')
def testPutAndGet(self):
isolate_infos = (
('Mac Builder Perf', change_test.Change(1), 'target_name', '7c7e90be'),
('Mac Builder Perf', change_test.Change(2), 'target_name', '38e2f262'),
)
isolate.Put('https://isolate.server', isolate_infos)
isolate_server, isolate_hash = isolate.Get(
'Mac Builder Perf', change_test.Change(1), 'target_name')
self.assertEqual(isolate_server, 'https://isolate.server')
self.assertEqual(isolate_hash, '7c7e90be')
def testBuilderNameMap(self):
# TODO(dtu): Remove 6 months after LUCI migration is complete.
isolate_infos = (('android_arm64-builder-perf', change_test.Change(1),
'target_name', 'https://isolate.server',
'abcd1234'), )
isolate.Put(isolate_infos)
isolate_server, isolate_hash = isolate.Get(
'Android arm64 Compile Perf', change_test.Change(1), 'target_name')
self.assertEqual(isolate_server, 'https://isolate.server')
self.assertEqual(isolate_hash, 'abcd1234')
with self.assertRaises(KeyError):
isolate.Get('Android arm64 Compile Perf', change_test.Change(2),
'target_name')
def testExpiredIsolate(self, mock_datetime):
isolate_infos = (('Mac Builder Perf', change_test.Change(1),
'target_name', 'https://isolate.server',
'7c7e90be'), )
isolate.Put(isolate_infos)
# Teleport to the future after the isolate is expired.
mock_datetime.datetime.now.return_value = (
datetime.datetime.now() + isolate.ISOLATE_EXPIRY_DURATION +
datetime.timedelta(days=1))
mock_datetime.timedelta = datetime.timedelta
with self.assertRaises(KeyError):
isolate.Get('Mac Builder Perf', change_test.Change(1),
'target_name')
def testBuildLifecycle(self, put, get_job_status):
change = change_module.Change(
(change_module.Commit('src', 'base git hash'),
change_module.Commit('v8', 'dep git hash')),
patch=change_module.Patch('https://example.org', 2565263002,
20001))
quest = find_isolate.FindIsolate('Mac Pro Perf',
'telemetry_perf_tests')
execution = quest.Start(change)
# Request a build.
put.return_value = {'build': {'id': 'build_id'}}
execution.Poll()
self.assertFalse(execution.completed)
put.assert_called_once_with(
find_isolate.BUCKET, {
'builder_name': 'Mac Builder',
'properties': {
'clobber': True,
'parent_got_revision': 'base git hash',
'deps_revision_overrides': {
'https://chromium.googlesource.com/v8/v8':
'dep git hash',
},
'patch_storage': 'rietveld',
'rietveld': 'https://example.org',
'issue': 2565263002,
'patchset': 20001,
}
})
# Check build status.
get_job_status.return_value = {'build': {'status': 'STARTED'}}
execution.Poll()
self.assertFalse(execution.completed)
get_job_status.assert_called_once_with('build_id')
# Look up isolate hash.
isolate.Put((('Mac Builder', change, 'telemetry_perf_tests',
'isolate git hash'), ))
execution.Poll()
self.assertExecutionSuccess(execution)
def setUp(self):
self.testbed = testbed.Testbed()
self.testbed.activate()
self.testbed.init_datastore_v3_stub()
self.testbed.init_memcache_stub()
ndb.get_context().clear_cache()
change = change_module.Change((change_module.Commit('src', 'f9f2b720'),))
isolate.Put((
('Mac Builder', change, 'telemetry_perf_tests', '7c7e90be'),
))
namespaced_stored_object.Set('repositories', {
'src': {
'repository_url': 'https://chromium.googlesource.com/chromium/src'
},
'v8': {
'repository_url': 'https://chromium.googlesource.com/v8/v8'
},
})
def _CheckBuildStatus(self):
"""Checks on the status of a previously requested build.
Raises:
BuildError: The build failed, was canceled, or didn't produce an isolate.
"""
build = buildbucket_service.GetJobStatus(self._build)['build']
logging.debug('buildbucket response: %s', build)
self._build_url = build.get('url')
if build['status'] != 'COMPLETED':
return
if build['result'] == 'FAILURE':
raise errors.BuildFailed(build['failure_reason'])
if build['result'] == 'CANCELED':
raise errors.BuildCancelled(build['cancelation_reason'])
# The build succeeded. Parse the result and complete this Quest.
properties = json.loads(build['result_details_json'])['properties']
commit_position = properties['got_revision_cp'].replace('@', '(at)')
suffix = 'with_patch' if 'patch_storage' in properties else 'without_patch'
key = '_'.join(('swarm_hashes', commit_position, suffix))
if self._target not in properties[key]:
raise errors.BuildIsolateNotFound()
# Cache the isolate information.
isolate.Put([
(self._builder_name, self._change, self._target,
properties['isolate_server'], properties[key][self._target])
])
result_arguments = {
'isolate_server': properties['isolate_server'],
'isolate_hash': properties[key][self._target],
}
self._Complete(result_arguments=result_arguments)
def testBuildLifecycle(self, build_parameters, put, get_job_status):
change = change_module.Change(
(change_module.Commit('chromium', 'base git hash'),
change_module.Commit('catapult', 'dep git hash')),
patch=change_module.GerritPatch('https://example.org', 672011,
'2f0d'))
quest = find_isolate.FindIsolate('Mac Builder', 'telemetry_perf_tests')
execution = quest.Start(change)
# Request a build.
build_parameters.return_value = {'patch_storage': 'gerrit'}
put.return_value = {'build': {'id': 'build_id'}}
execution.Poll()
self.assertFalse(execution.completed)
put.assert_called_once_with(
find_isolate.BUCKET, {
'builder_name': 'Mac Builder',
'properties': {
'clobber': True,
'parent_got_revision': 'base git hash',
'deps_revision_overrides': {
test.CATAPULT_URL: 'dep git hash'
},
'patch_storage': 'gerrit',
}
})
# Check build status.
get_job_status.return_value = {
'build': {
'status': 'STARTED',
'url': 'build_url',
}
}
execution.Poll()
self.assertFalse(execution.completed)
get_job_status.assert_called_once_with('build_id')
# Look up isolate hash.
isolate.Put('https://isolate.server',
(('Mac Builder', change, 'telemetry_perf_tests',
'isolate git hash'), ))
execution.Poll()
expected_result_arguments = {
'isolate_server': 'https://isolate.server',
'isolate_hash': 'isolate git hash',
}
expected_as_dict = {
'completed':
True,
'exception':
None,
'details': [
{
'key': 'builder',
'value': 'Mac Builder',
},
{
'key': 'build',
'value': 'build_id',
'url': 'build_url',
},
{
'key': 'isolate',
'value': 'isolate git hash',
'url':
'https://isolate.server/browse?digest=isolate git hash',
},
],
}
self.assertExecutionSuccess(execution)
self.assertEqual(execution.result_values, ())
self.assertEqual(execution.result_arguments, expected_result_arguments)
self.assertEqual(execution.AsDict(), expected_as_dict)
def testBuildLifecycle(self, put, get_job_status):
change = change_test.Change(123, 456, patch=True)
quest = find_isolate.FindIsolate('Mac Builder', 'telemetry_perf_tests')
execution = quest.Start(change)
# Request a build.
put.return_value = {'build': {'id': 'build_id'}}
execution.Poll()
self.assertFalse(execution.completed)
put.assert_called_once_with(find_isolate.BUCKET, {
'builder_name': 'Mac Builder',
'properties': {
'clobber': True,
'parent_got_revision': 'commit_123',
'deps_revision_overrides': {test.CATAPULT_URL: 'commit_456'},
'patch_gerrit_url': 'https://codereview.com',
'patch_issue': 567890,
'patch_project': 'project/name',
'patch_ref': 'refs/changes/90/567890/5',
'patch_repository_url': test.CHROMIUM_URL,
'patch_set': 5,
'patch_storage': 'gerrit',
}
})
# Check build status.
get_job_status.return_value = {'build': {
'status': 'STARTED',
'url': 'build_url',
}}
execution.Poll()
self.assertFalse(execution.completed)
get_job_status.assert_called_once_with('build_id')
# Look up isolate hash.
isolate.Put((
('Mac Builder', change, 'telemetry_perf_tests',
'https://isolate.server', 'isolate git hash'),
))
execution.Poll()
expected_result_arguments = {
'isolate_server': 'https://isolate.server',
'isolate_hash': 'isolate git hash',
}
expected_as_dict = {
'completed': True,
'exception': None,
'details': [
{
'key': 'builder',
'value': 'Mac Builder',
},
{
'key': 'build',
'value': 'build_id',
'url': 'build_url',
},
{
'key': 'isolate',
'value': 'isolate git hash',
'url': 'https://isolate.server/browse?digest=isolate git hash',
},
],
}
self.assertExecutionSuccess(execution)
self.assertEqual(execution.result_values, ())
self.assertEqual(execution.result_arguments, expected_result_arguments)
self.assertEqual(execution.AsDict(), expected_as_dict)