def _Poll(self): if not self._task_id: self._StartTask() return if not hasattr(self, '_swarming_server'): # TODO: Remove after data migration. crbug.com/822008 self._swarming_server = 'https://chromium-swarm.appspot.com' result = swarming.Swarming( self._swarming_server).Task(self._task_id).Result() if 'bot_id' in result: # Set bot_id to pass the info back to the Quest. self._bot_id = result['bot_id'] if result['state'] == 'PENDING' or result['state'] == 'RUNNING': return if result['state'] == 'EXPIRED': raise SwarmingExpiredError(self._task_id) if result['state'] != 'COMPLETED': raise SwarmingTaskError(self._task_id, result['state']) if result['failure']: raise SwarmingTestError(self._task_id, result['exit_code']) result_arguments = { 'isolate_server': result['outputs_ref']['isolatedserver'], 'isolate_hash': result['outputs_ref']['isolated'], } self._Complete(result_arguments=result_arguments)
def testNew(self): body = { 'name': 'name', 'user': '******', 'priority': '100', 'expiration_secs': '600', 'properties': { 'inputs_ref': { 'isolated': 'isolated_hash', }, 'extra_args': ['--output-format=histograms'], 'dimensions': [ { 'key': 'id', 'value': 'bot_id' }, { 'key': 'pool', 'value': 'Chrome-perf' }, ], 'execution_timeout_secs': '3600', 'io_timeout_secs': '3600', }, 'tags': [ 'id:bot_id', 'pool:Chrome-perf', ], } response = swarming.Swarming('https://server').Tasks().New(body) self._AssertCorrectResponse(response) self._AssertRequestMadeOnce('tasks/new', method='POST', body=body)
def _Poll(self): if not self._task_id: self._StartTask() return # TODO: Pass the Swarming server through the parameters. crbug.com/822008 result = swarming.Swarming('https://chromium-swarm.appspot.com').Task( self._task_id).Result() if 'bot_id' in result: # Set bot_id to pass the info back to the Quest. self._bot_id = result['bot_id'] if result['state'] == 'PENDING' or result['state'] == 'RUNNING': return if result['state'] == 'EXPIRED': raise SwarmingExpiredError(self._task_id) if result['state'] != 'COMPLETED': raise SwarmingTaskError(self._task_id, result['state']) if result['failure']: raise SwarmingTestError(self._task_id, result['exit_code']) isolate_hash = result['outputs_ref']['isolated'] result_arguments = {'isolate_hash': isolate_hash} self._Complete(result_arguments=result_arguments)
def _StartTask(self): """Kick off a Swarming task to run a test.""" if (self._previous_execution and not self._previous_execution.bot_id and self._previous_execution.failed): raise errors.SwarmingNoBots() properties = { 'inputs_ref': { 'isolatedserver': self._isolate_server, 'isolated': self._isolate_hash, }, 'extra_args': self._extra_args, 'dimensions': self._dimensions, # TODO(dberris): Make this configuration dependent. 'execution_timeout_secs': '2700', # 45 minutes for all tasks. 'io_timeout_secs': '2700', # Also set 45 minutes for all tasks. } body = { 'name': 'Pinpoint job', 'user': '******', 'priority': '100', 'service_account': _TESTER_SERVICE_ACCOUNT, 'task_slices': [{ 'properties': properties, 'expiration_secs': '86400', # 1 day. }], } if self._swarming_tags: # This means we have additional information available about the Pinpoint # tags, and we should add those to the Swarming Pub/Sub updates. body.update({ 'tags': ['%s:%s' % (k, v) for k, v in self._swarming_tags.items()], # TODO(dberris): Consolidate constants in environment vars? 'pubsub_topic': 'projects/chromeperf/topics/pinpoint-swarming-updates', 'pubsub_auth_token': 'UNUSED', 'pubsub_userdata': json.dumps({ 'job_id': self._swarming_tags.get('pinpoint_job_id'), 'task': { 'type': 'test', 'id': self._swarming_tags.get('pinpoint_task_id'), }, }), }) logging.debug('Requesting swarming task with parameters: %s', body) response = swarming.Swarming(self._swarming_server).Tasks().New(body) logging.debug('Response: %s', response) self._task_id = response['task_id']
def _StartTask(self): """Kick off a Swarming task to run a test.""" if self._previous_execution and not self._previous_execution.bot_id: if self._previous_execution.failed: # If the previous Execution fails before it gets a bot ID, it's likely # it couldn't find any device to run on. Subsequent Executions probably # wouldn't have any better luck, and failing fast is less complex than # handling retries. raise errors.SwarmingNoBots() else: return pool_dimension = None for dimension in self._dimensions: if dimension['key'] == 'pool': pool_dimension = dimension if self._previous_execution: dimensions = [ pool_dimension, { 'key': 'id', 'value': self._previous_execution.bot_id } ] else: dimensions = self._dimensions properties = { 'inputs_ref': { 'isolatedserver': self._isolate_server, 'isolated': self._isolate_hash, }, 'extra_args': self._extra_args, 'dimensions': dimensions, 'execution_timeout_secs': '21600', # 6 hours, for rendering.mobile. 'io_timeout_secs': '14400', # 4 hours, to match the perf bots. } properties.update(_VPYTHON_PARAMS) body = { 'name': 'Pinpoint job', 'user': '******', 'priority': '100', 'expiration_secs': '86400', # 1 day. 'properties': properties, } if self._swarming_tags: body['tags'] = [ '%s:%s' % (k, self._swarming_tags[k]) for k in self._swarming_tags ] logging.debug('Requesting swarming task with parameters: %s', body) response = swarming.Swarming(self._swarming_server).Tasks().New(body) logging.debug('Response: %s', response) self._task_id = response['task_id']
def __call__(self, _): swarming_server = self.task.payload.get('swarming_server') task_id = self.task.payload.get('swarming_task_id') swarming_task = swarming.Swarming(swarming_server).Task(task_id) result = swarming_task.Result() self.task.payload.update({ 'swarming_task_result': { k: v for k, v in result.items() if k in {'bot_id', 'state', 'failure'} } }) task_state = result.get('state') if task_state in {'PENDING', 'RUNNING'}: # Commit the task payload still. task_module.UpdateTask(self.job, self.task.id, payload=self.task.payload) return if task_state == 'EXPIRED': # TODO(dberris): Do a retry, reset the payload and run an "initiate"? self.task.payload.update({ 'errors': [{ 'reason': 'SwarmingExpired', 'message': 'Request to the Swarming service expired.', }] }) task_module.UpdateTask( self.job, self.task.id, new_state='failed', payload=self.task.payload) return if task_state != 'COMPLETED': task_module.UpdateTask( self.job, self.task.id, new_state='failed', payload=self.task.payload) return self.task.payload.update({ 'isolate_server': result.get('outputs_ref', {}).get('isolatedserver'), 'isolate_hash': result.get('outputs_ref', {}).get('isolated'), }) new_state = 'completed' if result.get('failure', False): new_state = 'failed' self.task.payload.update({ 'errors': self.task.payload.get('errors', []) + [{ 'reason': 'RunTestFailed', 'message': ('Running the test failed, see isolate output: ' 'https://%s/browse?digest=%s' % ( self.task.payload.get('isolate_server'), self.task.payload.get('isolate_hash'), )) }] }) task_module.UpdateTask( self.job, self.task.id, new_state=new_state, payload=self.task.payload)
def _StartTask(self): """Kick off a Swarming task to run a test.""" if self._previous_execution and not self._previous_execution.bot_id: if self._previous_execution.failed: # If the previous Execution fails before it gets a bot ID, it's likely # it couldn't find any device to run on. Subsequent Executions probably # wouldn't have any better luck, and failing fast is less complex than # handling retries. raise RunTestError('There are no bots available to run the test.') else: return pool_dimension = None for dimension in self._dimensions: if dimension['key'] == 'pool': pool_dimension = dimension if self._previous_execution: dimensions = [ # TODO: Remove fallback after data migration. crbug.com/822008 pool_dimension or {'key': 'pool', 'value': 'Chrome-perf-pinpoint'}, {'key': 'id', 'value': self._previous_execution.bot_id} ] else: dimensions = self._dimensions if not pool_dimension: # TODO: Remove after data migration. crbug.com/822008 dimensions.insert(0, {'key': 'pool', 'value': 'Chrome-perf-pinpoint'}) if not hasattr(self, '_isolate_server'): # TODO: Remove after data migration. crbug.com/822008 self._isolate_server = 'https://isolateserver.appspot.com' properties = { 'inputs_ref': { 'isolatedserver': self._isolate_server, 'isolated': self._isolate_hash, }, 'extra_args': self._extra_args, 'dimensions': dimensions, 'execution_timeout_secs': '21600', # 6 hours, for rendering.mobile. 'io_timeout_secs': '1200', # 20 minutes, to match the perf bots. } properties.update(_VPYTHON_PARAMS) body = { 'name': 'Pinpoint job', 'user': '******', 'priority': '100', 'expiration_secs': '86400', # 1 day. 'properties': properties, } if not hasattr(self, '_swarming_server'): # TODO: Remove after data migration. crbug.com/822008 self._swarming_server = 'https://chromium-swarm.appspot.com' response = swarming.Swarming(self._swarming_server).Tasks().New(body) self._task_id = response['task_id']
def __call__(self, _): logging.debug('Scheduling a Swarming task to run a test.') self.properties.update(run_test_quest.VPYTHON_PARAMS) body = { 'name': 'Pinpoint job', 'user': '******', # TODO(dberris): Make these constants configurable? 'priority': '100', 'task_slices': [{ 'properties': self.properties, 'expiration_secs': '86400', # 1 day. }], # Since we're always going to be using the PubSub handling, we add the # tags unconditionally. 'tags': [ '%s:%s' % (k, v) for k, v in run_test_quest.SwarmingTagsFromJob(self.job).items() ], # TODO(dberris): Consolidate constants in environment vars? 'pubsub_topic': 'projects/chromeperf/topics/pinpoint-swarming-updates', 'pubsub_auth_token': 'UNUSED', 'pubsub_userdata': json.dumps({ 'job_id': self.job.job_id, 'task': { 'type': 'run_test', 'id': self.task.id, }, }), } self.task.payload.update({ 'swarming_request_body': body, }) # At this point we know we were successful in transitioning to 'ongoing'. # TODO(dberris): Figure out error-handling for Swarming request failures? response = swarming.Swarming( self.task.payload.get('swarming_server')).Tasks().New(body) logging.debug('Swarming response: %s', response) self.task.payload.update({ 'swarming_task_id': response.get('task_id'), 'tries': self.task.payload.get('tries', 0) + 1 }) # Update the payload with the task id from the Swarming request. task_module.UpdateTask(self.job, self.task.id, new_state='ongoing', payload=self.task.payload)
def _StartTask(self): """Kick off a Swarming task to run a test.""" if (self._previous_execution and not self._previous_execution.bot_id and self._previous_execution.failed): # If the previous Execution fails before it gets a bot ID, it's likely # it couldn't find any device to run on. Subsequent Executions probably # wouldn't have any better luck, and failing fast is less complex than # handling retries. raise errors.SwarmingNoBots() properties = { 'inputs_ref': { 'isolatedserver': self._isolate_server, 'isolated': self._isolate_hash, }, 'extra_args': self._extra_args, 'dimensions': self._dimensions, 'execution_timeout_secs': '21600', # 6 hours, for rendering.mobile. 'io_timeout_secs': '14400', # 4 hours, to match the perf bots. } properties.update(_VPYTHON_PARAMS) body = { 'name': 'Pinpoint job', 'user': '******', 'priority': '100', 'expiration_secs': '86400', # 1 day. 'properties': properties, } if self._swarming_tags: # This means we have additional information available about the Pinpoint # tags, and we should add those to the Swarming Pub/Sub updates. body['tags'] = ['%s:%s' % (k, v) for k, v in self._swarming_tags.items()] body['pubsub_notification'] = { # TODO(dberris): Consolidate constants in environment vars? 'topic': 'projects/chromeperf/topics/pinpoint-swarming-updates', 'auth_token': 'UNUSED', 'userdata': json.dumps({ 'job_id': self._swarming_tags.get('pinpoint_job_id'), 'task': { 'type': 'test', 'id': self._swarming_tags.get('pinpoint_task_id'), }, }), } logging.debug('Requesting swarming task with parameters: %s', body) response = swarming.Swarming(self._swarming_server).Tasks().New(body) logging.debug('Response: %s', response) self._task_id = response['task_id']
def __call__(self, _): logging.debug('Polling a swarming task; task = %s', self.task) swarming_server = self.task.payload.get('swarming_server') task_id = self.task.payload.get('swarming_task_id') swarming_task = swarming.Swarming(swarming_server).Task(task_id) result = swarming_task.Result() self.task.payload.update({ 'swarming_task_result': { k: v for k, v in result.items() if k in {'bot_id', 'state', 'failure'} } }) task_state = result.get('state') if task_state in {'PENDING', 'RUNNING'}: return if task_state == 'EXPIRED': # TODO(dberris): Do a retry, reset the payload and run an "initiate"? self.task.payload.update({ 'errors': [{ 'reason': 'SwarmingExpired', 'message': 'Request to the Swarming service expired.', }] }) task_module.UpdateTask( self.job, self.task.id, new_state='failed', payload=self.task.payload) return if task_state != 'COMPLETED': task_module.UpdateTask( self.job, self.task.id, new_state='failed', payload=self.task.payload) return self.task.payload.update({ 'isolate_server': result.get('outputs_ref', {}).get('isolatedserver'), 'isolate_hash': result.get('outputs_ref', {}).get('isolated'), }) new_state = 'completed' if result.get('failure', False): new_state = 'failed' exception_string = run_test_quest.ParseException( swarming_task.Stdout()['output']) if not exception_string: exception_string = 'No exception found in Swarming task output.' self.task.payload.update({ 'errors': [{ 'reason': 'RunTestFailed', 'message': 'Running the test failed: %s' % (exception_string,) }] }) task_module.UpdateTask( self.job, self.task.id, new_state=new_state, payload=self.task.payload)
def testList(self): response = swarming.Swarming('https://server').Bots().List( 'CkMSPWoQ', { 'pool': 'Chrome-perf', 'a': 'b' }, False, 1, True) self._AssertCorrectResponse(response) path = ('bots/list') self._AssertRequestMadeOnce(path, cursor='CkMSPWoQ', dimensions=('a:b', 'pool:Chrome-perf'), is_dead=False, limit=1, quarantined=True)
def _Poll(self): if not self._task_id: self._StartTask() return logging.debug('_RunTestExecution Polling swarming: %s', self._task_id) swarming_task = swarming.Swarming(self._swarming_server).Task( self._task_id) result = swarming_task.Result() logging.debug('swarming response: %s', result) if 'bot_id' in result: # Set bot_id to pass the info back to the Quest. self._bot_id = result['bot_id'] if result['state'] == 'PENDING' or result['state'] == 'RUNNING': return if result['state'] == 'EXPIRED': raise errors.SwarmingExpired() if result['state'] != 'COMPLETED': raise errors.SwarmingTaskError(result['state']) if result['failure']: if 'outputs_ref' not in result: task_url = 'https://%s/task?id=%s' % (self._swarming_server, self._task_id) raise errors.SwarmingTaskFailed('<a href="%s">%s</a>' % (task_url, task_url)) else: isolate_output_url = 'https://%s/browse?digest=%s' % ( result['outputs_ref']['isolatedserver'], result['outputs_ref']['isolated']) raise errors.SwarmingTaskFailed( '<a href="%s">%s</a>' % (isolate_output_url, isolate_output_url)) result_arguments = { 'isolate_server': result['outputs_ref']['isolatedserver'], 'isolate_hash': result['outputs_ref']['isolated'], } self._Complete(result_arguments=result_arguments)
def _StartTask(self): """Kick off a Swarming task to run a test.""" if self._previous_execution and not self._previous_execution.bot_id: if self._previous_execution.failed: # If the previous Execution fails before it gets a bot ID, it's likely # it couldn't find any device to run on. Subsequent Executions probably # wouldn't have any better luck, and failing fast is less complex than # handling retries. raise RunTestError( 'There are no bots available to run the test.') else: return dimensions = [{'key': 'pool', 'value': 'Chrome-perf-pinpoint'}] if self._previous_execution: dimensions.append({ 'key': 'id', 'value': self._previous_execution.bot_id }) else: dimensions += self._dimensions body = { 'name': 'Pinpoint job', 'user': '******', 'priority': '100', 'expiration_secs': '86400', # 1 day. 'properties': { 'inputs_ref': { 'isolated': self._isolate_hash }, 'extra_args': self._extra_args, 'dimensions': dimensions, 'execution_timeout_secs': '7200', # 2 hours. 'io_timeout_secs': '1200', # 20 minutes, to match the perf bots. }, } if not hasattr(self, '_swarming_server'): # TODO: Remove after data migration. crbug.com/822008 self._swarming_server = 'https://chromium-swarm.appspot.com' response = swarming.Swarming(self._swarming_server).Tasks().New(body) self._task_id = response['task_id']
def _Poll(self): if not self._task_id: self._StartTask() return if not hasattr(self, '_swarming_server'): # TODO: Remove after data migration. crbug.com/822008 self._swarming_server = 'https://chromium-swarm.appspot.com' swarming_task = swarming.Swarming(self._swarming_server).Task( self._task_id) result = swarming_task.Result() if 'bot_id' in result: # Set bot_id to pass the info back to the Quest. self._bot_id = result['bot_id'] if result['state'] == 'PENDING' or result['state'] == 'RUNNING': return if result['state'] == 'EXPIRED': raise SwarmingExpiredError( 'The swarming task expired. The bots are ' 'likely overloaded, dead, or misconfigured.') if result['state'] != 'COMPLETED': raise SwarmingTaskError('The swarming task failed with ' 'state "%s".' % result['state']) if result['failure']: exception_string = _ParseException( swarming_task.Stdout()['output']) if exception_string: raise SwarmingTestError("The test failed. The test's error " 'message was:\n%s' % exception_string) else: raise SwarmingTestError('The test failed. No Python ' 'exception was found in the log.') result_arguments = { 'isolate_server': result['outputs_ref']['isolatedserver'], 'isolate_hash': result['outputs_ref']['isolated'], } self._Complete(result_arguments=result_arguments)
def _Poll(self): if not self._task_id: self._StartTask() return logging.debug('_RunTestExecution Polling swarming: %s', self._task_id) swarming_task = swarming.Swarming(self._swarming_server).Task( self._task_id) result = swarming_task.Result() logging.debug('swarming response: %s', result) if 'bot_id' in result: # Set bot_id to pass the info back to the Quest. self._bot_id = result['bot_id'] if result['state'] == 'PENDING' or result['state'] == 'RUNNING': return if result['state'] == 'EXPIRED': raise errors.SwarmingExpired() if result['state'] != 'COMPLETED': raise errors.SwarmingTaskError(result['state']) if result['failure']: exception_string = _ParseException( swarming_task.Stdout()['output']) if exception_string: raise errors.SwarmingTaskFailed(exception_string) else: raise errors.SwarmingTaskFailedNoException() result_arguments = { 'isolate_server': result['outputs_ref']['isolatedserver'], 'isolate_hash': result['outputs_ref']['isolated'], } self._Complete(result_arguments=result_arguments)
def __call__(self, _): logging.debug('Scheduling a Swarming task to run a test.') body = { 'name': 'Pinpoint job', 'user': '******', # TODO(dberris): Make these constants configurable? 'priority': '100', 'task_slices': [{ 'properties': self.properties, 'expiration_secs': '86400', # 1 day. }], # Since we're always going to be using the PubSub handling, we add the # tags unconditionally. 'tags': [ '%s:%s' % (k, v) for k, v in run_test_quest.SwarmingTagsFromJob(self.job).items() ], # Use an explicit service account. 'service_account': run_test_quest._TESTER_SERVICE_ACCOUNT, # TODO(dberris): Consolidate constants in environment vars? 'pubsub_topic': 'projects/chromeperf/topics/pinpoint-swarming-updates', 'pubsub_auth_token': 'UNUSED', 'pubsub_userdata': json.dumps({ 'job_id': self.job.job_id, 'task': { 'type': 'run_test', 'id': self.task.id, }, }), } self.task.payload.update({ 'swarming_request_body': body, }) # Ensure that this thread/process/handler is the first to mark this task # 'ongoing'. Only proceed in scheduling a Swarming request if we're the # first one to do so. task_module.UpdateTask(self.job, self.task.id, new_state='ongoing', payload=self.task.payload) # At this point we know we were successful in transitioning to 'ongoing'. try: response = swarming.Swarming( self.task.payload.get('swarming_server')).Tasks().New(body) self.task.payload.update({ 'swarming_task_id': response.get('task_id'), 'tries': self.task.payload.get('tries', 0) + 1 }) except request.RequestError as e: self.task.payload.update({ 'errors': self.task.payload.get('errors', []) + [{ 'reason': type(e).__name__, 'message': 'Encountered failure in swarming request: %s' % (e, ), }] }) # Update the payload with the task id from the Swarming request. Note that # this could also fail to commit. task_module.UpdateTask(self.job, self.task.id, payload=self.task.payload)
def testStdout(self): response = swarming.Swarming('https://server').Task('task_id').Stdout() self._AssertCorrectResponse(response) self._AssertRequestMadeOnce('task/task_id/stdout')
def testResultWithPerformanceStats(self): response = swarming.Swarming('https://server').Task('task_id').Result(True) self._AssertCorrectResponse(response) self._AssertRequestMadeOnce( 'task/task_id/result', include_performance_stats=True)
def testCancel(self): response = swarming.Swarming('https://server').Task('task_id').Cancel() self._AssertCorrectResponse(response) self._AssertRequestMadeOnce('task/task_id/cancel', method='POST')
def testTasks(self): response = swarming.Swarming('https://server').Bot('bot_id').Tasks() self._AssertCorrectResponse(response) self._AssertRequestMadeOnce('bot/bot_id/tasks')
def testGet(self): response = swarming.Swarming('https://server').Bot('bot_id').Get() self._AssertCorrectResponse(response) self._AssertRequestMadeOnce('bot/bot_id/get')
def _StartTask(self): """Kick off a Swarming task to run a test.""" if (self._previous_execution and not self._previous_execution.bot_id and self._previous_execution.failed): raise errors.SwarmingNoBots() properties = { 'inputs_ref': { 'isolatedserver': self._isolate_server, 'isolated': self._isolate_hash, }, 'extra_args': self._extra_args, 'dimensions': self._dimensions, 'execution_timeout_secs': str(self.execution_timeout_secs or 2700), 'io_timeout_secs': str(self.execution_timeout_secs or 2700), } if self.command: properties.update({ # Set the relative current working directory to be the root of the # isolate. 'relative_cwd': self.relative_cwd, # Use the command provided in the creation of the execution. 'command': self.command + self._extra_args, }) # Swarming requires that if 'command' is present in the request, that we # not provide 'extra_args'. del properties['extra_args'] body = { 'name': 'Pinpoint job', 'user': '******', 'priority': '100', 'service_account': _TESTER_SERVICE_ACCOUNT, 'task_slices': [{ 'properties': properties, 'expiration_secs': '86400', # 1 day. }], } if self._swarming_tags: # This means we have additional information available about the Pinpoint # tags, and we should add those to the Swarming Pub/Sub updates. body.update({ 'tags': ['%s:%s' % (k, v) for k, v in self._swarming_tags.items()], # TODO(dberris): Consolidate constants in environment vars? 'pubsub_topic': 'projects/chromeperf/topics/pinpoint-swarming-updates', 'pubsub_auth_token': 'UNUSED', 'pubsub_userdata': json.dumps({ 'job_id': self._swarming_tags.get('pinpoint_job_id'), 'task': { 'type': 'test', 'id': self._swarming_tags.get('pinpoint_task_id'), }, }), }) logging.debug('Requesting swarming task with parameters: %s', body) response = swarming.Swarming(self._swarming_server).Tasks().New(body) logging.debug('Response: %s', response) self._task_id = response['task_id']