def testSetTestRunState_finalState(self): # final test run state is immutable test_run = self._CreateTestRun(state=ndb_models.TestRunState.COMPLETED) test_run_manager.SetTestRunState(test_run_id=test_run.key.id(), state=ndb_models.TestRunState.ERROR) test_run = test_run.key.get() self.assertEqual(ndb_models.TestRunState.COMPLETED, test_run.state)
def testSetTestRunState(self): # can modify non-final test run state test_run = self._CreateTestRun(state=ndb_models.TestRunState.RUNNING) test_run_manager.SetTestRunState(test_run_id=test_run.key.id(), state=ndb_models.TestRunState.ERROR, error_reason='reason') test_run = test_run.key.get() self.assertEqual(ndb_models.TestRunState.ERROR, test_run.state) self.assertEqual('reason', test_run.error_reason)
def testSetTestRunState_cancel(self, mock_cancel_request): # switching to CANCELED state will also cancel the TFC request test_run = self._CreateTestRun(state=ndb_models.TestRunState.RUNNING) test_run.request_id = 'request_id' test_run.put() test_run_manager.SetTestRunState(test_run_id=test_run.key.id(), state=ndb_models.TestRunState.CANCELED) test_run = test_run.key.get() mock_cancel_request.assert_called_with(test_run.request_id) self.assertEqual(ndb_models.TestRunState.CANCELED, test_run.state)
def KickTestPlan(test_plan_id, task_name=None): """Kicks a test plan. Args: test_plan_id: a test plan ID. task_name: a task name. Returns: True if the test plan was kicked, False if skipped Raises: NoBuildError: if any of build channels doesn't return any builds. """ logging.info('Kicking test plan %s', test_plan_id) test_plan, test_plan_status = _GetTestPlanAndStatus(test_plan_id) if not test_plan or not test_plan_status: return False if task_name: if test_plan_status.next_run_task_name != task_name: logging.warning('Unexpected cron task name: %s != %s; ignoring', task_name, test_plan_status.next_run_task_name) return False test_plan_status.next_run_task_name = None # Schedule all test runs test_runs = [] exc_info = None try: for config in test_plan.test_run_configs: test_run = retry_wrapper(test_kicker.CreateTestRun, fkwargs=dict(labels=test_plan.labels, test_plan_key=test_plan.key, test_run_config=config)) test_runs.append(test_run) except Exception: # Record exception info and cancel all scheduled runs exc_info = sys.exc_info() for test_run in test_runs: test_run_manager.SetTestRunState( test_run_id=test_run.key.id(), state=ndb_models.TestRunState.CANCELED) # Update last run info test_plan_status.last_run_time = _GetCurrentTime() test_plan_status.last_run_keys = [test_run.key for test_run in test_runs] test_plan_status.last_run_error = str(exc_info[1]) if exc_info else None test_plan_status.put() # Re-raise any caught exception if exc_info: raise six.reraise(*exc_info) return True
def Cancel(self, request): """Cancels a test run. Parameters: test_run_id: Test run ID """ try: test_run_manager.SetTestRunState( test_run_id=request.test_run_id, state=ndb_models.TestRunState.CANCELED) except test_run_manager.TestRunNotFoundError: raise endpoints.NotFoundException('No test run found for ID %s' % request.test_run_id) return message_types.VoidMessage()
def CheckPendingTestRuns(): """Check test runs to decide if requeue/cancel according timeout parameter.""" pending_test_runs = (ndb_models.TestRun.query().order( -ndb_models.TestRun.create_time, ndb_models.TestRun.key).filter( ndb_models.TestRun.state == ndb_models.TestRunState.PENDING)) for test_run in pending_test_runs: pending_start_time = datetime.datetime.utcnow() - datetime.timedelta( seconds=_PENDING_TEST_RUN_TTL) if pending_start_time > test_run.create_time: test_run_manager.SetTestRunState( test_run_id=test_run.key.id(), state=ndb_models.TestRunState.CANCELED) continue logging.info('requeue the test run %s', test_run.key.id()) test_kicker.EnqueueTestRun(test_run.key.id())
@APP.route('/<path:fake>', methods=['POST']) def HandleTask(fake=None): """Handle tasks from the test kicker queue.""" del fake retry_count = int( flask.request.headers.get('X-AppEngine-TaskRetryCount', MAX_RETRY_COUNT)) payload = json.loads(flask.request.get_data()) test_run_id = payload['test_run_id'] try: KickTestRun(test_run_id) except Exception as e: if isinstance(e, errors.BaseError) and not e.retriable: logging.exception('Non retriable error %s, no retry needed', e) elif isinstance(e, messages.ValidationError): logging.exception('Non retriable error %s, no retry needed', e) elif isinstance(e, HttpError) and e.resp.status == 400: logging.exception('Non retriable error %s, no retry needed', e) elif retry_count < MAX_RETRY_COUNT: logging.exception( 'Fail to kick test run %s, retry_count = %d', test_run_id, retry_count + 1) raise else: logging.exception( 'Fail to kick test run %s after %d retries', test_run_id, MAX_RETRY_COUNT) test_run_manager.SetTestRunState( test_run_id=test_run_id, state=ndb_models.TestRunState.ERROR, error_reason=str(e)) return common.HTTP_OK
def testSetTestRunState_notFound(self): # trying to modify unknown test run throws error with self.assertRaises(test_run_manager.TestRunNotFoundError): test_run_manager.SetTestRunState(test_run_id=666, state=ndb_models.TestRunState.ERROR)