def testSetTestRunState_finalState(self):
# final test run state is immutable
test_run = self._CreateTestRun(state=ndb_models.TestRunState.COMPLETED)
test_run_manager.SetTestRunState(test_run_id=test_run.key.id(),
state=ndb_models.TestRunState.ERROR)
test_run = test_run.key.get()
self.assertEqual(ndb_models.TestRunState.COMPLETED, test_run.state)
def testSetTestRunState(self):
# can modify non-final test run state
test_run = self._CreateTestRun(state=ndb_models.TestRunState.RUNNING)
test_run_manager.SetTestRunState(test_run_id=test_run.key.id(),
state=ndb_models.TestRunState.ERROR,
error_reason='reason')
test_run = test_run.key.get()
self.assertEqual(ndb_models.TestRunState.ERROR, test_run.state)
self.assertEqual('reason', test_run.error_reason)
def testSetTestRunState_cancel(self, mock_cancel_request):
# switching to CANCELED state will also cancel the TFC request
test_run = self._CreateTestRun(state=ndb_models.TestRunState.RUNNING)
test_run.request_id = 'request_id'
test_run.put()
test_run_manager.SetTestRunState(test_run_id=test_run.key.id(),
state=ndb_models.TestRunState.CANCELED)
test_run = test_run.key.get()
mock_cancel_request.assert_called_with(test_run.request_id)
self.assertEqual(ndb_models.TestRunState.CANCELED, test_run.state)
def KickTestPlan(test_plan_id, task_name=None):
"""Kicks a test plan.
Args:
test_plan_id: a test plan ID.
task_name: a task name.
Returns:
True if the test plan was kicked, False if skipped
Raises:
NoBuildError: if any of build channels doesn't return any builds.
"""
logging.info('Kicking test plan %s', test_plan_id)
test_plan, test_plan_status = _GetTestPlanAndStatus(test_plan_id)
if not test_plan or not test_plan_status:
return False
if task_name:
if test_plan_status.next_run_task_name != task_name:
logging.warning('Unexpected cron task name: %s != %s; ignoring',
task_name, test_plan_status.next_run_task_name)
return False
test_plan_status.next_run_task_name = None
# Schedule all test runs
test_runs = []
exc_info = None
try:
for config in test_plan.test_run_configs:
test_run = retry_wrapper(test_kicker.CreateTestRun,
fkwargs=dict(labels=test_plan.labels,
test_plan_key=test_plan.key,
test_run_config=config))
test_runs.append(test_run)
except Exception: # Record exception info and cancel all scheduled runs
exc_info = sys.exc_info()
for test_run in test_runs:
test_run_manager.SetTestRunState(
test_run_id=test_run.key.id(),
state=ndb_models.TestRunState.CANCELED)
# Update last run info
test_plan_status.last_run_time = _GetCurrentTime()
test_plan_status.last_run_keys = [test_run.key for test_run in test_runs]
test_plan_status.last_run_error = str(exc_info[1]) if exc_info else None
test_plan_status.put()
# Re-raise any caught exception
if exc_info:
raise six.reraise(*exc_info)
return True
def Cancel(self, request):
"""Cancels a test run.
Parameters:
test_run_id: Test run ID
"""
try:
test_run_manager.SetTestRunState(
test_run_id=request.test_run_id,
state=ndb_models.TestRunState.CANCELED)
except test_run_manager.TestRunNotFoundError:
raise endpoints.NotFoundException('No test run found for ID %s' %
request.test_run_id)
return message_types.VoidMessage()
def CheckPendingTestRuns():
"""Check test runs to decide if requeue/cancel according timeout parameter."""
pending_test_runs = (ndb_models.TestRun.query().order(
-ndb_models.TestRun.create_time, ndb_models.TestRun.key).filter(
ndb_models.TestRun.state == ndb_models.TestRunState.PENDING))
for test_run in pending_test_runs:
pending_start_time = datetime.datetime.utcnow() - datetime.timedelta(
seconds=_PENDING_TEST_RUN_TTL)
if pending_start_time > test_run.create_time:
test_run_manager.SetTestRunState(
test_run_id=test_run.key.id(),
state=ndb_models.TestRunState.CANCELED)
continue
logging.info('requeue the test run %s', test_run.key.id())
test_kicker.EnqueueTestRun(test_run.key.id())
@APP.route('/<path:fake>', methods=['POST'])
def HandleTask(fake=None):
"""Handle tasks from the test kicker queue."""
del fake
retry_count = int(
flask.request.headers.get('X-AppEngine-TaskRetryCount', MAX_RETRY_COUNT))
payload = json.loads(flask.request.get_data())
test_run_id = payload['test_run_id']
try:
KickTestRun(test_run_id)
except Exception as e: if isinstance(e, errors.BaseError) and not e.retriable:
logging.exception('Non retriable error %s, no retry needed', e)
elif isinstance(e, messages.ValidationError):
logging.exception('Non retriable error %s, no retry needed', e)
elif isinstance(e, HttpError) and e.resp.status == 400:
logging.exception('Non retriable error %s, no retry needed', e)
elif retry_count < MAX_RETRY_COUNT:
logging.exception(
'Fail to kick test run %s, retry_count = %d',
test_run_id, retry_count + 1)
raise
else:
logging.exception(
'Fail to kick test run %s after %d retries', test_run_id,
MAX_RETRY_COUNT)
test_run_manager.SetTestRunState(
test_run_id=test_run_id,
state=ndb_models.TestRunState.ERROR,
error_reason=str(e))
return common.HTTP_OK
def testSetTestRunState_notFound(self):
# trying to modify unknown test run throws error
with self.assertRaises(test_run_manager.TestRunNotFoundError):
test_run_manager.SetTestRunState(test_run_id=666,
state=ndb_models.TestRunState.ERROR)