def testRecord_Estimate_Recorded(self):
now = datetime.datetime.now()
j1 = self._Job(
{
'configuration': 'linux',
'benchmark': 'foo',
'story': 'bar'
}, now - datetime.timedelta(minutes=5),
now - datetime.timedelta(minutes=4))
e1 = timing_record.GetSimilarHistoricalTimings(j1)
timing_record.RecordJobTiming(j1)
t1 = timing_record.TimingRecord.get_by_id(j1.job_id)
self.assertIsNone(e1)
self.assertIsNone(t1.estimate)
j2 = self._RecordTiming(
{
'configuration': 'linux',
'benchmark': 'foo',
'story': 'bar'
}, now - datetime.timedelta(minutes=3),
now - datetime.timedelta(minutes=2))
e2 = timing_record.GetSimilarHistoricalTimings(j2)
timing_record.RecordJobTiming(j2)
t2 = timing_record.TimingRecord.get_by_id(j2.job_id)
self.assertEqual(
int(e2[0][0].total_seconds()),
int((t2.estimate - j2.started_time).total_seconds()))
def _RecordTiming(self, args, started, completed, comparison_mode=None):
j = job.Job.New((), ())
j.comparison_mode = comparison_mode
j.arguments = args
j.started_time = started
j.updated = completed
timing_record.RecordJobTiming(j)
return j
def testRecord_Once(self):
j = self._RecordTiming(
{'configuration': 'linux', 'benchmark': 'foo', 'story': 'bar'},
datetime.datetime.now() - datetime.timedelta(minutes=1),
datetime.datetime.now())
timing_record.RecordJobTiming(j)
q = timing_record.TimingRecord.query()
results = q.fetch()
self.assertEqual(1, len(results))
def Run(self):
"""Runs this Job.
Loops through all Attempts and checks the status of each one, kicking off
tasks as needed. Does not block to wait for all tasks to finish. Also
compares adjacent Changes' results and adds any additional Attempts or
Changes as needed. If there are any incomplete tasks, schedules another
Run() call on the task queue.
"""
self.exception_details = None # In case the Job succeeds on retry.
self.task = None # In case an exception is thrown.
try:
if not self._IsTryJob():
self.state.Explore()
work_left = self.state.ScheduleWork()
# Schedule moar task.
if work_left:
self._Schedule()
else:
self._Complete()
self.retry_count = 0
except errors.RecoverableError:
try:
if not self._MaybeScheduleRetry():
self.Fail(errors.RETRY_LIMIT)
except errors.RecoverableError:
self.Fail(errors.RETRY_FAILED)
except BaseException:
self.Fail()
raise
finally:
# Don't use `auto_now` for `updated`. When we do data migration, we need
# to be able to modify the Job without changing the Job's completion time.
self.updated = datetime.datetime.now()
if self.completed:
timing_record.RecordJobTiming(self)
try:
self.put()
except (datastore_errors.Timeout,
datastore_errors.TransactionFailedError):
# Retry once.
self.put()
except datastore_errors.BadRequestError:
if self.task:
queue = taskqueue.Queue('job-queue')
queue.delete_tasks(taskqueue.Task(name=self.task))
self.task = None
# The _JobState is too large to fit in an ndb property.
# Load the Job from before we updated it, and fail it.
job = self.key.get(use_cache=False)
job.task = None
job.Fail()
job.updated = datetime.datetime.now()
job.put()
raise
def Run(self):
"""Runs this Job.
Loops through all Attempts and checks the status of each one, kicking off
tasks as needed. Does not block to wait for all tasks to finish. Also
compares adjacent Changes' results and adds any additional Attempts or
Changes as needed. If there are any incomplete tasks, schedules another
Run() call on the task queue.
"""
self.exception_details = None # In case the Job succeeds on retry.
self.task = None # In case an exception is thrown.
try:
if self.use_execution_engine:
# Treat this as if it's a poll, and run the handler here.
context = task_module.Evaluate(
self,
event_module.Event(type='initiate',
target_task=None,
payload={}),
task_evaluator.ExecutionEngine(self))
result_status = context.get('performance_bisection',
{}).get('status')
if result_status not in {'failed', 'completed'}:
return
if result_status == 'failed':
execution_errors = context['find_culprit'].get(
'errors', [])
if execution_errors:
self.exception_details = execution_errors[0]
self._Complete()
return
if not self._IsTryJob():
self.state.Explore()
work_left = self.state.ScheduleWork()
# Schedule moar task.
if work_left:
self._Schedule()
else:
self._Complete()
self.retry_count = 0
except errors.RecoverableError as e:
try:
if not self._MaybeScheduleRetry():
self.Fail(errors.JobRetryLimitExceededError(wrapped_exc=e))
except errors.RecoverableError as e:
self.Fail(errors.JobRetryFailed(wrapped_exc=e))
except BaseException:
self.Fail()
raise
finally:
# Don't use `auto_now` for `updated`. When we do data migration, we need
# to be able to modify the Job without changing the Job's completion time.
self.updated = datetime.datetime.now()
if self.completed:
timing_record.RecordJobTiming(self)
try:
self.put()
except (datastore_errors.Timeout,
datastore_errors.TransactionFailedError):
# Retry once.
self.put()
except datastore_errors.BadRequestError:
if self.task:
queue = taskqueue.Queue('job-queue')
queue.delete_tasks(taskqueue.Task(name=self.task))
self.task = None
# The _JobState is too large to fit in an ndb property.
# Load the Job from before we updated it, and fail it.
job = self.key.get(use_cache=False)
job.task = None
job.Fail()
job.updated = datetime.datetime.now()
job.put()
raise
def _RecordTiming(self, args, started, completed, comparison_mode=None):
j = self._Job(args, started, completed, comparison_mode)
timing_record.RecordJobTiming(j)
return j
