def test_set_from_run_result_two_tries(self):
request = mkreq(_gen_request())
result_summary = task_result.new_result_summary(request)
run_result_1 = task_result.new_run_result(
request, 1, 'localhost', 'abc', {})
run_result_2 = task_result.new_run_result(
request, 2, 'localhost', 'abc', {})
self.assertTrue(result_summary.need_update_from_run_result(run_result_1))
run_result_2.modified_ts = utils.utcnow()
result_summary.modified_ts = utils.utcnow()
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result_2)))
self.assertTrue(result_summary.need_update_from_run_result(run_result_1))
run_result_1.modified_ts = utils.utcnow()
result_summary.set_from_run_result(run_result_1, request)
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result_1)))
result_summary = result_summary.key.get()
self.assertFalse(result_summary.need_update_from_run_result(run_result_1))
self.assertTrue(result_summary.need_update_from_run_result(run_result_2))
run_result_2.modified_ts = utils.utcnow()
result_summary.set_from_run_result(run_result_2, request)
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result_2)))
result_summary = result_summary.key.get()
self.assertEqual(2, result_summary.try_number)
self.assertFalse(result_summary.need_update_from_run_result(run_result_1))
def test_set_from_run_result_two_tries(self):
request = task_request.make_request(_gen_request(), True)
result_summary = task_result.new_result_summary(request)
run_result_1 = task_result.new_run_result(
request, 1, 'localhost', 'abc', {})
run_result_2 = task_result.new_run_result(
request, 2, 'localhost', 'abc', {})
self.assertTrue(result_summary.need_update_from_run_result(run_result_1))
run_result_2.modified_ts = utils.utcnow()
result_summary.modified_ts = utils.utcnow()
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result_2)))
self.assertTrue(result_summary.need_update_from_run_result(run_result_1))
run_result_1.modified_ts = utils.utcnow()
result_summary.set_from_run_result(run_result_1, request)
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result_1)))
result_summary = result_summary.key.get()
self.assertFalse(result_summary.need_update_from_run_result(run_result_1))
self.assertTrue(result_summary.need_update_from_run_result(run_result_2))
run_result_2.modified_ts = utils.utcnow()
result_summary.set_from_run_result(run_result_2, request)
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result_2)))
result_summary = result_summary.key.get()
self.assertEqual(2, result_summary.try_number)
self.assertFalse(result_summary.need_update_from_run_result(run_result_1))
def test_new_run_result(self):
request = task_request.make_request(_gen_request(), True)
actual = task_result.new_run_result(
request, 1, "localhost", "abc", {"id": ["localhost"], "foo": ["bar", "biz"]}
)
expected = {
"abandoned_ts": None,
"bot_dimensions": {"id": ["localhost"], "foo": ["bar", "biz"]},
"bot_id": "localhost",
"bot_version": "abc",
"children_task_ids": [],
"completed_ts": None,
"cost_usd": 0.0,
"durations": [],
"exit_codes": [],
"failure": False,
"id": "1d69b9f088008811",
"internal_failure": False,
"modified_ts": None,
"outputs_ref": None,
"server_versions": ["v1a"],
"started_ts": self.now,
"state": task_result.State.RUNNING,
"try_number": 1,
}
self.assertEqual(expected, actual.to_dict())
self.assertEqual(50, actual.priority)
self.assertEqual(False, actual.can_be_canceled)
def run():
# 2 GET, 1 PUT at the end.
to_run_future = to_run_key.get_async()
result_summary_future = result_summary_key.get_async()
to_run = to_run_future.get_result()
result_summary = result_summary_future.get_result()
if not to_run:
logging.error('Missing TaskToRun?\n%s', result_summary.task_id)
return None
if not to_run.is_reapable:
logging.info('%s is not reapable', result_summary.task_id)
return None
if result_summary.bot_id == bot_id:
# This means two things, first it's a retry, second it's that the first
# try failed and the retry is being reaped by the same bot. Deny that, as
# the bot may be deeply broken and could be in a killing spree.
logging.warning('%s can\'t retry its own internal failure task',
result_summary.task_id)
return None
to_run.queue_number = None
run_result = task_result.new_run_result(
request, (result_summary.try_number or 0) + 1, bot_id, bot_version,
bot_dimensions)
run_result.modified_ts = now
result_summary.set_from_run_result(run_result, request)
ndb.put_multi([to_run, run_result, result_summary])
return run_result
def test_set_from_run_result_two_server_versions(self):
request = _gen_request()
result_summary = task_result.new_result_summary(request)
to_run = task_to_run.new_task_to_run(request, 1, 0)
run_result = task_result.new_run_result(request, to_run, 'localhost',
'abc', {})
run_result.started_ts = utils.utcnow()
self.assertTrue(result_summary.need_update_from_run_result(run_result))
result_summary.modified_ts = utils.utcnow()
run_result.modified_ts = utils.utcnow()
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
self.assertTrue(result_summary.need_update_from_run_result(run_result))
ndb.transaction(
lambda: result_summary.set_from_run_result(run_result, request))
ndb.transaction(lambda: ndb.put_multi([result_summary]))
run_result.signal_server_version('new-version')
run_result.modified_ts = utils.utcnow()
ndb.transaction(
lambda: result_summary.set_from_run_result(run_result, request))
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
self.assertEqual(['v1a', 'new-version'],
run_result.key.get().server_versions)
self.assertEqual(['v1a', 'new-version'],
result_summary.key.get().server_versions)
def test_new_run_result(self):
request = task_request.make_request(_gen_request(), True)
actual = task_result.new_run_result(
request, 1, 'localhost', 'abc',
{'id': ['localhost'], 'foo': ['bar', 'biz']})
expected = {
'abandoned_ts': None,
'bot_dimensions': {'id': ['localhost'], 'foo': ['bar', 'biz']},
'bot_id': 'localhost',
'bot_version': 'abc',
'children_task_ids': [],
'completed_ts': None,
'cost_usd': 0.,
'durations': [],
'exit_codes': [],
'failure': False,
'id': '1d69b9f088008811',
'internal_failure': False,
'modified_ts': None,
'outputs_ref': None,
'server_versions': ['v1a'],
'started_ts': self.now,
'state': task_result.State.RUNNING,
'try_number': 1,
}
self.assertEqual(expected, actual.to_dict())
self.assertEqual(50, actual.priority)
self.assertEqual(False, actual.can_be_canceled)
def test_new_run_result(self):
request = mkreq(_gen_request())
actual = task_result.new_run_result(
request, 1, 'localhost', 'abc',
{'id': ['localhost'], 'foo': ['bar', 'biz']})
expected = {
'abandoned_ts': None,
'bot_dimensions': {'id': ['localhost'], 'foo': ['bar', 'biz']},
'bot_id': 'localhost',
'bot_version': 'abc',
'children_task_ids': [],
'cipd_pins': None,
'completed_ts': None,
'cost_usd': 0.,
'duration': None,
'exit_code': None,
'failure': False,
'id': '1d69b9f088008811',
'internal_failure': False,
'modified_ts': None,
'outputs_ref': None,
'server_versions': ['v1a'],
'started_ts': self.now,
'state': task_result.State.RUNNING,
'try_number': 1,
}
self.assertEqual(expected, actual.to_dict())
self.assertEqual(50, actual.request.priority)
self.assertEqual(False, actual.can_be_canceled)
def setUp(self): super(TestOutput, self).setUp() request = task_request.make_request(_gen_request_data()) result_summary = task_result.new_result_summary(request) result_summary.modified_ts = utils.utcnow() ndb.transaction(result_summary.put) self.run_result = task_result.new_run_result(request, 1, 'localhost', 'abc') self.run_result.modified_ts = utils.utcnow() result_summary.set_from_run_result(self.run_result, request) ndb.transaction(lambda: ndb.put_multi((result_summary, self.run_result))) self.run_result = self.run_result.key.get()
def setUp(self):
super(TestOutput, self).setUp()
request = task_request.make_request(_gen_request(), True)
result_summary = task_result.new_result_summary(request)
result_summary.modified_ts = utils.utcnow()
ndb.transaction(result_summary.put)
self.run_result = task_result.new_run_result(request, 1, "localhost", "abc", {})
self.run_result.modified_ts = utils.utcnow()
result_summary.set_from_run_result(self.run_result, request)
ndb.transaction(lambda: ndb.put_multi((result_summary, self.run_result)))
self.run_result = self.run_result.key.get()
def test_set_from_run_result(self):
request = task_request.make_request(_gen_request(), True)
result_summary = task_result.new_result_summary(request)
run_result = task_result.new_run_result(request, 1, 'localhost', 'abc', {})
self.assertTrue(result_summary.need_update_from_run_result(run_result))
result_summary.modified_ts = utils.utcnow()
run_result.modified_ts = utils.utcnow()
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
self.assertTrue(result_summary.need_update_from_run_result(run_result))
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi([result_summary]))
self.assertFalse(result_summary.need_update_from_run_result(run_result))
def test_set_from_run_result(self):
request = mkreq(_gen_request())
result_summary = task_result.new_result_summary(request)
run_result = task_result.new_run_result(request, 1, 'localhost', 'abc', {})
self.assertTrue(result_summary.need_update_from_run_result(run_result))
result_summary.modified_ts = utils.utcnow()
run_result.modified_ts = utils.utcnow()
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
self.assertTrue(result_summary.need_update_from_run_result(run_result))
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi([result_summary]))
self.assertFalse(result_summary.need_update_from_run_result(run_result))
def test_run_result_timeout(self):
request = task_request.make_request(_gen_request(), True)
result_summary = task_result.new_result_summary(request)
result_summary.modified_ts = utils.utcnow()
ndb.transaction(result_summary.put)
run_result = task_result.new_run_result(request, 1, 'localhost', 'abc', {})
run_result.state = task_result.State.TIMED_OUT
run_result.completed_ts = utils.utcnow()
run_result.modified_ts = utils.utcnow()
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi((run_result, result_summary)))
run_result = run_result.key.get()
result_summary = result_summary.key.get()
self.assertEqual(True, run_result.failure)
self.assertEqual(True, result_summary.failure)
def test_run_result_timeout(self): request = task_request.make_request(_gen_request_data()) result_summary = task_result.new_result_summary(request) result_summary.modified_ts = utils.utcnow() ndb.transaction(result_summary.put) run_result = task_result.new_run_result(request, 1, 'localhost', 'abc') run_result.state = task_result.State.TIMED_OUT run_result.completed_ts = utils.utcnow() run_result.modified_ts = utils.utcnow() result_summary.set_from_run_result(run_result, request) ndb.transaction(lambda: ndb.put_multi((run_result, result_summary))) run_result = run_result.key.get() result_summary = result_summary.key.get() self.assertEqual(True, run_result.failure) self.assertEqual(True, result_summary.failure)
def setUp(self):
super(TestOutput, self).setUp()
request = _gen_request()
result_summary = task_result.new_result_summary(request)
result_summary.modified_ts = utils.utcnow()
ndb.transaction(result_summary.put)
to_run = task_to_run.new_task_to_run(request, 1, 0)
self.run_result = task_result.new_run_result(request, to_run,
'localhost', 'abc', {})
self.run_result.started_ts = result_summary.modified_ts
self.run_result.modified_ts = utils.utcnow()
ndb.transaction(lambda: result_summary.set_from_run_result(
self.run_result, request))
ndb.transaction(lambda: ndb.put_multi(
(result_summary, self.run_result)))
self.run_result = self.run_result.key.get()
def test_yield_run_result_keys_with_dead_bot(self):
request = task_request.make_request(_gen_request(), True)
result_summary = task_result.new_result_summary(request)
result_summary.modified_ts = utils.utcnow()
ndb.transaction(result_summary.put)
run_result = task_result.new_run_result(request, 1, "localhost", "abc", {})
run_result.completed_ts = utils.utcnow()
run_result.modified_ts = utils.utcnow()
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi((run_result, result_summary)))
self.mock_now(self.now + task_result.BOT_PING_TOLERANCE)
self.assertEqual([], list(task_result.yield_run_result_keys_with_dead_bot()))
self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 1)
self.assertEqual([run_result.key], list(task_result.yield_run_result_keys_with_dead_bot()))
def test_new_run_result(self):
request = _gen_request()
to_run = task_to_run.new_task_to_run(request, 1, 0)
actual = task_result.new_run_result(request, to_run, u'localhost',
u'abc', {
u'id': [u'localhost'],
u'foo': [u'bar', u'biz']
})
actual.modified_ts = self.now
actual.started_ts = self.now
# Trigger _pre_put_hook().
actual.put()
expected = self._gen_result(modified_ts=self.now, started_ts=self.now)
self.assertEqual(expected, actual.to_dict())
self.assertEqual(50, actual.request.priority)
self.assertEqual(True, actual.can_be_canceled)
self.assertEqual(0, actual.current_task_slice)
def test_set_from_run_result_two_server_versions(self):
request = task_request.make_request(_gen_request(), True)
result_summary = task_result.new_result_summary(request)
run_result = task_result.new_run_result(request, 1, "localhost", "abc", {})
self.assertTrue(result_summary.need_update_from_run_result(run_result))
result_summary.modified_ts = utils.utcnow()
run_result.modified_ts = utils.utcnow()
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
self.assertTrue(result_summary.need_update_from_run_result(run_result))
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi([result_summary]))
run_result.signal_server_version("new-version")
run_result.modified_ts = utils.utcnow()
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
self.assertEqual(["v1a", "new-version"], run_result.key.get().server_versions)
self.assertEqual(["v1a", "new-version"], result_summary.key.get().server_versions)
def test_yield_run_result_keys_with_dead_bot(self):
request = mkreq(_gen_request())
result_summary = task_result.new_result_summary(request)
result_summary.modified_ts = utils.utcnow()
ndb.transaction(result_summary.put)
run_result = task_result.new_run_result(request, 1, 'localhost', 'abc', {})
run_result.completed_ts = utils.utcnow()
run_result.modified_ts = utils.utcnow()
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi((run_result, result_summary)))
self.mock_now(self.now + task_result.BOT_PING_TOLERANCE)
self.assertEqual(
[], list(task_result.yield_run_result_keys_with_dead_bot()))
self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 1)
self.assertEqual(
[run_result.key],
list(task_result.yield_run_result_keys_with_dead_bot()))
def test_run_result_timeout(self):
request = _gen_request()
result_summary = task_result.new_result_summary(request)
result_summary.modified_ts = utils.utcnow()
ndb.transaction(result_summary.put)
to_run = task_to_run.new_task_to_run(request, 1, 0)
run_result = task_result.new_run_result(request, to_run, 'localhost',
'abc', {})
run_result.state = task_result.State.TIMED_OUT
run_result.duration = 0.1
run_result.exit_code = -1
run_result.started_ts = utils.utcnow()
run_result.completed_ts = run_result.started_ts
run_result.modified_ts = run_result.started_ts
ndb.transaction(
lambda: result_summary.set_from_run_result(run_result, request))
ndb.transaction(lambda: ndb.put_multi((run_result, result_summary)))
run_result = run_result.key.get()
result_summary = result_summary.key.get()
self.assertEqual(True, run_result.failure)
self.assertEqual(True, result_summary.failure)
def run():
# 2 GET, 1 PUT at the end.
to_run_future = to_run_key.get_async()
result_summary_future = result_summary_key.get_async()
to_run = to_run_future.get_result()
if not to_run or not to_run.is_reapable:
result_summary_future.wait()
return None
result_summary = result_summary_future.get_result()
if result_summary.bot_id == bot_id:
# This means two things, first it's a retry, second it's that the first
# try failed and the retry is being reaped by the same bot. Deny that, as
# the bot may be deeply broken and could be in a killing spree.
return None
to_run.queue_number = None
run_result = task_result.new_run_result(
request, (result_summary.try_number or 0) + 1, bot_id, bot_version)
run_result.modified_ts = now
result_summary.set_from_run_result(run_result, request)
ndb.put_multi([to_run, run_result, result_summary])
return run_result
def test_set_from_run_result_two_server_versions(self):
request = task_request.make_request(_gen_request_data())
result_summary = task_result.new_result_summary(request)
run_result = task_result.new_run_result(request, 1, 'localhost', 'abc')
self.assertTrue(result_summary.need_update_from_run_result(run_result))
result_summary.modified_ts = utils.utcnow()
run_result.modified_ts = utils.utcnow()
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
self.assertTrue(result_summary.need_update_from_run_result(run_result))
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi([result_summary]))
run_result.signal_server_version('new-version')
run_result.modified_ts = utils.utcnow()
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
self.assertEqual(
['v1a', 'new-version'], run_result.key.get().server_versions)
self.assertEqual(
['v1a', 'new-version'], result_summary.key.get().server_versions)
def run():
# 3 GET, 1 PUT at the end.
to_run_future = to_run_key.get_async()
result_summary_future = result_summary_key.get_async()
to_run = to_run_future.get_result()
t = request.task_slice(to_run.task_slice_index)
if t.properties.has_secret_bytes:
secret_bytes_future = request.secret_bytes_key.get_async()
result_summary = result_summary_future.get_result()
orig_summary_state = result_summary.state
secret_bytes = None
if t.properties.has_secret_bytes:
secret_bytes = secret_bytes_future.get_result()
if not to_run:
logging.error('Missing TaskToRun?\n%s', result_summary.task_id)
return None, None
if not to_run.is_reapable:
logging.info('%s is not reapable', result_summary.task_id)
return None, None
if result_summary.bot_id == bot_id:
# This means two things, first it's a retry, second it's that the first
# try failed and the retry is being reaped by the same bot. Deny that, as
# the bot may be deeply broken and could be in a killing spree.
# TODO(maruel): Allow retry for bot locked task using 'id' dimension.
logging.warning(
'%s can\'t retry its own internal failure task',
result_summary.task_id)
return None, None
to_run.queue_number = None
run_result = task_result.new_run_result(
request, to_run, bot_id, bot_version, bot_dimensions)
# Upon bot reap, both .started_ts and .modified_ts matches. They differ on
# the first ping.
run_result.started_ts = now
run_result.modified_ts = now
result_summary.set_from_run_result(run_result, request)
ndb.put_multi([to_run, run_result, result_summary])
if result_summary.state != orig_summary_state:
_maybe_pubsub_notify_via_tq(result_summary, request)
return run_result, secret_bytes
def run():
# 2 GET, 1 PUT at the end.
to_run_future = to_run_key.get_async()
result_summary_future = result_summary_key.get_async()
to_run = to_run_future.get_result()
if not to_run or not to_run.is_reapable:
result_summary_future.wait()
return None
result_summary = result_summary_future.get_result()
if result_summary.bot_id == bot_id:
# This means two things, first it's a retry, second it's that the first
# try failed and the retry is being reaped by the same bot. Deny that, as
# the bot may be deeply broken and could be in a killing spree.
return None
to_run.queue_number = None
run_result = task_result.new_run_result(
request, (result_summary.try_number or 0) + 1, bot_id, bot_version, bot_dimensions
)
run_result.modified_ts = now
result_summary.set_from_run_result(run_result, request)
ndb.put_multi([to_run, run_result, result_summary])
return run_result
def test_new_run_result_duration_no_exit_code(self):
request = _gen_request()
to_run = task_to_run.new_task_to_run(request, 1, 0)
actual = task_result.new_run_result(request, to_run, u'localhost',
u'abc', {
u'id': [u'localhost'],
u'foo': [u'bar', u'biz']
})
actual.modified_ts = self.now
actual.started_ts = self.now
actual.duration = 1.
actual.state = task_result.State.COMPLETED
# Trigger _pre_put_hook().
with self.assertRaises(datastore_errors.BadValueError):
actual.put()
actual.state = task_result.State.TIMED_OUT
actual.put()
expected = self._gen_result(duration=1.,
modified_ts=self.now,
failure=True,
started_ts=self.now,
state=task_result.State.TIMED_OUT)
self.assertEqual(expected, actual.to_dict())
def test_integration(self):
# Creates a TaskRequest, along its TaskResultSummary and TaskToRun. Have a
# bot reap the task, and complete the task. Ensure the resulting
# TaskResultSummary and TaskRunResult are properly updated.
request = mkreq(_gen_request())
result_summary = task_result.new_result_summary(request)
to_run = task_to_run.new_task_to_run(request)
result_summary.modified_ts = utils.utcnow()
ndb.transaction(lambda: ndb.put_multi([result_summary, to_run]))
expected = {
'abandoned_ts': None,
'bot_dimensions': None,
'bot_id': None,
'bot_version': None,
'cipd_pins': None,
'children_task_ids': [],
'completed_ts': None,
'costs_usd': [],
'cost_saved_usd': None,
'created_ts': self.now,
'deduped_from': None,
'duration': None,
'exit_code': None,
'failure': False,
'id': '1d69b9f088008810',
'internal_failure': False,
'modified_ts': self.now,
'name': u'Request name',
'outputs_ref': None,
'properties_hash': None,
'server_versions': [],
'started_ts': None,
'state': task_result.State.PENDING,
'try_number': None,
'tags': [
u'pool:default',
u'priority:50',
u'service_account:none',
u'tag:1',
u'user:Jesus',
],
'user': u'Jesus',
}
self.assertEqual(expected, result_summary.to_dict())
# Nothing changed 2 secs later except latency.
self.mock_now(self.now, 2)
self.assertEqual(expected, result_summary.to_dict())
# Task is reaped after 2 seconds (4 secs total).
reap_ts = self.now + datetime.timedelta(seconds=4)
self.mock_now(reap_ts)
to_run.queue_number = None
to_run.put()
run_result = task_result.new_run_result(request, 1, 'localhost', 'abc', {})
run_result.modified_ts = utils.utcnow()
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
expected = {
'abandoned_ts': None,
'bot_dimensions': {},
'bot_id': u'localhost',
'bot_version': u'abc',
'cipd_pins': None,
'children_task_ids': [],
'completed_ts': None,
'costs_usd': [0.],
'cost_saved_usd': None,
'created_ts': self.now,
'deduped_from': None,
'duration': None,
'exit_code': None,
'failure': False,
'id': '1d69b9f088008810',
'internal_failure': False,
'modified_ts': reap_ts,
'name': u'Request name',
'outputs_ref': None,
'properties_hash': None,
'server_versions': [u'v1a'],
'started_ts': reap_ts,
'state': task_result.State.RUNNING,
'tags': [
u'pool:default',
u'priority:50',
u'service_account:none',
u'tag:1',
u'user:Jesus',
],
'try_number': 1,
'user': u'Jesus',
}
self.assertEqual(expected, result_summary.key.get().to_dict())
# Task completed after 2 seconds (6 secs total), the task has been running
# for 2 seconds.
complete_ts = self.now + datetime.timedelta(seconds=6)
self.mock_now(complete_ts)
run_result.completed_ts = complete_ts
run_result.duration = 0.1
run_result.exit_code = 0
run_result.state = task_result.State.COMPLETED
run_result.modified_ts = utils.utcnow()
task_result.PerformanceStats(
key=task_pack.run_result_key_to_performance_stats_key(run_result.key),
bot_overhead=0.1,
isolated_download=task_result.OperationStats(
duration=0.05, initial_number_items=10, initial_size=10000,
items_cold='foo', items_hot='bar'),
isolated_upload=task_result.OperationStats(
duration=0.01, items_cold='foo')).put()
ndb.transaction(lambda: ndb.put_multi(run_result.append_output('foo', 0)))
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
expected = {
'abandoned_ts': None,
'bot_dimensions': {},
'bot_id': u'localhost',
'bot_version': u'abc',
'cipd_pins': None,
'children_task_ids': [],
'completed_ts': complete_ts,
'costs_usd': [0.],
'cost_saved_usd': None,
'created_ts': self.now,
'deduped_from': None,
'duration': 0.1,
'exit_code': 0,
'failure': False,
'id': '1d69b9f088008810',
'internal_failure': False,
'modified_ts': complete_ts,
'name': u'Request name',
'outputs_ref': None,
'properties_hash': None,
'server_versions': [u'v1a'],
'started_ts': reap_ts,
'state': task_result.State.COMPLETED,
'tags': [
u'pool:default',
u'priority:50',
u'service_account:none',
u'tag:1',
u'user:Jesus',
],
'try_number': 1,
'user': u'Jesus',
}
self.assertEqual(expected, result_summary.key.get().to_dict())
expected = {
'bot_overhead': 0.1,
'isolated_download': {
'duration': 0.05,
'initial_number_items': 10,
'initial_size': 10000,
'items_cold': 'foo',
'items_hot': 'bar',
},
'isolated_upload': {
'duration': 0.01,
'initial_number_items': None,
'initial_size': None,
'items_cold': 'foo',
'items_hot': None,
},
'package_installation': {
'duration': None,
'initial_number_items': None,
'initial_size': None,
'items_cold': None,
'items_hot': None,
},
}
self.assertEqual(expected, result_summary.performance_stats.to_dict())
self.assertEqual('foo', result_summary.get_output())
self.assertEqual(
datetime.timedelta(seconds=2),
result_summary.duration_as_seen_by_server)
self.assertEqual(
datetime.timedelta(seconds=0.1),
result_summary.duration_now(utils.utcnow()))
self.assertEqual(
datetime.timedelta(seconds=4), result_summary.pending)
self.assertEqual(
datetime.timedelta(seconds=4),
result_summary.pending_now(utils.utcnow()))
self.assertEqual(
task_pack.pack_result_summary_key(result_summary.key),
result_summary.task_id)
self.assertEqual(complete_ts, result_summary.ended_ts)
self.assertEqual(
task_pack.pack_run_result_key(run_result.key),
run_result.task_id)
self.assertEqual(complete_ts, run_result.ended_ts)
def test_integration(self):
# Creates a TaskRequest, along its TaskResultSummary and TaskToRun. Have a
# bot reap the task, and complete the task. Ensure the resulting
# TaskResultSummary and TaskRunResult are properly updated.
request = task_request.make_request(_gen_request(), True)
result_summary = task_result.new_result_summary(request)
to_run = task_to_run.new_task_to_run(request)
result_summary.modified_ts = utils.utcnow()
ndb.transaction(lambda: ndb.put_multi([result_summary, to_run]))
expected = {
'abandoned_ts': None,
'bot_dimensions': None,
'bot_id': None,
'bot_version': None,
'children_task_ids': [],
'completed_ts': None,
'costs_usd': [],
'cost_saved_usd': None,
'created_ts': self.now,
'deduped_from': None,
'durations': [],
'exit_codes': [],
'failure': False,
'id': '1d69b9f088008810',
'internal_failure': False,
'modified_ts': self.now,
'name': u'Request name',
'outputs_ref': None,
'properties_hash': None,
'server_versions': [],
'started_ts': None,
'state': task_result.State.PENDING,
'try_number': None,
'tags': [u'priority:50', u'tag:1', u'user:Jesus'],
'user': u'Jesus',
}
self.assertEqual(expected, result_summary.to_dict())
# Nothing changed 2 secs later except latency.
self.mock_now(self.now, 2)
self.assertEqual(expected, result_summary.to_dict())
# Task is reaped after 2 seconds (4 secs total).
reap_ts = self.now + datetime.timedelta(seconds=4)
self.mock_now(reap_ts)
to_run.queue_number = None
to_run.put()
run_result = task_result.new_run_result(request, 1, 'localhost', 'abc',
{})
run_result.modified_ts = utils.utcnow()
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
expected = {
'abandoned_ts': None,
'bot_dimensions': {},
'bot_id': u'localhost',
'bot_version': u'abc',
'children_task_ids': [],
'completed_ts': None,
'costs_usd': [0.],
'cost_saved_usd': None,
'created_ts': self.now,
'deduped_from': None,
'durations': [],
'exit_codes': [],
'failure': False,
'id': '1d69b9f088008810',
'internal_failure': False,
'modified_ts': reap_ts,
'name': u'Request name',
'outputs_ref': None,
'properties_hash': None,
'server_versions': [u'v1a'],
'started_ts': reap_ts,
'state': task_result.State.RUNNING,
'tags': [u'priority:50', u'tag:1', u'user:Jesus'],
'try_number': 1,
'user': u'Jesus',
}
self.assertEqual(expected, result_summary.key.get().to_dict())
# Task completed after 2 seconds (6 secs total), the task has been running
# for 2 seconds.
complete_ts = self.now + datetime.timedelta(seconds=6)
self.mock_now(complete_ts)
run_result.completed_ts = complete_ts
run_result.exit_codes.append(0)
run_result.state = task_result.State.COMPLETED
run_result.modified_ts = utils.utcnow()
ndb.transaction(
lambda: ndb.put_multi(run_result.append_output(0, 'foo', 0)))
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
expected = {
'abandoned_ts': None,
'bot_dimensions': {},
'bot_id': u'localhost',
'bot_version': u'abc',
'children_task_ids': [],
'completed_ts': complete_ts,
'costs_usd': [0.],
'cost_saved_usd': None,
'created_ts': self.now,
'deduped_from': None,
'durations': [],
'exit_codes': [0],
'failure': False,
'id': '1d69b9f088008810',
'internal_failure': False,
'modified_ts': complete_ts,
'name': u'Request name',
'outputs_ref': None,
'properties_hash': None,
'server_versions': [u'v1a'],
'started_ts': reap_ts,
'state': task_result.State.COMPLETED,
'tags': [u'priority:50', u'tag:1', u'user:Jesus'],
'try_number': 1,
'user': u'Jesus',
}
self.assertEqual(expected, result_summary.key.get().to_dict())
self.assertEqual(['foo'], list(result_summary.get_outputs()))
self.assertEqual(datetime.timedelta(seconds=2),
result_summary.duration_total)
self.assertEqual(datetime.timedelta(seconds=2),
result_summary.duration_now(utils.utcnow()))
self.assertEqual(datetime.timedelta(seconds=4), result_summary.pending)
self.assertEqual(datetime.timedelta(seconds=4),
result_summary.pending_now(utils.utcnow()))
self.assertEqual(task_pack.pack_result_summary_key(result_summary.key),
result_summary.task_id)
self.assertEqual(complete_ts, result_summary.ended_ts)
self.assertEqual(task_pack.pack_run_result_key(run_result.key),
run_result.task_id)
self.assertEqual(complete_ts, run_result.ended_ts)
def test_integration(self):
# Creates a TaskRequest, along its TaskResultSummary and TaskToRun. Have a
# bot reap the task, and complete the task. Ensure the resulting
# TaskResultSummary and TaskRunResult are properly updated.
request = task_request.make_request(_gen_request(), True)
result_summary = task_result.new_result_summary(request)
to_run = task_to_run.new_task_to_run(request)
result_summary.modified_ts = utils.utcnow()
ndb.transaction(lambda: ndb.put_multi([result_summary, to_run]))
expected = {
'abandoned_ts': None,
'bot_dimensions': None,
'bot_id': None,
'bot_version': None,
'children_task_ids': [],
'completed_ts': None,
'costs_usd': [],
'cost_saved_usd': None,
'created_ts': self.now,
'deduped_from': None,
'durations': [],
'exit_codes': [],
'failure': False,
'id': '1d69b9f088008810',
'internal_failure': False,
'modified_ts': self.now,
'name': u'Request name',
'outputs_ref': None,
'properties_hash': None,
'server_versions': [],
'started_ts': None,
'state': task_result.State.PENDING,
'try_number': None,
'tags': [u'priority:50', u'tag:1', u'user:Jesus'],
'user': u'Jesus',
}
self.assertEqual(expected, result_summary.to_dict())
# Nothing changed 2 secs later except latency.
self.mock_now(self.now, 2)
self.assertEqual(expected, result_summary.to_dict())
# Task is reaped after 2 seconds (4 secs total).
reap_ts = self.now + datetime.timedelta(seconds=4)
self.mock_now(reap_ts)
to_run.queue_number = None
to_run.put()
run_result = task_result.new_run_result(request, 1, 'localhost', 'abc', {})
run_result.modified_ts = utils.utcnow()
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
expected = {
'abandoned_ts': None,
'bot_dimensions': {},
'bot_id': u'localhost',
'bot_version': u'abc',
'children_task_ids': [],
'completed_ts': None,
'costs_usd': [0.],
'cost_saved_usd': None,
'created_ts': self.now,
'deduped_from': None,
'durations': [],
'exit_codes': [],
'failure': False,
'id': '1d69b9f088008810',
'internal_failure': False,
'modified_ts': reap_ts,
'name': u'Request name',
'outputs_ref': None,
'properties_hash': None,
'server_versions': [u'v1a'],
'started_ts': reap_ts,
'state': task_result.State.RUNNING,
'tags': [u'priority:50', u'tag:1', u'user:Jesus'],
'try_number': 1,
'user': u'Jesus',
}
self.assertEqual(expected, result_summary.key.get().to_dict())
# Task completed after 2 seconds (6 secs total), the task has been running
# for 2 seconds.
complete_ts = self.now + datetime.timedelta(seconds=6)
self.mock_now(complete_ts)
run_result.completed_ts = complete_ts
run_result.exit_codes.append(0)
run_result.state = task_result.State.COMPLETED
run_result.modified_ts = utils.utcnow()
ndb.transaction(
lambda: ndb.put_multi(run_result.append_output(0, 'foo', 0)))
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
expected = {
'abandoned_ts': None,
'bot_dimensions': {},
'bot_id': u'localhost',
'bot_version': u'abc',
'children_task_ids': [],
'completed_ts': complete_ts,
'costs_usd': [0.],
'cost_saved_usd': None,
'created_ts': self.now,
'deduped_from': None,
'durations': [],
'exit_codes': [0],
'failure': False,
'id': '1d69b9f088008810',
'internal_failure': False,
'modified_ts': complete_ts,
'name': u'Request name',
'outputs_ref': None,
'properties_hash': None,
'server_versions': [u'v1a'],
'started_ts': reap_ts,
'state': task_result.State.COMPLETED,
'tags': [u'priority:50', u'tag:1', u'user:Jesus'],
'try_number': 1,
'user': u'Jesus',
}
self.assertEqual(expected, result_summary.key.get().to_dict())
self.assertEqual(['foo'], list(result_summary.get_outputs()))
self.assertEqual(
datetime.timedelta(seconds=2), result_summary.duration_total)
self.assertEqual(
datetime.timedelta(seconds=2),
result_summary.duration_now(utils.utcnow()))
self.assertEqual(
datetime.timedelta(seconds=4), result_summary.pending)
self.assertEqual(
datetime.timedelta(seconds=4),
result_summary.pending_now(utils.utcnow()))
self.assertEqual(
task_pack.pack_result_summary_key(result_summary.key),
result_summary.task_id)
self.assertEqual(complete_ts, result_summary.ended_ts)
self.assertEqual(
task_pack.pack_run_result_key(run_result.key),
run_result.task_id)
self.assertEqual(complete_ts, run_result.ended_ts)
def test_integration(self):
# Creates a TaskRequest, along its TaskResultSummary and TaskToRun. Have a
# bot reap the task, and complete the task. Ensure the resulting
# TaskResultSummary and TaskRunResult are properly updated.
request = _gen_request()
result_summary = task_result.new_result_summary(request)
to_run = task_to_run.new_task_to_run(request, 1, 0)
result_summary.modified_ts = utils.utcnow()
ndb.transaction(lambda: ndb.put_multi([result_summary, to_run]))
expected = self._gen_summary(modified_ts=self.now)
self.assertEqual(expected, result_summary.to_dict())
# Nothing changed 2 secs later except latency.
self.mock_now(self.now, 2)
self.assertEqual(expected, result_summary.to_dict())
# Task is reaped after 2 seconds (4 secs total).
reap_ts = self.now + datetime.timedelta(seconds=4)
self.mock_now(reap_ts)
to_run.queue_number = None
to_run.put()
run_result = task_result.new_run_result(request, to_run, u'localhost',
u'abc', {})
run_result.started_ts = utils.utcnow()
run_result.modified_ts = run_result.started_ts
ndb.transaction(
lambda: result_summary.set_from_run_result(run_result, request))
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
expected = self._gen_summary(bot_dimensions={},
bot_version=u'abc',
bot_id=u'localhost',
costs_usd=[0.],
modified_ts=reap_ts,
state=task_result.State.RUNNING,
started_ts=reap_ts,
try_number=1)
self.assertEqual(expected, result_summary.key.get().to_dict())
# Task completed after 2 seconds (6 secs total), the task has been running
# for 2 seconds.
complete_ts = self.now + datetime.timedelta(seconds=6)
self.mock_now(complete_ts)
run_result.completed_ts = complete_ts
run_result.duration = 0.1
run_result.exit_code = 0
run_result.state = task_result.State.COMPLETED
run_result.modified_ts = utils.utcnow()
task_result.PerformanceStats(
key=task_pack.run_result_key_to_performance_stats_key(
run_result.key),
bot_overhead=0.1,
isolated_download=task_result.OperationStats(
duration=0.05,
initial_number_items=10,
initial_size=10000,
items_cold=large.pack([1, 2]),
items_hot=large.pack([3, 4, 5])),
isolated_upload=task_result.OperationStats(duration=0.01,
items_cold=large.pack(
[10]))).put()
ndb.transaction(
lambda: ndb.put_multi(run_result.append_output('foo', 0)))
ndb.transaction(
lambda: result_summary.set_from_run_result(run_result, request))
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
expected = self._gen_summary(bot_dimensions={},
bot_version=u'abc',
bot_id=u'localhost',
completed_ts=complete_ts,
costs_usd=[0.],
duration=0.1,
exit_code=0,
modified_ts=complete_ts,
state=task_result.State.COMPLETED,
started_ts=reap_ts,
try_number=1)
self.assertEqual(expected, result_summary.key.get().to_dict())
expected = {
'bot_overhead': 0.1,
'isolated_download': {
'duration': 0.05,
'initial_number_items': 10,
'initial_size': 10000,
'items_cold': large.pack([1, 2]),
'items_hot': large.pack([3, 4, 5]),
'num_items_cold': 2,
'total_bytes_items_cold': 3,
'num_items_hot': 3,
'total_bytes_items_hot': 12,
},
'isolated_upload': {
'duration': 0.01,
'initial_number_items': None,
'initial_size': None,
'items_cold': large.pack([10]),
'items_hot': None,
'num_items_cold': 1,
'total_bytes_items_cold': 10,
'num_items_hot': None,
'total_bytes_items_hot': None,
},
'package_installation': {
'duration': None,
'initial_number_items': None,
'initial_size': None,
'items_cold': None,
'items_hot': None,
'num_items_cold': None,
'total_bytes_items_cold': None,
'num_items_hot': None,
'total_bytes_items_hot': None,
},
}
self.assertEqual(expected, result_summary.performance_stats.to_dict())
self.assertEqual('foo', result_summary.get_output())
self.assertEqual(datetime.timedelta(seconds=2),
result_summary.duration_as_seen_by_server)
self.assertEqual(datetime.timedelta(seconds=0.1),
result_summary.duration_now(utils.utcnow()))
self.assertEqual(datetime.timedelta(seconds=4), result_summary.pending)
self.assertEqual(datetime.timedelta(seconds=4),
result_summary.pending_now(utils.utcnow()))
self.assertEqual(task_pack.pack_result_summary_key(result_summary.key),
result_summary.task_id)
self.assertEqual(complete_ts, result_summary.ended_ts)
self.assertEqual(task_pack.pack_run_result_key(run_result.key),
run_result.task_id)
self.assertEqual(complete_ts, run_result.ended_ts)
def test_integration(self):
# Creates a TaskRequest, along its TaskResultSummary and TaskToRun. Have a
# bot reap the task, and complete the task. Ensure the resulting
# TaskResultSummary and TaskRunResult are properly updated.
request = task_request.make_request(_gen_request(), True)
result_summary = task_result.new_result_summary(request)
to_run = task_to_run.new_task_to_run(request)
result_summary.modified_ts = utils.utcnow()
ndb.transaction(lambda: ndb.put_multi([result_summary, to_run]))
expected = {
"abandoned_ts": None,
"bot_dimensions": None,
"bot_id": None,
"bot_version": None,
"children_task_ids": [],
"completed_ts": None,
"costs_usd": [],
"cost_saved_usd": None,
"created_ts": self.now,
"deduped_from": None,
"durations": [],
"exit_codes": [],
"failure": False,
"id": "1d69b9f088008810",
"internal_failure": False,
"modified_ts": self.now,
"name": u"Request name",
"outputs_ref": None,
"properties_hash": None,
"server_versions": [],
"started_ts": None,
"state": task_result.State.PENDING,
"try_number": None,
"tags": [u"priority:50", u"tag:1", u"user:Jesus"],
"user": u"Jesus",
}
self.assertEqual(expected, result_summary.to_dict())
# Nothing changed 2 secs later except latency.
self.mock_now(self.now, 2)
self.assertEqual(expected, result_summary.to_dict())
# Task is reaped after 2 seconds (4 secs total).
reap_ts = self.now + datetime.timedelta(seconds=4)
self.mock_now(reap_ts)
to_run.queue_number = None
to_run.put()
run_result = task_result.new_run_result(request, 1, "localhost", "abc", {})
run_result.modified_ts = utils.utcnow()
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
expected = {
"abandoned_ts": None,
"bot_dimensions": {},
"bot_id": u"localhost",
"bot_version": u"abc",
"children_task_ids": [],
"completed_ts": None,
"costs_usd": [0.0],
"cost_saved_usd": None,
"created_ts": self.now,
"deduped_from": None,
"durations": [],
"exit_codes": [],
"failure": False,
"id": "1d69b9f088008810",
"internal_failure": False,
"modified_ts": reap_ts,
"name": u"Request name",
"outputs_ref": None,
"properties_hash": None,
"server_versions": [u"v1a"],
"started_ts": reap_ts,
"state": task_result.State.RUNNING,
"tags": [u"priority:50", u"tag:1", u"user:Jesus"],
"try_number": 1,
"user": u"Jesus",
}
self.assertEqual(expected, result_summary.key.get().to_dict())
# Task completed after 2 seconds (6 secs total), the task has been running
# for 2 seconds.
complete_ts = self.now + datetime.timedelta(seconds=6)
self.mock_now(complete_ts)
run_result.completed_ts = complete_ts
run_result.exit_codes.append(0)
run_result.state = task_result.State.COMPLETED
run_result.modified_ts = utils.utcnow()
ndb.transaction(lambda: ndb.put_multi(run_result.append_output(0, "foo", 0)))
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
expected = {
"abandoned_ts": None,
"bot_dimensions": {},
"bot_id": u"localhost",
"bot_version": u"abc",
"children_task_ids": [],
"completed_ts": complete_ts,
"costs_usd": [0.0],
"cost_saved_usd": None,
"created_ts": self.now,
"deduped_from": None,
"durations": [],
"exit_codes": [0],
"failure": False,
"id": "1d69b9f088008810",
"internal_failure": False,
"modified_ts": complete_ts,
"name": u"Request name",
"outputs_ref": None,
"properties_hash": None,
"server_versions": [u"v1a"],
"started_ts": reap_ts,
"state": task_result.State.COMPLETED,
"tags": [u"priority:50", u"tag:1", u"user:Jesus"],
"try_number": 1,
"user": u"Jesus",
}
self.assertEqual(expected, result_summary.key.get().to_dict())
self.assertEqual(["foo"], list(result_summary.get_outputs()))
self.assertEqual(datetime.timedelta(seconds=2), result_summary.duration)
self.assertEqual(datetime.timedelta(seconds=2), result_summary.duration_now(utils.utcnow()))
self.assertEqual(datetime.timedelta(seconds=4), result_summary.pending)
self.assertEqual(datetime.timedelta(seconds=4), result_summary.pending_now(utils.utcnow()))
self.assertEqual(task_pack.pack_result_summary_key(result_summary.key), result_summary.task_id)
self.assertEqual(complete_ts, result_summary.ended_ts)
self.assertEqual(task_pack.pack_run_result_key(run_result.key), run_result.task_id)
self.assertEqual(complete_ts, run_result.ended_ts)