def test_make_request_parent(self):
parent = task_request.make_request(_gen_request_data())
# Hack: Would need to know about TaskResultSummary.
parent_id = task_pack.pack_request_key(parent.key) + '1'
child = task_request.make_request(
_gen_request_data(parent_task_id=parent_id))
self.assertEqual(parent_id, child.parent_task_id)
def test_make_request_clone(self):
# Compare with test_make_request().
parent = task_request.make_request(_gen_request_data())
# Hack: Would need to know about TaskResultSummary.
parent_id = task_pack.pack_request_key(parent.key) + '1'
data = _gen_request_data(
properties=dict(idempotent=True), parent_task_id=parent_id)
request = task_request.make_request_clone(task_request.make_request(data))
# Differences from make_request() are:
# - idempotent was reset to False.
# - parent_task_id was reset to None.
expected_properties = {
'commands': [[u'command1', u'arg1']],
'data': [
# Items were sorted.
[u'http://localhost/bar', u'bar.zip'],
[u'http://localhost/foo', u'foo.zip'],
],
'dimensions': {u'OS': u'Windows-3.1.1', u'hostname': u'localhost'},
'env': {u'foo': u'bar', u'joe': u'2'},
'execution_timeout_secs': 30,
'extra_args': None,
'grace_period_secs': 30,
'idempotent': False,
'io_timeout_secs': None,
'isolated': None,
'isolatedserver': None,
'namespace': None,
}
# Differences from make_request() are:
# - parent_task_id was reset to None.
# - tag 'user:'******'authenticated': auth_testing.DEFAULT_MOCKED_IDENTITY,
'name': u'Request name (Retry #1)',
'parent_task_id': None,
'priority': 49,
'properties': expected_properties,
'properties_hash': None,
'tags': [
u'OS:Windows-3.1.1',
u'hostname:localhost',
u'priority:49',
u'tag:1',
u'user:[email protected]',
],
'user': u'*****@*****.**',
}
actual = request.to_dict()
# expiration_ts - created_ts == deadline_to_run.
created = actual.pop('created_ts')
expiration = actual.pop('expiration_ts')
self.assertEqual(
int(round((expiration - created).total_seconds())),
data['scheduling_expiration_secs'])
self.assertEqual(expected_request, actual)
self.assertEqual(
data['scheduling_expiration_secs'], request.scheduling_expiration_secs)
def test_different(self):
# Two TestRequest with different properties.
request_1 = task_request.make_request(
_gen_request(properties=dict(execution_timeout_secs=30, idempotent=True)), True
)
request_2 = task_request.make_request(
_gen_request(properties=dict(execution_timeout_secs=129, idempotent=True)), True
)
self.assertNotEqual(request_1.properties.properties_hash, request_2.properties.properties_hash)
def test_new_task_to_run(self):
self.mock(random, "getrandbits", lambda _: 0x12)
request_dimensions = {u"OS": u"Windows-3.1.1"}
data = _gen_request_data(
properties={
"commands": [[u"command1", u"arg1"]],
"data": [[u"http://localhost/foo", u"foo.zip"]],
"dimensions": request_dimensions,
"env": {u"foo": u"bar"},
"execution_timeout_secs": 30,
},
priority=20,
scheduling_expiration_secs=31,
)
task_to_run.new_task_to_run(task_request.make_request(data)).put()
# Create a second with higher priority.
self.mock(random, "getrandbits", lambda _: 0x23)
data = _gen_request_data(
properties={
"commands": [[u"command1", u"arg1"]],
"data": [[u"http://localhost/foo", u"foo.zip"]],
"dimensions": request_dimensions,
"env": {u"foo": u"bar"},
"execution_timeout_secs": 30,
},
priority=10,
scheduling_expiration_secs=31,
)
task_to_run.new_task_to_run(task_request.make_request(data)).put()
expected = [
{
"dimensions_hash": _hash_dimensions(request_dimensions),
"expiration_ts": self.now + datetime.timedelta(seconds=31),
"request_key": "0x7e296460f77ffdce",
# Lower priority value means higher priority.
"queue_number": "0x00060dc5849f1346",
},
{
"dimensions_hash": _hash_dimensions(request_dimensions),
"expiration_ts": self.now + datetime.timedelta(seconds=31),
"request_key": "0x7e296460f77ffede",
"queue_number": "0x00072c96fd65d346",
},
]
def flatten(i):
out = _task_to_run_to_dict(i)
out["request_key"] = "0x%016x" % i.request_key.integer_id()
return out
# Warning: Ordering by key doesn't work because of TaskToRunShard; e.g.
# the entity key ordering DOES NOT correlate with .queue_number
# Ensure they come out in expected order.
q = task_to_run.TaskToRun.query().order(task_to_run.TaskToRun.queue_number)
self.assertEqual(expected, map(flatten, q.fetch()))
def test_make_request_isolated(self):
parent = task_request.make_request(
_gen_request(
properties={
'commands': None,
'data': None,
'inputs_ref': {
'isolated': '0123456789012345678901234567890123456789',
'isolatedserver': 'http://localhost:1',
'namespace': 'default-gzip',
},
}),
True)
# Hack: Would need to know about TaskResultSummary.
parent_id = task_pack.pack_request_key(parent.key) + '1'
request = task_request.make_request(
_gen_request(properties={'idempotent':True}, parent_task_id=parent_id),
True)
expected_properties = {
'commands': [[u'command1', u'arg1']],
'data': [
# Items were sorted.
[u'http://localhost/bar', u'bar.zip'],
[u'http://localhost/foo', u'foo.zip'],
],
'dimensions': {u'OS': u'Windows-3.1.1', u'hostname': u'localhost'},
'env': {u'foo': u'bar', u'joe': u'2'},
'extra_args': [],
'execution_timeout_secs': 30,
'grace_period_secs': 30,
'idempotent': True,
'inputs_ref': None,
'io_timeout_secs': None,
}
expected_request = {
'authenticated': auth_testing.DEFAULT_MOCKED_IDENTITY,
'name': u'Request name',
'parent_task_id': unicode(parent_id),
'priority': 49,
'properties': expected_properties,
'properties_hash': 'b45f6f868f9227c3035cd82b4a5b0360f5ce6f61',
'tags': [
u'OS:Windows-3.1.1',
u'hostname:localhost',
u'priority:49',
u'tag:1',
u'user:Jesus',
],
'user': u'Jesus',
}
actual = request.to_dict()
# expiration_ts - created_ts == scheduling_expiration_secs.
actual.pop('created_ts')
actual.pop('expiration_ts')
self.assertEqual(expected_request, actual)
self.assertEqual(30, request.expiration_secs)
def test_new_task_to_run(self):
self.mock(random, 'getrandbits', lambda _: 0x12)
request_dimensions = {u'OS': u'Windows-3.1.1', u'pool': u'default'}
now = utils.utcnow()
data = _gen_request(
properties={
'command': [u'command1', u'arg1'],
'dimensions': request_dimensions,
'env': {u'foo': u'bar'},
'execution_timeout_secs': 30,
},
priority=20,
created_ts=now,
expiration_ts=now+datetime.timedelta(seconds=31))
task_to_run.new_task_to_run(task_request.make_request(data, True)).put()
# Create a second with higher priority.
self.mock(random, 'getrandbits', lambda _: 0x23)
data = _gen_request(
properties={
'command': [u'command1', u'arg1'],
'dimensions': request_dimensions,
'env': {u'foo': u'bar'},
'execution_timeout_secs': 30,
},
priority=10,
created_ts=now,
expiration_ts=now+datetime.timedelta(seconds=31))
task_to_run.new_task_to_run(task_request.make_request(data, True)).put()
expected = [
{
'dimensions_hash': _hash_dimensions(request_dimensions),
'expiration_ts': self.now + datetime.timedelta(seconds=31),
'request_key': '0x7e296460f77ffdce',
# Lower priority value means higher priority.
'queue_number': '0x00060dc5849f1346',
},
{
'dimensions_hash': _hash_dimensions(request_dimensions),
'expiration_ts': self.now + datetime.timedelta(seconds=31),
'request_key': '0x7e296460f77ffede',
'queue_number': '0x00072c96fd65d346',
},
]
def flatten(i):
out = _task_to_run_to_dict(i)
out['request_key'] = '0x%016x' % i.request_key.integer_id()
return out
# Warning: Ordering by key doesn't work because of TaskToRunShard; e.g.
# the entity key ordering DOES NOT correlate with .queue_number
# Ensure they come out in expected order.
q = task_to_run.TaskToRun.query().order(task_to_run.TaskToRun.queue_number)
self.assertEqual(expected, map(flatten, q.fetch()))
def test_make_request_isolated(self):
parent = task_request.make_request(
_gen_request(
properties={
"commands": None,
"data": None,
"inputs_ref": {
"isolated": "0123456789012345678901234567890123456789",
"isolatedserver": "http://localhost:1",
"namespace": "default-gzip",
},
}
),
True,
)
# Hack: Would need to know about TaskResultSummary.
parent_id = task_pack.pack_request_key(parent.key) + "1"
request = task_request.make_request(
_gen_request(properties={"idempotent": True}, parent_task_id=parent_id), True
)
expected_properties = {
"commands": [[u"command1", u"arg1"]],
"data": [
# Items were sorted.
[u"http://localhost/bar", u"bar.zip"],
[u"http://localhost/foo", u"foo.zip"],
],
"dimensions": {u"OS": u"Windows-3.1.1", u"hostname": u"localhost"},
"env": {u"foo": u"bar", u"joe": u"2"},
"extra_args": [],
"execution_timeout_secs": 30,
"grace_period_secs": 30,
"idempotent": True,
"inputs_ref": None,
"io_timeout_secs": None,
}
expected_request = {
"authenticated": auth_testing.DEFAULT_MOCKED_IDENTITY,
"name": u"Request name",
"parent_task_id": unicode(parent_id),
"priority": 49,
"properties": expected_properties,
"properties_hash": "b45f6f868f9227c3035cd82b4a5b0360f5ce6f61",
"pubsub_topic": None,
"pubsub_userdata": None,
"tags": [u"OS:Windows-3.1.1", u"hostname:localhost", u"priority:49", u"tag:1", u"user:Jesus"],
"user": u"Jesus",
}
actual = request.to_dict()
# expiration_ts - created_ts == scheduling_expiration_secs.
actual.pop("created_ts")
actual.pop("expiration_ts")
self.assertEqual(expected_request, actual)
self.assertEqual(30, request.expiration_secs)
def test_make_request(self):
# Compare with test_make_request_clone().
parent = task_request.make_request(_gen_request_data())
# Hack: Would need to know about TaskResultSummary.
parent_id = task_pack.pack_request_key(parent.key) + '1'
data = _gen_request_data(
properties=dict(idempotent=True), parent_task_id=parent_id)
request = task_request.make_request(data)
expected_properties = {
'commands': [[u'command1', u'arg1']],
'data': [
# Items were sorted.
[u'http://localhost/bar', u'bar.zip'],
[u'http://localhost/foo', u'foo.zip'],
],
'dimensions': {u'OS': u'Windows-3.1.1', u'hostname': u'localhost'},
'env': {u'foo': u'bar', u'joe': u'2'},
'extra_args': None,
'execution_timeout_secs': 30,
'grace_period_secs': 30,
'idempotent': True,
'io_timeout_secs': None,
'isolated': None,
'isolatedserver': None,
'namespace': None,
}
expected_request = {
'authenticated': auth_testing.DEFAULT_MOCKED_IDENTITY,
'name': u'Request name',
'parent_task_id': unicode(parent_id),
'priority': 49,
'properties': expected_properties,
'properties_hash': 'e7276ca9999756de386d26d39ae648e641ae1eac',
'tags': [
u'OS:Windows-3.1.1',
u'hostname:localhost',
u'priority:49',
u'tag:1',
u'user:Jesus',
],
'user': u'Jesus',
}
actual = request.to_dict()
# expiration_ts - created_ts == scheduling_expiration_secs.
created = actual.pop('created_ts')
expiration = actual.pop('expiration_ts')
self.assertEqual(
int(round((expiration - created).total_seconds())),
data['scheduling_expiration_secs'])
self.assertEqual(expected_request, actual)
self.assertEqual(
data['scheduling_expiration_secs'], request.scheduling_expiration_secs)
def test_duped(self):
# Two TestRequest with the same properties.
request_1 = task_request.make_request(
_gen_request_data(properties=dict(idempotent=True)))
request_2 = task_request.make_request(
_gen_request_data(
name='Other', user='******', priority=201,
scheduling_expiration_secs=129, tags=['tag:2'],
properties=dict(idempotent=True)))
self.assertEqual(
request_1.properties.properties_hash,
request_2.properties.properties_hash)
self.assertTrue(request_1.properties.properties_hash)
def test_make_request_clone(self):
# Compare with test_make_request().
parent = task_request.make_request(_gen_request(), True)
# Hack: Would need to know about TaskResultSummary.
parent_id = task_pack.pack_request_key(parent.key) + "1"
data = _gen_request(properties=dict(idempotent=True), parent_task_id=parent_id)
request = task_request.make_request_clone(task_request.make_request(data, True))
# Differences from make_request() are:
# - idempotent was reset to False.
# - parent_task_id was reset to None.
expected_properties = {
"commands": [[u"command1", u"arg1"]],
"data": [
# Items were sorted.
[u"http://localhost/bar", u"bar.zip"],
[u"http://localhost/foo", u"foo.zip"],
],
"dimensions": {u"OS": u"Windows-3.1.1", u"hostname": u"localhost"},
"env": {u"foo": u"bar", u"joe": u"2"},
"execution_timeout_secs": 30,
"extra_args": [],
"grace_period_secs": 30,
"idempotent": False,
"inputs_ref": None,
"io_timeout_secs": None,
}
# Differences from make_request() are:
# - parent_task_id was reset to None.
# - tag 'user:' was replaced
# - user was replaced.
expected_request = {
"authenticated": auth_testing.DEFAULT_MOCKED_IDENTITY,
"name": u"Request name (Retry #1)",
"parent_task_id": None,
"priority": 49,
"properties": expected_properties,
"properties_hash": None,
"pubsub_topic": None,
"pubsub_userdata": None,
"tags": [u"OS:Windows-3.1.1", u"hostname:localhost", u"priority:49", u"tag:1", u"user:[email protected]"],
"user": u"*****@*****.**",
}
actual = request.to_dict()
# expiration_ts - created_ts == deadline_to_run.
actual.pop("created_ts")
actual.pop("expiration_ts")
self.assertEqual(expected_request, actual)
self.assertEqual(30, request.expiration_secs)
def test_set_from_run_result_two_tries(self):
request = task_request.make_request(_gen_request(), True)
result_summary = task_result.new_result_summary(request)
run_result_1 = task_result.new_run_result(
request, 1, 'localhost', 'abc', {})
run_result_2 = task_result.new_run_result(
request, 2, 'localhost', 'abc', {})
self.assertTrue(result_summary.need_update_from_run_result(run_result_1))
run_result_2.modified_ts = utils.utcnow()
result_summary.modified_ts = utils.utcnow()
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result_2)))
self.assertTrue(result_summary.need_update_from_run_result(run_result_1))
run_result_1.modified_ts = utils.utcnow()
result_summary.set_from_run_result(run_result_1, request)
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result_1)))
result_summary = result_summary.key.get()
self.assertFalse(result_summary.need_update_from_run_result(run_result_1))
self.assertTrue(result_summary.need_update_from_run_result(run_result_2))
run_result_2.modified_ts = utils.utcnow()
result_summary.set_from_run_result(run_result_2, request)
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result_2)))
result_summary = result_summary.key.get()
self.assertEqual(2, result_summary.try_number)
self.assertFalse(result_summary.need_update_from_run_result(run_result_1))
def test_new_run_result(self):
request = task_request.make_request(_gen_request(), True)
actual = task_result.new_run_result(
request, 1, 'localhost', 'abc',
{'id': ['localhost'], 'foo': ['bar', 'biz']})
expected = {
'abandoned_ts': None,
'bot_dimensions': {'id': ['localhost'], 'foo': ['bar', 'biz']},
'bot_id': 'localhost',
'bot_version': 'abc',
'children_task_ids': [],
'completed_ts': None,
'cost_usd': 0.,
'durations': [],
'exit_codes': [],
'failure': False,
'id': '1d69b9f088008811',
'internal_failure': False,
'modified_ts': None,
'outputs_ref': None,
'server_versions': ['v1a'],
'started_ts': self.now,
'state': task_result.State.RUNNING,
'try_number': 1,
}
self.assertEqual(expected, actual.to_dict())
self.assertEqual(50, actual.priority)
self.assertEqual(False, actual.can_be_canceled)
def test_bot_kill_task(self):
self.mock(random, 'getrandbits', lambda _: 0x88)
dimensions = {u'OS': u'Windows-3.1.1'}
request = task_request.make_request(
_gen_request(properties={'dimensions': dimensions}), True)
result_summary = task_scheduler.schedule_request(request)
reaped_request, run_result = task_scheduler.bot_reap_task(
{'OS': 'Windows-3.1.1'}, 'localhost', 'abc')
self.assertEqual(
None, task_scheduler.bot_kill_task(run_result.key, 'localhost'))
expected = {
'abandoned_ts': self.now,
'bot_dimensions': dimensions,
'bot_id': u'localhost',
'bot_version': u'abc',
'children_task_ids': [],
'completed_ts': None,
'costs_usd': [0.],
'cost_saved_usd': None,
'created_ts': self.now,
'deduped_from': None,
'durations': [],
'exit_codes': [],
'failure': False,
'id': '1d69b9f088008810',
'internal_failure': True,
'modified_ts': self.now,
'name': u'Request name',
'outputs_ref': None,
'properties_hash': None,
'server_versions': [u'v1a'],
'started_ts': self.now,
'state': State.BOT_DIED,
'tags': [u'OS:Windows-3.1.1', u'priority:50', u'tag:1', u'user:Jesus'],
'try_number': 1,
'user': u'Jesus',
}
self.assertEqual(expected, result_summary.key.get().to_dict())
expected = {
'abandoned_ts': self.now,
'bot_dimensions': dimensions,
'bot_id': u'localhost',
'bot_version': u'abc',
'children_task_ids': [],
'completed_ts': None,
'cost_usd': 0.,
'durations': [],
'exit_codes': [],
'failure': False,
'id': '1d69b9f088008811',
'internal_failure': True,
'modified_ts': self.now,
'outputs_ref': None,
'server_versions': [u'v1a'],
'started_ts': self.now,
'state': State.BOT_DIED,
'try_number': 1,
}
self.assertEqual(expected, run_result.key.get().to_dict())
def test_cron_abort_expired_task_to_run(self):
self.mock(random, 'getrandbits', lambda _: 0x88)
data = _gen_request_data(
properties=dict(dimensions={u'OS': u'Windows-3.1.1'}))
request = task_request.make_request(data)
result_summary = task_scheduler.schedule_request(request)
abandoned_ts = self.mock_now(self.now, data['scheduling_expiration_secs']+1)
self.assertEqual(1, task_scheduler.cron_abort_expired_task_to_run())
self.assertEqual([], task_result.TaskRunResult.query().fetch())
expected = {
'abandoned_ts': abandoned_ts,
'bot_id': None,
'bot_version': None,
'children_task_ids': [],
'completed_ts': None,
'costs_usd': [],
'cost_saved_usd': None,
'created_ts': self.now,
'deduped_from': None,
'durations': [],
'exit_codes': [],
'failure': False,
'id': '1d69b9f088008810',
'internal_failure': False,
'modified_ts': abandoned_ts,
'name': u'Request name',
'properties_hash': None,
'server_versions': [],
'started_ts': None,
'state': task_result.State.EXPIRED,
'try_number': None,
'user': u'Jesus',
}
self.assertEqual(expected, result_summary.key.get().to_dict())
def test_request_to_task_to_run_key(self):
self.mock(random, 'getrandbits', lambda _: 0x88)
request = task_request.make_request(_gen_request(), True)
# Ensures that the hash value is constant for the same input.
self.assertEqual(
ndb.Key('TaskRequest', 0x7e296460f77ff77e, 'TaskToRun', 3420117132),
task_to_run.request_to_task_to_run_key(request))
def test_new_run_result(self):
request = task_request.make_request(_gen_request(), True)
actual = task_result.new_run_result(
request, 1, "localhost", "abc", {"id": ["localhost"], "foo": ["bar", "biz"]}
)
expected = {
"abandoned_ts": None,
"bot_dimensions": {"id": ["localhost"], "foo": ["bar", "biz"]},
"bot_id": "localhost",
"bot_version": "abc",
"children_task_ids": [],
"completed_ts": None,
"cost_usd": 0.0,
"durations": [],
"exit_codes": [],
"failure": False,
"id": "1d69b9f088008811",
"internal_failure": False,
"modified_ts": None,
"outputs_ref": None,
"server_versions": ["v1a"],
"started_ts": self.now,
"state": task_result.State.RUNNING,
"try_number": 1,
}
self.assertEqual(expected, actual.to_dict())
self.assertEqual(50, actual.priority)
self.assertEqual(False, actual.can_be_canceled)
def test_request_to_task_to_run_key(self):
self.mock(random, "getrandbits", lambda _: 0x88)
request = task_request.make_request(_gen_request_data())
self.assertEqual(
ndb.Key("TaskRequest", 0x7E296460F77FF77E, "TaskToRun", 2471203225),
task_to_run.request_to_task_to_run_key(request),
)
def test_bot_update_pubsub_error(self):
data = _gen_request(
properties=dict(dimensions={u'OS': u'Windows-3.1.1'}),
pubsub_topic='projects/abc/topics/def')
request = task_request.make_request(data, True)
task_scheduler.schedule_request(request)
bot_dimensions = {
u'OS': [u'Windows', u'Windows-3.1.1'],
u'hostname': u'localhost',
u'foo': u'bar',
}
_, run_result = task_scheduler.bot_reap_task(
bot_dimensions, 'localhost', 'abc')
self.assertEqual('localhost', run_result.bot_id)
# Attempt to terminate the task with success, but make PubSub call fail.
self.mock_pub_sub(publish_successful=False)
self.assertEqual(
(False, False),
task_scheduler.bot_update_task(
run_result.key, 'localhost', 'Foo1', 0, 0, 0.1, False, False,
0.1, None))
# Bot retries bot_update, now PubSub works and notification is sent.
pub_sub_calls = self.mock_pub_sub(publish_successful=True)
self.assertEqual(
(True, True),
task_scheduler.bot_update_task(
run_result.key, 'localhost', 'Foo1', 0, 0, 0.1, False, False,
0.1, None))
self.assertEqual(1, len(pub_sub_calls)) # notification is sent
def new(self, request):
"""Creates a new task.
The task will be enqueued in the tasks list and will be executed at the
earliest opportunity by a bot that has at least the dimensions as described
in the task request.
"""
logging.info('%s', request)
try:
request = message_conversion.new_task_request_from_rpc(
request, utils.utcnow())
posted_request = task_request.make_request(request, acl.is_bot_or_admin())
except (datastore_errors.BadValueError, TypeError, ValueError) as e:
raise endpoints.BadRequestException(e.message)
result_summary = task_scheduler.schedule_request(posted_request)
previous_result = None
if result_summary.deduped_from:
previous_result = message_conversion.task_result_to_rpc(result_summary)
return swarming_rpcs.TaskRequestMetadata(
request=message_conversion.task_request_to_rpc(posted_request),
task_id=task_pack.pack_result_summary_key(result_summary.key),
task_result=previous_result)
def test_task_parent_isolated(self):
request = task_request.make_request(
_gen_request(
properties={
'commands': None,
'dimensions': {u'OS': u'Windows-3.1.1'},
'inputs_ref': {
'isolated': '1' * 40,
'isolatedserver': 'http://localhost:1',
'namespace': 'default-gzip',
},
}),
True)
_result_summary = task_scheduler.schedule_request(request)
bot_dimensions = {
u'OS': [u'Windows', u'Windows-3.1.1'],
u'hostname': u'localhost',
u'foo': u'bar',
}
actual_request, run_result = task_scheduler.bot_reap_task(
bot_dimensions, 'localhost', 'abc')
self.assertEqual(request, actual_request)
self.assertEqual('localhost', run_result.bot_id)
self.assertEqual(None, task_to_run.TaskToRun.query().get().queue_number)
# It's important to terminate the task with success.
self.assertEqual(
(True, True),
task_scheduler.bot_update_task(
run_result.key, 'localhost', 'Foo1', 0, 0, 0.1, False, False,
0.1, None))
parent_id = run_result.task_id
request = task_request.make_request(
_gen_request(
parent_task_id=parent_id,
properties={'dimensions':{u'OS': u'Windows-3.1.1'}}),
True)
result_summary = task_scheduler.schedule_request(request)
self.assertEqual([], result_summary.children_task_ids)
self.assertEqual(parent_id, request.parent_task_id)
parent_run_result_key = task_pack.unpack_run_result_key(parent_id)
parent_res_summary_key = task_pack.run_result_key_to_result_summary_key(
parent_run_result_key)
expected = [result_summary.task_id]
self.assertEqual(expected, parent_run_result_key.get().children_task_ids)
self.assertEqual(expected, parent_res_summary_key.get().children_task_ids)
def test_make_request_idempotent(self):
request = task_request.make_request(
_gen_request_data(properties=dict(idempotent=True)))
as_dict = request.to_dict()
self.assertEqual(True, as_dict['properties']['idempotent'])
# Ensure the algorithm is deterministic.
self.assertEqual(
'e7276ca9999756de386d26d39ae648e641ae1eac', as_dict['properties_hash'])
def test_make_request_idempotent(self):
request = task_request.make_request(
_gen_request(properties=dict(idempotent=True)), True)
as_dict = request.to_dict()
self.assertEqual(True, as_dict['properties']['idempotent'])
# Ensure the algorithm is deterministic.
self.assertEqual(
'b45f6f868f9227c3035cd82b4a5b0360f5ce6f61', as_dict['properties_hash'])
def _bot_update_timeouts(self, hard, io):
self.mock(random, 'getrandbits', lambda _: 0x88)
data = _gen_request_data(
properties=dict(dimensions={u'OS': u'Windows-3.1.1'}))
request = task_request.make_request(data)
result_summary = task_scheduler.schedule_request(request)
reaped_request, run_result = task_scheduler.bot_reap_task(
{'OS': 'Windows-3.1.1'}, 'localhost', 'abc')
self.assertEqual(
(True, True),
task_scheduler.bot_update_task(
run_result.key, 'localhost', 'hi', 0, 0, 0.1, hard, io, 0.1))
expected = {
'abandoned_ts': None,
'bot_id': u'localhost',
'bot_version': u'abc',
'children_task_ids': [],
'completed_ts': self.now,
'costs_usd': [0.1],
'cost_saved_usd': None,
'created_ts': self.now,
'deduped_from': None,
'durations': [0.1],
'exit_codes': [0],
'failure': True,
'id': '1d69b9f088008810',
'internal_failure': False,
'modified_ts': self.now,
'name': u'Request name',
'properties_hash': None,
'server_versions': [u'v1a'],
'started_ts': self.now,
'state': State.TIMED_OUT,
'tags': [u'OS:Windows-3.1.1', u'priority:50', u'tag:1', u'user:Jesus'],
'try_number': 1,
'user': u'Jesus',
}
self.assertEqual(expected, result_summary.key.get().to_dict())
expected = {
'abandoned_ts': None,
'bot_id': u'localhost',
'bot_version': u'abc',
'children_task_ids': [],
'completed_ts': self.now,
'cost_usd': 0.1,
'durations': [0.1],
'exit_codes': [0],
'failure': True,
'id': '1d69b9f088008811',
'internal_failure': False,
'modified_ts': self.now,
'server_versions': [u'v1a'],
'started_ts': self.now,
'state': State.TIMED_OUT,
'try_number': 1,
}
self.assertEqual(expected, run_result.key.get().to_dict())
def test_cron_handle_bot_died_second(self):
# Test two tries internal_failure's leading to a BOT_DIED status.
self.mock(random, 'getrandbits', lambda _: 0x88)
now = utils.utcnow()
data = _gen_request(
properties=dict(dimensions={u'OS': u'Windows-3.1.1'}),
created_ts=now,
expiration_ts=now+datetime.timedelta(seconds=600))
request = task_request.make_request(data, True)
_result_summary = task_scheduler.schedule_request(request)
bot_dimensions = {
u'OS': [u'Windows', u'Windows-3.1.1'],
u'hostname': u'localhost',
u'foo': u'bar',
}
_request, run_result = task_scheduler.bot_reap_task(
bot_dimensions, 'localhost', 'abc')
self.assertEqual(1, run_result.try_number)
self.assertEqual(task_result.State.RUNNING, run_result.state)
self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 1)
self.assertEqual(([], 1, 0), task_scheduler.cron_handle_bot_died('f.local'))
now_1 = self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 2)
# It must be a different bot.
_request, run_result = task_scheduler.bot_reap_task(
bot_dimensions, 'localhost-second', 'abc')
now_2 = self.mock_now(self.now + 2 * task_result.BOT_PING_TOLERANCE, 3)
self.assertEqual(
(['1d69b9f088008812'], 0, 0),
task_scheduler.cron_handle_bot_died('f.local'))
self.assertEqual(([], 0, 0), task_scheduler.cron_handle_bot_died('f.local'))
expected = {
'abandoned_ts': now_2,
'bot_dimensions': bot_dimensions,
'bot_id': u'localhost-second',
'bot_version': u'abc',
'children_task_ids': [],
'completed_ts': None,
'costs_usd': [0., 0.],
'cost_saved_usd': None,
'created_ts': self.now,
'deduped_from': None,
'durations': [],
'exit_codes': [],
'failure': False,
'id': '1d69b9f088008810',
'internal_failure': True,
'modified_ts': now_2,
'name': u'Request name',
'outputs_ref': None,
'properties_hash': None,
'server_versions': [u'v1a'],
'started_ts': now_1,
'state': task_result.State.BOT_DIED,
'tags': [u'OS:Windows-3.1.1', u'priority:50', u'tag:1', u'user:Jesus'],
'try_number': 2,
'user': u'Jesus',
}
self.assertEqual(expected, run_result.result_summary_key.get().to_dict())
def test_search_by_name_broken_tasks(self):
# Create tasks where task_scheduler.schedule_request() fails in the middle.
# This is done by mocking the functions to fail every SKIP call and running
# it in a loop.
class RandomFailure(Exception):
pass
# First call fails ndb.put_multi(), second call fails search.Index.put(),
# third call work.
index = [0]
SKIP = 3
def put_multi(*args, **kwargs):
callers = [i[3] for i in inspect.stack()]
self.assertTrue(
'make_request' in callers or 'schedule_request' in callers, callers)
if (index[0] % SKIP) == 1:
raise RandomFailure()
return old_put_multi(*args, **kwargs)
def put_async(*args, **kwargs):
callers = [i[3] for i in inspect.stack()]
self.assertIn('schedule_request', callers)
out = ndb.Future()
if (index[0] % SKIP) == 2:
out.set_exception(search.Error())
else:
out.set_result(old_put_async(*args, **kwargs).get_result())
return out
old_put_multi = self.mock(ndb, 'put_multi', put_multi)
old_put_async = self.mock(search.Index, 'put_async', put_async)
saved = []
for i in xrange(100):
index[0] = i
data = _gen_request(
name='Request %d' % i,
properties=dict(dimensions={u'OS': u'Windows-3.1.1'}))
try:
request = task_request.make_request(data, True)
result_summary = task_scheduler.schedule_request(request)
saved.append(result_summary)
except RandomFailure:
pass
self.assertEqual(67, len(saved))
self.assertEqual(67, task_request.TaskRequest.query().count())
self.assertEqual(67, task_result.TaskResultSummary.query().count())
# Now the DB is full of half-corrupted entities.
cursor = None
actual, cursor = task_result._search_by_name('Request', cursor, 31)
self.assertEqual(31, len(actual))
actual, cursor = task_result._search_by_name('Request', cursor, 31)
self.assertEqual(3, len(actual))
actual, cursor = task_result._search_by_name('Request', cursor, 31)
self.assertEqual(0, len(actual))
def _quick_reap():
"""Reaps a task."""
data = _gen_request(
properties=dict(dimensions={u'OS': u'Windows-3.1.1'}))
request = task_request.make_request(data, True)
_result_summary = task_scheduler.schedule_request(request)
reaped_request, run_result = task_scheduler.bot_reap_task(
{'OS': 'Windows-3.1.1'}, 'localhost', 'abc')
return run_result
def test_duped(self):
# Two TestRequest with the same properties.
request_1 = task_request.make_request(_gen_request(properties=dict(idempotent=True)), True)
now = utils.utcnow()
request_2 = task_request.make_request(
_gen_request(
name="Other",
user="******",
priority=201,
created_ts=now,
expiration_ts=now + datetime.timedelta(seconds=129),
tags=["tag:2"],
properties=dict(idempotent=True),
),
True,
)
self.assertEqual(request_1.properties.properties_hash, request_2.properties.properties_hash)
self.assertTrue(request_1.properties.properties_hash)
def test_cron_abort_expired_task_to_run_retry(self):
self.mock(random, 'getrandbits', lambda _: 0x88)
now = utils.utcnow()
data = _gen_request(
properties=dict(dimensions={u'OS': u'Windows-3.1.1'}),
created_ts=now,
expiration_ts=now+datetime.timedelta(seconds=600))
request = task_request.make_request(data, True)
result_summary = task_scheduler.schedule_request(request)
# Fake first try bot died.
bot_dimensions = {
u'OS': [u'Windows', u'Windows-3.1.1'],
u'hostname': u'localhost',
u'foo': u'bar',
}
_request, run_result = task_scheduler.bot_reap_task(
bot_dimensions, 'localhost', 'abc')
now_1 = self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 1)
self.assertEqual((0, 1, 0), task_scheduler.cron_handle_bot_died())
self.assertEqual(task_result.State.BOT_DIED, run_result.key.get().state)
self.assertEqual(
task_result.State.PENDING, run_result.result_summary_key.get().state)
# BOT_DIED is kept instead of EXPIRED.
abandoned_ts = self.mock_now(self.now, request.expiration_secs+1)
self.assertEqual(1, task_scheduler.cron_abort_expired_task_to_run())
self.assertEqual(1, len(task_result.TaskRunResult.query().fetch()))
expected = {
'abandoned_ts': abandoned_ts,
'bot_dimensions': bot_dimensions,
'bot_id': u'localhost',
'bot_version': u'abc',
'children_task_ids': [],
'completed_ts': None,
'costs_usd': [0.],
'cost_saved_usd': None,
'created_ts': self.now,
'deduped_from': None,
'durations': [],
'exit_codes': [],
'failure': False,
'id': '1d69b9f088008810',
'internal_failure': True,
'modified_ts': abandoned_ts,
'name': u'Request name',
'outputs_ref': None,
'properties_hash': None,
'server_versions': [u'v1a'],
'started_ts': self.now,
'state': task_result.State.BOT_DIED,
'tags': [u'OS:Windows-3.1.1', u'priority:50', u'tag:1', u'user:Jesus'],
'try_number': 1,
'user': u'Jesus',
}
self.assertEqual(expected, result_summary.key.get().to_dict())
def test_cancel_task(self):
data = _gen_request_data(
properties=dict(dimensions={u'OS': u'Windows-3.1.1'}))
request = task_request.make_request(data)
result_summary = task_scheduler.schedule_request(request)
ok, was_running = task_scheduler.cancel_task(result_summary.key)
self.assertEqual(True, ok)
self.assertEqual(False, was_running)
result_summary = result_summary.key.get()
self.assertEqual(task_result.State.CANCELED, result_summary.state)
def test_search_by_name_failures(self):
data = _gen_request(
properties=dict(dimensions={u'OS': u'Windows-3.1.1'}))
request = task_request.make_request(data, True)
result_summary = task_scheduler.schedule_request(request)
actual, _cursor = task_result._search_by_name('foo', None, 10)
self.assertEqual([], actual)
# Partial match doesn't work.
actual, _cursor = task_result._search_by_name('nam', None, 10)
self.assertEqual([], actual)
def test_bot_reap_task(self):
data = _gen_request_data(properties=dict(
dimensions={u'OS': u'Windows-3.1.1'}))
request = task_request.make_request(data)
_result_summary = task_scheduler.schedule_request(request)
bot_dimensions = {
u'OS': [u'Windows', u'Windows-3.1.1'],
u'hostname': u'localhost',
u'foo': u'bar',
}
actual_request, run_result = task_scheduler.bot_reap_task(
bot_dimensions, 'localhost', 'abc')
self.assertEqual(request, actual_request)
self.assertEqual('localhost', run_result.bot_id)
self.assertEqual(None,
task_to_run.TaskToRun.query().get().queue_number)
def test_set_from_run_result(self):
request = task_request.make_request(_gen_request(), True)
result_summary = task_result.new_result_summary(request)
run_result = task_result.new_run_result(request, 1, 'localhost', 'abc',
{})
self.assertTrue(result_summary.need_update_from_run_result(run_result))
result_summary.modified_ts = utils.utcnow()
run_result.modified_ts = utils.utcnow()
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
self.assertTrue(result_summary.need_update_from_run_result(run_result))
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi([result_summary]))
self.assertFalse(
result_summary.need_update_from_run_result(run_result))
def test_run_result_timeout(self):
request = task_request.make_request(_gen_request(), True)
result_summary = task_result.new_result_summary(request)
result_summary.modified_ts = utils.utcnow()
ndb.transaction(result_summary.put)
run_result = task_result.new_run_result(request, 1, 'localhost', 'abc',
{})
run_result.state = task_result.State.TIMED_OUT
run_result.completed_ts = utils.utcnow()
run_result.modified_ts = utils.utcnow()
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi((run_result, result_summary)))
run_result = run_result.key.get()
result_summary = result_summary.key.get()
self.assertEqual(True, run_result.failure)
self.assertEqual(True, result_summary.failure)
def test_new_result_summary(self):
request = task_request.make_request(_gen_request(), True)
actual = task_result.new_result_summary(request)
expected = {
'abandoned_ts': None,
'bot_dimensions': None,
'bot_id': None,
'bot_version': None,
'children_task_ids': [],
'completed_ts': None,
'costs_usd': [],
'cost_saved_usd': None,
'created_ts': self.now,
'deduped_from': None,
'durations': [],
'exit_codes': [],
'failure': False,
'id': '1d69b9f088008810',
'internal_failure': False,
'modified_ts': None,
'name': u'Request name',
'outputs_ref': None,
'properties_hash': None,
'server_versions': [],
'started_ts': None,
'state': task_result.State.PENDING,
'tags': [u'priority:50', u'tag:1', u'user:Jesus'],
'try_number': None,
'user': u'Jesus',
}
self.assertEqual(expected, actual.to_dict())
self.assertEqual(50, actual.priority)
self.assertEqual(True, actual.can_be_canceled)
actual.state = task_result.State.RUNNING
self.assertEqual(False, actual.can_be_canceled)
actual.children_task_ids = [
'1d69ba3ea8008810',
'3d69ba3ea8008810',
'2d69ba3ea8008810',
]
actual.modified_ts = utils.utcnow()
ndb.transaction(actual.put)
expected = [
u'1d69ba3ea8008810', u'2d69ba3ea8008810', u'3d69ba3ea8008810'
]
self.assertEqual(expected, actual.key.get().children_task_ids)
def test_task_parent_children(self):
# Parent task creates a child task.
parent_id = self._task_ran_successfully()
data = _gen_request(
parent_task_id=parent_id,
properties=dict(dimensions={u'OS': u'Windows-3.1.1'}))
request = task_request.make_request(data, True)
result_summary = task_scheduler.schedule_request(request)
self.assertEqual([], result_summary.children_task_ids)
self.assertEqual(parent_id, request.parent_task_id)
parent_run_result_key = task_pack.unpack_run_result_key(parent_id)
parent_res_summary_key = task_pack.run_result_key_to_result_summary_key(
parent_run_result_key)
expected = [result_summary.task_id]
self.assertEqual(expected, parent_run_result_key.get().children_task_ids)
self.assertEqual(expected, parent_res_summary_key.get().children_task_ids)
def post(self):
request_data = self.parse_body()
# If the priority is below 100, make the the user has right to do so.
if request_data.get('priority', 255) < 100 and not acl.is_bot_or_admin():
# Silently drop the priority of normal users.
request_data['priority'] = 100
try:
request = task_request.make_request(request_data)
except (datastore_errors.BadValueError, TypeError, ValueError) as e:
self.abort_with_error(400, error=str(e))
result_summary = task_scheduler.schedule_request(request)
data = {
'request': request.to_dict(),
'task_id': task_pack.pack_result_summary_key(result_summary.key),
}
self.send_response(utils.to_json_encodable(data))
def test_cron_handle_bot_died_ignored_expired(self):
self.mock(random, 'getrandbits', lambda _: 0x88)
data = _gen_request_data(
properties=dict(dimensions={u'OS': u'Windows-3.1.1'}),
scheduling_expiration_secs=600)
request = task_request.make_request(data)
_result_summary = task_scheduler.schedule_request(request)
bot_dimensions = {
u'OS': [u'Windows', u'Windows-3.1.1'],
u'hostname': u'localhost',
u'foo': u'bar',
}
_request, run_result = task_scheduler.bot_reap_task(
bot_dimensions, 'localhost', 'abc')
self.assertEqual(1, run_result.try_number)
self.assertEqual(task_result.State.RUNNING, run_result.state)
self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 601)
self.assertEqual((1, 0, 0), task_scheduler.cron_handle_bot_died())
def test_task_idempotent_old(self):
self.mock(random, 'getrandbits', lambda _: 0x88)
# First task is idempotent.
self._task_ran_successfully()
# Second task is scheduled, first task is too old to be reused.
new_ts = self.mock_now(self.now,
config.settings().reusable_task_age_secs)
data = _gen_request_data(name='yay',
user='******',
properties=dict(
dimensions={u'OS': u'Windows-3.1.1'},
idempotent=True))
request = task_request.make_request(data)
_result_summary = task_scheduler.schedule_request(request)
# The task was enqueued for execution.
self.assertNotEqual(None,
task_to_run.TaskToRun.query().get().queue_number)
def terminate(self, request):
"""Asks a bot to terminate itself gracefully.
The bot will stay in the DB, use 'delete' to remove it from the DB
afterward. This request returns a pseudo-taskid that can be waited for to
wait for the bot to turn down.
This command is particularly useful when a privileged user needs to safely
debug a machine specific issue. The user can trigger a terminate for one of
the bot exhibiting the issue, wait for the pseudo-task to run then access
the machine with the guarantee that the bot is not running anymore.
"""
# TODO(maruel): Disallow a terminate task when there's one currently
# pending or if the bot is considered 'dead', e.g. no contact since 10
# minutes.
logging.info('%s', request)
bot_key = bot_management.get_info_key(request.bot_id)
get_or_raise(bot_key) # raises 404 if there is no such bot
try:
# Craft a special priority 0 task to tell the bot to shutdown.
properties = task_request.TaskProperties(
dimensions={u'id': request.bot_id},
execution_timeout_secs=0,
grace_period_secs=0,
io_timeout_secs=0)
now = utils.utcnow()
request = task_request.TaskRequest(
created_ts=now,
expiration_ts=now + datetime.timedelta(days=1),
name='Terminate %s' % request.bot_id,
priority=0,
properties=properties,
tags=['terminate:1'],
user=auth.get_current_identity().to_bytes())
assert request.properties.is_terminate
posted_request = task_request.make_request(request,
acl.is_bot_or_admin())
except (datastore_errors.BadValueError, TypeError, ValueError) as e:
raise endpoints.BadRequestException(e.message)
result_summary = task_scheduler.schedule_request(posted_request)
return swarming_rpcs.TerminateResponse(
task_id=task_pack.pack_result_summary_key(result_summary.key))
def new(self, request):
"""Creates a new task.
The task will be enqueued in the tasks list and will be executed at the
earliest opportunity by a bot that has at least the dimensions as described
in the task request.
"""
logging.info('%s', request)
try:
request = message_conversion.new_task_request_from_rpc(
request, utils.utcnow())
posted_request = task_request.make_request(request, acl.is_bot_or_admin())
except (datastore_errors.BadValueError, TypeError, ValueError) as e:
raise endpoints.BadRequestException(e.message)
result_summary = task_scheduler.schedule_request(posted_request)
return swarming_rpcs.TaskRequestMetadata(
request=message_conversion.task_request_to_rpc(posted_request),
task_id=task_pack.pack_result_summary_key(result_summary.key))
def test_yield_run_result_keys_with_dead_bot(self):
request = task_request.make_request(_gen_request_data())
result_summary = task_result.new_result_summary(request)
result_summary.modified_ts = utils.utcnow()
ndb.transaction(result_summary.put)
run_result = task_result.new_run_result(request, 1, 'localhost', 'abc')
run_result.completed_ts = utils.utcnow()
run_result.modified_ts = utils.utcnow()
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi((run_result, result_summary)))
self.mock_now(self.now + task_result.BOT_PING_TOLERANCE)
self.assertEqual(
[], list(task_result.yield_run_result_keys_with_dead_bot()))
self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 1)
self.assertEqual(
[run_result.key],
list(task_result.yield_run_result_keys_with_dead_bot()))
def test_task_idempotent_three(self):
self.mock(random, 'getrandbits', lambda _: 0x88)
# First task is idempotent.
task_id = self._task_ran_successfully()
# Second task is deduped against first task.
new_ts = self.mock_now(self.now, config.settings().reusable_task_age_secs-1)
self._task_deduped(new_ts, task_id)
# Third task is scheduled, second task is not dedupable, first task is too
# old.
new_ts = self.mock_now(self.now, config.settings().reusable_task_age_secs)
data = _gen_request(
name='yay',
user='******',
properties=dict(dimensions={u'OS': u'Windows-3.1.1'}, idempotent=True))
request = task_request.make_request(data, True)
_result_summary = task_scheduler.schedule_request(request)
# The task was enqueued for execution.
self.assertNotEqual(None, task_to_run.TaskToRun.query().get().queue_number)
def test_set_from_run_result_two_server_versions(self):
request = task_request.make_request(_gen_request_data())
result_summary = task_result.new_result_summary(request)
run_result = task_result.new_run_result(request, 1, 'localhost', 'abc')
self.assertTrue(result_summary.need_update_from_run_result(run_result))
result_summary.modified_ts = utils.utcnow()
run_result.modified_ts = utils.utcnow()
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
self.assertTrue(result_summary.need_update_from_run_result(run_result))
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi([result_summary]))
run_result.signal_server_version('new-version')
run_result.modified_ts = utils.utcnow()
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
self.assertEqual(
['v1a', 'new-version'], run_result.key.get().server_versions)
self.assertEqual(
['v1a', 'new-version'], result_summary.key.get().server_versions)
def new(self, request):
"""Provides a TaskRequest and receive its metadata."""
request_dict = json.loads(remote.protojson.encode_message(request))
_transform_request(request_dict)
# If the priority is below 100, make the the user has right to do so.
if request_dict.get('priority',
255) < 100 and not acl.is_bot_or_admin():
# Silently drop the priority of normal users.
request_dict['priority'] = 100
try:
posted_request = task_request.make_request(request_dict)
except (datastore_errors.BadValueError, TypeError, ValueError) as e:
raise endpoints.BadRequestException(e.message)
result_summary = task_scheduler.schedule_request(posted_request)
posted_dict = utils.to_json_encodable(posted_request)
return swarming_rpcs.TaskRequestMetadata(
request=message_conversion.task_request_from_dict(posted_dict),
task_id=task_pack.pack_result_summary_key(result_summary.key))
def _task_ran_successfully(self):
"""Runs a task successfully and returns the task_id."""
data = _gen_request_data(properties=dict(
dimensions={u'OS': u'Windows-3.1.1'}, idempotent=True))
request = task_request.make_request(data)
_result_summary = task_scheduler.schedule_request(request)
bot_dimensions = {
u'OS': [u'Windows', u'Windows-3.1.1'],
u'hostname': u'localhost',
u'foo': u'bar',
}
actual_request, run_result = task_scheduler.bot_reap_task(
bot_dimensions, 'localhost', 'abc')
self.assertEqual(request, actual_request)
self.assertEqual('localhost', run_result.bot_id)
self.assertEqual(None,
task_to_run.TaskToRun.query().get().queue_number)
# It's important to terminate the task with success.
self.assertEqual(
(True, True),
task_scheduler.bot_update_task(run_result.key, 'localhost', 'Foo1',
0, 0, 0.1, False, False, 0.1))
return unicode(run_result.key_packed)
def test_cron_abort_expired_task_to_run(self):
self.mock(random, 'getrandbits', lambda _: 0x88)
request = task_request.make_request(
_gen_request(properties={'dimensions': {u'OS': u'Windows-3.1.1'}}),
True)
result_summary = task_scheduler.schedule_request(request)
abandoned_ts = self.mock_now(self.now, request.expiration_secs+1)
self.assertEqual(1, task_scheduler.cron_abort_expired_task_to_run())
self.assertEqual([], task_result.TaskRunResult.query().fetch())
expected = {
'abandoned_ts': abandoned_ts,
'bot_dimensions': None,
'bot_id': None,
'bot_version': None,
'children_task_ids': [],
'completed_ts': None,
'costs_usd': [],
'cost_saved_usd': None,
'created_ts': self.now,
'deduped_from': None,
'durations': [],
'exit_codes': [],
'failure': False,
'id': '1d69b9f088008810',
'internal_failure': False,
'modified_ts': abandoned_ts,
'name': u'Request name',
'outputs_ref': None,
'properties_hash': None,
'server_versions': [],
'started_ts': None,
'state': task_result.State.EXPIRED,
'tags': [u'OS:Windows-3.1.1', u'priority:50', u'tag:1', u'user:Jesus'],
'try_number': None,
'user': u'Jesus',
}
self.assertEqual(expected, result_summary.key.get().to_dict())
def test_new_run_result(self):
request = task_request.make_request(_gen_request_data())
actual = task_result.new_run_result(request, 1, 'localhost', 'abc')
expected = {
'abandoned_ts': None,
'bot_id': 'localhost',
'bot_version': 'abc',
'children_task_ids': [],
'completed_ts': None,
'cost_usd': 0.,
'durations': [],
'exit_codes': [],
'failure': False,
'id': '1d69b9f088008811',
'internal_failure': False,
'modified_ts': None,
'server_versions': ['v1a'],
'started_ts': self.now,
'state': task_result.State.RUNNING,
'try_number': 1,
}
self.assertEqual(expected, actual.to_dict())
self.assertEqual(50, actual.priority)
self.assertEqual(False, actual.can_be_canceled)
def test_make_request_invalid_parent_id(self):
# Must ends with '1' or '2', not '0'
data = _gen_request_data(parent_task_id='1d69b9f088008810')
with self.assertRaises(ValueError):
task_request.make_request(data)
def test_integration(self):
# Creates a TaskRequest, along its TaskResultSummary and TaskToRun. Have a
# bot reap the task, and complete the task. Ensure the resulting
# TaskResultSummary and TaskRunResult are properly updated.
request = task_request.make_request(_gen_request(), True)
result_summary = task_result.new_result_summary(request)
to_run = task_to_run.new_task_to_run(request)
result_summary.modified_ts = utils.utcnow()
ndb.transaction(lambda: ndb.put_multi([result_summary, to_run]))
expected = {
'abandoned_ts': None,
'bot_dimensions': None,
'bot_id': None,
'bot_version': None,
'children_task_ids': [],
'completed_ts': None,
'costs_usd': [],
'cost_saved_usd': None,
'created_ts': self.now,
'deduped_from': None,
'durations': [],
'exit_codes': [],
'failure': False,
'id': '1d69b9f088008810',
'internal_failure': False,
'modified_ts': self.now,
'name': u'Request name',
'outputs_ref': None,
'properties_hash': None,
'server_versions': [],
'started_ts': None,
'state': task_result.State.PENDING,
'try_number': None,
'tags': [u'priority:50', u'tag:1', u'user:Jesus'],
'user': u'Jesus',
}
self.assertEqual(expected, result_summary.to_dict())
# Nothing changed 2 secs later except latency.
self.mock_now(self.now, 2)
self.assertEqual(expected, result_summary.to_dict())
# Task is reaped after 2 seconds (4 secs total).
reap_ts = self.now + datetime.timedelta(seconds=4)
self.mock_now(reap_ts)
to_run.queue_number = None
to_run.put()
run_result = task_result.new_run_result(request, 1, 'localhost', 'abc',
{})
run_result.modified_ts = utils.utcnow()
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
expected = {
'abandoned_ts': None,
'bot_dimensions': {},
'bot_id': u'localhost',
'bot_version': u'abc',
'children_task_ids': [],
'completed_ts': None,
'costs_usd': [0.],
'cost_saved_usd': None,
'created_ts': self.now,
'deduped_from': None,
'durations': [],
'exit_codes': [],
'failure': False,
'id': '1d69b9f088008810',
'internal_failure': False,
'modified_ts': reap_ts,
'name': u'Request name',
'outputs_ref': None,
'properties_hash': None,
'server_versions': [u'v1a'],
'started_ts': reap_ts,
'state': task_result.State.RUNNING,
'tags': [u'priority:50', u'tag:1', u'user:Jesus'],
'try_number': 1,
'user': u'Jesus',
}
self.assertEqual(expected, result_summary.key.get().to_dict())
# Task completed after 2 seconds (6 secs total), the task has been running
# for 2 seconds.
complete_ts = self.now + datetime.timedelta(seconds=6)
self.mock_now(complete_ts)
run_result.completed_ts = complete_ts
run_result.exit_codes.append(0)
run_result.state = task_result.State.COMPLETED
run_result.modified_ts = utils.utcnow()
ndb.transaction(
lambda: ndb.put_multi(run_result.append_output(0, 'foo', 0)))
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
expected = {
'abandoned_ts': None,
'bot_dimensions': {},
'bot_id': u'localhost',
'bot_version': u'abc',
'children_task_ids': [],
'completed_ts': complete_ts,
'costs_usd': [0.],
'cost_saved_usd': None,
'created_ts': self.now,
'deduped_from': None,
'durations': [],
'exit_codes': [0],
'failure': False,
'id': '1d69b9f088008810',
'internal_failure': False,
'modified_ts': complete_ts,
'name': u'Request name',
'outputs_ref': None,
'properties_hash': None,
'server_versions': [u'v1a'],
'started_ts': reap_ts,
'state': task_result.State.COMPLETED,
'tags': [u'priority:50', u'tag:1', u'user:Jesus'],
'try_number': 1,
'user': u'Jesus',
}
self.assertEqual(expected, result_summary.key.get().to_dict())
self.assertEqual(['foo'], list(result_summary.get_outputs()))
self.assertEqual(datetime.timedelta(seconds=2),
result_summary.duration_total)
self.assertEqual(datetime.timedelta(seconds=2),
result_summary.duration_now(utils.utcnow()))
self.assertEqual(datetime.timedelta(seconds=4), result_summary.pending)
self.assertEqual(datetime.timedelta(seconds=4),
result_summary.pending_now(utils.utcnow()))
self.assertEqual(task_pack.pack_result_summary_key(result_summary.key),
result_summary.task_id)
self.assertEqual(complete_ts, result_summary.ended_ts)
self.assertEqual(task_pack.pack_run_result_key(run_result.key),
run_result.task_id)
self.assertEqual(complete_ts, run_result.ended_ts)
def test_bad_values(self):
with self.assertRaises(ValueError):
task_request.make_request({})
with self.assertRaises(ValueError):
task_request.make_request(
_gen_request_data(properties={'foo': 'bar'}))
task_request.make_request(_gen_request_data())
with self.assertRaises(datastore_errors.BadValueError):
task_request.make_request(
_gen_request_data(properties=dict(commands=None)))
with self.assertRaises(datastore_errors.BadValueError):
task_request.make_request(
_gen_request_data(properties=dict(commands=[])))
with self.assertRaises(TypeError):
task_request.make_request(
_gen_request_data(properties=dict(commands={'a': 'b'})))
with self.assertRaises(TypeError):
task_request.make_request(
_gen_request_data(properties=dict(commands=['python'])))
with self.assertRaises(TypeError):
task_request.make_request(
_gen_request_data(properties=dict(commands=[['python']])))
task_request.make_request(
_gen_request_data(properties=dict(commands=[[u'python']])))
with self.assertRaises(TypeError):
task_request.make_request(
_gen_request_data(properties=dict(env=[])))
with self.assertRaises(TypeError):
task_request.make_request(
_gen_request_data(properties=dict(env={u'a': 1})))
task_request.make_request(_gen_request_data(properties=dict(env={})))
with self.assertRaises(TypeError):
task_request.make_request(
_gen_request_data(properties=dict(data=[[
'a',
]])))
with self.assertRaises(TypeError):
task_request.make_request(
_gen_request_data(properties=dict(data=[('a', '1')])))
with self.assertRaises(TypeError):
task_request.make_request(
_gen_request_data(properties=dict(data=[(u'a', u'1')])))
task_request.make_request(
_gen_request_data(properties=dict(data=[[u'a', u'1']])))
with self.assertRaises(datastore_errors.BadValueError):
task_request.make_request(
_gen_request_data(priority=task_request.MAXIMUM_PRIORITY + 1))
task_request.make_request(
_gen_request_data(priority=task_request.MAXIMUM_PRIORITY))
with self.assertRaises(datastore_errors.BadValueError):
task_request.make_request(
_gen_request_data(properties=dict(
execution_timeout_secs=task_request._ONE_DAY_SECS + 1)))
task_request.make_request(
_gen_request_data(properties=dict(
execution_timeout_secs=task_request._ONE_DAY_SECS)))
with self.assertRaises(datastore_errors.BadValueError):
task_request.make_request(
_gen_request_data(
scheduling_expiration_secs=task_request._MIN_TIMEOUT_SECS -
1))
task_request.make_request(
_gen_request_data(
scheduling_expiration_secs=task_request._MIN_TIMEOUT_SECS))
def _gen_new_task_to_run(**kwargs):
"""Returns a TaskToRun saved in the DB."""
request = task_request.make_request(_gen_request(**kwargs), True)
to_run = task_to_run.new_task_to_run(request)
to_run.put()
return to_run
def test_make_request_isolated(self):
parent = task_request.make_request(
_gen_request(
properties={
'commands': None,
'data': None,
'inputs_ref': {
'isolated': '0123456789012345678901234567890123456789',
'isolatedserver': 'http://localhost:1',
'namespace': 'default-gzip',
},
}), True)
# Hack: Would need to know about TaskResultSummary.
parent_id = task_pack.pack_request_key(parent.key) + '1'
request = task_request.make_request(
_gen_request(properties={'idempotent': True},
parent_task_id=parent_id), True)
expected_properties = {
'commands': [[u'command1', u'arg1']],
'data': [
# Items were sorted.
[u'http://localhost/bar', u'bar.zip'],
[u'http://localhost/foo', u'foo.zip'],
],
'dimensions': {
u'OS': u'Windows-3.1.1',
u'hostname': u'localhost'
},
'env': {
u'foo': u'bar',
u'joe': u'2'
},
'extra_args': [],
'execution_timeout_secs':
30,
'grace_period_secs':
30,
'idempotent':
True,
'inputs_ref':
None,
'io_timeout_secs':
None,
}
expected_request = {
'authenticated':
auth_testing.DEFAULT_MOCKED_IDENTITY,
'name':
u'Request name',
'parent_task_id':
unicode(parent_id),
'priority':
49,
'properties':
expected_properties,
'properties_hash':
'b45f6f868f9227c3035cd82b4a5b0360f5ce6f61',
'pubsub_topic':
None,
'pubsub_userdata':
None,
'tags': [
u'OS:Windows-3.1.1',
u'hostname:localhost',
u'priority:49',
u'tag:1',
u'user:Jesus',
],
'user':
u'Jesus',
}
actual = request.to_dict()
# expiration_ts - created_ts == scheduling_expiration_secs.
actual.pop('created_ts')
actual.pop('expiration_ts')
self.assertEqual(expected_request, actual)
self.assertEqual(30, request.expiration_secs)
def test_bad_values(self):
with self.assertRaises(AssertionError):
task_request.make_request(None, True)
with self.assertRaises(AssertionError):
task_request.make_request({}, True)
with self.assertRaises(AttributeError):
task_request.make_request(_gen_request(properties={'foo': 'bar'}),
True)
task_request.make_request(_gen_request(), True)
with self.assertRaises(datastore_errors.BadValueError):
task_request.make_request(
_gen_request(properties=dict(commands=[])), True)
with self.assertRaises(TypeError):
task_request.make_request(
_gen_request(properties=dict(commands={'a': 'b'})), True)
with self.assertRaises(TypeError):
task_request.make_request(
_gen_request(properties=dict(commands=['python'])), True)
with self.assertRaises(TypeError):
task_request.make_request(
_gen_request(properties=dict(commands=[['python']])), True)
task_request.make_request(
_gen_request(properties=dict(commands=[[u'python']])), True)
with self.assertRaises(TypeError):
task_request.make_request(_gen_request(properties=dict(env=[])),
True)
with self.assertRaises(TypeError):
task_request.make_request(
_gen_request(properties=dict(env={u'a': 1})), True)
task_request.make_request(_gen_request(properties=dict(env={})), True)
with self.assertRaises(TypeError):
task_request.make_request(
_gen_request(properties=dict(data=[[
'a',
]])), True)
with self.assertRaises(TypeError):
task_request.make_request(
_gen_request(properties=dict(data=[('a', '1')])), True)
with self.assertRaises(TypeError):
task_request.make_request(
_gen_request(properties=dict(data=[(u'a', u'1')])), True)
task_request.make_request(
_gen_request(properties=dict(data=[[u'a', u'1']])), True)
with self.assertRaises(datastore_errors.BadValueError):
task_request.make_request(
_gen_request(priority=task_request.MAXIMUM_PRIORITY + 1), True)
task_request.make_request(
_gen_request(priority=task_request.MAXIMUM_PRIORITY), True)
with self.assertRaises(datastore_errors.BadValueError):
task_request.make_request(
_gen_request(properties=dict(
execution_timeout_secs=task_request._ONE_DAY_SECS + 1)),
True)
task_request.make_request(
_gen_request(properties=dict(
execution_timeout_secs=task_request._ONE_DAY_SECS)), True)
now = utils.utcnow()
with self.assertRaises(datastore_errors.BadValueError):
task_request.make_request(
_gen_request(created_ts=now,
expiration_ts=now + datetime.timedelta(
seconds=task_request._MIN_TIMEOUT_SECS - 1)),
True)
task_request.make_request(
_gen_request(
created_ts=now,
expiration_ts=now +
datetime.timedelta(seconds=task_request._MIN_TIMEOUT_SECS)),
True)
# Try with isolated/isolatedserver/namespace.
with self.assertRaises(datastore_errors.BadValueError):
task_request.make_request(
_gen_request(properties=dict(commands=['see', 'spot', 'run'],
isolated='something.isolated')),
True)
def test_make_request(self):
# Compare with test_make_request_clone().
parent = task_request.make_request(_gen_request_data())
# Hack: Would need to know about TaskResultSummary.
parent_id = task_pack.pack_request_key(parent.key) + '1'
data = _gen_request_data(properties=dict(idempotent=True),
parent_task_id=parent_id)
request = task_request.make_request(data)
expected_properties = {
'commands': [[u'command1', u'arg1']],
'data': [
# Items were sorted.
[u'http://localhost/bar', u'bar.zip'],
[u'http://localhost/foo', u'foo.zip'],
],
'dimensions': {
u'OS': u'Windows-3.1.1',
u'hostname': u'localhost'
},
'env': {
u'foo': u'bar',
u'joe': u'2'
},
'execution_timeout_secs':
30,
'grace_period_secs':
30,
'idempotent':
True,
'io_timeout_secs':
None,
}
expected_request = {
'authenticated':
auth_testing.DEFAULT_MOCKED_IDENTITY,
'name':
u'Request name',
'parent_task_id':
unicode(parent_id),
'priority':
49,
'properties':
expected_properties,
'properties_hash':
'6ec96bdc40fad2bdaec3cbbe43594961ad72f02e',
'tags': [
u'OS:Windows-3.1.1',
u'hostname:localhost',
u'priority:49',
u'tag:1',
u'user:Jesus',
],
'user':
u'Jesus',
}
actual = request.to_dict()
# expiration_ts - created_ts == scheduling_expiration_secs.
created = actual.pop('created_ts')
expiration = actual.pop('expiration_ts')
self.assertEqual(int(round((expiration - created).total_seconds())),
data['scheduling_expiration_secs'])
self.assertEqual(expected_request, actual)
self.assertEqual(data['scheduling_expiration_secs'],
request.scheduling_expiration_secs)
def test_make_request_clone(self):
# Compare with test_make_request().
parent = task_request.make_request(_gen_request(), True)
# Hack: Would need to know about TaskResultSummary.
parent_id = task_pack.pack_request_key(parent.key) + '1'
data = _gen_request(properties=dict(idempotent=True),
parent_task_id=parent_id)
request = task_request.make_request_clone(
task_request.make_request(data, True))
# Differences from make_request() are:
# - idempotent was reset to False.
# - parent_task_id was reset to None.
expected_properties = {
'commands': [[u'command1', u'arg1']],
'data': [
# Items were sorted.
[u'http://localhost/bar', u'bar.zip'],
[u'http://localhost/foo', u'foo.zip'],
],
'dimensions': {
u'OS': u'Windows-3.1.1',
u'hostname': u'localhost'
},
'env': {
u'foo': u'bar',
u'joe': u'2'
},
'execution_timeout_secs':
30,
'extra_args': [],
'grace_period_secs':
30,
'idempotent':
False,
'inputs_ref':
None,
'io_timeout_secs':
None,
}
# Differences from make_request() are:
# - parent_task_id was reset to None.
# - tag 'user:'******'authenticated':
auth_testing.DEFAULT_MOCKED_IDENTITY,
'name':
u'Request name (Retry #1)',
'parent_task_id':
None,
'priority':
49,
'properties':
expected_properties,
'properties_hash':
None,
'pubsub_topic':
None,
'pubsub_userdata':
None,
'tags': [
u'OS:Windows-3.1.1',
u'hostname:localhost',
u'priority:49',
u'tag:1',
u'user:[email protected]',
],
'user':
u'*****@*****.**',
}
actual = request.to_dict()
# expiration_ts - created_ts == deadline_to_run.
actual.pop('created_ts')
actual.pop('expiration_ts')
self.assertEqual(expected_request, actual)
self.assertEqual(30, request.expiration_secs)
def test_new_task_to_run(self):
self.mock(random, 'getrandbits', lambda _: 0x12)
request_dimensions = {u'OS': u'Windows-3.1.1'}
data = _gen_request_data(properties={
'commands': [[u'command1', u'arg1']],
'data': [[u'http://localhost/foo', u'foo.zip']],
'dimensions':
request_dimensions,
'env': {
u'foo': u'bar'
},
'execution_timeout_secs':
30,
},
priority=20,
scheduling_expiration_secs=31)
task_to_run.new_task_to_run(task_request.make_request(data)).put()
# Create a second with higher priority.
self.mock(random, 'getrandbits', lambda _: 0x23)
data = _gen_request_data(properties={
'commands': [[u'command1', u'arg1']],
'data': [[u'http://localhost/foo', u'foo.zip']],
'dimensions':
request_dimensions,
'env': {
u'foo': u'bar'
},
'execution_timeout_secs':
30,
},
priority=10,
scheduling_expiration_secs=31)
task_to_run.new_task_to_run(task_request.make_request(data)).put()
expected = [
{
'dimensions_hash': _hash_dimensions(request_dimensions),
'expiration_ts': self.now + datetime.timedelta(seconds=31),
'request_key': '0x7e296460f77ffdce',
# Lower priority value means higher priority.
'queue_number': '0x00060dc5849f1346',
},
{
'dimensions_hash': _hash_dimensions(request_dimensions),
'expiration_ts': self.now + datetime.timedelta(seconds=31),
'request_key': '0x7e296460f77ffede',
'queue_number': '0x00072c96fd65d346',
},
]
def flatten(i):
out = _task_to_run_to_dict(i)
out['request_key'] = '0x%016x' % i.request_key.integer_id()
return out
# Warning: Ordering by key doesn't work because of TaskToRunShard; e.g.
# the entity key ordering DOES NOT correlate with .queue_number
# Ensure they come out in expected order.
q = task_to_run.TaskToRun.query().order(
task_to_run.TaskToRun.queue_number)
self.assertEqual(expected, map(flatten, q.fetch()))
def mkreq(req):
return task_request.make_request(req, True)
def test_task_to_run_key_to_request_key(self):
request = task_request.make_request(_gen_request_data())
task_key = task_to_run.request_to_task_to_run_key(request)
actual = task_to_run.task_to_run_key_to_request_key(task_key)
self.assertEqual(request.key, actual)
def _gen_new_task_to_run(**kwargs):
"""Returns a TaskToRun saved in the DB."""
data = _gen_request_data(**kwargs)
to_run = task_to_run.new_task_to_run(task_request.make_request(data))
to_run.put()
return to_run
def test_cron_handle_bot_died_same_bot_denied(self):
# Test first retry, then success.
self.mock(random, 'getrandbits', lambda _: 0x88)
data = _gen_request_data(
properties=dict(dimensions={u'OS': u'Windows-3.1.1'}),
scheduling_expiration_secs=600)
request = task_request.make_request(data)
_result_summary = task_scheduler.schedule_request(request)
bot_dimensions = {
u'OS': [u'Windows', u'Windows-3.1.1'],
u'hostname': u'localhost',
u'foo': u'bar',
}
_request, run_result = task_scheduler.bot_reap_task(
bot_dimensions, 'localhost', 'abc')
self.assertEqual(1, run_result.try_number)
self.assertEqual(task_result.State.RUNNING, run_result.state)
now_1 = self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 1)
self.assertEqual((0, 1, 0), task_scheduler.cron_handle_bot_died())
# Refresh and compare:
expected = {
'abandoned_ts': now_1,
'bot_id': u'localhost',
'bot_version': u'abc',
'children_task_ids': [],
'completed_ts': None,
'cost_usd': 0.,
'durations': [],
'exit_codes': [],
'failure': False,
'id': '1d69b9f088008811',
'internal_failure': True,
'modified_ts': now_1,
'server_versions': [u'v1a'],
'started_ts': self.now,
'state': task_result.State.BOT_DIED,
'try_number': 1,
}
self.assertEqual(expected, run_result.key.get().to_dict())
expected = {
'abandoned_ts': None,
'bot_id': u'localhost',
'bot_version': u'abc',
'children_task_ids': [],
'completed_ts': None,
'costs_usd': [0.],
'cost_saved_usd': None,
'created_ts': self.now,
'deduped_from': None,
'durations': [],
'exit_codes': [],
'failure': False,
'id': '1d69b9f088008810',
'internal_failure': False,
'modified_ts': now_1,
'name': u'Request name',
'properties_hash': None,
'server_versions': [u'v1a'],
'started_ts': None,
'state': task_result.State.PENDING,
'try_number': 1,
'user': u'Jesus',
}
self.assertEqual(expected,
run_result.result_summary_key.get().to_dict())
# Task was retried but the same bot polls again, it's denied the task.
now_2 = self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 2)
request, run_result = task_scheduler.bot_reap_task(
bot_dimensions, 'localhost', 'abc')
self.assertEqual(None, request)
self.assertEqual(None, run_result)
logging.info('%s',
[t.to_dict() for t in task_to_run.TaskToRun.query()])
