def _gen_request(properties=None):
"""Creates a TaskRequest."""
now = utils.utcnow()
args = {
'created_ts': now,
'manual_tags': [u'tag:1'],
'name': 'Request name',
'priority': 50,
'task_slices': [
task_request.TaskSlice(
expiration_secs=60,
properties=properties or _gen_properties()),
],
'user': '******',
}
req = task_request.TaskRequest(**args)
task_request.init_new_request(req, True)
return req
def test_duped(self):
# Two TestRequest with the same properties.
request_1 = _gen_request(properties=_gen_properties(idempotent=True))
now = utils.utcnow()
request_2 = _gen_request_slices(
name='Other',
user='******',
priority=201,
created_ts=now,
manual_tags=['tag:2'],
task_slices=[
task_request.TaskSlice(
expiration_secs=129,
properties=_gen_properties(idempotent=True)),
])
self.assertEqual(
request_1.task_slice(0).properties_hash(),
request_2.task_slice(0).properties_hash())
self.assertTrue(request_1.task_slice(0).properties_hash())
def test_new_task_to_run_limits(self):
# Generate a TaskRequest with eight TaskSlice.
slices = [
task_request.TaskSlice(
expiration_secs=60,
properties=_gen_properties(
dimensions={u'pool': [u'default'], u'v': [unicode(i)]}))
for i in xrange(8)
]
request = self.mkreq(8, _gen_request_slices(task_slices=slices))
with self.assertRaises(AssertionError):
task_to_run.new_task_to_run(request, 0, 0)
task_to_run.new_task_to_run(request, 1, 0)
task_to_run.new_task_to_run(request, 2, 0)
with self.assertRaises(AssertionError):
task_to_run.new_task_to_run(request, 3, 0)
task_to_run.new_task_to_run(request, 1, 7)
with self.assertRaises(IndexError):
task_to_run.new_task_to_run(request, 1, 8)
def _gen_request_slices(**kwargs):
"""Creates a TaskRequest."""
now = utils.utcnow()
args = {
u'created_ts':
now,
u'manual_tags': [u'tag:1'],
u'name':
u'Request name',
u'priority':
50,
u'task_slices': [
task_request.TaskSlice(expiration_secs=30,
properties=_gen_properties()),
],
u'user':
u'Jesus',
}
args.update(kwargs)
# Note that ndb model constructor accepts dicts for structured properties.
req = task_request.TaskRequest(**args)
task_request.init_new_request(req, True)
return req
def _gen_request_slice(**kwargs):
"""Creates a TaskRequest."""
now = utils.utcnow()
args = {
'created_ts':
now,
'manual_tags': [u'tag:1'],
'name':
'Request name',
'priority':
50,
'task_slices': [
task_request.TaskSlice(expiration_secs=60,
properties=_gen_properties()),
],
'user':
'******',
}
args.update(kwargs)
ret = task_request.TaskRequest(**args)
task_request.init_new_request(ret, True)
ret.key = task_request.new_request_key()
ret.put()
return ret
def make_task_request(self,
service_account,
service_account_token=None,
realm=None):
now = utils.utcnow()
args = {
'created_ts':
now,
'manual_tags': [u'tag:1'],
'name':
'Request with %s' % service_account,
'priority':
50,
'task_slices': [
task_request.TaskSlice(
expiration_secs=60,
properties=task_request.TaskProperties(
command=[u'command1'],
dimensions_data={u'pool': [u'default']},
execution_timeout_secs=24 * 60 * 60)),
],
'user':
'******',
}
req = task_request.TaskRequest(**args)
task_request.init_new_request(req, True, task_request.TEMPLATE_AUTO)
req.key = task_request.new_request_key()
req.service_account = service_account
req.service_account_token = service_account_token
req.realm = realm
req.put()
summary_key = task_pack.request_key_to_result_summary_key(req.key)
run_result_key = task_pack.result_summary_key_to_run_result_key(
summary_key, 1)
return task_pack.pack_run_result_key(run_result_key)
def _gen_slice(**props):
return task_request.TaskSlice(
expiration_secs=60, properties=_gen_properties(**props))
def test_new_task_to_run_list(self):
self.mock(random, 'getrandbits', lambda _: 0x12)
request_dimensions = {u'os': [u'Windows-3.1.1'], u'pool': [u'default']}
data = _gen_request_slices(
priority=20,
created_ts=self.now,
task_slices=[
task_request.TaskSlice(
expiration_secs=31,
properties=_gen_properties(
command=[u'command1', u'arg1'],
dimensions=request_dimensions,
env={u'foo': u'bar'},
execution_timeout_secs=30)),
])
request = self.mkreq(1, data)
task_to_run.new_task_to_run(request, 1, 0).put()
# Create a second with higher priority.
self.mock(random, 'getrandbits', lambda _: 0x23)
data = _gen_request_slices(
priority=10,
created_ts=self.now,
task_slices=[
task_request.TaskSlice(
expiration_secs=31,
properties=_gen_properties(
command=[u'command1', u'arg1'],
dimensions=request_dimensions,
env={u'foo': u'bar'},
execution_timeout_secs=30)),
])
task_to_run.new_task_to_run(self.mkreq(0, data), 1, 0).put()
expected = [
{
'created_ts': self.now,
'expiration_ts': self.now + datetime.timedelta(seconds=31),
'request_key': '0x7e296460f77ffdce',
# Lower priority value means higher priority.
'queue_number': '0x1a3aa663153d248e',
'task_slice_index': 0,
'try_number': 1,
},
{
'created_ts': self.now,
'expiration_ts': self.now + datetime.timedelta(seconds=31),
'request_key': '0x7e296460f77ffede',
'queue_number': '0x1a3aa66317bd248e',
'task_slice_index': 0,
'try_number': 1,
},
]
def flatten(i):
out = i.to_dict()
out['request_key'] = '0x%016x' % i.request_key.integer_id()
return out
# Warning: Ordering by key doesn't work because of TaskToRunShard; e.g.
# the entity key ordering DOES NOT correlate with .queue_number
# Ensure they come out in expected order.
q = task_to_run.TaskToRun.query().order(task_to_run.TaskToRun.queue_number)
self.assertEqual(expected, map(flatten, q.fetch()))
def new_task_request_from_rpc(msg, now):
""""Returns a (task_request.TaskRequest, task_request.SecretBytes,
task_request.TemplateApplyEnum) from a swarming_rpcs.NewTaskRequest.
If secret_bytes were not included in the rpc, the SecretBytes entity will be
None.
"""
assert msg.__class__ is swarming_rpcs.NewTaskRequest
if msg.task_slices and msg.properties:
raise ValueError('Specify one of properties or task_slices, not both')
if msg.properties:
logging.info('Properties is still used')
if not msg.expiration_secs:
raise ValueError('missing expiration_secs')
props, secret_bytes = _taskproperties_from_rpc(msg.properties)
slices = [
task_request.TaskSlice(properties=props,
expiration_secs=msg.expiration_secs),
]
elif msg.task_slices:
if msg.expiration_secs:
raise ValueError(
'When using task_slices, do not specify a global expiration_secs'
)
secret_bytes = None
slices = []
for t in (msg.task_slices or []):
sl, se = _taskslice_from_rpc(t)
slices.append(sl)
if se:
if secret_bytes and se != secret_bytes:
raise ValueError(
'When using secret_bytes multiple times, all values must match'
)
secret_bytes = se
else:
raise ValueError('Specify one of properties or task_slices')
pttf = swarming_rpcs.PoolTaskTemplateField
template_apply = {
pttf.AUTO: task_request.TEMPLATE_AUTO,
pttf.CANARY_NEVER: task_request.TEMPLATE_CANARY_NEVER,
pttf.CANARY_PREFER: task_request.TEMPLATE_CANARY_PREFER,
pttf.SKIP: task_request.TEMPLATE_SKIP,
}[msg.pool_task_template]
req = _rpc_to_ndb(
task_request.TaskRequest,
msg,
created_ts=now,
expiration_ts=None,
# It is set in task_request.init_new_request().
authenticated=None,
properties=None,
task_slices=slices,
# 'tags' is now generated from manual_tags plus automatic tags.
tags=None,
manual_tags=msg.tags,
# This is internal field not settable via RPC.
service_account_token=None,
pool_task_template=None) # handled out of band
return req, secret_bytes, template_apply