def get(self, bot_id):
logging.error('Unexpected old client')
bot = bot_management.get_info_key(bot_id).get()
if not bot:
self.abort_with_error(404, error='Bot not found')
now = utils.utcnow()
self.send_response(utils.to_json_encodable(bot.to_dict_with_now(now)))
def _cmd_run(self, request, run_result_key, bot_id):
# Only one of 'command' or 'inputs_ref' can be set.
out = {
'cmd': 'run',
'manifest': {
'bot_id':
bot_id,
'command':
request.properties.commands[0]
if request.properties.commands else None,
'data':
request.properties.data,
'dimensions':
request.properties.dimensions,
'env':
request.properties.env,
'extra_args':
request.properties.extra_args,
'grace_period':
request.properties.grace_period_secs,
'hard_timeout':
request.properties.execution_timeout_secs,
'host':
utils.get_versioned_hosturl(),
'io_timeout':
request.properties.io_timeout_secs,
'inputs_ref':
request.properties.inputs_ref,
'task_id':
task_pack.pack_run_result_key(run_result_key),
},
}
self.send_response(utils.to_json_encodable(out))
def get(self):
extra = frozenset(self.request.GET) - self.EXPECTED
if extra:
self.abort_with_error(
400,
error='Extraneous query parameters. Did you make a typo? %s' %
','.join(sorted(extra)))
# Use a similar query to /user/tasks.
name = self.request.get('name')
tags = self.request.get_all('tag')
cursor_str = self.request.get('cursor')
limit = int(self.request.get('limit', 100))
sort = self.request.get('sort', 'created_ts')
state = self.request.get('state', 'all')
uses = bool(name) + bool(tags) + bool(state!='all')
if uses > 1:
self.abort_with_error(
400, error='Only one of name, tag (1 or many) or state can be used')
items, cursor_str, sort, state = task_result.get_tasks(
name, tags, cursor_str, limit, sort, state)
data = {
'cursor': cursor_str,
'items': items,
'limit': limit,
'sort': sort,
'state': state,
}
self.send_response(utils.to_json_encodable(data))
def get(self):
extra = frozenset(self.request.GET) - self.EXPECTED
if extra:
self.abort_with_error(
400,
error='Extraneous query parameters. Did you make a typo? %s' %
','.join(sorted(extra)))
# Use a similar query to /user/tasks.
name = self.request.get('name')
tags = self.request.get_all('tag')
cursor_str = self.request.get('cursor')
limit = int(self.request.get('limit', 100))
sort = self.request.get('sort', 'created_ts')
state = self.request.get('state', 'all')
uses = bool(name) + bool(tags) + bool(state!='all')
if uses > 1:
self.abort_with_error(
400, error='Only one of name, tag (1 or many) or state can be used')
items, cursor_str, sort, state = task_result.get_tasks(
name, tags, cursor_str, limit, sort, state)
data = {
'cursor': cursor_str,
'items': items,
'limit': limit,
'sort': sort,
'state': state,
}
self.send_response(utils.to_json_encodable(data))
def request(self, request):
"""Returns the task request corresponding to a task ID."""
_, summary_key = get_result_key(request.task_id)
request_key = task_pack.result_summary_key_to_request_key(summary_key)
entity = get_or_raise(request_key)
return message_conversion.task_request_from_dict(
utils.to_json_encodable(entity))
def get(self):
extra = frozenset(self.request.GET) - self.EXPECTED
if extra:
self.abort_with_error(
400,
error='Extraneous query parameters. Did you make a typo? %s' %
','.join(sorted(extra)))
interval = self.request.get('interval', 24 * 3600)
state = self.request.get('state', 'all')
tags = self.request.get_all('tag')
try:
interval = int(interval)
if interval <= 0:
raise ValueError()
except ValueError:
self.abort_with_error(
400, error='"interval" must be a positive integer number of seconds')
if state not in self.VALID_STATES:
self.abort_with_error(
400,
error='Invalid state "%s", expecting on of %s' %
(state, ', '.join(sorted(self.VALID_STATES))))
# Cutoff deadline => request key for filtering (it embeds timestamp).
cutoff = utils.utcnow() - datetime.timedelta(seconds=interval)
request_id = task_request.datetime_to_request_base_id(cutoff)
request_key = task_request.request_id_to_key(request_id)
query = task_result.get_result_summary_query(None, state, tags)
query = query.filter(task_result.TaskResultSummary.key <= request_key)
self.send_response(utils.to_json_encodable({'count': query.count()}))
def get(self, bot_id):
logging.error('Unexpected old client')
bot = bot_management.get_info_key(bot_id).get()
if not bot:
self.abort_with_error(404, error='Bot not found')
now = utils.utcnow()
self.send_response(utils.to_json_encodable(bot.to_dict_with_now(now)))
def get(self):
extra = frozenset(self.request.GET) - self.EXPECTED
if extra:
self.abort_with_error(
400,
error='Extraneous query parameters. Did you make a typo? %s' %
','.join(sorted(extra)))
interval = self.request.get('interval', 24 * 3600)
state = self.request.get('state', 'all')
tags = self.request.get_all('tag')
try:
interval = int(interval)
if interval <= 0:
raise ValueError()
except ValueError:
self.abort_with_error(
400, error='"interval" must be a positive integer number of seconds')
if state not in self.VALID_STATES:
self.abort_with_error(
400,
error='Invalid state "%s", expecting on of %s' %
(state, ', '.join(sorted(self.VALID_STATES))))
# Cutoff deadline => request key for filtering (it embeds timestamp).
cutoff = utils.utcnow() - datetime.timedelta(seconds=interval)
request_id = task_request.datetime_to_request_base_id(cutoff)
request_key = task_request.request_id_to_key(request_id)
query = task_result.get_result_summary_query(None, state, tags)
query = query.filter(task_result.TaskResultSummary.key <= request_key)
self.send_response(utils.to_json_encodable({'count': query.count()}))
def get(self):
logging.error('Unexpected old client')
extra = frozenset(self.request.GET) - self.EXPECTED
if extra:
self.abort_with_error(
400,
error='Extraneous query parameters. Did you make a typo? %s' %
','.join(sorted(extra)))
interval = self.request.get('interval', 24 * 3600)
state = self.request.get('state', 'all')
tags = self.request.get_all('tag')
try:
interval = int(interval)
if interval <= 0:
raise ValueError()
except ValueError:
self.abort_with_error(
400,
error='"interval" must be a positive integer number of seconds'
)
if state not in self.VALID_STATES:
self.abort_with_error(
400,
error='Invalid state "%s", expecting on of %s' %
(state, ', '.join(sorted(self.VALID_STATES))))
cutoff = utils.utcnow() - datetime.timedelta(seconds=interval)
query = task_result.get_result_summaries_query(cutoff, None,
'created_ts', state,
tags)
self.send_response(utils.to_json_encodable({'count': query.count()}))
def get(self):
logging.error('Unexpected old client')
extra = frozenset(self.request.GET) - self.EXPECTED
if extra:
self.abort_with_error(
400,
error='Extraneous query parameters. Did you make a typo? %s' %
','.join(sorted(extra)))
interval = self.request.get('interval', 24 * 3600)
state = self.request.get('state', 'all')
tags = self.request.get_all('tag')
try:
interval = int(interval)
if interval <= 0:
raise ValueError()
except ValueError:
self.abort_with_error(
400, error='"interval" must be a positive integer number of seconds')
if state not in self.VALID_STATES:
self.abort_with_error(
400,
error='Invalid state "%s", expecting on of %s' %
(state, ', '.join(sorted(self.VALID_STATES))))
cutoff = utils.utcnow() - datetime.timedelta(seconds=interval)
query = task_result.get_result_summaries_query(
cutoff, None, 'created_ts', state, tags)
self.send_response(utils.to_json_encodable({'count': query.count()}))
def get(self, task_id, command_index):
result = self.get_result_entity(task_id)
output = result.get_command_output_async(int(command_index)).get_result()
if output:
output = output.decode('utf-8', 'replace')
# JSON then reencodes to ascii compatible encoded strings, which explodes
# the size.
data = {
'output': output,
}
self.send_response(utils.to_json_encodable(data))
def get(self, task_id):
result = self.get_result_entity(task_id)
# JSON then reencodes to ascii compatible encoded strings, which explodes
# the size.
data = {
'outputs': [
i.decode('utf-8', 'replace') if i else i
for i in result.get_outputs()
],
}
self.send_response(utils.to_json_encodable(data))
def get(self, task_id):
result = self.get_result_entity(task_id)
# JSON then reencodes to ascii compatible encoded strings, which explodes
# the size.
data = {
'outputs': [
i.decode('utf-8', 'replace') if i else i
for i in result.get_outputs()
],
}
self.send_response(utils.to_json_encodable(data))
def get(self, task_id, command_index):
result = self.get_result_entity(task_id)
output = result.get_command_output_async(int(command_index)).get_result()
if output:
output = output.decode('utf-8', 'replace')
# JSON then reencodes to ascii compatible encoded strings, which explodes
# the size.
data = {
'output': output,
}
self.send_response(utils.to_json_encodable(data))
def post(self):
logging.error('Unexpected old client')
data = self.parse_body()
msg = log_unexpected_subset_keys(self._EXPECTED_DATA_KEYS,
self._REQUIRED_DATA_KEYS, data,
self.request, 'client',
'request keys')
if msg:
self.abort_with_error(400, error=msg)
data_properties = data['properties']
msg = log_unexpected_subset_keys(self._EXPECTED_PROPERTIES_KEYS,
self._REQUIRED_PROPERTIES_KEYS,
data_properties, self.request,
'client', 'request properties keys')
if msg:
self.abort_with_error(400, error=msg)
# Class TaskProperties takes care of making everything deterministic.
properties = task_request.TaskProperties(
commands=data_properties['commands'],
data=data_properties['data'],
dimensions=data_properties['dimensions'],
env=data_properties['env'],
execution_timeout_secs=data_properties['execution_timeout_secs'],
grace_period_secs=data_properties.get('grace_period_secs', 30),
idempotent=data_properties.get('idempotent', False),
io_timeout_secs=data_properties['io_timeout_secs'])
now = utils.utcnow()
expiration_ts = now + datetime.timedelta(
seconds=data['scheduling_expiration_secs'])
request = task_request.TaskRequest(
created_ts=now,
expiration_ts=expiration_ts,
name=data['name'],
parent_task_id=data.get('parent_task_id'),
priority=data['priority'],
properties=properties,
tags=data['tags'],
user=data['user'] or '')
try:
request = task_request.make_request(request, acl.is_bot_or_admin())
except (AttributeError, datastore_errors.BadValueError, TypeError,
ValueError) as e:
self.abort_with_error(400, error=str(e))
result_summary = task_scheduler.schedule_request(request)
data = {
'request': request.to_dict(),
'task_id': task_pack.pack_result_summary_key(result_summary.key),
}
self.send_response(utils.to_json_encodable(data))
def post(self):
logging.error('Unexpected old client')
data = self.parse_body()
msg = log_unexpected_subset_keys(
self._EXPECTED_DATA_KEYS, self._REQUIRED_DATA_KEYS, data, self.request,
'client', 'request keys')
if msg:
self.abort_with_error(400, error=msg)
data_properties = data['properties']
msg = log_unexpected_subset_keys(
self._EXPECTED_PROPERTIES_KEYS, self._REQUIRED_PROPERTIES_KEYS,
data_properties, self.request, 'client', 'request properties keys')
if msg:
self.abort_with_error(400, error=msg)
# Class TaskProperties takes care of making everything deterministic.
properties = task_request.TaskProperties(
commands=data_properties['commands'],
data=data_properties['data'],
dimensions=data_properties['dimensions'],
env=data_properties['env'],
execution_timeout_secs=data_properties['execution_timeout_secs'],
grace_period_secs=data_properties.get('grace_period_secs', 30),
idempotent=data_properties.get('idempotent', False),
io_timeout_secs=data_properties['io_timeout_secs'])
now = utils.utcnow()
expiration_ts = now + datetime.timedelta(
seconds=data['scheduling_expiration_secs'])
request = task_request.TaskRequest(
created_ts=now,
expiration_ts=expiration_ts,
name=data['name'],
parent_task_id=data.get('parent_task_id'),
priority=data['priority'],
properties=properties,
tags=data['tags'],
user=data['user'] or '')
try:
request = task_request.make_request(request, acl.is_bot_or_admin())
except (
AttributeError, datastore_errors.BadValueError, TypeError,
ValueError) as e:
self.abort_with_error(400, error=str(e))
result_summary = task_scheduler.schedule_request(request)
data = {
'request': request.to_dict(),
'task_id': task_pack.pack_result_summary_key(result_summary.key),
}
self.send_response(utils.to_json_encodable(data))
def test_json(self):
r = Rambling(a=2,
b=0.2,
c=datetime.datetime(2012, 1, 2, 3, 4, 5, 6),
d=datetime.date(2012, 1, 2))
actual = utils.to_json_encodable([r])
# Confirm that default is tight encoding and sorted keys.
expected = [
{
'a': 2,
'b': 0.2,
'c': u'2012-01-02 03:04:05',
'd': u'2012-01-02',
'e': 1,
'f': u'\u0129',
},
]
self.assertEqual(expected, actual)
self.assertEqual([0, 1], utils.to_json_encodable(range(2)))
self.assertEqual([0, 1], utils.to_json_encodable(i for i in (0, 1)))
self.assertEqual([0, 1], utils.to_json_encodable(xrange(2)))
def add_machines(requests):
"""Add machines to the Machine Provider's Catalog.
Args:
requests: A list of rpc_messages.CatalogMachineAdditionRequest instances.
"""
logging.info('Sending batched add_machines request')
return net.json_request(
'%s/add_machines' % CATALOG_BASE_URL,
method='POST',
payload=utils.to_json_encodable({'requests': requests}),
scopes=MACHINE_PROVIDER_SCOPES,
)
def lease_machine(request):
"""Lease a machine from the Machine Provider.
Args:
request: An rpc_messages.LeaseRequest instance.
"""
return net.json_request(
'%s/_ah/api/machine_provider/v1/lease' %
MachineProviderConfiguration.get_instance_url(),
method='POST',
payload=utils.to_json_encodable(request),
scopes=MACHINE_PROVIDER_SCOPES,
)
def lease_machines(requests):
"""Lease machines from the Machine Provider.
Args:
requests: A list of rpc_messages.LeaseRequest instances.
"""
logging.info('Sending batched lease_machines request')
return net.json_request(
'%s/_ah/api/machine_provider/v1/batched_lease' %
MachineProviderConfiguration.get_instance_url(),
method='POST',
payload=utils.to_json_encodable({'requests': requests}),
scopes=MACHINE_PROVIDER_SCOPES,
)
def get(self):
now = utils.utcnow()
limit = int(self.request.get('limit', 1000))
cursor = datastore_query.Cursor(urlsafe=self.request.get('cursor'))
q = bot_management.BotInfo.query().order(bot_management.BotInfo.key)
bots, cursor, more = q.fetch_page(limit, start_cursor=cursor)
data = {
'cursor': cursor.urlsafe() if cursor and more else None,
'death_timeout': config.settings().bot_death_timeout_secs,
'items': [b.to_dict_with_now(now) for b in bots],
'limit': limit,
'now': now,
}
self.send_response(utils.to_json_encodable(data))
def add_machines(requests):
"""Add machines to the Machine Provider's Catalog.
Args:
requests: A list of rpc_messages.CatalogMachineAdditionRequest instances.
"""
logging.info('Sending batched add_machines request')
return net.json_request(
'%s/_ah/api/catalog/v1/add_machines' %
MachineProviderConfiguration.get_instance_url(),
method='POST',
payload=utils.to_json_encodable({'requests': requests}),
scopes=MACHINE_PROVIDER_SCOPES,
)
def test_json(self):
r = Rambling(
a=2,
b=0.2,
c=datetime.datetime(2012, 1, 2, 3, 4, 5, 6),
d=datetime.date(2012, 1, 2))
actual = utils.to_json_encodable([r])
# Confirm that default is tight encoding and sorted keys.
expected = [
{
'a': 2,
'b': 0.2,
'c': u'2012-01-02 03:04:05',
'd': u'2012-01-02',
'e': 1,
'f': u'\u0129',
},
]
self.assertEqual(expected, actual)
self.assertEqual([0, 1], utils.to_json_encodable(range(2)))
self.assertEqual([0, 1], utils.to_json_encodable(i for i in (0, 1)))
self.assertEqual([0, 1], utils.to_json_encodable(xrange(2)))
def add_machines(requests):
"""Add machines to the Machine Provider's Catalog.
Args:
requests: A list of rpc_messages.CatalogMachineAdditionRequest instances.
"""
logging.info('Sending batched add_machines request')
return net.json_request(
'%s/_ah/api/catalog/v1/add_machines' %
MachineProviderConfiguration.get_instance_url(),
method='POST',
payload=utils.to_json_encodable({'requests': requests}),
scopes=MACHINE_PROVIDER_SCOPES,
)
def lease_machines(requests):
"""Lease machines from the Machine Provider.
Args:
requests: A list of rpc_messages.LeaseRequest instances.
"""
logging.info('Sending batched lease_machines request')
return net.json_request(
'%s/_ah/api/machine_provider/v1/batched_lease' %
MachineProviderConfiguration.get_instance_url(),
method='POST',
payload=utils.to_json_encodable({'requests': requests}),
scopes=MACHINE_PROVIDER_SCOPES,
)
def release_machine(client_request_id):
"""Voluntarily releases a leased machine back to Machine Provider.
Args:
client_request_id: Request ID originally used by the client when creating
the lease request.
"""
return net.json_request(
'%s/_ah/api/machine_provider/v1/release' %
MachineProviderConfiguration.get_instance_url(),
method='POST',
payload=utils.to_json_encodable({'request_id': client_request_id}),
scopes=MACHINE_PROVIDER_SCOPES,
)
def get(self, bot_id):
limit = int(self.request.get('limit', 100))
cursor = datastore_query.Cursor(urlsafe=self.request.get('cursor'))
run_results, cursor, more = task_result.TaskRunResult.query(
task_result.TaskRunResult.bot_id == bot_id).order(
-task_result.TaskRunResult.started_ts).fetch_page(
limit, start_cursor=cursor)
now = utils.utcnow()
data = {
'cursor': cursor.urlsafe() if cursor and more else None,
'items': run_results,
'limit': limit,
'now': now,
}
self.send_response(utils.to_json_encodable(data))
def get(self, bot_id):
limit = int(self.request.get('limit', 100))
cursor = datastore_query.Cursor(urlsafe=self.request.get('cursor'))
run_results, cursor, more = task_result.TaskRunResult.query(
task_result.TaskRunResult.bot_id == bot_id).order(
-task_result.TaskRunResult.started_ts).fetch_page(
limit, start_cursor=cursor)
now = utils.utcnow()
data = {
'cursor': cursor.urlsafe() if cursor and more else None,
'items': run_results,
'limit': limit,
'now': now,
}
self.send_response(utils.to_json_encodable(data))
def tasks(self, request):
"""Lists all of a given bot's tasks."""
cursor = datastore_query.Cursor(urlsafe=request.cursor)
run_results, cursor, more = task_result.TaskRunResult.query(
task_result.TaskRunResult.bot_id == request.bot_id).order(
-task_result.TaskRunResult.started_ts).fetch_page(
request.limit, start_cursor=cursor)
result_list = [message_conversion.task_run_result_from_dict(
utils.to_json_encodable(run_result)) for run_result in run_results]
response = swarming_rpcs.BotTask(
cursor=cursor.urlsafe() if cursor and more else None,
items=result_list,
limit=request.limit,
now=utils.utcnow())
return response
def delete_machine(dimensions):
"""Deletes a machine from the Machine Provider's Catalog.
Args:
dimensions: Dimensions for the machine.
"""
logging.info('Sending delete_machine request')
return net.json_request(
'%s/_ah/api/catalog/v1/delete_machine' %
MachineProviderConfiguration.get_instance_url(),
method='POST',
payload=utils.to_json_encodable({
'dimensions': dimensions,
}),
scopes=MACHINE_PROVIDER_SCOPES,
)
def delete_machine(dimensions):
"""Deletes a machine from the Machine Provider's Catalog.
Args:
dimensions: Dimensions for the machine.
"""
logging.info('Sending delete_machine request')
return net.json_request(
'%s/_ah/api/catalog/v1/delete_machine' %
MachineProviderConfiguration.get_instance_url(),
method='POST',
payload=utils.to_json_encodable({
'dimensions': dimensions,
}),
scopes=MACHINE_PROVIDER_SCOPES,
)
def tasks(self, request):
"""Lists a given bot's tasks within the specified date range."""
try:
start, end = _get_range(request)
run_results, cursor, more = task_result.get_run_results(
request.cursor, request.bot_id, start, end, request.batch_size)
except ValueError as e:
raise endpoints.BadRequestException(
'Inappropriate batch_size for bots/list: %s' % e)
result_list = [message_conversion.task_run_result_from_dict(
utils.to_json_encodable(run_result)) for run_result in run_results]
response = swarming_rpcs.BotTasks(
cursor=cursor.urlsafe() if cursor and more else None,
items=result_list,
now=utils.utcnow())
return response
def retrieve_machine(hostname, backend=None):
"""Requests information about a machine from the Machine Provider's Catalog.
Args:
hostname: Hostname of the machine to request information about.
backend: Backend the machine belongs to.
"""
return net.json_request(
'%s/_ah/api/catalog/v1/get' %
MachineProviderConfiguration.get_instance_url(),
method='POST',
payload=utils.to_json_encodable({
'backend': backend,
'hostname': hostname,
}),
scopes=MACHINE_PROVIDER_SCOPES,
)
def post(self):
request_data = self.parse_body()
# If the priority is below 100, make the the user has right to do so.
if request_data.get('priority', 255) < 100 and not acl.is_bot_or_admin():
# Silently drop the priority of normal users.
request_data['priority'] = 100
try:
request = task_request.make_request(request_data)
except (datastore_errors.BadValueError, TypeError, ValueError) as e:
self.abort_with_error(400, error=str(e))
result_summary = task_scheduler.schedule_request(request)
data = {
'request': request.to_dict(),
'task_id': task_pack.pack_result_summary_key(result_summary.key),
}
self.send_response(utils.to_json_encodable(data))
def add_machine(dimensions, policies):
"""Add a machine to the Machine Provider's Catalog.
Args:
dimensions: Dimensions for this machine.
policies: Policies governing this machine.
"""
logging.info('Sending add_machine request')
return net.json_request(
'%s/_ah/api/catalog/v1/add_machine' %
MachineProviderConfiguration.get_instance_url(),
method='POST',
payload=utils.to_json_encodable({
'dimensions': dimensions,
'policies': policies,
}),
scopes=MACHINE_PROVIDER_SCOPES,
)
def add_machine(dimensions, policies):
"""Add a machine to the Machine Provider's Catalog.
Args:
dimensions: Dimensions for this machine.
policies: Policies governing this machine.
"""
logging.info('Sending add_machine request')
return net.json_request(
'%s/_ah/api/catalog/v1/add_machine' %
MachineProviderConfiguration.get_instance_url(),
method='POST',
payload=utils.to_json_encodable({
'dimensions': dimensions,
'policies': policies,
}),
scopes=MACHINE_PROVIDER_SCOPES,
)
def tasks(self, request):
"""Lists all of a given bot's tasks."""
cursor = datastore_query.Cursor(urlsafe=request.cursor)
run_results, cursor, more = task_result.TaskRunResult.query(
task_result.TaskRunResult.bot_id == request.bot_id).order(
-task_result.TaskRunResult.started_ts).fetch_page(
request.limit, start_cursor=cursor)
result_list = [
message_conversion.task_run_result_from_dict(
utils.to_json_encodable(run_result))
for run_result in run_results
]
response = swarming_rpcs.BotTask(
cursor=cursor.urlsafe() if cursor and more else None,
items=result_list,
limit=request.limit,
now=utils.utcnow())
return response
def post(self):
request_data = self.parse_body()
# If the priority is below 100, make the the user has right to do so.
if request_data.get('priority', 255) < 100 and not acl.is_bot_or_admin():
# Silently drop the priority of normal users.
request_data['priority'] = 100
try:
request = task_request.make_request(request_data)
except (datastore_errors.BadValueError, TypeError, ValueError) as e:
self.abort_with_error(400, error=str(e))
result_summary = task_scheduler.schedule_request(request)
data = {
'request': request.to_dict(),
'task_id': task_pack.pack_result_summary_key(result_summary.key),
}
self.send_response(utils.to_json_encodable(data))
def _cmd_run(self, request, run_result_key, bot_id):
# Only one of 'command' or 'inputs_ref' can be set.
out = {
"cmd": "run",
"manifest": {
"bot_id": bot_id,
"command": request.properties.commands[0] if request.properties.commands else None,
"data": request.properties.data,
"dimensions": request.properties.dimensions,
"env": request.properties.env,
"extra_args": request.properties.extra_args,
"grace_period": request.properties.grace_period_secs,
"hard_timeout": request.properties.execution_timeout_secs,
"host": utils.get_versioned_hosturl(),
"io_timeout": request.properties.io_timeout_secs,
"inputs_ref": request.properties.inputs_ref,
"task_id": task_pack.pack_run_result_key(run_result_key),
},
}
self.send_response(utils.to_json_encodable(out))
def instruct_machine(request_id, swarming_server):
"""Instruct a leased machine to connect to a Swarming server.
Args:
request_id: Request ID for the fulfilled lease whose machine to send
the instruction to.
swarming_server: URL of the Swarming server to connect to.
"""
return net.json_request(
'%s/api/machine_provider/v1/instruct' %
MachineProviderConfiguration.get_instance_url(),
method='POST',
payload=utils.to_json_encodable({
'instruction': {
'swarming_server': swarming_server,
},
'request_id': request_id,
}),
scopes=MACHINE_PROVIDER_SCOPES,
)
def new(self, request):
"""Provides a TaskRequest and receives its metadata."""
request_dict = json.loads(remote.protojson.encode_message(request))
_transform_request(request_dict)
# If the priority is below 100, make the the user has right to do so.
if request_dict.get('priority', 255) < 100 and not acl.is_bot_or_admin():
# Silently drop the priority of normal users.
request_dict['priority'] = 100
try:
posted_request = task_request.make_request(request_dict)
except (datastore_errors.BadValueError, TypeError, ValueError) as e:
raise endpoints.BadRequestException(e.message)
result_summary = task_scheduler.schedule_request(posted_request)
posted_dict = utils.to_json_encodable(posted_request)
return swarming_rpcs.TaskRequestMetadata(
request=message_conversion.task_request_from_dict(posted_dict),
task_id=task_pack.pack_result_summary_key(result_summary.key))
def list(self, request):
"""Provides a list of available tasks."""
state = request.state.name.lower()
uses = sum([
request.name is not None, bool(request.tag), state != 'all'])
if uses > 1:
raise endpoints.BadRequestException(
'Only one of name, tag (1 or many) or state can be used.')
# get the tasks
items, cursor_str, sort, state = task_result.get_tasks(
request.name, request.tag, request.cursor, request.limit, request.sort,
state)
return swarming_rpcs.TaskList(
cursor=cursor_str,
items=[message_conversion.task_result_summary_from_dict(
utils.to_json_encodable(item)) for item in items],
limit=request.limit,
sort=sort,
state=state)
def new(self, request):
"""Provides a TaskRequest and receive its metadata."""
request_dict = json.loads(remote.protojson.encode_message(request))
_transform_request(request_dict)
# If the priority is below 100, make the the user has right to do so.
if request_dict.get('priority',
255) < 100 and not acl.is_bot_or_admin():
# Silently drop the priority of normal users.
request_dict['priority'] = 100
try:
posted_request = task_request.make_request(request_dict)
except (datastore_errors.BadValueError, TypeError, ValueError) as e:
raise endpoints.BadRequestException(e.message)
result_summary = task_scheduler.schedule_request(posted_request)
posted_dict = utils.to_json_encodable(posted_request)
return swarming_rpcs.TaskRequestMetadata(
request=message_conversion.task_request_from_dict(posted_dict),
task_id=task_pack.pack_result_summary_key(result_summary.key))
def _cmd_run(self, request, run_result_key, bot_id):
# Only one of 'command' or 'inputs_ref' can be set.
out = {
'cmd': 'run',
'manifest': {
'bot_id': bot_id,
'command':
request.properties.commands[0]
if request.properties.commands else None,
'data': request.properties.data,
'dimensions': request.properties.dimensions,
'env': request.properties.env,
'extra_args': request.properties.extra_args,
'grace_period': request.properties.grace_period_secs,
'hard_timeout': request.properties.execution_timeout_secs,
'host': utils.get_versioned_hosturl(),
'io_timeout': request.properties.io_timeout_secs,
'inputs_ref': request.properties.inputs_ref,
'task_id': task_pack.pack_run_result_key(run_result_key),
},
}
self.send_response(utils.to_json_encodable(out))
def _cmd_run(self, request, run_result_key, bot_id):
cmd = None
if request.properties.commands:
cmd = request.properties.commands[0]
elif request.properties.command:
cmd = request.properties.command
out = {
'cmd': 'run',
'manifest': {
'bot_id': bot_id,
'caches': [c.to_dict() for c in request.properties.caches],
'cipd_input': {
'client_package':
(request.properties.cipd_input.client_package.to_dict()),
'packages': [
p.to_dict()
for p in request.properties.cipd_input.packages
],
'server':
request.properties.cipd_input.server,
} if request.properties.cipd_input else None,
'command': cmd,
'dimensions': request.properties.dimensions,
'env': request.properties.env,
'extra_args': request.properties.extra_args,
'grace_period': request.properties.grace_period_secs,
'hard_timeout': request.properties.execution_timeout_secs,
'host': utils.get_versioned_hosturl(),
'io_timeout': request.properties.io_timeout_secs,
'isolated': {
'input': request.properties.inputs_ref.isolated,
'namespace': request.properties.inputs_ref.namespace,
'server': request.properties.inputs_ref.isolatedserver,
} if request.properties.inputs_ref else None,
'service_account': request.service_account,
'task_id': task_pack.pack_run_result_key(run_result_key),
},
}
self.send_response(utils.to_json_encodable(out))
def _cmd_run(self, request, run_result_key, bot_id):
cmd = None
if request.properties.commands:
cmd = request.properties.commands[0]
elif request.properties.command:
cmd = request.properties.command
out = {
'cmd': 'run',
'manifest': {
'bot_id':
bot_id,
'command':
cmd,
'dimensions':
request.properties.dimensions,
'env':
request.properties.env,
'extra_args':
request.properties.extra_args,
'grace_period':
request.properties.grace_period_secs,
'hard_timeout':
request.properties.execution_timeout_secs,
'host':
utils.get_versioned_hosturl(),
'io_timeout':
request.properties.io_timeout_secs,
'inputs_ref':
request.properties.inputs_ref,
'packages': [{
'package_name': p.package_name,
'version': p.version,
} for p in request.properties.packages],
'task_id':
task_pack.pack_run_result_key(run_result_key),
},
}
self.send_response(utils.to_json_encodable(out))
def get(self):
logging.error('Unexpected old client')
now = utils.utcnow()
limit = int(self.request.get('limit', 1000))
filter_by = self.request.get('filter')
if filter_by and filter_by not in self.ACCEPTABLE_FILTERS:
self.abort_with_error(400, error='Invalid filter query parameter')
q = bot_management.BotInfo.query()
if not filter_by:
q = q.order(bot_management.BotInfo.key)
recheck = lambda _: True
elif filter_by == 'quarantined':
q = q.order(bot_management.BotInfo.key)
q = q.filter(bot_management.BotInfo.quarantined == True)
recheck = lambda b: b.quarantined
elif filter_by == 'is_dead':
# The first sort key must be the same as used in the filter, otherwise
# datastore raises BadRequestError.
deadline = now - datetime.timedelta(
seconds=config.settings().bot_death_timeout_secs)
q = q.order(bot_management.BotInfo.last_seen_ts)
q = q.filter(bot_management.BotInfo.last_seen_ts < deadline)
recheck = lambda b: b.last_seen_ts < deadline
else:
raise AssertionError('Impossible')
cursor = datastore_query.Cursor(urlsafe=self.request.get('cursor'))
bots, cursor, more = q.fetch_page(limit, start_cursor=cursor)
data = {
'cursor': cursor.urlsafe() if cursor and more else None,
'death_timeout': config.settings().bot_death_timeout_secs,
'items': [b.to_dict_with_now(now) for b in bots if recheck(b)],
'limit': limit,
'now': now,
}
self.send_response(utils.to_json_encodable(data))
def get(self):
logging.error('Unexpected old client')
now = utils.utcnow()
limit = int(self.request.get('limit', 1000))
filter_by = self.request.get('filter')
if filter_by and filter_by not in self.ACCEPTABLE_FILTERS:
self.abort_with_error(400, error='Invalid filter query parameter')
q = bot_management.BotInfo.query()
if not filter_by:
q = q.order(bot_management.BotInfo.key)
recheck = lambda _: True
elif filter_by == 'quarantined':
q = q.order(bot_management.BotInfo.key)
q = q.filter(bot_management.BotInfo.quarantined == True)
recheck = lambda b: b.quarantined
elif filter_by == 'is_dead':
# The first sort key must be the same as used in the filter, otherwise
# datastore raises BadRequestError.
deadline = now - datetime.timedelta(
seconds=config.settings().bot_death_timeout_secs)
q = q.order(bot_management.BotInfo.last_seen_ts)
q = q.filter(bot_management.BotInfo.last_seen_ts < deadline)
recheck = lambda b: b.last_seen_ts < deadline
else:
raise AssertionError('Impossible')
cursor = datastore_query.Cursor(urlsafe=self.request.get('cursor'))
bots, cursor, more = q.fetch_page(limit, start_cursor=cursor)
data = {
'cursor': cursor.urlsafe() if cursor and more else None,
'death_timeout': config.settings().bot_death_timeout_secs,
'items': [b.to_dict_with_now(now) for b in bots if recheck(b)],
'limit': limit,
'now': now,
}
self.send_response(utils.to_json_encodable(data))
def list(self, request):
"""Provides a list of available tasks."""
state = request.state.name.lower()
uses = sum(
[request.name is not None,
bool(request.tag), state != 'all'])
if uses > 1:
raise endpoints.BadRequestException(
'Only one of name, tag (1 or many) or state can be used.')
# get the tasks
items, cursor_str, sort, state = task_result.get_tasks(
request.name, request.tag, request.cursor, request.limit,
request.sort, state)
return swarming_rpcs.TaskList(
cursor=cursor_str,
items=[
message_conversion.task_result_summary_from_dict(
utils.to_json_encodable(item)) for item in items
],
limit=request.limit,
sort=sort,
state=state)
def list(self, request):
"""Provides a list of available tasks."""
state = request.state.name.lower()
uses = sum([bool(request.tag), state != 'all'])
if state != 'all':
raise endpoints.BadRequestException(
'Querying by state is not yet supported. '
'Received argument state=%s.' % state)
if uses > 1:
raise endpoints.BadRequestException(
'Only one of tag (1 or many) or state can be used.')
# get the tasks
try:
start, end = _get_range(request)
items, cursor_str, state = task_result.get_result_summaries(
request.tag, request.cursor, start, end, state, request.batch_size)
return swarming_rpcs.TaskList(
cursor=cursor_str,
items=[message_conversion.task_result_summary_from_dict(
utils.to_json_encodable(item)) for item in items])
except ValueError as e:
raise endpoints.BadRequestException(
'Inappropriate batch_size for tasks/list: %s' % e)
def _cmd_run(self, request, secret_bytes, run_result, bot_id,
bot_group_cfg):
logging.info('Run: %s', request.task_id)
props = request.task_slice(run_result.current_task_slice).properties
out = {
'cmd': 'run',
'manifest': {
'bot_id':
bot_id,
'bot_authenticated_as':
auth.get_peer_identity().to_bytes(),
'caches': [c.to_dict() for c in props.caches],
'cipd_input': {
'client_package':
props.cipd_input.client_package.to_dict(),
'packages':
[p.to_dict() for p in props.cipd_input.packages],
'server': props.cipd_input.server,
} if props.cipd_input else None,
'command':
props.command,
'dimensions':
props.dimensions,
'env':
props.env,
'env_prefixes':
props.env_prefixes,
'extra_args':
props.extra_args,
'grace_period':
props.grace_period_secs,
'hard_timeout':
props.execution_timeout_secs,
'host':
utils.get_versioned_hosturl(),
'io_timeout':
props.io_timeout_secs,
'secret_bytes': (secret_bytes.secret_bytes.encode('base64')
if secret_bytes else None),
'isolated': {
'input': props.inputs_ref.isolated,
'namespace': props.inputs_ref.namespace,
'server': props.inputs_ref.isolatedserver,
} if props.inputs_ref else None,
'outputs':
props.outputs,
'relative_cwd':
props.relative_cwd,
'service_accounts': {
'system': {
# 'none', 'bot' or email. Bot interprets 'none' and 'bot' locally.
# When it sees something else, it uses /oauth_token API endpoint to
# grab tokens through server.
'service_account':
bot_group_cfg.system_service_account or 'none',
},
'task': {
# Same here.
'service_account': request.service_account,
},
},
'task_id':
task_pack.pack_run_result_key(run_result.key),
},
}
self.send_response(utils.to_json_encodable(out))
def get(self, task_id):
logging.error('Unexpected old client')
_, summary_key = self.get_result_key(task_id)
request_key = task_pack.result_summary_key_to_request_key(summary_key)
self.send_response(utils.to_json_encodable(request_key.get()))
def get(self, task_id):
logging.error('Unexpected old client')
result = self.get_result_entity(task_id)
self.send_response(utils.to_json_encodable(result))
def get(self, task_id): result = self.get_result_entity(task_id) self.send_response(utils.to_json_encodable(result))
def get(self):
logging.error('Unexpected old client')
data = {
'bot_version': bot_code.get_bot_version(self.request.host_url),
}
self.send_response(utils.to_json_encodable(data))
def get(self, task_id): _, summary_key = self.get_result_key(task_id) request_key = task_pack.result_summary_key_to_request_key(summary_key) self.send_response(utils.to_json_encodable(request_key.get()))
def get(self):
data = {
'bot_version': bot_code.get_bot_version(self.request.host_url),
}
self.send_response(utils.to_json_encodable(data))
def result(self, request):
"""Reports the result of the task corresponding to a task ID."""
entity = get_result_entity(request.task_id)
return message_conversion.task_result_summary_from_dict(
utils.to_json_encodable(entity))
def get(self):
data = {
'bot_version': bot_code.get_bot_version(self.request.host_url),
}
self.send_response(utils.to_json_encodable(data))
def result(self, request):
"""Reports the result of the task corresponding to a task ID."""
entity = get_result_entity(request.task_id)
return message_conversion.task_result_summary_from_dict(
utils.to_json_encodable(entity))
