def test_task_result_to_rpc_empty_cipd_pins_packages(self):
message_conversion.task_result_to_rpc(
task_result.TaskResultSummary(
key=ndb.Key('TaskResultSummary',
1,
parent=ndb.Key('TaskRequest', 0x7fffffffff447fde)),
cipd_pins=task_result.CipdPins()), False)
def list(self, request):
"""Provides a list of available tasks."""
logging.info('%s', request)
try:
start = message_conversion.epoch_to_datetime(request.start)
end = message_conversion.epoch_to_datetime(request.end)
now = utils.utcnow()
query = task_result.get_result_summaries_query(
start, end,
request.sort.name.lower(),
request.state.name.lower(),
request.tags)
items, cursor = datastore_utils.fetch_page(
query, request.limit, request.cursor)
except ValueError as e:
raise endpoints.BadRequestException(
'Inappropriate filter for tasks/list: %s' % e)
except datastore_errors.NeedIndexError as e:
logging.error('%s', e)
raise endpoints.BadRequestException(
'Requires new index, ask admin to create one.')
except datastore_errors.BadArgumentError as e:
logging.error('%s', e)
raise endpoints.BadRequestException(
'This combination is unsupported, sorry.')
return swarming_rpcs.TaskList(
cursor=cursor,
items=[message_conversion.task_result_to_rpc(i) for i in items],
now=now)
def new(self, request):
"""Creates a new task.
The task will be enqueued in the tasks list and will be executed at the
earliest opportunity by a bot that has at least the dimensions as described
in the task request.
"""
logging.info('%s', request)
try:
request = message_conversion.new_task_request_from_rpc(
request, utils.utcnow())
posted_request = task_request.make_request(request, acl.is_bot_or_admin())
except (datastore_errors.BadValueError, TypeError, ValueError) as e:
raise endpoints.BadRequestException(e.message)
result_summary = task_scheduler.schedule_request(posted_request)
previous_result = None
if result_summary.deduped_from:
previous_result = message_conversion.task_result_to_rpc(result_summary)
return swarming_rpcs.TaskRequestMetadata(
request=message_conversion.task_request_to_rpc(posted_request),
task_id=task_pack.pack_result_summary_key(result_summary.key),
task_result=previous_result)
def list(self, request):
"""Returns tasks results based on the filters.
This endpoint is significantly slower than 'count'. Use 'count' when
possible.
"""
# TODO(maruel): Rename 'list' to 'results'.
# TODO(maruel): Rename 'TaskList' to 'TaskResults'.
logging.info('%s', request)
now = utils.utcnow()
try:
items, cursor = datastore_utils.fetch_page(
self._query_from_request(request), request.limit, request.cursor)
except ValueError as e:
raise endpoints.BadRequestException(
'Inappropriate filter for tasks/list: %s' % e)
except datastore_errors.NeedIndexError as e:
logging.error('%s', e)
raise endpoints.BadRequestException(
'Requires new index, ask admin to create one.')
except datastore_errors.BadArgumentError as e:
logging.error('%s', e)
raise endpoints.BadRequestException(
'This combination is unsupported, sorry.')
return swarming_rpcs.TaskList(
cursor=cursor,
items=[
message_conversion.task_result_to_rpc(
i, request.include_performance_stats)
for i in items
],
now=now)
def list(self, request):
"""Provides a list of available tasks."""
logging.info('%s', request)
state = request.state.name.lower()
uses = sum([bool(request.tags), state != 'all'])
if state != 'all':
raise endpoints.BadRequestException(
'Querying by state is not yet supported. '
'Received argument state=%s.' % state)
if uses > 1:
raise endpoints.BadRequestException(
'Only one of tag (1 or many) or state can be used.')
# get the tasks
try:
start = message_conversion.epoch_to_datetime(request.start)
end = message_conversion.epoch_to_datetime(request.end)
items, cursor_str, state = task_result.get_result_summaries(
request.tags, request.cursor, start, end, state, request.limit)
return swarming_rpcs.TaskList(
cursor=cursor_str,
items=[message_conversion.task_result_to_rpc(i) for i in items])
except ValueError as e:
raise endpoints.BadRequestException(
'Inappropriate limit for tasks/list: %s' % e)
def list(self, request):
"""Provides a list of available tasks."""
logging.info('%s', request)
try:
start = message_conversion.epoch_to_datetime(request.start)
end = message_conversion.epoch_to_datetime(request.end)
now = utils.utcnow()
query = task_result.get_result_summaries_query(
start, end, request.sort.name.lower(),
request.state.name.lower(), request.tags)
items, cursor = datastore_utils.fetch_page(query, request.limit,
request.cursor)
except ValueError as e:
raise endpoints.BadRequestException(
'Inappropriate filter for tasks/list: %s' % e)
except datastore_errors.NeedIndexError as e:
logging.error('%s', e)
raise endpoints.BadRequestException(
'Requires new index, ask admin to create one.')
except datastore_errors.BadArgumentError as e:
logging.error('%s', e)
raise endpoints.BadRequestException(
'This combination is unsupported, sorry.')
return swarming_rpcs.TaskList(
cursor=cursor,
items=[message_conversion.task_result_to_rpc(i) for i in items],
now=now)
def new(self, request):
"""Creates a new task.
The task will be enqueued in the tasks list and will be executed at the
earliest opportunity by a bot that has at least the dimensions as described
in the task request.
"""
logging.info('%s', request)
try:
request = message_conversion.new_task_request_from_rpc(
request, utils.utcnow())
posted_request = task_request.make_request(request,
acl.is_bot_or_admin())
except (datastore_errors.BadValueError, TypeError, ValueError) as e:
raise endpoints.BadRequestException(e.message)
result_summary = task_scheduler.schedule_request(posted_request)
previous_result = None
if result_summary.deduped_from:
previous_result = message_conversion.task_result_to_rpc(
result_summary)
return swarming_rpcs.TaskRequestMetadata(
request=message_conversion.task_request_to_rpc(posted_request),
task_id=task_pack.pack_result_summary_key(result_summary.key),
task_result=previous_result)
def list(self, request):
"""Returns tasks results based on the filters.
This endpoint is significantly slower than 'count'. Use 'count' when
possible.
"""
# TODO(maruel): Rename 'list' to 'results'.
# TODO(maruel): Rename 'TaskList' to 'TaskResults'.
logging.debug('%s', request)
now = utils.utcnow()
try:
items, cursor = datastore_utils.fetch_page(
self._query_from_request(request), request.limit,
request.cursor)
except ValueError as e:
raise endpoints.BadRequestException(
'Inappropriate filter for tasks/list: %s' % e)
except datastore_errors.NeedIndexError as e:
logging.error('%s', e)
raise endpoints.BadRequestException(
'Requires new index, ask admin to create one.')
except datastore_errors.BadArgumentError as e:
logging.error('%s', e)
raise endpoints.BadRequestException(
'This combination is unsupported, sorry.')
return swarming_rpcs.TaskList(
cursor=cursor,
items=[
message_conversion.task_result_to_rpc(
i, request.include_performance_stats) for i in items
],
now=now)
def tasks(self, request):
"""Lists a given bot's tasks within the specified date range.
In this case, the tasks are effectively TaskRunResult since it's individual
task tries sent to this specific bot.
It is impossible to search by both tags and bot id. If there's a need,
TaskRunResult.tags will be added (via a copy from TaskRequest.tags).
"""
logging.info('%s', request)
try:
start = message_conversion.epoch_to_datetime(request.start)
end = message_conversion.epoch_to_datetime(request.end)
now = utils.utcnow()
query = task_result.get_run_results_query(
start, end,
request.sort.name.lower(),
request.state.name.lower(),
request.bot_id)
items, cursor = datastore_utils.fetch_page(
query, request.limit, request.cursor)
except ValueError as e:
raise endpoints.BadRequestException(
'Inappropriate filter for bot.tasks: %s' % e)
return swarming_rpcs.BotTasks(
cursor=cursor,
items=[message_conversion.task_result_to_rpc(r) for r in items],
now=now)
def tasks(self, request):
"""Lists a given bot's tasks within the specified date range.
In this case, the tasks are effectively TaskRunResult since it's individual
task tries sent to this specific bot.
It is impossible to search by both tags and bot id. If there's a need,
TaskRunResult.tags will be added (via a copy from TaskRequest.tags).
"""
logging.debug('%s', request)
try:
start = message_conversion.epoch_to_datetime(request.start)
end = message_conversion.epoch_to_datetime(request.end)
now = utils.utcnow()
query = task_result.get_run_results_query(
start, end, request.sort.name.lower(),
request.state.name.lower(), request.bot_id)
items, cursor = datastore_utils.fetch_page(query, request.limit,
request.cursor)
except ValueError as e:
raise endpoints.BadRequestException(
'Inappropriate filter for bot.tasks: %s' % e)
return swarming_rpcs.BotTasks(
cursor=cursor,
items=[
message_conversion.task_result_to_rpc(
r, request.include_performance_stats) for r in items
],
now=now)
def result(self, request):
"""Reports the result of the task corresponding to a task ID.
It can be a 'run' ID specifying a specific retry or a 'summary' ID hidding
the fact that a task may have been retried transparently, when a bot reports
BOT_DIED.
A summary ID ends with '0', a run ID ends with '1' or '2'.
"""
logging.info('%s', request)
return message_conversion.task_result_to_rpc(
get_result_entity(request.task_id))
def result(self, request):
"""Reports the result of the task corresponding to a task ID.
It can be a 'run' ID specifying a specific retry or a 'summary' ID hidding
the fact that a task may have been retried transparently, when a bot reports
BOT_DIED.
A summary ID ends with '0', a run ID ends with '1' or '2'.
"""
logging.info('%s', request)
return message_conversion.task_result_to_rpc(
get_result_entity(request.task_id))
def tasks(self, request):
"""Lists a given bot's tasks within the specified date range."""
logging.info('%s', request)
try:
start = message_conversion.epoch_to_datetime(request.start)
end = message_conversion.epoch_to_datetime(request.end)
run_results, cursor, more = task_result.get_run_results(
request.cursor, request.bot_id, start, end, request.limit)
except ValueError as e:
raise endpoints.BadRequestException(
'Inappropriate limit for bots/list: %s' % e)
return swarming_rpcs.BotTasks(
cursor=cursor.urlsafe() if cursor and more else None,
items=[message_conversion.task_result_to_rpc(r) for r in run_results],
now=utils.utcnow())
def result(self, request):
"""Reports the result of the task corresponding to a task ID.
It can be a 'run' ID specifying a specific retry or a 'summary' ID hidding
the fact that a task may have been retried transparently, when a bot reports
BOT_DIED.
A summary ID ends with '0', a run ID ends with '1' or '2'.
"""
logging.info('%s', request)
key, _, = get_result_key(request.task_id)
result = key.get()
if not result:
raise endpoints.NotFoundException('%s not found.' % key.id())
return message_conversion.task_result_to_rpc(
result, request.include_performance_stats)
def result(self, request):
"""Reports the result of the task corresponding to a task ID.
It can be a 'run' ID specifying a specific retry or a 'summary' ID hidding
the fact that a task may have been retried transparently, when a bot reports
BOT_DIED.
A summary ID ends with '0', a run ID ends with '1' or '2'.
"""
logging.info('%s', request)
key, _, = get_result_key(request.task_id)
result = key.get()
if not result:
raise endpoints.NotFoundException('%s not found.' % key.id())
return message_conversion.task_result_to_rpc(
result, request.include_performance_stats)
def result(self, request):
"""Reports the result of the task corresponding to a task ID.
It can be a 'run' ID specifying a specific retry or a 'summary' ID hidding
the fact that a task may have been retried transparently, when a bot reports
BOT_DIED.
A summary ID ends with '0', a run ID ends with '1' or '2'.
"""
logging.debug('%s', request)
# Workaround a bug in ndb where if a memcache set fails, a stale copy can be
# kept in memcache indefinitely. In the case of task result, if this happens
# on the very last store where the task is saved to NDB to be marked as
# completed, and that the DB store succeeds *but* the memcache update fails,
# this API will *always* return the stale version.
_, result = _get_request_and_result(request.task_id, _VIEW, False)
return message_conversion.task_result_to_rpc(
result, request.include_performance_stats)
def new(self, request):
"""Creates a new task.
The task will be enqueued in the tasks list and will be executed at the
earliest opportunity by a bot that has at least the dimensions as described
in the task request.
"""
sb = (request.properties.secret_bytes
if request.properties is not None else None)
if sb is not None:
request.properties.secret_bytes = "HIDDEN"
logging.debug('%s', request)
if sb is not None:
request.properties.secret_bytes = sb
try:
request_obj, secret_bytes = message_conversion.new_task_request_from_rpc(
request, utils.utcnow())
for index in xrange(request_obj.num_task_slices):
apply_server_property_defaults(
request_obj.task_slice(index).properties)
task_request.init_new_request(
request_obj, acl.can_schedule_high_priority_tasks())
# We need to call the ndb.Model pre-put check earlier because the
# following checks assume that the request itself is valid and could crash
# otherwise.
request_obj._pre_put_hook()
except (datastore_errors.BadValueError, TypeError, ValueError) as e:
logging.exception(
'Here\'s what was wrong in the user new task request:')
raise endpoints.BadRequestException(e.message)
# Make sure the caller is actually allowed to schedule the task before
# asking the token server for a service account token.
task_scheduler.check_schedule_request_acl(request_obj)
# If request_obj.service_account is an email, contact the token server to
# generate "OAuth token grant" (or grab a cached one). By doing this we
# check that the given service account usage is allowed by the token server
# rules at the time the task is posted. This check is also performed later
# (when running the task), when we get the actual OAuth access token.
if service_accounts.is_service_account(request_obj.service_account):
if not service_accounts.has_token_server():
raise endpoints.BadRequestException(
'This Swarming server doesn\'t support task service accounts '
'because Token Server URL is not configured')
max_lifetime_secs = request_obj.max_lifetime_secs
try:
# Note: this raises AuthorizationError if the user is not allowed to use
# the requested account or service_accounts.InternalError if something
# unexpected happens.
duration = datetime.timedelta(seconds=max_lifetime_secs)
request_obj.service_account_token = (
service_accounts.get_oauth_token_grant(
service_account=request_obj.service_account,
validity_duration=duration))
except service_accounts.InternalError as exc:
raise endpoints.InternalServerErrorException(exc.message)
# If the user only wanted to evaluate scheduling the task, but not actually
# schedule it, return early without a task_id.
if request.evaluate_only:
request_obj._pre_put_hook()
return swarming_rpcs.TaskRequestMetadata(
request=message_conversion.task_request_to_rpc(request_obj))
try:
result_summary = task_scheduler.schedule_request(
request_obj, secret_bytes)
except (datastore_errors.BadValueError, TypeError, ValueError) as e:
raise endpoints.BadRequestException(e.message)
previous_result = None
if result_summary.deduped_from:
previous_result = message_conversion.task_result_to_rpc(
result_summary, False)
return swarming_rpcs.TaskRequestMetadata(
request=message_conversion.task_request_to_rpc(request_obj),
task_id=task_pack.pack_result_summary_key(result_summary.key),
task_result=previous_result)