def list(self, request):
"""Provides list of known bots.
Deleted bots will not be listed.
"""
logging.debug('%s', request)
now = utils.utcnow()
q = bot_management.BotInfo.query()
try:
q = bot_management.filter_dimensions(q, request.dimensions)
q = bot_management.filter_availability(
q, swarming_rpcs.to_bool(request.quarantined),
swarming_rpcs.to_bool(request.in_maintenance),
swarming_rpcs.to_bool(request.is_dead),
swarming_rpcs.to_bool(request.is_busy),
swarming_rpcs.to_bool(request.is_mp))
except ValueError as e:
raise endpoints.BadRequestException(str(e))
bots, cursor = datastore_utils.fetch_page(q, request.limit,
request.cursor)
return swarming_rpcs.BotList(
cursor=cursor,
death_timeout=config.settings().bot_death_timeout_secs,
items=[message_conversion.bot_info_to_rpc(bot) for bot in bots],
now=now)
def list(self, request):
"""Returns tasks results based on the filters.
This endpoint is significantly slower than 'count'. Use 'count' when
possible.
"""
# TODO(maruel): Rename 'list' to 'results'.
# TODO(maruel): Rename 'TaskList' to 'TaskResults'.
logging.info('%s', request)
now = utils.utcnow()
try:
items, cursor = datastore_utils.fetch_page(
self._query_from_request(request), request.limit, request.cursor)
except ValueError as e:
raise endpoints.BadRequestException(
'Inappropriate filter for tasks/list: %s' % e)
except datastore_errors.NeedIndexError as e:
logging.error('%s', e)
raise endpoints.BadRequestException(
'Requires new index, ask admin to create one.')
except datastore_errors.BadArgumentError as e:
logging.error('%s', e)
raise endpoints.BadRequestException(
'This combination is unsupported, sorry.')
return swarming_rpcs.TaskList(
cursor=cursor,
items=[
message_conversion.task_result_to_rpc(
i, request.include_performance_stats)
for i in items
],
now=now)
def list(self, request):
"""Provides a list of available tasks."""
logging.info('%s', request)
try:
start = message_conversion.epoch_to_datetime(request.start)
end = message_conversion.epoch_to_datetime(request.end)
now = utils.utcnow()
query = task_result.get_result_summaries_query(
start, end, request.sort.name.lower(),
request.state.name.lower(), request.tags)
items, cursor = datastore_utils.fetch_page(query, request.limit,
request.cursor)
except ValueError as e:
raise endpoints.BadRequestException(
'Inappropriate filter for tasks/list: %s' % e)
except datastore_errors.NeedIndexError as e:
logging.error('%s', e)
raise endpoints.BadRequestException(
'Requires new index, ask admin to create one.')
except datastore_errors.BadArgumentError as e:
logging.error('%s', e)
raise endpoints.BadRequestException(
'This combination is unsupported, sorry.')
return swarming_rpcs.TaskList(
cursor=cursor,
items=[message_conversion.task_result_to_rpc(i) for i in items],
now=now)
def get_tasks(limit, cursor_str, sort, state, tags, task_name):
"""Returns TaskResultSummary entities for this query.
This function is synchronous.
Arguments:
limit: Maximum number of items to return.
cursor_str: query-dependent string encoded cursor to continue a previous
search.
sort: Order to use. Must default to 'created_ts' to use the default.
state: State to filter on.
tags: List of search for one or multiple task tags.
task_name: search for task name whole word.
Returns:
tuple(list of tasks, str encoded cursor, updated sort, updated state)
"""
if task_name:
# Task name based word based search. Override the flags.
sort = "created_ts"
state = "all"
tasks, cursor_str = _search_by_name(task_name, cursor_str, limit)
else:
# Normal listing.
if tags:
# Add TaskResultSummary indexes if desired.
sort = "created_ts"
query = get_result_summaries_query(None, None, sort, state, tags)
tasks, cursor_str = datastore_utils.fetch_page(query, limit, cursor_str)
return tasks, cursor_str, sort, state
def tasks(self, request):
"""Lists a given bot's tasks within the specified date range.
In this case, the tasks are effectively TaskRunResult since it's individual
task tries sent to this specific bot.
It is impossible to search by both tags and bot id. If there's a need,
TaskRunResult.tags will be added (via a copy from TaskRequest.tags).
"""
logging.info('%s', request)
try:
start = message_conversion.epoch_to_datetime(request.start)
end = message_conversion.epoch_to_datetime(request.end)
now = utils.utcnow()
query = task_result.get_run_results_query(
start, end,
request.sort.name.lower(),
request.state.name.lower(),
request.bot_id)
items, cursor = datastore_utils.fetch_page(
query, request.limit, request.cursor)
except ValueError as e:
raise endpoints.BadRequestException(
'Inappropriate filter for bot.tasks: %s' % e)
return swarming_rpcs.BotTasks(
cursor=cursor,
items=[message_conversion.task_result_to_rpc(r) for r in items],
now=now)
def list(self, request):
"""Returns tasks results based on the filters.
This endpoint is significantly slower than 'count'. Use 'count' when
possible.
"""
# TODO(maruel): Rename 'list' to 'results'.
# TODO(maruel): Rename 'TaskList' to 'TaskResults'.
logging.debug('%s', request)
now = utils.utcnow()
try:
items, cursor = datastore_utils.fetch_page(
self._query_from_request(request), request.limit,
request.cursor)
except ValueError as e:
raise endpoints.BadRequestException(
'Inappropriate filter for tasks/list: %s' % e)
except datastore_errors.NeedIndexError as e:
logging.error('%s', e)
raise endpoints.BadRequestException(
'Requires new index, ask admin to create one.')
except datastore_errors.BadArgumentError as e:
logging.error('%s', e)
raise endpoints.BadRequestException(
'This combination is unsupported, sorry.')
return swarming_rpcs.TaskList(
cursor=cursor,
items=[
message_conversion.task_result_to_rpc(
i, request.include_performance_stats) for i in items
],
now=now)
def events(self, request):
"""Returns events that happened on a bot."""
logging.debug('%s', request)
try:
now = utils.utcnow()
start = message_conversion.epoch_to_datetime(request.start)
end = message_conversion.epoch_to_datetime(request.end)
order = not (start or end)
query = bot_management.get_events_query(request.bot_id, order)
if not order:
query = query.order(-bot_management.BotEvent.ts,
bot_management.BotEvent.key)
if start:
query = query.filter(bot_management.BotEvent.ts >= start)
if end:
query = query.filter(bot_management.BotEvent.ts < end)
items, cursor = datastore_utils.fetch_page(query, request.limit,
request.cursor)
except ValueError as e:
raise endpoints.BadRequestException(
'Inappropriate filter for bot.events: %s' % e)
return swarming_rpcs.BotEvents(
cursor=cursor,
items=[message_conversion.bot_event_to_rpc(r) for r in items],
now=now)
def get(self):
ndb.get_context().set_cache_policy(lambda _: False)
seen = {}
now = utils.utcnow()
count = 0
q = task_result.TaskResultSummary.query(
task_result.TaskResultSummary.modified_ts > now -
datetime.timedelta(hours=1))
cursor = None
while True:
tasks, cursor = datastore_utils.fetch_page(q, 1000, cursor)
count += len(tasks)
for t in tasks:
for i in t.tags:
k, v = i.split(':', 1)
s = seen.setdefault(k, set())
if s is not None:
s.add(v)
# 128 is arbitrary large number to avoid OOM
if len(s) >= 128:
logging.info(
'Limiting tag %s because there are too many',
k)
seen[k] = None
if not cursor or len(tasks) == 0:
break
tags = [
task_result.TagValues(tag=k, values=sorted(values or []))
for k, values in sorted(seen.iteritems())
]
logging.info('From %d tasks, saw %d tags', count, len(tags))
task_result.TagAggregation(key=task_result.TagAggregation.KEY,
tags=tags,
ts=now).put()
def list(self, request):
logging.debug('%s', request)
now = utils.utcnow()
# Disable the in-process local cache. This is important, as there can be up
# to a thousand entities loaded in memory, and this is a pure memory leak,
# as there's no chance this specific instance will need these again,
# therefore this leads to 'Exceeded soft memory limit' AppEngine errors.
q = task_queues.TaskDimensions.query(default_options=ndb.QueryOptions(
use_cache=False))
cursor = request.cursor
out = []
count = 0
# As there can be a lot of terminate tasks, try to loop a few times (max 5)
# to get more items.
while len(out) < request.limit and (cursor or not count) and count < 5:
items, cursor = datastore_utils.fetch_page(
q, request.limit - len(out), cursor)
for i in items:
for s in i.sets:
# Ignore the tasks that are only id specific, since they are
# termination tasks. There may be one per bot, and it is not really
# useful for the user, the user may just query the list of bots.
if (len(s.dimensions_flat) == 1
and s.dimensions_flat[0].startswith('id:')):
# A terminate task.
continue
out.append(
swarming_rpcs.TaskQueue(
dimensions=s.dimensions_flat,
valid_until_ts=s.valid_until_ts))
count += 1
return swarming_rpcs.TaskQueueList(cursor=cursor, items=out, now=now)
def list(self, request):
"""Provides list of known bots.
Deleted bots will not be listed.
"""
logging.debug('%s', request)
now = utils.utcnow()
# Disable the in-process local cache. This is important, as there can be up
# to a thousand entities loaded in memory, and this is a pure memory leak,
# as there's no chance this specific instance will need these again,
# therefore this leads to 'Exceeded soft memory limit' AppEngine errors.
q = bot_management.BotInfo.query(default_options=ndb.QueryOptions(
use_cache=False))
try:
q = bot_management.filter_dimensions(q, request.dimensions)
q = bot_management.filter_availability(
q, swarming_rpcs.to_bool(request.quarantined),
swarming_rpcs.to_bool(request.in_maintenance),
swarming_rpcs.to_bool(request.is_dead),
swarming_rpcs.to_bool(request.is_busy))
except ValueError as e:
raise endpoints.BadRequestException(str(e))
bots, cursor = datastore_utils.fetch_page(q, request.limit,
request.cursor)
return swarming_rpcs.BotList(
cursor=cursor,
death_timeout=config.settings().bot_death_timeout_secs,
items=[message_conversion.bot_info_to_rpc(bot) for bot in bots],
now=now)
def requests(self, request):
"""Returns tasks requests based on the filters.
This endpoint is slightly slower than 'list'. Use 'list' or 'count' when
possible.
"""
logging.info('%s', request)
if request.include_performance_stats:
raise endpoints.BadRequestException(
'Can\'t set include_performance_stats for tasks/list')
now = utils.utcnow()
try:
# Get the TaskResultSummary keys, then fetch the corresponding
# TaskRequest entities.
keys, cursor = datastore_utils.fetch_page(
self._query_from_request(request),
request.limit, request.cursor, keys_only=True)
items = ndb.get_multi(
task_pack.result_summary_key_to_request_key(k) for k in keys)
except ValueError as e:
raise endpoints.BadRequestException(
'Inappropriate filter for tasks/requests: %s' % e)
except datastore_errors.NeedIndexError as e:
logging.error('%s', e)
raise endpoints.BadRequestException(
'Requires new index, ask admin to create one.')
except datastore_errors.BadArgumentError as e:
logging.error('%s', e)
raise endpoints.BadRequestException(
'This combination is unsupported, sorry.')
return swarming_rpcs.TaskRequests(
cursor=cursor,
items=[message_conversion.task_request_to_rpc(i) for i in items],
now=now)
def tasks(self, request):
"""Lists a given bot's tasks within the specified date range.
In this case, the tasks are effectively TaskRunResult since it's individual
task tries sent to this specific bot.
It is impossible to search by both tags and bot id. If there's a need,
TaskRunResult.tags will be added (via a copy from TaskRequest.tags).
"""
logging.debug('%s', request)
try:
start = message_conversion.epoch_to_datetime(request.start)
end = message_conversion.epoch_to_datetime(request.end)
now = utils.utcnow()
query = task_result.get_run_results_query(
start, end, request.sort.name.lower(),
request.state.name.lower(), request.bot_id)
items, cursor = datastore_utils.fetch_page(query, request.limit,
request.cursor)
except ValueError as e:
raise endpoints.BadRequestException(
'Inappropriate filter for bot.tasks: %s' % e)
return swarming_rpcs.BotTasks(
cursor=cursor,
items=[
message_conversion.task_result_to_rpc(
r, request.include_performance_stats) for r in items
],
now=now)
def list(self, request):
logging.debug('%s', request)
now = utils.utcnow()
q = task_queues.TaskDimensions.query()
cursor = request.cursor
out = []
count = 0
# As there can be a lot of terminate tasks, try to loop a few times (max 5)
# to get more items.
while len(out) < request.limit and (cursor or not count) and count < 5:
items, cursor = datastore_utils.fetch_page(
q, request.limit - len(out), cursor)
for i in items:
for s in i.sets:
# Ignore the tasks that are only id specific, since they are
# termination tasks. This happens in particular with Machine Provider
# managed bots, since they are recycled on a continuous basis. There
# may be one per bot, and it is not really useful for the user, the
# user may just query the list of bots.
if (len(s.dimensions_flat) == 1
and s.dimensions_flat[0].startswith('id:')):
# A terminate task.
continue
out.append(
swarming_rpcs.TaskQueue(
dimensions=s.dimensions_flat,
valid_until_ts=s.valid_until_ts))
count += 1
return swarming_rpcs.TaskQueueList(cursor=cursor, items=out, now=now)
def events(self, request):
"""Returns events that happened on a bot."""
logging.info('%s', request)
try:
now = utils.utcnow()
start = message_conversion.epoch_to_datetime(request.start)
end = message_conversion.epoch_to_datetime(request.end)
order = not (start or end)
query = bot_management.get_events_query(request.bot_id, order)
if not order:
query = query.order(
-bot_management.BotEvent.ts, bot_management.BotEvent.key)
if start:
query = query.filter(bot_management.BotEvent.ts >= start)
if end:
query = query.filter(bot_management.BotEvent.ts < end)
items, cursor = datastore_utils.fetch_page(
query, request.limit, request.cursor)
except ValueError as e:
raise endpoints.BadRequestException(
'Inappropriate filter for bot.events: %s' % e)
return swarming_rpcs.BotEvents(
cursor=cursor,
items=[message_conversion.bot_event_to_rpc(r) for r in items],
now=now)
def list(self, request):
"""Provides a list of available tasks."""
logging.info('%s', request)
try:
start = message_conversion.epoch_to_datetime(request.start)
end = message_conversion.epoch_to_datetime(request.end)
now = utils.utcnow()
query = task_result.get_result_summaries_query(
start, end,
request.sort.name.lower(),
request.state.name.lower(),
request.tags)
items, cursor = datastore_utils.fetch_page(
query, request.limit, request.cursor)
except ValueError as e:
raise endpoints.BadRequestException(
'Inappropriate filter for tasks/list: %s' % e)
except datastore_errors.NeedIndexError as e:
logging.error('%s', e)
raise endpoints.BadRequestException(
'Requires new index, ask admin to create one.')
except datastore_errors.BadArgumentError as e:
logging.error('%s', e)
raise endpoints.BadRequestException(
'This combination is unsupported, sorry.')
return swarming_rpcs.TaskList(
cursor=cursor,
items=[message_conversion.task_result_to_rpc(i) for i in items],
now=now)
def get_tasks(limit, cursor_str, sort, state, tags, task_name):
"""Returns TaskResultSummary entities for this query.
This function is synchronous.
Arguments:
limit: Maximum number of items to return.
cursor_str: query-dependent string encoded cursor to continue a previous
search.
sort: Order to use. Must default to 'created_ts' to use the default.
state: State to filter on.
tags: List of search for one or multiple task tags.
task_name: search for task name whole word.
Returns:
tuple(list of tasks, str encoded cursor, updated sort, updated state)
"""
if task_name:
# Task name based word based search. Override the flags.
sort = 'created_ts'
state = 'all'
tasks, cursor_str = _search_by_name(task_name, cursor_str, limit)
else:
# Normal listing.
if tags:
# Add TaskResultSummary indexes if desired.
sort = 'created_ts'
query = get_result_summaries_query(None, None, sort, state, tags)
tasks, cursor_str = datastore_utils.fetch_page(query, limit,
cursor_str)
return tasks, cursor_str, sort, state
def Events(self, request, context):
logging.debug('%s', request)
try:
if not request.bot_id:
# TODO(maruel): Allows not specifying one. Or specifying a pool.
raise ValueError('specify bot_id')
# Transparently limit to 1000, default to 200.
page_size = request.page_size or 200
if page_size > 1000:
page_size = 1000
if page_size < 0:
raise ValueError('page_size must be positive')
start = None
end = None
if request.HasField('start_time'):
start = request.start_time.ToDatetime()
if request.HasField('end_time'):
end = request.end_time.ToDatetime()
if (start and end) and start >= end:
raise ValueError('start_time must be before end_time')
# The BotEvent key is already in the right chronological order, but
# querying per BotEvent.ts *requires* ordering per BotEvent.ts.
order = not (start or end)
q = bot_management.get_events_query(request.bot_id, order)
if not order:
q = q.order(-bot_management.BotEvent.ts,
bot_management.BotEvent.key)
if start:
q = q.filter(bot_management.BotEvent.ts >= start)
if end:
q = q.filter(bot_management.BotEvent.ts < end)
items, cursor = datastore_utils.fetch_page(q, page_size,
request.page_token)
if not items:
# Check if the bot exists, if not, return a 404. We check BotRoot, not
# BotInfo, so that even deleted bots can be queried. See bot_management
# for more information.
if not bot_management.get_root_key(request.bot_id).get():
context.set_code(StatusCode.NOT_FOUND)
context.set_details('Bot does not exist')
return None
except ValueError as e:
context.set_code(StatusCode.INVALID_ARGUMENT)
context.set_details(str(e))
return None
logging.info('Returning %d events', len(items))
out = swarming_pb2.BotEventsResponse(next_page_token=cursor)
for r in items:
i = out.events.add()
r.to_proto(i)
return out
def list(self, request):
"""Provides list of known bots.
Deleted bots will not be listed.
"""
logging.info('%s', request)
now = utils.utcnow()
q = bot_management.BotInfo.query().order(bot_management.BotInfo.key)
bots, cursor = datastore_utils.fetch_page(q, request.limit, request.cursor)
return swarming_rpcs.BotList(
cursor=cursor,
death_timeout=config.settings().bot_death_timeout_secs,
items=[message_conversion.bot_info_to_rpc(bot, now) for bot in bots],
now=now)
def list(self, request):
"""Provides list of known bots.
Deleted bots will not be listed.
"""
logging.info('%s', request)
now = utils.utcnow()
q = bot_management.BotInfo.query().order(bot_management.BotInfo.key)
bots, cursor = datastore_utils.fetch_page(q, request.limit,
request.cursor)
return swarming_rpcs.BotList(
cursor=cursor,
death_timeout=config.settings().bot_death_timeout_secs,
items=[
message_conversion.bot_info_to_rpc(bot, now) for bot in bots
],
now=now)
def cancel(self, request):
"""Cancel a subset of pending tasks based on the tags.
Cancellation happens asynchronously, so when this call returns,
cancellations will not have completed yet.
"""
logging.debug('%s', request)
if not request.tags:
# Prevent accidental cancellation of everything.
raise endpoints.BadRequestException(
'You must specify tags when cancelling multiple tasks.')
now = utils.utcnow()
cond = task_result.TaskResultSummary.state == task_result.State.PENDING
if request.kill_running:
cond = ndb.OR(
cond, task_result.TaskResultSummary.state ==
task_result.State.RUNNING)
q = task_result.TaskResultSummary.query(cond).order(
task_result.TaskResultSummary.key)
for tag in request.tags:
q = q.filter(task_result.TaskResultSummary.tags == tag)
tasks, cursor = datastore_utils.fetch_page(q, request.limit,
request.cursor)
if tasks:
payload = json.dumps({
'tasks': [t.task_id for t in tasks],
'kill_running': request.kill_running or False,
})
ok = utils.enqueue_task(
'/internal/taskqueue/important/tasks/cancel',
'cancel-tasks',
payload=payload)
if not ok:
raise endpoints.InternalServerErrorException(
'Could not enqueue cancel request, try again later')
else:
logging.info('No tasks to cancel.')
return swarming_rpcs.TasksCancelResponse(cursor=cursor,
matched=len(tasks),
now=now)
def get(self, lease_id=None):
params = {
'lease_requests': [],
'next_page_token': None,
'now_ts': utils.time_time(),
}
if lease_id:
lease_request = models.LeaseRequest.get_by_id(lease_id)
if not lease_request:
self.abort(404)
params['lease_requests'] = [lease_request]
else:
query = models.LeaseRequest.query().order(
-models.LeaseRequest.last_modified_ts)
page_token = self.request.get('page_token') or ''
params['lease_requests'], params['next_page_token'] = (
datastore_utils.fetch_page(query, 50, page_token))
self.response.write(template.render('templates/leases.html', params=params))
def get(self, machine_id=None):
params = {
'machines': [],
'next_page_token': None,
}
if machine_id:
machine = models.CatalogMachineEntry.get_by_id(machine_id)
if not machine:
self.abort(404)
params['machines'] = [machine]
else:
query = models.CatalogMachineEntry.query().order(
models.CatalogMachineEntry.dimensions.hostname)
page_token = self.request.get('page_token') or ''
params['machines'], params['next_page_token'] = (
datastore_utils.fetch_page(query, 50, page_token))
self.response.write(
template.render('templates/catalog.html', params=params))
def list(self, request):
"""Provides list of known bots.
Deleted bots will not be listed.
"""
logging.info('%s', request)
now = utils.utcnow()
q = bot_management.BotInfo.query().order(bot_management.BotInfo.key)
for d in request.dimensions:
if not ':' in d:
raise endpoints.BadRequestException('Invalid dimensions')
parts = d.split(':', 1)
if len(parts) != 2 or any(i.strip() != i or not i for i in parts):
raise endpoints.BadRequestException('Invalid dimensions')
q = q.filter(bot_management.BotInfo.dimensions_flat == d)
bots, cursor = datastore_utils.fetch_page(q, request.limit, request.cursor)
return swarming_rpcs.BotList(
cursor=cursor,
death_timeout=config.settings().bot_death_timeout_secs,
items=[message_conversion.bot_info_to_rpc(bot, now) for bot in bots],
now=now)
def list(self, request):
"""Provides list of known bots.
Deleted bots will not be listed.
"""
logging.info('%s', request)
now = utils.utcnow()
q = bot_management.BotInfo.query().order(bot_management.BotInfo.key)
for d in request.dimensions:
if not ':' in d:
raise endpoints.BadRequestException('Invalid dimensions')
parts = d.split(':', 1)
if len(parts) != 2 or any(i.strip() != i or not i for i in parts):
raise endpoints.BadRequestException('Invalid dimensions')
q = q.filter(bot_management.BotInfo.dimensions_flat == d)
bots, cursor = datastore_utils.fetch_page(q, request.limit,
request.cursor)
return swarming_rpcs.BotList(
cursor=cursor,
death_timeout=config.settings().bot_death_timeout_secs,
items=[
message_conversion.bot_info_to_rpc(bot, now) for bot in bots
],
now=now)
def requests(self, request):
"""Returns tasks requests based on the filters.
This endpoint is slightly slower than 'list'. Use 'list' or 'count' when
possible.
"""
logging.debug('%s', request)
if request.include_performance_stats:
raise endpoints.BadRequestException(
'Can\'t set include_performance_stats for tasks/list')
now = utils.utcnow()
try:
# Get the TaskResultSummary keys, then fetch the corresponding
# TaskRequest entities.
keys, cursor = datastore_utils.fetch_page(
self._query_from_request(request),
request.limit,
request.cursor,
keys_only=True)
items = ndb.get_multi(
task_pack.result_summary_key_to_request_key(k) for k in keys)
except ValueError as e:
raise endpoints.BadRequestException(
'Inappropriate filter for tasks/requests: %s' % e)
except datastore_errors.NeedIndexError as e:
logging.error('%s', e)
raise endpoints.BadRequestException(
'Requires new index, ask admin to create one.')
except datastore_errors.BadArgumentError as e:
logging.error('%s', e)
raise endpoints.BadRequestException(
'This combination is unsupported, sorry.')
return swarming_rpcs.TaskRequests(
cursor=cursor,
items=[message_conversion.task_request_to_rpc(i) for i in items],
now=now)
def get(self):
cursor_str = self.request.get('cursor')
limit = int(self.request.get('limit', 100))
sort = self.request.get('sort', self.SORT_CHOICES[0][0])
state = self.request.get('state', self.STATE_CHOICES[0][0][0])
counts = self.request.get('counts', '').strip()
task_tags = [
line for line in self.request.get('task_tag', '').splitlines() if line
]
if not any(sort == i[0] for i in self.SORT_CHOICES):
self.abort(400, 'Invalid sort')
if not any(any(state == i[0] for i in j) for j in self.STATE_CHOICES):
self.abort(400, 'Invalid state')
if sort != 'created_ts':
# Zap all filters in this case to reduce the number of required indexes.
# Revisit according to the user requests.
state = 'all'
now = utils.utcnow()
# "Temporarily" disable the count. This is too slow on the prod server
# (>10s). The fix is to have the web page do a XHR query to get the values
# asynchronously.
counts_future = None
if counts == 'true':
counts_future = self._get_counts_future(now)
try:
if task_tags:
# Enforce created_ts when tags are used.
sort = 'created_ts'
query = task_result.get_result_summaries_query(
None, None, sort, state, task_tags)
tasks, cursor_str = datastore_utils.fetch_page(query, limit, cursor_str)
# Prefetch the TaskRequest all at once, so that ndb's in-process cache has
# it instead of fetching them one at a time indirectly when using
# TaskResultSummary.request_key.get().
futures = ndb.get_multi_async(t.request_key for t in tasks)
# Evaluate the counts to print the filtering columns with the associated
# numbers.
state_choices = self._get_state_choices(counts_future)
except ValueError as e:
self.abort(400, str(e))
def safe_sum(items):
return sum(items, datetime.timedelta())
def avg(items):
if not items:
return 0.
return safe_sum(items) / len(items)
def median(items):
if not items:
return 0.
middle = len(items) / 2
if len(items) % 2:
return items[middle]
return (items[middle-1]+items[middle]) / 2
gen = (t.duration_now(now) for t in tasks)
durations = sorted(t for t in gen if t is not None)
gen = (t.pending_now(now) for t in tasks)
pendings = sorted(t for t in gen if t is not None)
total_cost_usd = sum(t.cost_usd for t in tasks)
total_cost_saved_usd = sum(
t.cost_saved_usd for t in tasks if t.cost_saved_usd)
# Include the overhead in the total amount of time saved, since it's
# overhead saved.
# In theory, t.duration_as_seen_by_server should always be set when
# t.deduped_from is set but there has some broken entities in the datastore.
total_saved = safe_sum(
t.duration_as_seen_by_server for t in tasks
if t.deduped_from and t.duration_as_seen_by_server)
duration_sum = safe_sum(durations)
total_saved_percent = (
(100. * total_saved.total_seconds() / duration_sum.total_seconds())
if duration_sum else 0.)
try_link = '/tasklist?l=%d' % limit
if task_tags:
try_link += '&f=' + '&f='.join(task_tags)
params = {
'cursor': cursor_str,
'duration_average': avg(durations),
'duration_median': median(durations),
'duration_sum': duration_sum,
'has_pending': any(t.is_pending for t in tasks),
'has_running': any(t.is_running for t in tasks),
'is_admin': acl.is_admin(),
'is_privileged_user': acl.is_privileged_user(),
'limit': limit,
'now': now,
'pending_average': avg(pendings),
'pending_median': median(pendings),
'pending_sum': safe_sum(pendings),
'show_footer': bool(pendings or durations),
'sort': sort,
'sort_choices': self.SORT_CHOICES,
'state': state,
'state_choices': state_choices,
'task_tag': '\n'.join(task_tags),
'tasks': tasks,
'total_cost_usd': total_cost_usd,
'total_cost_saved_usd': total_cost_saved_usd,
'total_saved': total_saved,
'total_saved_percent': total_saved_percent,
'try_link': try_link,
'xsrf_token': self.generate_xsrf_token(),
}
# TODO(maruel): If admin or if the user is task's .user, show the Cancel
# button. Do not show otherwise.
self.response.write(template.render('swarming/user_tasks.html', params))
# Do not let dangling futures linger around.
ndb.Future.wait_all(futures)
