def get(self):
params = {
"host_url": self.request.host_url,
"is_admin": acl.is_admin(),
"is_bot": acl.is_bot(),
"is_privileged_user": acl.is_privileged_user(),
"is_user": acl.is_user(),
"mapreduce_jobs": [],
"user_type": acl.get_user_type(),
}
if acl.is_admin():
params["mapreduce_jobs"] = [
{"id": job_id, "name": job_def["name"]} for job_id, job_def in mapreduce_jobs.MAPREDUCE_JOBS.iteritems()
]
params["xsrf_token"] = self.generate_xsrf_token()
self.response.write(template.render("swarming/root.html", params))
def _check_dimension_acls(request):
"""Raises AuthorizationError if some requested dimensions are forbidden.
Uses 'dimension_acls' field from the settings. See proto/config.proto.
"""
dim_acls = config.settings().dimension_acls
if not dim_acls or not dim_acls.entry:
return # not configured, this is fine
ident = request.authenticated
dims = request.properties.dimensions
assert 'id' in dims or 'pool' in dims, dims # see _validate_dimensions
assert ident is not None # see task_request.init_new_request
# Forbid targeting individual bots for non-admins, but allow using 'id' if
# 'pool' is used as well (so whoever can posts tasks to 'pool', can target an
# individual bot in that pool).
if 'id' in dims and 'pool' not in dims:
if not acl.is_admin():
raise auth.AuthorizationError(
'Only Swarming administrators can post tasks with "id" dimension '
'without specifying a "pool" dimension.')
for k, v in sorted(dims.iteritems()):
if not _can_use_dimension(dim_acls, ident, k, v):
raise auth.AuthorizationError(
'User %s is not allowed to schedule tasks with dimension "%s:%s"'
% (ident.to_bytes(), k, v))
def get(self):
limit = int(self.request.get('limit', 100))
cursor = datastore_query.Cursor(urlsafe=self.request.get('cursor'))
sort_by = self.request.get('sort_by', '__key__')
if sort_by not in self.ACCEPTABLE_BOTS_SORTS:
self.abort(400, 'Invalid sort_by query parameter')
if sort_by[0] == '-':
order = datastore_query.PropertyOrder(
sort_by[1:], datastore_query.PropertyOrder.DESCENDING)
else:
order = datastore_query.PropertyOrder(
sort_by, datastore_query.PropertyOrder.ASCENDING)
now = utils.utcnow()
cutoff = now - datetime.timedelta(
seconds=config.settings().bot_death_timeout_secs)
num_bots_busy_future = bot_management.BotInfo.query(
bot_management.BotInfo.is_busy == True).count_async()
num_bots_dead_future = bot_management.BotInfo.query(
bot_management.BotInfo.last_seen_ts < cutoff).count_async()
num_bots_quarantined_future = bot_management.BotInfo.query(
bot_management.BotInfo.quarantined == True).count_async()
num_bots_total_future = bot_management.BotInfo.query().count_async()
fetch_future = bot_management.BotInfo.query().order(order).fetch_page_async(
limit, start_cursor=cursor)
# TODO(maruel): self.request.host_url should be the default AppEngine url
# version and not the current one. It is only an issue when
# version-dot-appid.appspot.com urls are used to access this page.
version = bot_code.get_bot_version(self.request.host_url)
bots, cursor, more = fetch_future.get_result()
# Prefetch the tasks. We don't actually use the value here, it'll be
# implicitly used by ndb local's cache when refetched by the html template.
tasks = filter(None, (b.task for b in bots))
ndb.get_multi(tasks)
num_bots_busy = num_bots_busy_future.get_result()
num_bots_dead = num_bots_dead_future.get_result()
num_bots_quarantined = num_bots_quarantined_future.get_result()
num_bots_total = num_bots_total_future.get_result()
params = {
'bots': bots,
'current_version': version,
'cursor': cursor.urlsafe() if cursor and more else '',
'is_admin': acl.is_admin(),
'is_privileged_user': acl.is_privileged_user(),
'limit': limit,
'now': now,
'num_bots_alive': num_bots_total - num_bots_dead,
'num_bots_busy': num_bots_busy,
'num_bots_dead': num_bots_dead,
'num_bots_quarantined': num_bots_quarantined,
'sort_by': sort_by,
'sort_options': self.SORT_OPTIONS,
'xsrf_token': self.generate_xsrf_token(),
}
self.response.write(
template.render('swarming/restricted_botslist.html', params))
def get(self):
limit = int(self.request.get('limit', 100))
cursor = datastore_query.Cursor(urlsafe=self.request.get('cursor'))
sort_by = self.request.get('sort_by', '__key__')
if sort_by not in self.ACCEPTABLE_BOTS_SORTS:
self.abort(400, 'Invalid sort_by query parameter')
if sort_by[0] == '-':
order = datastore_query.PropertyOrder(
sort_by[1:], datastore_query.PropertyOrder.DESCENDING)
else:
order = datastore_query.PropertyOrder(
sort_by, datastore_query.PropertyOrder.ASCENDING)
now = utils.utcnow()
cutoff = now - datetime.timedelta(
seconds=config.settings().bot_death_timeout_secs)
num_bots_busy_future = bot_management.BotInfo.query(
bot_management.BotInfo.is_busy == True).count_async()
num_bots_dead_future = bot_management.BotInfo.query(
bot_management.BotInfo.last_seen_ts < cutoff).count_async()
num_bots_quarantined_future = bot_management.BotInfo.query(
bot_management.BotInfo.quarantined == True).count_async()
num_bots_total_future = bot_management.BotInfo.query().count_async()
fetch_future = bot_management.BotInfo.query().order(
order).fetch_page_async(limit, start_cursor=cursor)
# TODO(maruel): self.request.host_url should be the default AppEngine url
# version and not the current one. It is only an issue when
# version-dot-appid.appspot.com urls are used to access this page.
version = bot_code.get_bot_version(self.request.host_url)
bots, cursor, more = fetch_future.get_result()
# Prefetch the tasks. We don't actually use the value here, it'll be
# implicitly used by ndb local's cache when refetched by the html template.
tasks = filter(None, (b.task for b in bots))
ndb.get_multi(tasks)
num_bots_busy = num_bots_busy_future.get_result()
num_bots_dead = num_bots_dead_future.get_result()
num_bots_quarantined = num_bots_quarantined_future.get_result()
num_bots_total = num_bots_total_future.get_result()
params = {
'bots': bots,
'current_version': version,
'cursor': cursor.urlsafe() if cursor and more else '',
'is_admin': acl.is_admin(),
'is_privileged_user': acl.is_privileged_user(),
'limit': limit,
'now': now,
'num_bots_alive': num_bots_total - num_bots_dead,
'num_bots_busy': num_bots_busy,
'num_bots_dead': num_bots_dead,
'num_bots_quarantined': num_bots_quarantined,
'sort_by': sort_by,
'sort_options': self.SORT_OPTIONS,
'xsrf_token': self.generate_xsrf_token(),
}
self.response.write(
template.render('swarming/restricted_botslist.html', params))
def get(self):
params = {
'host_url': self.request.host_url,
'is_admin': acl.is_admin(),
'is_bot': acl.is_bot(),
'is_privileged_user': acl.is_privileged_user(),
'is_user': acl.is_user(),
'mapreduce_jobs': [],
'user_type': acl.get_user_type(),
}
if acl.is_admin():
params['mapreduce_jobs'] = [
{'id': job_id, 'name': job_def['name']}
for job_id, job_def in mapreduce_jobs.MAPREDUCE_JOBS.iteritems()
]
params['xsrf_token'] = self.generate_xsrf_token()
self.response.write(template.render('swarming/root.html', params))
def get(self):
params = {
'host_url': self.request.host_url,
'is_admin': acl.is_admin(),
'is_bot': acl.is_bot(),
'is_privileged_user': acl.is_privileged_user(),
'is_user': acl.is_user(),
'mapreduce_jobs': [],
'user_type': acl.get_user_type(),
}
if acl.is_admin():
params['mapreduce_jobs'] = [
{'id': job_id, 'name': job_def['job_name']}
for job_id, job_def in mapreduce_jobs.MAPREDUCE_JOBS.iteritems()
]
params['xsrf_token'] = self.generate_xsrf_token()
self.response.write(template.render('swarming/root.html', params))
def post(self, task_id):
original_request, _ = self.get_request_and_result(task_id)
# Retrying a task is essentially reusing the same task request as the
# original one, but with new parameters.
new_request = task_request.new_request_clone(
original_request, allow_high_priority=acl.is_admin())
result_summary = task_scheduler.schedule_request(new_request)
self.redirect('/user/task/%s' % result_summary.task_id)
def get(self, bot_id):
# pagination is currently for tasks, not events.
limit = int(self.request.get('limit', 100))
cursor = datastore_query.Cursor(urlsafe=self.request.get('cursor'))
bot_future = bot_management.get_info_key(bot_id).get_async()
run_results, cursor, more = task_result.TaskRunResult.query(
task_result.TaskRunResult.bot_id == bot_id).order(
-task_result.TaskRunResult.started_ts).fetch_page(
limit, start_cursor=cursor)
events_future = bot_management.get_events_query(bot_id).fetch_async(
100)
now = utils.utcnow()
bot = bot_future.get_result()
# Calculate the time this bot was idle.
idle_time = datetime.timedelta()
run_time = datetime.timedelta()
if run_results:
run_time = run_results[0].duration_now(now) or datetime.timedelta()
if not cursor and run_results[0].state != task_result.State.RUNNING:
# Add idle time since last task completed. Do not do this when a cursor
# is used since it's not representative.
idle_time = now - run_results[0].ended_ts
for index in xrange(1, len(run_results)):
# .started_ts will always be set by definition but .ended_ts may be None
# if the task was abandoned. We can't count idle time since the bot may
# have been busy running *another task*.
# TODO(maruel): One option is to add a third value "broken_time".
# Looking at timestamps specifically could help too, e.g. comparing
# ended_ts of this task vs the next one to see if the bot was assigned
# two tasks simultaneously.
if run_results[index].ended_ts:
idle_time += (run_results[index - 1].started_ts -
run_results[index].ended_ts)
duration = run_results[index].duration
if duration:
run_time += duration
params = {
'bot': bot,
'bot_id': bot_id,
'current_version': bot_code.get_bot_version(self.request.host_url),
'cursor': cursor.urlsafe() if cursor and more else None,
'events': events_future.get_result(),
'idle_time': idle_time,
'is_admin': acl.is_admin(),
'limit': limit,
'now': now,
'run_results': run_results,
'run_time': run_time,
'xsrf_token': self.generate_xsrf_token(),
}
self.response.write(
template.render('swarming/restricted_bot.html', params))
def get(self, bot_id):
# pagination is currently for tasks, not events.
limit = int(self.request.get('limit', 100))
cursor = datastore_query.Cursor(urlsafe=self.request.get('cursor'))
bot_future = bot_management.get_info_key(bot_id).get_async()
run_results, cursor, more = task_result.TaskRunResult.query(
task_result.TaskRunResult.bot_id == bot_id).order(
-task_result.TaskRunResult.started_ts).fetch_page(
limit, start_cursor=cursor)
events_future = bot_management.get_events_query(bot_id).fetch_async(100)
now = utils.utcnow()
bot = bot_future.get_result()
# Calculate the time this bot was idle.
idle_time = datetime.timedelta()
run_time = datetime.timedelta()
if run_results:
run_time = run_results[0].duration_now(now) or datetime.timedelta()
if not cursor and run_results[0].state != task_result.State.RUNNING:
# Add idle time since last task completed. Do not do this when a cursor
# is used since it's not representative.
idle_time = now - run_results[0].ended_ts
for index in xrange(1, len(run_results)):
# .started_ts will always be set by definition but .ended_ts may be None
# if the task was abandoned. We can't count idle time since the bot may
# have been busy running *another task*.
# TODO(maruel): One option is to add a third value "broken_time".
# Looking at timestamps specifically could help too, e.g. comparing
# ended_ts of this task vs the next one to see if the bot was assigned
# two tasks simultaneously.
if run_results[index].ended_ts:
idle_time += (
run_results[index-1].started_ts - run_results[index].ended_ts)
duration = run_results[index].duration
if duration:
run_time += duration
params = {
'bot': bot,
'bot_id': bot_id,
'current_version': bot_code.get_bot_version(self.request.host_url),
'cursor': cursor.urlsafe() if cursor and more else None,
'events': events_future.get_result(),
'idle_time': idle_time,
'is_admin': acl.is_admin(),
'limit': limit,
'now': now,
'run_results': run_results,
'run_time': run_time,
'xsrf_token': self.generate_xsrf_token(),
}
self.response.write(
template.render('swarming/restricted_bot.html', params))
def get(self, bot_id):
# pagination is currently for tasks, not events.
limit = int(self.request.get('limit', 100))
cursor = datastore_query.Cursor(urlsafe=self.request.get('cursor'))
run_results_future = task_result.TaskRunResult.query(
task_result.TaskRunResult.bot_id == bot_id).order(
-task_result.TaskRunResult.started_ts).fetch_page_async(
limit, start_cursor=cursor)
bot_future = bot_management.get_info_key(bot_id).get_async()
events_future = bot_management.get_events_query(
bot_id, True).fetch_async(100)
now = utils.utcnow()
# Calculate the time this bot was idle.
idle_time = datetime.timedelta()
run_time = datetime.timedelta()
run_results, cursor, more = run_results_future.get_result()
if run_results:
run_time = run_results[0].duration_now(now) or datetime.timedelta()
if not cursor and run_results[0].state != task_result.State.RUNNING:
# Add idle time since last task completed. Do not do this when a cursor
# is used since it's not representative.
idle_time = now - run_results[0].ended_ts
for index in xrange(1, len(run_results)):
# .started_ts will always be set by definition but .ended_ts may be None
# if the task was abandoned. We can't count idle time since the bot may
# have been busy running *another task*.
# TODO(maruel): One option is to add a third value "broken_time".
# Looking at timestamps specifically could help too, e.g. comparing
# ended_ts of this task vs the next one to see if the bot was assigned
# two tasks simultaneously.
if run_results[index].ended_ts:
idle_time += (
run_results[index-1].started_ts - run_results[index].ended_ts)
# We are taking the whole time the bot was doing work, not just the
# duration associated with the task.
duration = run_results[index].duration_total
if duration:
run_time += duration
events = events_future.get_result()
bot = bot_future.get_result()
if not bot and events:
# If there is not BotInfo, look if there are BotEvent child of this
# entity. If this is the case, it means the bot was deleted but it's
# useful to show information about it to the user even if the bot was
# deleted. For example, it could be an auto-scaled bot.
bot = bot_management.BotInfo(
key=bot_management.get_info_key(bot_id),
dimensions=events[0].dimensions,
state=events[0].state,
external_ip=events[0].external_ip,
version=events[0].version,
quarantined=events[0].quarantined,
task_id=events[0].task_id,
last_seen_ts=events[0].ts)
params = {
'bot': bot,
'bot_id': bot_id,
'current_version': bot_code.get_bot_version(self.request.host_url),
'cursor': cursor.urlsafe() if cursor and more else None,
'events': events,
'idle_time': idle_time,
'is_admin': acl.is_admin(),
'limit': limit,
'now': now,
'run_results': run_results,
'run_time': run_time,
'xsrf_token': self.generate_xsrf_token(),
}
self.response.write(
template.render('swarming/restricted_bot.html', params))
def get(self, task_id):
request, result = self.get_request_and_result(task_id)
parent_task_future = None
if request.parent_task_id:
parent_key = task_pack.unpack_run_result_key(request.parent_task_id)
parent_task_future = parent_key.get_async()
children_tasks_futures = [
task_pack.unpack_result_summary_key(c).get_async()
for c in result.children_task_ids
]
bot_id = result.bot_id
following_task_future = None
previous_task_future = None
if result.started_ts:
# Use a shortcut name because it becomes unwieldy otherwise.
cls = task_result.TaskRunResult
# Note that the links will be to the TaskRunResult, not to
# TaskResultSummary.
following_task_future = cls.query(
cls.bot_id == bot_id,
cls.started_ts > result.started_ts,
).order(cls.started_ts).get_async()
previous_task_future = cls.query(
cls.bot_id == bot_id,
cls.started_ts < result.started_ts,
).order(-cls.started_ts).get_async()
bot_future = (
bot_management.get_info_key(bot_id).get_async() if bot_id else None)
following_task = None
if following_task_future:
following_task = following_task_future.get_result()
previous_task = None
if previous_task_future:
previous_task = previous_task_future.get_result()
parent_task = None
if parent_task_future:
parent_task = parent_task_future.get_result()
children_tasks = [c.get_result() for c in children_tasks_futures]
cipd = None
if request.properties.cipd_input:
cipd = {
'server': request.properties.cipd_input.server,
'client_package': request.properties.cipd_input.client_package,
'packages': self.packages_grouped_by_path(
request.properties.cipd_input.packages),
}
cipd_pins = None
if result.cipd_pins:
cipd_pins = {
'client_package': result.cipd_pins.client_package,
'packages': self.packages_grouped_by_path(result.cipd_pins.packages),
}
params = {
'bot': bot_future.get_result() if bot_future else None,
'children_tasks': children_tasks,
'cipd': cipd,
'cipd_pins': cipd_pins,
'is_admin': acl.is_admin(),
'is_gae_admin': users.is_current_user_admin(),
'is_privileged_user': acl.is_privileged_user(),
'following_task': following_task,
'full_appid': os.environ['APPLICATION_ID'],
'host_url': self.request.host_url,
'is_running': result.state == task_result.State.RUNNING,
'parent_task': parent_task,
'previous_task': previous_task,
'request': request,
'task': result,
'try_link': '/task?id=%s' % task_id,
'xsrf_token': self.generate_xsrf_token(),
}
self.response.write(template.render('swarming/user_task.html', params))
def get(self):
cursor_str = self.request.get('cursor')
limit = int(self.request.get('limit', 100))
sort = self.request.get('sort', self.SORT_CHOICES[0][0])
state = self.request.get('state', self.STATE_CHOICES[0][0][0])
counts = self.request.get('counts', '').strip()
task_tags = [
line for line in self.request.get('task_tag', '').splitlines() if line
]
if not any(sort == i[0] for i in self.SORT_CHOICES):
self.abort(400, 'Invalid sort')
if not any(any(state == i[0] for i in j) for j in self.STATE_CHOICES):
self.abort(400, 'Invalid state')
if sort != 'created_ts':
# Zap all filters in this case to reduce the number of required indexes.
# Revisit according to the user requests.
state = 'all'
now = utils.utcnow()
# "Temporarily" disable the count. This is too slow on the prod server
# (>10s). The fix is to have the web page do a XHR query to get the values
# asynchronously.
counts_future = None
if counts == 'true':
counts_future = self._get_counts_future(now)
try:
if task_tags:
# Enforce created_ts when tags are used.
sort = 'created_ts'
query = task_result.get_result_summaries_query(
None, None, sort, state, task_tags)
tasks, cursor_str = datastore_utils.fetch_page(query, limit, cursor_str)
# Prefetch the TaskRequest all at once, so that ndb's in-process cache has
# it instead of fetching them one at a time indirectly when using
# TaskResultSummary.request_key.get().
futures = ndb.get_multi_async(t.request_key for t in tasks)
# Evaluate the counts to print the filtering columns with the associated
# numbers.
state_choices = self._get_state_choices(counts_future)
except ValueError as e:
self.abort(400, str(e))
def safe_sum(items):
return sum(items, datetime.timedelta())
def avg(items):
if not items:
return 0.
return safe_sum(items) / len(items)
def median(items):
if not items:
return 0.
middle = len(items) / 2
if len(items) % 2:
return items[middle]
return (items[middle-1]+items[middle]) / 2
gen = (t.duration_now(now) for t in tasks)
durations = sorted(t for t in gen if t is not None)
gen = (t.pending_now(now) for t in tasks)
pendings = sorted(t for t in gen if t is not None)
total_cost_usd = sum(t.cost_usd for t in tasks)
total_cost_saved_usd = sum(
t.cost_saved_usd for t in tasks if t.cost_saved_usd)
# Include the overhead in the total amount of time saved, since it's
# overhead saved.
# In theory, t.duration_as_seen_by_server should always be set when
# t.deduped_from is set but there has some broken entities in the datastore.
total_saved = safe_sum(
t.duration_as_seen_by_server for t in tasks
if t.deduped_from and t.duration_as_seen_by_server)
duration_sum = safe_sum(durations)
total_saved_percent = (
(100. * total_saved.total_seconds() / duration_sum.total_seconds())
if duration_sum else 0.)
try_link = '/tasklist?l=%d' % limit
if task_tags:
try_link += '&f=' + '&f='.join(task_tags)
params = {
'cursor': cursor_str,
'duration_average': avg(durations),
'duration_median': median(durations),
'duration_sum': duration_sum,
'has_pending': any(t.is_pending for t in tasks),
'has_running': any(t.is_running for t in tasks),
'is_admin': acl.is_admin(),
'is_privileged_user': acl.is_privileged_user(),
'limit': limit,
'now': now,
'pending_average': avg(pendings),
'pending_median': median(pendings),
'pending_sum': safe_sum(pendings),
'show_footer': bool(pendings or durations),
'sort': sort,
'sort_choices': self.SORT_CHOICES,
'state': state,
'state_choices': state_choices,
'task_tag': '\n'.join(task_tags),
'tasks': tasks,
'total_cost_usd': total_cost_usd,
'total_cost_saved_usd': total_cost_saved_usd,
'total_saved': total_saved,
'total_saved_percent': total_saved_percent,
'try_link': try_link,
'xsrf_token': self.generate_xsrf_token(),
}
# TODO(maruel): If admin or if the user is task's .user, show the Cancel
# button. Do not show otherwise.
self.response.write(template.render('swarming/user_tasks.html', params))
# Do not let dangling futures linger around.
ndb.Future.wait_all(futures)
def get(self, bot_id):
# pagination is currently for tasks, not events.
limit = int(self.request.get('limit', 100))
cursor = datastore_query.Cursor(urlsafe=self.request.get('cursor'))
run_results_future = task_result.TaskRunResult.query(
task_result.TaskRunResult.bot_id == bot_id).order(
-task_result.TaskRunResult.started_ts).fetch_page_async(
limit, start_cursor=cursor)
bot_future = bot_management.get_info_key(bot_id).get_async()
events_future = bot_management.get_events_query(
bot_id, True).fetch_async(100)
now = utils.utcnow()
# Calculate the time this bot was idle.
idle_time = datetime.timedelta()
run_time = datetime.timedelta()
run_results, cursor, more = run_results_future.get_result()
if run_results:
run_time = run_results[0].duration_now(now) or datetime.timedelta()
if not cursor and run_results[0].state != task_result.State.RUNNING:
# Add idle time since last task completed. Do not do this when a cursor
# is used since it's not representative.
idle_time = now - run_results[0].ended_ts
for index in xrange(1, len(run_results)):
# .started_ts will always be set by definition but .ended_ts may be None
# if the task was abandoned. We can't count idle time since the bot may
# have been busy running *another task*.
# TODO(maruel): One option is to add a third value "broken_time".
# Looking at timestamps specifically could help too, e.g. comparing
# ended_ts of this task vs the next one to see if the bot was assigned
# two tasks simultaneously.
if run_results[index].ended_ts:
idle_time += (
run_results[index-1].started_ts - run_results[index].ended_ts)
# We are taking the whole time the bot was doing work, not just the
# duration associated with the task.
duration = run_results[index].duration_as_seen_by_server
if duration:
run_time += duration
events = events_future.get_result()
bot = bot_future.get_result()
if not bot and events:
# If there is not BotInfo, look if there are BotEvent child of this
# entity. If this is the case, it means the bot was deleted but it's
# useful to show information about it to the user even if the bot was
# deleted. For example, it could be an auto-scaled bot.
bot = bot_management.BotInfo(
key=bot_management.get_info_key(bot_id),
dimensions_flat=bot_management.dimensions_to_flat(
events[0].dimensions),
state=events[0].state,
external_ip=events[0].external_ip,
authenticated_as=events[0].authenticated_as,
version=events[0].version,
quarantined=events[0].quarantined,
task_id=events[0].task_id,
last_seen_ts=events[0].ts)
params = {
'bot': bot,
'bot_id': bot_id,
'current_version': bot_code.get_bot_version(self.request.host_url),
'cursor': cursor.urlsafe() if cursor and more else None,
'events': events,
'idle_time': idle_time,
'is_admin': acl.is_admin(),
'limit': limit,
'now': now,
'run_results': run_results,
'run_time': run_time,
'try_link': '/bot?id=%s' % bot_id,
'xsrf_token': self.generate_xsrf_token(),
}
self.response.write(
template.render('swarming/restricted_bot.html', params))
def get(self):
"""Handles both ndb.Query searches and search.Index().search() queries.
If |task_name| is set or not affects the meaning of |cursor|. When set, the
cursor is for search.Index, otherwise the cursor is for a ndb.Query.
"""
cursor_str = self.request.get('cursor')
limit = int(self.request.get('limit', 100))
sort = self.request.get('sort', self.SORT_CHOICES[0][0])
state = self.request.get('state', self.STATE_CHOICES[0][0][0])
task_name = self.request.get('task_name', '').strip()
task_tags = [
line for line in self.request.get('task_tag', '').splitlines()
if line
]
if not any(sort == i[0] for i in self.SORT_CHOICES):
self.abort(400, 'Invalid sort')
if not any(any(state == i[0] for i in j) for j in self.STATE_CHOICES):
self.abort(400, 'Invalid state')
if sort != 'created_ts':
# Zap all filters in this case to reduce the number of required indexes.
# Revisit according to the user requests.
state = 'all'
now = utils.utcnow()
counts_future = self._get_counts_future(now)
# This call is synchronous.
try:
tasks, cursor_str, sort, state = task_result.get_tasks(
limit, cursor_str, sort, state, task_tags, task_name)
# Prefetch the TaskRequest all at once, so that ndb's in-process cache has
# it instead of fetching them one at a time indirectly when using
# TaskResultSummary.request_key.get().
futures = ndb.get_multi_async(t.request_key for t in tasks)
# Evaluate the counts to print the filtering columns with the associated
# numbers.
state_choices = self._get_state_choices(counts_future)
except (search.QueryError, ValueError) as e:
self.abort(400, str(e))
def safe_sum(items):
return sum(items, datetime.timedelta())
def avg(items):
if not items:
return 0.
return safe_sum(items) / len(items)
def median(items):
if not items:
return 0.
middle = len(items) / 2
if len(items) % 2:
return items[middle]
return (items[middle - 1] + items[middle]) / 2
gen = (t.duration_now(now) for t in tasks)
durations = sorted(t for t in gen if t is not None)
gen = (t.pending_now(now) for t in tasks)
pendings = sorted(t for t in gen if t is not None)
total_cost_usd = sum(t.cost_usd for t in tasks)
total_cost_saved_usd = sum(t.cost_saved_usd for t in tasks
if t.cost_saved_usd)
# Include the overhead in the total amount of time saved, since it's
# overhead saved.
# In theory, t.duration_total should always be set when t.deduped_from is
# set but there has some broken entities in the datastore.
total_saved = safe_sum(t.duration_total for t in tasks
if t.deduped_from and t.duration_total)
duration_sum = safe_sum(durations)
total_saved_percent = ((100. * total_saved.total_seconds() /
duration_sum.total_seconds())
if duration_sum else 0.)
params = {
'cursor': cursor_str,
'duration_average': avg(durations),
'duration_median': median(durations),
'duration_sum': duration_sum,
'has_pending': any(t.is_pending for t in tasks),
'has_running': any(t.is_running for t in tasks),
'is_admin': acl.is_admin(),
'is_privileged_user': acl.is_privileged_user(),
'limit': limit,
'now': now,
'pending_average': avg(pendings),
'pending_median': median(pendings),
'pending_sum': safe_sum(pendings),
'show_footer': bool(pendings or durations),
'sort': sort,
'sort_choices': self.SORT_CHOICES,
'state': state,
'state_choices': state_choices,
'task_name': task_name,
'task_tag': '\n'.join(task_tags),
'tasks': tasks,
'total_cost_usd': total_cost_usd,
'total_cost_saved_usd': total_cost_saved_usd,
'total_saved': total_saved,
'total_saved_percent': total_saved_percent,
'xsrf_token': self.generate_xsrf_token(),
}
# TODO(maruel): If admin or if the user is task's .user, show the Cancel
# button. Do not show otherwise.
self.response.write(template.render('swarming/user_tasks.html',
params))
# Do not let dangling futures linger around.
ndb.Future.wait_all(futures)
def get(self, task_id):
try:
key = task_pack.unpack_result_summary_key(task_id)
request_key = task_pack.result_summary_key_to_request_key(key)
except ValueError:
try:
key = task_pack.unpack_run_result_key(task_id)
request_key = task_pack.result_summary_key_to_request_key(
task_pack.run_result_key_to_result_summary_key(key))
except (NotImplementedError, ValueError):
self.abort(404, 'Invalid key format.')
# 'result' can be either a TaskRunResult or TaskResultSummary.
result_future = key.get_async()
request_future = request_key.get_async()
result = result_future.get_result()
if not result:
self.abort(404, 'Invalid key.')
if not acl.is_privileged_user():
self.abort(403, 'Implement access control based on the user')
request = request_future.get_result()
parent_task_future = None
if request.parent_task_id:
parent_key = task_pack.unpack_run_result_key(request.parent_task_id)
parent_task_future = parent_key.get_async()
children_tasks_futures = [
task_pack.unpack_result_summary_key(c).get_async()
for c in result.children_task_ids
]
bot_id = result.bot_id
following_task_future = None
previous_task_future = None
if result.started_ts:
# Use a shortcut name because it becomes unwieldy otherwise.
cls = task_result.TaskRunResult
# Note that the links will be to the TaskRunResult, not to
# TaskResultSummary.
following_task_future = cls.query(
cls.bot_id == bot_id,
cls.started_ts > result.started_ts,
).order(cls.started_ts).get_async()
previous_task_future = cls.query(
cls.bot_id == bot_id,
cls.started_ts < result.started_ts,
).order(-cls.started_ts).get_async()
bot_future = (
bot_management.get_info_key(bot_id).get_async() if bot_id else None)
following_task = None
if following_task_future:
following_task = following_task_future.get_result()
previous_task = None
if previous_task_future:
previous_task = previous_task_future.get_result()
parent_task = None
if parent_task_future:
parent_task = parent_task_future.get_result()
children_tasks = [c.get_result() for c in children_tasks_futures]
params = {
'bot': bot_future.get_result() if bot_future else None,
'children_tasks': children_tasks,
'is_admin': acl.is_admin(),
'is_gae_admin': users.is_current_user_admin(),
'is_privileged_user': acl.is_privileged_user(),
'following_task': following_task,
'full_appid': os.environ['APPLICATION_ID'],
'host_url': self.request.host_url,
'is_running': result.state == task_result.State.RUNNING,
'now': utils.utcnow(),
'parent_task': parent_task,
'previous_task': previous_task,
'request': request,
'task': result,
'xsrf_token': self.generate_xsrf_token(),
}
self.response.write(template.render('swarming/user_task.html', params))
def get(self):
"""Handles both ndb.Query searches and search.Index().search() queries.
If |task_name| is set or not affects the meaning of |cursor|. When set, the
cursor is for search.Index, otherwise the cursor is for a ndb.Query.
"""
cursor_str = self.request.get('cursor')
limit = int(self.request.get('limit', 100))
sort = self.request.get('sort', self.SORT_CHOICES[0][0])
state = self.request.get('state', self.STATE_CHOICES[0][0][0])
task_name = self.request.get('task_name', '').strip()
task_tags = [
line for line in self.request.get('task_tag', '').splitlines() if line
]
if not any(sort == i[0] for i in self.SORT_CHOICES):
self.abort(400, 'Invalid sort')
if not any(any(state == i[0] for i in j) for j in self.STATE_CHOICES):
self.abort(400, 'Invalid state')
if sort != 'created_ts':
# Zap all filters in this case to reduce the number of required indexes.
# Revisit according to the user requests.
state = 'all'
now = utils.utcnow()
counts_future = self._get_counts_future(now)
# This call is synchronous.
try:
tasks, cursor_str, sort, state = task_result.get_tasks(
limit, cursor_str, sort, state, task_tags, task_name)
# Prefetch the TaskRequest all at once, so that ndb's in-process cache has
# it instead of fetching them one at a time indirectly when using
# TaskResultSummary.request_key.get().
futures = ndb.get_multi_async(t.request_key for t in tasks)
# Evaluate the counts to print the filtering columns with the associated
# numbers.
state_choices = self._get_state_choices(counts_future)
except (search.QueryError, ValueError) as e:
self.abort(400, str(e))
def safe_sum(items):
return sum(items, datetime.timedelta())
def avg(items):
if not items:
return 0.
return safe_sum(items) / len(items)
def median(items):
if not items:
return 0.
middle = len(items) / 2
if len(items) % 2:
return items[middle]
return (items[middle-1]+items[middle]) / 2
gen = (t.duration_now(now) for t in tasks)
durations = sorted(t for t in gen if t is not None)
gen = (t.pending_now(now) for t in tasks)
pendings = sorted(t for t in gen if t is not None)
total_cost_usd = sum(t.cost_usd for t in tasks)
total_cost_saved_usd = sum(
t.cost_saved_usd for t in tasks if t.cost_saved_usd)
# Include the overhead in the total amount of time saved, since it's
# overhead saved.
# In theory, t.duration_total should always be set when t.deduped_from is
# set but there has some broken entities in the datastore.
total_saved = safe_sum(
t.duration_total for t in tasks if t.deduped_from and t.duration_total)
duration_sum = safe_sum(durations)
total_saved_percent = (
(100. * total_saved.total_seconds() / duration_sum.total_seconds())
if duration_sum else 0.)
params = {
'cursor': cursor_str,
'duration_average': avg(durations),
'duration_median': median(durations),
'duration_sum': duration_sum,
'has_pending': any(t.is_pending for t in tasks),
'has_running': any(t.is_running for t in tasks),
'is_admin': acl.is_admin(),
'is_privileged_user': acl.is_privileged_user(),
'limit': limit,
'now': now,
'pending_average': avg(pendings),
'pending_median': median(pendings),
'pending_sum': safe_sum(pendings),
'show_footer': bool(pendings or durations),
'sort': sort,
'sort_choices': self.SORT_CHOICES,
'state': state,
'state_choices': state_choices,
'task_name': task_name,
'task_tag': '\n'.join(task_tags),
'tasks': tasks,
'total_cost_usd': total_cost_usd,
'total_cost_saved_usd': total_cost_saved_usd,
'total_saved': total_saved,
'total_saved_percent': total_saved_percent,
'xsrf_token': self.generate_xsrf_token(),
}
# TODO(maruel): If admin or if the user is task's .user, show the Cancel
# button. Do not show otherwise.
self.response.write(template.render('swarming/user_tasks.html', params))
# Do not let dangling futures linger around.
ndb.Future.wait_all(futures)
def get(self, task_id):
try:
key = task_pack.unpack_result_summary_key(task_id)
request_key = task_pack.result_summary_key_to_request_key(key)
except ValueError:
try:
key = task_pack.unpack_run_result_key(task_id)
request_key = task_pack.result_summary_key_to_request_key(
task_pack.run_result_key_to_result_summary_key(key)
)
except (NotImplementedError, ValueError):
self.abort(404, "Invalid key format.")
# 'result' can be either a TaskRunResult or TaskResultSummary.
result_future = key.get_async()
request_future = request_key.get_async()
result = result_future.get_result()
if not result:
self.abort(404, "Invalid key.")
if not acl.is_privileged_user():
self.abort(403, "Implement access control based on the user")
request = request_future.get_result()
parent_task_future = None
if request.parent_task_id:
parent_key = task_pack.unpack_run_result_key(request.parent_task_id)
parent_task_future = parent_key.get_async()
children_tasks_futures = [task_pack.unpack_result_summary_key(c).get_async() for c in result.children_task_ids]
bot_id = result.bot_id
following_task_future = None
previous_task_future = None
if result.started_ts:
# Use a shortcut name because it becomes unwieldy otherwise.
cls = task_result.TaskRunResult
# Note that the links will be to the TaskRunResult, not to
# TaskResultSummary.
following_task_future = (
cls.query(cls.bot_id == bot_id, cls.started_ts > result.started_ts).order(cls.started_ts).get_async()
)
previous_task_future = (
cls.query(cls.bot_id == bot_id, cls.started_ts < result.started_ts).order(-cls.started_ts).get_async()
)
bot_future = bot_management.get_info_key(bot_id).get_async() if bot_id else None
following_task = None
if following_task_future:
following_task = following_task_future.get_result()
previous_task = None
if previous_task_future:
previous_task = previous_task_future.get_result()
parent_task = None
if parent_task_future:
parent_task = parent_task_future.get_result()
children_tasks = [c.get_result() for c in children_tasks_futures]
params = {
"bot": bot_future.get_result() if bot_future else None,
"children_tasks": children_tasks,
"is_admin": acl.is_admin(),
"is_gae_admin": users.is_current_user_admin(),
"is_privileged_user": acl.is_privileged_user(),
"following_task": following_task,
"full_appid": os.environ["APPLICATION_ID"],
"host_url": self.request.host_url,
"is_running": result.state == task_result.State.RUNNING,
"now": utils.utcnow(),
"parent_task": parent_task,
"previous_task": previous_task,
"request": request,
"task": result,
"xsrf_token": self.generate_xsrf_token(),
}
self.response.write(template.render("swarming/user_task.html", params))
def get(self, task_id):
try:
key = task_pack.unpack_result_summary_key(task_id)
request_key = task_pack.result_summary_key_to_request_key(key)
except ValueError:
try:
key = task_pack.unpack_run_result_key(task_id)
request_key = task_pack.result_summary_key_to_request_key(
task_pack.run_result_key_to_result_summary_key(key))
except (NotImplementedError, ValueError):
self.abort(404, 'Invalid key format.')
# 'result' can be either a TaskRunResult or TaskResultSummary.
result_future = key.get_async()
request_future = request_key.get_async()
result = result_future.get_result()
if not result:
self.abort(404, 'Invalid key.')
if not acl.is_privileged_user():
self.abort(403, 'Implement access control based on the user')
request = request_future.get_result()
parent_task_future = None
if request.parent_task_id:
parent_key = task_pack.unpack_run_result_key(request.parent_task_id)
parent_task_future = parent_key.get_async()
children_tasks_futures = [
task_pack.unpack_result_summary_key(c).get_async()
for c in result.children_task_ids
]
bot_id = result.bot_id
following_task_future = None
previous_task_future = None
if result.started_ts:
# Use a shortcut name because it becomes unwieldy otherwise.
cls = task_result.TaskRunResult
# Note that the links will be to the TaskRunResult, not to
# TaskResultSummary.
following_task_future = cls.query(
cls.bot_id == bot_id,
cls.started_ts > result.started_ts,
).order(cls.started_ts).get_async()
previous_task_future = cls.query(
cls.bot_id == bot_id,
cls.started_ts < result.started_ts,
).order(-cls.started_ts).get_async()
bot_future = (
bot_management.get_info_key(bot_id).get_async() if bot_id else None)
following_task = None
if following_task_future:
following_task = following_task_future.get_result()
previous_task = None
if previous_task_future:
previous_task = previous_task_future.get_result()
parent_task = None
if parent_task_future:
parent_task = parent_task_future.get_result()
children_tasks = [c.get_result() for c in children_tasks_futures]
params = {
'bot': bot_future.get_result() if bot_future else None,
'children_tasks': children_tasks,
'is_admin': acl.is_admin(),
'is_gae_admin': users.is_current_user_admin(),
'is_privileged_user': acl.is_privileged_user(),
'following_task': following_task,
'full_appid': os.environ['APPLICATION_ID'],
'host_url': self.request.host_url,
'is_running': result.state == task_result.State.RUNNING,
'now': utils.utcnow(),
'parent_task': parent_task,
'previous_task': previous_task,
'request': request,
'task': result,
'xsrf_token': self.generate_xsrf_token(),
}
self.response.write(template.render('swarming/user_task.html', params))
