def get_tasks(task_name, task_tags, cursor_str, limit, sort, state):
"""Returns TaskResultSummary entities for this query.
This function is synchronous.
Arguments:
task_name: search for task name whole word.
task_tags: list of search for one or multiple task tags.
cursor_str: query-dependent string encoded cursor to continue a previous
search.
limit: Maximum number of items to return.
sort: get_result_summary_query() argument. Only used if both task_name and
task_tags are empty.
state: get_result_summary_query() argument. Only used if both task_name and
task_tags are empty.
Returns:
tuple(list of tasks, str encoded cursor, updated sort, updated state)
"""
# TODO(vadimsh): Use tags with get_result_summary_query. Will require existing
# entities to be updated first to include 'tags' fields (otherwise they'll
# disappear from the search).
if task_tags:
# Tag based search. Override the flags.
sort = 'created_ts'
state = 'all'
# Only the TaskRequest has the tags. So first query all the keys to
# requests; then fetch the TaskResultSummary.
order = _sort_property(sort)
query = task_request.TaskRequest.query().order(order)
task_tags = task_tags[:]
tags_filter = task_request.TaskRequest.tags == task_tags.pop(0)
while task_tags:
tags_filter = ndb.AND(
tags_filter, task_request.TaskRequest.tags == task_tags.pop(0))
query = query.filter(tags_filter)
cursor = datastore_query.Cursor(urlsafe=cursor_str)
requests, cursor, more = query.fetch_page(limit,
start_cursor=cursor,
keys_only=True)
keys = [
task_pack.request_key_to_result_summary_key(k) for k in requests
]
tasks = ndb.get_multi(keys)
cursor_str = cursor.urlsafe() if cursor and more else None
elif task_name:
# Task name based word based search. Override the flags.
sort = 'created_ts'
state = 'all'
tasks, cursor_str = search_by_name(task_name, cursor_str, limit)
else:
# Normal listing.
query = get_result_summary_query(sort, state, None)
cursor = datastore_query.Cursor(urlsafe=cursor_str)
tasks, cursor, more = query.fetch_page(limit, start_cursor=cursor)
cursor_str = cursor.urlsafe() if cursor and more else None
return tasks, cursor_str, sort, state
def get_question_ids_linked_to_skill_ids(cls, question_count, skill_ids,
start_cursor):
"""Fetches the list of question ids linked to the skill in batches.
Args:
question_count: int. The number of questions to be returned.
skill_ids: list(str). The ids of skills for which the linked
question ids are to be retrieved.
start_cursor: str. The starting point from which the batch of
questions are to be returned. This value should be urlsafe.
Returns:
list(str), str|None. The question ids linked to given skills and the
next cursor value to be used for the next page (or None if no
more pages are left). The returned next cursor value is urlsafe.
"""
if not start_cursor == '':
cursor = datastore_query.Cursor(urlsafe=start_cursor)
question_skill_link_models, next_cursor, more = cls.query(
cls.skill_id.IN(skill_ids)).order(cls.key).fetch_page(
question_count, start_cursor=cursor)
else:
question_skill_link_models, next_cursor, more = cls.query(
cls.skill_id.IN(skill_ids)).order(
cls.key).fetch_page(question_count)
question_ids = [
model.question_id for model in question_skill_link_models
]
next_cursor_str = (next_cursor.urlsafe() if
(next_cursor and more) else None)
return question_ids, next_cursor_str
def fetch_exploration_task_history_page(exploration,
urlsafe_start_cursor=None):
"""Fetches a page from the given exploration's history of resolved tasks.
Args:
exploration: exp_domain.Exploration. The exploration to fetch the
history page for.
urlsafe_start_cursor: str or None. Starting point for the search. When
None, the starting point is the very beginning of the history
results (i.e. starting from the most recently resolved task entry).
Returns:
tuple. Contains the following 3 items:
results: list(improvements_domain.TaskEntry). The query results.
urlsafe_cursor: str or None. a query cursor pointing to the "next"
batch of results. If there are no more results, this might be
None.
more: bool. Indicates whether there are (likely) more results after
this batch. If False, there are no more results; if True, there
are probably more results.
"""
start_cursor = (urlsafe_start_cursor
and datastore_query.Cursor(urlsafe=urlsafe_start_cursor))
results, cursor, more = (improvements_models.TaskEntryModel.query(
improvements_models.TaskEntryModel.entity_type == (
improvements_models.TASK_ENTITY_TYPE_EXPLORATION),
improvements_models.TaskEntryModel.entity_id == exploration.id,
improvements_models.TaskEntryModel.status == (
improvements_models.TASK_STATUS_RESOLVED
)).order(-improvements_models.TaskEntryModel.resolved_on).fetch_page(
feconf.MAX_TASK_MODELS_PER_HISTORY_PAGE,
start_cursor=start_cursor))
return ([get_task_entry_from_model(model) for model in results], cursor
and cursor.urlsafe(), more)
def get_question_ids_linked_to_skill_ids(cls, question_count, skill_ids,
start_cursor):
"""Fetches the list of question ids linked to the skill in batches.
Args:
question_count: int. The number of questions to be returned.
skill_ids: list(str). The ids of skills for which the linked
question ids are to be retrieved.
start_cursor: str. The starting point from which the batch of
questions are to be returned. This value should be urlsafe.
Returns:
list(str), str|None. The question ids linked to given skills and the
next cursor value to be used for the next page (or None if no
more pages are left). The returned next cursor value is urlsafe.
"""
if not start_cursor == '':
cursor = datastore_query.Cursor(urlsafe=start_cursor)
question_skill_link_models, next_cursor, more = cls.query(
cls.skill_id.IN(skill_ids)
# Order by cls.key is needed alongside cls.last_updated so as to
# resolve conflicts, if any.
# Reference SO link: https://stackoverflow.com/q/12449197
).order(-cls.last_updated, cls.key).fetch_page(question_count,
start_cursor=cursor)
else:
question_skill_link_models, next_cursor, more = cls.query(
cls.skill_id.IN(skill_ids)).order(
-cls.last_updated, cls.key).fetch_page(question_count)
question_ids = [
model.question_id for model in question_skill_link_models
]
next_cursor_str = (next_cursor.urlsafe() if
(next_cursor and more) else None)
return question_ids, next_cursor_str
def get_skill_opportunities(cls, page_size, urlsafe_start_cursor):
"""Returns a list of skill opportunities available for adding questions.
Args:
page_size: int. The maximum number of entities to be returned.
urlsafe_start_cursor: str or None. If provided, the list of
returned entities starts from this datastore cursor.
Otherwise, the returned entities start from the beginning
of the full list of entities.
Returns:
3-tuple of (results, cursor, more) as described in fetch_page() at:
https://developers.google.com/appengine/docs/python/ndb/queryclass,
where:
results: list(SkillOpportunityModel)|None. A list
of query results.
cursor: str or None. A query cursor pointing to the next
batch of results. If there are no more results, this might
be None.
more: bool. If True, there are (probably) more results after
this batch. If False, there are no further results after
this batch.
"""
if urlsafe_start_cursor:
start_cursor = datastore_query.Cursor(urlsafe=urlsafe_start_cursor)
else:
start_cursor = None
results, cursor, more = cls.get_all().order(cls.created_on).fetch_page(
page_size, start_cursor=start_cursor)
return (results, (cursor.urlsafe() if cursor else None), more)
def _Migrate(status, cursor=None):
if cursor:
cursor = datastore_query.Cursor(urlsafe=cursor)
query = job.Job.query(job.Job.task == None)
jobs, next_cursor, more = query.fetch_page(_BATCH_SIZE,
start_cursor=cursor)
# Because individual job instances might fail to be persisted for some reason
# (e.g. entities exceeding the entity size limit) we'll perform the updates
# one at a time. This is not an ideal state, since we'll want to be able to
# migrate all jobs to an alternative structure in the future, but we recognise
# that partial success is better than total failure.
for j in jobs:
try:
j.put()
status['count'] += 1
except datastore_errors.BadRequestError as e:
logging.error('Failed migrating job %s: %s', j.job_id, e)
status['errors'] += 1
if more:
stored_object.Set(_STATUS_KEY, status)
deferred.defer(_Migrate, status, next_cursor.urlsafe())
else:
stored_object.Set(_STATUS_KEY, None)
def fetch_page(cls, page_size, cursor):
"""Fetches a list of all query_models sorted by creation date.
Args:
page_size: int. The maximum number of entities to be returned.
cursor: str or None. The list of returned entities starts from this
datastore cursor.
Returns:
3-tuple of (query_models, cursor, more) as described in fetch_page()
at:
https://developers.google.com/appengine/docs/python/ndb/queryclass,
where:
query_models: List of UserQueryModel instances.
next_cursor: str or None. A query cursor pointing to the next
batch of results. If there are no more results, this might
be None.
more: bool. If True, there are probably more results after
this batch. If False, there are no further results after
this batch.
"""
cursor = datastore_query.Cursor(urlsafe=cursor)
query_models, next_cursor, more = (
cls.query().order(-cls.created_on).fetch_page(page_size,
start_cursor=cursor))
next_cursor = next_cursor.urlsafe() if (next_cursor and more) else None
return query_models, next_cursor, more
def get(self):
cursor = datastore_query.Cursor(urlsafe=self.request.get('cursor'))
repo = self.request.get('repo')
number = int(self.request.get('number', 0)) or None
count = int(self.request.get('count', 500))
if repo is not None and number is not None:
q = models.GithubWebhookRaw.query(
models.GithubWebhookRaw.repo == repo,
models.GithubWebhookRaw.number == number)
else:
q = models.GithubWebhookRaw.query()
q = q.order(models.GithubWebhookRaw.timestamp)
events, next_cursor, more = q.fetch_page(count, start_cursor=cursor)
out = []
for event in events:
out.append({
'repo': event.repo,
'event': event.event,
'guid': event.guid,
'timestamp': str(event.timestamp),
'body': json.loads(event.body)
})
resp = {'next': more and next_cursor.urlsafe(), 'calls': out}
self.response.headers['content-type'] = 'text/json'
self.response.write(json.dumps(resp, indent=4, sort_keys=True))
def fetch_page(query, batch_size, cursor_str, **kwargs):
"""Fetches a page from a query.
Arguments:
query: ndb.Query.
batch_size: Maximum number of items to return.
cursor_str: query-dependent string encoded cursor to continue a previous
search.
Returns:
- items
- str encoded cursor if relevant or None.
"""
assert isinstance(query, ndb.Query), query
if not 0 < batch_size <= 1000 or not isinstance(batch_size, int):
raise ValueError(
'batch_size must be between 1 and 1000, got %r', batch_size)
if cursor_str:
if not isinstance(cursor_str, basestring):
raise ValueError(
'cursor must be between valid string, got %r', cursor_str)
cursor = datastore_query.Cursor(urlsafe=cursor_str)
else:
cursor = None
items, cursor, more = query.fetch_page(
batch_size, start_cursor=cursor, **kwargs)
if not more:
return items, None
return items, cursor.urlsafe()
def get(self):
"""
1) Filter by subject
2) Filter by Completion
"""
user = users.get_current_user()
if check_creds(user, self.check_u(), admin=True):
admin = hold_creds(user, self.check_u())
cursor_str = self.request.get('c', None)
cursor = None
if cursor_str:
cursor = datastore_query.Cursor(urlsafe=cursor_str)
var1 = self.request.get('q')
# var2 = self.request.get('o')
subject_list = m.Standard.admin_get_subject_list()
results, new_cursor, more = self.organise(var1, subject_list,
cursor)
if more:
urlcursor = new_cursor.urlsafe()
else:
urlcursor = None
self.render('standards/all_standard_page_admin.html',
var=results,
subject_list=subject_list,
q=var1,
urlcursor=urlcursor,
admin=admin)
else:
self.redirect('/ouch')
def post(self):
"""Query for tests, and put ones with no new data on the delete queue."""
datastore_hooks.SetPrivilegedRequest()
cursor = datastore_query.Cursor(urlsafe=self.request.get('cursor'))
tests, next_cursor, more = graph_data.TestMetadata.query().fetch_page(
_TESTS_TO_CHECK_AT_ONCE, keys_only=True, start_cursor=cursor)
if more:
taskqueue.add(url='/delete_old_tests',
params={'cursor': next_cursor.urlsafe()},
queue_name=_TASK_QUEUE_NAME)
for test in tests:
# Delete this test if:
# 1) It has no Rows newer than the cutoff
# 2) It has no descendant tests
no_new_rows = False
last_row = graph_data.Row.query(
graph_data.Row.parent_test == utils.OldStyleTestKey(
test)).order(-graph_data.Row.timestamp).get()
if last_row:
if last_row.timestamp < datetime.datetime.today(
) - _CUTOFF_DATE:
no_new_rows = True
else:
no_new_rows = True
descendants = list_tests.GetTestDescendants(test, keys_only=True)
descendants.remove(test)
if not descendants and no_new_rows:
taskqueue.add(
url='/delete_test_data',
params={
'test_path':
utils.TestPath(test), # For manual inspection.
'test_key': test.urlsafe(),
},
queue_name=_DELETE_TASK_QUEUE_NAME)
def _fetch_page_sorted_by_last_updated(cls, query, page_size,
urlsafe_start_cursor):
"""Fetches a page of entities sorted by their last_updated attribute in
descending order (newly updated first).
Args:
query: ndb.Query.
page_size: int. The maximum number of entities to be returned.
urlsafe_start_cursor: str or None. If provided, the list of returned
entities starts from this datastore cursor. Otherwise,
the returned entities start from the beginning of the full
list of entities.
Returns:
3-tuple of (results, cursor, more) as described in fetch_page() at:
https://developers.google.com/appengine/docs/python/ndb/queryclass,
where:
results: List of query results.
cursor: str or None. A query cursor pointing to the next batch
of results. If there are no more results, this will be None.
more: bool. If True, there are (probably) more results after
this batch. If False, there are no further results after
this batch.
"""
if urlsafe_start_cursor:
start_cursor = datastore_query.Cursor(urlsafe=urlsafe_start_cursor)
else:
start_cursor = None
result = query.order(-cls.last_updated).fetch_page(
page_size, start_cursor=start_cursor)
return (result[0], (result[1].urlsafe() if result[1] else None),
result[2])
def get(self):
limit = int(self.request.get('limit', 100))
cursor = datastore_query.Cursor(urlsafe=self.request.get('cursor'))
sort_by = self.request.get('sort_by', '__key__')
if sort_by not in self.ACCEPTABLE_BOTS_SORTS:
self.abort(400, 'Invalid sort_by query parameter')
if sort_by[0] == '-':
order = datastore_query.PropertyOrder(
sort_by[1:], datastore_query.PropertyOrder.DESCENDING)
else:
order = datastore_query.PropertyOrder(
sort_by, datastore_query.PropertyOrder.ASCENDING)
now = utils.utcnow()
cutoff = now - datetime.timedelta(
seconds=config.settings().bot_death_timeout_secs)
num_bots_busy_future = bot_management.BotInfo.query(
bot_management.BotInfo.is_busy == True).count_async()
num_bots_dead_future = bot_management.BotInfo.query(
bot_management.BotInfo.last_seen_ts < cutoff).count_async()
num_bots_quarantined_future = bot_management.BotInfo.query(
bot_management.BotInfo.quarantined == True).count_async()
num_bots_total_future = bot_management.BotInfo.query().count_async()
fetch_future = bot_management.BotInfo.query().order(
order).fetch_page_async(limit, start_cursor=cursor)
# TODO(maruel): self.request.host_url should be the default AppEngine url
# version and not the current one. It is only an issue when
# version-dot-appid.appspot.com urls are used to access this page.
version = bot_code.get_bot_version(self.request.host_url)
bots, cursor, more = fetch_future.get_result()
# Prefetch the tasks. We don't actually use the value here, it'll be
# implicitly used by ndb local's cache when refetched by the html template.
tasks = filter(None, (b.task for b in bots))
ndb.get_multi(tasks)
num_bots_busy = num_bots_busy_future.get_result()
num_bots_dead = num_bots_dead_future.get_result()
num_bots_quarantined = num_bots_quarantined_future.get_result()
num_bots_total = num_bots_total_future.get_result()
params = {
'bots': bots,
'current_version': version,
'cursor': cursor.urlsafe() if cursor and more else '',
'is_admin': acl.is_admin(),
'is_privileged_user': acl.is_privileged_user(),
'limit': limit,
'now': now,
'num_bots_alive': num_bots_total - num_bots_dead,
'num_bots_busy': num_bots_busy,
'num_bots_dead': num_bots_dead,
'num_bots_quarantined': num_bots_quarantined,
'sort_by': sort_by,
'sort_options': self.SORT_OPTIONS,
'xsrf_token': self.generate_xsrf_token(),
}
self.response.write(
template.render('swarming/restricted_botslist.html', params))
def exportDb(self, cursor=None, backupkey=None, endcursor=None, kind=None, *args, **kwargs):
global backupKey
assert safeStringComparison(backupKey, backupkey)
if cursor:
c = datastore_query.Cursor(urlsafe=cursor)
else:
c = None
if endcursor:
endCursor = datastore_query.Cursor(urlsafe=endcursor)
else:
endCursor = None
q = datastore.Query(kind, cursor=c, end_cursor=endCursor)
logging.error((cursor, backupkey, endcursor, kind))
r = []
for res in q.Run(limit=5):
r.append(self.genDict(res))
return (pickle.dumps({"cursor": str(q.GetCursor().urlsafe()), "values": r}).encode("HEX"))
def post(self, uuid):
# Prepare the query
cursor = self.parsed_json.get(santa_const.RULE_DOWNLOAD.CURSOR)
if self.host.rule_sync_dt is None:
logging.info('%s clean rule sync', 'Continuing' if cursor else 'Starting')
# pylint:disable=g-explicit-bool-comparison
query = santa_db.SantaRule.query(
santa_db.SantaRule.in_effect == True,
santa_db.SantaRule.updated_dt >= self.host.rule_sync_dt,
santa_db.SantaRule.host_id.IN(['', uuid])
).order(santa_db.SantaRule.updated_dt, santa_db.SantaRule.key)
# pylint:enable=g-explicit-bool-comparison
# Fetch
rules, next_cursor, more = query.fetch_page(
settings.SANTA_RULE_BATCH_SIZE,
start_cursor=datastore_query.Cursor(urlsafe=cursor))
# Process the received rules.
response_rules = []
for rule in rules:
creation_timestamp = common_utils.ToUtcTimestamp(rule.updated_dt)
rule_dict = {
santa_const.RULE_DOWNLOAD.SHA256: rule.key.parent().id(),
santa_const.RULE_DOWNLOAD.RULE_TYPE: rule.rule_type,
santa_const.RULE_DOWNLOAD.POLICY: rule.policy,
santa_const.RULE_DOWNLOAD.CUSTOM_MSG: rule.custom_msg,
santa_const.RULE_DOWNLOAD.CREATION_TIME: creation_timestamp}
if rule.rule_type == common_const.RULE_TYPE.PACKAGE:
# For Bundles, each binary member should have a separate rule generated
# with a policy type matching that of the PACKAGE rule.
binary_ids = rule.bundle_binary_ids
binary_count = len(binary_ids)
logging.debug('Syncing %s bundle rules', binary_ids)
for id_ in binary_ids:
dict_ = rule_dict.copy()
dict_.update({
santa_const.RULE_DOWNLOAD.SHA256: id_,
santa_const.RULE_DOWNLOAD.RULE_TYPE:
common_const.RULE_TYPE.BINARY,
santa_const.RULE_DOWNLOAD.FILE_BUNDLE_BINARY_COUNT: binary_count,
santa_const.RULE_DOWNLOAD.FILE_BUNDLE_HASH: rule.key.parent().id()
})
response_rules.append(dict_)
else:
response_rules.append(rule_dict)
# Prepare the response, include the cursor if there are more rules.
response = {santa_const.RULE_DOWNLOAD.RULES: response_rules}
if more:
response[santa_const.RULE_DOWNLOAD.CURSOR] = next_cursor.urlsafe()
self.respond_json(response)
def get(self, bot_id):
# pagination is currently for tasks, not events.
limit = int(self.request.get('limit', 100))
cursor = datastore_query.Cursor(urlsafe=self.request.get('cursor'))
bot_future = bot_management.get_info_key(bot_id).get_async()
run_results, cursor, more = task_result.TaskRunResult.query(
task_result.TaskRunResult.bot_id == bot_id).order(
-task_result.TaskRunResult.started_ts).fetch_page(
limit, start_cursor=cursor)
events_future = bot_management.get_events_query(bot_id).fetch_async(
100)
now = utils.utcnow()
bot = bot_future.get_result()
# Calculate the time this bot was idle.
idle_time = datetime.timedelta()
run_time = datetime.timedelta()
if run_results:
run_time = run_results[0].duration_now(now) or datetime.timedelta()
if not cursor and run_results[0].state != task_result.State.RUNNING:
# Add idle time since last task completed. Do not do this when a cursor
# is used since it's not representative.
idle_time = now - run_results[0].ended_ts
for index in xrange(1, len(run_results)):
# .started_ts will always be set by definition but .ended_ts may be None
# if the task was abandoned. We can't count idle time since the bot may
# have been busy running *another task*.
# TODO(maruel): One option is to add a third value "broken_time".
# Looking at timestamps specifically could help too, e.g. comparing
# ended_ts of this task vs the next one to see if the bot was assigned
# two tasks simultaneously.
if run_results[index].ended_ts:
idle_time += (run_results[index - 1].started_ts -
run_results[index].ended_ts)
duration = run_results[index].duration
if duration:
run_time += duration
params = {
'bot': bot,
'bot_id': bot_id,
'current_version': bot_code.get_bot_version(self.request.host_url),
'cursor': cursor.urlsafe() if cursor and more else None,
'events': events_future.get_result(),
'idle_time': idle_time,
'is_admin': acl.is_admin(),
'limit': limit,
'now': now,
'run_results': run_results,
'run_time': run_time,
'xsrf_token': self.generate_xsrf_token(),
}
self.response.write(
template.render('swarming/restricted_bot.html', params))
def get(self):
"""Fixes rows for one or more tests and queues the next task to fix more.
Request parameters:
ancestor: A slash-separated path to the ancestor to start from.
cursor: An urlsafe string for a datastore_query.Cursor object.
Outputs:
Some indication of the results.
"""
# Get the ancestor of the tests to change, and abort if not given.
ancestor = self.request.get('ancestor')
if not ancestor:
self.ReportError('Missing ancestor parameter.')
return
ancestor_key = utils.TestKey(ancestor)
# Get the query cursor if given.
urlsafe_cursor = self.request.get('cursor')
cursor = None
if urlsafe_cursor:
cursor = datastore_query.Cursor(urlsafe=urlsafe_cursor)
more = False
test_query = graph_data.Test.query(ancestor=ancestor_key)
test_query = test_query.filter(graph_data.Test.has_rows == True)
keys, next_cursor, more = test_query.fetch_page(_NUM_TESTS,
keys_only=True,
start_cursor=cursor)
futures = []
for key in keys:
futures.extend(_FixTest(key))
ndb.Future.wait_all(futures)
if not futures:
cursor = next_cursor
urlsafe_cursor = cursor.urlsafe() if cursor else ''
if more or futures:
taskqueue.add(queue_name=_QUEUE_NAME,
url='/shrink_timestamp_revisions',
params={
'cursor': urlsafe_cursor or '',
'ancestor': ancestor
})
logging.info('Task added, cursor: %s', urlsafe_cursor)
# Display some information, to verify that something is happening.
self.RenderHtml(
'result.html',
{'results': [{
'name': 'cursor',
'value': urlsafe_cursor
}]})
def cursor(self, cursor, endCursor=None):
"""
Sets the start cursor for this query.
The result set will only include results behind that cursor.
The cursor is generated by an earlier query with exactly the same configuration.
Its safe to use client-supplied cursors, a cursor can't be abused to access entities
which don't match the current filters.
:param cursor: The cursor key to set to the Query.
:type cursor: str | datastore_query.Cursor
:returns: Returns the query itself for chaining.
:rtype: server.db.Query
"""
if isinstance(cursor, basestring):
cursor = datastore_query.Cursor(urlsafe=cursor)
elif isinstance(cursor, datastore_query.Cursor) or cursor == None:
pass
else:
raise ValueError(
"Cursor must be String, datastore_query.Cursor or None")
if endCursor is not None:
if isinstance(endCursor, basestring):
endCursor = datastore_query.Cursor(urlsafe=endCursor)
elif isinstance(cursor,
datastore_query.Cursor) or endCursor == None:
pass
else:
raise ValueError(
"endCursor must be String, datastore_query.Cursor or None")
qo = self.datastoreQuery.__query_options
self.datastoreQuery.__query_options = datastore_query.QueryOptions(
keys_only=qo.keys_only,
produce_cursors=qo.produce_cursors,
start_cursor=cursor,
end_cursor=endCursor or qo.end_cursor,
projection=qo.projection)
self._origCursor = cursor
return (self)
def getListData(self, list_id, query, start=None, limit=50):
"""See ListDataReader.getListData for specification."""
start_cursor = datastore_query.Cursor(urlsafe=start)
entities, next_cursor, more = query.fetch_page(
limit, start_cursor=start_cursor)
next_cursor = next_cursor.urlsafe() if more else FINAL_BATCH
col_funcs = [(c.col_id, c.getValue) for c in getList(list_id).columns]
items = [toListItemDict(entity, col_funcs) for entity in entities]
return ListData(items[:limit], next_cursor)
def _fetch_page_sorted_by_last_updated(cls, query, page_size,
urlsafe_start_cursor):
if urlsafe_start_cursor:
start_cursor = datastore_query.Cursor(urlsafe=urlsafe_start_cursor)
else:
start_cursor = None
result = query.order(-cls.last_updated).fetch_page(
page_size, start_cursor=start_cursor)
return (result[0], (result[1].urlsafe() if result[1] else None),
result[2])
def search(cls, query_string=None, cursor=None, limit=None):
limit = limit or 200
start_cursor = datastore_query.Cursor(urlsafe=cursor) \
if cursor else None
query = cls.query()
if query_string:
# TODO: Support query language.
query = query.filter(cls.email == query_string)
query = query.order(-cls.created)
results, next_cursor, has_more = \
query.fetch_page(limit, start_cursor=start_cursor)
return (results, next_cursor, has_more)
def _Migrate(self, query, status):
cursor = datastore_query.Cursor(urlsafe=self.request.get('cursor'))
jobs, next_cursor, more = query.fetch_page(_BATCH_SIZE, start_cursor=cursor)
ndb.put_multi(jobs)
if more:
status['count'] += len(jobs)
stored_object.Set(_STATUS_KEY, status)
params = {'cursor': next_cursor.urlsafe()}
taskqueue.add(url='/api/migrate', params=params)
else:
stored_object.Set(_STATUS_KEY, None)
def get(self):
limit = int(self.request.get('limit', 100))
cursor = datastore_query.Cursor(urlsafe=self.request.get('cursor'))
errors_found, cursor, more = models.Error.query().order(
-models.Error.created_ts).fetch_page(limit, start_cursor=cursor)
params = {
'cursor': cursor.urlsafe() if cursor and more else None,
'errors': errors_found,
'limit': limit,
'now': utils.utcnow(),
}
self.response.out.write(
template.render('ereporter2/errors.html', params))
def get(self):
now = utils.utcnow()
limit = int(self.request.get('limit', 1000))
cursor = datastore_query.Cursor(urlsafe=self.request.get('cursor'))
q = bot_management.BotInfo.query().order(bot_management.BotInfo.key)
bots, cursor, more = q.fetch_page(limit, start_cursor=cursor)
data = {
'cursor': cursor.urlsafe() if cursor and more else None,
'death_timeout': config.settings().bot_death_timeout_secs,
'items': [b.to_dict_with_now(now) for b in bots],
'limit': limit,
'now': now,
}
self.send_response(utils.to_json_encodable(data))
def convertProposals(self, request, *args, **kwargs):
"""Start tasks to convert proposals for all organizations.
POST Args:
program_key: the key of the program whose proposals should be converted
org_cursor: the cursor indicating at which org we currently are
"""
params = dicts.merge(request.POST, request.GET)
if 'program_key' not in params:
logging.error("missing program_key in params: '%s'", params)
return responses.terminateTask()
program = GSoCProgram.get_by_key_name(params['program_key'])
if not program:
logging.error("invalid program_key in params: '%s'", params)
return responses.terminateTask()
query = soc_org_model.SOCOrganization.query(
soc_org_model.SOCOrganization.program ==
ndb.Key.from_old_key(program.key()),
soc_org_model.SOCOrganization.status == org_model.Status.ACCEPTED)
org_cursor = params.get('org_cursor')
start_cursor = (
datastore_query.Cursor(urlsafe=urllib.unquote_plus(org_cursor))
if org_cursor else None)
# Add a task for a single organization
organizations, next_cursor, _ = query.fetch_page(
1, start_cursor=start_cursor)
if organizations:
organization = organizations[0]
logging.info(
'Enqueing task to accept proposals for %s.', organization.name)
# Compounded accept/reject taskflow
taskqueue.add(
url='/tasks/gsoc/accept_proposals/accept',
params={
'org_key': organization.key.id(),
})
# Enqueue a new task to do the next organization
params['org_cursor'] = next_cursor.urlsafe()
taskqueue.add(url=request.path, params=params)
# Exit this task successfully
return responses.terminateTask()
def get_all_translation_opportunities(cls, page_size, urlsafe_start_cursor,
language_code):
"""Returns a list of opportunities available for translation in a
specific language.
Args:
page_size: int. The maximum number of entities to be returned.
urlsafe_start_cursor: str or None. If provided, the list of
returned entities starts from this datastore cursor.
Otherwise, the returned entities start from the beginning
of the full list of entities.
language_code: str. The language for which translation opportunities
are to be fetched.
Returns:
3-tuple of (results, cursor, more) as described in fetch_page() at:
https://developers.google.com/appengine/docs/python/ndb/queryclass,
where:
results: list(ExplorationOpportunitySummaryModel)|None. A list
of query results.
cursor: str or None. A query cursor pointing to the next
batch of results. If there are no more results, this might
be None.
more: bool. If True, there are (probably) more results after
this batch. If False, there are no further results after
this batch.
"""
if urlsafe_start_cursor:
start_cursor = datastore_query.Cursor(urlsafe=urlsafe_start_cursor)
else:
start_cursor = datastore_query.Cursor()
results, cursor, more = cls.query(
cls.incomplete_translation_language_codes == language_code).order(
cls.incomplete_translation_language_codes).fetch_page(
page_size, start_cursor=start_cursor)
return (results, (cursor.urlsafe() if cursor else None), more)
def list(self, request):
"""Provides list of known bots.
Deleted bots will not be listed.
"""
logging.info('%s', request)
now = utils.utcnow()
cursor = datastore_query.Cursor(urlsafe=request.cursor)
q = bot_management.BotInfo.query().order(bot_management.BotInfo.key)
bots, cursor, more = q.fetch_page(request.limit, start_cursor=cursor)
return swarming_rpcs.BotList(
cursor=cursor.urlsafe() if cursor and more else None,
death_timeout=config.settings().bot_death_timeout_secs,
items=[message_conversion.bot_info_to_rpc(bot, now) for bot in bots],
now=now)
def exportDb(self, cursor=None, key=None, *args, **kwargs):
if not self._checkKey(key, export=True):
raise errors.Forbidden()
if cursor:
c = datastore_query.Cursor(urlsafe=cursor)
else:
c = None
q = datastore.Query(None, cursor=c)
r = []
for res in q.Run(limit=5):
r.append(self.genDict(res))
return (pickle.dumps({
"cursor": str(q.GetCursor().urlsafe()),
"values": r
}).encode("HEX"))
def list(self, request):
"""Provides list of bots."""
now = utils.utcnow()
cursor = datastore_query.Cursor(urlsafe=request.cursor)
q = bot_management.BotInfo.query().order(bot_management.BotInfo.key)
bots, cursor, more = q.fetch_page(request.limit, start_cursor=cursor)
return swarming_rpcs.BotList(
cursor=cursor.urlsafe() if cursor and more else None,
death_timeout=config.settings().bot_death_timeout_secs,
items=[
message_conversion.bot_info_from_dict(
bot.to_dict_with_now(now)) for bot in bots
],
limit=request.limit,
now=now)
def get(self, bot_id):
limit = int(self.request.get('limit', 100))
cursor = datastore_query.Cursor(urlsafe=self.request.get('cursor'))
run_results, cursor, more = task_result.TaskRunResult.query(
task_result.TaskRunResult.bot_id == bot_id).order(
-task_result.TaskRunResult.started_ts).fetch_page(
limit, start_cursor=cursor)
now = utils.utcnow()
data = {
'cursor': cursor.urlsafe() if cursor and more else None,
'items': run_results,
'limit': limit,
'now': now,
}
self.send_response(utils.to_json_encodable(data))
