def GetRowsForTestBeforeAfterRev(test_key,
rev,
num_rows_before,
num_rows_after,
privileged=False):
"""Gets up to |num_points| Row entities for a test centered on a revision."""
test_key = utils.OldStyleTestKey(test_key)
if privileged:
datastore_hooks.SetSinglePrivilegedRequest()
query_up_to_rev = Row.query(Row.parent_test == test_key,
Row.revision <= rev)
query_up_to_rev = query_up_to_rev.order(-Row.revision)
rows_up_to_rev = list(
reversed(query_up_to_rev.fetch(limit=num_rows_before, batch_size=100)))
if privileged:
datastore_hooks.SetSinglePrivilegedRequest()
query_after_rev = Row.query(Row.parent_test == test_key,
Row.revision > rev)
query_after_rev = query_after_rev.order(Row.revision)
rows_after_rev = query_after_rev.fetch(limit=num_rows_after,
batch_size=100)
return rows_up_to_rev + rows_after_rev
def _UpdateCache(test_key):
"""Queries Rows for a test then updates the cache.
Args:
test_key: ndb.Key for a TestMetadata entity.
Returns:
The list of triplets that was just fetched and set in the cache.
"""
test = test_key.get()
if not test:
return []
assert utils.IsInternalUser() or not test.internal_only
datastore_hooks.SetSinglePrivilegedRequest()
# A projection query queries just for the values of particular properties;
# this is faster than querying for whole entities.
query = graph_data.Row.query(projection=['revision', 'value', 'timestamp'])
query = query.filter(
graph_data.Row.parent_test == utils.OldStyleTestKey(test_key))
# Using a large batch_size speeds up queries with > 1000 Rows.
rows = map(_MakeTriplet, query.iter(batch_size=1000))
# Note: Unit tests do not call datastore_hooks with the above query, but
# it is called in production and with more recent SDK.
datastore_hooks.CancelSinglePrivilegedRequest()
SetCache(utils.TestPath(test_key), rows)
return rows
def _UpdateNewestRevInMilestoneDict(bots, tests, milestone_dict):
"""Updates the most recent rev in the milestone dict.
The global milestone dicts are declared with 'None' for the end of the
current milestone range. If we might be using the last milestone, update
the end of the current milestone range to be the most recent revision.
"""
if bots and tests:
test_path = bots[0] + '/' + tests[0]
test_key = utils.TestKey(test_path)
# Need to allow set this request as privileged in order to bypass datastore
# hooks. This is okay here because table_config is internal_only protected
# and will ensure that only the correct users can see internal_only data.
datastore_hooks.SetSinglePrivilegedRequest()
query = graph_data.Row.query()
query = query.filter(
graph_data.Row.parent_test == utils.OldStyleTestKey(test_key))
query = query.order(-graph_data.Row.revision)
row = query.get()
if row:
milestone_dict[CURRENT_MILESTONE] = (
milestone_dict[CURRENT_MILESTONE][0], row.revision)
else:
milestone_dict[CURRENT_MILESTONE] = (
milestone_dict[CURRENT_MILESTONE][0],
milestone_dict[CURRENT_MILESTONE][0])
def GetRowsForTestInRange(test_key, start_rev, end_rev, privileged=False):
"""Gets all the Row entities for a test between a given start and end."""
test_key = utils.OldStyleTestKey(test_key)
if privileged:
datastore_hooks.SetSinglePrivilegedRequest()
query = Row.query(Row.parent_test == test_key, Row.revision >= start_rev,
Row.revision <= end_rev)
return query.fetch(batch_size=100)
def GetLatestRowsForTest(
test_key, num_points, keys_only=False, privileged=False):
"""Gets the latest num_points Row entities for a test."""
test_key = utils.OldStyleTestKey(test_key)
if privileged:
datastore_hooks.SetSinglePrivilegedRequest()
query = Row.query(Row.parent_test == test_key)
query = query.order(-Row.revision)
return query.fetch(limit=num_points, batch_size=100, keys_only=keys_only)
def testQuery_SinglePrivilegedRequest_InternalOnlyFetched(self):
self.UnsetCurrentUser()
datastore_hooks.SetSinglePrivilegedRequest()
# Not using _CheckQueryResults because this only affects a single query.
# First query has internal results.
bots = graph_data.Bot.query().fetch()
self.assertEqual(2, len(bots))
# Second query does not.
bots = graph_data.Bot.query().fetch()
self.assertEqual(1, len(bots))
def AuthorizedPost(self, *args):
"""Returns timeseries data in response to API requests.
Argument:
test_path: Full path of test timeseries
Outputs:
JSON timeseries data for the test_path, see README.md.
"""
try:
days = int(self.request.get('num_days', 30))
except ValueError:
raise api_request_handler.BadRequestError(
'Invalid num_days parameter %s' % self.request.get('num_days'))
if days <= 0:
raise api_request_handler.BadRequestError(
'num_days cannot be negative (%s)' % days)
before = datetime.datetime.now() - datetime.timedelta(days=days)
test_path = args[0]
test_key = utils.TestKey(test_path)
test = test_key.get()
if not test:
raise api_request_handler.BadRequestError('Invalid test_path %s' %
test_path)
assert (datastore_hooks.IsUnalteredQueryPermitted()
or not test.internal_only)
datastore_hooks.SetSinglePrivilegedRequest()
q = graph_data.Row.query()
q = q.filter(
graph_data.Row.parent_test == utils.OldStyleTestKey(test_key))
q = q.filter(graph_data.Row.timestamp > before)
rows = q.fetch()
if not rows:
return []
revisions = [rev for rev in rows[0].to_dict() if rev.startswith('r_')]
header = ['revision', 'value', 'timestamp'] + revisions
timeseries = [header]
for row in sorted(rows, key=lambda r: r.revision):
timeseries.append([self._GetValue(row, a) for a in header])
return {
'timeseries': timeseries,
'test_path': test_path,
'revision_logs': namespaced_stored_object.Get('revision_info'),
'improvement_direction': test.improvement_direction,
}
def _FetchLatestRows(test, num_points):
"""Does a query for the latest Row entities in the given test.
Args:
test: A TestMetadata entity to fetch Row entities for.
num_points: Number of points to fetch.
Returns:
A list of Row entities, ordered by revision. The number to fetch is limited
to the number that is expected to be processed at once by GASP.
"""
assert utils.IsInternalUser() or not test.internal_only
datastore_hooks.SetSinglePrivilegedRequest()
return list(reversed(graph_data.GetLatestRowsForTest(test.key, num_points)))
def get(self):
"""Gets CSV from data store and outputs it.
Request parameters:
test_path: Full test path of one trace.
rev: End revision number; if not given, latest revision is used.
num_points: Number of Rows to get data for.
attr: Comma-separated list of attributes (columns) to return.
Outputs:
CSV file contents.
"""
test_path = self.request.get('test_path')
rev = self.request.get('rev')
num_points = int(self.request.get('num_points', 500))
attributes = self.request.get('attr', 'revision,value').split(',')
if not test_path:
self.ReportError('No test path given.', status=400)
return
logging.info('Got request to /graph_csv for test: "%s".', test_path)
test_key = utils.TestKey(test_path)
test = test_key.get()
assert (datastore_hooks.IsUnalteredQueryPermitted()
or not test.internal_only)
datastore_hooks.SetSinglePrivilegedRequest()
q = graph_data.Row.query()
q = q.filter(
graph_data.Row.parent_test == utils.OldStyleTestKey(test_key))
if rev:
q = q.filter(graph_data.Row.revision <= int(rev))
q = q.order(-graph_data.Row.revision)
points = reversed(q.fetch(limit=num_points))
rows = self._GenerateRows(points, attributes)
output = StringIO.StringIO()
csv.writer(output).writerows(rows)
self.response.headers['Content-Type'] = 'text/csv'
self.response.headers['Content-Disposition'] = (
'attachment; filename=%s.csv' % test.test_name)
self.response.out.write(output.getvalue())
