def _GetLatestRowsForTest(test_key, num_points, privileged=False):
"""Gets the latest num_points Row entities for a Test."""
if privileged:
datastore_hooks.SetSinglePrivilegedRequest()
query = graph_data.Row.query(graph_data.Row.parent_test == test_key)
query = query.order(-graph_data.Row.revision)
return query.fetch(limit=num_points, batch_size=100)
def _UpdateCache(test_key):
"""Queries Rows for a test then updates the cache.
Args:
test_key: ndb.Key for a Test entity.
Returns:
The list of triplets that was just fetched and set in the cache.
"""
test = test_key.get()
if not test:
return []
assert utils.IsInternalUser() or not test.internal_only
datastore_hooks.SetSinglePrivilegedRequest()
# A projection query queries just for the values of particular properties;
# this is faster than querying for whole entities.
query = graph_data.Row.query(projection=['revision', 'value', 'timestamp'])
query = query.filter(graph_data.Row.parent_test == test_key)
# Using a large batch_size speeds up queries with > 1000 Rows.
rows = map(_MakeTriplet, query.iter(batch_size=1000))
# Note: Unit tests do not call datastore_hooks with the above query, but
# it is called in production and with more recent SDK.
datastore_hooks.CancelSinglePrivilegedRequest()
SetCache(utils.TestPath(test_key), rows)
return rows
def _GetRowsForTestInRange(test_key, start_rev, end_rev, privileged=False):
"""Gets all the Row entities for a Test between a given start and end."""
if privileged:
datastore_hooks.SetSinglePrivilegedRequest()
query = graph_data.Row.query(graph_data.Row.parent_test == test_key,
graph_data.Row.revision >= start_rev,
graph_data.Row.revision <= end_rev)
return query.fetch(batch_size=100)
def GetRowsForTestInRange(test_key, start_rev, end_rev, privileged=False):
"""Gets all the Row entities for a test between a given start and end."""
test_key = utils.OldStyleTestKey(test_key)
if privileged:
datastore_hooks.SetSinglePrivilegedRequest()
query = Row.query(Row.parent_test == test_key, Row.revision >= start_rev,
Row.revision <= end_rev)
return query.fetch(batch_size=100)
def testQuery_SinglePrivilegedRequest_InternalOnlyFetched(self): self.UnsetCurrentUser() datastore_hooks.SetSinglePrivilegedRequest() # Not using _CheckQueryResults because this only affects a single query. # First query has internal results. rows = graph_data.Row.query().filter(graph_data.Row.value == 20).fetch() self.assertEqual(2, len(rows)) # Second query does not. rows = graph_data.Row.query().filter(graph_data.Row.value == 20).fetch() self.assertEqual(1, len(rows))
def GetLatestRowsForTest(test_key,
num_points,
keys_only=False,
privileged=False):
"""Gets the latest num_points Row entities for a test."""
test_key = utils.OldStyleTestKey(test_key)
if privileged:
datastore_hooks.SetSinglePrivilegedRequest()
query = Row.query(Row.parent_test == test_key)
query = query.order(-Row.revision)
return query.fetch(limit=num_points, batch_size=100, keys_only=keys_only)
def _GetRowsForTestAroundRev(test_key, rev, num_points, privileged=False):
"""Gets up to num_points Row entities for a Test centered on a revision."""
num_rows_before = int(num_points / 2) + 1
num_rows_after = int(num_points / 2)
if privileged:
datastore_hooks.SetSinglePrivilegedRequest()
query_up_to_rev = graph_data.Row.query(
graph_data.Row.parent_test == test_key, graph_data.Row.revision <= rev)
query_up_to_rev = query_up_to_rev.order(-graph_data.Row.revision)
rows_up_to_rev = query_up_to_rev.fetch(limit=num_rows_before,
batch_size=100)
if privileged:
datastore_hooks.SetSinglePrivilegedRequest()
query_after_rev = graph_data.Row.query(
graph_data.Row.parent_test == test_key, graph_data.Row.revision > rev)
query_after_rev = query_after_rev.order(graph_data.Row.revision)
rows_after_rev = query_after_rev.fetch(limit=num_rows_after,
batch_size=100)
return rows_up_to_rev + rows_after_rev
def _FetchLatestRows(test, num_points):
"""Does a query for the latest Row entities in the given test.
Args:
test: A Test entity to fetch Row entities for.
num_points: Number of points to fetch.
Returns:
A list of Row entities, ordered by revision. The number to fetch is limited
to the number that is expected to be processed at once by GASP.
"""
assert utils.IsInternalUser() or not test.internal_only
datastore_hooks.SetSinglePrivilegedRequest()
q = graph_data.Row.query(projection=['revision', 'value'])
q = q.filter(graph_data.Row.parent_test == test.key)
q = q.order(-graph_data.Row.revision)
rows = list(reversed(q.fetch(limit=num_points)))
return rows
def _FetchLatestRows(test, num_points):
"""Does a query for the latest Row entities in the given test.
Args:
test: A TestMetadata entity to fetch Row entities for.
num_points: Number of points to fetch.
Returns:
A list of Row entities, ordered by revision. The number to fetch is limited
to the number that is expected to be processed at once by GASP.
"""
assert utils.IsInternalUser() or not test.internal_only
datastore_hooks.SetSinglePrivilegedRequest()
return list(
reversed(
graph_data.GetLatestRowsForTest(test.key,
num_points,
privileged=True)))
def get(self):
"""Gets CSV from data store and outputs it.
Request parameters:
test_path: Full test path of one trace.
rev: End revision number; if not given, latest revision is used.
num_points: Number of Rows to get data for.
attr: Comma-separated list of attributes (columns) to return.
Outputs:
CSV file contents.
"""
test_path = self.request.get('test_path')
rev = self.request.get('rev')
num_points = int(self.request.get('num_points', 500))
attributes = self.request.get('attr', 'revision,value').split(',')
if not test_path:
self.ReportError('No test path given.', status=400)
return
logging.info('Got request to /graph_csv for test: "%s".', test_path)
test_key = utils.TestKey(test_path)
test = test_key.get()
assert (datastore_hooks.IsUnalteredQueryPermitted()
or not test.internal_only)
datastore_hooks.SetSinglePrivilegedRequest()
q = graph_data.Row.query()
q = q.filter(
graph_data.Row.parent_test == utils.OldStyleTestKey(test_key))
if rev:
q = q.filter(graph_data.Row.revision <= int(rev))
q = q.order(-graph_data.Row.revision)
points = reversed(q.fetch(limit=num_points))
rows = self._GenerateRows(points, attributes)
output = StringIO.StringIO()
csv.writer(output).writerows(rows)
self.response.headers['Content-Type'] = 'text/csv'
self.response.headers['Content-Disposition'] = (
'attachment; filename=%s.csv' % test.test_name)
self.response.out.write(output.getvalue())