def _UpdateCache(test_key):
"""Queries Rows for a test then updates the cache.
Args:
test_key: ndb.Key for a TestMetadata entity.
Returns:
The list of triplets that was just fetched and set in the cache.
"""
test = test_key.get()
if not test:
return []
assert utils.IsInternalUser() or not test.internal_only
datastore_hooks.SetSinglePrivilegedRequest()
# A projection query queries just for the values of particular properties;
# this is faster than querying for whole entities.
query = graph_data.Row.query(projection=['revision', 'value', 'timestamp'])
query = query.filter(
graph_data.Row.parent_test == utils.OldStyleTestKey(test_key))
# Using a large batch_size speeds up queries with > 1000 Rows.
rows = map(_MakeTriplet, query.iter(batch_size=1000))
# Note: Unit tests do not call datastore_hooks with the above query, but
# it is called in production and with more recent SDK.
datastore_hooks.CancelSinglePrivilegedRequest()
SetCache(utils.TestPath(test_key), rows)
return rows
def post(self):
"""Query for tests, and put ones with no new data on the delete queue."""
datastore_hooks.SetPrivilegedRequest()
cursor = datastore_query.Cursor(urlsafe=self.request.get('cursor'))
tests, next_cursor, more = graph_data.TestMetadata.query().fetch_page(
_TESTS_TO_CHECK_AT_ONCE, keys_only=True, start_cursor=cursor)
if more:
taskqueue.add(url='/delete_old_tests',
params={'cursor': next_cursor.urlsafe()},
queue_name=_TASK_QUEUE_NAME)
for test in tests:
# Delete this test if:
# 1) It has no Rows newer than the cutoff
# 2) It has no descendant tests
no_new_rows = False
last_row = graph_data.Row.query(
graph_data.Row.parent_test == utils.OldStyleTestKey(
test)).order(-graph_data.Row.timestamp).get()
if last_row:
if last_row.timestamp < datetime.datetime.today(
) - _CUTOFF_DATE:
no_new_rows = True
else:
no_new_rows = True
descendants = list_tests.GetTestDescendants(test, keys_only=True)
descendants.remove(test)
if not descendants and no_new_rows:
taskqueue.add(
url='/delete_test_data',
params={
'test_path':
utils.TestPath(test), # For manual inspection.
'test_key': test.urlsafe(),
},
queue_name=_DELETE_TASK_QUEUE_NAME)
def GetRowsForTestBeforeAfterRev(test_key,
rev,
num_rows_before,
num_rows_after,
privileged=False):
"""Gets up to |num_points| Row entities for a test centered on a revision."""
test_key = utils.OldStyleTestKey(test_key)
if privileged:
datastore_hooks.SetSinglePrivilegedRequest()
query_up_to_rev = Row.query(Row.parent_test == test_key,
Row.revision <= rev)
query_up_to_rev = query_up_to_rev.order(-Row.revision)
rows_up_to_rev = list(
reversed(query_up_to_rev.fetch(limit=num_rows_before, batch_size=100)))
if privileged:
datastore_hooks.SetSinglePrivilegedRequest()
query_after_rev = Row.query(Row.parent_test == test_key,
Row.revision > rev)
query_after_rev = query_after_rev.order(Row.revision)
rows_after_rev = query_after_rev.fetch(limit=num_rows_after,
batch_size=100)
return rows_up_to_rev + rows_after_rev
def _AssertNotExists(self, test_paths):
for test_path in test_paths:
test_key = utils.TestKey(test_path)
num_rows = graph_data.Row.query(
graph_data.Row.parent_test == utils.OldStyleTestKey(test_key)).count()
self.assertEqual(0, num_rows)
self.assertIsNone(test_key.get())
def testOldStyleTestKey_String(self):
key = utils.OldStyleTestKey('m/b/suite/metric')
self.assertEqual('Test', key.kind())
self.assertEqual('metric', key.id())
self.assertEqual(
('Master', 'm', 'Bot', 'b', 'Test', 'suite', 'Test', 'metric'),
key.flat())
def GetRowsForTestInRange(test_key, start_rev, end_rev, privileged=False):
"""Gets all the Row entities for a test between a given start and end."""
test_key = utils.OldStyleTestKey(test_key)
if privileged:
datastore_hooks.SetSinglePrivilegedRequest()
query = Row.query(Row.parent_test == test_key, Row.revision >= start_rev,
Row.revision <= end_rev)
return query.fetch(batch_size=100)
def GetRowsForTestAroundRev(test_key, rev, num_points, privileged=False):
"""Gets up to |num_points| Row entities for a test centered on a revision."""
test_key = utils.OldStyleTestKey(test_key)
num_rows_before = int(num_points / 2) + 1
num_rows_after = int(num_points / 2)
return GetRowsForTestBeforeAfterRev(test_key, rev, num_rows_before,
num_rows_after, privileged)
def parent_test(self): # pylint: disable=invalid-name
# The Test entity that a Row belongs to isn't actually its parent in
# the datastore. Rather, the parent key of each Row contains a test path,
# which contains the information necessary to get the actual Test
# key. The Test key will need to be converted back to a new style
# TestMetadata key to get information back out. This is because we have
# over 3 trillion Rows in the datastore and cannot convert them all :(
return utils.OldStyleTestKey(
utils.TestKey(self.key.parent().string_id()))
def _HighestRevision(test_key):
"""Gets the revision number of the Row with the highest ID for a test."""
query = graph_data.Row.query(
graph_data.Row.parent_test == utils.OldStyleTestKey(test_key))
query = query.order(-graph_data.Row.revision)
highest_row_key = query.get(keys_only=True)
if highest_row_key:
return highest_row_key.id()
return None
def DeprecateTestsMapper(entity):
"""Marks a TestMetadata entity as deprecated if the last row is too old.
What is considered "too old" is defined by _DEPRECATION_REVISION_DELTA. Also,
if all of the subtests in a test have been marked as deprecated, then that
parent test will be marked as deprecated.
This mapper doesn't un-deprecate tests if new data has been added; that
happens in add_point.py.
Args:
entity: The TestMetadata entity to check.
Yields:
Zero or more datastore mutation operations.
"""
# Fetch the last row.
datastore_hooks.SetPrivilegedRequest()
query = graph_data.Row.query(
graph_data.Row.parent_test == utils.OldStyleTestKey(entity.key))
query = query.order(-graph_data.Row.timestamp)
last_row = query.get()
# Check if the test should be deleted entirely.
now = datetime.datetime.now()
logging.info('checking %s', entity.test_path)
if not last_row or last_row.timestamp < now - _REMOVAL_REVISON_DELTA:
descendants = list_tests.GetTestDescendants(entity.key, keys_only=True)
if entity.key in descendants:
descendants.remove(entity.key)
if not descendants:
logging.info('removing')
if last_row:
logging.info('last row timestamp: %s', last_row.timestamp)
else:
logging.info('no last row, no descendants')
taskqueue.add(
url='/delete_test_data',
params={
'test_path':
utils.TestPath(entity.key), # For manual inspection.
'test_key': entity.key.urlsafe(),
'notify': 'false',
},
queue_name=_DELETE_TASK_QUEUE_NAME)
return
if entity.deprecated or not last_row:
return
if last_row.timestamp < now - _DEPRECATION_REVISION_DELTA:
for operation in _MarkDeprecated(entity):
yield operation
for operation in _CreateStoppageAlerts(entity, last_row):
yield operation
def testGetAlertsForTest(self):
old_style_key1 = utils.OldStyleTestKey('master/bot/test1/metric')
new_style_key1 = utils.TestMetadataKey('master/bot/test1/metric')
old_style_key2 = utils.OldStyleTestKey('master/bot/test2/metric')
new_style_key2 = utils.TestMetadataKey('master/bot/test2/metric')
anomaly.Anomaly(id="old_1", test=old_style_key1).put()
anomaly.Anomaly(id="old_1a", test=old_style_key1).put()
anomaly.Anomaly(id="old_2", test=old_style_key2).put()
anomaly.Anomaly(id="new_1", test=new_style_key1).put()
anomaly.Anomaly(id="new_2", test=new_style_key2).put()
anomaly.Anomaly(id="new_2a", test=new_style_key2).put()
key1_alerts = anomaly.Anomaly.GetAlertsForTest(new_style_key1)
self.assertEqual(['new_1', 'old_1', 'old_1a'],
[a.key.id() for a in key1_alerts])
key2_alerts = anomaly.Anomaly.GetAlertsForTest(old_style_key2)
self.assertEqual(['new_2', 'new_2a', 'old_2'],
[a.key.id() for a in key2_alerts])
key2_alerts_limit = anomaly.Anomaly.GetAlertsForTest(old_style_key2,
limit=2)
self.assertEqual(['new_2', 'new_2a'],
[a.key.id() for a in key2_alerts_limit])
def GetLatestRowsForTest(test_key,
num_points,
keys_only=False,
privileged=False):
"""Gets the latest num_points Row entities for a test."""
test_key = utils.OldStyleTestKey(test_key)
if privileged:
datastore_hooks.SetSinglePrivilegedRequest()
query = Row.query(Row.parent_test == test_key)
query = query.order(-Row.revision)
return query.fetch(limit=num_points, batch_size=100, keys_only=keys_only)
def _FetchRowsAsync(self, test_keys, num_points):
"""Fetches recent Row asynchronously across all 'test_keys'."""
rows = []
futures = []
for test_key in test_keys:
q = graph_data.Row.query()
q = q.filter(
graph_data.Row.parent_test == utils.OldStyleTestKey(test_key))
q = q.order(-graph_data.Row.revision)
futures.append(q.fetch_async(limit=num_points))
ndb.Future.wait_all(futures)
for future in futures:
rows.extend(future.get_result())
return rows
def testDeprecateTestsMapper_AllSubtestsDeprecated_UpdatesSuite(self):
(trace_a, trace_b, suite) = self._AddMockDataForDeprecatedTests()
last_b = graph_data.Row.query(
graph_data.Row.parent_test == utils.OldStyleTestKey(trace_b.key),
graph_data.Row.revision == 4).get()
last_b.timestamp = datetime.datetime.now() - datetime.timedelta(days=20)
last_b.put()
for operation in mr.DeprecateTestsMapper(trace_a):
self._ExecOperation(operation)
for operation in mr.DeprecateTestsMapper(trace_b):
self._ExecOperation(operation)
self.assertTrue(trace_a.deprecated)
self.assertTrue(trace_b.deprecated)
self.assertTrue(suite.deprecated)
def _DumpTestData(self):
"""Dumps data for the requested test.
Request parameters:
test_path: A single full test path, including master/bot.
num_points: Max number of Row entities (optional).
end_rev: Ending revision number, inclusive (optional).
Outputs:
JSON array of encoded protobuf messages, which encode all of
the datastore entities relating to one test (including Master, Bot,
TestMetadata, Row, Anomaly and Sheriff entities).
"""
test_path = self.request.get('test_path')
num_points = int(self.request.get('num_points', _DEFAULT_MAX_POINTS))
end_rev = self.request.get('end_rev')
test_key = utils.TestKey(test_path)
if not test_key or test_key.kind() != 'TestMetadata':
# Bad test_path passed in.
self.response.out.write(json.dumps([]))
return
# List of datastore entities that will be dumped.
entities = []
entities.extend(self._GetTestAncestors([test_key]))
# Get the Row entities.
q = graph_data.Row.query()
print test_key
q = q.filter(
graph_data.Row.parent_test == utils.OldStyleTestKey(test_key))
if end_rev:
q = q.filter(graph_data.Row.revision <= int(end_rev))
q = q.order(-graph_data.Row.revision)
entities += q.fetch(limit=num_points)
# Get the Anomaly and Sheriff entities.
alerts = anomaly.Anomaly.GetAlertsForTest(test_key)
sheriff_keys = {alert.sheriff for alert in alerts}
sheriffs = [sheriff.get() for sheriff in sheriff_keys]
entities += alerts
entities += sheriffs
# Convert the entities to protobuf message strings and output as JSON.
protobuf_strings = map(EntityToBinaryProtobuf, entities)
self.response.out.write(json.dumps(protobuf_strings))
def _CheckRows(self, test_path, multiplier=2):
"""Checks the rows match the expected sample data for a given test.
The expected revisions that should be present are based on the sample data
added in _AddMockData.
Args:
test_path: Test path of the test to get rows for.
multiplier: Number to multiply with revision to get expected value.
"""
rows = graph_data.Row.query(
graph_data.Row.parent_test == utils.OldStyleTestKey(test_path)).fetch()
self.assertEqual(50, len(rows))
self.assertEqual(15000, rows[0].revision)
self.assertEqual(15000 * multiplier, rows[0].value)
self.assertEqual(15098, rows[49].revision)
self.assertEqual(15098 * multiplier, rows[49].value)
def get(self):
"""Gets CSV from data store and outputs it.
Request parameters:
test_path: Full test path of one trace.
rev: End revision number; if not given, latest revision is used.
num_points: Number of Rows to get data for.
attr: Comma-separated list of attributes (columns) to return.
Outputs:
CSV file contents.
"""
test_path = self.request.get('test_path')
rev = self.request.get('rev')
num_points = int(self.request.get('num_points', 500))
attributes = self.request.get('attr', 'revision,value').split(',')
if not test_path:
self.ReportError('No test path given.', status=400)
return
logging.info('Got request to /graph_csv for test: "%s".', test_path)
test_key = utils.TestKey(test_path)
test = test_key.get()
assert (datastore_hooks.IsUnalteredQueryPermitted()
or not test.internal_only)
datastore_hooks.SetSinglePrivilegedRequest()
q = graph_data.Row.query()
q = q.filter(
graph_data.Row.parent_test == utils.OldStyleTestKey(test_key))
if rev:
q = q.filter(graph_data.Row.revision <= int(rev))
q = q.order(-graph_data.Row.revision)
points = reversed(q.fetch(limit=num_points))
rows = self._GenerateRows(points, attributes)
output = StringIO.StringIO()
csv.writer(output).writerows(rows)
self.response.headers['Content-Type'] = 'text/csv'
self.response.headers['Content-Disposition'] = (
'attachment; filename=%s.csv' % test.test_name)
self.response.out.write(output.getvalue())
def testGet_ThreeAlertsOneSheriff_EmailSent(self):
self._AddSampleData()
for name in ('foo', 'bar', 'baz'):
test = utils.TestKey('M/b/suite/%s' % name).get()
row = graph_data.Row.query(graph_data.Row.parent_test ==
utils.OldStyleTestKey(test.key)).get()
stoppage_alert.CreateStoppageAlert(test, row).put()
self.testapp.get('/send_stoppage_alert_emails')
messages = self.mail_stub.get_sent_messages()
self.assertEqual(1, len(messages))
self.assertEqual('*****@*****.**', messages[0].sender)
self.assertEqual('*****@*****.**', messages[0].to)
self.assertIn('3', messages[0].subject)
self.assertIn('foo', str(messages[0].body))
self.assertIn('bar', str(messages[0].body))
self.assertIn('baz', str(messages[0].body))
stoppage_alerts = stoppage_alert.StoppageAlert.query().fetch()
for alert in stoppage_alerts:
self.assertTrue(alert.mail_sent)
def DeprecateTestsMapper(entity):
"""Marks a TestMetadata entity as deprecated if the last row is too old.
What is considered "too old" is defined by _OLDEST_REVISION_DELTA. Also,
if all of the subtests in a test have been marked as deprecated, then that
parent test will be marked as deprecated.
This mapper doesn't un-deprecate tests if new data has been added; that
happens in add_point.py.
Args:
entity: The TestMetadata entity to check.
Yields:
Zero or more datastore mutation operations.
"""
# Make sure that we have a non-deprecated TestMetadata with Rows.
if (entity.key.kind() != 'TestMetadata' or not entity.has_rows
or entity.deprecated):
# TODO(qyearsley): Add test coverage. See catapult:#1346.
logging.error(
'Got bad entity in mapreduce! Kind: %s, has_rows: %s, deprecated: %s',
entity.key.kind(), entity.has_rows, entity.deprecated)
return
# Fetch the last row.
datastore_hooks.SetPrivilegedRequest()
query = graph_data.Row.query(
graph_data.Row.parent_test == utils.OldStyleTestKey(entity.key))
query = query.order(-graph_data.Row.timestamp)
last_row = query.get()
if not last_row:
# TODO(qyearsley): Add test coverage. See catapult:#1346.
logging.error('No rows for %s (but has_rows=True)', entity.key)
return
now = datetime.datetime.now()
if last_row.timestamp < now - _OLDEST_REVISION_DELTA:
for operation in _MarkDeprecated(entity):
yield operation
for operation in _CreateStoppageAlerts(entity, last_row):
yield operation
def _UpdateTest(self, test_key_urlsafe, internal_only, cursor=None):
"""Updates the given TestMetadata and associated Row entities."""
test_key = ndb.Key(urlsafe=test_key_urlsafe)
if not cursor:
# First time updating for this TestMetadata.
test_entity = test_key.get()
if test_entity.internal_only != internal_only:
test_entity.internal_only = internal_only
test_entity.put()
# Update all of the Anomaly entities for this test.
# Assuming that this should be fast enough to do in one request
# for any one test.
anomalies = anomaly.Anomaly.GetAlertsForTest(test_key)
for anomaly_entity in anomalies:
if anomaly_entity.internal_only != internal_only:
anomaly_entity.internal_only = internal_only
ndb.put_multi(anomalies)
else:
cursor = datastore_query.Cursor(urlsafe=cursor)
# Fetch a certain number of Row entities starting from cursor.
rows_query = graph_data.Row.query(
graph_data.Row.parent_test == utils.OldStyleTestKey(test_key))
rows, next_cursor, more = rows_query.fetch_page(_MAX_ROWS_TO_PUT,
start_cursor=cursor)
for row in rows:
if row.internal_only != internal_only:
row.internal_only = internal_only
ndb.put_multi(rows)
if more:
taskqueue.add(url='/change_internal_only',
params={
'test':
test_key_urlsafe,
'cursor':
next_cursor.urlsafe(),
'internal_only':
'true' if internal_only else 'false',
},
queue_name=_QUEUE_NAME)
def GetRowsToAnalyze(test, max_num_rows):
"""Gets the Row entities that we want to analyze.
Args:
test: The TestMetadata entity to get data for.
max_num_rows: The maximum number of points to get.
Returns:
A list of the latest Rows after the last alerted revision, ordered by
revision. These rows are fetched with t a projection query so they only
have the revision and value properties.
"""
query = graph_data.Row.query(projection=['revision', 'value'])
query = query.filter(
graph_data.Row.parent_test == utils.OldStyleTestKey(test.key))
# The query is ordered in descending order by revision because we want
# to get the newest points.
query = query.filter(graph_data.Row.revision > test.last_alerted_revision)
query = query.order(-graph_data.Row.revision)
# However, we want to analyze them in ascending order.
return list(reversed(query.fetch(limit=max_num_rows)))
def testGetGraphJson_WithSelectedTrace(self):
self._AddTestColumns(start_rev=15000, end_rev=15050)
rows = graph_data.Row.query(
graph_data.Row.parent_test == utils.OldStyleTestKey(
'ChromiumGPU/win7/dromaeo/jslib')).fetch()
for row in rows:
row.error = 1 + ((row.revision - 15000) * 0.25)
ndb.put_multi(rows)
flot_json_str = graph_json.GetGraphJson(
{
'ChromiumGPU/win7/dromaeo/jslib': ['jslib'],
},
rev=15000,
num_points=8,
is_selected=True)
flot = json.loads(flot_json_str)
self.assertEqual(1, len(flot['data']))
self.assertEqual(5, len(flot['data']['0']['data']))
self.assertEqual(1, len(flot['annotations']['series']))
self.assertEqual(5, len(flot['annotations'].get('0').keys()))
self.assertEqual(5, len(flot['error_bars']['0'][0]['data']))
self.assertEqual(5, len(flot['error_bars']['0'][1]['data']))
def testOldStyleTestKey_Test(self):
original_key = ndb.Key('Master', 'm', 'Bot', 'b', 'Test', 'suite',
'Test', 'metric')
key = utils.OldStyleTestKey(original_key)
self.assertEqual(original_key, key)
def testOldStyleTestKey_None(self):
key = utils.OldStyleTestKey(None)
self.assertIsNone(key)
def GetAlertsForTest(cls, test_key, limit=None):
return cls.query(
cls.test.IN([
utils.TestMetadataKey(test_key),
utils.OldStyleTestKey(test_key)
])).fetch(limit=limit)
