def _UpdateCache(test_key): """Queries Rows for a test then updates the cache. Args: test_key: ndb.Key for a TestMetadata entity. Returns: The list of triplets that was just fetched and set in the cache. """ test = test_key.get() if not test: return [] assert utils.IsInternalUser() or not test.internal_only datastore_hooks.SetSinglePrivilegedRequest() # A projection query queries just for the values of particular properties; # this is faster than querying for whole entities. query = graph_data.Row.query(projection=['revision', 'value', 'timestamp']) query = query.filter( graph_data.Row.parent_test == utils.OldStyleTestKey(test_key)) # Using a large batch_size speeds up queries with > 1000 Rows. rows = map(_MakeTriplet, query.iter(batch_size=1000)) # Note: Unit tests do not call datastore_hooks with the above query, but # it is called in production and with more recent SDK. datastore_hooks.CancelSinglePrivilegedRequest() SetCache(utils.TestPath(test_key), rows) return rows
def post(self): """Query for tests, and put ones with no new data on the delete queue.""" datastore_hooks.SetPrivilegedRequest() cursor = datastore_query.Cursor(urlsafe=self.request.get('cursor')) tests, next_cursor, more = graph_data.TestMetadata.query().fetch_page( _TESTS_TO_CHECK_AT_ONCE, keys_only=True, start_cursor=cursor) if more: taskqueue.add(url='/delete_old_tests', params={'cursor': next_cursor.urlsafe()}, queue_name=_TASK_QUEUE_NAME) for test in tests: # Delete this test if: # 1) It has no Rows newer than the cutoff # 2) It has no descendant tests no_new_rows = False last_row = graph_data.Row.query( graph_data.Row.parent_test == utils.OldStyleTestKey( test)).order(-graph_data.Row.timestamp).get() if last_row: if last_row.timestamp < datetime.datetime.today( ) - _CUTOFF_DATE: no_new_rows = True else: no_new_rows = True descendants = list_tests.GetTestDescendants(test, keys_only=True) descendants.remove(test) if not descendants and no_new_rows: taskqueue.add( url='/delete_test_data', params={ 'test_path': utils.TestPath(test), # For manual inspection. 'test_key': test.urlsafe(), }, queue_name=_DELETE_TASK_QUEUE_NAME)
def GetRowsForTestBeforeAfterRev(test_key, rev, num_rows_before, num_rows_after, privileged=False): """Gets up to |num_points| Row entities for a test centered on a revision.""" test_key = utils.OldStyleTestKey(test_key) if privileged: datastore_hooks.SetSinglePrivilegedRequest() query_up_to_rev = Row.query(Row.parent_test == test_key, Row.revision <= rev) query_up_to_rev = query_up_to_rev.order(-Row.revision) rows_up_to_rev = list( reversed(query_up_to_rev.fetch(limit=num_rows_before, batch_size=100))) if privileged: datastore_hooks.SetSinglePrivilegedRequest() query_after_rev = Row.query(Row.parent_test == test_key, Row.revision > rev) query_after_rev = query_after_rev.order(Row.revision) rows_after_rev = query_after_rev.fetch(limit=num_rows_after, batch_size=100) return rows_up_to_rev + rows_after_rev
def _AssertNotExists(self, test_paths): for test_path in test_paths: test_key = utils.TestKey(test_path) num_rows = graph_data.Row.query( graph_data.Row.parent_test == utils.OldStyleTestKey(test_key)).count() self.assertEqual(0, num_rows) self.assertIsNone(test_key.get())
def testOldStyleTestKey_String(self): key = utils.OldStyleTestKey('m/b/suite/metric') self.assertEqual('Test', key.kind()) self.assertEqual('metric', key.id()) self.assertEqual( ('Master', 'm', 'Bot', 'b', 'Test', 'suite', 'Test', 'metric'), key.flat())
def GetRowsForTestInRange(test_key, start_rev, end_rev, privileged=False): """Gets all the Row entities for a test between a given start and end.""" test_key = utils.OldStyleTestKey(test_key) if privileged: datastore_hooks.SetSinglePrivilegedRequest() query = Row.query(Row.parent_test == test_key, Row.revision >= start_rev, Row.revision <= end_rev) return query.fetch(batch_size=100)
def GetRowsForTestAroundRev(test_key, rev, num_points, privileged=False): """Gets up to |num_points| Row entities for a test centered on a revision.""" test_key = utils.OldStyleTestKey(test_key) num_rows_before = int(num_points / 2) + 1 num_rows_after = int(num_points / 2) return GetRowsForTestBeforeAfterRev(test_key, rev, num_rows_before, num_rows_after, privileged)
def parent_test(self): # pylint: disable=invalid-name # The Test entity that a Row belongs to isn't actually its parent in # the datastore. Rather, the parent key of each Row contains a test path, # which contains the information necessary to get the actual Test # key. The Test key will need to be converted back to a new style # TestMetadata key to get information back out. This is because we have # over 3 trillion Rows in the datastore and cannot convert them all :( return utils.OldStyleTestKey( utils.TestKey(self.key.parent().string_id()))
def _HighestRevision(test_key): """Gets the revision number of the Row with the highest ID for a test.""" query = graph_data.Row.query( graph_data.Row.parent_test == utils.OldStyleTestKey(test_key)) query = query.order(-graph_data.Row.revision) highest_row_key = query.get(keys_only=True) if highest_row_key: return highest_row_key.id() return None
def DeprecateTestsMapper(entity): """Marks a TestMetadata entity as deprecated if the last row is too old. What is considered "too old" is defined by _DEPRECATION_REVISION_DELTA. Also, if all of the subtests in a test have been marked as deprecated, then that parent test will be marked as deprecated. This mapper doesn't un-deprecate tests if new data has been added; that happens in add_point.py. Args: entity: The TestMetadata entity to check. Yields: Zero or more datastore mutation operations. """ # Fetch the last row. datastore_hooks.SetPrivilegedRequest() query = graph_data.Row.query( graph_data.Row.parent_test == utils.OldStyleTestKey(entity.key)) query = query.order(-graph_data.Row.timestamp) last_row = query.get() # Check if the test should be deleted entirely. now = datetime.datetime.now() logging.info('checking %s', entity.test_path) if not last_row or last_row.timestamp < now - _REMOVAL_REVISON_DELTA: descendants = list_tests.GetTestDescendants(entity.key, keys_only=True) if entity.key in descendants: descendants.remove(entity.key) if not descendants: logging.info('removing') if last_row: logging.info('last row timestamp: %s', last_row.timestamp) else: logging.info('no last row, no descendants') taskqueue.add( url='/delete_test_data', params={ 'test_path': utils.TestPath(entity.key), # For manual inspection. 'test_key': entity.key.urlsafe(), 'notify': 'false', }, queue_name=_DELETE_TASK_QUEUE_NAME) return if entity.deprecated or not last_row: return if last_row.timestamp < now - _DEPRECATION_REVISION_DELTA: for operation in _MarkDeprecated(entity): yield operation for operation in _CreateStoppageAlerts(entity, last_row): yield operation
def testGetAlertsForTest(self): old_style_key1 = utils.OldStyleTestKey('master/bot/test1/metric') new_style_key1 = utils.TestMetadataKey('master/bot/test1/metric') old_style_key2 = utils.OldStyleTestKey('master/bot/test2/metric') new_style_key2 = utils.TestMetadataKey('master/bot/test2/metric') anomaly.Anomaly(id="old_1", test=old_style_key1).put() anomaly.Anomaly(id="old_1a", test=old_style_key1).put() anomaly.Anomaly(id="old_2", test=old_style_key2).put() anomaly.Anomaly(id="new_1", test=new_style_key1).put() anomaly.Anomaly(id="new_2", test=new_style_key2).put() anomaly.Anomaly(id="new_2a", test=new_style_key2).put() key1_alerts = anomaly.Anomaly.GetAlertsForTest(new_style_key1) self.assertEqual(['new_1', 'old_1', 'old_1a'], [a.key.id() for a in key1_alerts]) key2_alerts = anomaly.Anomaly.GetAlertsForTest(old_style_key2) self.assertEqual(['new_2', 'new_2a', 'old_2'], [a.key.id() for a in key2_alerts]) key2_alerts_limit = anomaly.Anomaly.GetAlertsForTest(old_style_key2, limit=2) self.assertEqual(['new_2', 'new_2a'], [a.key.id() for a in key2_alerts_limit])
def GetLatestRowsForTest(test_key, num_points, keys_only=False, privileged=False): """Gets the latest num_points Row entities for a test.""" test_key = utils.OldStyleTestKey(test_key) if privileged: datastore_hooks.SetSinglePrivilegedRequest() query = Row.query(Row.parent_test == test_key) query = query.order(-Row.revision) return query.fetch(limit=num_points, batch_size=100, keys_only=keys_only)
def _FetchRowsAsync(self, test_keys, num_points): """Fetches recent Row asynchronously across all 'test_keys'.""" rows = [] futures = [] for test_key in test_keys: q = graph_data.Row.query() q = q.filter( graph_data.Row.parent_test == utils.OldStyleTestKey(test_key)) q = q.order(-graph_data.Row.revision) futures.append(q.fetch_async(limit=num_points)) ndb.Future.wait_all(futures) for future in futures: rows.extend(future.get_result()) return rows
def testDeprecateTestsMapper_AllSubtestsDeprecated_UpdatesSuite(self): (trace_a, trace_b, suite) = self._AddMockDataForDeprecatedTests() last_b = graph_data.Row.query( graph_data.Row.parent_test == utils.OldStyleTestKey(trace_b.key), graph_data.Row.revision == 4).get() last_b.timestamp = datetime.datetime.now() - datetime.timedelta(days=20) last_b.put() for operation in mr.DeprecateTestsMapper(trace_a): self._ExecOperation(operation) for operation in mr.DeprecateTestsMapper(trace_b): self._ExecOperation(operation) self.assertTrue(trace_a.deprecated) self.assertTrue(trace_b.deprecated) self.assertTrue(suite.deprecated)
def _DumpTestData(self): """Dumps data for the requested test. Request parameters: test_path: A single full test path, including master/bot. num_points: Max number of Row entities (optional). end_rev: Ending revision number, inclusive (optional). Outputs: JSON array of encoded protobuf messages, which encode all of the datastore entities relating to one test (including Master, Bot, TestMetadata, Row, Anomaly and Sheriff entities). """ test_path = self.request.get('test_path') num_points = int(self.request.get('num_points', _DEFAULT_MAX_POINTS)) end_rev = self.request.get('end_rev') test_key = utils.TestKey(test_path) if not test_key or test_key.kind() != 'TestMetadata': # Bad test_path passed in. self.response.out.write(json.dumps([])) return # List of datastore entities that will be dumped. entities = [] entities.extend(self._GetTestAncestors([test_key])) # Get the Row entities. q = graph_data.Row.query() print test_key q = q.filter( graph_data.Row.parent_test == utils.OldStyleTestKey(test_key)) if end_rev: q = q.filter(graph_data.Row.revision <= int(end_rev)) q = q.order(-graph_data.Row.revision) entities += q.fetch(limit=num_points) # Get the Anomaly and Sheriff entities. alerts = anomaly.Anomaly.GetAlertsForTest(test_key) sheriff_keys = {alert.sheriff for alert in alerts} sheriffs = [sheriff.get() for sheriff in sheriff_keys] entities += alerts entities += sheriffs # Convert the entities to protobuf message strings and output as JSON. protobuf_strings = map(EntityToBinaryProtobuf, entities) self.response.out.write(json.dumps(protobuf_strings))
def _CheckRows(self, test_path, multiplier=2): """Checks the rows match the expected sample data for a given test. The expected revisions that should be present are based on the sample data added in _AddMockData. Args: test_path: Test path of the test to get rows for. multiplier: Number to multiply with revision to get expected value. """ rows = graph_data.Row.query( graph_data.Row.parent_test == utils.OldStyleTestKey(test_path)).fetch() self.assertEqual(50, len(rows)) self.assertEqual(15000, rows[0].revision) self.assertEqual(15000 * multiplier, rows[0].value) self.assertEqual(15098, rows[49].revision) self.assertEqual(15098 * multiplier, rows[49].value)
def get(self): """Gets CSV from data store and outputs it. Request parameters: test_path: Full test path of one trace. rev: End revision number; if not given, latest revision is used. num_points: Number of Rows to get data for. attr: Comma-separated list of attributes (columns) to return. Outputs: CSV file contents. """ test_path = self.request.get('test_path') rev = self.request.get('rev') num_points = int(self.request.get('num_points', 500)) attributes = self.request.get('attr', 'revision,value').split(',') if not test_path: self.ReportError('No test path given.', status=400) return logging.info('Got request to /graph_csv for test: "%s".', test_path) test_key = utils.TestKey(test_path) test = test_key.get() assert (datastore_hooks.IsUnalteredQueryPermitted() or not test.internal_only) datastore_hooks.SetSinglePrivilegedRequest() q = graph_data.Row.query() q = q.filter( graph_data.Row.parent_test == utils.OldStyleTestKey(test_key)) if rev: q = q.filter(graph_data.Row.revision <= int(rev)) q = q.order(-graph_data.Row.revision) points = reversed(q.fetch(limit=num_points)) rows = self._GenerateRows(points, attributes) output = StringIO.StringIO() csv.writer(output).writerows(rows) self.response.headers['Content-Type'] = 'text/csv' self.response.headers['Content-Disposition'] = ( 'attachment; filename=%s.csv' % test.test_name) self.response.out.write(output.getvalue())
def testGet_ThreeAlertsOneSheriff_EmailSent(self): self._AddSampleData() for name in ('foo', 'bar', 'baz'): test = utils.TestKey('M/b/suite/%s' % name).get() row = graph_data.Row.query(graph_data.Row.parent_test == utils.OldStyleTestKey(test.key)).get() stoppage_alert.CreateStoppageAlert(test, row).put() self.testapp.get('/send_stoppage_alert_emails') messages = self.mail_stub.get_sent_messages() self.assertEqual(1, len(messages)) self.assertEqual('*****@*****.**', messages[0].sender) self.assertEqual('*****@*****.**', messages[0].to) self.assertIn('3', messages[0].subject) self.assertIn('foo', str(messages[0].body)) self.assertIn('bar', str(messages[0].body)) self.assertIn('baz', str(messages[0].body)) stoppage_alerts = stoppage_alert.StoppageAlert.query().fetch() for alert in stoppage_alerts: self.assertTrue(alert.mail_sent)
def DeprecateTestsMapper(entity): """Marks a TestMetadata entity as deprecated if the last row is too old. What is considered "too old" is defined by _OLDEST_REVISION_DELTA. Also, if all of the subtests in a test have been marked as deprecated, then that parent test will be marked as deprecated. This mapper doesn't un-deprecate tests if new data has been added; that happens in add_point.py. Args: entity: The TestMetadata entity to check. Yields: Zero or more datastore mutation operations. """ # Make sure that we have a non-deprecated TestMetadata with Rows. if (entity.key.kind() != 'TestMetadata' or not entity.has_rows or entity.deprecated): # TODO(qyearsley): Add test coverage. See catapult:#1346. logging.error( 'Got bad entity in mapreduce! Kind: %s, has_rows: %s, deprecated: %s', entity.key.kind(), entity.has_rows, entity.deprecated) return # Fetch the last row. datastore_hooks.SetPrivilegedRequest() query = graph_data.Row.query( graph_data.Row.parent_test == utils.OldStyleTestKey(entity.key)) query = query.order(-graph_data.Row.timestamp) last_row = query.get() if not last_row: # TODO(qyearsley): Add test coverage. See catapult:#1346. logging.error('No rows for %s (but has_rows=True)', entity.key) return now = datetime.datetime.now() if last_row.timestamp < now - _OLDEST_REVISION_DELTA: for operation in _MarkDeprecated(entity): yield operation for operation in _CreateStoppageAlerts(entity, last_row): yield operation
def _UpdateTest(self, test_key_urlsafe, internal_only, cursor=None): """Updates the given TestMetadata and associated Row entities.""" test_key = ndb.Key(urlsafe=test_key_urlsafe) if not cursor: # First time updating for this TestMetadata. test_entity = test_key.get() if test_entity.internal_only != internal_only: test_entity.internal_only = internal_only test_entity.put() # Update all of the Anomaly entities for this test. # Assuming that this should be fast enough to do in one request # for any one test. anomalies = anomaly.Anomaly.GetAlertsForTest(test_key) for anomaly_entity in anomalies: if anomaly_entity.internal_only != internal_only: anomaly_entity.internal_only = internal_only ndb.put_multi(anomalies) else: cursor = datastore_query.Cursor(urlsafe=cursor) # Fetch a certain number of Row entities starting from cursor. rows_query = graph_data.Row.query( graph_data.Row.parent_test == utils.OldStyleTestKey(test_key)) rows, next_cursor, more = rows_query.fetch_page(_MAX_ROWS_TO_PUT, start_cursor=cursor) for row in rows: if row.internal_only != internal_only: row.internal_only = internal_only ndb.put_multi(rows) if more: taskqueue.add(url='/change_internal_only', params={ 'test': test_key_urlsafe, 'cursor': next_cursor.urlsafe(), 'internal_only': 'true' if internal_only else 'false', }, queue_name=_QUEUE_NAME)
def GetRowsToAnalyze(test, max_num_rows): """Gets the Row entities that we want to analyze. Args: test: The TestMetadata entity to get data for. max_num_rows: The maximum number of points to get. Returns: A list of the latest Rows after the last alerted revision, ordered by revision. These rows are fetched with t a projection query so they only have the revision and value properties. """ query = graph_data.Row.query(projection=['revision', 'value']) query = query.filter( graph_data.Row.parent_test == utils.OldStyleTestKey(test.key)) # The query is ordered in descending order by revision because we want # to get the newest points. query = query.filter(graph_data.Row.revision > test.last_alerted_revision) query = query.order(-graph_data.Row.revision) # However, we want to analyze them in ascending order. return list(reversed(query.fetch(limit=max_num_rows)))
def testGetGraphJson_WithSelectedTrace(self): self._AddTestColumns(start_rev=15000, end_rev=15050) rows = graph_data.Row.query( graph_data.Row.parent_test == utils.OldStyleTestKey( 'ChromiumGPU/win7/dromaeo/jslib')).fetch() for row in rows: row.error = 1 + ((row.revision - 15000) * 0.25) ndb.put_multi(rows) flot_json_str = graph_json.GetGraphJson( { 'ChromiumGPU/win7/dromaeo/jslib': ['jslib'], }, rev=15000, num_points=8, is_selected=True) flot = json.loads(flot_json_str) self.assertEqual(1, len(flot['data'])) self.assertEqual(5, len(flot['data']['0']['data'])) self.assertEqual(1, len(flot['annotations']['series'])) self.assertEqual(5, len(flot['annotations'].get('0').keys())) self.assertEqual(5, len(flot['error_bars']['0'][0]['data'])) self.assertEqual(5, len(flot['error_bars']['0'][1]['data']))
def testOldStyleTestKey_Test(self): original_key = ndb.Key('Master', 'm', 'Bot', 'b', 'Test', 'suite', 'Test', 'metric') key = utils.OldStyleTestKey(original_key) self.assertEqual(original_key, key)
def testOldStyleTestKey_None(self): key = utils.OldStyleTestKey(None) self.assertIsNone(key)
def GetAlertsForTest(cls, test_key, limit=None): return cls.query( cls.test.IN([ utils.TestMetadataKey(test_key), utils.OldStyleTestKey(test_key) ])).fetch(limit=limit)