def _DeleteTestData(test_key): logging.info('DELETING TEST DATA FOR %s', utils.TestPath(test_key)) futures = [] num_tests_processed = 0 finished = True descendants = list_tests.GetTestDescendants(test_key) for descendant in descendants: rows = graph_data.GetLatestRowsForTest(descendant, _ROWS_TO_DELETE_AT_ONCE, keys_only=True) if rows: futures.extend(ndb.delete_multi_async(rows)) finished = False num_tests_processed += 1 if num_tests_processed > _MAX_DELETIONS_PER_TASK: break # Only delete TestMetadata entities after all Row entities have been deleted. if finished: descendants = ndb.get_multi(descendants) for descendant in descendants: _SendNotificationEmail(descendant) futures.append(descendant.key.delete_async()) ndb.Future.wait_all(futures) return finished
def post(self): """Query for tests, and put ones with no new data on the delete queue.""" datastore_hooks.SetPrivilegedRequest() cursor = datastore_query.Cursor(urlsafe=self.request.get('cursor')) tests, next_cursor, more = graph_data.TestMetadata.query().fetch_page( _TESTS_TO_CHECK_AT_ONCE, keys_only=True, start_cursor=cursor) if more: taskqueue.add(url='/delete_old_tests', params={'cursor': next_cursor.urlsafe()}, queue_name=_TASK_QUEUE_NAME) for test in tests: # Delete this test if: # 1) It has no Rows newer than the cutoff # 2) It has no descendant tests no_new_rows = False last_row = graph_data.Row.query( graph_data.Row.parent_test == utils.OldStyleTestKey( test)).order(-graph_data.Row.timestamp).get() if last_row: if last_row.timestamp < datetime.datetime.today( ) - _CUTOFF_DATE: no_new_rows = True else: no_new_rows = True descendants = list_tests.GetTestDescendants(test, keys_only=True) descendants.remove(test) if not descendants and no_new_rows: taskqueue.add( url='/delete_test_data', params={ 'test_path': utils.TestPath(test), # For manual inspection. 'test_key': test.urlsafe(), }, queue_name=_DELETE_TASK_QUEUE_NAME)
def GuessStoryFilter(test_path): """Returns a suitable "story filter" to use in the bisect config. Args: test_path: The slash-separated test path used by the dashboard. Returns: A regex pattern that matches the story referred to by the test_path, or an empty string if the test_path does not refer to a story and no story filter should be used. """ test_path_parts = test_path.split('/') suite_name, story_name = test_path_parts[2], test_path_parts[-1] if any([ suite_name in _DISABLE_STORY_FILTER, suite_name.startswith('media.') and '.html?' not in story_name, suite_name.startswith('webrtc.') ]): return '' test_key = utils.TestKey(test_path) subtest_keys = list_tests.GetTestDescendants(test_key) try: subtest_keys.remove(test_key) except ValueError: pass if subtest_keys: # Stories do not have subtests. return '' # During import, some chars in story names got replaced by "_" so they # could be safely included in the test_path. At this point we don't know # what the original characters were. Additionally, some special characters # and argument quoting are not interpreted correctly, e.g. by bisect # scripts (crbug.com/662472). We thus keep only a small set of "safe chars" # and replace all others with match-any-character regex dots. return re.sub(r'[^a-zA-Z0-9]', '.', story_name)
def _DeleteTestData(test_key, notify): futures = [] num_tests_processed = 0 more = False descendants = list_tests.GetTestDescendants(test_key) for descendant in descendants: rows = graph_data.GetLatestRowsForTest(descendant, _ROWS_TO_DELETE_AT_ONCE, keys_only=True) if rows: futures.extend(ndb.delete_multi_async(rows)) more = True num_tests_processed += 1 if num_tests_processed > _MAX_DELETIONS_PER_TASK: break if not more: more = _DeleteTestHistogramData(descendant) # Only delete TestMetadata entities after all Row entities have been deleted. if not more: descendants = ndb.get_multi(descendants) for descendant in descendants: _SendNotificationEmail(descendant, notify) futures.append(descendant.key.delete_async()) ndb.Future.wait_all(futures) return not more
def GuessStoryFilter(test_path): """Returns a suitable "story filter" to use in the bisect config. Args: test_path: The slash-separated test path used by the dashboard. Returns: A regex pattern that matches the story referred to by the test_path, or an empty string if the test_path does not refer to a story and no story filter should be used. """ test_path_parts = test_path.split('/') suite_name, story_name = test_path_parts[2], test_path_parts[-1] if suite_name in _NON_TELEMETRY_TEST_COMMANDS: return '' test_key = utils.TestKey(test_path) subtest_keys = list_tests.GetTestDescendants(test_key) try: subtest_keys.remove(test_key) except ValueError: pass if subtest_keys: # Stories do not have subtests. return '' if story_name.startswith('after_'): # TODO(perezju,#1811): Remove this hack after deprecating the # memory.top_10_mobile benchmark. story_name = story_name[len('after_'):] # During import, some chars in story names got replaced by "_" so they # could be safely included in the test_path. At this point we don't know # what the original characters were. Additionally, some special characters # and argument quoting are not interpreted correctly, e.g. by bisect # scripts (crbug.com/662472). We thus keep only a small set of "safe chars" # and replace all others with match-any-character regex dots. return re.sub(r'[^a-zA-Z0-9]', '.', story_name)
def _PrefillInfo(test_path): """Pre-fills some best guesses config form based on the test path. Args: test_path: Test path string. Returns: A dictionary indicating the result. If successful, this should contain the the fields "suite", "email", "all_metrics", and "default_metric". If not successful this will contain the field "error". """ if not test_path: return {'error': 'No test specified'} suite_path = '/'.join(test_path.split('/')[:3]) suite = utils.TestKey(suite_path).get() if not suite: return {'error': 'Invalid test %s' % test_path} graph_path = '/'.join(test_path.split('/')[:4]) graph_key = utils.TestKey(graph_path) info = {'suite': suite.test_name} info['master'] = suite.master_name info['internal_only'] = suite.internal_only info['use_archive'] = _CanDownloadBuilds(suite.master_name) info['all_bots'] = _GetAvailableBisectBots(suite.master_name) info['bisect_bot'] = GuessBisectBot(suite.master_name, suite.bot_name) user = users.get_current_user() if not user: return {'error': 'User not logged in.'} # Secondary check for bisecting internal only tests. if suite.internal_only and not utils.IsInternalUser(): return { 'error': 'Unauthorized access, please use corp account to login.' } if users.is_current_user_admin(): info['is_admin'] = True else: info['is_admin'] = False info['email'] = user.email() info['all_metrics'] = [] metric_keys = list_tests.GetTestDescendants(graph_key, has_rows=True) for metric_key in metric_keys: metric_path = utils.TestPath(metric_key) if metric_path.endswith('/ref') or metric_path.endswith('_ref'): continue info['all_metrics'].append(GuessMetric(metric_path)) info['default_metric'] = GuessMetric(test_path) info['story_filter'] = GuessStoryFilter(test_path) return info
def DeprecateTestsMapper(entity): """Marks a TestMetadata entity as deprecated if the last row is too old. What is considered "too old" is defined by _DEPRECATION_REVISION_DELTA. Also, if all of the subtests in a test have been marked as deprecated, then that parent test will be marked as deprecated. This mapper doesn't un-deprecate tests if new data has been added; that happens in add_point.py. Args: entity: The TestMetadata entity to check. Yields: Zero or more datastore mutation operations. """ # Fetch the last row. datastore_hooks.SetPrivilegedRequest() query = graph_data.Row.query( graph_data.Row.parent_test == utils.OldStyleTestKey(entity.key)) query = query.order(-graph_data.Row.timestamp) last_row = query.get() # Check if the test should be deleted entirely. now = datetime.datetime.now() logging.info('checking %s', entity.test_path) if not last_row or last_row.timestamp < now - _REMOVAL_REVISON_DELTA: descendants = list_tests.GetTestDescendants(entity.key, keys_only=True) if entity.key in descendants: descendants.remove(entity.key) if not descendants: logging.info('removing') if last_row: logging.info('last row timestamp: %s', last_row.timestamp) else: logging.info('no last row, no descendants') taskqueue.add( url='/delete_test_data', params={ 'test_path': utils.TestPath(entity.key), # For manual inspection. 'test_key': entity.key.urlsafe(), 'notify': 'false', }, queue_name=_DELETE_TASK_QUEUE_NAME) return if entity.deprecated or not last_row: return if last_row.timestamp < now - _DEPRECATION_REVISION_DELTA: for operation in _MarkDeprecated(entity): yield operation for operation in _CreateStoppageAlerts(entity, last_row): yield operation
def testGetDescendants(self): self._AddSampleData() self.assertEqual( list_tests.GetTestDescendants( ndb.Key('TestMetadata', 'Chromium/mac/dromaeo')), [ ndb.Key('TestMetadata', 'Chromium/mac/dromaeo'), ndb.Key('TestMetadata', 'Chromium/mac/dromaeo/dom'), ndb.Key('TestMetadata', 'Chromium/mac/dromaeo/jslib'),]) self.assertEqual( list_tests.GetTestDescendants( ndb.Key('TestMetadata', 'Chromium/win7/really/nested')), [ ndb.Key('TestMetadata', 'Chromium/win7/really/nested'), ndb.Key('TestMetadata', 'Chromium/win7/really/nested/very'), ndb.Key('TestMetadata', 'Chromium/win7/really/nested/very/deeply'), ndb.Key('TestMetadata', 'Chromium/win7/really/nested/very/deeply/subtest'), ndb.Key('TestMetadata', 'Chromium/win7/really/nested/very_very'),])
def GuessStoryFilter(test_path): """Returns a suitable "story filter" to use in the bisect config. Args: test_path: The slash-separated test path used by the dashboard. Returns: A regex pattern that matches the story referred to by the test_path, or an empty string if the test_path does not refer to a story and no story filter should be used. """ test_path_parts = test_path.split('/') suite_name, story_name = test_path_parts[2], test_path_parts[-1] if any([ _IsNonTelemetrySuiteName(suite_name), suite_name in _DISABLE_STORY_FILTER_SUITE_LIST, suite_name.startswith('media.') and '.html?' not in story_name, suite_name.startswith('webrtc.'), story_name in _DISABLE_STORY_FILTER_STORY_LIST ]): return '' test_key = utils.TestKey(test_path) subtest_keys = list_tests.GetTestDescendants(test_key) try: subtest_keys.remove(test_key) except ValueError: pass if subtest_keys: # Stories do not have subtests. return '' # memory.top_10_mobile runs pairs of "{url}" and "after_{url}" stories to # gather foreground and background measurements. Story filters may be used, # but we need to strip off the "after_" prefix so that both stories in the # pair are always run together. # TODO(crbug.com/761014): Remove when benchmark is deprecated. if suite_name == 'memory.top_10_mobile' and story_name.startswith( 'after_'): story_name = story_name[len('after_'):] # During import, some chars in story names got replaced by "_" so they # could be safely included in the test_path. At this point we don't know # what the original characters were. Additionally, some special characters # and argument quoting are not interpreted correctly, e.g. by bisect # scripts (crbug.com/662472). We thus keep only a small set of "safe chars" # and replace all others with match-any-character regex dots. return re.sub(r'[^a-zA-Z0-9]', '.', story_name)
def _AllSubtestsDeprecated(test): """Checks whether all descendant tests are marked as deprecated.""" descendant_tests = list_tests.GetTestDescendants( test.key, has_rows=True, keys_only=False) return all(t.deprecated for t in descendant_tests)
def _PrefillInfo(test_path): """Pre-fills some best guesses config form based on the test path. Args: test_path: Test path string. Returns: A dictionary indicating the result. If successful, this should contain the the fields "suite", "email", "all_metrics", and "default_metric". If not successful this will contain the field "error". """ if not test_path: return {'error': 'No test specified'} suite_path = '/'.join(test_path.split('/')[:3]) suite = utils.TestKey(suite_path).get() if not suite: return {'error': 'Invalid test %s' % test_path} graph_path = '/'.join(test_path.split('/')[:4]) graph_key = utils.TestKey(graph_path) info = {'suite': suite.test_name} info['master'] = suite.master_name info['internal_only'] = suite.internal_only info['use_archive'] = _CanDownloadBuilds(suite.master_name) info['all_bots'] = _GetAvailableBisectBots(suite.master_name) info['bisect_bot'] = GuessBisectBot(suite.master_name, suite.bot_name) user = users.get_current_user() if not user: return {'error': 'User not logged in.'} # Secondary check for bisecting internal only tests. if suite.internal_only and not utils.IsInternalUser(): return { 'error': 'Unauthorized access, please use corp account to login.' } info['email'] = user.email() info['all_metrics'] = [] metric_keys = list_tests.GetTestDescendants(graph_key, has_rows=True) should_add_story_filter = ( suite.test_name not in _NON_TELEMETRY_TEST_COMMANDS and # is not a top-level test_path, those are usually not story names '/' in test_path) test_path_prefix = test_path + '/' for metric_key in metric_keys: metric_path = utils.TestPath(metric_key) if metric_path.endswith('/ref') or metric_path.endswith('_ref'): continue if metric_path.startswith(test_path_prefix): should_add_story_filter = False # Stories do not have sub-tests. info['all_metrics'].append(GuessMetric(metric_path)) info['default_metric'] = GuessMetric(test_path) if should_add_story_filter: _, story_name = test_path.rsplit('/', 1) # During import, some chars in story names got replaced by "_" so they # could be safely included in the test_path. At this point we don't know # what the original characters were, so we pass a regex where each # underscore is replaced back with a match-any-character dot. info['story_filter'] = re.sub(r'\\_', '.', re.escape(story_name)) else: info['story_filter'] = '' return info
def _PrefillInfo(test_path): """Pre-fills some best guesses config form based on the test path. Args: test_path: Test path string. Returns: A dictionary indicating the result. If successful, this should contain the the fields "suite", "email", "all_metrics", and "default_metric". If not successful this will contain the field "error". """ if not test_path: return {'error': 'No test specified'} suite_path = '/'.join(test_path.split('/')[:3]) suite = utils.TestKey(suite_path).get() if not suite: return {'error': 'Invalid test %s' % test_path} graph_path = '/'.join(test_path.split('/')[:4]) graph_key = utils.TestKey(graph_path) info = {'suite': suite.test_name} info['master'] = suite.master_name info['internal_only'] = suite.internal_only info['use_archive'] = _CanDownloadBuilds(suite.master_name) info['all_bots'] = _GetAvailableBisectBots(suite.master_name) info['bisect_bot'] = GuessBisectBot(suite.master_name, suite.bot_name) user = users.get_current_user() if not user: return {'error': 'User not logged in.'} # Secondary check for bisecting internal only tests. if suite.internal_only and not utils.IsInternalUser(): return { 'error': 'Unauthorized access, please use corp account to login.' } info['email'] = user.email() info['all_metrics'] = [] metric_keys = list_tests.GetTestDescendants(graph_key, has_rows=True) should_add_story_filter = ( suite.test_name not in _NON_TELEMETRY_TEST_COMMANDS and # is not a top-level test_path, those are usually not story names '/' in test_path) test_path_prefix = test_path + '/' for metric_key in metric_keys: metric_path = utils.TestPath(metric_key) if metric_path.endswith('/ref') or metric_path.endswith('_ref'): continue if metric_path.startswith(test_path_prefix): should_add_story_filter = False # Stories do not have sub-tests. info['all_metrics'].append(GuessMetric(metric_path)) info['default_metric'] = GuessMetric(test_path) if should_add_story_filter: _, story_name = test_path.rsplit('/', 1) if story_name.startswith('after_'): # TODO(perezju,#1811): Remove this hack after deprecating the # memory.top_10_mobile benchmark. story_name = story_name[len('after_'):] # During import, some chars in story names got replaced by "_" so they # could be safely included in the test_path. At this point we don't know # what the original characters were. Additionally, some special characters # and argument quoting are not interpreted correctly, e.g. by bisect # scripts (crbug.com/662472). We thus keep only a small set of "safe chars" # and replace all others with match-any-character regex dots. info['story_filter'] = re.sub(r'[^a-zA-Z0-9]', '.', story_name) else: info['story_filter'] = '' return info