def _DeleteTestData(test_key):
logging.info('DELETING TEST DATA FOR %s', utils.TestPath(test_key))
futures = []
num_tests_processed = 0
finished = True
descendants = list_tests.GetTestDescendants(test_key)
for descendant in descendants:
rows = graph_data.GetLatestRowsForTest(descendant,
_ROWS_TO_DELETE_AT_ONCE,
keys_only=True)
if rows:
futures.extend(ndb.delete_multi_async(rows))
finished = False
num_tests_processed += 1
if num_tests_processed > _MAX_DELETIONS_PER_TASK:
break
# Only delete TestMetadata entities after all Row entities have been deleted.
if finished:
descendants = ndb.get_multi(descendants)
for descendant in descendants:
_SendNotificationEmail(descendant)
futures.append(descendant.key.delete_async())
ndb.Future.wait_all(futures)
return finished
def post(self):
"""Query for tests, and put ones with no new data on the delete queue."""
datastore_hooks.SetPrivilegedRequest()
cursor = datastore_query.Cursor(urlsafe=self.request.get('cursor'))
tests, next_cursor, more = graph_data.TestMetadata.query().fetch_page(
_TESTS_TO_CHECK_AT_ONCE, keys_only=True, start_cursor=cursor)
if more:
taskqueue.add(url='/delete_old_tests',
params={'cursor': next_cursor.urlsafe()},
queue_name=_TASK_QUEUE_NAME)
for test in tests:
# Delete this test if:
# 1) It has no Rows newer than the cutoff
# 2) It has no descendant tests
no_new_rows = False
last_row = graph_data.Row.query(
graph_data.Row.parent_test == utils.OldStyleTestKey(
test)).order(-graph_data.Row.timestamp).get()
if last_row:
if last_row.timestamp < datetime.datetime.today(
) - _CUTOFF_DATE:
no_new_rows = True
else:
no_new_rows = True
descendants = list_tests.GetTestDescendants(test, keys_only=True)
descendants.remove(test)
if not descendants and no_new_rows:
taskqueue.add(
url='/delete_test_data',
params={
'test_path':
utils.TestPath(test), # For manual inspection.
'test_key': test.urlsafe(),
},
queue_name=_DELETE_TASK_QUEUE_NAME)
def GuessStoryFilter(test_path):
"""Returns a suitable "story filter" to use in the bisect config.
Args:
test_path: The slash-separated test path used by the dashboard.
Returns:
A regex pattern that matches the story referred to by the test_path, or
an empty string if the test_path does not refer to a story and no story
filter should be used.
"""
test_path_parts = test_path.split('/')
suite_name, story_name = test_path_parts[2], test_path_parts[-1]
if any([
suite_name in _DISABLE_STORY_FILTER,
suite_name.startswith('media.') and '.html?' not in story_name,
suite_name.startswith('webrtc.')
]):
return ''
test_key = utils.TestKey(test_path)
subtest_keys = list_tests.GetTestDescendants(test_key)
try:
subtest_keys.remove(test_key)
except ValueError:
pass
if subtest_keys: # Stories do not have subtests.
return ''
# During import, some chars in story names got replaced by "_" so they
# could be safely included in the test_path. At this point we don't know
# what the original characters were. Additionally, some special characters
# and argument quoting are not interpreted correctly, e.g. by bisect
# scripts (crbug.com/662472). We thus keep only a small set of "safe chars"
# and replace all others with match-any-character regex dots.
return re.sub(r'[^a-zA-Z0-9]', '.', story_name)
def _DeleteTestData(test_key, notify):
futures = []
num_tests_processed = 0
more = False
descendants = list_tests.GetTestDescendants(test_key)
for descendant in descendants:
rows = graph_data.GetLatestRowsForTest(descendant,
_ROWS_TO_DELETE_AT_ONCE,
keys_only=True)
if rows:
futures.extend(ndb.delete_multi_async(rows))
more = True
num_tests_processed += 1
if num_tests_processed > _MAX_DELETIONS_PER_TASK:
break
if not more:
more = _DeleteTestHistogramData(descendant)
# Only delete TestMetadata entities after all Row entities have been deleted.
if not more:
descendants = ndb.get_multi(descendants)
for descendant in descendants:
_SendNotificationEmail(descendant, notify)
futures.append(descendant.key.delete_async())
ndb.Future.wait_all(futures)
return not more
def GuessStoryFilter(test_path):
"""Returns a suitable "story filter" to use in the bisect config.
Args:
test_path: The slash-separated test path used by the dashboard.
Returns:
A regex pattern that matches the story referred to by the test_path, or
an empty string if the test_path does not refer to a story and no story
filter should be used.
"""
test_path_parts = test_path.split('/')
suite_name, story_name = test_path_parts[2], test_path_parts[-1]
if suite_name in _NON_TELEMETRY_TEST_COMMANDS:
return ''
test_key = utils.TestKey(test_path)
subtest_keys = list_tests.GetTestDescendants(test_key)
try:
subtest_keys.remove(test_key)
except ValueError:
pass
if subtest_keys: # Stories do not have subtests.
return ''
if story_name.startswith('after_'):
# TODO(perezju,#1811): Remove this hack after deprecating the
# memory.top_10_mobile benchmark.
story_name = story_name[len('after_'):]
# During import, some chars in story names got replaced by "_" so they
# could be safely included in the test_path. At this point we don't know
# what the original characters were. Additionally, some special characters
# and argument quoting are not interpreted correctly, e.g. by bisect
# scripts (crbug.com/662472). We thus keep only a small set of "safe chars"
# and replace all others with match-any-character regex dots.
return re.sub(r'[^a-zA-Z0-9]', '.', story_name)
def _PrefillInfo(test_path):
"""Pre-fills some best guesses config form based on the test path.
Args:
test_path: Test path string.
Returns:
A dictionary indicating the result. If successful, this should contain the
the fields "suite", "email", "all_metrics", and "default_metric". If not
successful this will contain the field "error".
"""
if not test_path:
return {'error': 'No test specified'}
suite_path = '/'.join(test_path.split('/')[:3])
suite = utils.TestKey(suite_path).get()
if not suite:
return {'error': 'Invalid test %s' % test_path}
graph_path = '/'.join(test_path.split('/')[:4])
graph_key = utils.TestKey(graph_path)
info = {'suite': suite.test_name}
info['master'] = suite.master_name
info['internal_only'] = suite.internal_only
info['use_archive'] = _CanDownloadBuilds(suite.master_name)
info['all_bots'] = _GetAvailableBisectBots(suite.master_name)
info['bisect_bot'] = GuessBisectBot(suite.master_name, suite.bot_name)
user = users.get_current_user()
if not user:
return {'error': 'User not logged in.'}
# Secondary check for bisecting internal only tests.
if suite.internal_only and not utils.IsInternalUser():
return {
'error': 'Unauthorized access, please use corp account to login.'
}
if users.is_current_user_admin():
info['is_admin'] = True
else:
info['is_admin'] = False
info['email'] = user.email()
info['all_metrics'] = []
metric_keys = list_tests.GetTestDescendants(graph_key, has_rows=True)
for metric_key in metric_keys:
metric_path = utils.TestPath(metric_key)
if metric_path.endswith('/ref') or metric_path.endswith('_ref'):
continue
info['all_metrics'].append(GuessMetric(metric_path))
info['default_metric'] = GuessMetric(test_path)
info['story_filter'] = GuessStoryFilter(test_path)
return info
def DeprecateTestsMapper(entity):
"""Marks a TestMetadata entity as deprecated if the last row is too old.
What is considered "too old" is defined by _DEPRECATION_REVISION_DELTA. Also,
if all of the subtests in a test have been marked as deprecated, then that
parent test will be marked as deprecated.
This mapper doesn't un-deprecate tests if new data has been added; that
happens in add_point.py.
Args:
entity: The TestMetadata entity to check.
Yields:
Zero or more datastore mutation operations.
"""
# Fetch the last row.
datastore_hooks.SetPrivilegedRequest()
query = graph_data.Row.query(
graph_data.Row.parent_test == utils.OldStyleTestKey(entity.key))
query = query.order(-graph_data.Row.timestamp)
last_row = query.get()
# Check if the test should be deleted entirely.
now = datetime.datetime.now()
logging.info('checking %s', entity.test_path)
if not last_row or last_row.timestamp < now - _REMOVAL_REVISON_DELTA:
descendants = list_tests.GetTestDescendants(entity.key, keys_only=True)
if entity.key in descendants:
descendants.remove(entity.key)
if not descendants:
logging.info('removing')
if last_row:
logging.info('last row timestamp: %s', last_row.timestamp)
else:
logging.info('no last row, no descendants')
taskqueue.add(
url='/delete_test_data',
params={
'test_path': utils.TestPath(entity.key), # For manual inspection.
'test_key': entity.key.urlsafe(),
'notify': 'false',
},
queue_name=_DELETE_TASK_QUEUE_NAME)
return
if entity.deprecated or not last_row:
return
if last_row.timestamp < now - _DEPRECATION_REVISION_DELTA:
for operation in _MarkDeprecated(entity):
yield operation
for operation in _CreateStoppageAlerts(entity, last_row):
yield operation
def testGetDescendants(self):
self._AddSampleData()
self.assertEqual(
list_tests.GetTestDescendants(
ndb.Key('TestMetadata', 'Chromium/mac/dromaeo')), [
ndb.Key('TestMetadata', 'Chromium/mac/dromaeo'),
ndb.Key('TestMetadata', 'Chromium/mac/dromaeo/dom'),
ndb.Key('TestMetadata', 'Chromium/mac/dromaeo/jslib'),])
self.assertEqual(
list_tests.GetTestDescendants(
ndb.Key('TestMetadata', 'Chromium/win7/really/nested')), [
ndb.Key('TestMetadata', 'Chromium/win7/really/nested'),
ndb.Key('TestMetadata', 'Chromium/win7/really/nested/very'),
ndb.Key('TestMetadata',
'Chromium/win7/really/nested/very/deeply'),
ndb.Key('TestMetadata',
'Chromium/win7/really/nested/very/deeply/subtest'),
ndb.Key('TestMetadata',
'Chromium/win7/really/nested/very_very'),])
def GuessStoryFilter(test_path):
"""Returns a suitable "story filter" to use in the bisect config.
Args:
test_path: The slash-separated test path used by the dashboard.
Returns:
A regex pattern that matches the story referred to by the test_path, or
an empty string if the test_path does not refer to a story and no story
filter should be used.
"""
test_path_parts = test_path.split('/')
suite_name, story_name = test_path_parts[2], test_path_parts[-1]
if any([
_IsNonTelemetrySuiteName(suite_name), suite_name
in _DISABLE_STORY_FILTER_SUITE_LIST,
suite_name.startswith('media.') and '.html?' not in story_name,
suite_name.startswith('webrtc.'), story_name
in _DISABLE_STORY_FILTER_STORY_LIST
]):
return ''
test_key = utils.TestKey(test_path)
subtest_keys = list_tests.GetTestDescendants(test_key)
try:
subtest_keys.remove(test_key)
except ValueError:
pass
if subtest_keys: # Stories do not have subtests.
return ''
# memory.top_10_mobile runs pairs of "{url}" and "after_{url}" stories to
# gather foreground and background measurements. Story filters may be used,
# but we need to strip off the "after_" prefix so that both stories in the
# pair are always run together.
# TODO(crbug.com/761014): Remove when benchmark is deprecated.
if suite_name == 'memory.top_10_mobile' and story_name.startswith(
'after_'):
story_name = story_name[len('after_'):]
# During import, some chars in story names got replaced by "_" so they
# could be safely included in the test_path. At this point we don't know
# what the original characters were. Additionally, some special characters
# and argument quoting are not interpreted correctly, e.g. by bisect
# scripts (crbug.com/662472). We thus keep only a small set of "safe chars"
# and replace all others with match-any-character regex dots.
return re.sub(r'[^a-zA-Z0-9]', '.', story_name)
def _AllSubtestsDeprecated(test):
"""Checks whether all descendant tests are marked as deprecated."""
descendant_tests = list_tests.GetTestDescendants(
test.key, has_rows=True, keys_only=False)
return all(t.deprecated for t in descendant_tests)
def _PrefillInfo(test_path):
"""Pre-fills some best guesses config form based on the test path.
Args:
test_path: Test path string.
Returns:
A dictionary indicating the result. If successful, this should contain the
the fields "suite", "email", "all_metrics", and "default_metric". If not
successful this will contain the field "error".
"""
if not test_path:
return {'error': 'No test specified'}
suite_path = '/'.join(test_path.split('/')[:3])
suite = utils.TestKey(suite_path).get()
if not suite:
return {'error': 'Invalid test %s' % test_path}
graph_path = '/'.join(test_path.split('/')[:4])
graph_key = utils.TestKey(graph_path)
info = {'suite': suite.test_name}
info['master'] = suite.master_name
info['internal_only'] = suite.internal_only
info['use_archive'] = _CanDownloadBuilds(suite.master_name)
info['all_bots'] = _GetAvailableBisectBots(suite.master_name)
info['bisect_bot'] = GuessBisectBot(suite.master_name, suite.bot_name)
user = users.get_current_user()
if not user:
return {'error': 'User not logged in.'}
# Secondary check for bisecting internal only tests.
if suite.internal_only and not utils.IsInternalUser():
return {
'error': 'Unauthorized access, please use corp account to login.'
}
info['email'] = user.email()
info['all_metrics'] = []
metric_keys = list_tests.GetTestDescendants(graph_key, has_rows=True)
should_add_story_filter = (
suite.test_name not in _NON_TELEMETRY_TEST_COMMANDS and
# is not a top-level test_path, those are usually not story names
'/' in test_path)
test_path_prefix = test_path + '/'
for metric_key in metric_keys:
metric_path = utils.TestPath(metric_key)
if metric_path.endswith('/ref') or metric_path.endswith('_ref'):
continue
if metric_path.startswith(test_path_prefix):
should_add_story_filter = False # Stories do not have sub-tests.
info['all_metrics'].append(GuessMetric(metric_path))
info['default_metric'] = GuessMetric(test_path)
if should_add_story_filter:
_, story_name = test_path.rsplit('/', 1)
# During import, some chars in story names got replaced by "_" so they
# could be safely included in the test_path. At this point we don't know
# what the original characters were, so we pass a regex where each
# underscore is replaced back with a match-any-character dot.
info['story_filter'] = re.sub(r'\\_', '.', re.escape(story_name))
else:
info['story_filter'] = ''
return info
def _PrefillInfo(test_path):
"""Pre-fills some best guesses config form based on the test path.
Args:
test_path: Test path string.
Returns:
A dictionary indicating the result. If successful, this should contain the
the fields "suite", "email", "all_metrics", and "default_metric". If not
successful this will contain the field "error".
"""
if not test_path:
return {'error': 'No test specified'}
suite_path = '/'.join(test_path.split('/')[:3])
suite = utils.TestKey(suite_path).get()
if not suite:
return {'error': 'Invalid test %s' % test_path}
graph_path = '/'.join(test_path.split('/')[:4])
graph_key = utils.TestKey(graph_path)
info = {'suite': suite.test_name}
info['master'] = suite.master_name
info['internal_only'] = suite.internal_only
info['use_archive'] = _CanDownloadBuilds(suite.master_name)
info['all_bots'] = _GetAvailableBisectBots(suite.master_name)
info['bisect_bot'] = GuessBisectBot(suite.master_name, suite.bot_name)
user = users.get_current_user()
if not user:
return {'error': 'User not logged in.'}
# Secondary check for bisecting internal only tests.
if suite.internal_only and not utils.IsInternalUser():
return {
'error': 'Unauthorized access, please use corp account to login.'
}
info['email'] = user.email()
info['all_metrics'] = []
metric_keys = list_tests.GetTestDescendants(graph_key, has_rows=True)
should_add_story_filter = (
suite.test_name not in _NON_TELEMETRY_TEST_COMMANDS and
# is not a top-level test_path, those are usually not story names
'/' in test_path)
test_path_prefix = test_path + '/'
for metric_key in metric_keys:
metric_path = utils.TestPath(metric_key)
if metric_path.endswith('/ref') or metric_path.endswith('_ref'):
continue
if metric_path.startswith(test_path_prefix):
should_add_story_filter = False # Stories do not have sub-tests.
info['all_metrics'].append(GuessMetric(metric_path))
info['default_metric'] = GuessMetric(test_path)
if should_add_story_filter:
_, story_name = test_path.rsplit('/', 1)
if story_name.startswith('after_'):
# TODO(perezju,#1811): Remove this hack after deprecating the
# memory.top_10_mobile benchmark.
story_name = story_name[len('after_'):]
# During import, some chars in story names got replaced by "_" so they
# could be safely included in the test_path. At this point we don't know
# what the original characters were. Additionally, some special characters
# and argument quoting are not interpreted correctly, e.g. by bisect
# scripts (crbug.com/662472). We thus keep only a small set of "safe chars"
# and replace all others with match-any-character regex dots.
info['story_filter'] = re.sub(r'[^a-zA-Z0-9]', '.', story_name)
else:
info['story_filter'] = ''
return info
