def testToTestPaths_Pct(self):
expected = {'master/bot/suite/measure_pct_090'}
self.assertEqual(expected, descriptor.Descriptor(
bot='master:bot',
test_suite='suite',
measurement='measure',
statistic='pct_090').ToTestPathsSync())
def testToTestPaths_Ci_Lower(self):
expected = {'master/bot/suite/measure_ci_090_lower'}
self.assertEqual(expected, descriptor.Descriptor(
bot='master:bot',
test_suite='suite',
measurement='measure',
statistic='ci_090_lower').ToTestPathsSync())
def AuthorizedPost(self):
desc = descriptor.Descriptor(
test_suite=self.request.get('test_suite'),
measurement=self.request.get('measurement'),
bot=self.request.get('bot'),
test_case=self.request.get('test_case'),
statistic=None,
build_type=self.request.get('build_type'))
min_revision = self.request.get('min_revision')
min_revision = int(min_revision) if min_revision else None
max_revision = self.request.get('max_revision')
max_revision = int(max_revision) if max_revision else None
query = TimeseriesQuery(
desc,
self.request.get('columns').split(','), min_revision, max_revision,
api_utils.ParseISO8601(self.request.get('min_timestamp', None)),
api_utils.ParseISO8601(self.request.get('max_timestamp', None)))
try:
result = query.FetchSync()
except AssertionError:
# The caller has requested internal-only data but is not authorized.
raise api_request_handler.NotFoundError
self.response.headers['Cache-Control'] = '%s, max-age=%d' % (
'private' if query.private else 'public', CACHE_SECONDS)
return result
def testToTestPath_ResourceSizes(self):
expected = 'master/bot/resource_sizes (foo)/a/b/c'
self.assertEqual({expected},
descriptor.Descriptor(
bot='master:bot',
test_suite='resource_sizes:foo',
measurement='a:b:c').ToTestPathsSync())
def _QueryCaseTags(test_suite, bots):
test_paths = set()
for bot in bots:
desc = descriptor.Descriptor(test_suite=test_suite, bot=bot)
for test_path in desc.ToTestPathsSync():
test_paths.add(test_path)
futures = []
for test_path in test_paths:
futures.append(
histogram.SparseDiagnostic.GetMostRecentDataByNamesAsync(
utils.TestKey(test_path), [reserved_infos.TAG_MAP.name]))
ndb.Future.wait_all(futures)
tag_map = histogram_module.TagMap({})
for future in futures:
data = future.get_result().get(reserved_infos.TAG_MAP.name)
if not data:
continue
tag_map.AddDiagnostic(histogram_module.TagMap.FromDict(data))
return {
tag: list(sorted(cases))
for tag, cases in tag_map.tags_to_story_names.iteritems()
}
def testToTestPaths_Statistic(self):
self.assertEqual({'master/bot/suite/measure_avg'},
descriptor.Descriptor(
bot='master:bot',
test_suite='suite',
measurement='measure',
statistic='avg').ToTestPathsSync())
def testToTestPath_TwoTwo(self):
expected = 'master/bot/TEST_PARTIAL_TEST_SUITE/two_two/a/b/c/d'
self.assertEqual({expected}, descriptor.Descriptor(
bot='master:bot',
test_suite='TEST_PARTIAL_TEST_SUITE:two_two',
measurement='a:b',
test_case='c:d').ToTestPathsSync())
def testToTestPaths_TestCase(self):
self.assertEqual({'master/bot/suite/measure/case'},
descriptor.Descriptor(
bot='master:bot',
test_suite='suite',
measurement='measure',
test_case='case').ToTestPathsSync())
def testToTestPaths_Composite(self):
expected = 'master/bot/TEST_PARTIAL_TEST_SUITE/COMPOSITE'
self.assertEqual({expected},
descriptor.Descriptor(
bot='master:bot',
test_suite='TEST_PARTIAL_TEST_SUITE:COMPOSITE').
ToTestPathsSync())
def testToTestPath_Memory(self):
expected = 'master/bot/memory.top_10_mobile/measure/ground/page'
self.assertEqual({expected}, descriptor.Descriptor(
bot='master:bot',
test_suite='memory.top_10_mobile',
measurement='measure',
test_case='ground:page').ToTestPathsSync())
def testToTestPath_Sizes(self):
expected = 'master/bot/sizes/a/b/c/d/e/f/g/h/i'
self.assertEqual({expected}, descriptor.Descriptor(
bot='master:bot',
test_suite='sizes',
measurement='a:b:c:d:e:f',
test_case='g:h:i').ToTestPathsSync())
def testToTestPaths_V8Browsing(self):
expected = {'master/bot/v8.browsing_desktop/a/c_d/c_d_e'}
self.assertEqual(expected, descriptor.Descriptor(
bot='master:bot',
test_suite='v8:browsing_desktop',
measurement='a',
test_case='c:d:e').ToTestPathsSync())
def UpdateDescriptor(test_suite, namespace):
test_path = descriptor.Descriptor(
test_suite=test_suite,
bot='place:holder').ToTestPathsSync()[0].split('/')
measurements = set()
bots = set()
cases = set()
# TODO(4549) Tagmaps.
query = graph_data.TestMetadata.query()
query = query.filter(graph_data.TestMetadata.suite_name == test_path[2])
if len(test_path) > 3:
# test_suite is composite.
query = query.filter(
graph_data.TestMetadata.test_part1_name == test_path[3])
query = query.filter(graph_data.TestMetadata.deprecated == False)
query = query.filter(graph_data.TestMetadata.has_rows == True)
for key in query.fetch(keys_only=True):
desc = descriptor.Descriptor.FromTestPathSync(utils.TestPath(key))
bots.add(desc.bot)
if desc.measurement:
measurements.add(desc.measurement)
if desc.test_case:
cases.add(desc.test_case)
desc = {
'measurements': list(sorted(measurements)),
'bots': list(sorted(bots)),
'cases': list(sorted(cases)),
}
key = namespaced_stored_object.NamespaceKey(CacheKey(test_suite),
namespace)
stored_object.Set(key, desc)
def TableRowDescriptors(table_row):
for test_suite in table_row['testSuites']:
for bot in table_row['bots']:
for case in table_row['testCases']:
yield descriptor.Descriptor(test_suite, table_row['measurement'], bot,
case)
if not table_row['testCases']:
yield descriptor.Descriptor(test_suite, table_row['measurement'], bot)
def testToTestPath_LongRunningTools(self):
expected = ('master/bot/system_health.common_desktop/measurement/'
'long_running_tools/long_running_tools_gmail')
self.assertEqual({expected}, descriptor.Descriptor(
bot='master:bot',
test_suite='system_health.common_desktop',
measurement='measurement',
test_case='long_running_tools:gmail').ToTestPathsSync())
def testToTestPath_SystemHealth(self):
expected = ('master/bot/system_health.common_desktop/measurement/'
'browse_news/browse_news_cnn')
self.assertEqual({expected}, descriptor.Descriptor(
bot='master:bot',
test_suite='system_health.common_desktop',
measurement='measurement',
test_case='browse:news:cnn').ToTestPathsSync())
def testToTestPaths_Ref(self):
test_path = 'master/bot/suite/measure'
expected = {test_path + '_ref', test_path + '/ref'}
self.assertEqual(expected, descriptor.Descriptor(
bot='master:bot',
test_suite='suite',
measurement='measure',
build_type=descriptor.REFERENCE_BUILD_TYPE).ToTestPathsSync())
def testToTestPath_OneTwo(self):
expected = 'master/bot/one_two/measure/case_part_a/case_part_b'
self.assertEqual({expected},
descriptor.Descriptor(
bot='master:bot',
test_suite='one_two',
measurement='measure',
test_case='case_part_a:case_part_b').ToTestPathsSync())
def Post(self):
params = dict((a, self.request.get(a)) for a in self.request.arguments())
desc = descriptor.Descriptor(
params['suite'], params['measurement'], params['bot'],
params.get('case'), params.get('statistic'))
params['test_path'] = list(desc.ToTestPathsSync())[0]
# TODO Find the first test_path that exists, maybe strip statistic.
params['story_filter'] = params.get('case')
return pinpoint_request.NewPinpointBisect(params)
def testToTestPaths_Loading(self):
expected = {
'master/bot/loading.foo/measure/cold/24h_cold',
'master/bot/loading.foo/measure/cold/24h',
}
self.assertEqual(expected, descriptor.Descriptor(
bot='master:bot',
test_suite='loading.foo',
measurement='measure',
test_case='cold:24h').ToTestPathsSync())
def testToTestPath_BotAliases(self):
expected = {
'master/a/suite/measure',
'master/b/suite/measure',
'master/c/suite/measure',
}
self.assertEqual(expected, descriptor.Descriptor(
bot='master:b',
test_suite='suite',
measurement='measure').ToTestPathsSync())
def testToTestPaths_All(self):
test_path = 'master/bot/suite/measure_avg/case'
expected = [test_path + '_ref', test_path + '/ref']
self.assertEqual(
expected,
descriptor.Descriptor(
bot='master:bot',
test_suite='suite',
measurement='measure',
test_case='case',
statistic='avg',
build_type=descriptor.REFERENCE_BUILD_TYPE).ToTestPaths())
def _QueryTestSuite(test_suite):
desc = descriptor.Descriptor(test_suite=test_suite, bot='place:holder')
test_path = list(desc.ToTestPathsSync())[0].split('/')
query = graph_data.TestMetadata.query()
query = query.filter(graph_data.TestMetadata.suite_name == test_path[2])
if len(test_path) > 3:
# test_suite is composite.
query = query.filter(
graph_data.TestMetadata.test_part1_name == test_path[3])
query = query.filter(graph_data.TestMetadata.deprecated == False)
query = query.filter(graph_data.TestMetadata.has_rows == True)
return query
def _UpdateDescriptor(test_suite, namespace):
logging.info('%s %s', test_suite, namespace)
# This function always runs in the taskqueue as an anonymous user.
if namespace == datastore_hooks.INTERNAL:
datastore_hooks.SetPrivilegedRequest()
desc = descriptor.Descriptor(test_suite=test_suite, bot='place:holder')
test_path = list(desc.ToTestPathsSync())[0].split('/')
measurements = set()
bots = set()
cases = set()
# TODO(4549) Tagmaps.
query = graph_data.TestMetadata.query()
query = query.filter(graph_data.TestMetadata.suite_name == test_path[2])
if len(test_path) > 3:
# test_suite is composite.
query = query.filter(
graph_data.TestMetadata.test_part1_name == test_path[3])
query = query.filter(graph_data.TestMetadata.deprecated == False)
query = query.filter(graph_data.TestMetadata.has_rows == True)
# Use an iterator because some test suites have more keys than can fit in
# memory.
for key in query.iter(keys_only=True):
desc = descriptor.Descriptor.FromTestPathSync(utils.TestPath(key))
bots.add(desc.bot)
if desc.measurement:
measurements.add(desc.measurement)
if desc.test_case:
cases.add(desc.test_case)
logging.info('%d measurements, %d bots, %d cases', len(measurements),
len(bots), len(cases))
desc = {
'measurements': list(sorted(measurements)),
'bots': list(sorted(bots)),
'cases': list(sorted(cases)),
}
key = namespaced_stored_object.NamespaceKey(CacheKey(test_suite),
namespace)
stored_object.Set(key, desc)
def Post(self):
desc = descriptor.Descriptor(
test_suite=self.request.get('test_suite'),
measurement=self.request.get('measurement'),
bot=self.request.get('bot'),
test_case=self.request.get('test_case'),
statistic=None,
build_type=self.request.get('build_type'))
min_revision = self.request.get('min_revision')
min_revision = int(min_revision) if min_revision else None
max_revision = self.request.get('max_revision')
max_revision = int(max_revision) if max_revision else None
query = TimeseriesQuery(
desc, self.request.get('columns').split(','),
min_revision, max_revision,
api_utils.ParseISO8601(self.request.get('min_timestamp', None)),
api_utils.ParseISO8601(self.request.get('max_timestamp', None)))
try:
result = query.FetchSync()
except AssertionError:
# The caller has requested internal-only data but is not authorized.
raise api_request_handler.NotFoundError
return result
def testToTestPaths_Groupable(self):
self.assertEqual(
{'master/bot/TEST_GROUPABLE%FOO'},
descriptor.Descriptor(
bot='master:bot',
test_suite='TEST_GROUPABLE:FOO').ToTestPathsSync())
def testToTestPaths_Suite(self):
self.assertEqual(
{'master/bot/suite'},
descriptor.Descriptor(bot='master:bot',
test_suite='suite').ToTestPathsSync())
def testToTestPaths_Bot(self):
self.assertEqual(
{'master/bot'},
descriptor.Descriptor(bot='master:bot').ToTestPathsSync())
def testToTestPaths_Empty(self):
self.assertEqual(set(), descriptor.Descriptor().ToTestPathsSync())
def testToTestPaths_Measurement(self):
self.assertEqual(['master/bot/suite/measure'],
descriptor.Descriptor(
bot='master:bot',
test_suite='suite',
measurement='measure').ToTestPaths())
