def setUp(self):
super(ConfigTest, self).setUp()
self.SetUpApp([(r'/api/config', config.ConfigHandler)])
self.SetCurrentClientIdOAuth(api_auth.OAUTH_CLIENT_ID_WHITELIST[0])
external_key = namespaced_stored_object.NamespaceKey(
config.WHITELIST[0], datastore_hooks.EXTERNAL)
stored_object.Set(external_key, datastore_hooks.EXTERNAL)
internal_key = namespaced_stored_object.NamespaceKey(
config.WHITELIST[0], datastore_hooks.INTERNAL)
stored_object.Set(internal_key, datastore_hooks.INTERNAL)
def setUp(self):
super(TestSuitesTest, self).setUp()
self.SetUpApp([('/api/test_suites', test_suites.TestSuitesHandler)])
self.SetCurrentClientIdOAuth(api_auth.OAUTH_CLIENT_ID_WHITELIST[0])
external_key = namespaced_stored_object.NamespaceKey(
update_test_suites.TEST_SUITES_2_CACHE_KEY,
datastore_hooks.EXTERNAL)
stored_object.Set(external_key, ['external'])
internal_key = namespaced_stored_object.NamespaceKey(
update_test_suites.TEST_SUITES_2_CACHE_KEY,
datastore_hooks.INTERNAL)
stored_object.Set(internal_key, ['external', 'internal'])
def UpdateTestSuites(permissions_namespace):
"""Updates test suite data for either internal or external users."""
logging.info('Updating test suite data for: %s', permissions_namespace)
suite_dict = _CreateTestSuiteDict()
key = namespaced_stored_object.NamespaceKey(_LIST_SUITES_CACHE_KEY,
permissions_namespace)
stored_object.Set(key, suite_dict)
stored_object.Set(
namespaced_stored_object.NamespaceKey(TEST_SUITES_2_CACHE_KEY,
permissions_namespace),
_ListTestSuites())
def testPost_ForcesCacheUpdate(self):
key = namespaced_stored_object.NamespaceKey(
update_test_suites._LIST_SUITES_CACHE_KEY)
stored_object.Set(key, {'foo': 'bar'})
self.assertEqual({'foo': 'bar'},
update_test_suites.FetchCachedTestSuites())
self._AddSampleData()
# Because there is something cached, the cache is
# not automatically updated when new data is added.
self.assertEqual({'foo': 'bar'},
update_test_suites.FetchCachedTestSuites())
stored_object.Set(
namespaced_stored_object.NamespaceKey(
update_test_suites.TEST_SUITES_2_CACHE_KEY), ['foo'])
self.assertEqual(['foo'], update_test_suites.FetchCachedTestSuites2())
# Making a request to /udate_test_suites forces an update.
self.testapp.post('/update_test_suites')
self.assertEqual(
{
'dromaeo': {
'mas': {
'Chromium': {
'mac': False,
'win7': False
}
},
},
'scrolling': {
'mas': {
'Chromium': {
'mac': False,
'win7': False
}
},
},
'really': {
'mas': {
'Chromium': {
'mac': False,
'win7': False
}
},
},
}, update_test_suites.FetchCachedTestSuites())
self.assertEqual(['dromaeo', 'really', 'scrolling'],
update_test_suites.FetchCachedTestSuites2())
def testUnparsed(self):
external_key = namespaced_stored_object.NamespaceKey(
update_test_suites.TEST_SUITES_2_CACHE_KEY, datastore_hooks.EXTERNAL)
stored_object.Set(external_key, ['unparsed'])
testing_common.AddTests(
['master'],
['bot'],
{
'unparsed': {
'a': {
'b': {
'c': {},
},
},
},
})
test = utils.TestKey('master/bot/unparsed/a/b/c').get()
test.has_rows = True
test.put()
self.Post('/update_test_suite_descriptors')
with self.assertRaises(ValueError):
self.ExecuteDeferredTasks('default')
actual = update_test_suite_descriptors.FetchCachedTestSuiteDescriptor(
'unparsed')
self.assertEqual(None, actual)
def testFetchCachedTestSuites_NotEmpty(self):
# If the cache is set, then whatever's there is returned.
key = namespaced_stored_object.NamespaceKey(
update_test_suites._LIST_SUITES_CACHE_KEY)
stored_object.Set(key, {'foo': 'bar'})
self.assertEqual({'foo': 'bar'},
update_test_suites.FetchCachedTestSuites())
def testComposite(self):
external_key = namespaced_stored_object.NamespaceKey(
update_test_suites.TEST_SUITES_2_CACHE_KEY,
datastore_hooks.EXTERNAL)
stored_object.Set(external_key, ['TEST_PARTIAL_TEST_SUITE:COMPOSITE'])
testing_common.AddTests(
['master'], ['bot'], {
'TEST_PARTIAL_TEST_SUITE': {
'COMPOSITE': {
'measurement': {
'test_case': {},
},
},
},
})
test = utils.TestKey('master/bot/TEST_PARTIAL_TEST_SUITE/COMPOSITE/' +
'measurement/test_case').get()
test.has_rows = True
test.put()
self.Post('/update_test_suite_descriptors')
self.ExecuteDeferredTasks('default')
expected = {
'measurements': ['measurement'],
'bots': ['master:bot'],
'cases': ['test_case'],
'caseTags': {},
}
actual = update_test_suite_descriptors.FetchCachedTestSuiteDescriptor(
'TEST_PARTIAL_TEST_SUITE:COMPOSITE')
self.assertEqual(expected, actual)
def UpdateDescriptor(test_suite, namespace):
test_path = descriptor.Descriptor(
test_suite=test_suite,
bot='place:holder').ToTestPathsSync()[0].split('/')
measurements = set()
bots = set()
cases = set()
# TODO(4549) Tagmaps.
query = graph_data.TestMetadata.query()
query = query.filter(graph_data.TestMetadata.suite_name == test_path[2])
if len(test_path) > 3:
# test_suite is composite.
query = query.filter(
graph_data.TestMetadata.test_part1_name == test_path[3])
query = query.filter(graph_data.TestMetadata.deprecated == False)
query = query.filter(graph_data.TestMetadata.has_rows == True)
for key in query.fetch(keys_only=True):
desc = descriptor.Descriptor.FromTestPathSync(utils.TestPath(key))
bots.add(desc.bot)
if desc.measurement:
measurements.add(desc.measurement)
if desc.test_case:
cases.add(desc.test_case)
desc = {
'measurements': list(sorted(measurements)),
'bots': list(sorted(bots)),
'cases': list(sorted(cases)),
}
key = namespaced_stored_object.NamespaceKey(CacheKey(test_suite),
namespace)
stored_object.Set(key, desc)
def Set(key, value, days_to_keep=None, namespace=None):
"""Sets the value in the datastore.
Args:
key: The key name, which will be namespaced.
value: The value to set.
days_to_keep: Number of days to keep entity in datastore, default is None.
Entity will not expire when this value is 0 or None.
namespace: Optional namespace, otherwise namespace will be retrieved
using datastore_hooks.GetNamespace().
"""
# When number of days to keep is given, calculate expiration time for
# the entity and store it in datastore.
# Once the entity expires, it will be deleted from the datastore.
expire_time = None
if days_to_keep:
expire_time = datetime.datetime.now() + datetime.timedelta(
days=days_to_keep)
namespaced_key = namespaced_stored_object.NamespaceKey(key, namespace)
try:
CachedPickledString(id=namespaced_key,
value=cPickle.dumps(value),
expire_time=expire_time).put()
except datastore_errors.BadRequestError as e:
logging.warning('BadRequestError for key %s: %s', key, e)
except apiproxy_errors.RequestTooLargeError as e:
stored_object.Set(key, value)
def testInternal(self):
internal_key = namespaced_stored_object.NamespaceKey(
update_test_suites.TEST_SUITES_2_CACHE_KEY,
datastore_hooks.INTERNAL)
stored_object.Set(internal_key, ['internal'])
testing_common.AddTests(['master'], ['bot'], {
'internal': {
'measurement': {
'test_case': {},
},
},
})
test = utils.TestKey('master/bot/internal/measurement/test_case').get()
test.has_rows = True
test.put()
self.Post('/update_test_suite_descriptors?internal_only=true')
self.ExecuteDeferredTasks('default')
expected = {
'measurements': ['measurement'],
'bots': ['master:bot'],
'cases': ['test_case'],
}
self.SetCurrentUser('*****@*****.**')
actual = update_test_suite_descriptors.FetchCachedTestSuiteDescriptor(
'internal')
self.assertEqual(expected, actual)
def setUp(self):
super(_NewTest, self).setUp()
self.SetCurrentUserOAuth(testing_common.INTERNAL_USER)
self.SetCurrentClientIdOAuth(api_auth.OAUTH_CLIENT_ID_WHITELIST[0])
key = namespaced_stored_object.NamespaceKey('bot_configurations',
datastore_hooks.INTERNAL)
stored_object.Set(key,
{'chromium-rel-mac11-pro': _CONFIGURATION_ARGUMENTS})
def Get(key):
"""Gets the value from the datastore."""
if key is None:
return None
namespaced_key = namespaced_stored_object.NamespaceKey(key)
entity = ndb.Key('CachedPickledString',
namespaced_key).get(read_policy=ndb.EVENTUAL_CONSISTENCY)
if entity:
return cPickle.loads(entity.value)
else:
return stored_object.Get(key)
def setUp(self):
super(DescribeTest, self).setUp()
self.SetUpApp([(r'/api/describe', describe.DescribeHandler)])
self.SetCurrentClientIdOAuth(api_auth.OAUTH_CLIENT_ID_WHITELIST[0])
external_key = namespaced_stored_object.NamespaceKey(
update_test_suite_descriptors.CacheKey(TEST_SUITE_NAME),
datastore_hooks.EXTERNAL)
stored_object.Set(external_key, {
'measurements': ['measurement'],
'bots': ['external:bot'],
'cases': ['case'],
})
internal_key = namespaced_stored_object.NamespaceKey(
update_test_suite_descriptors.CacheKey(TEST_SUITE_NAME),
datastore_hooks.INTERNAL)
stored_object.Set(internal_key, {
'measurements': ['measurement'],
'bots': ['external:bot', 'internal:bot'],
'cases': ['case'],
})
def testCaseTags(self):
external_key = namespaced_stored_object.NamespaceKey(
update_test_suites.TEST_SUITES_2_CACHE_KEY,
datastore_hooks.EXTERNAL)
stored_object.Set(external_key, ['suite'])
testing_common.AddTests(['master'], ['a', 'b'], {
'suite': {
'measurement': {
'x': {},
'y': {},
'z': {},
},
},
})
for bot in 'ab':
for case in 'xyz':
test = utils.TestKey('master/%s/suite/measurement/%s' %
(bot, case)).get()
test.has_rows = True
test.put()
histogram.SparseDiagnostic(test=utils.TestKey('master/a/suite'),
name=reserved_infos.TAG_MAP.name,
end_revision=sys.maxint,
data=histogram_module.TagMap({
'tagsToStoryNames': {
'j': ['x']
}
}).AsDict()).put()
histogram.SparseDiagnostic(test=utils.TestKey('master/b/suite'),
name=reserved_infos.TAG_MAP.name,
end_revision=sys.maxint,
data=histogram_module.TagMap({
'tagsToStoryNames': {
'j': ['y'],
'k': ['y']
}
}).AsDict()).put()
self.Post('/update_test_suite_descriptors')
self.ExecuteDeferredTasks('default')
expected = {
'measurements': ['measurement'],
'bots': ['master:a', 'master:b'],
'cases': ['x', 'y', 'z'],
'caseTags': {
'j': ['x', 'y'],
'k': ['y']
},
}
actual = update_test_suite_descriptors.FetchCachedTestSuiteDescriptor(
'suite')
self.assertEqual(expected, actual)
def GetExternal(key):
"""Gets the value from the datastore for the externally namespaced key."""
if key is None:
return None
namespaced_key = namespaced_stored_object.NamespaceKey(
key, datastore_hooks.EXTERNAL)
entity = ndb.Key('CachedPickledString',
namespaced_key).get(read_policy=ndb.EVENTUAL_CONSISTENCY)
if entity:
return cPickle.loads(entity.value)
else:
return stored_object.Get(key)
def setUp(self):
super(_NewTest, self).setUp()
self.SetCurrentUserOAuth(testing_common.INTERNAL_USER)
self.SetCurrentClientIdOAuth(api_auth.OAUTH_CLIENT_ID_WHITELIST[0])
key = namespaced_stored_object.NamespaceKey('bot_configurations',
datastore_hooks.INTERNAL)
config_with_args = _CONFIGURATION_ARGUMENTS.copy()
config_with_args.update({'extra_test_args': '--experimental-flag'})
stored_object.Set(
key, {
'chromium-rel-mac11-pro': _CONFIGURATION_ARGUMENTS,
'test-config-with-args': config_with_args,
})
def _UpdateDescriptor(test_suite, namespace):
logging.info('%s %s', test_suite, namespace)
# This function always runs in the taskqueue as an anonymous user.
if namespace == datastore_hooks.INTERNAL:
datastore_hooks.SetPrivilegedRequest()
desc = descriptor.Descriptor(test_suite=test_suite, bot='place:holder')
test_path = list(desc.ToTestPathsSync())[0].split('/')
measurements = set()
bots = set()
cases = set()
# TODO(4549) Tagmaps.
query = graph_data.TestMetadata.query()
query = query.filter(graph_data.TestMetadata.suite_name == test_path[2])
if len(test_path) > 3:
# test_suite is composite.
query = query.filter(
graph_data.TestMetadata.test_part1_name == test_path[3])
query = query.filter(graph_data.TestMetadata.deprecated == False)
query = query.filter(graph_data.TestMetadata.has_rows == True)
# Use an iterator because some test suites have more keys than can fit in
# memory.
for key in query.iter(keys_only=True):
desc = descriptor.Descriptor.FromTestPathSync(utils.TestPath(key))
bots.add(desc.bot)
if desc.measurement:
measurements.add(desc.measurement)
if desc.test_case:
cases.add(desc.test_case)
logging.info('%d measurements, %d bots, %d cases', len(measurements),
len(bots), len(cases))
desc = {
'measurements': list(sorted(measurements)),
'bots': list(sorted(bots)),
'cases': list(sorted(cases)),
}
key = namespaced_stored_object.NamespaceKey(CacheKey(test_suite),
namespace)
stored_object.Set(key, desc)
def testInternal(self):
internal_key = namespaced_stored_object.NamespaceKey(
update_test_suites.TEST_SUITES_2_CACHE_KEY,
datastore_hooks.INTERNAL)
stored_object.Set(internal_key, ['internal'])
testing_common.AddTests(['master'], ['bot'], {
'internal': {
'measurement': {
'test_case': {},
},
},
})
test = utils.TestKey('master/bot/internal/measurement/test_case').get()
test.unescaped_story_name = 'test_case'
test.has_rows = True
test.internal_only = True
test.put()
self.Post('/update_test_suite_descriptors?internal_only=true')
# deferred.Defer() packages up the function call and arguments, not changes
# to global state like SetPrivilegedRequest, so set privileged=False as the
# taskqueue does, and test that UpdateDescriptor sets it back to True so
# that it gets the internal TestMetadata.
class FakeRequest(object):
def __init__(self):
self.registry = {'privileged': False}
webapp2._local.request = FakeRequest()
self.ExecuteDeferredTasks('default')
expected = {
'measurements': ['measurement'],
'bots': ['bot'],
'cases': ['test_case'],
'caseTags': {},
}
self.SetCurrentUser('*****@*****.**')
actual = update_test_suite_descriptors.FetchCachedTestSuiteDescriptor(
'master', 'internal')
self.assertEqual(expected, actual)
def NamespacedKey(cls, key, namespace):
return ndb.Key(cls.__name__,
namespaced_stored_object.NamespaceKey(key, namespace))
def _UpdateDescriptor(test_suite, namespace, start_cursor=None,
measurements=(), bots=(), cases=()):
logging.info('%s %s %d %d %d', test_suite, namespace,
len(measurements), len(bots), len(cases))
# This function always runs in the taskqueue as an anonymous user.
if namespace == datastore_hooks.INTERNAL:
datastore_hooks.SetPrivilegedRequest()
start = time.time()
deadline = start + DEADLINE_SECONDS
key_count = 0
measurements = set(measurements)
bots = set(bots)
cases = set(cases)
# Some test suites have more keys than can fit in memory or can be processed
# in 10 minutes, so use an iterator instead of a page limit.
query_iter = _QueryTestSuite(test_suite).iter(
keys_only=True, produce_cursors=True, start_cursor=start_cursor,
use_cache=False, use_memcache=False)
try:
for key in query_iter:
key_count += 1
desc = descriptor.Descriptor.FromTestPathSync(utils.TestPath(key))
bots.add(desc.bot)
if desc.measurement:
measurements.add(desc.measurement)
if desc.test_case:
cases.add(desc.test_case)
if time.time() > deadline:
break
except db.BadRequestError:
pass
logging.info('%d keys, %d measurements, %d bots, %d cases',
key_count, len(measurements), len(bots), len(cases))
if key_count:
logging.info('per_key:wall_us=%f',
round(1e6 * (time.time() - start) / key_count))
if query_iter.probably_has_next():
logging.info('continuing')
deferred.defer(_UpdateDescriptor, test_suite, namespace,
query_iter.cursor_before(), measurements, bots, cases)
return
desc = {
'measurements': list(sorted(measurements)),
'bots': list(sorted(bots)),
'cases': list(sorted(cases)),
}
case_tags = _QueryCaseTags(test_suite, bots)
if case_tags:
desc['caseTags'] = case_tags
key = namespaced_stored_object.NamespaceKey(
CacheKey(test_suite), namespace)
stored_object.Set(key, desc)
def UpdateTestSuiteDescriptors(namespace):
key = namespaced_stored_object.NamespaceKey(
update_test_suites.TEST_SUITES_2_CACHE_KEY, namespace)
for test_suite in stored_object.Get(key):
ScheduleUpdateDescriptor(test_suite, namespace)
def _UpdateDescriptor(test_suite,
namespace,
start_cursor=None,
measurements=(),
bots=(),
cases=(),
case_tags=None):
logging.info('%s %s %d %d %d', test_suite, namespace, len(measurements),
len(bots), len(cases))
# This function always runs in the taskqueue as an anonymous user.
if namespace == datastore_hooks.INTERNAL:
datastore_hooks.SetPrivilegedRequest()
measurements = set(measurements)
bots = set(bots)
cases = set(cases)
case_tags = case_tags or {}
tags_futures = []
tests, next_cursor, more = _QueryTestSuite(test_suite).fetch_page(
TESTS_TO_FETCH,
start_cursor=start_cursor,
use_cache=False,
use_memcache=False)
for test in tests:
bots.add(test.bot_name)
try:
_, measurement, story = utils.ParseTelemetryMetricParts(
test.test_path)
except utils.ParseTelemetryMetricFailed as e:
# Log the error and process the rest of the test suite.
logging.error('Parsing error encounted: %s', e)
continue
if test.unescaped_story_name:
story = test.unescaped_story_name
if measurement:
measurements.add(measurement)
if story and story not in cases:
cases.add(story)
tags_futures.append(_QueryCaseTags(test.test_path, story))
_CollectCaseTags(tags_futures, case_tags)
logging.info('%d keys, %d measurements, %d bots, %d cases, %d tags',
len(tests), len(measurements), len(bots), len(cases),
len(case_tags))
if more:
logging.info('continuing')
deferred.defer(_UpdateDescriptor, test_suite, namespace, next_cursor,
measurements, bots, cases, case_tags)
return
desc = {
'measurements': list(sorted(measurements)),
'bots': list(sorted(bots)),
'cases': list(sorted(cases)),
'caseTags':
{tag: sorted(cases)
for tag, cases in list(case_tags.items())}
}
key = namespaced_stored_object.NamespaceKey(CacheKey(test_suite),
namespace)
stored_object.Set(key, desc)
