예제 #1
0
 def setUp(self):
     super(ConfigTest, self).setUp()
     self.SetUpApp([(r'/api/config', config.ConfigHandler)])
     self.SetCurrentClientIdOAuth(api_auth.OAUTH_CLIENT_ID_WHITELIST[0])
     external_key = namespaced_stored_object.NamespaceKey(
         config.WHITELIST[0], datastore_hooks.EXTERNAL)
     stored_object.Set(external_key, datastore_hooks.EXTERNAL)
     internal_key = namespaced_stored_object.NamespaceKey(
         config.WHITELIST[0], datastore_hooks.INTERNAL)
     stored_object.Set(internal_key, datastore_hooks.INTERNAL)
 def setUp(self):
     super(TestSuitesTest, self).setUp()
     self.SetUpApp([('/api/test_suites', test_suites.TestSuitesHandler)])
     self.SetCurrentClientIdOAuth(api_auth.OAUTH_CLIENT_ID_WHITELIST[0])
     external_key = namespaced_stored_object.NamespaceKey(
         update_test_suites.TEST_SUITES_2_CACHE_KEY,
         datastore_hooks.EXTERNAL)
     stored_object.Set(external_key, ['external'])
     internal_key = namespaced_stored_object.NamespaceKey(
         update_test_suites.TEST_SUITES_2_CACHE_KEY,
         datastore_hooks.INTERNAL)
     stored_object.Set(internal_key, ['external', 'internal'])
예제 #3
0
def UpdateTestSuites(permissions_namespace):
    """Updates test suite data for either internal or external users."""
    logging.info('Updating test suite data for: %s', permissions_namespace)
    suite_dict = _CreateTestSuiteDict()
    key = namespaced_stored_object.NamespaceKey(_LIST_SUITES_CACHE_KEY,
                                                permissions_namespace)
    stored_object.Set(key, suite_dict)

    stored_object.Set(
        namespaced_stored_object.NamespaceKey(TEST_SUITES_2_CACHE_KEY,
                                              permissions_namespace),
        _ListTestSuites())
    def testPost_ForcesCacheUpdate(self):
        key = namespaced_stored_object.NamespaceKey(
            update_test_suites._LIST_SUITES_CACHE_KEY)
        stored_object.Set(key, {'foo': 'bar'})
        self.assertEqual({'foo': 'bar'},
                         update_test_suites.FetchCachedTestSuites())
        self._AddSampleData()
        # Because there is something cached, the cache is
        # not automatically updated when new data is added.
        self.assertEqual({'foo': 'bar'},
                         update_test_suites.FetchCachedTestSuites())

        stored_object.Set(
            namespaced_stored_object.NamespaceKey(
                update_test_suites.TEST_SUITES_2_CACHE_KEY), ['foo'])
        self.assertEqual(['foo'], update_test_suites.FetchCachedTestSuites2())

        # Making a request to /udate_test_suites forces an update.
        self.testapp.post('/update_test_suites')
        self.assertEqual(
            {
                'dromaeo': {
                    'mas': {
                        'Chromium': {
                            'mac': False,
                            'win7': False
                        }
                    },
                },
                'scrolling': {
                    'mas': {
                        'Chromium': {
                            'mac': False,
                            'win7': False
                        }
                    },
                },
                'really': {
                    'mas': {
                        'Chromium': {
                            'mac': False,
                            'win7': False
                        }
                    },
                },
            }, update_test_suites.FetchCachedTestSuites())

        self.assertEqual(['dromaeo', 'really', 'scrolling'],
                         update_test_suites.FetchCachedTestSuites2())
예제 #5
0
  def testUnparsed(self):
    external_key = namespaced_stored_object.NamespaceKey(
        update_test_suites.TEST_SUITES_2_CACHE_KEY, datastore_hooks.EXTERNAL)
    stored_object.Set(external_key, ['unparsed'])
    testing_common.AddTests(
        ['master'],
        ['bot'],
        {
            'unparsed': {
                'a': {
                    'b': {
                        'c': {},
                    },
                },
            },
        })
    test = utils.TestKey('master/bot/unparsed/a/b/c').get()
    test.has_rows = True
    test.put()

    self.Post('/update_test_suite_descriptors')
    with self.assertRaises(ValueError):
      self.ExecuteDeferredTasks('default')
    actual = update_test_suite_descriptors.FetchCachedTestSuiteDescriptor(
        'unparsed')
    self.assertEqual(None, actual)
 def testFetchCachedTestSuites_NotEmpty(self):
     # If the cache is set, then whatever's there is returned.
     key = namespaced_stored_object.NamespaceKey(
         update_test_suites._LIST_SUITES_CACHE_KEY)
     stored_object.Set(key, {'foo': 'bar'})
     self.assertEqual({'foo': 'bar'},
                      update_test_suites.FetchCachedTestSuites())
예제 #7
0
    def testComposite(self):
        external_key = namespaced_stored_object.NamespaceKey(
            update_test_suites.TEST_SUITES_2_CACHE_KEY,
            datastore_hooks.EXTERNAL)
        stored_object.Set(external_key, ['TEST_PARTIAL_TEST_SUITE:COMPOSITE'])
        testing_common.AddTests(
            ['master'], ['bot'], {
                'TEST_PARTIAL_TEST_SUITE': {
                    'COMPOSITE': {
                        'measurement': {
                            'test_case': {},
                        },
                    },
                },
            })
        test = utils.TestKey('master/bot/TEST_PARTIAL_TEST_SUITE/COMPOSITE/' +
                             'measurement/test_case').get()
        test.has_rows = True
        test.put()

        self.Post('/update_test_suite_descriptors')
        self.ExecuteDeferredTasks('default')
        expected = {
            'measurements': ['measurement'],
            'bots': ['master:bot'],
            'cases': ['test_case'],
            'caseTags': {},
        }
        actual = update_test_suite_descriptors.FetchCachedTestSuiteDescriptor(
            'TEST_PARTIAL_TEST_SUITE:COMPOSITE')
        self.assertEqual(expected, actual)
def UpdateDescriptor(test_suite, namespace):
    test_path = descriptor.Descriptor(
        test_suite=test_suite,
        bot='place:holder').ToTestPathsSync()[0].split('/')

    measurements = set()
    bots = set()
    cases = set()
    # TODO(4549) Tagmaps.

    query = graph_data.TestMetadata.query()
    query = query.filter(graph_data.TestMetadata.suite_name == test_path[2])
    if len(test_path) > 3:
        # test_suite is composite.
        query = query.filter(
            graph_data.TestMetadata.test_part1_name == test_path[3])
    query = query.filter(graph_data.TestMetadata.deprecated == False)
    query = query.filter(graph_data.TestMetadata.has_rows == True)
    for key in query.fetch(keys_only=True):
        desc = descriptor.Descriptor.FromTestPathSync(utils.TestPath(key))
        bots.add(desc.bot)
        if desc.measurement:
            measurements.add(desc.measurement)
        if desc.test_case:
            cases.add(desc.test_case)

    desc = {
        'measurements': list(sorted(measurements)),
        'bots': list(sorted(bots)),
        'cases': list(sorted(cases)),
    }
    key = namespaced_stored_object.NamespaceKey(CacheKey(test_suite),
                                                namespace)
    stored_object.Set(key, desc)
예제 #9
0
def Set(key, value, days_to_keep=None, namespace=None):
    """Sets the value in the datastore.

  Args:
    key: The key name, which will be namespaced.
    value: The value to set.
    days_to_keep: Number of days to keep entity in datastore, default is None.
    Entity will not expire when this value is 0 or None.
    namespace: Optional namespace, otherwise namespace will be retrieved
        using datastore_hooks.GetNamespace().
  """
    # When number of days to keep is given, calculate expiration time for
    # the entity and store it in datastore.
    # Once the entity expires, it will be deleted from the datastore.
    expire_time = None
    if days_to_keep:
        expire_time = datetime.datetime.now() + datetime.timedelta(
            days=days_to_keep)
    namespaced_key = namespaced_stored_object.NamespaceKey(key, namespace)

    try:
        CachedPickledString(id=namespaced_key,
                            value=cPickle.dumps(value),
                            expire_time=expire_time).put()
    except datastore_errors.BadRequestError as e:
        logging.warning('BadRequestError for key %s: %s', key, e)
    except apiproxy_errors.RequestTooLargeError as e:
        stored_object.Set(key, value)
    def testInternal(self):
        internal_key = namespaced_stored_object.NamespaceKey(
            update_test_suites.TEST_SUITES_2_CACHE_KEY,
            datastore_hooks.INTERNAL)
        stored_object.Set(internal_key, ['internal'])
        testing_common.AddTests(['master'], ['bot'], {
            'internal': {
                'measurement': {
                    'test_case': {},
                },
            },
        })
        test = utils.TestKey('master/bot/internal/measurement/test_case').get()
        test.has_rows = True
        test.put()

        self.Post('/update_test_suite_descriptors?internal_only=true')
        self.ExecuteDeferredTasks('default')
        expected = {
            'measurements': ['measurement'],
            'bots': ['master:bot'],
            'cases': ['test_case'],
        }
        self.SetCurrentUser('*****@*****.**')
        actual = update_test_suite_descriptors.FetchCachedTestSuiteDescriptor(
            'internal')
        self.assertEqual(expected, actual)
예제 #11
0
    def setUp(self):
        super(_NewTest, self).setUp()

        self.SetCurrentUserOAuth(testing_common.INTERNAL_USER)
        self.SetCurrentClientIdOAuth(api_auth.OAUTH_CLIENT_ID_WHITELIST[0])

        key = namespaced_stored_object.NamespaceKey('bot_configurations',
                                                    datastore_hooks.INTERNAL)
        stored_object.Set(key,
                          {'chromium-rel-mac11-pro': _CONFIGURATION_ARGUMENTS})
예제 #12
0
def Get(key):
    """Gets the value from the datastore."""
    if key is None:
        return None
    namespaced_key = namespaced_stored_object.NamespaceKey(key)
    entity = ndb.Key('CachedPickledString',
                     namespaced_key).get(read_policy=ndb.EVENTUAL_CONSISTENCY)
    if entity:
        return cPickle.loads(entity.value)
    else:
        return stored_object.Get(key)
예제 #13
0
 def setUp(self):
   super(DescribeTest, self).setUp()
   self.SetUpApp([(r'/api/describe', describe.DescribeHandler)])
   self.SetCurrentClientIdOAuth(api_auth.OAUTH_CLIENT_ID_WHITELIST[0])
   external_key = namespaced_stored_object.NamespaceKey(
       update_test_suite_descriptors.CacheKey(TEST_SUITE_NAME),
       datastore_hooks.EXTERNAL)
   stored_object.Set(external_key, {
       'measurements': ['measurement'],
       'bots': ['external:bot'],
       'cases': ['case'],
   })
   internal_key = namespaced_stored_object.NamespaceKey(
       update_test_suite_descriptors.CacheKey(TEST_SUITE_NAME),
       datastore_hooks.INTERNAL)
   stored_object.Set(internal_key, {
       'measurements': ['measurement'],
       'bots': ['external:bot', 'internal:bot'],
       'cases': ['case'],
   })
    def testCaseTags(self):
        external_key = namespaced_stored_object.NamespaceKey(
            update_test_suites.TEST_SUITES_2_CACHE_KEY,
            datastore_hooks.EXTERNAL)
        stored_object.Set(external_key, ['suite'])
        testing_common.AddTests(['master'], ['a', 'b'], {
            'suite': {
                'measurement': {
                    'x': {},
                    'y': {},
                    'z': {},
                },
            },
        })
        for bot in 'ab':
            for case in 'xyz':
                test = utils.TestKey('master/%s/suite/measurement/%s' %
                                     (bot, case)).get()
                test.has_rows = True
                test.put()
        histogram.SparseDiagnostic(test=utils.TestKey('master/a/suite'),
                                   name=reserved_infos.TAG_MAP.name,
                                   end_revision=sys.maxint,
                                   data=histogram_module.TagMap({
                                       'tagsToStoryNames': {
                                           'j': ['x']
                                       }
                                   }).AsDict()).put()
        histogram.SparseDiagnostic(test=utils.TestKey('master/b/suite'),
                                   name=reserved_infos.TAG_MAP.name,
                                   end_revision=sys.maxint,
                                   data=histogram_module.TagMap({
                                       'tagsToStoryNames': {
                                           'j': ['y'],
                                           'k': ['y']
                                       }
                                   }).AsDict()).put()

        self.Post('/update_test_suite_descriptors')

        self.ExecuteDeferredTasks('default')

        expected = {
            'measurements': ['measurement'],
            'bots': ['master:a', 'master:b'],
            'cases': ['x', 'y', 'z'],
            'caseTags': {
                'j': ['x', 'y'],
                'k': ['y']
            },
        }
        actual = update_test_suite_descriptors.FetchCachedTestSuiteDescriptor(
            'suite')
        self.assertEqual(expected, actual)
예제 #15
0
def GetExternal(key):
    """Gets the value from the datastore for the externally namespaced key."""
    if key is None:
        return None
    namespaced_key = namespaced_stored_object.NamespaceKey(
        key, datastore_hooks.EXTERNAL)
    entity = ndb.Key('CachedPickledString',
                     namespaced_key).get(read_policy=ndb.EVENTUAL_CONSISTENCY)
    if entity:
        return cPickle.loads(entity.value)
    else:
        return stored_object.Get(key)
예제 #16
0
    def setUp(self):
        super(_NewTest, self).setUp()

        self.SetCurrentUserOAuth(testing_common.INTERNAL_USER)
        self.SetCurrentClientIdOAuth(api_auth.OAUTH_CLIENT_ID_WHITELIST[0])

        key = namespaced_stored_object.NamespaceKey('bot_configurations',
                                                    datastore_hooks.INTERNAL)
        config_with_args = _CONFIGURATION_ARGUMENTS.copy()
        config_with_args.update({'extra_test_args': '--experimental-flag'})
        stored_object.Set(
            key, {
                'chromium-rel-mac11-pro': _CONFIGURATION_ARGUMENTS,
                'test-config-with-args': config_with_args,
            })
예제 #17
0
def _UpdateDescriptor(test_suite, namespace):
    logging.info('%s %s', test_suite, namespace)
    # This function always runs in the taskqueue as an anonymous user.
    if namespace == datastore_hooks.INTERNAL:
        datastore_hooks.SetPrivilegedRequest()

    desc = descriptor.Descriptor(test_suite=test_suite, bot='place:holder')
    test_path = list(desc.ToTestPathsSync())[0].split('/')

    measurements = set()
    bots = set()
    cases = set()
    # TODO(4549) Tagmaps.

    query = graph_data.TestMetadata.query()
    query = query.filter(graph_data.TestMetadata.suite_name == test_path[2])
    if len(test_path) > 3:
        # test_suite is composite.
        query = query.filter(
            graph_data.TestMetadata.test_part1_name == test_path[3])
    query = query.filter(graph_data.TestMetadata.deprecated == False)
    query = query.filter(graph_data.TestMetadata.has_rows == True)

    # Use an iterator because some test suites have more keys than can fit in
    # memory.
    for key in query.iter(keys_only=True):
        desc = descriptor.Descriptor.FromTestPathSync(utils.TestPath(key))
        bots.add(desc.bot)
        if desc.measurement:
            measurements.add(desc.measurement)
        if desc.test_case:
            cases.add(desc.test_case)

    logging.info('%d measurements, %d bots, %d cases', len(measurements),
                 len(bots), len(cases))
    desc = {
        'measurements': list(sorted(measurements)),
        'bots': list(sorted(bots)),
        'cases': list(sorted(cases)),
    }
    key = namespaced_stored_object.NamespaceKey(CacheKey(test_suite),
                                                namespace)
    stored_object.Set(key, desc)
    def testInternal(self):
        internal_key = namespaced_stored_object.NamespaceKey(
            update_test_suites.TEST_SUITES_2_CACHE_KEY,
            datastore_hooks.INTERNAL)
        stored_object.Set(internal_key, ['internal'])
        testing_common.AddTests(['master'], ['bot'], {
            'internal': {
                'measurement': {
                    'test_case': {},
                },
            },
        })
        test = utils.TestKey('master/bot/internal/measurement/test_case').get()
        test.unescaped_story_name = 'test_case'
        test.has_rows = True
        test.internal_only = True
        test.put()

        self.Post('/update_test_suite_descriptors?internal_only=true')

        # deferred.Defer() packages up the function call and arguments, not changes
        # to global state like SetPrivilegedRequest, so set privileged=False as the
        # taskqueue does, and test that UpdateDescriptor sets it back to True so
        # that it gets the internal TestMetadata.
        class FakeRequest(object):
            def __init__(self):
                self.registry = {'privileged': False}

        webapp2._local.request = FakeRequest()
        self.ExecuteDeferredTasks('default')

        expected = {
            'measurements': ['measurement'],
            'bots': ['bot'],
            'cases': ['test_case'],
            'caseTags': {},
        }
        self.SetCurrentUser('*****@*****.**')
        actual = update_test_suite_descriptors.FetchCachedTestSuiteDescriptor(
            'master', 'internal')
        self.assertEqual(expected, actual)
예제 #19
0
 def NamespacedKey(cls, key, namespace):
     return ndb.Key(cls.__name__,
                    namespaced_stored_object.NamespaceKey(key, namespace))
예제 #20
0
def _UpdateDescriptor(test_suite, namespace, start_cursor=None,
                      measurements=(), bots=(), cases=()):
  logging.info('%s %s %d %d %d', test_suite, namespace,
               len(measurements), len(bots), len(cases))

  # This function always runs in the taskqueue as an anonymous user.
  if namespace == datastore_hooks.INTERNAL:
    datastore_hooks.SetPrivilegedRequest()

  start = time.time()
  deadline = start + DEADLINE_SECONDS
  key_count = 0
  measurements = set(measurements)
  bots = set(bots)
  cases = set(cases)

  # Some test suites have more keys than can fit in memory or can be processed
  # in 10 minutes, so use an iterator instead of a page limit.
  query_iter = _QueryTestSuite(test_suite).iter(
      keys_only=True, produce_cursors=True, start_cursor=start_cursor,
      use_cache=False, use_memcache=False)

  try:
    for key in query_iter:
      key_count += 1
      desc = descriptor.Descriptor.FromTestPathSync(utils.TestPath(key))
      bots.add(desc.bot)
      if desc.measurement:
        measurements.add(desc.measurement)
      if desc.test_case:
        cases.add(desc.test_case)
      if time.time() > deadline:
        break
  except db.BadRequestError:
    pass

  logging.info('%d keys, %d measurements, %d bots, %d cases',
               key_count, len(measurements), len(bots), len(cases))
  if key_count:
    logging.info('per_key:wall_us=%f',
                 round(1e6 * (time.time() - start) / key_count))

  if query_iter.probably_has_next():
    logging.info('continuing')
    deferred.defer(_UpdateDescriptor, test_suite, namespace,
                   query_iter.cursor_before(), measurements, bots, cases)
    return

  desc = {
      'measurements': list(sorted(measurements)),
      'bots': list(sorted(bots)),
      'cases': list(sorted(cases)),
  }

  case_tags = _QueryCaseTags(test_suite, bots)
  if case_tags:
    desc['caseTags'] = case_tags

  key = namespaced_stored_object.NamespaceKey(
      CacheKey(test_suite), namespace)
  stored_object.Set(key, desc)
예제 #21
0
def UpdateTestSuiteDescriptors(namespace):
  key = namespaced_stored_object.NamespaceKey(
      update_test_suites.TEST_SUITES_2_CACHE_KEY, namespace)
  for test_suite in stored_object.Get(key):
    ScheduleUpdateDescriptor(test_suite, namespace)
예제 #22
0
def _UpdateDescriptor(test_suite,
                      namespace,
                      start_cursor=None,
                      measurements=(),
                      bots=(),
                      cases=(),
                      case_tags=None):
    logging.info('%s %s %d %d %d', test_suite, namespace, len(measurements),
                 len(bots), len(cases))
    # This function always runs in the taskqueue as an anonymous user.
    if namespace == datastore_hooks.INTERNAL:
        datastore_hooks.SetPrivilegedRequest()

    measurements = set(measurements)
    bots = set(bots)
    cases = set(cases)
    case_tags = case_tags or {}
    tags_futures = []

    tests, next_cursor, more = _QueryTestSuite(test_suite).fetch_page(
        TESTS_TO_FETCH,
        start_cursor=start_cursor,
        use_cache=False,
        use_memcache=False)

    for test in tests:
        bots.add(test.bot_name)

        try:
            _, measurement, story = utils.ParseTelemetryMetricParts(
                test.test_path)
        except utils.ParseTelemetryMetricFailed as e:
            # Log the error and process the rest of the test suite.
            logging.error('Parsing error encounted: %s', e)
            continue

        if test.unescaped_story_name:
            story = test.unescaped_story_name

        if measurement:
            measurements.add(measurement)

        if story and story not in cases:
            cases.add(story)
            tags_futures.append(_QueryCaseTags(test.test_path, story))

    _CollectCaseTags(tags_futures, case_tags)

    logging.info('%d keys, %d measurements, %d bots, %d cases, %d tags',
                 len(tests), len(measurements), len(bots), len(cases),
                 len(case_tags))

    if more:
        logging.info('continuing')
        deferred.defer(_UpdateDescriptor, test_suite, namespace, next_cursor,
                       measurements, bots, cases, case_tags)
        return

    desc = {
        'measurements': list(sorted(measurements)),
        'bots': list(sorted(bots)),
        'cases': list(sorted(cases)),
        'caseTags':
        {tag: sorted(cases)
         for tag, cases in list(case_tags.items())}
    }

    key = namespaced_stored_object.NamespaceKey(CacheKey(test_suite),
                                                namespace)
    stored_object.Set(key, desc)