def testPost_EndToEnd(self): expected = { 'count': 0, 'started': 'Date Time', 'total': 100, 'errors': 0, } job_state.JobState.__setstate__ = _JobStateSetState response = self.testapp.post('/api/migrate', status=200) self.assertEqual(response.normal_body, json.dumps(expected)) expected = { 'count': 50, 'started': 'Date Time', 'total': 100, 'errors': 0, } self.ExecuteDeferredTasks('default', recurse=False) status = stored_object.Get(migrate._STATUS_KEY) self.assertEqual(status, expected) self.ExecuteDeferredTasks('default', recurse=False) status = stored_object.Get(migrate._STATUS_KEY) self.assertEqual(status, None) del job_state.JobState.__setstate__ jobs = job.Job.query().fetch() for j in jobs: self.assertEqual(j.state._new_field, 'new value')
def FetchCachedTestSuites(): """Fetches cached test suite data.""" cache_key = _NamespaceKey(_LIST_SUITES_CACHE_KEY) cached = stored_object.Get(cache_key) if cached is None: # If the cache test suite list is not set, update it before fetching. # This is for convenience when testing sending of data to a local instance. namespace = datastore_hooks.GetNamespace() UpdateTestSuites(namespace) cached = stored_object.Get(cache_key) return cached
def ServiceAccountHttp(*args, **kwargs): """Returns the Credentials of the service account. Just prior to the first request, the Http object makes a request to https://accounts.google.com/o/oauth2/token to get an oauth token. The Http object is cached in memcache to reduce the number of extraneous requests. Arguments: args, kwargs: Arguments passed through to httplib2.Http(). Raises: KeyError: The service account credentials were not found. """ http = memcache.get(SERVICE_ACCOUNT_KEY) if not http: account_details = stored_object.Get(SERVICE_ACCOUNT_KEY) if not account_details: raise KeyError('Service account credentials not found.') client.logger.setLevel(logging.WARNING) credentials = client.SignedJwtAssertionCredentials( service_account_name=account_details['client_email'], private_key=account_details['private_key'], scope=EMAIL_SCOPE) http = httplib2.Http(*args, **kwargs) credentials.authorize(http) memcache.add(SERVICE_ACCOUNT_KEY, http, time=60 * 50) # Less than 1 hour. return http
def testPost_WithKey_UpdatesNonNamespacedValues(self): self.testapp.post('/edit_site_config', { 'key': 'foo', 'value': '[1, 2, 3]', 'xsrf_token': xsrf.GenerateToken(users.get_current_user()), }) self.assertEqual([1, 2, 3], stored_object.Get('foo'))
def TickMonitoringCustomMetric(metric_name): """Increments the stackdriver custom metric with the given name. This is used for cron job monitoring; if these metrics stop being received an alert mail is sent. For more information on custom metrics, see https://cloud.google.com/monitoring/custom-metrics/using-custom-metrics Args: metric_name: The name of the metric being monitored. """ credentials = client.GoogleCredentials.get_application_default() monitoring = discovery.build('monitoring', 'v3', credentials=credentials) now = _GetNowRfc3339() project_id = stored_object.Get(_PROJECT_ID_KEY) points = [{ 'interval': { 'startTime': now, 'endTime': now, }, 'value': { 'int64Value': _DEFAULT_CUSTOM_METRIC_VAL, }, }] write_request = monitoring.projects().timeSeries().create( name='projects/%s' % project_id, body={ 'timeSeries': [{ 'metric': { 'type': 'custom.googleapis.com/%s' % metric_name, }, 'points': points }] }) write_request.execute()
def testAddRowsToCache(self): self._AddMockData() rows = [] stored_object.Set( 'externally_visible__num_revisions_ChromiumPerf/win7/dromaeo/dom', [[10, 2, 3], [15, 4, 5], [100, 6, 7]]) test_key = utils.TestKey('ChromiumPerf/win7/dromaeo/dom') test_container_key = utils.GetTestContainerKey(test_key) ts1 = datetime.datetime(2013, 1, 1) row1 = graph_data.Row(parent=test_container_key, id=1, value=9, timestamp=ts1) rows.append(row1) ts2 = datetime.datetime(2013, 1, 2) row2 = graph_data.Row(parent=test_container_key, id=12, value=90, timestamp=ts2) rows.append(row2) ts3 = datetime.datetime(2013, 1, 3) row3 = graph_data.Row(parent=test_container_key, id=102, value=99, timestamp=ts3) rows.append(row3) graph_revisions.AddRowsToCache(rows) self.assertEqual( [[1, 9, utils.TimestampMilliseconds(ts1)], [10, 2, 3], [12, 90, utils.TimestampMilliseconds(ts2)], [15, 4, 5], [100, 6, 7], [102, 99, utils.TimestampMilliseconds(ts3)]], stored_object.Get('externally_visible__num_revisions_' 'ChromiumPerf/win7/dromaeo/dom'))
def ServiceAccountEmail(scope=EMAIL_SCOPE): account_details = stored_object.Get(SERVICE_ACCOUNT_KEY) if not account_details: raise KeyError('Service account credentials not found.') assert scope, "ServiceAccountHttp scope must not be None." return account_details['client_email'],
def testSetAndGet_NoMemcache(self, async_mock): async_mock.return_value = ndb.Future() async_mock.return_value.set_result(None) new_account = SampleSerializableClass('Some account data.') stored_object.Set('chris', new_account) chris_account = stored_object.Get('chris') self.assertEqual(new_account, chris_account)
def testSetAndGet_CacheNotExist_CacheSet(self): new_account = SampleSerializableClass('Some account data.') stored_object.Set('chris', new_account) stored_object.MultipartCache.Delete('chris') chris_account = stored_object.Get('chris') self.assertEqual(new_account, chris_account) cache_values = self._GetCachedValues('chris') self.assertGreater(len(cache_values), 0)
def testPost_WithSomeInvalidJSON_ShowsErrorAndDoesNotModify(self): stored_object.Set('foo', 'XXX') response = self.testapp.post('/edit_site_config', { 'key': 'foo', 'value': '[1, 2, this is not json', 'xsrf_token': xsrf.GenerateToken(users.get_current_user()), }) self.assertIn('Invalid JSON', response.body) self.assertEqual('XXX', stored_object.Get('foo'))
def _PartialTestSuites(cls): """Returns a list of test path component strings that must be joined with the subsequent test path component in order to form a composite test suite. Some are internal-only, but there's no need to store a separate list for external users. """ if cls.PARTIAL_TEST_SUITES is None: cls.PARTIAL_TEST_SUITES = stored_object.Get( PARTIAL_TEST_SUITES_KEY) return cls.PARTIAL_TEST_SUITES
def IsValidSheriffUser(): """Checks whether the user should be allowed to triage alerts.""" user = users.get_current_user() sheriff_domains = stored_object.Get(SHERIFF_DOMAINS_KEY) if user: domain_matched = sheriff_domains and any( user.email().endswith('@' + domain) for domain in sheriff_domains) return domain_matched or IsGroupMember( identity=user, group='project-chromium-tryjob-access') return False
def IsValidSheriffUser(): """Checks whether the user should be allowed to triage alerts.""" email = GetEmail() if not email: return False sheriff_domains = stored_object.Get(SHERIFF_DOMAINS_KEY) domain_matched = sheriff_domains and any( email.endswith('@' + domain) for domain in sheriff_domains) return domain_matched or IsTryjobUser()
def _GuessStagingBot(master_name, production_bot_name): staging_bot_map = stored_object.Get('staging_bot_map') or { 'ChromiumPerf': [['win', 'staging_win_perf_bisect'], ['mac', 'staging_mac_10_10_perf_bisect'], ['linux', 'staging_linux_perf_bisect'], ['android', 'staging_android_nexus5X_perf_bisect']] } for infix, staging_bot in staging_bot_map[master_name]: if infix in production_bot_name: return staging_bot
def GetSuiteKey(histograms): assert len(histograms) > 0 # TODO(eakuefner): Refactor this to coalesce the boilerplate (note that this # is all also being done in add_histograms_queue's post handler) master, bot, benchmark = _GetMasterBotBenchmarkFromHistogram( histograms.GetFirstHistogram()) bot_whitelist = stored_object.Get(add_point_queue.BOT_WHITELIST_KEY) internal_only = add_point_queue.BotInternalOnly(bot, bot_whitelist) return add_point_queue.GetOrCreateAncestors(master, bot, benchmark, internal_only).key
def post(self): """Adds a single histogram or sparse shared diagnostic to the datastore. The |data| request parameter can be either a histogram or a sparse shared diagnostic; the set of diagnostics that are considered sparse (meaning that they don't normally change on every upload for a given benchmark from a given bot) is shown in add_histograms.SPARSE_DIAGNOSTIC_TYPES. See https://goo.gl/lHzea6 for detailed information on the JSON format for histograms and diagnostics. Request parameters: data: JSON encoding of a histogram or shared diagnostic. revision: a revision, given as an int. test_path: the test path to which this diagnostic or histogram should be attached. """ datastore_hooks.SetPrivilegedRequest() data = self.request.get('data') revision = int(self.request.get('revision')) test_path = self.request.get('test_path') data_dict = json.loads(data) guid = data_dict['guid'] is_diagnostic = 'type' in data_dict test_path_parts = test_path.split('/') master = test_path_parts[0] bot = test_path_parts[1] test_name = '/'.join(test_path_parts[2:]) bot_whitelist = stored_object.Get(add_point_queue.BOT_WHITELIST_KEY) internal_only = add_point_queue.BotInternalOnly(bot, bot_whitelist) extra_args = {} if is_diagnostic else GetUnitArgs(data_dict['unit']) # TDOO(eakuefner): Populate benchmark_description once it appears in # diagnostics. test_key = add_point_queue.GetOrCreateAncestors( master, bot, test_name, internal_only, **extra_args).key if is_diagnostic: entity = histogram.SparseDiagnostic(id=guid, data=data, test=test_key, start_revision=revision, end_revision=revision, internal_only=internal_only) else: entity = histogram.Histogram(id=guid, data=data, test=test_key, revision=revision, internal_only=internal_only) AddRow(data_dict, test_key, revision, test_path, internal_only) entity.put()
def Get(key): """Gets the value from the datastore.""" if key is None: return None namespaced_key = namespaced_stored_object.NamespaceKey(key) entity = ndb.Key('CachedPickledString', namespaced_key).get(read_policy=ndb.EVENTUAL_CONSISTENCY) if entity: return cPickle.loads(entity.value) else: return stored_object.Get(key)
def GetExternal(key): """Gets the value from the datastore for the externally namespaced key.""" if key is None: return None namespaced_key = _NamespaceKey(key, datastore_hooks.EXTERNAL) entity = ndb.Key('CachedPickledString', namespaced_key).get(read_policy=ndb.EVENTUAL_CONSISTENCY) if entity: return cPickle.loads(entity.value) else: return stored_object.Get(key)
def _GroupableTestSuitePrefixes(cls): """ Returns a list of prefixes of test suites that are transformed to allow the UI to group them. Some are internal-only, but there's no need to store a separate list for external users. """ if cls.GROUPABLE_TEST_SUITE_PREFIXES is None: cls.GROUPABLE_TEST_SUITE_PREFIXES = stored_object.Get( GROUPABLE_TEST_SUITE_PREFIXES_KEY) return cls.GROUPABLE_TEST_SUITE_PREFIXES
def post(self): query = job.Job.query(job.Job.task == None) status = stored_object.Get(_STATUS_KEY) if not status: self._Start(query) self.get() return self._Migrate(query, status) self.get()
def testPost_SetInternalOnlyToFalse(self): # First change to internal only. self._AddSampleData() stored_object.Set(add_point_queue.BOT_WHITELIST_KEY, ['win7']) self.testapp.post('/change_internal_only', [ ('internal_only', 'true'), ('bots', 'ChromiumPerf/win7'), ('bots', 'ChromiumGPU/mac'), ]) self.ExecuteTaskQueueTasks( '/change_internal_only', change_internal_only._QUEUE_NAME) bot_whitelist = stored_object.Get(add_point_queue.BOT_WHITELIST_KEY) self.assertEqual(bot_whitelist, []) # Then change back. self._AddSampleData() self.testapp.post('/change_internal_only', [ ('internal_only', 'false'), ('bots', 'ChromiumPerf/win7'), ('bots', 'ChromiumGPU/mac'), ]) self.ExecuteTaskQueueTasks( '/change_internal_only', change_internal_only._QUEUE_NAME) bot_whitelist = stored_object.Get(add_point_queue.BOT_WHITELIST_KEY) self.assertTrue('win7' in bot_whitelist) self.assertTrue('mac' in bot_whitelist) # No entities should be marked as internal only. bots = graph_data.Bot.query().fetch() for bot in bots: self.assertFalse(bot.internal_only) tests = graph_data.TestMetadata.query().fetch() for test in tests: self.assertFalse(test.internal_only) rows = graph_data.Row.query().fetch() for row in rows: self.assertFalse(row.internal_only)
def testPost_SetInternalOnlyToTrue(self): self._AddSampleData() stored_object.Set(add_point_queue.BOT_WHITELIST_KEY, ['win7']) self.testapp.post('/change_internal_only', [ ('internal_only', 'true'), ('bots', 'ChromiumPerf/win7'), ('bots', 'ChromiumGPU/mac'), ]) self.ExecuteTaskQueueTasks( '/change_internal_only', change_internal_only._QUEUE_NAME) bot_whitelist = stored_object.Get(add_point_queue.BOT_WHITELIST_KEY) self.assertEqual(bot_whitelist, []) # Verify that Bot entities were changed. bots = graph_data.Bot.query().fetch() for bot in bots: path = bot.key.parent().string_id() + '/' + bot.key.string_id() if path == 'ChromiumPerf/win7' or path == 'ChromiumGPU/mac': self.assertTrue(bot.internal_only) else: self.assertFalse(bot.internal_only) # Verify that TestMetadata entities were changed. tests = graph_data.TestMetadata.query().fetch() for test in tests: if (test.test_path.startswith('ChromiumPerf/win7') or test.test_path.startswith('ChromiumGPU/mac')): self.assertTrue(test.internal_only) else: self.assertFalse(test.internal_only) # Verify that Row entities were changed. rows = graph_data.Row.query().fetch() for row in rows: test_path = utils.TestPath(row.key.parent()) if (test_path.startswith('ChromiumPerf/win7') or test_path.startswith('ChromiumGPU/mac')): self.assertTrue(row.internal_only) else: self.assertFalse(row.internal_only) # Verify that Anomaly entities were changed. anomalies = anomaly.Anomaly.query().fetch() for a in anomalies: test_path = utils.TestPath(a.test) if (test_path.startswith('ChromiumPerf/win7') or test_path.startswith('ChromiumGPU/mac')): self.assertTrue(a.internal_only) else: self.assertFalse(a.internal_only)
def testSetAndGet_LargeObject(self): a_large_string = '0' * 2097152 new_account = SampleSerializableClass(a_large_string) stored_object.Set('chris', new_account) chris_account = stored_object.Get('chris') part_entities = stored_object.PartEntity.query().fetch() self.assertEqual(new_account, chris_account) # chris_account object should be stored over 3 PartEntity entities. self.assertEqual(3, len(part_entities))
def _CompositeTestSuites(cls): """ Returns a list of test suites composed of 2 or more test path components. All composite test suites start with a partial test suite, but not all test suites that start with a partial test suite are composite. Some are internal-only, but there's no need to store a separate list for external users. """ if cls.COMPOSITE_TEST_SUITES is None: cls.COMPOSITE_TEST_SUITES = stored_object.Get( COMPOSITE_TEST_SUITES_KEY) return cls.COMPOSITE_TEST_SUITES
def _UpdateBotWhitelist(self, bot_master_names, internal_only): """Updates the global bot_whitelist object, otherwise subsequent add_point calls will overwrite our work.""" bot_whitelist = stored_object.Get(add_point_queue.BOT_WHITELIST_KEY) bot_names = [b.split('/')[1] for b in bot_master_names] if internal_only: bot_whitelist = [b for b in bot_whitelist if b not in bot_names] else: bot_whitelist.extend(bot_names) bot_whitelist = list(set(bot_whitelist)) bot_whitelist.sort() stored_object.Set(add_point_queue.BOT_WHITELIST_KEY, bot_whitelist)
def ServiceAccountHttp(): """Returns the Credentials of the service account if available.""" account_details = stored_object.Get(SERVICE_ACCOUNT_KEY) if not account_details: raise KeyError('Service account credentials not found.') credentials = client.SignedJwtAssertionCredentials( service_account_name=account_details['client_email'], private_key=account_details['private_key'], scope=EMAIL_SCOPE) http = httplib2.Http() credentials.authorize(http) return http
def testPost_ReturnsAndCachesCorrectRevisions(self): self._AddMockData() response = self.testapp.post( '/graph_revisions', {'test_path': 'ChromiumPerf/win7/dromaeo/dom'}) cached_rows = stored_object.Get( 'externally_visible__num_revisions_ChromiumPerf/win7/dromaeo/dom') for index, row in enumerate(json.loads(response.body)): expected_rev = 15000 + (index * 5) expected_value = int(expected_rev) * 2.5 expected_timestamp = utils.TimestampMilliseconds( datetime.datetime(2013, 8, 1)) self.assertEqual([expected_rev, expected_value, expected_timestamp], row) self.assertEqual([expected_rev, expected_value, expected_timestamp], cached_rows[index])
def get(self): """Renders the UI with the form.""" key = self.request.get('key') if not key: self.RenderHtml('edit_site_config.html', {}) return value = stored_object.Get(key) external_value = namespaced_stored_object.GetExternal(key) internal_value = namespaced_stored_object.Get(key) self.RenderHtml('edit_site_config.html', { 'key': key, 'value': _FormatJson(value), 'external_value': _FormatJson(external_value), 'internal_value': _FormatJson(internal_value), })
def testSetAndGet_LargeObject(self): a_large_string = '0' * 2097152 new_account = SampleSerializableClass(a_large_string) stored_object.Set('chris', new_account) chris_account = stored_object.Get('chris') part_entities = stored_object.PartEntity.query().fetch() self.assertEqual(new_account, chris_account) # chris_account object should be stored over 3 PartEntity entities. self.assertEqual(3, len(part_entities)) # Stored over 4 caches here, one extra for the head cache. cache_values = self._GetCachedValues('chris') self.assertEqual(4, len(cache_values))
def post(self): """Accepts posted values, makes changes, and shows the form again.""" key = self.request.get('key') if not utils.IsInternalUser(): self.RenderHtml('edit_site_config.html', { 'error': 'Only internal users can post to this end-point.' }) return if not key: self.RenderHtml('edit_site_config.html', {}) return new_value_json = self.request.get('value').strip() new_external_value_json = self.request.get('external_value').strip() new_internal_value_json = self.request.get('internal_value').strip() template_params = { 'key': key, 'value': new_value_json, 'external_value': new_external_value_json, 'internal_value': new_internal_value_json, } try: new_value = json.loads(new_value_json or 'null') new_external_value = json.loads(new_external_value_json or 'null') new_internal_value = json.loads(new_internal_value_json or 'null') except ValueError: template_params['error'] = 'Invalid JSON in at least one field.' self.RenderHtml('edit_site_config.html', template_params) return old_value = stored_object.Get(key) old_external_value = namespaced_stored_object.GetExternal(key) old_internal_value = namespaced_stored_object.Get(key) stored_object.Set(key, new_value) namespaced_stored_object.SetExternal(key, new_external_value) namespaced_stored_object.Set(key, new_internal_value) _SendNotificationEmail( key, old_value, old_external_value, old_internal_value, new_value, new_external_value, new_internal_value) self.RenderHtml('edit_site_config.html', template_params)