def testPost_EndToEnd(self):
expected = {
'count': 0,
'started': 'Date Time',
'total': 100,
'errors': 0,
}
job_state.JobState.__setstate__ = _JobStateSetState
response = self.testapp.post('/api/migrate', status=200)
self.assertEqual(response.normal_body, json.dumps(expected))
expected = {
'count': 50,
'started': 'Date Time',
'total': 100,
'errors': 0,
}
self.ExecuteDeferredTasks('default', recurse=False)
status = stored_object.Get(migrate._STATUS_KEY)
self.assertEqual(status, expected)
self.ExecuteDeferredTasks('default', recurse=False)
status = stored_object.Get(migrate._STATUS_KEY)
self.assertEqual(status, None)
del job_state.JobState.__setstate__
jobs = job.Job.query().fetch()
for j in jobs:
self.assertEqual(j.state._new_field, 'new value')
def FetchCachedTestSuites():
"""Fetches cached test suite data."""
cache_key = _NamespaceKey(_LIST_SUITES_CACHE_KEY)
cached = stored_object.Get(cache_key)
if cached is None:
# If the cache test suite list is not set, update it before fetching.
# This is for convenience when testing sending of data to a local instance.
namespace = datastore_hooks.GetNamespace()
UpdateTestSuites(namespace)
cached = stored_object.Get(cache_key)
return cached
def ServiceAccountHttp(*args, **kwargs):
"""Returns the Credentials of the service account.
Just prior to the first request, the Http object makes a request to
https://accounts.google.com/o/oauth2/token to get an oauth token. The Http
object is cached in memcache to reduce the number of extraneous requests.
Arguments:
args, kwargs: Arguments passed through to httplib2.Http().
Raises:
KeyError: The service account credentials were not found.
"""
http = memcache.get(SERVICE_ACCOUNT_KEY)
if not http:
account_details = stored_object.Get(SERVICE_ACCOUNT_KEY)
if not account_details:
raise KeyError('Service account credentials not found.')
client.logger.setLevel(logging.WARNING)
credentials = client.SignedJwtAssertionCredentials(
service_account_name=account_details['client_email'],
private_key=account_details['private_key'],
scope=EMAIL_SCOPE)
http = httplib2.Http(*args, **kwargs)
credentials.authorize(http)
memcache.add(SERVICE_ACCOUNT_KEY, http,
time=60 * 50) # Less than 1 hour.
return http
def testPost_WithKey_UpdatesNonNamespacedValues(self):
self.testapp.post('/edit_site_config', {
'key': 'foo',
'value': '[1, 2, 3]',
'xsrf_token': xsrf.GenerateToken(users.get_current_user()),
})
self.assertEqual([1, 2, 3], stored_object.Get('foo'))
def TickMonitoringCustomMetric(metric_name):
"""Increments the stackdriver custom metric with the given name.
This is used for cron job monitoring; if these metrics stop being received
an alert mail is sent. For more information on custom metrics, see
https://cloud.google.com/monitoring/custom-metrics/using-custom-metrics
Args:
metric_name: The name of the metric being monitored.
"""
credentials = client.GoogleCredentials.get_application_default()
monitoring = discovery.build('monitoring', 'v3', credentials=credentials)
now = _GetNowRfc3339()
project_id = stored_object.Get(_PROJECT_ID_KEY)
points = [{
'interval': {
'startTime': now,
'endTime': now,
},
'value': {
'int64Value': _DEFAULT_CUSTOM_METRIC_VAL,
},
}]
write_request = monitoring.projects().timeSeries().create(
name='projects/%s' % project_id,
body={
'timeSeries': [{
'metric': {
'type': 'custom.googleapis.com/%s' % metric_name,
},
'points': points
}]
})
write_request.execute()
def testAddRowsToCache(self):
self._AddMockData()
rows = []
stored_object.Set(
'externally_visible__num_revisions_ChromiumPerf/win7/dromaeo/dom',
[[10, 2, 3], [15, 4, 5], [100, 6, 7]])
test_key = utils.TestKey('ChromiumPerf/win7/dromaeo/dom')
test_container_key = utils.GetTestContainerKey(test_key)
ts1 = datetime.datetime(2013, 1, 1)
row1 = graph_data.Row(parent=test_container_key,
id=1, value=9, timestamp=ts1)
rows.append(row1)
ts2 = datetime.datetime(2013, 1, 2)
row2 = graph_data.Row(parent=test_container_key,
id=12, value=90, timestamp=ts2)
rows.append(row2)
ts3 = datetime.datetime(2013, 1, 3)
row3 = graph_data.Row(parent=test_container_key,
id=102, value=99, timestamp=ts3)
rows.append(row3)
graph_revisions.AddRowsToCache(rows)
self.assertEqual(
[[1, 9, utils.TimestampMilliseconds(ts1)],
[10, 2, 3],
[12, 90, utils.TimestampMilliseconds(ts2)],
[15, 4, 5], [100, 6, 7],
[102, 99, utils.TimestampMilliseconds(ts3)]],
stored_object.Get('externally_visible__num_revisions_'
'ChromiumPerf/win7/dromaeo/dom'))
def ServiceAccountEmail(scope=EMAIL_SCOPE):
account_details = stored_object.Get(SERVICE_ACCOUNT_KEY)
if not account_details:
raise KeyError('Service account credentials not found.')
assert scope, "ServiceAccountHttp scope must not be None."
return account_details['client_email'],
def testSetAndGet_NoMemcache(self, async_mock):
async_mock.return_value = ndb.Future()
async_mock.return_value.set_result(None)
new_account = SampleSerializableClass('Some account data.')
stored_object.Set('chris', new_account)
chris_account = stored_object.Get('chris')
self.assertEqual(new_account, chris_account)
def testSetAndGet_CacheNotExist_CacheSet(self):
new_account = SampleSerializableClass('Some account data.')
stored_object.Set('chris', new_account)
stored_object.MultipartCache.Delete('chris')
chris_account = stored_object.Get('chris')
self.assertEqual(new_account, chris_account)
cache_values = self._GetCachedValues('chris')
self.assertGreater(len(cache_values), 0)
def testPost_WithSomeInvalidJSON_ShowsErrorAndDoesNotModify(self):
stored_object.Set('foo', 'XXX')
response = self.testapp.post('/edit_site_config', {
'key': 'foo',
'value': '[1, 2, this is not json',
'xsrf_token': xsrf.GenerateToken(users.get_current_user()),
})
self.assertIn('Invalid JSON', response.body)
self.assertEqual('XXX', stored_object.Get('foo'))
def _PartialTestSuites(cls):
"""Returns a list of test path component strings that must be joined with
the subsequent test path component in order to form a composite test suite.
Some are internal-only, but there's no need to store a separate list for
external users.
"""
if cls.PARTIAL_TEST_SUITES is None:
cls.PARTIAL_TEST_SUITES = stored_object.Get(
PARTIAL_TEST_SUITES_KEY)
return cls.PARTIAL_TEST_SUITES
def IsValidSheriffUser():
"""Checks whether the user should be allowed to triage alerts."""
user = users.get_current_user()
sheriff_domains = stored_object.Get(SHERIFF_DOMAINS_KEY)
if user:
domain_matched = sheriff_domains and any(
user.email().endswith('@' + domain) for domain in sheriff_domains)
return domain_matched or IsGroupMember(
identity=user, group='project-chromium-tryjob-access')
return False
def IsValidSheriffUser():
"""Checks whether the user should be allowed to triage alerts."""
email = GetEmail()
if not email:
return False
sheriff_domains = stored_object.Get(SHERIFF_DOMAINS_KEY)
domain_matched = sheriff_domains and any(
email.endswith('@' + domain) for domain in sheriff_domains)
return domain_matched or IsTryjobUser()
def _GuessStagingBot(master_name, production_bot_name):
staging_bot_map = stored_object.Get('staging_bot_map') or {
'ChromiumPerf': [['win', 'staging_win_perf_bisect'],
['mac', 'staging_mac_10_10_perf_bisect'],
['linux', 'staging_linux_perf_bisect'],
['android', 'staging_android_nexus5X_perf_bisect']]
}
for infix, staging_bot in staging_bot_map[master_name]:
if infix in production_bot_name:
return staging_bot
def GetSuiteKey(histograms):
assert len(histograms) > 0
# TODO(eakuefner): Refactor this to coalesce the boilerplate (note that this
# is all also being done in add_histograms_queue's post handler)
master, bot, benchmark = _GetMasterBotBenchmarkFromHistogram(
histograms.GetFirstHistogram())
bot_whitelist = stored_object.Get(add_point_queue.BOT_WHITELIST_KEY)
internal_only = add_point_queue.BotInternalOnly(bot, bot_whitelist)
return add_point_queue.GetOrCreateAncestors(master, bot, benchmark,
internal_only).key
def post(self):
"""Adds a single histogram or sparse shared diagnostic to the datastore.
The |data| request parameter can be either a histogram or a sparse shared
diagnostic; the set of diagnostics that are considered sparse (meaning that
they don't normally change on every upload for a given benchmark from a
given bot) is shown in add_histograms.SPARSE_DIAGNOSTIC_TYPES.
See https://goo.gl/lHzea6 for detailed information on the JSON format for
histograms and diagnostics.
Request parameters:
data: JSON encoding of a histogram or shared diagnostic.
revision: a revision, given as an int.
test_path: the test path to which this diagnostic or histogram should be
attached.
"""
datastore_hooks.SetPrivilegedRequest()
data = self.request.get('data')
revision = int(self.request.get('revision'))
test_path = self.request.get('test_path')
data_dict = json.loads(data)
guid = data_dict['guid']
is_diagnostic = 'type' in data_dict
test_path_parts = test_path.split('/')
master = test_path_parts[0]
bot = test_path_parts[1]
test_name = '/'.join(test_path_parts[2:])
bot_whitelist = stored_object.Get(add_point_queue.BOT_WHITELIST_KEY)
internal_only = add_point_queue.BotInternalOnly(bot, bot_whitelist)
extra_args = {} if is_diagnostic else GetUnitArgs(data_dict['unit'])
# TDOO(eakuefner): Populate benchmark_description once it appears in
# diagnostics.
test_key = add_point_queue.GetOrCreateAncestors(
master, bot, test_name, internal_only, **extra_args).key
if is_diagnostic:
entity = histogram.SparseDiagnostic(id=guid,
data=data,
test=test_key,
start_revision=revision,
end_revision=revision,
internal_only=internal_only)
else:
entity = histogram.Histogram(id=guid,
data=data,
test=test_key,
revision=revision,
internal_only=internal_only)
AddRow(data_dict, test_key, revision, test_path, internal_only)
entity.put()
def Get(key):
"""Gets the value from the datastore."""
if key is None:
return None
namespaced_key = namespaced_stored_object.NamespaceKey(key)
entity = ndb.Key('CachedPickledString',
namespaced_key).get(read_policy=ndb.EVENTUAL_CONSISTENCY)
if entity:
return cPickle.loads(entity.value)
else:
return stored_object.Get(key)
def GetExternal(key):
"""Gets the value from the datastore for the externally namespaced key."""
if key is None:
return None
namespaced_key = _NamespaceKey(key, datastore_hooks.EXTERNAL)
entity = ndb.Key('CachedPickledString',
namespaced_key).get(read_policy=ndb.EVENTUAL_CONSISTENCY)
if entity:
return cPickle.loads(entity.value)
else:
return stored_object.Get(key)
def _GroupableTestSuitePrefixes(cls):
"""
Returns a list of prefixes of test suites that are transformed to allow the
UI to group them.
Some are internal-only, but there's no need to store a separate list for
external users.
"""
if cls.GROUPABLE_TEST_SUITE_PREFIXES is None:
cls.GROUPABLE_TEST_SUITE_PREFIXES = stored_object.Get(
GROUPABLE_TEST_SUITE_PREFIXES_KEY)
return cls.GROUPABLE_TEST_SUITE_PREFIXES
def post(self):
query = job.Job.query(job.Job.task == None)
status = stored_object.Get(_STATUS_KEY)
if not status:
self._Start(query)
self.get()
return
self._Migrate(query, status)
self.get()
def testPost_SetInternalOnlyToFalse(self):
# First change to internal only.
self._AddSampleData()
stored_object.Set(add_point_queue.BOT_WHITELIST_KEY, ['win7'])
self.testapp.post('/change_internal_only', [
('internal_only', 'true'),
('bots', 'ChromiumPerf/win7'),
('bots', 'ChromiumGPU/mac'),
])
self.ExecuteTaskQueueTasks(
'/change_internal_only', change_internal_only._QUEUE_NAME)
bot_whitelist = stored_object.Get(add_point_queue.BOT_WHITELIST_KEY)
self.assertEqual(bot_whitelist, [])
# Then change back.
self._AddSampleData()
self.testapp.post('/change_internal_only', [
('internal_only', 'false'),
('bots', 'ChromiumPerf/win7'),
('bots', 'ChromiumGPU/mac'),
])
self.ExecuteTaskQueueTasks(
'/change_internal_only', change_internal_only._QUEUE_NAME)
bot_whitelist = stored_object.Get(add_point_queue.BOT_WHITELIST_KEY)
self.assertTrue('win7' in bot_whitelist)
self.assertTrue('mac' in bot_whitelist)
# No entities should be marked as internal only.
bots = graph_data.Bot.query().fetch()
for bot in bots:
self.assertFalse(bot.internal_only)
tests = graph_data.TestMetadata.query().fetch()
for test in tests:
self.assertFalse(test.internal_only)
rows = graph_data.Row.query().fetch()
for row in rows:
self.assertFalse(row.internal_only)
def testPost_SetInternalOnlyToTrue(self):
self._AddSampleData()
stored_object.Set(add_point_queue.BOT_WHITELIST_KEY, ['win7'])
self.testapp.post('/change_internal_only', [
('internal_only', 'true'),
('bots', 'ChromiumPerf/win7'),
('bots', 'ChromiumGPU/mac'),
])
self.ExecuteTaskQueueTasks(
'/change_internal_only', change_internal_only._QUEUE_NAME)
bot_whitelist = stored_object.Get(add_point_queue.BOT_WHITELIST_KEY)
self.assertEqual(bot_whitelist, [])
# Verify that Bot entities were changed.
bots = graph_data.Bot.query().fetch()
for bot in bots:
path = bot.key.parent().string_id() + '/' + bot.key.string_id()
if path == 'ChromiumPerf/win7' or path == 'ChromiumGPU/mac':
self.assertTrue(bot.internal_only)
else:
self.assertFalse(bot.internal_only)
# Verify that TestMetadata entities were changed.
tests = graph_data.TestMetadata.query().fetch()
for test in tests:
if (test.test_path.startswith('ChromiumPerf/win7') or
test.test_path.startswith('ChromiumGPU/mac')):
self.assertTrue(test.internal_only)
else:
self.assertFalse(test.internal_only)
# Verify that Row entities were changed.
rows = graph_data.Row.query().fetch()
for row in rows:
test_path = utils.TestPath(row.key.parent())
if (test_path.startswith('ChromiumPerf/win7') or
test_path.startswith('ChromiumGPU/mac')):
self.assertTrue(row.internal_only)
else:
self.assertFalse(row.internal_only)
# Verify that Anomaly entities were changed.
anomalies = anomaly.Anomaly.query().fetch()
for a in anomalies:
test_path = utils.TestPath(a.test)
if (test_path.startswith('ChromiumPerf/win7') or
test_path.startswith('ChromiumGPU/mac')):
self.assertTrue(a.internal_only)
else:
self.assertFalse(a.internal_only)
def testSetAndGet_LargeObject(self):
a_large_string = '0' * 2097152
new_account = SampleSerializableClass(a_large_string)
stored_object.Set('chris', new_account)
chris_account = stored_object.Get('chris')
part_entities = stored_object.PartEntity.query().fetch()
self.assertEqual(new_account, chris_account)
# chris_account object should be stored over 3 PartEntity entities.
self.assertEqual(3, len(part_entities))
def _CompositeTestSuites(cls):
"""
Returns a list of test suites composed of 2 or more test path components.
All composite test suites start with a partial test suite, but not all test
suites that start with a partial test suite are composite.
Some are internal-only, but there's no need to store a separate list for
external users.
"""
if cls.COMPOSITE_TEST_SUITES is None:
cls.COMPOSITE_TEST_SUITES = stored_object.Get(
COMPOSITE_TEST_SUITES_KEY)
return cls.COMPOSITE_TEST_SUITES
def _UpdateBotWhitelist(self, bot_master_names, internal_only):
"""Updates the global bot_whitelist object, otherwise subsequent add_point
calls will overwrite our work."""
bot_whitelist = stored_object.Get(add_point_queue.BOT_WHITELIST_KEY)
bot_names = [b.split('/')[1] for b in bot_master_names]
if internal_only:
bot_whitelist = [b for b in bot_whitelist if b not in bot_names]
else:
bot_whitelist.extend(bot_names)
bot_whitelist = list(set(bot_whitelist))
bot_whitelist.sort()
stored_object.Set(add_point_queue.BOT_WHITELIST_KEY, bot_whitelist)
def ServiceAccountHttp():
"""Returns the Credentials of the service account if available."""
account_details = stored_object.Get(SERVICE_ACCOUNT_KEY)
if not account_details:
raise KeyError('Service account credentials not found.')
credentials = client.SignedJwtAssertionCredentials(
service_account_name=account_details['client_email'],
private_key=account_details['private_key'],
scope=EMAIL_SCOPE)
http = httplib2.Http()
credentials.authorize(http)
return http
def testPost_ReturnsAndCachesCorrectRevisions(self):
self._AddMockData()
response = self.testapp.post(
'/graph_revisions', {'test_path': 'ChromiumPerf/win7/dromaeo/dom'})
cached_rows = stored_object.Get(
'externally_visible__num_revisions_ChromiumPerf/win7/dromaeo/dom')
for index, row in enumerate(json.loads(response.body)):
expected_rev = 15000 + (index * 5)
expected_value = int(expected_rev) * 2.5
expected_timestamp = utils.TimestampMilliseconds(
datetime.datetime(2013, 8, 1))
self.assertEqual([expected_rev, expected_value, expected_timestamp], row)
self.assertEqual([expected_rev, expected_value, expected_timestamp],
cached_rows[index])
def get(self):
"""Renders the UI with the form."""
key = self.request.get('key')
if not key:
self.RenderHtml('edit_site_config.html', {})
return
value = stored_object.Get(key)
external_value = namespaced_stored_object.GetExternal(key)
internal_value = namespaced_stored_object.Get(key)
self.RenderHtml('edit_site_config.html', {
'key': key,
'value': _FormatJson(value),
'external_value': _FormatJson(external_value),
'internal_value': _FormatJson(internal_value),
})
def testSetAndGet_LargeObject(self):
a_large_string = '0' * 2097152
new_account = SampleSerializableClass(a_large_string)
stored_object.Set('chris', new_account)
chris_account = stored_object.Get('chris')
part_entities = stored_object.PartEntity.query().fetch()
self.assertEqual(new_account, chris_account)
# chris_account object should be stored over 3 PartEntity entities.
self.assertEqual(3, len(part_entities))
# Stored over 4 caches here, one extra for the head cache.
cache_values = self._GetCachedValues('chris')
self.assertEqual(4, len(cache_values))
def post(self):
"""Accepts posted values, makes changes, and shows the form again."""
key = self.request.get('key')
if not utils.IsInternalUser():
self.RenderHtml('edit_site_config.html', {
'error': 'Only internal users can post to this end-point.'
})
return
if not key:
self.RenderHtml('edit_site_config.html', {})
return
new_value_json = self.request.get('value').strip()
new_external_value_json = self.request.get('external_value').strip()
new_internal_value_json = self.request.get('internal_value').strip()
template_params = {
'key': key,
'value': new_value_json,
'external_value': new_external_value_json,
'internal_value': new_internal_value_json,
}
try:
new_value = json.loads(new_value_json or 'null')
new_external_value = json.loads(new_external_value_json or 'null')
new_internal_value = json.loads(new_internal_value_json or 'null')
except ValueError:
template_params['error'] = 'Invalid JSON in at least one field.'
self.RenderHtml('edit_site_config.html', template_params)
return
old_value = stored_object.Get(key)
old_external_value = namespaced_stored_object.GetExternal(key)
old_internal_value = namespaced_stored_object.Get(key)
stored_object.Set(key, new_value)
namespaced_stored_object.SetExternal(key, new_external_value)
namespaced_stored_object.Set(key, new_internal_value)
_SendNotificationEmail(
key, old_value, old_external_value, old_internal_value,
new_value, new_external_value, new_internal_value)
self.RenderHtml('edit_site_config.html', template_params)
