def _UpdateCache(test_key):
"""Queries Rows for a test then updates the cache.
Args:
test_key: ndb.Key for a TestMetadata entity.
Returns:
The list of triplets that was just fetched and set in the cache.
"""
test = test_key.get()
if not test:
return []
assert utils.IsInternalUser() or not test.internal_only
datastore_hooks.SetSinglePrivilegedRequest()
# A projection query queries just for the values of particular properties;
# this is faster than querying for whole entities.
query = graph_data.Row.query(projection=['revision', 'value', 'timestamp'])
query = query.filter(
graph_data.Row.parent_test == utils.OldStyleTestKey(test_key))
# Using a large batch_size speeds up queries with > 1000 Rows.
rows = map(_MakeTriplet, query.iter(batch_size=1000))
# Note: Unit tests do not call datastore_hooks with the above query, but
# it is called in production and with more recent SDK.
datastore_hooks.CancelSinglePrivilegedRequest()
SetCache(utils.TestPath(test_key), rows)
return rows
def testPost_ExternalUserRequestsInternalOnlySheriff_ErrorMessage(self):
self.UnsetCurrentUser()
sheriff.Sheriff(id='Foo', internal_only=True).put()
self.assertFalse(utils.IsInternalUser())
response = self.testapp.post('/alerts?sheriff=Foo')
error = self.GetJsonValue(response, 'error')
self.assertIsNotNone(error)
def Authorize():
try:
email = utils.GetEmail()
except oauth.OAuthRequestError:
raise OAuthError
if not email:
raise NotLoggedInError
try:
# TODO(dberris): Migrate to using Cloud IAM and checking roles instead, to
# allow for dynamic management of the accounts.
if not email.endswith('.gserviceaccount.com'):
# For non-service accounts, need to verify that the OAuth client ID
# is in our allowlist.
client_id = oauth.get_client_id(utils.OAUTH_SCOPES)
if client_id not in OAUTH_CLIENT_ID_ALLOWLIST:
logging.error('OAuth client id %s for user %s not in allowlist',
client_id, email)
raise OAuthError
except oauth.OAuthRequestError:
# Transient errors when checking the token result should result in HTTP 500,
# so catch oauth.OAuthRequestError here, not oauth.Error (which would catch
# both fatal and transient errors).
raise OAuthError
logging.info('OAuth user logged in as: %s', email)
if utils.IsInternalUser():
datastore_hooks.SetPrivilegedRequest()
def post(self, *args):
"""Returns alert data in response to API requests.
Outputs:
JSON results.
"""
self._SetCorsHeadersIfAppropriate()
try:
api_auth.Authorize()
except api_auth.NotLoggedInError as e:
if not self._AllowAnonymous():
self.WriteErrorMessage(e.message, 401)
return
except api_auth.OAuthError as e:
self.WriteErrorMessage(e.message, 403)
return
# Allow oauth.Error to manifest as HTTP 500.
try:
if utils.IsInternalUser():
results = self.PrivilegedPost(*args)
else:
results = self.UnprivilegedPost(*args)
self.response.out.write(json.dumps(results))
except NotFoundError as e:
self.WriteErrorMessage(e.message, 404)
except ForbiddenError as e:
self.WriteErrorMessage(e.message, 403)
except (BadRequestError, KeyError, TypeError, ValueError) as e:
self.WriteErrorMessage(e.message, 400)
def Authorize():
try:
email = utils.GetEmail()
except oauth.OAuthRequestError:
raise OAuthError
if not email:
raise NotLoggedInError
try:
if not email.endswith('.gserviceaccount.com'):
# For non-service account, need to verify that the OAuth client ID
# is in our whitelist.
client_id = oauth.get_client_id(utils.OAUTH_SCOPES)
if client_id not in OAUTH_CLIENT_ID_WHITELIST:
logging.error('OAuth client id %s for user %s not in whitelist',
client_id, email)
email = None
raise OAuthError
except oauth.OAuthRequestError:
# Transient errors when checking the token result should result in HTTP 500,
# so catch oauth.OAuthRequestError here, not oauth.Error (which would catch
# both fatal and transient errors).
raise OAuthError
logging.info('OAuth user logged in as: %s', email)
if utils.IsInternalUser():
datastore_hooks.SetPrivilegedRequest()
def post(self):
"""POSTS the data to the datastore."""
user = users.get_current_user()
if not user:
self.response.out.write(
json.dumps({'error': 'User not logged in.'}))
return
if not utils.IsInternalUser():
self.response.out.write(
json.dumps({
'error':
'Unauthorized access, please use chromium account to login.'
}))
return
get_token = self.request.get('getToken')
if get_token == 'true':
values = {}
self.GetDynamicVariables(values)
self.response.out.write(
json.dumps({
'xsrf_token': values['xsrf_token'],
}))
else:
self._CreateTableConfig()
def testGetHistogram_Internal_Fails(self):
self.UnsetCurrentUser()
self._AddMockData(_TEST_HISTOGRAM_DATA, True)
self.assertFalse(utils.IsInternalUser())
self.testapp.post(
'/get_histogram', {'guid': _TEST_HISTOGRAM_DATA['guid']}, status=400)
def _PrefillInfo(test_path):
"""Pre-fills some best guesses config form based on the test path.
Args:
test_path: Test path string.
Returns:
A dictionary indicating the result. If successful, this should contain the
the fields "suite", "email", "all_metrics", and "default_metric". If not
successful this will contain the field "error".
"""
if not test_path:
return {'error': 'No test specified'}
suite_path = '/'.join(test_path.split('/')[:3])
suite = utils.TestKey(suite_path).get()
if not suite:
return {'error': 'Invalid test %s' % test_path}
graph_path = '/'.join(test_path.split('/')[:4])
graph_key = utils.TestKey(graph_path)
info = {'suite': suite.test_name}
info['master'] = suite.master_name
info['internal_only'] = suite.internal_only
info['use_archive'] = _CanDownloadBuilds(suite.master_name)
info['all_bots'] = _GetAvailableBisectBots(suite.master_name)
info['bisect_bot'] = GuessBisectBot(suite.master_name, suite.bot_name)
user = users.get_current_user()
if not user:
return {'error': 'User not logged in.'}
# Secondary check for bisecting internal only tests.
if suite.internal_only and not utils.IsInternalUser():
return {
'error': 'Unauthorized access, please use corp account to login.'
}
if users.is_current_user_admin():
info['is_admin'] = True
else:
info['is_admin'] = False
info['email'] = user.email()
info['all_metrics'] = []
metric_keys = list_tests.GetTestDescendants(graph_key, has_rows=True)
for metric_key in metric_keys:
metric_path = utils.TestPath(metric_key)
if metric_path.endswith('/ref') or metric_path.endswith('_ref'):
continue
info['all_metrics'].append(GuessMetric(metric_path))
info['default_metric'] = GuessMetric(test_path)
info['story_filter'] = GuessStoryFilter(test_path)
return info
def _ShowErrorIfNotLoggedIn(self):
"""Shows an error message if not logged in with an internal account.
Returns:
True if error message shown, False otherwise.
"""
if not utils.IsInternalUser():
self.RenderHtml('result.html', {
'errors': ['Only logged-in internal users can set warnings.']
})
return True
return False
def _FetchLatestRows(test, num_points):
"""Does a query for the latest Row entities in the given test.
Args:
test: A TestMetadata entity to fetch Row entities for.
num_points: Number of points to fetch.
Returns:
A list of Row entities, ordered by revision. The number to fetch is limited
to the number that is expected to be processed at once by GASP.
"""
assert utils.IsInternalUser() or not test.internal_only
datastore_hooks.SetSinglePrivilegedRequest()
return list(reversed(graph_data.GetLatestRowsForTest(test.key, num_points)))
def testPost_ExternalUserRequestsInternalOnlySheriff_ErrorMessage(self):
self.UnsetCurrentUser()
self.assertFalse(utils.IsInternalUser())
with mock.patch.object(
SheriffConfigClient, 'List',
mock.MagicMock(return_value=([
Subscription(
name='Chromium Perf Sheriff',
notification_email='*****@*****.**',
)
], None))):
response = self.testapp.post('/alerts?sheriff=Foo')
error = self.GetJsonValue(response, 'error')
self.assertIsNotNone(error)
def post(self):
"""Accepts posted values, makes changes, and shows the form again."""
key = self.request.get('key')
if not utils.IsInternalUser():
self.RenderHtml('edit_site_config.html', {
'error': 'Only internal users can post to this end-point.'
})
return
if not key:
self.RenderHtml('edit_site_config.html', {})
return
new_value_json = self.request.get('value').strip()
new_external_value_json = self.request.get('external_value').strip()
new_internal_value_json = self.request.get('internal_value').strip()
template_params = {
'key': key,
'value': new_value_json,
'external_value': new_external_value_json,
'internal_value': new_internal_value_json,
}
try:
new_value = json.loads(new_value_json or 'null')
new_external_value = json.loads(new_external_value_json or 'null')
new_internal_value = json.loads(new_internal_value_json or 'null')
except ValueError:
template_params['error'] = 'Invalid JSON in at least one field.'
self.RenderHtml('edit_site_config.html', template_params)
return
old_value = stored_object.Get(key)
old_external_value = namespaced_stored_object.GetExternal(key)
old_internal_value = namespaced_stored_object.Get(key)
stored_object.Set(key, new_value)
namespaced_stored_object.SetExternal(key, new_external_value)
namespaced_stored_object.Set(key, new_internal_value)
_SendNotificationEmail(
key, old_value, old_external_value, old_internal_value,
new_value, new_external_value, new_internal_value)
self.RenderHtml('edit_site_config.html', template_params)
def IsUnalteredQueryPermitted():
"""Checks if the current user is internal, or the request is privileged.
"Internal users" are users whose email address belongs to a certain
privileged domain; but some privileged requests, such as task queue tasks,
are also considered privileged.
Returns:
True for users with google.com emails and privileged requests.
"""
if utils.IsInternalUser():
return True
if users.is_current_user_admin():
# It's possible to be an admin with a non-internal account; For example,
# the default login for dev appserver instances is [email protected].
return True
return _IsServicingPrivilegedRequest()
def _FetchRowsAroundRev(test, revision, num_before, num_after):
"""Fetches Row entities before and after a given revision.
Args:
test: A TestMetadata entity.
revision: A Row ID.
num_before: Maximum number of Rows before |revision| to fetch.
num_after: Max number of Rows starting from |revision| to fetch.
Returns:
A list of Row entities ordered by ID. The Row entities will have at least
the "revision" and "value" properties, which are the only ones relevant
to their use in this module.
"""
assert utils.IsInternalUser() or not test.internal_only
return graph_data.GetRowsForTestBeforeAfterRev(test.key, revision,
num_before, num_after)
def GetDynamicVariables(self, template_values, request_path=None):
"""Gets the values that vary for every page.
Args:
template_values: dict of name/value pairs.
request_path: path for login urls, None if using the current path.
"""
user_info = ''
xsrf_token = ''
user = users.get_current_user()
display_username = '******'
title = 'Sign in to an account'
is_admin = False
if user:
display_username = user.email()
title = 'Switch user'
xsrf_token = xsrf.GenerateToken(user)
is_admin = users.is_current_user_admin()
try:
login_url = users.create_login_url(request_path
or self.request.path_qs)
except users.RedirectTooLongError:
# On the bug filing pages, the full login URL can be too long. Drop
# the correct redirect URL, since the user should already be logged in at
# this point anyway.
login_url = users.create_login_url('/')
user_info = '<a href="%s" title="%s">%s</a>' % (login_url, title,
display_username)
# Force out of passive login, as it creates multilogin issues.
login_url = login_url.replace('passive=true', 'passive=false')
template_values['login_url'] = login_url
template_values['display_username'] = display_username
template_values['user_info'] = user_info
template_values['is_admin'] = is_admin
template_values['is_internal_user'] = utils.IsInternalUser()
template_values['xsrf_token'] = xsrf_token
template_values['xsrf_input'] = (
'<input type="hidden" name="xsrf_token" value="%s">' % xsrf_token)
template_values['login_url'] = login_url
return template_values
def post(self):
if not utils.IsInternalUser():
self.response.out.write(json.dumps({
'error': 'You are not authorized to post to this endpoint.',
}))
return
job = buildbucket_job.BisectJob(
'linux_perf_bisector',
self.request.get('good_revision'),
self.request.get('bad_revision'),
self.request.get('command'),
self.request.get('metric'),
self.request.get('repeat_count'),
self.request.get('max_time_minutes'),
self.request.get('bug_id'),
self.request.get('gs_bucket'),
self.request.get('recipe_tester_name'),
self.request.get('builder_host'),
self.request.get('builder_port'))
buildbucket_service.PutJob(job)
self.response.out.write(job.response_fields)
def _CheckIsInternalUser(self):
if utils.IsDevAppserver():
return
self._CheckIsLoggedIn()
if not utils.IsInternalUser():
raise ForbiddenError()
def ListStaticTemplates():
return [
handler for handler in STATIC_TEMPLATES
if (not handler.template.internal_only) or utils.IsInternalUser()
]
def _PrefillInfo(test_path):
"""Pre-fills some best guesses config form based on the test path.
Args:
test_path: Test path string.
Returns:
A dictionary indicating the result. If successful, this should contain the
the fields "suite", "email", "all_metrics", and "default_metric". If not
successful this will contain the field "error".
"""
if not test_path:
return {'error': 'No test specified'}
suite_path = '/'.join(test_path.split('/')[:3])
suite = utils.TestKey(suite_path).get()
if not suite:
return {'error': 'Invalid test %s' % test_path}
graph_path = '/'.join(test_path.split('/')[:4])
graph_key = utils.TestKey(graph_path)
info = {'suite': suite.test_name}
info['master'] = suite.master_name
info['internal_only'] = suite.internal_only
info['use_archive'] = _CanDownloadBuilds(suite.master_name)
info['all_bots'] = _GetAvailableBisectBots(suite.master_name)
info['bisect_bot'] = GuessBisectBot(suite.master_name, suite.bot_name)
user = users.get_current_user()
if not user:
return {'error': 'User not logged in.'}
# Secondary check for bisecting internal only tests.
if suite.internal_only and not utils.IsInternalUser():
return {
'error': 'Unauthorized access, please use corp account to login.'
}
info['email'] = user.email()
info['all_metrics'] = []
metric_keys = list_tests.GetTestDescendants(graph_key, has_rows=True)
should_add_story_filter = (
suite.test_name not in _NON_TELEMETRY_TEST_COMMANDS and
# is not a top-level test_path, those are usually not story names
'/' in test_path)
test_path_prefix = test_path + '/'
for metric_key in metric_keys:
metric_path = utils.TestPath(metric_key)
if metric_path.endswith('/ref') or metric_path.endswith('_ref'):
continue
if metric_path.startswith(test_path_prefix):
should_add_story_filter = False # Stories do not have sub-tests.
info['all_metrics'].append(GuessMetric(metric_path))
info['default_metric'] = GuessMetric(test_path)
if should_add_story_filter:
_, story_name = test_path.rsplit('/', 1)
# During import, some chars in story names got replaced by "_" so they
# could be safely included in the test_path. At this point we don't know
# what the original characters were, so we pass a regex where each
# underscore is replaced back with a match-any-character dot.
info['story_filter'] = re.sub(r'\\_', '.', re.escape(story_name))
else:
info['story_filter'] = ''
return info
def _PrefillInfo(test_path):
"""Pre-fills some best guesses config form based on the test path.
Args:
test_path: Test path string.
Returns:
A dictionary indicating the result. If successful, this should contain the
the fields "suite", "email", "all_metrics", and "default_metric". If not
successful this will contain the field "error".
"""
if not test_path:
return {'error': 'No test specified'}
suite_path = '/'.join(test_path.split('/')[:3])
suite = utils.TestKey(suite_path).get()
if not suite:
return {'error': 'Invalid test %s' % test_path}
graph_path = '/'.join(test_path.split('/')[:4])
graph_key = utils.TestKey(graph_path)
info = {'suite': suite.test_name}
info['master'] = suite.master_name
info['internal_only'] = suite.internal_only
info['use_archive'] = _CanDownloadBuilds(suite.master_name)
info['all_bots'] = _GetAvailableBisectBots(suite.master_name)
info['bisect_bot'] = GuessBisectBot(suite.master_name, suite.bot_name)
user = users.get_current_user()
if not user:
return {'error': 'User not logged in.'}
# Secondary check for bisecting internal only tests.
if suite.internal_only and not utils.IsInternalUser():
return {
'error': 'Unauthorized access, please use corp account to login.'
}
info['email'] = user.email()
info['all_metrics'] = []
metric_keys = list_tests.GetTestDescendants(graph_key, has_rows=True)
should_add_story_filter = (
suite.test_name not in _NON_TELEMETRY_TEST_COMMANDS and
# is not a top-level test_path, those are usually not story names
'/' in test_path)
test_path_prefix = test_path + '/'
for metric_key in metric_keys:
metric_path = utils.TestPath(metric_key)
if metric_path.endswith('/ref') or metric_path.endswith('_ref'):
continue
if metric_path.startswith(test_path_prefix):
should_add_story_filter = False # Stories do not have sub-tests.
info['all_metrics'].append(GuessMetric(metric_path))
info['default_metric'] = GuessMetric(test_path)
if should_add_story_filter:
_, story_name = test_path.rsplit('/', 1)
if story_name.startswith('after_'):
# TODO(perezju,#1811): Remove this hack after deprecating the
# memory.top_10_mobile benchmark.
story_name = story_name[len('after_'):]
# During import, some chars in story names got replaced by "_" so they
# could be safely included in the test_path. At this point we don't know
# what the original characters were. Additionally, some special characters
# and argument quoting are not interpreted correctly, e.g. by bisect
# scripts (crbug.com/662472). We thus keep only a small set of "safe chars"
# and replace all others with match-any-character regex dots.
info['story_filter'] = re.sub(r'[^a-zA-Z0-9]', '.', story_name)
else:
info['story_filter'] = ''
return info
