def testGet_ExternalUserRequestsInternalOnlySheriff_ErrorMessage(self):
sheriff.Sheriff(id='Foo', internal_only=True).put()
self.assertFalse(utils.IsInternalUser())
response = self.testapp.get('/alerts?sheriff=Foo')
self.assertIn('class="error"', response.body)
self.assertEqual(0, len(response.html('img')))
self.assertEqual(0, len(response.html('alerts-table')))
def _UpdateCache(test_key):
"""Queries Rows for a test then updates the cache.
Args:
test_key: ndb.Key for a TestMetadata entity.
Returns:
The list of triplets that was just fetched and set in the cache.
"""
test = test_key.get()
if not test:
return []
assert utils.IsInternalUser() or not test.internal_only
datastore_hooks.SetSinglePrivilegedRequest()
# A projection query queries just for the values of particular properties;
# this is faster than querying for whole entities.
query = graph_data.Row.query(projection=['revision', 'value', 'timestamp'])
query = query.filter(
graph_data.Row.parent_test == utils.OldStyleTestKey(test_key))
# Using a large batch_size speeds up queries with > 1000 Rows.
rows = map(_MakeTriplet, query.iter(batch_size=1000))
# Note: Unit tests do not call datastore_hooks with the above query, but
# it is called in production and with more recent SDK.
datastore_hooks.CancelSinglePrivilegedRequest()
SetCache(utils.TestPath(test_key), rows)
return rows
def _FetchRowsAroundRev(test, revision, num_before, num_after):
"""Fetches Row entities before and after a given revision.
Args:
test: A Test entity.
revision: A Row ID.
num_before: Maximum number of Rows before |revision| to fetch.
num_after: Max number of Rows starting from |revision| to fetch.
Returns:
A list of Row entities ordered by ID. The Row entities will have at least
the "revision" and "value" properties, which are the only ones relevant
to their use in this module.
"""
assert utils.IsInternalUser() or not test.internal_only
query = graph_data.Row.query(projection=['revision', 'value'])
query = query.filter(graph_data.Row.parent_test == test.key)
before_query = query.filter(graph_data.Row.revision < revision)
before_query = before_query.order(-graph_data.Row.revision)
datastore_hooks.SetSinglePrivilegedRequest()
rows_before = list(reversed(before_query.fetch(limit=num_before)))
after_query = query.filter(graph_data.Row.revision >= revision)
after_query = after_query.order(graph_data.Row.revision)
datastore_hooks.SetSinglePrivilegedRequest()
rows_at_and_after = after_query.fetch(num_after)
return rows_before + rows_at_and_after
def _PrefillInfo(test_path):
"""Pre-fills some best guesses config form based on the test path.
Args:
test_path: Test path string.
Returns:
A dictionary indicating the result. If successful, this should contain the
the fields "suite", "email", "all_metrics", and "default_metric". If not
successful this will contain the field "error".
"""
if not test_path:
return {'error': 'No test specified'}
suite_path = '/'.join(test_path.split('/')[:3])
suite = utils.TestKey(suite_path).get()
if not suite:
return {'error': 'Invalid test %s' % test_path}
graph_path = '/'.join(test_path.split('/')[:4])
graph_key = utils.TestKey(graph_path)
info = {'suite': suite.key.string_id()}
info['master'] = suite.master_name
info['internal_only'] = suite.internal_only
info['use_archive'] = _CanDownloadBuilds(suite.master_name)
info['all_bots'] = _GetAvailableBisectBots(suite.master_name)
info['bisect_bot'] = GuessBisectBot(suite.master_name, suite.bot_name)
user = users.get_current_user()
if not user:
return {'error': 'User not logged in.'}
# Secondary check for bisecting internal only tests.
if suite.internal_only and not utils.IsInternalUser():
return {
'error': 'Unauthorized access, please use corp account to login.'
}
info['email'] = user.email()
info['all_metrics'] = []
metric_keys_query = graph_data.Test.query(graph_data.Test.has_rows == True,
ancestor=graph_key)
metric_keys = metric_keys_query.fetch(keys_only=True)
for metric_key in metric_keys:
metric_path = utils.TestPath(metric_key)
if metric_path.endswith('/ref') or metric_path.endswith('_ref'):
continue
info['all_metrics'].append(GuessMetric(metric_path))
info['default_metric'] = GuessMetric(test_path)
return info
def _ShowErrorIfNotLoggedIn(self):
"""Shows an error message if not logged in with an internal account.
Returns:
True if error message shown, False otherwise.
"""
if not utils.IsInternalUser():
self.RenderHtml('result.html', {
'errors': ['Only logged-in internal users can set warnings.']
})
return True
return False
def get(self):
"""Shows a set of statistics, or a form for producing stats."""
if not utils.IsInternalUser():
self.RenderHtml('result.html', {
'errors': ['Only logged-in internal users can access stats.']
})
return
key = self.request.get('key')
if key:
self._DisplayResults(key)
else:
self._DisplayForm()
def post(self):
"""Accepts posted values, makes changes, and shows the form again."""
key = self.request.get('key')
if not utils.IsInternalUser():
self.RenderHtml('edit_site_config.html', {
'error': 'Only internal users can post to this end-point.'
})
return
if not key:
self.RenderHtml('edit_site_config.html', {})
return
new_value_json = self.request.get('value').strip()
new_external_value_json = self.request.get('external_value').strip()
new_internal_value_json = self.request.get('internal_value').strip()
template_params = {
'key': key,
'value': new_value_json,
'external_value': new_external_value_json,
'internal_value': new_internal_value_json,
}
try:
new_value = json.loads(new_value_json or 'null')
new_external_value = json.loads(new_external_value_json or 'null')
new_internal_value = json.loads(new_internal_value_json or 'null')
except ValueError:
template_params['error'] = 'Invalid JSON in at least one field.'
self.RenderHtml('edit_site_config.html', template_params)
return
old_value = stored_object.Get(key)
old_external_value = namespaced_stored_object.GetExternal(key)
old_internal_value = namespaced_stored_object.Get(key)
stored_object.Set(key, new_value)
namespaced_stored_object.SetExternal(key, new_external_value)
namespaced_stored_object.Set(key, new_internal_value)
_SendNotificationEmail(
key, old_value, old_external_value, old_internal_value,
new_value, new_external_value, new_internal_value)
self.RenderHtml('edit_site_config.html', template_params)
def IsUnalteredQueryPermitted():
"""Checks if the current user is internal, or the request is privileged.
"Internal users" are users whose email address belongs to a certain
privileged domain; but some privileged requests, such as task queue tasks,
are also considered privileged.
Returns:
True for users with google.com emails and privileged requests.
"""
if utils.IsInternalUser():
return True
if users.is_current_user_admin():
# It's possible to be an admin with a non-internal account; For example,
# the default login for dev appserver instances is [email protected].
return True
return _IsServicingPrivilegedRequest()
def post(self):
"""Kicks off a task on the task queue to generate the requested stats."""
if not utils.IsInternalUser():
self.RenderHtml('result.html', {
'errors': ['Only logged-in internal users can access stats.']
})
return
datastore_hooks.SetPrivilegedRequest()
stat_type = self.request.get('type')
stat_container = StatContainer(stat_type=stat_type)
if stat_type == 'around_revision':
self._StartGeneratingStatsAroundRevision(stat_container)
elif stat_type == 'alert_summary':
self._StartGeneratingStatsForAlerts(stat_container)
self.redirect('/stats?key=%s' % stat_container.key.urlsafe())
def _FetchLatestRows(test, num_points):
"""Does a query for the latest Row entities in the given test.
Args:
test: A TestMetadata entity to fetch Row entities for.
num_points: Number of points to fetch.
Returns:
A list of Row entities, ordered by revision. The number to fetch is limited
to the number that is expected to be processed at once by GASP.
"""
assert utils.IsInternalUser() or not test.internal_only
datastore_hooks.SetSinglePrivilegedRequest()
return list(
reversed(
graph_data.GetLatestRowsForTest(test.key,
num_points,
privileged=True)))
def _FetchLatestRows(test, num_points):
"""Does a query for the latest Row entities in the given test.
Args:
test: A Test entity to fetch Row entities for.
num_points: Number of points to fetch.
Returns:
A list of Row entities, ordered by revision. The number to fetch is limited
to the number that is expected to be processed at once by GASP.
"""
assert utils.IsInternalUser() or not test.internal_only
datastore_hooks.SetSinglePrivilegedRequest()
q = graph_data.Row.query(projection=['revision', 'value'])
q = q.filter(graph_data.Row.parent_test == test.key)
q = q.order(-graph_data.Row.revision)
rows = list(reversed(q.fetch(limit=num_points)))
return rows
def _FetchRowsAroundRev(test, revision, num_before, num_after):
"""Fetches Row entities before and after a given revision.
Args:
test: A TestMetadata entity.
revision: A Row ID.
num_before: Maximum number of Rows before |revision| to fetch.
num_after: Max number of Rows starting from |revision| to fetch.
Returns:
A list of Row entities ordered by ID. The Row entities will have at least
the "revision" and "value" properties, which are the only ones relevant
to their use in this module.
"""
assert utils.IsInternalUser() or not test.internal_only
return graph_data.GetRowsForTestBeforeAfterRev(test.key,
revision,
num_before,
num_after,
privileged=True)
def RenderHtml(self, template_file, template_values, status=200):
"""Renders HTML given template and values.
Args:
template_file: string. File name under templates directory.
template_values: dict. Mapping of template variables to corresponding
values.
status: int. HTTP status code.
"""
self.response.set_status(status)
template = JINJA2_ENVIRONMENT.get_template(template_file)
user_info = ''
xsrf_token = ''
user = users.get_current_user()
display_username = '******'
title = 'Sign in to an account'
is_admin = False
if user:
display_username = user.email()
title = 'Switch user'
xsrf_token = xsrf.GenerateToken(user)
is_admin = users.is_current_user_admin()
try:
login_url = users.create_login_url(self.request.path_qs)
except users.RedirectTooLongError:
# On the bug filing pages, the full login URL can be too long. Drop
# the correct redirect URL, since the user should already be logged in at
# this point anyway.
login_url = users.create_login_url('/')
user_info = '<a href="%s" title="%s">%s</a>' % (login_url, title,
display_username)
template_values['user_info'] = user_info
template_values['is_admin'] = is_admin
template_values['is_internal_user'] = utils.IsInternalUser()
template_values['xsrf_token'] = xsrf_token
template_values['xsrf_input'] = (
'<input type="hidden" name="xsrf_token" value="%s">' % xsrf_token)
template_values['login_url'] = login_url
self.response.out.write(template.render(template_values))
def GetDynamicVariables(self, template_values, request_path=None):
"""Gets the values that vary for every page.
Args:
template_values: dict of name/value pairs.
request_path: path for login urls, None if using the current path.
"""
user_info = ''
xsrf_token = ''
user = users.get_current_user()
display_username = '******'
title = 'Sign in to an account'
is_admin = False
if user:
display_username = user.email()
title = 'Switch user'
xsrf_token = xsrf.GenerateToken(user)
is_admin = users.is_current_user_admin()
try:
login_url = users.create_login_url(request_path
or self.request.path_qs)
except users.RedirectTooLongError:
# On the bug filing pages, the full login URL can be too long. Drop
# the correct redirect URL, since the user should already be logged in at
# this point anyway.
login_url = users.create_login_url('/')
user_info = '<a href="%s" title="%s">%s</a>' % (login_url, title,
display_username)
template_values['login_url'] = login_url
template_values['display_username'] = display_username
template_values['user_info'] = user_info
template_values['is_admin'] = is_admin
template_values['is_internal_user'] = utils.IsInternalUser()
template_values['xsrf_token'] = xsrf_token
template_values['xsrf_input'] = (
'<input type="hidden" name="xsrf_token" value="%s">' % xsrf_token)
template_values['login_url'] = login_url
return template_values
def post(self):
if not utils.IsInternalUser():
self.response.out.write(
json.dumps({
'error':
'You are not authorized to post to this endpoint.',
}))
return
job = buildbucket_job.BisectJob(self.request.get('platform'),
self.request.get('good_revision'),
self.request.get('bad_revision'),
self.request.get('command'),
self.request.get('metric'),
self.request.get('repeat_count'),
self.request.get('truncate_percent'),
self.request.get('max_time_minutes'),
self.request.get('bug_id'),
self.request.get('gs_bucket'),
self.request.get('builder_host'),
self.request.get('builder_port'))
buildbucket_service.PutJob(job)
self.response.out.write(job.response_fields)
def post(self):
"""Accepts posted values, makes changes, and shows the form again."""
key = self.request.get('key')
if not utils.IsInternalUser():
self.RenderHtml('edit_site_config.html', {
'error': 'Only internal users can post to this end-point.'
})
return
if not key:
self.RenderHtml('edit_site_config.html', {})
return
value = self.request.get('value').strip()
external_value = self.request.get('external_value').strip()
internal_value = self.request.get('internal_value').strip()
template_params = {
'key': key,
'value': value,
'external_value': external_value,
'internal_value': internal_value,
}
try:
if value:
stored_object.Set(key, json.loads(value))
if external_value:
namespaced_stored_object.SetExternal(key, json.loads(external_value))
if internal_value:
namespaced_stored_object.Set(key, json.loads(internal_value))
except ValueError:
template_params['error'] = 'Invalid JSON in at least one field.'
_SendNotificationEmail(key, template_params)
self.RenderHtml('edit_site_config.html', template_params)
def testPost_ExternalUserRequestsInternalOnlySheriff_ErrorMessage(self):
sheriff.Sheriff(id='Foo', internal_only=True).put()
self.assertFalse(utils.IsInternalUser())
response = self.testapp.post('/alerts?sheriff=Foo')
error = self.GetJsonValue(response, 'error')
self.assertIsNotNone(error)
def _GetFlotJson(revision_map, tests, show_old_data_warning):
"""Constructs JSON in the format expected by Flot.
Args:
revision_map: A dict which maps revision numbers to data point info.
tests: A list of Test entities.
show_old_data_warning: Whether to a show a warning to the user that
the graph data is out of date.
Returns:
JSON serialization of a dict with line data, annotations, error range data,
and possibly warning information. (This data may not be passed exactly
as-is to the Flot plot funciton, but it will all be used when plotting.)
"""
# TODO(qyearsley): Break this function into smaller functions.
# Each entry in the following dict is one Flot series object. The actual
# x-y values will be put into the 'data' properties for each object.
cols = {i: _FlotSeries(i) for i in range(len(tests))}
flot_annotations = {}
flot_annotations['series'] = _GetSeriesAnnotations(tests)
# For each Test (which corresponds to a trace line), the shaded error
# region is specified by two series objects. For a demo, see:
# http://www.flotcharts.org/flot/examples/percentiles/index.html
error_bars = {
x: [{
'id': 'bottom_%d' % x,
'data': [],
'color': x,
'clickable': False,
'hoverable': False,
'lines': {
'show': True,
'lineWidth': 0,
'fill': 0.2,
},
'fillBetween': 'line_%d' % x,
}, {
'id': 'top_%d' % x,
'data': [],
'color': x,
'clickable': False,
'hoverable': False,
'lines': {
'show': True,
'lineWidth': 0,
'fill': 0.2,
},
'fillBetween': 'line_%d' % x,
}]
for x, _ in enumerate(tests)
}
test_keys = [t.key.urlsafe() for t in tests]
last_timestamp = None
has_points = False
for revision in sorted(revision_map.keys()):
for series_index, key in enumerate(test_keys):
point_info = revision_map[revision].get(key, None)
if not point_info:
continue
has_points = True
timestamp = point_info.get('timestamp')
if timestamp:
if type(timestamp) is datetime.datetime:
point_info['timestamp'] = utils.TimestampMilliseconds(
timestamp)
if not last_timestamp or point_info[
'timestamp'] > last_timestamp:
last_timestamp = point_info['timestamp']
point_list = [revision, point_info['value']]
if 'error' in point_info:
error = point_info['error']
error_bars[series_index][0]['data'].append(
[revision, point_info['value'] - error])
error_bars[series_index][1]['data'].append(
[revision, point_info['value'] + error])
cols[series_index]['data'].append(point_list)
data_index = len(cols[series_index]['data']) - 1
series_dict = flot_annotations.setdefault(series_index, {})
data_dict = copy.deepcopy(point_info)
del data_dict['value']
series_dict.setdefault(data_index, data_dict)
warning = None
if show_old_data_warning and last_timestamp:
last_timestamp = datetime.datetime.fromtimestamp(last_timestamp / 1000)
if last_timestamp < datetime.datetime.now() - _STALE_DATA_DELTA:
warning = ('Graph out of date! Last data received: %s' %
last_timestamp.strftime('%Y/%m/%d %H:%M'))
elif not has_points:
warning = 'No data available.'
if not utils.IsInternalUser():
warning += ' Note that some data is only available when logged in.'
return json.dumps(
{
'data': cols,
'annotations': flot_annotations,
'error_bars': error_bars,
'warning': warning
},
allow_nan=False)
