def post(self):
"""Starts migration of old TestMetadata entity names to new ones.
The form that's used to kick off migrations will give the parameters
old_pattern and new_pattern, which are both test path pattern strings.
When this handler is called from the task queue, however, it will be given
the parameters old_test_key and new_test_key, which should both be keys
of TestMetadata entities in urlsafe form.
"""
datastore_hooks.SetPrivilegedRequest()
old_pattern = self.request.get('old_pattern')
new_pattern = self.request.get('new_pattern')
old_test_key = self.request.get('old_test_key')
new_test_key = self.request.get('new_test_key')
if old_pattern and new_pattern:
try:
_AddTasksForPattern(old_pattern, new_pattern)
self.RenderHtml(
'result.html',
{'headline': 'Test name migration task started.'})
except BadInputPatternError as error:
self.ReportError('Error: %s' % error.message, status=400)
elif old_test_key and new_test_key:
_MigrateOldTest(old_test_key, new_test_key)
else:
self.ReportError(
'Missing required parameters of /migrate_test_names.')
def get(self):
"""The get handler method is called from a cron job.
It expects no parameters and has no output. It checks all current bisect try
jobs and send comments to an issue on the issue tracker if a bisect job has
completed.
"""
issue_tracker = issue_tracker_service.IssueTrackerService(
utils.ServiceAccountHttp())
# Set privilege so we can also fetch internal try_job entities.
datastore_hooks.SetPrivilegedRequest()
jobs_to_check = try_job.TryJob.query(
try_job.TryJob.status.IN(['started', 'pending'])).fetch()
all_successful = True
for job in jobs_to_check:
try:
_CheckJob(job, issue_tracker)
except Exception as e: # pylint: disable=broad-except
logging.error('Caught Exception %s: %s\n%s',
type(e).__name__, e, traceback.format_exc())
all_successful = False
if all_successful:
utils.TickMonitoringCustomMetric('UpdateBugWithResults')
def Authorize():
try:
email = utils.GetEmail()
except oauth.OAuthRequestError:
raise OAuthError
if not email:
raise NotLoggedInError
try:
if not email.endswith('.gserviceaccount.com'):
# For non-service account, need to verify that the OAuth client ID
# is in our whitelist.
client_id = oauth.get_client_id(utils.OAUTH_SCOPES)
if client_id not in OAUTH_CLIENT_ID_WHITELIST:
logging.error('OAuth client id %s for user %s not in whitelist',
client_id, email)
email = None
raise OAuthError
except oauth.OAuthRequestError:
# Transient errors when checking the token result should result in HTTP 500,
# so catch oauth.OAuthRequestError here, not oauth.Error (which would catch
# both fatal and transient errors).
raise OAuthError
logging.info('OAuth user logged in as: %s', email)
if utils.IsInternalUser():
datastore_hooks.SetPrivilegedRequest()
def AuthorizedPost(self):
datastore_hooks.SetPrivilegedRequest()
try:
data_str = self.request.get('data')
if not data_str:
raise api_request_handler.BadRequestError(
'Missing "data" parameter')
logging.info('Received data: %s', data_str)
histogram_dicts = json.loads(data_str)
ProcessHistogramSet(histogram_dicts)
except api_request_handler.BadRequestError:
# TODO(simonhatch, eakuefner: Remove this later.
# When this has all stabilized a bit, remove and let this 400 to clients,
# but for now to preven the waterfall from re-uploading over and over
# while we bug fix, let's just log the error.
# https://github.com/catapult-project/catapult/issues/4019
logging.error(traceback.format_exc())
except Exception: # pylint: disable=broad-except
# TODO(simonhatch, eakuefner: Remove this later.
# We shouldn't be catching ALL exceptions, this is just while the
# stability of the endpoint is being worked on.
logging.error(traceback.format_exc())
def Authorize():
try:
email = utils.GetEmail()
except oauth.OAuthRequestError:
raise OAuthError
if not email:
raise NotLoggedInError
try:
# TODO(dberris): Migrate to using Cloud IAM and checking roles instead, to
# allow for dynamic management of the accounts.
if not email.endswith('.gserviceaccount.com'):
# For non-service accounts, need to verify that the OAuth client ID
# is in our allowlist.
client_id = oauth.get_client_id(utils.OAUTH_SCOPES)
if client_id not in OAUTH_CLIENT_ID_ALLOWLIST:
logging.error(
'OAuth client id %s for user %s not in allowlist',
client_id, email)
email = None
raise OAuthError
except oauth.OAuthRequestError:
# Transient errors when checking the token result should result in HTTP 500,
# so catch oauth.OAuthRequestError here, not oauth.Error (which would catch
# both fatal and transient errors).
raise OAuthError
logging.info('OAuth user logged in as: %s', email)
if utils.IsInternalUser():
datastore_hooks.SetPrivilegedRequest()
def Authorize():
try:
user = oauth.get_current_user(OAUTH_SCOPES)
except oauth.Error:
raise NotLoggedInError
if not user:
raise NotLoggedInError
try:
if not user.email().endswith('.gserviceaccount.com'):
# For non-service account, need to verify that the OAuth client ID
# is in our whitelist.
client_id = oauth.get_client_id(OAUTH_SCOPES)
if client_id not in OAUTH_CLIENT_ID_WHITELIST:
logging.info('OAuth client id %s for user %s not in whitelist',
client_id, user.email())
user = None
raise OAuthError
except oauth.Error:
raise OAuthError
logging.info('OAuth user logged in as: %s', user.email())
if utils.IsGroupMember(user.email(), 'chromeperf-access'):
datastore_hooks.SetPrivilegedRequest()
def post(self):
"""Recursively deletes TestMetadata and Row data.
The form that's used to kick off migrations will give the parameter
pattern, which is a test path pattern string.
When this handler is called from the task queue, however, it will be given
the parameter test_key, which should be a key of a TestMetadata entity in
urlsafe form.
"""
datastore_hooks.SetPrivilegedRequest()
pattern = self.request.get('pattern')
test_key = self.request.get('test_key')
notify = self.request.get('notify', 'true')
if notify.lower() == 'true':
notify = True
else:
notify = False
if pattern:
try:
_AddTasksForPattern(pattern, notify)
self.RenderHtml('result.html',
{'headline': 'Test deletion task started.'})
except BadInputPatternError as error:
self.ReportError('Error: %s' % error.message, status=400)
elif test_key:
_DeleteTest(test_key, notify)
else:
self.ReportError(
'Missing required parameters of /delete_test_data.')
def get(self):
"""Emails sheriffs about new stoppage alerts."""
datastore_hooks.SetPrivilegedRequest()
sheriffs_to_email_query = sheriff.Sheriff.query(
sheriff.Sheriff.stoppage_alert_delay > 0)
for sheriff_entity in sheriffs_to_email_query:
_SendStoppageAlertEmail(sheriff_entity)
def get(self):
"""Emails sheriffs with anomalies identified in most-recent 24 hours."""
if self.request.get('internal_only') == '1':
datastore_hooks.SetPrivilegedRequest()
_QueryAndSendSummaryEmails(True)
else:
_QueryAndSendSummaryEmails(False)
def post(self):
"""Runs auto bisects."""
if 'stats' in self.request.query_string:
self.RenderHtml('result.html', _PrintStartedAndFailedBisectJobs())
return
datastore_hooks.SetPrivilegedRequest()
if _RestartFailedBisectJobs():
utils.TickMonitoringCustomMetric('RestartFailedBisectJobs')
def post(self):
"""Adds a set of points from the post data.
Request parameters:
data: JSON encoding of a list of dictionaries. Each dictionary represents
one point to add. For each dict, one Row entity will be added, and
any required TestMetadata or Master or Bot entities will be created.
"""
datastore_hooks.SetPrivilegedRequest()
data = json.loads(self.request.get('data'))
_PrewarmGets(data)
all_put_futures = []
added_rows = []
parent_tests = []
for row_dict in data:
try:
new_row, parent_test, put_futures = _AddRow(row_dict)
added_rows.append(new_row)
parent_tests.append(parent_test)
all_put_futures.extend(put_futures)
except add_point.BadRequestError as e:
logging.error('Could not add %s, it was invalid.', e.message)
except datastore_errors.BadRequestError as e:
logging.info('While trying to store %s', row_dict)
logging.error('Datastore request failed: %s.', e.message)
return
ndb.Future.wait_all(all_put_futures)
client = sheriff_config_client.GetSheriffConfigClient()
tests_keys = []
for t in parent_tests:
reason = []
subscriptions, _ = client.Match(t.test_path, check=True)
if not subscriptions:
reason.append('subscriptions')
if not t.has_rows:
reason.append('has_rows')
if IsRefBuild(t.key):
reason.append('RefBuild')
if reason:
logging.info('Skip test: %s reason=%s', t.key,
','.join(reason))
continue
logging.info('Process test: %s', t.key)
tests_keys.append(t.key)
# Updating of the cached graph revisions should happen after put because
# it requires the new row to have a timestamp, which happens upon put.
futures = [
graph_revisions.AddRowsToCacheAsync(added_rows),
find_anomalies.ProcessTestsAsync(tests_keys)
]
ndb.Future.wait_all(futures)
def post(self):
"""Adds a single histogram or sparse shared diagnostic to the datastore.
The |data| request parameter can be either a histogram or a sparse shared
diagnostic; the set of diagnostics that are considered sparse (meaning that
they don't normally change on every upload for a given benchmark from a
given bot) is shown in histogram_helpers.SPARSE_DIAGNOSTIC_TYPES.
See https://goo.gl/lHzea6 for detailed information on the JSON format for
histograms and diagnostics.
Request parameters:
data: JSON encoding of a histogram or shared diagnostic.
revision: a revision, given as an int.
test_path: the test path to which this diagnostic or histogram should be
attached.
"""
datastore_hooks.SetPrivilegedRequest()
params = json.loads(self.request.body)
_PrewarmGets(params)
# Roughly, the processing of histograms and the processing of rows can be
# done in parallel since there are no dependencies.
histogram_futures = []
token_state_futures = []
try:
for p in params:
histogram_futures.append((p, _ProcessRowAndHistogram(p)))
except Exception as e: # pylint: disable=broad-except
for param, futures_info in itertools.izip_longest(
params, histogram_futures):
if futures_info is not None:
continue
token_state_futures.append(
upload_completion_token.Measurement.UpdateStateByIdAsync(
param.get('test_path'), param.get('token'),
upload_completion_token.State.FAILED, e.message))
ndb.Future.wait_all(token_state_futures)
raise
for info, futures in histogram_futures:
operation_state = upload_completion_token.State.COMPLETED
error_message = None
for f in futures:
exception = f.get_exception()
if exception is not None:
operation_state = upload_completion_token.State.FAILED
error_message = exception.message
token_state_futures.append(
upload_completion_token.Measurement.UpdateStateByIdAsync(
info.get('test_path'), info.get('token'), operation_state,
error_message))
ndb.Future.wait_all(token_state_futures)
def AuthorizedPost(self):
datastore_hooks.SetPrivilegedRequest()
data_str = self.request.get('data')
if not data_str:
raise api_request_handler.BadRequestError('Missing "data" parameter')
histogram_dicts = json.loads(data_str)
ProcessHistogramSet(histogram_dicts)
def DeprecateTestsMapper(entity):
"""Marks a TestMetadata entity as deprecated if the last row is too old.
What is considered "too old" is defined by _DEPRECATION_REVISION_DELTA. Also,
if all of the subtests in a test have been marked as deprecated, then that
parent test will be marked as deprecated.
This mapper doesn't un-deprecate tests if new data has been added; that
happens in add_point.py.
Args:
entity: The TestMetadata entity to check.
Yields:
Zero or more datastore mutation operations.
"""
# Fetch the last row.
datastore_hooks.SetPrivilegedRequest()
query = graph_data.Row.query(
graph_data.Row.parent_test == utils.OldStyleTestKey(entity.key))
query = query.order(-graph_data.Row.timestamp)
last_row = query.get()
# Check if the test should be deleted entirely.
now = datetime.datetime.now()
logging.info('checking %s', entity.test_path)
if not last_row or last_row.timestamp < now - _REMOVAL_REVISON_DELTA:
descendants = list_tests.GetTestDescendants(entity.key, keys_only=True)
if entity.key in descendants:
descendants.remove(entity.key)
if not descendants:
logging.info('removing')
if last_row:
logging.info('last row timestamp: %s', last_row.timestamp)
else:
logging.info('no last row, no descendants')
taskqueue.add(
url='/delete_test_data',
params={
'test_path': utils.TestPath(entity.key), # For manual inspection.
'test_key': entity.key.urlsafe(),
'notify': 'false',
},
queue_name=_DELETE_TASK_QUEUE_NAME)
return
if entity.deprecated or not last_row:
return
if last_row.timestamp < now - _DEPRECATION_REVISION_DELTA:
for operation in _MarkDeprecated(entity):
yield operation
for operation in _CreateStoppageAlerts(entity, last_row):
yield operation
def post(self):
"""Adds a single histogram or sparse shared diagnostic to the datastore.
The |data| request parameter can be either a histogram or a sparse shared
diagnostic; the set of diagnostics that are considered sparse (meaning that
they don't normally change on every upload for a given benchmark from a
given bot) is shown in add_histograms.SPARSE_DIAGNOSTIC_TYPES.
See https://goo.gl/lHzea6 for detailed information on the JSON format for
histograms and diagnostics.
Request parameters:
data: JSON encoding of a histogram or shared diagnostic.
revision: a revision, given as an int.
test_path: the test path to which this diagnostic or histogram should be
attached.
"""
datastore_hooks.SetPrivilegedRequest()
data = self.request.get('data')
revision = int(self.request.get('revision'))
test_path = self.request.get('test_path')
data_dict = json.loads(data)
guid = data_dict['guid']
is_diagnostic = 'type' in data_dict
test_path_parts = test_path.split('/')
master = test_path_parts[0]
bot = test_path_parts[1]
test_name = '/'.join(test_path_parts[2:])
bot_whitelist = stored_object.Get(add_point_queue.BOT_WHITELIST_KEY)
internal_only = add_point_queue.BotInternalOnly(bot, bot_whitelist)
extra_args = {} if is_diagnostic else GetUnitArgs(data_dict['unit'])
# TDOO(eakuefner): Populate benchmark_description once it appears in
# diagnostics.
test_key = add_point_queue.GetOrCreateAncestors(
master, bot, test_name, internal_only, **extra_args).key
if is_diagnostic:
entity = histogram.SparseDiagnostic(id=guid,
data=data,
test=test_key,
start_revision=revision,
end_revision=revision,
internal_only=internal_only)
else:
entity = histogram.Histogram(id=guid,
data=data,
test=test_key,
revision=revision,
internal_only=internal_only)
AddRow(data_dict, test_key, revision, test_path, internal_only)
entity.put()
def post(self):
"""Refreshes the cached test suites list."""
if self.request.get('internal_only') == 'true':
logging.info('Going to update internal-only test suites data.')
# Update internal-only test suites data.
datastore_hooks.SetPrivilegedRequest()
UpdateTestSuites(datastore_hooks.INTERNAL)
else:
logging.info('Going to update externally-visible test suites data.')
# Update externally-visible test suites data.
UpdateTestSuites(datastore_hooks.EXTERNAL)
def post(self):
"""Updates the selected bots internal_only property.
POST requests will be made by the task queue; tasks are added to the task
queue either by a kick-off POST from the front-end form, or by this handler
itself.
Request parameters:
internal_only: "true" if turning on internal_only, else "false".
bots: Bots to update. Multiple bots parameters are possible; the value
of each should be a string like "MasterName/platform-name".
test: An urlsafe Key for a TestMetadata entity.
cursor: An urlsafe Cursor; this parameter is only given if we're part-way
through processing a Bot or a TestMetadata.
Outputs:
A message to the user if this request was started by the web form,
or an error message if something went wrong, or nothing.
"""
# /change_internal_only should be only accessible if one has administrator
# privileges, so requests are guaranteed to be authorized.
datastore_hooks.SetPrivilegedRequest()
internal_only_string = self.request.get('internal_only')
if internal_only_string == 'true':
internal_only = True
elif internal_only_string == 'false':
internal_only = False
else:
self.ReportError('No internal_only field')
return
bot_names = self.request.get_all('bots')
test_key_urlsafe = self.request.get('test')
cursor = self.request.get('cursor', None)
if bot_names and len(bot_names) > 1:
self._UpdateMultipleBots(bot_names, internal_only)
self.RenderHtml(
'result.html', {
'headline':
('Updating internal_only. This may take some time '
'depending on the data to update. Check the task queue '
'to determine whether the job is still in progress.'),
})
elif bot_names and len(bot_names) == 1:
self._UpdateBot(bot_names[0], internal_only, cursor=cursor)
elif test_key_urlsafe:
self._UpdateTest(test_key_urlsafe, internal_only, cursor=cursor)
def post(self):
datastore_hooks.SetPrivilegedRequest()
data_str = self.request.get('data')
if not data_str:
self.ReportError('Missing "data" parameter', status=400)
return
try:
histogram_dicts = json.loads(data_str)
ProcessHistogramSet(histogram_dicts)
except ValueError:
self.ReportError('Invalid JSON string', status=400)
except BadRequestError as e:
self.ReportError(e.message, status=400)
def AuthorizedPost(self):
datastore_hooks.SetPrivilegedRequest()
try:
data_str = zlib.decompress(self.request.body)
logging.info('Recieved compressed data.')
except zlib.error:
data_str = self.request.get('data')
logging.info('Recieved uncompressed data.')
if not data_str:
raise api_request_handler.BadRequestError('Missing "data" parameter')
logging.info('Received data: %s', data_str[:100])
histogram_dicts = json.loads(data_str)
ProcessHistogramSet(histogram_dicts)
def post(self):
"""Adds a set of points from the post data.
Request parameters:
data: JSON encoding of a list of dictionaries. Each dictionary represents
one point to add. For each dict, one Row entity will be added, and
any required TestMetadata or Master or Bot entities will be created.
"""
datastore_hooks.SetPrivilegedRequest()
data = json.loads(self.request.get('data'))
_PrewarmGets(data)
bot_whitelist = stored_object.Get(BOT_WHITELIST_KEY)
all_put_futures = []
added_rows = []
parent_tests = []
for row_dict in data:
try:
new_row, parent_test, put_futures = _AddRow(
row_dict, bot_whitelist)
added_rows.append(new_row)
parent_tests.append(parent_test)
all_put_futures.extend(put_futures)
except add_point.BadRequestError as e:
logging.error('Could not add %s, it was invalid.', e.message)
except datastore_errors.BadRequestError as e:
logging.info('While trying to store %s', row_dict)
logging.error('Datastore request failed: %s.', e.message)
return
ndb.Future.wait_all(all_put_futures)
monitored_test_keys = [
t.key for t in parent_tests if t.sheriff and t.has_rows
]
tests_keys = [k for k in monitored_test_keys if not IsRefBuild(k)]
# Updating of the cached graph revisions should happen after put because
# it requires the new row to have a timestamp, which happens upon put.
futures = [
graph_revisions.AddRowsToCacheAsync(added_rows),
find_anomalies.ProcessTestsAsync(tests_keys)
]
ndb.Future.wait_all(futures)
def post(self):
"""Validates data parameter and saves to TryJob entity.
Bisect results come from a "data" parameter, which is a JSON encoding of a
dictionary.
The required fields are "master", "bot", "test".
Request parameters:
data: JSON encoding of a dictionary.
Outputs:
Empty 200 response with if successful,
200 response with warning message if optional data is invalid,
403 response with error message if sender IP is not white-listed,
400 response with error message if required data is invalid.
500 with error message otherwise.
"""
datastore_hooks.SetPrivilegedRequest()
if not self._CheckIpAgainstWhitelist():
return
data = self.request.get('data')
if not data:
self.ReportError('Missing "data" parameter.', status=400)
return
logging.info('Received data: %s', data)
try:
data = json.loads(self.request.get('data'))
except ValueError:
self.ReportError('Invalid JSON string.', status=400)
return
try:
_ValidateResultsData(data)
job = _GetTryJob(data)
if not job:
self.ReportWarning('No try job found.')
return
_UpdateTryJob(job, data)
update_bug_with_results.UpdateQuickLog(job, in_progress=True)
except BadRequestError as error:
self.ReportError(error.message, status=400)
def post(self):
datastore_hooks.SetPrivilegedRequest()
task_type = self.request.get('type')
if not task_type or task_type == 'fetch-and-process-tests':
start_cursor = self.request.get('start_cursor', None)
if start_cursor:
start_cursor = Cursor(urlsafe=start_cursor)
_DeprecateTestsTask(start_cursor)
return
if task_type == 'deprecate-test':
test_key = ndb.Key(urlsafe=self.request.get('test_key'))
_MarkTestDeprecated(test_key)
return
logging.error(
'Unknown task_type posted to /deprecate_tests: %s', task_type)
def post(self):
"""Starts migration of old TestMetadata entity names to new ones.
The form that's used to kick off migrations will give the parameters
old_pattern and new_pattern, which are both test path pattern strings.
When this handler is called from the task queue, however, it will be given
the parameters old_test_key and new_test_key, which should both be keys
of TestMetadata entities in urlsafe form.
"""
datastore_hooks.SetPrivilegedRequest()
status = self.request.get('status')
if not status:
try:
old_pattern = self.request.get('old_pattern')
new_pattern = self.request.get('new_pattern')
_MigrateTestBegin(old_pattern, new_pattern)
self.RenderHtml(
'result.html',
{'headline': 'Test name migration task started.'})
except BadInputPatternError as error:
self.ReportError('Error: %s' % error.message, status=400)
elif status:
if status == _MIGRATE_TEST_LOOKUP_PATTERNS:
old_pattern = self.request.get('old_pattern')
new_pattern = self.request.get('new_pattern')
_MigrateTestLookupPatterns(old_pattern, new_pattern)
elif status == _MIGRATE_TEST_CREATE:
old_test_key = ndb.Key(
urlsafe=self.request.get('old_test_key'))
new_test_key = ndb.Key(
urlsafe=self.request.get('new_test_key'))
_MigrateTestCreateTest(old_test_key, new_test_key)
elif status == _MIGRATE_TEST_COPY_DATA:
old_test_key = ndb.Key(
urlsafe=self.request.get('old_test_key'))
new_test_key = ndb.Key(
urlsafe=self.request.get('new_test_key'))
_MigrateTestCopyData(old_test_key, new_test_key)
else:
self.ReportError(
'Missing required parameters of /migrate_test_names.')
def post(self):
"""Checks if alerts have recovered, and marks them if so.
This includes checking untriaged alerts, as well as alerts associated with
open bugs..
"""
datastore_hooks.SetPrivilegedRequest()
# Handle task queue requests.
bug_id = self.request.get('bug_id')
project_id = self.request.get('project_id')
if bug_id:
bug_id = int(bug_id)
if not project_id:
project_id = 'chromium'
if self.request.get('check_alert'):
self.MarkAlertAndBugIfRecovered(self.request.get('alert_key'),
bug_id, project_id)
return
if self.request.get('check_bug'):
self.CheckRecoveredAlertsForBug(bug_id, project_id)
return
# Kick off task queue jobs for untriaged anomalies.
alerts = self._FetchUntriagedAnomalies()
logging.info('Kicking off tasks for %d alerts', len(alerts))
for alert in alerts:
taskqueue.add(url='/mark_recovered_alerts',
params={
'check_alert': 1,
'alert_key': alert.urlsafe()
},
queue_name=_TASK_QUEUE_NAME)
# Kick off task queue jobs for open bugs.
bugs = self._FetchOpenBugs()
logging.info('Kicking off tasks for %d bugs', len(bugs))
for bug in bugs:
taskqueue.add(url='/mark_recovered_alerts',
params={
'check_bug': 1,
'bug_id': bug['id']
},
queue_name=_TASK_QUEUE_NAME)
def _UpdateDescriptor(test_suite, namespace):
logging.info('%s %s', test_suite, namespace)
# This function always runs in the taskqueue as an anonymous user.
if namespace == datastore_hooks.INTERNAL:
datastore_hooks.SetPrivilegedRequest()
desc = descriptor.Descriptor(test_suite=test_suite, bot='place:holder')
test_path = list(desc.ToTestPathsSync())[0].split('/')
measurements = set()
bots = set()
cases = set()
# TODO(4549) Tagmaps.
query = graph_data.TestMetadata.query()
query = query.filter(graph_data.TestMetadata.suite_name == test_path[2])
if len(test_path) > 3:
# test_suite is composite.
query = query.filter(
graph_data.TestMetadata.test_part1_name == test_path[3])
query = query.filter(graph_data.TestMetadata.deprecated == False)
query = query.filter(graph_data.TestMetadata.has_rows == True)
# Use an iterator because some test suites have more keys than can fit in
# memory.
for key in query.iter(keys_only=True):
desc = descriptor.Descriptor.FromTestPathSync(utils.TestPath(key))
bots.add(desc.bot)
if desc.measurement:
measurements.add(desc.measurement)
if desc.test_case:
cases.add(desc.test_case)
logging.info('%d measurements, %d bots, %d cases', len(measurements),
len(bots), len(cases))
desc = {
'measurements': list(sorted(measurements)),
'bots': list(sorted(bots)),
'cases': list(sorted(cases)),
}
key = namespaced_stored_object.NamespaceKey(CacheKey(test_suite),
namespace)
stored_object.Set(key, desc)
def AuthorizedPost(self):
datastore_hooks.SetPrivilegedRequest()
try:
data_str = self.request.get('data')
if not data_str:
raise api_request_handler.BadRequestError(
'Missing "data" parameter')
logging.info('Received data: %s', data_str)
histogram_dicts = json.loads(data_str)
ProcessHistogramSet(histogram_dicts)
except api_request_handler.BadRequestError as e:
# TODO(simonhatch, eakuefner: Remove this later.
# When this has all stabilized a bit, remove and let this 400 to clients,
# but for now to preven the waterfall from re-uploading over and over
# while we bug fix, let's just log the error.
# https://github.com/catapult-project/catapult/issues/4019
logging.error(e.message)
def post(self):
"""Saves the given entities."""
datastore_hooks.SetPrivilegedRequest()
urlsafe_keys = self.request.get('keys').split(',')
keys = [ndb.Key(urlsafe=k) for k in urlsafe_keys]
results = ndb.get_multi(keys)
tests = []
entities = []
for e in results:
if e.key.kind() == 'TestMetadata':
tests.append(e)
else:
entities.append(e)
for t in tests:
t.UpdateSheriff()
t.put()
ndb.put_multi(entities)
def post(self):
"""Endpoint to create a new health report.
When there is a request to create a new health report, it's made with:
report_name: Optional name of report, defaults to date
num_days: Optional number of days to report on, defaults to 90
master: Optional master to report on, defaults to ChromiumPerf
Since querying all the different alerts bugs, etc. required to create the
report takes quite a while, the entry point to create a new health report
queues up tasks to this same endpoint which fill in the details, with:
benchmark: The name of the benchmark to fill in.
"""
datastore_hooks.SetPrivilegedRequest()
# This is the entry point for tasks which have already been queued up
# for individual benchmarks. If the benchmark name is specified, fill in
# report data for the benchmark.
benchmark = self.request.get('benchmark')
if benchmark:
report_name = self.request.get('report_name')
if not report_name:
self.ReportError('No name for report')
return
num_days = self.request.get('num_days')
if not num_days:
self.ReportError('No number of days for report')
return
self._FillBenchmarkDetailsToHealthReport(
benchmark, report_name, num_days,
self.request.get('master', 'ChromiumPerf'))
return
# This is called for requests to create a new health report. It creates
# taskqueue tasks which queue up smaller tasks with benchmark args
# which fill in details for individual benchmarks.
self._CreateHealthReport(self.request.get('report_name'),
self.request.get('num_days', '90'),
self.request.get('master', 'ChromiumPerf'))
def post(self):
datastore_hooks.SetPrivilegedRequest()
try:
params = json.loads(self.request.body)
gcs_file_path = params['gcs_file_path']
try:
gcs_file = cloudstorage.open(
gcs_file_path, 'r', retry_params=_RETRY_PARAMS)
with DecompressFileWrapper(gcs_file) as decompressing_file:
histogram_dicts = _LoadHistogramList(decompressing_file)
gcs_file.close()
ProcessHistogramSet(histogram_dicts)
finally:
cloudstorage.delete(gcs_file_path, retry_params=_RETRY_PARAMS)
except Exception as e: # pylint: disable=broad-except
logging.error('Error processing histograms: %r', e.message)
self.response.out.write(json.dumps({'error': e.message}))
def post(self):
"""Performs any automatic triaging operations.
This will include updating Anomaly entities, and checking whether they
should be marked as "recovered", as well as updating Bug entities, and
commenting on the issue tracker if all alerts for a bug are recovered.
"""
datastore_hooks.SetPrivilegedRequest()
# Handle task queue requests.
if self.request.get('update_recovered_bug'):
bug_id = int(self.request.get('bug_id'))
TriageBugs.UpdateRecoveredBugs(bug_id)
return
logging.info('Triaging anomalies')
TriageAnomalies.Process()
utils.TickMonitoringCustomMetric('TriageAnomalies')
logging.info('Triaging bugs')
TriageBugs.Process()
utils.TickMonitoringCustomMetric('TriageBugs')
logging.info('/auto_triage complete')