def testGenericRetry(self): """Test basic semantics of retry and success recording.""" source = functools.partial(next, iter(range(5))) def _TestMain(): val = source() if val < 4: raise ValueError() return val handler = lambda ex: isinstance(ex, ValueError) callback_args = [] with self.assertRaises(ValueError): retry_util.GenericRetry( handler, 3, _TestMain, status_callback=lambda *args: callback_args.append(args)) self.assertEqual(callback_args, [(0, False), (1, False), (2, False), (3, False)]) callback_args = [] self.assertEqual( 4, retry_util.GenericRetry( handler, 1, _TestMain, status_callback=lambda *args: callback_args.append(args))) self.assertEqual(callback_args, [(0, True)]) callback_args = [] with self.assertRaises(StopIteration): retry_util.GenericRetry( handler, 3, _TestMain, status_callback=lambda *args: callback_args.append(args)) self.assertEqual(callback_args, [(0, False)])
def _wait_for_update_service(self): """Ensure that the update engine daemon is running, possibly by waiting for it a bit in case the DUT just rebooted and the service hasn't started yet. """ def handler(e): """Retry exception handler. Assumes that the error is due to the update service not having started yet. @param e: the exception intercepted by the retry util. """ if isinstance(e, error.AutoservRunError): logging.debug( 'update service check exception: %s\n' 'retrying...', e) return True else: return False # Retry at most three times, every 5s. status = retry_util.GenericRetry(handler, 3, self.check_update_status, sleep=5) # Expect the update engine to be idle. if status != UPDATER_IDLE: raise ChromiumOSError('%s is not in an installable state' % self.host.hostname)
def GetAccessToken(**kwargs): """Returns an OAuth2 access token using luci-auth. Retry the _TokenAndLoginIfNeed function when the error threw is an AccessTokenError. Args: kwargs: A list of keyword arguments to pass to _TokenAndLoginIfNeed. Returns: The access token string or None if failed to get access token. """ service_account_json = kwargs.get('service_account_json') force_token_renew = kwargs.get('force_token_renew', False) retry = lambda e: isinstance(e, AccessTokenError) try: result = retry_util.GenericRetry( retry, RETRY_GET_ACCESS_TOKEN, _TokenAndLoginIfNeed, service_account_json=service_account_json, force_token_renew=force_token_renew, sleep=3) return result except AccessTokenError as e: logging.error('Failed at getting the access token: %s ', e) # Do not raise the AccessTokenError here. # Let the response returned by the request handler # tell the status and errors. return
def testStatustCallbackExceptionForSuccess(self): """Exception from |status_callback| should be raised even on success.""" with self.assertRaises(TestRetries.CheckException): retry_util.GenericRetry(lambda _: True, 1, lambda: None, status_callback=self._RaiseCheckException)
def GetAccessToken(service_account_json=None): """Returns an OAuth2 access token using authutil. Retry the _TokenAndLoginIfNeed function when the error threw is an AccessTokenError. Args: service_account_json: A optional path to a service account. Returns: The access token string. """ retry = lambda e: isinstance(e, AccessTokenError) try: result = retry_util.GenericRetry(retry, RETRY_GET_ACCESS_TOKEN, _TokenAndLoginIfNeed, service_account_json, sleep=3) return result except AccessTokenError as e: logging.error('Failed at getting the access token: %s ', e) # Do not raise the AccessTokenError here. # Let the response returned by the request handler # tell the status and errors. return
def _base_update_handler(self, run_args, err_msg_prefix=None): """Handle a remote update ssh call, possibly with retries. @param run_args: Dictionary of args passed to ssh_host.run function. @param err_msg_prefix: Prefix of the exception error message. """ def exception_handler(e): """Examines exceptions and returns True if the update handler should be retried. @param e: the exception intercepted by the retry util. """ return (isinstance(e, error.AutoservSSHTimeout) or (isinstance(e, error.GenericHostRunError) and hasattr(e, 'description') and (re.search('ERROR_CODE=37', e.description) or re.search('generic error .255.', e.description)))) try: # Try the update twice (arg 2 is max_retry, not including the first # call). Some exceptions may be caught by the retry handler. retry_util.GenericRetry(exception_handler, 1, self._base_update_handler_no_retry, run_args) except Exception as e: message = err_msg_prefix + ': ' + str(e) raise RootFSUpdateError(message)
def LockDb(db): """Lock an account database. We use the same algorithm as shadow/user.eclass. This way we don't race and corrupt things in parallel. """ lock = '%s.lock' % db _, tmplock = tempfile.mkstemp(prefix='%s.platform.' % lock) # First try forever to grab the lock. retry = lambda e: e.errno == errno.EEXIST # Retry quickly at first, but slow down over time. try: retry_util.GenericRetry(retry, 60, os.link, tmplock, lock, sleep=0.1) except Exception: print('error: could not grab lock %s' % lock) raise # Yield while holding the lock, but try to clean it no matter what. try: os.unlink(tmplock) yield lock finally: os.unlink(lock)
def testGenericRetryBadArgs(self): """Test bad retry related arguments to GenericRetry raise ValueError.""" def _AlwaysRaise(): raise Exception('Not a ValueError') # |max_retry| must be non-negative number. with self.assertRaises(ValueError): retry_util.GenericRetry(lambda _: True, -1, _AlwaysRaise) # |backoff_factor| must be 1 or greator. with self.assertRaises(ValueError): retry_util.GenericRetry(lambda _: True, 3, _AlwaysRaise, backoff_factor=0.9) # Sleep must be non-negative number. with self.assertRaises(ValueError): retry_util.GenericRetry(lambda _: True, 3, _AlwaysRaise, sleep=-1)
def testRetryWithBackoff(self): sleep_history = [] self.PatchObject(time, 'sleep', new=sleep_history.append) def _AlwaysFail(): raise ValueError() with self.assertRaises(ValueError): retry_util.GenericRetry(lambda _: True, 5, _AlwaysFail, sleep=1, backoff_factor=2) self.assertEqual(sleep_history, [1, 2, 4, 8, 16])
def testRaisedException(self): """Test which exception gets raised by repeated failure.""" def _GetTestMain(): """Get function that fails once with ValueError, Then AssertionError.""" source = itertools.count() def _TestMain(): if next(source) == 0: raise ValueError() else: raise AssertionError() return _TestMain with self.assertRaises(ValueError): retry_util.GenericRetry(lambda _: True, 3, _GetTestMain()) with self.assertRaises(AssertionError): retry_util.GenericRetry(lambda _: True, 3, _GetTestMain(), raise_first_exception_on_failure=False)
def send_email(to, subject, message_text, retry=True, creds_path=None): """Send email. @param to: The recipients, separated by comma. @param subject: Subject of the email. @param message_text: Text to send. @param retry: If retry on retriable failures as defined in RETRIABLE_MSGS. @param creds_path: The credential path for gmail account, if None, will use DEFAULT_CREDS_FILE. """ auth_creds = server_utils.get_creds_abspath(creds_path or DEFAULT_CREDS_FILE) if not auth_creds or not os.path.isfile(auth_creds): logging.error( 'Failed to send email to %s: Credential file does not' 'exist: %s. If this is a prod server, puppet should' 'install it. If you need to be able to send email, ' 'find the credential file from chromeos-admin repo and ' 'copy it to %s', to, auth_creds, auth_creds) return client = GmailApiClient(oauth_credentials=auth_creds) m = Message(to, subject, message_text) retry_count = MAX_RETRY if retry else 0 def _run(): """Send the message.""" client.send_message(m, ignore_error=False) def handler(exc): """Check if exc is an HttpError and is retriable. @param exc: An exception. @return: True if is an retriable HttpError. """ if not isinstance(exc, apiclient_errors.HttpError): return False error_msg = str(exc) should_retry = any([msg in error_msg for msg in RETRIABLE_MSGS]) if should_retry: logging.warning('Will retry error %s', exc) return should_retry success = False try: retry_util.GenericRetry(handler, retry_count, _run, sleep=RETRY_DELAY, backoff_factor=RETRY_BACKOFF_FACTOR) success = True finally: metrics.Counter('chromeos/autotest/send_email/count').increment( fields={'success': success})
def testStatusCallbackExceptionForRetry(self): """Exception from |status_callback| should stop retry.""" counter = [0] # Counter to track how many times _functor is called. def _TestMain(): counter[0] += 1 raise Exception() # Let it fail. with self.assertRaises(TestRetries.CheckException): retry_util.GenericRetry(lambda _: True, 10, _TestMain, status_callback=self._RaiseCheckException) # Do not expect retry in case |status_callback| raises an exception. self.assertEqual(counter[0], 1)
def GetExperimentalBuilders(status_url=None, timeout=1): """Polls |status_url| and returns the list of experimental builders. This function gets a JSON response from |status_url|, and returns the list of builders marked as experimental in the tree status' message. Args: status_url: The status url to check i.e. 'https://status.appspot.com/current?format=json' timeout: How long to wait for the tree status (in seconds). Returns: A list of strings, where each string is a builder. Returns an empty list if there are no experimental builders listed in the tree status. Raises: TimeoutError if the request takes longer than |timeout| to complete. """ if not status_url: status_url = CROS_TREE_STATUS_JSON_URL site_config = config_lib.GetConfig() @timeout_util.TimeoutDecorator(timeout) def _get_status_dict(): experimental = [] status_dict = _GetStatusDict(status_url) if status_dict: for match in EXPERIMENTAL_BUILDERS_RE.findall( status_dict.get(TREE_STATUS_MESSAGE)): # The value for EXPERIMENTAL= could be a comma-separated list # of builders. for builder in match.split(','): if builder in site_config: experimental.append(builder) else: logging.warning( 'Got unknown build config "%s" in list of ' 'EXPERIMENTAL-BUILDERS.', builder) if experimental: logging.info('Got experimental build configs %s from tree status.', experimental) return experimental return retry_util.GenericRetry(lambda _: True, 3, _get_status_dict, sleep=1)
def run(self, call, **dargs): if retry_util is None: raise ImportError('Unable to import chromite. Please consider to ' 'run build_externals to build site packages.') # exc_retry: We retry if this exception is raised. # blacklist: Exceptions that we raise immediately if caught. exc_retry = Exception blacklist = (ImportError, error.RPCException, proxy.JSONRPCException, timeout_util.TimeoutError) backoff = 2 max_retry = convert_timeout_to_retry(backoff, self.timeout_min, self.delay_sec) def _run(self, call, **dargs): return super(RetryingAFE, self).run(call, **dargs) def handler(exc): """Check if exc is an exc_retry or if it's blacklisted. @param exc: An exception. @return: True if exc is an exc_retry and is not blacklisted. False otherwise. """ is_exc_to_check = isinstance(exc, exc_retry) is_blacklisted = isinstance(exc, blacklist) return is_exc_to_check and not is_blacklisted # If the call is not in main thread, signal can't be used to abort the # call. In that case, use a basic retry which does not enforce timeout # if the process hangs. @retry.retry(Exception, timeout_min=self.timeout_min, delay_sec=self.delay_sec, blacklist=[ImportError, error.RPCException, proxy.ValidationError]) def _run_in_child_thread(self, call, **dargs): return super(RetryingAFE, self).run(call, **dargs) if isinstance(threading.current_thread(), threading._MainThread): # Set the keyword argument for GenericRetry dargs['sleep'] = self.delay_sec dargs['backoff_factor'] = backoff with timeout_util.Timeout(self.timeout_min * 60): return retry_util.GenericRetry(handler, max_retry, _run, self, call, **dargs) else: return _run_in_child_thread(self, call, **dargs)
def RetryWithStats(category, handler, max_retry, functor, *args, **kwargs): """Wrapper around retry_util.GenericRetry that collects stats. This wrapper collects statistics about each failure or retry. Each category is defined by a unique string. Each category should be setup before use (actually, before processes are forked). All other arguments are blindly passed to retry_util.GenericRetry. Args: category: A string that defines the 'namespace' for these stats. handler: See retry_util.GenericRetry. max_retry: See retry_util.GenericRetry. functor: See retry_util.GenericRetry. args: See retry_util.GenericRetry. kwargs: See retry_util.GenericRetry. Returns: See retry_util.GenericRetry raises. Raises: See retry_util.GenericRetry raises. """ statEntry = StatEntry(category, attempts=[]) # Wrap the work method, so we can gather info. def wrapper(*args, **kwargs): start = datetime.datetime.now() try: result = functor(*args, **kwargs) except Exception as e: end = datetime.datetime.now() e_description = '%s: %s' % (type(e).__name__, e) statEntry.attempts.append(Attempt(end - start, e_description)) raise end = datetime.datetime.now() statEntry.attempts.append(Attempt(end - start, None)) return result try: return retry_util.GenericRetry(handler, max_retry, wrapper, *args, **kwargs) finally: if _STATS_COLLECTION is not None: _STATS_COLLECTION.append(statEntry)
def _PostConfigToBuildBucket(self, testjob=False, dryrun=False): """Posts the tryjob config to buildbucket. Args: dryrun: Whether to skip the request to buildbucket. testjob: Whether to use the test instance of the buildbucket server. Returns: A (response, body) tuple of the response from the buildbucket service. """ http = self._BuildBucketAuth() host = topology.topology[ topology.BUILDBUCKET_TEST_HOST_KEY if testjob else topology.BUILDBUCKET_HOST_KEY] buildbucket_put_url = ( 'https://{hostname}/_ah/api/buildbucket/v1/builds'.format( hostname=host)) body = json.dumps({ 'bucket': 'master.chromiumos.tryserver', 'parameters_json': json.dumps(self.values), }) def try_put(): response, _ = http.request( buildbucket_put_url, 'PUT', body=body, headers={'Content-Type': 'application/json'}, ) if int(response['status']) // 100 != 2: raise Exception('Got a %s response from Buildbucket.' % response['status']) if dryrun: logging.info('dryrun mode is on; skipping request to buildbucket. ' 'Would have made a request with body:\n%s', body) return return retry_util.GenericRetry(lambda _: True, 3, try_put)
def _lock_backing_file(self): """Context to lock the backing store file. @raises StoreError if the backing file can not be locked. """ def _retry_locking_failures(exc): return isinstance(exc, locking.LockNotAcquiredError) try: retry_util.GenericRetry( handler=_retry_locking_failures, functor=self._lock.write_lock, max_retry=self._lock_max_retry, sleep=self._lock_sleep) # If self._lock fails to write the locking file, it'll leak an OSError except (locking.LockNotAcquiredError, OSError) as e: raise host_info.StoreError(e) with self._lock: yield
def SendBuildbucketRequest(self, url, method, body, dryrun): """Generic buildbucket request. Args: url: Buildbucket url to send requests. method: HTTP method to perform, such as GET, POST, DELETE. body: The entity body to be sent with the request (a string object). See httplib2.Http.request for details. dryrun: Whether a dryrun. Returns: A dict of response entity body if the request succeeds; else, None. See httplib2.Http.request for details. Raises: BuildbucketResponseException when response['status'] is invalid. """ if dryrun: logging.info( 'Dryrun mode is on; Would have made a request ' 'with url %s method %s body:\n%s', url, method, body) return def try_method(): response, content = self.http.request( url, method, body=body, headers={'Content-Type': 'application/json'}, ) if int(response['status']) // 100 != 2: raise BuildbucketResponseException( 'Got a %s response from buildbucket with url: %s\n' 'content: %s' % (response['status'], url, content)) # Deserialize the content into a python dict. return json.loads(content) return retry_util.GenericRetry(lambda _: True, 3, try_method)
def RunGit(git_repo, cmd, retry=True, **kwargs): """RunCommand wrapper for git commands. This suppresses print_cmd, and suppresses output by default. Git functionality w/in this module should use this unless otherwise warranted, to standardize git output (primarily, keeping it quiet and being able to throw useful errors for it). Args: git_repo: Pathway to the git repo to operate on. cmd: A sequence of the git subcommand to run. The 'git' prefix is added automatically. If you wished to run 'git remote update', this would be ['remote', 'update'] for example. retry: If set, retry on transient errors. Defaults to True. kwargs: Any RunCommand or GenericRetry options/overrides to use. Returns: A CommandResult object. """ def _ShouldRetry(exc): """Returns True if push operation failed with a transient error.""" if (isinstance(exc, cros_build_lib.RunCommandError) and exc.result and exc.result.error and GIT_TRANSIENT_ERRORS_RE.search(exc.result.error)): logging.warning('git reported transient error (cmd=%s); retrying', cros_build_lib.CmdToStr(cmd), exc_info=True) return True return False max_retry = kwargs.pop('max_retry', DEFAULT_RETRIES if retry else 0) kwargs.setdefault('print_cmd', False) kwargs.setdefault('sleep', DEFAULT_RETRY_INTERVAL) kwargs.setdefault('cwd', git_repo) kwargs.setdefault('capture_output', True) return retry_util.GenericRetry( _ShouldRetry, max_retry, cros_build_lib.RunCommand, ['git'] + cmd, **kwargs)
def BuildBucketRequest(http, url, method, body, dryrun): """Generic buildbucket request. Args: http: Http instance. url: Buildbucket url to send requests. method: Request method. body: Body of http request (string object). dryrun: Whether a dryrun. Returns: Content if request succeeds. Raises: BuildbucketResponseException when response['status'] is invalid. """ if dryrun: logging.info('Dryrun mode is on; Would have made a request ' 'with url %s method %s body:\n%s', url, method, body) return def try_method(): response, content = http.request( url, method, body=body, headers={'Content-Type': 'application/json'}, ) if int(response['status']) // 100 != 2: raise BuildbucketResponseException( 'Got a %s response from buildbucket with url: %s\n' 'content: %s' % (response['status'], url, content)) # Return content_dict return json.loads(content) return retry_util.GenericRetry(lambda _: True, 3, try_method)
def run(self, call, **dargs): if retry_util is None: raise ImportError('Unable to import chromite. Please consider to ' 'run build_externals to build site packages.') # exc_retry: We retry if this exception is raised. # blacklist: Exceptions that we raise immediately if caught. exc_retry = Exception blacklist = (ImportError, error.RPCException, proxy.JSONRPCException, timeout_util.TimeoutError) backoff = 2 max_retry = convert_timeout_to_retry(backoff, self.timeout_min, self.delay_sec) def _run(self, call, **dargs): return super(RetryingAFE, self).run(call, **dargs) def handler(exc): """Check if exc is an exc_retry or if it's blacklisted. @param exc: An exception. @return: True if exc is an exc_retry and is not blacklisted. False otherwise. """ is_exc_to_check = isinstance(exc, exc_retry) is_blacklisted = isinstance(exc, blacklist) return is_exc_to_check and not is_blacklisted # If the call is not in main thread, signal can't be used to abort the # call. In that case, use a basic retry which does not enforce timeout # if the process hangs. @retry.retry( Exception, timeout_min=self.timeout_min, delay_sec=self.delay_sec, blacklist=[ImportError, error.RPCException, proxy.ValidationError]) def _run_in_child_thread(self, call, **dargs): return super(RetryingAFE, self).run(call, **dargs) if isinstance(threading.current_thread(), threading._MainThread): # Set the keyword argument for GenericRetry dargs['sleep'] = self.delay_sec dargs['backoff_factor'] = backoff # timeout_util.Timeout fundamentally relies on sigalrm, and doesn't # work at all in wsgi environment (just emits logs spam). So, don't # use it in wsgi. try: if env.IN_MOD_WSGI: return retry_util.GenericRetry(handler, max_retry, _run, self, call, **dargs) with timeout_util.Timeout(self.timeout_min * 60): return retry_util.GenericRetry(handler, max_retry, _run, self, call, **dargs) except timeout_util.TimeoutError: c = metrics.Counter( 'chromeos/autotest/retrying_afe/retry_timeout') # Reserve field job_details for future use. f = { 'destination_server': self.server.split(':')[0], 'call': call, 'job_details': '' } c.increment(fields=f) raise else: return _run_in_child_thread(self, call, **dargs)
def SendRequest(self, service, method, body=None, dryrun=False, timeout_secs=None, retry_count=3): """Generic pRPC request. Args: service: The pRPC service. method: The pRPC method. body: The entity body to be sent with the request (a string object). See httplib2.Http.request for details. dryrun: Whether a dryrun. timeout_secs: Maximum number of seconds for remote server to process request. retry_count: Number of retry attempts on transient failures. Returns: A dict of the decoded JSON response. Raises: PRPCResponseException if the pRPC response is invalid. """ url = self.ConstructURL(service, method) if dryrun: logging.info( 'Dryrun mode is on; Would have made a request with url %s body:\n%s', url, body) return {} headers = { 'Accept': 'application/json', 'Content-Type': 'application/json', } if timeout_secs: headers['X-Prpc-Timeout'] = '%dS' % (timeout_secs) def AllowRetry(e): return (isinstance(e, httplib2.ServerNotFoundError) or isinstance(e, socket.error) or isinstance(e, socket.timeout) or (isinstance(e, PRPCResponseException) and e.transient)) def IsTransientHTTPStatus(status): return status >= 500 def IsTransientPRPCCode(code): return code in (PRPCCode.Unknown, PRPCCode.Internal, PRPCCode.Unavailable) def TryMethod(): response, content = self.http.request(url, POST_METHOD, body=body, headers=headers) # Check HTTP status code. if 'status' not in response: raise PRPCResponseException( 'Missing HTTP response code with url: %s\n' 'content: %s' % (url, content)) status = int(response['status']) if status not in (httplib.OK, httplib.NO_CONTENT): raise PRPCResponseException( 'Got a %s response with url: %s\ncontent: %s' % (response['status'], url, content), transient=IsTransientHTTPStatus(status)) # Check pRPC status code. if 'x-prpc-grpc-code' not in response: raise PRPCResponseException( 'Missing pRPC response code with url: %s\n' 'content: %s' % (url, content)) prpc_code = int(response['x-prpc-grpc-code']) if prpc_code != PRPCCode.OK: raise PRPCResponseException( 'Got a %s (%s) pRPC response code with url: %s\ncontent: %s' % (response['x-prpc-grpc-code'], GetCodeString(prpc_code), url, content), transient=IsTransientPRPCCode(prpc_code)) # Verify XSSI prefix. if content[:5] != ')]}\'\n': # Unwrap the gRPC message by removing XSSI prefix. raise PRPCResponseException('Got a non-matching XSSI prefix') return json.loads(content[5:]) return retry_util.GenericRetry(AllowRetry, retry_count, TryMethod)
def UploadPerfValues(perf_values, platform_name, test_name, revision=None, cros_version=None, chrome_version=None, dashboard=DASHBOARD_URL, master_name=None, test_prefix=None, platform_prefix=None, dry_run=False): """Uploads any perf data associated with a test to the perf dashboard. Note: If |revision| is used, then |cros_version| & |chrome_version| are not necessary. Conversely, if |revision| is not used, then |cros_version| and |chrome_version| must both be specified. Args: perf_values: List of PerformanceValue objects. platform_name: A string identifying platform e.g. 'x86-release'. 'cros-' will be prepended to |platform_name| internally, by _FormatForUpload. test_name: A string identifying the test revision: The raw X-axis value; normally it represents a VCS repo, but may be any monotonic increasing value integer. cros_version: A string identifying Chrome OS version e.g. '6052.0.0'. chrome_version: A string identifying Chrome version e.g. '38.0.2091.2'. dashboard: The dashboard to upload data to. master_name: The "master" field to use; by default it is looked up in the perf_dashboard_config.json database. test_prefix: Arbitrary string to automatically prefix to the test name. If None, then 'cbuildbot.' is used to guarantee namespacing. platform_prefix: Arbitrary string to automatically prefix to |platform_name|. If None, then 'cros-' is used to guarantee namespacing. dry_run: Do everything but upload the data to the server. """ if not perf_values: return # Aggregate values from multiple iterations together. perf_data = _AggregateIterations(perf_values) # Compute averages and standard deviations as needed for measured perf # values that exist in multiple iterations. Ultimately, we only upload a # single measurement (with standard deviation) for every unique measured # perf metric. _ComputeAvgStddev(perf_data) # Format the perf data for the upload, then upload it. if revision is None: # No "revision" field, calculate one. Chrome and CrOS fields must be given. cros_version = chrome_version[:chrome_version.find('.') + 1] + cros_version revision = _ComputeRevisionFromVersions(chrome_version, cros_version) try: if master_name is None: presentation_info = _GetPresentationInfo(test_name) else: presentation_info = PresentationInfo(master_name, test_name) formatted_data = _FormatForUpload(perf_data, platform_name, presentation_info, revision=revision, cros_version=cros_version, chrome_version=chrome_version, test_prefix=test_prefix, platform_prefix=platform_prefix) if dry_run: logging.debug('UploadPerfValues: skipping upload due to dry-run') else: retry_util.GenericRetry(_RetryIfServerError, 3, _SendToDashboard, formatted_data, dashboard=dashboard) except PerfUploadingError: logging.exception('Error when uploading perf data to the perf ' 'dashboard for test %s.', test_name) raise else: logging.info('Successfully uploaded perf data to the perf ' 'dashboard for test %s.', test_name)