def results_url(self): return gsutil.gs_to_public_url(self.results_gs_url)
def process_report(params): # Mandatory fields: uploader = params['uploader'] gcs_paths = params.getlist('gcs') result_type = params['type'] # Optional fields: callback_url = params.get('callback_url') labels = params.get('labels', '') assert ( (result_type == 'single' and len(gcs_paths) == 1) or (result_type == 'multiple' and len(gcs_paths) > 1) ) report = wptreport.WPTReport() try: for gcs_path in gcs_paths: _process_chunk(report, gcs_path) # To be deprecated once all reports have all the required metadata. report.update_metadata( revision=params.get('revision'), browser_name=params.get('browser_name'), browser_version=params.get('browser_version'), os_name=params.get('os_name'), os_version=params.get('os_version'), ) report.finalize() except wptreport.WPTReportError: etype, e, tb = sys.exc_info() e.path = str(gcs_paths) # This will register an error in Stackdriver. traceback.print_exception(etype, e, tb) # The input is invalid and there is no point to retry, so we return an # empty (but successful) response to tell TaskQueue to drop the task. return '' resp = "{} results loaded from: {}\n".format( len(report.results), ' '.join(gcs_paths)) raw_results_gs_url = 'gs://{}/{}/report.json'.format( config.raw_results_bucket(), report.sha_product_path) raw_results_url = gsutil.gs_to_public_url(raw_results_gs_url) # Abort early if the result already exists in Datastore. This is safe to do # because raw_results_url contains both the full revision & checksum of the # report content, unique enough to use as a UID. if _find_run_by_raw_results(raw_results_url): _log.warning( 'Skipping the task because RawResultsURL already exists: %s', raw_results_url) return '' if result_type == 'single': # If the original report isn't chunked, we store it directly without # the roundtrip to serialize it back. gsutil.copy('gs:/' + gcs_paths[0], raw_results_gs_url) else: with tempfile.NamedTemporaryFile(suffix='.json.gz') as temp: report.serialize_gzip(temp.name) gsutil.copy(temp.name, raw_results_gs_url, gzipped=True) tempdir = tempfile.mkdtemp() try: report.populate_upload_directory(output_dir=tempdir) # First copy [ID]-summary.json.gz to /wptd/[SHA]/[ID]-summary.json.gz. results_gs_url = 'gs://{}/{}'.format( config.results_bucket(), report.sha_summary_path) gsutil.copy( os.path.join(tempdir, report.sha_summary_path), results_gs_url, gzipped=True) # Now /wptd/[SHA] is guaranteed to exist. According to `gsutil cp # --help`, copy [ID] to /wptd/[SHA] will create /wptd/[SHA]/[ID]. gsutil.copy( os.path.join(tempdir, report.sha_product_path), 'gs://{}/{}'.format(config.results_bucket(), report.run_info['revision']), gzipped=True, quiet=True) resp += "Uploaded to {}\n".format(results_gs_url) finally: shutil.rmtree(tempdir) # Check again because the upload takes a long time. # Datastore does not support a query-and-put transaction, so this is only a # best effort to avoid duplicate runs. if _find_run_by_raw_results(raw_results_url): _log.warning( 'Skipping the task because RawResultsURL already exists: %s', raw_results_url) return '' # Authenticate as "_processor" for create-test-run API. secret = _get_uploader_password('_processor') test_run_id = wptreport.create_test_run( report, labels, uploader, secret, gsutil.gs_to_public_url(results_gs_url), raw_results_url, callback_url) assert test_run_id success = _after_new_run(report, test_run_id) if success: resp += "Successfully ran hooks: {}\n".format(', '.join(success)) return resp