def results_url(self):
return gsutil.gs_to_public_url(self.results_gs_url)
def process_report(params):
# Mandatory fields:
uploader = params['uploader']
gcs_paths = params.getlist('gcs')
result_type = params['type']
# Optional fields:
callback_url = params.get('callback_url')
labels = params.get('labels', '')
assert (
(result_type == 'single' and len(gcs_paths) == 1) or
(result_type == 'multiple' and len(gcs_paths) > 1)
)
report = wptreport.WPTReport()
try:
for gcs_path in gcs_paths:
_process_chunk(report, gcs_path)
# To be deprecated once all reports have all the required metadata.
report.update_metadata(
revision=params.get('revision'),
browser_name=params.get('browser_name'),
browser_version=params.get('browser_version'),
os_name=params.get('os_name'),
os_version=params.get('os_version'),
)
report.finalize()
except wptreport.WPTReportError:
etype, e, tb = sys.exc_info()
e.path = str(gcs_paths)
# This will register an error in Stackdriver.
traceback.print_exception(etype, e, tb)
# The input is invalid and there is no point to retry, so we return an
# empty (but successful) response to tell TaskQueue to drop the task.
return ''
resp = "{} results loaded from: {}\n".format(
len(report.results), ' '.join(gcs_paths))
raw_results_gs_url = 'gs://{}/{}/report.json'.format(
config.raw_results_bucket(), report.sha_product_path)
raw_results_url = gsutil.gs_to_public_url(raw_results_gs_url)
# Abort early if the result already exists in Datastore. This is safe to do
# because raw_results_url contains both the full revision & checksum of the
# report content, unique enough to use as a UID.
if _find_run_by_raw_results(raw_results_url):
_log.warning(
'Skipping the task because RawResultsURL already exists: %s',
raw_results_url)
return ''
if result_type == 'single':
# If the original report isn't chunked, we store it directly without
# the roundtrip to serialize it back.
gsutil.copy('gs:/' + gcs_paths[0], raw_results_gs_url)
else:
with tempfile.NamedTemporaryFile(suffix='.json.gz') as temp:
report.serialize_gzip(temp.name)
gsutil.copy(temp.name, raw_results_gs_url, gzipped=True)
tempdir = tempfile.mkdtemp()
try:
report.populate_upload_directory(output_dir=tempdir)
# First copy [ID]-summary.json.gz to /wptd/[SHA]/[ID]-summary.json.gz.
results_gs_url = 'gs://{}/{}'.format(
config.results_bucket(), report.sha_summary_path)
gsutil.copy(
os.path.join(tempdir, report.sha_summary_path),
results_gs_url,
gzipped=True)
# Now /wptd/[SHA] is guaranteed to exist. According to `gsutil cp
# --help`, copy [ID] to /wptd/[SHA] will create /wptd/[SHA]/[ID].
gsutil.copy(
os.path.join(tempdir, report.sha_product_path),
'gs://{}/{}'.format(config.results_bucket(),
report.run_info['revision']),
gzipped=True, quiet=True)
resp += "Uploaded to {}\n".format(results_gs_url)
finally:
shutil.rmtree(tempdir)
# Check again because the upload takes a long time.
# Datastore does not support a query-and-put transaction, so this is only a
# best effort to avoid duplicate runs.
if _find_run_by_raw_results(raw_results_url):
_log.warning(
'Skipping the task because RawResultsURL already exists: %s',
raw_results_url)
return ''
# Authenticate as "_processor" for create-test-run API.
secret = _get_uploader_password('_processor')
test_run_id = wptreport.create_test_run(
report,
labels,
uploader,
secret,
gsutil.gs_to_public_url(results_gs_url),
raw_results_url,
callback_url)
assert test_run_id
success = _after_new_run(report, test_run_id)
if success:
resp += "Successfully ran hooks: {}\n".format(', '.join(success))
return resp