def __init__(self,
filename: str,
run_info: Optional[wptreport.RunInfo] = None,
api: Optional[str] = None,
auth: Optional[Tuple[str, str]] = None,
processes: Optional[int] = None):
"""Creates a WPTScreenshot context manager.
Usage:
with WPTScreenshot(...) as s:
s.process()
Args:
filename: Filename of the screenshots database (the file can be
gzipped if the extension is ".gz").
run_info: A finalized WPTReport.run_info dict (important fields:
product, browser_version, os, os_version) (optional).
api: The URL of the API (optional).
auth: A (username, password) tuple for HTTP basic auth (optional).
processes: The number of worker processes (defaults to cpu*2).
"""
self._filename: str = filename
self._run_info: wptreport.RunInfo = run_info or {}
self._api: str = (api or
config.project_baseurl() + '/api/screenshots/upload')
self._auth = auth
if processes is None:
processes = (os.cpu_count() or 2) * 2
self._processes: int = processes
self._f: Optional[IO[str]] = None
self._pool: Optional[multiprocessing.pool.Pool] = None
def update_status(self, run_id, stage, error=None, callback_url=None):
assert stage, "stage cannot be empty"
if int(run_id) == 0:
_log.error('Cannot update run status: missing run_id')
return
if callback_url is None:
callback_url = config.project_baseurl()
parsed_url = urlparse(callback_url)
api = '%s://%s/api/status/%s' % (parsed_url.scheme, parsed_url.netloc,
run_id)
payload = {'id': int(run_id), 'stage': stage}
if error:
payload['error'] = error
if self.report.run_info.get('revision'):
payload['full_revision_hash'] = self.report.run_info['revision']
if self.report.run_info.get('product'):
payload['browser_name'] = self.report.run_info['product']
if self.report.run_info.get('browser_version'):
payload['browser_version'] = self.report.run_info[
'browser_version']
if self.report.run_info.get('os'):
payload['os_name'] = self.report.run_info['os']
if self.report.run_info.get('os_version'):
payload['os_version'] = self.report.run_info['os_version']
try:
response = requests.patch(api, auth=self.auth, json=payload)
response.raise_for_status()
_log.debug('Updated run %s to %s', run_id, stage)
except requests.RequestException as e:
_log.error('Cannot update status for run %s: %s', run_id, str(e))
def create_test_run(report, labels_str, uploader, secret, results_gcs_path,
raw_results_gcs_path):
"""Creates a TestRun on the dashboard.
By posting to the /api/results/create endpoint.
Args:
report: A WPTReport.
labels_str: A comma-separated string of labels from the uploader.
uploader: The name of the uploader.
secret: A secret token.
results_gcs_path: The GCS path to the gzipped summary file.
(e.g. '/wptd/0123456789/chrome-62.0-linux-summary.json.gz')
raw_results_gcs_path: The GCS path to the raw full report.
(e.g. '/wptd-results/[full SHA]/chrome-62.0-linux/report.json')
"""
assert results_gcs_path.startswith('/')
assert raw_results_gcs_path.startswith('/')
labels = prepare_labels(report, labels_str, uploader)
assert len(labels) > 0
payload = report.test_run_metadata
payload['results_url'] = GCS_PUBLIC_DOMAIN + results_gcs_path
payload['raw_results_url'] = GCS_PUBLIC_DOMAIN + raw_results_gcs_path
payload['labels'] = labels
response = requests.post(config.project_baseurl() + '/api/results/create',
auth=('_processor', secret),
data=json.dumps(payload))
response.raise_for_status()
def __init__(self,
filename,
run_info=None,
api=None,
auth=None,
processes=None):
"""Creates a WPTScreenshot context manager.
Usage:
with WPTScreenshot(...) as s:
s.process()
Args:
filename: Filename of the screenshots database (the file can be
gzipped if the extension is ".gz").
run_info: A finalized WPTReport.run_info dict (important fields:
product, browser_version, os, os_version) (optional).
api: The URL of the API (optional).
auth: A (username, password) tuple for HTTP basic auth (optional).
processes: The number of worker processes (defaults to cpu*2).
"""
self._filename = filename
self._run_info = run_info or {}
self._api = api or config.project_baseurl() + '/api/screenshots/upload'
self._auth = auth
self._processes = processes or os.cpu_count() * 2
self._f = None
self._pool = None
def _push_to_spanner(_, test_run_id):
# Authenticate as "_spanner" for push-to-spanner API.
secret = _get_uploader_password('_spanner')
response = requests.put('{}/api/spanner_push_run?run_id={}'.format(
config.project_baseurl(), test_run_id),
auth=('_spanner', secret))
if not response.ok:
_log.error('Bad status code from push-to-spanner API: %d',
response.status_code)
def create_test_run(report,
run_id,
labels_str,
uploader,
auth,
results_url,
raw_results_url,
callback_url=None):
"""Creates a TestRun on the dashboard.
By posting to the /api/results/create endpoint.
Args:
report: A WPTReport.
run_id: The pre-allocated Datastore ID for this run.
labels_str: A comma-separated string of labels from the uploader.
uploader: The name of the uploader.
auth: A (username, password) tuple for HTTP basic auth.
results_url: URL of the gzipped summary file. (e.g.
'https://.../wptd/0123456789/chrome-62.0-linux-summary.json.gz')
raw_results_url: URL of the raw full report. (e.g.
'https://.../wptd-results/[FullSHA]/chrome-62.0-linux/report.json')
Returns:
The integral ID associated with the created test run.
"""
if callback_url is None:
callback_url = config.project_baseurl() + '/api/results/create'
_log.info('Creating run %s from %s using %s', run_id, uploader,
callback_url)
labels = prepare_labels(report, labels_str, uploader)
assert len(labels) > 0
labels |= normalize_product(report)
payload = report.test_run_metadata
if int(run_id) != 0:
payload['id'] = int(run_id)
payload['results_url'] = results_url
payload['raw_results_url'] = raw_results_url
payload['labels'] = sorted(labels)
response = requests.post(callback_url, auth=auth, json=payload)
response.raise_for_status()
response_data = response.json()
return response_data['id']
def create_test_run(report,
labels_str,
uploader,
secret,
results_url,
raw_results_url,
callback_url=None):
"""Creates a TestRun on the dashboard.
By posting to the /api/results/create endpoint.
Args:
report: A WPTReport.
labels_str: A comma-separated string of labels from the uploader.
uploader: The name of the uploader.
secret: A secret token.
results_url: URL of the gzipped summary file. (e.g.
'https://.../wptd/0123456789/chrome-62.0-linux-summary.json.gz')
raw_results_url: URL of the raw full report. (e.g.
'https://.../wptd-results/[FullSHA]/chrome-62.0-linux/report.json')
Returns:
The integral ID associated with the created test run.
"""
if callback_url is None:
callback_url = config.project_baseurl() + '/api/results/create'
labels = prepare_labels(report, labels_str, uploader)
assert len(labels) > 0
payload = report.test_run_metadata
payload['results_url'] = results_url
payload['raw_results_url'] = raw_results_url
payload['labels'] = labels
response = requests.post(callback_url,
auth=('_processor', secret),
data=json.dumps(payload))
response.raise_for_status()
response_data = response.json()
return response_data['id']
def task_handler():
_atomic_write(TIMESTAMP_FILE, str(time.time()))
params = flask.request.form
# Mandatory fields:
uploader = params['uploader']
gcs_paths = params.getlist('gcs')
result_type = params['type']
# Optional fields:
labels = params.get('labels', '')
assert ((result_type == 'single' and len(gcs_paths) == 1)
or (result_type == 'multiple' and len(gcs_paths) > 1))
report = wptreport.WPTReport()
try:
for gcs_path in gcs_paths:
_process_chunk(report, gcs_path)
# To be deprecated once all reports have all the required metadata.
report.update_metadata(
revision=params.get('revision'),
browser_name=params.get('browser_name'),
browser_version=params.get('browser_version'),
os_name=params.get('os_name'),
os_version=params.get('os_version'),
)
report.finalize()
except wptreport.WPTReportError:
etype, e, tb = sys.exc_info()
e.path = str(gcs_paths)
# This will register an error in Stackdriver.
traceback.print_exception(etype, e, tb)
# The input is invalid and there is no point to retry, so we return 2XX
# to tell TaskQueue to drop the task.
return ('', HTTPStatus.NO_CONTENT)
resp = "{} results loaded from {}\n".format(len(report.results),
str(gcs_paths))
raw_results_gcs_path = '/{}/{}/report.json'.format(
config.raw_results_bucket(), report.sha_product_path)
if result_type == 'single':
# If the original report isn't chunked, we store it directly without
# the roundtrip to serialize it back.
gsutil.copy('gs:/' + gcs_paths[0], 'gs:/' + raw_results_gcs_path)
else:
with tempfile.NamedTemporaryFile(suffix='.json.gz') as temp:
report.serialize_gzip(temp.name)
gsutil.copy(temp.name, 'gs:/' + raw_results_gcs_path, gzipped=True)
tempdir = tempfile.mkdtemp()
try:
report.populate_upload_directory(output_dir=tempdir)
results_gcs_path = '/{}/{}'.format(config.results_bucket(),
report.sha_summary_path)
gsutil.copy(os.path.join(tempdir, report.sha_summary_path),
'gs:/' + results_gcs_path,
gzipped=True)
# TODO(Hexcles): Consider switching to gsutil.copy.
gsutil.rsync_gzip(
os.path.join(tempdir, report.sha_product_path),
# The trailing slash is crucial (wpt.fyi#275).
'gs://{}/{}/'.format(config.results_bucket(),
report.sha_product_path),
quiet=True)
resp += "Uploaded to gs:/{}\n".format(results_gcs_path)
finally:
shutil.rmtree(tempdir)
# Authenticate as "_processor" for create-test-run API.
ds = datastore.Client()
secret = ds.get(ds.key('Uploader', '_processor'))['Password']
test_run_id = wptreport.create_test_run(report, labels, uploader, secret,
results_gcs_path,
raw_results_gcs_path)
assert test_run_id
# Authenticate as "_spanner" for push-to-spanner API.
secret = ds.get(ds.key('Uploader', '_spanner'))['Password']
response = requests.put('%s/api/spanner_push_run?run_id=%d' %
(config.project_baseurl(), test_run_id),
auth=('_spanner', secret))
if not response.ok:
app.logger.error('Bad status code from push-to-spanner API: %d' %
response.status_code)
return (resp, HTTPStatus.CREATED)