def main(platform_id, platform, args, config):
loggingLevel = getattr(logging, args.log.upper(), None)
logging.basicConfig(level=loggingLevel)
logger = logging.getLogger()
print('PLATFORM_ID:', platform_id)
print('PLATFORM INFO:', platform)
if args.path:
print('Running tests in path: %s' % args.path)
else:
print('Running all tests!')
if args.upload:
print('Setting up storage client')
from google.cloud import storage
storage_client = storage.Client(project='wptdashboard')
bucket = storage_client.get_bucket(config['gs_results_bucket'])
verify_gsutil_installed(config)
if args.create_testrun:
assert len(config['secret']) == 64, (
'Valid secret required to create TestRun')
if not platform.get('sauce'):
if platform['browser_name'] == 'chrome':
browser_binary = config['chrome_binary']
elif platform['browser_name'] == 'firefox':
browser_binary = config['firefox_binary']
if platform['browser_name'] == 'chrome':
verify_browser_binary_version(platform, browser_binary)
verify_os_name(platform)
verify_or_set_os_version(platform)
print('Platform information:')
print('Browser version: %s' % platform['browser_version'])
print('OS name: %s' % platform['os_name'])
print('OS version: %s' % platform['os_version'])
print('==================================================')
print('Setting up WPT checkout')
setup_wpt(config)
print('Getting WPT commit SHA and Date')
wpt_sha, wpt_commit_date = get_commit_details(args, config, logger)
print('WPT SHA: %s' % wpt_sha)
print('WPT Commit Date: %s' % wpt_commit_date)
short_wpt_sha = wpt_sha[0:10]
abs_report_log_path = "%s/wptd-%s-%s-report.log" % (
config['build_path'], short_wpt_sha, platform_id)
abs_report_chunks_path = "%s/%s/%s-report-chunks" % (
config['build_path'], short_wpt_sha, platform_id)
abs_current_chunk_path = os.path.join(abs_report_chunks_path,
'current.json')
mkdirp(abs_report_chunks_path)
sha_summary_gz_path = '%s/%s-summary.json.gz' % (short_wpt_sha,
platform_id)
abs_sha_summary_gz_path = "%s/%s" % (config['build_path'],
sha_summary_gz_path)
gs_results_base_path = "%s/%s/%s" % (config['build_path'], short_wpt_sha,
platform_id)
gs_results_url = 'https://storage.googleapis.com/%s/%s' % (
config['gs_results_bucket'], sha_summary_gz_path)
print('==================================================')
print('Running WPT')
report = Report(args.total_chunks, abs_report_chunks_path)
expected_test_count = 0
for this_chunk in range(1, args.total_chunks + 1):
if platform.get('sauce'):
if platform['browser_name'] == 'edge':
sauce_browser_name = 'MicrosoftEdge'
else:
sauce_browser_name = platform['browser_name']
command = [
'./wpt',
'run',
'sauce:%s:%s' %
(sauce_browser_name, platform['browser_version']),
'--sauce-platform=%s %s' %
(platform['os_name'], platform['os_version']),
'--sauce-key=%s' % config['sauce_key'],
'--sauce-user=%s' % config['sauce_user'],
'--sauce-connect-binary=%s' % config['sauce_connect_binary'],
'--sauce-tunnel-id=%s' % config['sauce_tunnel_id'],
'--no-restart-on-unexpected',
'--processes=2',
'--run-by-dir=3',
]
if args.path:
command.insert(3, args.path)
else:
command = [
'xvfb-run',
'--auto-servernum',
'./wpt',
'run',
platform['browser_name'],
]
if args.path:
command.insert(5, args.path)
if platform['browser_name'] == 'chrome':
command.extend(['--binary', browser_binary])
if platform['browser_name'] == 'firefox':
command.extend(['--install-browser', '--yes'])
command.append('--certutil-binary=certutil')
# temporary fix to allow WebRTC tests to call getUserMedia
command.extend(
['--setpref', 'media.navigator.streams.fake=true'])
command.append('--log-mach=-')
raw_log_filename = tempfile.mkstemp('-wptdashboard')[1]
command.extend(['--log-raw', raw_log_filename])
command.extend(['--log-wptreport', abs_current_chunk_path])
command.append('--install-fonts')
command.extend([
'--this-chunk',
str(this_chunk), '--total-chunks',
str(args.total_chunks)
])
for attempt_number in range(1, args.max_attempts + 1):
print('Running chunk %s of %s (attempt %s of %s)' %
(this_chunk, args.total_chunks, attempt_number,
args.max_attempts))
# In the event of a failed attempt, previously-created files will
# still be available on disk. Remove these to guard against errors
# where the next attempt fails to write new results.
for name in (abs_current_chunk_path, raw_log_filename):
try:
os.remove(name)
except OSError:
pass
return_code = subprocess.call(command, cwd=config['wpt_path'])
print('Return code from wptrunner: %s' % return_code)
chunk_test_count = len(get_expected_tests(raw_log_filename))
print('%s tests defined in chunk' % chunk_test_count)
expected_test_count += chunk_test_count
try:
data = report.load_chunk(this_chunk, abs_current_chunk_path)
print 'Report contains %s results' % len(data['results'])
break
except InsufficientData:
pass
else:
print('No results found after %s attempts. Giving up.' %
args.max_attempts)
os.remove(raw_log_filename)
print('==================================================')
print('Finished WPT run')
if platform['browser_name'] == 'firefox':
print('Verifying installed firefox matches platform ID')
firefox_path = '%s/_venv/firefox/firefox' % config['wpt_path']
verify_browser_binary_version(platform, firefox_path)
print('Creating summary of results')
try:
summary = report.summarize()
actual_test_count = len(summary.keys())
actual_percentage = actual_test_count / expected_test_count
if actual_percentage < args.partial_threshold / 100:
raise InsufficientData('%s of %s is below threshold of %s%%' %
(actual_test_count, expected_test_count,
args.partial_threshold))
except InsufficientData as exc:
logging.fatal('Insufficient report data (%s). Stopping.', exc)
exit(1)
print('==================================================')
print('Writing summary.json.gz to local filesystem')
write_gzip_json(abs_sha_summary_gz_path, summary)
print('Wrote file %s' % abs_sha_summary_gz_path)
print('==================================================')
print('Writing individual result files to local filesystem')
for result in report.each_result():
test_file = result['test']
filepath = '%s%s' % (gs_results_base_path, test_file)
write_gzip_json(filepath, result)
print('Wrote file %s' % filepath)
if not args.upload:
print('==================================================')
print('Stopping here (pass --upload to upload results to WPTD).')
return
print('==================================================')
print('Uploading results to gs://%s' % config['gs_results_bucket'])
command = [
'gsutil', '-m', '-h', 'Content-Encoding:gzip', 'rsync', '-r',
short_wpt_sha,
'gs://wptd/%s' % short_wpt_sha
]
return_code = subprocess.check_call(command, cwd=config['build_path'])
assert return_code == 0
print('Successfully uploaded!')
print('HTTP summary URL: %s' % gs_results_url)
if not args.create_testrun:
print('==================================================')
print('Stopping here')
print('pass --create-testrun to create and promote this TestRun).')
return
print('==================================================')
print('Creating new TestRun in the dashboard...')
url = '%s/api/run' % config['wptd_prod_host']
response = requests.post(url,
params={'secret': config['secret']},
data=json.dumps({
'browser_name':
platform['browser_name'],
'browser_version':
platform['browser_version'],
'commit_date':
wpt_commit_date,
'os_name':
platform['os_name'],
'os_version':
platform['os_version'],
'revision':
short_wpt_sha,
'results_url':
gs_results_url
}))
if response.status_code == 201:
print('Run created!')
else:
print('There was an issue creating the TestRun.')
print('Response status code:', response.status_code)
print('Response text:', response.text)
