def main(platform_id, platform, args, config): loggingLevel = getattr(logging, args.log.upper(), None) logging.basicConfig(level=loggingLevel) logger = logging.getLogger() print('PLATFORM_ID:', platform_id) print('PLATFORM INFO:', platform) if args.path: print('Running tests in path: %s' % args.path) else: print('Running all tests!') if args.upload: print('Setting up storage client') from google.cloud import storage storage_client = storage.Client(project='wptdashboard') bucket = storage_client.get_bucket(config['gs_results_bucket']) verify_gsutil_installed(config) if args.create_testrun: assert len(config['secret']) == 64, ( 'Valid secret required to create TestRun') if not platform.get('sauce'): if platform['browser_name'] == 'chrome': browser_binary = config['chrome_binary'] elif platform['browser_name'] == 'firefox': browser_binary = config['firefox_binary'] if platform['browser_name'] == 'chrome': verify_browser_binary_version(platform, browser_binary) verify_os_name(platform) verify_or_set_os_version(platform) print('Platform information:') print('Browser version: %s' % platform['browser_version']) print('OS name: %s' % platform['os_name']) print('OS version: %s' % platform['os_version']) print('==================================================') print('Setting up WPT checkout') setup_wpt(config) print('Getting WPT commit SHA and Date') wpt_sha, wpt_commit_date = get_commit_details(args, config, logger) print('WPT SHA: %s' % wpt_sha) print('WPT Commit Date: %s' % wpt_commit_date) short_wpt_sha = wpt_sha[0:10] abs_report_log_path = "%s/wptd-%s-%s-report.log" % ( config['build_path'], short_wpt_sha, platform_id) abs_report_chunks_path = "%s/%s/%s-report-chunks" % ( config['build_path'], short_wpt_sha, platform_id) abs_current_chunk_path = os.path.join(abs_report_chunks_path, 'current.json') mkdirp(abs_report_chunks_path) sha_summary_gz_path = '%s/%s-summary.json.gz' % (short_wpt_sha, platform_id) abs_sha_summary_gz_path = "%s/%s" % (config['build_path'], sha_summary_gz_path) gs_results_base_path = "%s/%s/%s" % (config['build_path'], short_wpt_sha, platform_id) gs_results_url = 'https://storage.googleapis.com/%s/%s' % ( config['gs_results_bucket'], sha_summary_gz_path) print('==================================================') print('Running WPT') report = Report(args.total_chunks, abs_report_chunks_path) expected_test_count = 0 for this_chunk in range(1, args.total_chunks + 1): if platform.get('sauce'): if platform['browser_name'] == 'edge': sauce_browser_name = 'MicrosoftEdge' else: sauce_browser_name = platform['browser_name'] command = [ './wpt', 'run', 'sauce:%s:%s' % (sauce_browser_name, platform['browser_version']), '--sauce-platform=%s %s' % (platform['os_name'], platform['os_version']), '--sauce-key=%s' % config['sauce_key'], '--sauce-user=%s' % config['sauce_user'], '--sauce-connect-binary=%s' % config['sauce_connect_binary'], '--sauce-tunnel-id=%s' % config['sauce_tunnel_id'], '--no-restart-on-unexpected', '--processes=2', '--run-by-dir=3', ] if args.path: command.insert(3, args.path) else: command = [ 'xvfb-run', '--auto-servernum', './wpt', 'run', platform['browser_name'], ] if args.path: command.insert(5, args.path) if platform['browser_name'] == 'chrome': command.extend(['--binary', browser_binary]) if platform['browser_name'] == 'firefox': command.extend(['--install-browser', '--yes']) command.append('--certutil-binary=certutil') # temporary fix to allow WebRTC tests to call getUserMedia command.extend( ['--setpref', 'media.navigator.streams.fake=true']) command.append('--log-mach=-') raw_log_filename = tempfile.mkstemp('-wptdashboard')[1] command.extend(['--log-raw', raw_log_filename]) command.extend(['--log-wptreport', abs_current_chunk_path]) command.append('--install-fonts') command.extend([ '--this-chunk', str(this_chunk), '--total-chunks', str(args.total_chunks) ]) for attempt_number in range(1, args.max_attempts + 1): print('Running chunk %s of %s (attempt %s of %s)' % (this_chunk, args.total_chunks, attempt_number, args.max_attempts)) # In the event of a failed attempt, previously-created files will # still be available on disk. Remove these to guard against errors # where the next attempt fails to write new results. for name in (abs_current_chunk_path, raw_log_filename): try: os.remove(name) except OSError: pass return_code = subprocess.call(command, cwd=config['wpt_path']) print('Return code from wptrunner: %s' % return_code) chunk_test_count = len(get_expected_tests(raw_log_filename)) print('%s tests defined in chunk' % chunk_test_count) expected_test_count += chunk_test_count try: data = report.load_chunk(this_chunk, abs_current_chunk_path) print 'Report contains %s results' % len(data['results']) break except InsufficientData: pass else: print('No results found after %s attempts. Giving up.' % args.max_attempts) os.remove(raw_log_filename) print('==================================================') print('Finished WPT run') if platform['browser_name'] == 'firefox': print('Verifying installed firefox matches platform ID') firefox_path = '%s/_venv/firefox/firefox' % config['wpt_path'] verify_browser_binary_version(platform, firefox_path) print('Creating summary of results') try: summary = report.summarize() actual_test_count = len(summary.keys()) actual_percentage = actual_test_count / expected_test_count if actual_percentage < args.partial_threshold / 100: raise InsufficientData('%s of %s is below threshold of %s%%' % (actual_test_count, expected_test_count, args.partial_threshold)) except InsufficientData as exc: logging.fatal('Insufficient report data (%s). Stopping.', exc) exit(1) print('==================================================') print('Writing summary.json.gz to local filesystem') write_gzip_json(abs_sha_summary_gz_path, summary) print('Wrote file %s' % abs_sha_summary_gz_path) print('==================================================') print('Writing individual result files to local filesystem') for result in report.each_result(): test_file = result['test'] filepath = '%s%s' % (gs_results_base_path, test_file) write_gzip_json(filepath, result) print('Wrote file %s' % filepath) if not args.upload: print('==================================================') print('Stopping here (pass --upload to upload results to WPTD).') return print('==================================================') print('Uploading results to gs://%s' % config['gs_results_bucket']) command = [ 'gsutil', '-m', '-h', 'Content-Encoding:gzip', 'rsync', '-r', short_wpt_sha, 'gs://wptd/%s' % short_wpt_sha ] return_code = subprocess.check_call(command, cwd=config['build_path']) assert return_code == 0 print('Successfully uploaded!') print('HTTP summary URL: %s' % gs_results_url) if not args.create_testrun: print('==================================================') print('Stopping here') print('pass --create-testrun to create and promote this TestRun).') return print('==================================================') print('Creating new TestRun in the dashboard...') url = '%s/api/run' % config['wptd_prod_host'] response = requests.post(url, params={'secret': config['secret']}, data=json.dumps({ 'browser_name': platform['browser_name'], 'browser_version': platform['browser_version'], 'commit_date': wpt_commit_date, 'os_name': platform['os_name'], 'os_version': platform['os_version'], 'revision': short_wpt_sha, 'results_url': gs_results_url })) if response.status_code == 201: print('Run created!') else: print('There was an issue creating the TestRun.') print('Response status code:', response.status_code) print('Response text:', response.text)