def test_setup_wptrunner(venv, logger, product):
parser = run.create_parser()
kwargs = vars(parser.parse_args(["--channel=nightly", product]))
kwargs["prompt"] = False
# Hack to get a real existing path
kwargs["binary"] = sys.argv[0]
kwargs["webdriver_binary"] = sys.argv[0]
if kwargs["product"] == "sauce":
kwargs["product"] = "sauce:firefox:63"
run.setup_wptrunner(venv, **kwargs)
def test_setup_wptrunner(venv, product):
parser = run.create_parser()
kwargs = vars(parser.parse_args(["--channel=nightly", product]))
kwargs["prompt"] = False
# Hack to get a real existing path
kwargs["binary"] = sys.argv[0]
kwargs["webdriver_binary"] = sys.argv[0]
if kwargs["product"] == "sauce":
kwargs["product"] = "sauce:firefox:63"
run.setup_wptrunner(venv, **kwargs)
def test_setup_wptrunner(venv, logger, product):
if product == "firefox_android":
pytest.skip("Android emulator doesn't work on docker")
parser = run.create_parser()
kwargs = vars(parser.parse_args(["--channel=nightly", product]))
kwargs["prompt"] = False
# Hack to get a real existing path
kwargs["binary"] = sys.argv[0]
kwargs["webdriver_binary"] = sys.argv[0]
if kwargs["product"] == "sauce":
kwargs["product"] = "sauce:firefox:63"
run.setup_wptrunner(venv, **kwargs)
def run(venv, wpt_args, **kwargs):
global logger
do_delayed_imports()
retcode = 0
parser = get_parser()
wpt_args = create_parser().parse_args(wpt_args)
with open(kwargs["config_file"], 'r') as config_fp:
config = SafeConfigParser()
config.readfp(config_fp)
skip_tests = config.get("file detection", "skip_tests").split()
ignore_changes = set(config.get("file detection", "ignore_changes").split())
results_url = config.get("file detection", "results_url")
if kwargs["output_bytes"] is not None:
replace_streams(kwargs["output_bytes"],
"Log reached capacity (%s bytes); output disabled." % kwargs["output_bytes"])
wpt_args.metadata_root = kwargs["metadata_root"]
try:
os.makedirs(wpt_args.metadata_root)
except OSError:
pass
setup_logging()
browser_name = wpt_args.product.split(":")[0]
if browser_name == "sauce" and not wpt_args.sauce_key:
logger.warning("Cannot run tests on Sauce Labs. No access key.")
return retcode
pr_number = pr()
with TravisFold("browser_setup"):
logger.info(markdown.format_comment_title(wpt_args.product))
if pr is not None:
deepen_checkout(kwargs["user"])
# Ensure we have a branch called "master"
fetch_wpt(kwargs["user"], "master:master")
head_sha1 = get_sha1()
logger.info("Testing web-platform-tests at revision %s" % head_sha1)
wpt_kwargs = Kwargs(vars(wpt_args))
if not wpt_kwargs["test_list"]:
manifest_path = os.path.join(wpt_kwargs["metadata_root"], "MANIFEST.json")
tests_changed, files_affected = get_changed_files(manifest_path, kwargs["rev"],
ignore_changes, skip_tests)
if not (tests_changed or files_affected):
logger.info("No tests changed")
return 0
if tests_changed:
logger.debug("Tests changed:\n%s" % "".join(" * %s\n" % item for item in tests_changed))
if files_affected:
logger.debug("Affected tests:\n%s" % "".join(" * %s\n" % item for item in files_affected))
wpt_kwargs["test_list"] = list(tests_changed | files_affected)
set_default_args(wpt_kwargs)
do_delayed_imports()
wpt_kwargs["stability"] = True
wpt_kwargs["prompt"] = False
wpt_kwargs["install_browser"] = True
wpt_kwargs["install"] = wpt_kwargs["product"].split(":")[0] == "firefox"
wpt_kwargs = setup_wptrunner(venv, **wpt_kwargs)
logger.info("Using binary %s" % wpt_kwargs["binary"])
with TravisFold("running_tests"):
logger.info("Starting tests")
wpt_logger = wptrunner.logger
iterations, results, inconsistent = run(venv, wpt_logger, **wpt_kwargs)
if results:
if inconsistent:
write_inconsistent(logger.error, inconsistent, iterations)
retcode = 2
else:
logger.info("All results were stable\n")
with TravisFold("full_results"):
write_results(logger.info, results, iterations,
pr_number=pr_number,
use_details=True)
if pr_number:
post_results(results, iterations=iterations, url=results_url,
product=wpt_args.product, pr_number=pr_number,
status="failed" if inconsistent else "passed")
else:
logger.info("No tests run.")
return retcode
def run(venv, wpt_args, **kwargs):
global logger
do_delayed_imports()
retcode = 0
parser = get_parser()
wpt_args = create_parser().parse_args(wpt_args)
with open(kwargs["config_file"], 'r') as config_fp:
config = SafeConfigParser()
config.readfp(config_fp)
skip_tests = config.get("file detection", "skip_tests").split()
ignore_changes = set(config.get("file detection", "ignore_changes").split())
if kwargs["output_bytes"] is not None:
replace_streams(kwargs["output_bytes"],
"Log reached capacity (%s bytes); output disabled." % kwargs["output_bytes"])
wpt_args.metadata_root = kwargs["metadata_root"]
try:
os.makedirs(wpt_args.metadata_root)
except OSError:
pass
logger = logging.getLogger(os.path.splitext(__file__)[0])
setup_logging()
browser_name = wpt_args.product.split(":")[0]
if browser_name == "sauce" and not wpt_args.sauce_key:
logger.warning("Cannot run tests on Sauce Labs. No access key.")
return retcode
pr_number = pr()
with TravisFold("browser_setup"):
logger.info(markdown.format_comment_title(wpt_args.product))
if pr is not None:
deepen_checkout(kwargs["user"])
# Ensure we have a branch called "master"
fetch_wpt(kwargs["user"], "master:master")
head_sha1 = get_sha1()
logger.info("Testing web-platform-tests at revision %s" % head_sha1)
if not kwargs["rev"]:
branch_point = testfiles.branch_point()
revish = "%s..HEAD" % branch_point
else:
revish = kwargs["rev"]
files_changed, files_ignored = testfiles.files_changed(revish, ignore_changes)
if files_ignored:
logger.info("Ignoring %s changed files:\n%s" % (len(files_ignored),
"".join(" * %s\n" % item for item in files_ignored)))
tests_changed, files_affected = testfiles.affected_testfiles(files_changed, skip_tests,
manifest_path=os.path.join(
wpt_args.metadata_root,
"MANIFEST.json"))
if not (tests_changed or files_affected):
logger.info("No tests changed")
return 0
wpt_kwargs = Kwargs(vars(wpt_args))
wpt_kwargs["test_list"] = list(tests_changed | files_affected)
set_default_args(wpt_kwargs)
do_delayed_imports()
wpt_kwargs["stability"] = True
wpt_kwargs["prompt"] = False
wpt_kwargs["install_browser"] = True
wpt_kwargs["install"] = wpt_kwargs["product"].split(":")[0] == "firefox"
wpt_kwargs = setup_wptrunner(venv, **wpt_kwargs)
logger.info("Using binary %s" % wpt_kwargs["binary"])
if tests_changed:
logger.debug("Tests changed:\n%s" % "".join(" * %s\n" % item for item in tests_changed))
if files_affected:
logger.debug("Affected tests:\n%s" % "".join(" * %s\n" % item for item in files_affected))
with TravisFold("running_tests"):
logger.info("Starting tests")
wpt_logger = wptrunner.logger
iterations, results, inconsistent = run(venv, wpt_logger, **wpt_kwargs)
if results:
if inconsistent:
write_inconsistent(logger.error, inconsistent, iterations)
retcode = 2
else:
logger.info("All results were stable\n")
with TravisFold("full_results"):
write_results(logger.info, results, iterations,
pr_number=kwargs["comment_pr"],
use_details=True)
else:
logger.info("No tests run.")
return retcode
def run(venv, wpt_args, **kwargs):
do_delayed_imports()
setup_logging()
retcode = 0
wpt_args = create_parser().parse_args(wpt_args)
with open(kwargs["config_file"], 'r') as config_fp:
config = SafeConfigParser()
config.readfp(config_fp)
ignore_changes = set(
config.get("file detection", "ignore_changes").split())
if kwargs["output_bytes"] is not None:
replace_streams(
kwargs["output_bytes"],
"Log reached capacity (%s bytes); output disabled." %
kwargs["output_bytes"])
wpt_args.metadata_root = kwargs["metadata_root"]
try:
os.makedirs(wpt_args.metadata_root)
except OSError:
pass
pr_number = pr()
with TravisFold("browser_setup"):
logger.info(markdown.format_comment_title(wpt_args.product))
if pr is not None:
deepen_checkout(kwargs["user"])
# Ensure we have a branch called "master"
fetch_wpt(kwargs["user"], "master:master")
head_sha1 = get_sha1()
logger.info("Testing web-platform-tests at revision %s" % head_sha1)
wpt_kwargs = Kwargs(vars(wpt_args))
if not wpt_kwargs["test_list"]:
manifest_path = os.path.join(wpt_kwargs["metadata_root"],
"MANIFEST.json")
tests_changed, files_affected = get_changed_files(
manifest_path, kwargs["rev"], ignore_changes)
if not (tests_changed or files_affected):
logger.info("No tests changed")
return 0
if tests_changed:
logger.debug("Tests changed:\n%s" %
"".join(" * %s\n" % item
for item in tests_changed))
if files_affected:
logger.debug("Affected tests:\n%s" %
"".join(" * %s\n" % item
for item in files_affected))
wpt_kwargs["test_list"] = list(tests_changed | files_affected)
do_delayed_imports()
wpt_kwargs["prompt"] = False
wpt_kwargs["install_browser"] = wpt_kwargs["product"].split(
":")[0] == "firefox"
wpt_kwargs["pause_after_test"] = False
wpt_kwargs["verify_log_full"] = False
if wpt_kwargs["repeat"] == 1:
wpt_kwargs["repeat"] = 10
wpt_kwargs["headless"] = False
wpt_kwargs["log_tbpl"] = [sys.stdout]
wpt_kwargs = setup_wptrunner(venv, **wpt_kwargs)
logger.info("Using binary %s" % wpt_kwargs["binary"])
with TravisFold("running_tests"):
logger.info("Starting tests")
wpt_logger = wptrunner.logger
results, inconsistent, slow, iterations = run_step(
wpt_logger, wpt_kwargs["repeat"], True, {}, **wpt_kwargs)
if results:
if inconsistent:
write_inconsistent(logger.error, inconsistent, iterations)
retcode = 2
elif slow:
write_slow_tests(logger.error, slow)
retcode = 2
else:
logger.info("All results were stable\n")
with TravisFold("full_results"):
write_results(logger.info,
results,
iterations,
pr_number=pr_number,
use_details=True)
else:
logger.info("No tests run.")
# Be conservative and only return errors when we know for sure tests are changed.
if tests_changed:
retcode = 3
return retcode
def run(venv, wpt_args, **kwargs):
do_delayed_imports()
retcode = 0
wpt_args = create_parser().parse_args(wpt_args)
with open(kwargs["config_file"], 'r') as config_fp:
config = SafeConfigParser()
config.readfp(config_fp)
skip_tests = config.get("file detection", "skip_tests").split()
ignore_changes = set(config.get("file detection", "ignore_changes").split())
results_url = config.get("file detection", "results_url")
if kwargs["output_bytes"] is not None:
replace_streams(kwargs["output_bytes"],
"Log reached capacity (%s bytes); output disabled." % kwargs["output_bytes"])
wpt_args.metadata_root = kwargs["metadata_root"]
try:
os.makedirs(wpt_args.metadata_root)
except OSError:
pass
setup_logging()
browser_name = wpt_args.product.split(":")[0]
pr_number = pr()
with TravisFold("browser_setup"):
logger.info(markdown.format_comment_title(wpt_args.product))
if pr is not None:
deepen_checkout(kwargs["user"])
# Ensure we have a branch called "master"
fetch_wpt(kwargs["user"], "master:master")
head_sha1 = get_sha1()
logger.info("Testing web-platform-tests at revision %s" % head_sha1)
wpt_kwargs = Kwargs(vars(wpt_args))
if not wpt_kwargs["test_list"]:
manifest_path = os.path.join(wpt_kwargs["metadata_root"], "MANIFEST.json")
tests_changed, files_affected = get_changed_files(manifest_path, kwargs["rev"],
ignore_changes, skip_tests)
if not (tests_changed or files_affected):
logger.info("No tests changed")
return 0
if tests_changed:
logger.debug("Tests changed:\n%s" % "".join(" * %s\n" % item for item in tests_changed))
if files_affected:
logger.debug("Affected tests:\n%s" % "".join(" * %s\n" % item for item in files_affected))
wpt_kwargs["test_list"] = list(tests_changed | files_affected)
do_delayed_imports()
wpt_kwargs["stability"] = True
wpt_kwargs["prompt"] = False
wpt_kwargs["install_browser"] = True
wpt_kwargs["install"] = wpt_kwargs["product"].split(":")[0] == "firefox"
wpt_kwargs = setup_wptrunner(venv, **wpt_kwargs)
logger.info("Using binary %s" % wpt_kwargs["binary"])
with TravisFold("running_tests"):
logger.info("Starting tests")
wpt_logger = wptrunner.logger
iterations, results, inconsistent = stability_run(venv, wpt_logger, **wpt_kwargs)
if results:
if inconsistent:
write_inconsistent(logger.error, inconsistent, iterations)
retcode = 2
else:
logger.info("All results were stable\n")
with TravisFold("full_results"):
write_results(logger.info, results, iterations,
pr_number=pr_number,
use_details=True)
if pr_number:
post_results(results, iterations=iterations, url=results_url,
product=wpt_args.product, pr_number=pr_number,
status="failed" if inconsistent else "passed")
else:
logger.info("No tests run.")
# Be conservative and only return errors when we know for sure tests are changed.
if tests_changed:
retcode = 3
return retcode
