def main(
test_name: str,
test_collection_file: Optional[str] = None,
smoke_test: bool = False,
report: bool = False,
ray_wheels: Optional[str] = None,
cluster_id: Optional[str] = None,
cluster_env_id: Optional[str] = None,
no_terminate: bool = False,
):
test_collection_file = test_collection_file or os.path.join(
os.path.dirname(__file__), "..", "..", "release_tests.yaml")
test_collection = read_and_validate_release_test_collection(
test_collection_file)
test = find_test(test_collection, test_name)
if not test:
raise ReleaseTestCLIError(
f"Test `{test_name}` not found in collection file: "
f"{test_collection_file}")
if smoke_test:
test = as_smoke_test(test)
ray_wheels_url = find_and_wait_for_ray_wheels_url(
ray_wheels, timeout=DEFAULT_WHEEL_WAIT_TIMEOUT)
anyscale_project = os.environ.get("ANYSCALE_PROJECT", None)
if not anyscale_project:
raise ReleaseTestCLIError(
"You have to set the ANYSCALE_PROJECT environment variable!")
maybe_fetch_api_token()
result = Result()
reporters = [LogReporter()]
if report:
reporters.append(LegacyRDSReporter())
try:
result = run_release_test(
test,
anyscale_project=anyscale_project,
result=result,
ray_wheels_url=ray_wheels_url,
reporters=reporters,
smoke_test=smoke_test,
cluster_id=cluster_id,
cluster_env_id=cluster_env_id,
no_terminate=no_terminate,
)
except ReleaseTestError as e:
logger.exception(e)
logger.info(f"Release test pipeline for test {test['name']} completed. "
f"Returning with exit code = {result.return_code}")
sys.exit(result.return_code)
def get_step(
test: Test,
report: bool = False,
smoke_test: bool = False,
ray_wheels: Optional[str] = None,
env: Optional[Dict] = None,
priority_val: int = 0,
):
env = env or {}
step = copy.deepcopy(DEFAULT_STEP_TEMPLATE)
cmd = f"./release/run_release_test.sh \"{test['name']}\" "
if report and not bool(int(os.environ.get("NO_REPORT_OVERRIDE", "0"))):
cmd += " --report"
if smoke_test:
cmd += " --smoke-test"
if ray_wheels:
cmd += f" --ray-wheels {ray_wheels}"
step["command"] = cmd
env_to_use = test.get("env", DEFAULT_ENVIRONMENT)
env_dict = load_environment(env_to_use)
env_dict.update(env)
step["env"].update(env_dict)
if "python" in test:
python_version = parse_python_version(test["python"])
else:
python_version = DEFAULT_PYTHON_VERSION
step["plugins"][0]["docker#v3.9.0"][
"image"] = f"rayproject/ray:latest-py{python_version_str(python_version)}"
commit = get_test_env_var("RAY_COMMIT")
branch = get_test_env_var("RAY_BRANCH")
label = commit[:7] if commit else branch
concurrency_group = test.get("concurrency_group", None)
if concurrency_group:
if concurrency_group not in CONCURRENY_GROUPS:
raise ReleaseTestConfigError(
f"Unknown concurrency group: {concurrency_group}")
concurrency_limit = CONCURRENY_GROUPS[concurrency_group]
else:
if smoke_test:
concurrency_test = as_smoke_test(test)
else:
concurrency_test = test
concurrency_group, concurrency_limit = get_concurrency_group(
concurrency_test)
step["concurrency_group"] = concurrency_group
step["concurrency"] = concurrency_limit
step["priority"] = priority_val
# If a test is not stable, allow to soft fail
stable = test.get("stable", True)
if not stable:
step["soft_fail"] = True
full_label = "[unstable] "
else:
full_label = ""
full_label += test["name"]
if smoke_test:
full_label += " [smoke test] "
full_label += f" ({label})"
step["label"] = full_label
return step
def main(
test_name: str,
test_collection_file: Optional[str] = None,
smoke_test: bool = False,
report: bool = False,
ray_wheels: Optional[str] = None,
cluster_id: Optional[str] = None,
cluster_env_id: Optional[str] = None,
env: Optional[str] = None,
no_terminate: bool = False,
):
test_collection_file = test_collection_file or os.path.join(
os.path.dirname(__file__), "..", "..", "release_tests.yaml")
test_collection = read_and_validate_release_test_collection(
test_collection_file)
test = find_test(test_collection, test_name)
if not test:
raise ReleaseTestCLIError(
f"Test `{test_name}` not found in collection file: "
f"{test_collection_file}")
if smoke_test:
test = as_smoke_test(test)
env_to_use = env or test.get("env", DEFAULT_ENVIRONMENT)
env_dict = load_environment(env_to_use)
populate_os_env(env_dict)
if "python" in test:
python_version = parse_python_version(test["python"])
else:
python_version = DEFAULT_PYTHON_VERSION
ray_wheels_url = find_and_wait_for_ray_wheels_url(
ray_wheels,
python_version=python_version,
timeout=DEFAULT_WHEEL_WAIT_TIMEOUT)
anyscale_project = os.environ.get("ANYSCALE_PROJECT", None)
if not anyscale_project:
raise ReleaseTestCLIError(
"You have to set the ANYSCALE_PROJECT environment variable!")
maybe_fetch_api_token()
result = Result()
reporters = [LogReporter()]
if "BUILDKITE" in os.environ:
reporters.append(ArtifactsReporter())
if report:
reporters.append(LegacyRDSReporter())
reporters.append(DBReporter())
try:
result = run_release_test(
test,
anyscale_project=anyscale_project,
result=result,
ray_wheels_url=ray_wheels_url,
reporters=reporters,
smoke_test=smoke_test,
cluster_id=cluster_id,
cluster_env_id=cluster_env_id,
no_terminate=no_terminate,
)
return_code = result.return_code
except ReleaseTestError as e:
logger.exception(e)
return_code = e.exit_code.value
logger.info(f"Release test pipeline for test {test['name']} completed. "
f"Returning with exit code = {return_code}")
sys.exit(result.return_code)
def get_step(
test: Test,
report: bool = False,
smoke_test: bool = False,
ray_wheels: Optional[str] = None,
env: Optional[Dict] = None,
priority_val: int = 0,
):
env = env or {}
step = copy.deepcopy(DEFAULT_STEP_TEMPLATE)
cmd = f"./release/run_release_test.sh \"{test['name']}\" "
if report and not bool(int(os.environ.get("NO_REPORT_OVERRIDE", "0"))):
cmd += " --report"
if smoke_test:
cmd += " --smoke-test"
if ray_wheels:
cmd += f" --ray-wheels {ray_wheels}"
step["command"] = cmd
step["env"].update(env)
commit = get_test_env_var("RAY_COMMIT")
branch = get_test_env_var("RAY_BRANCH")
label = commit[:7] if commit else branch
concurrency_group = test.get("concurrency_group", None)
if concurrency_group:
if concurrency_group not in CONCURRENY_GROUPS:
raise ReleaseTestConfigError(
f"Unknown concurrency group: {concurrency_group}"
)
concurrency_limit = CONCURRENY_GROUPS[concurrency_group]
else:
if smoke_test:
concurrency_test = as_smoke_test(test)
else:
concurrency_test = test
concurrency_group, concurrency_limit = get_concurrency_group(concurrency_test)
step["concurrency_group"] = concurrency_group
step["concurrency"] = concurrency_limit
step["priority"] = priority_val
# If a test is not stable, allow to soft fail
stable = test.get("stable", True)
if not stable:
step["soft_fail"] = True
full_label = "[unstable] "
else:
full_label = ""
full_label += test["name"]
if smoke_test:
full_label += " [smoke test] "
full_label += f" ({label})"
step["label"] = full_label
return step