def parse_wheels_filename(
filename: str, ) -> Tuple[Optional[str], Optional[Tuple[int, int]]]:
"""Parse filename and return Ray version + python version"""
matched = re.search(
r"ray-([0-9a-z\.]+)-cp([0-9]{2,3})-cp([0-9]{2,3})m?-manylinux2014_x86_64\.whl$",
filename,
)
if not matched:
return None, None
ray_version = matched.group(1)
py_version_str = matched.group(2)
try:
python_version = parse_python_version(py_version_str)
except Exception:
return ray_version, None
return ray_version, python_version
def populate_cluster_env_variables(test: Test, ray_wheels_url: str) -> Dict:
env = get_test_environment()
commit = env.get("RAY_COMMIT", None)
if not commit:
match = re.search(r"/([a-f0-9]{40})/", ray_wheels_url)
if match:
commit = match.group(1)
env["RAY_WHEELS_SANITY_CHECK"] = get_wheels_sanity_check(commit)
env["RAY_WHEELS"] = ray_wheels_url
if "python" in test:
python_version = parse_python_version(test["python"])
else:
python_version = DEFAULT_PYTHON_VERSION
env["RAY_IMAGE_NIGHTLY_CPU"] = f"anyscale/ray:1.12.1-py{python_version_str(python_version)}"
env["RAY_IMAGE_ML_NIGHTLY_GPU"] = f"anyscale/ray-ml:1.12.1-py{python_version_str(python_version)}-gpu"
return env
def get_step(
test: Test,
report: bool = False,
smoke_test: bool = False,
ray_wheels: Optional[str] = None,
env: Optional[Dict] = None,
priority_val: int = 0,
):
env = env or {}
step = copy.deepcopy(DEFAULT_STEP_TEMPLATE)
cmd = f"./release/run_release_test.sh \"{test['name']}\" "
if report and not bool(int(os.environ.get("NO_REPORT_OVERRIDE", "0"))):
cmd += " --report"
if smoke_test:
cmd += " --smoke-test"
if ray_wheels:
cmd += f" --ray-wheels {ray_wheels}"
step["command"] = cmd
env_to_use = test.get("env", DEFAULT_ENVIRONMENT)
env_dict = load_environment(env_to_use)
env_dict.update(env)
step["env"].update(env_dict)
if "python" in test:
python_version = parse_python_version(test["python"])
else:
python_version = DEFAULT_PYTHON_VERSION
step["plugins"][0]["docker#v3.9.0"][
"image"] = f"rayproject/ray:latest-py{python_version_str(python_version)}"
commit = get_test_env_var("RAY_COMMIT")
branch = get_test_env_var("RAY_BRANCH")
label = commit[:7] if commit else branch
concurrency_group = test.get("concurrency_group", None)
if concurrency_group:
if concurrency_group not in CONCURRENY_GROUPS:
raise ReleaseTestConfigError(
f"Unknown concurrency group: {concurrency_group}")
concurrency_limit = CONCURRENY_GROUPS[concurrency_group]
else:
if smoke_test:
concurrency_test = as_smoke_test(test)
else:
concurrency_test = test
concurrency_group, concurrency_limit = get_concurrency_group(
concurrency_test)
step["concurrency_group"] = concurrency_group
step["concurrency"] = concurrency_limit
step["priority"] = priority_val
# If a test is not stable, allow to soft fail
stable = test.get("stable", True)
if not stable:
step["soft_fail"] = True
full_label = "[unstable] "
else:
full_label = ""
full_label += test["name"]
if smoke_test:
full_label += " [smoke test] "
full_label += f" ({label})"
step["label"] = full_label
return step
def main(test_collection_file: Optional[str] = None):
settings = get_pipeline_settings()
repo = settings["ray_test_repo"]
branch = settings["ray_test_branch"]
tmpdir = None
env = {}
if repo:
# If the Ray test repo is set, we clone that repo to fetch
# the test configuration file. Otherwise we might be missing newly
# added test.
repo = settings["ray_test_repo"]
tmpdir = tempfile.mktemp()
clone_cmd = f"git clone --depth 1 --branch {branch} {repo} {tmpdir}"
try:
subprocess.check_output(clone_cmd, shell=True)
except Exception as e:
raise ReleaseTestCLIError(f"Could not clone test repository "
f"{repo} (branch {branch}): {e}") from e
test_collection_file = os.path.join(tmpdir, "release",
"release_tests.yaml")
env = {
"RAY_TEST_REPO": repo,
"RAY_TEST_BRANCH": branch,
}
else:
test_collection_file = test_collection_file or os.path.join(
os.path.dirname(__file__), "..", "..", "release_tests.yaml")
test_collection = read_and_validate_release_test_collection(
test_collection_file)
if tmpdir:
shutil.rmtree(tmpdir, ignore_errors=True)
frequency = settings["frequency"]
prefer_smoke_tests = settings["prefer_smoke_tests"]
test_attr_regex_filters = settings["test_attr_regex_filters"]
ray_wheels = settings["ray_wheels"]
priority = settings["priority"]
logger.info(
f"Found the following buildkite pipeline settings:\n\n"
f" frequency = {settings['frequency']}\n"
f" prefer_smoke_tests = {settings['prefer_smoke_tests']}\n"
f" test_attr_regex_filters = {settings['test_attr_regex_filters']}\n"
f" ray_wheels = {settings['ray_wheels']}\n"
f" ray_test_repo = {settings['ray_test_repo']}\n"
f" ray_test_branch = {settings['ray_test_branch']}\n"
f" priority = {settings['priority']}\n"
f" no_concurrency_limit = {settings['no_concurrency_limit']}\n")
filtered_tests = filter_tests(
test_collection,
frequency=frequency,
test_attr_regex_filters=test_attr_regex_filters,
prefer_smoke_tests=prefer_smoke_tests,
)
logger.info(f"Found {len(filtered_tests)} tests to run.")
if len(filtered_tests) == 0:
raise ReleaseTestCLIError(
"Empty test collection. The selected frequency or filter did "
"not return any tests to run. Adjust your filters.")
grouped_tests = group_tests(filtered_tests)
group_str = ""
for group, tests in grouped_tests.items():
group_str += f"\n{group}:\n"
for test, smoke in tests:
group_str += f" {test['name']}"
if smoke:
group_str += " [smoke test]"
group_str += "\n"
logger.info(f"Tests to run:\n{group_str}")
# Wait for wheels here so we have them ready before we kick off
# the other workers
ray_wheels_url = find_and_wait_for_ray_wheels_url(
ray_wheels, timeout=DEFAULT_WHEEL_WAIT_TIMEOUT)
logger.info(f"Starting pipeline for Ray wheel: {ray_wheels_url}")
no_concurrency_limit = settings["no_concurrency_limit"]
if no_concurrency_limit:
logger.warning("Concurrency is not limited for this run!")
# Report if REPORT=1 or BUILDKITE_SOURCE=schedule
report = (bool(int(os.environ.get("REPORT", "0")))
or os.environ.get("BUILDKITE_SOURCE", "manual") == "schedule")
steps = []
for group in sorted(grouped_tests):
tests = grouped_tests[group]
group_steps = []
for test, smoke_test in tests:
# If the python version is defined, we need a different Ray wheels URL
if "python" in test:
python_version = parse_python_version(test["python"])
this_ray_wheels_url = find_ray_wheels_url(
ray_wheels, python_version=python_version)
else:
this_ray_wheels_url = ray_wheels_url
step = get_step(
test,
report=report,
smoke_test=smoke_test,
ray_wheels=this_ray_wheels_url,
env=env,
priority_val=priority.value,
)
if no_concurrency_limit:
step.pop("concurrency", None)
step.pop("concurrency_group", None)
group_steps.append(step)
group_step = {"group": group, "steps": group_steps}
steps.append(group_step)
if "BUILDKITE" in os.environ:
if os.path.exists(PIPELINE_ARTIFACT_PATH):
shutil.rmtree(PIPELINE_ARTIFACT_PATH)
os.makedirs(PIPELINE_ARTIFACT_PATH, exist_ok=True, mode=0o755)
with open(os.path.join(PIPELINE_ARTIFACT_PATH, "pipeline.json"),
"wt") as fp:
json.dump(steps, fp)
settings["frequency"] = settings["frequency"].value
settings["priority"] = settings["priority"].value
with open(os.path.join(PIPELINE_ARTIFACT_PATH, "settings.json"),
"wt") as fp:
json.dump(settings, fp)
steps_str = json.dumps(steps)
print(steps_str)
def main(
test_name: str,
test_collection_file: Optional[str] = None,
smoke_test: bool = False,
report: bool = False,
ray_wheels: Optional[str] = None,
cluster_id: Optional[str] = None,
cluster_env_id: Optional[str] = None,
no_terminate: bool = False,
):
test_collection_file = test_collection_file or os.path.join(
os.path.dirname(__file__), "..", "..", "release_tests.yaml")
test_collection = read_and_validate_release_test_collection(
test_collection_file)
test = find_test(test_collection, test_name)
if not test:
raise ReleaseTestCLIError(
f"Test `{test_name}` not found in collection file: "
f"{test_collection_file}")
if smoke_test:
test = as_smoke_test(test)
if "python" in test:
python_version = parse_python_version(test["python"])
else:
python_version = DEFAULT_PYTHON_VERSION
ray_wheels_url = find_and_wait_for_ray_wheels_url(
ray_wheels,
python_version=python_version,
timeout=DEFAULT_WHEEL_WAIT_TIMEOUT)
anyscale_project = os.environ.get("ANYSCALE_PROJECT", None)
if not anyscale_project:
raise ReleaseTestCLIError(
"You have to set the ANYSCALE_PROJECT environment variable!")
maybe_fetch_api_token()
result = Result()
reporters = [LogReporter()]
if "BUILDKITE" in os.environ:
reporters.append(ArtifactsReporter())
if report:
reporters.append(LegacyRDSReporter())
reporters.append(DBReporter())
try:
result = run_release_test(
test,
anyscale_project=anyscale_project,
result=result,
ray_wheels_url=ray_wheels_url,
reporters=reporters,
smoke_test=smoke_test,
cluster_id=cluster_id,
cluster_env_id=cluster_env_id,
no_terminate=no_terminate,
)
except ReleaseTestError as e:
logger.exception(e)
logger.info(f"Release test pipeline for test {test['name']} completed. "
f"Returning with exit code = {result.return_code}")
sys.exit(result.return_code)