def testConcurrencyGroupSmokeTest(self):
with tempfile.TemporaryDirectory() as tmpdir:
cluster_config_full = {
"head_node_type": {
"instance_type": "n1-standard-16" # 16 CPUs, 0 GPUs
},
"worker_node_types": [
{
"instance_type":
"random-str-xxx-32", # 32 CPUS, 0 GPUs
"max_workers": 10,
},
],
}
cluster_config_smoke = {
"head_node_type": {
"instance_type": "n1-standard-16" # 16 CPUs, 0 GPUs
},
"worker_node_types": [
{
"instance_type":
"random-str-xxx-32", # 32 CPUS, 0 GPUs
"max_workers": 1,
},
],
}
cluster_config_full_path = os.path.join(tmpdir, "full.yaml")
with open(cluster_config_full_path, "w") as fp:
yaml.safe_dump(cluster_config_full, fp)
cluster_config_smoke_path = os.path.join(tmpdir, "smoke.yaml")
with open(cluster_config_smoke_path, "w") as fp:
yaml.safe_dump(cluster_config_smoke, fp)
test = Test({
"name": "test_1",
"cluster": {
"cluster_compute": cluster_config_full_path
},
"smoke_test": {
"cluster": {
"cluster_compute": cluster_config_smoke_path
},
},
})
step = get_step(test, smoke_test=False)
self.assertEquals(step["concurrency_group"], "medium")
step = get_step(test, smoke_test=True)
self.assertEquals(step["concurrency_group"], "small")
def testGetStep(self):
test = Test({
"name": "test",
"frequency": "nightly",
"run": {
"script": "test_script.py"
},
"smoke_test": {
"frequency": "multi"
},
})
step = get_step(test, smoke_test=False)
self.assertNotIn("--smoke-test", step["command"])
step = get_step(test, smoke_test=True)
self.assertIn("--smoke-test", step["command"])
def main(test_collection_file: Optional[str] = None):
settings = get_pipeline_settings()
repo = settings["ray_test_repo"]
branch = settings["ray_test_branch"]
tmpdir = None
env = {}
if repo:
# If the Ray test repo is set, we clone that repo to fetch
# the test configuration file. Otherwise we might be missing newly
# added test.
repo = settings["ray_test_repo"]
tmpdir = tempfile.mktemp()
clone_cmd = f"git clone --depth 1 --branch {branch} {repo} {tmpdir}"
try:
subprocess.check_output(clone_cmd, shell=True)
except Exception as e:
raise ReleaseTestCLIError(f"Could not clone test repository "
f"{repo} (branch {branch}): {e}") from e
test_collection_file = os.path.join(tmpdir, "release",
"release_tests.yaml")
env = {
"RAY_TEST_REPO": repo,
"RAY_TEST_BRANCH": branch,
}
else:
test_collection_file = test_collection_file or os.path.join(
os.path.dirname(__file__), "..", "..", "release_tests.yaml")
test_collection = read_and_validate_release_test_collection(
test_collection_file)
if tmpdir:
shutil.rmtree(tmpdir, ignore_errors=True)
frequency = settings["frequency"]
prefer_smoke_tests = settings["prefer_smoke_tests"]
test_attr_regex_filters = settings["test_attr_regex_filters"]
ray_wheels = settings["ray_wheels"]
priority = settings["priority"]
logger.info(
f"Found the following buildkite pipeline settings:\n\n"
f" frequency = {settings['frequency']}\n"
f" prefer_smoke_tests = {settings['prefer_smoke_tests']}\n"
f" test_attr_regex_filters = {settings['test_attr_regex_filters']}\n"
f" ray_wheels = {settings['ray_wheels']}\n"
f" ray_test_repo = {settings['ray_test_repo']}\n"
f" ray_test_branch = {settings['ray_test_branch']}\n"
f" priority = {settings['priority']}\n"
f" no_concurrency_limit = {settings['no_concurrency_limit']}\n")
filtered_tests = filter_tests(
test_collection,
frequency=frequency,
test_attr_regex_filters=test_attr_regex_filters,
prefer_smoke_tests=prefer_smoke_tests,
)
logger.info(f"Found {len(filtered_tests)} tests to run.")
if len(filtered_tests) == 0:
raise ReleaseTestCLIError(
"Empty test collection. The selected frequency or filter did "
"not return any tests to run. Adjust your filters.")
grouped_tests = group_tests(filtered_tests)
group_str = ""
for group, tests in grouped_tests.items():
group_str += f"\n{group}:\n"
for test, smoke in tests:
group_str += f" {test['name']}"
if smoke:
group_str += " [smoke test]"
group_str += "\n"
logger.info(f"Tests to run:\n{group_str}")
# Wait for wheels here so we have them ready before we kick off
# the other workers
ray_wheels_url = find_and_wait_for_ray_wheels_url(
ray_wheels, timeout=DEFAULT_WHEEL_WAIT_TIMEOUT)
logger.info(f"Starting pipeline for Ray wheel: {ray_wheels_url}")
no_concurrency_limit = settings["no_concurrency_limit"]
if no_concurrency_limit:
logger.warning("Concurrency is not limited for this run!")
# Report if REPORT=1 or BUILDKITE_SOURCE=schedule
report = (bool(int(os.environ.get("REPORT", "0")))
or os.environ.get("BUILDKITE_SOURCE", "manual") == "schedule")
steps = []
for group in sorted(grouped_tests):
tests = grouped_tests[group]
group_steps = []
for test, smoke_test in tests:
# If the python version is defined, we need a different Ray wheels URL
if "python" in test:
python_version = parse_python_version(test["python"])
this_ray_wheels_url = find_ray_wheels_url(
ray_wheels, python_version=python_version)
else:
this_ray_wheels_url = ray_wheels_url
step = get_step(
test,
report=report,
smoke_test=smoke_test,
ray_wheels=this_ray_wheels_url,
env=env,
priority_val=priority.value,
)
if no_concurrency_limit:
step.pop("concurrency", None)
step.pop("concurrency_group", None)
group_steps.append(step)
group_step = {"group": group, "steps": group_steps}
steps.append(group_step)
if "BUILDKITE" in os.environ:
if os.path.exists(PIPELINE_ARTIFACT_PATH):
shutil.rmtree(PIPELINE_ARTIFACT_PATH)
os.makedirs(PIPELINE_ARTIFACT_PATH, exist_ok=True, mode=0o755)
with open(os.path.join(PIPELINE_ARTIFACT_PATH, "pipeline.json"),
"wt") as fp:
json.dump(steps, fp)
settings["frequency"] = settings["frequency"].value
settings["priority"] = settings["priority"].value
with open(os.path.join(PIPELINE_ARTIFACT_PATH, "settings.json"),
"wt") as fp:
json.dump(settings, fp)
steps_str = json.dumps(steps)
print(steps_str)
def main(test_collection_file: Optional[str] = None):
settings = get_pipeline_settings()
repo = settings["ray_test_repo"]
branch = settings["ray_test_branch"]
tmpdir = None
env = {}
if repo:
# If the Ray test repo is set, we clone that repo to fetch
# the test configuration file. Otherwise we might be missing newly
# added test.
repo = settings["ray_test_repo"]
tmpdir = tempfile.mktemp()
clone_cmd = f"git clone --depth 1 --branch {branch} {repo} {tmpdir}"
try:
subprocess.check_output(clone_cmd, shell=True)
except Exception as e:
raise ReleaseTestCLIError(f"Could not clone test repository "
f"{repo} (branch {branch}): {e}") from e
test_collection_file = os.path.join(tmpdir, "release",
"release_tests.yaml")
env = {
"RAY_TEST_REPO": repo,
"RAY_TEST_BRANCH": branch,
}
else:
test_collection_file = test_collection_file or os.path.join(
os.path.dirname(__file__), "..", "..", "release_tests.yaml")
test_collection = read_and_validate_release_test_collection(
test_collection_file)
if tmpdir:
shutil.rmtree(tmpdir, ignore_errors=True)
frequency = settings["frequency"]
test_name_filter = settings["test_name_filter"]
ray_wheels = settings["ray_wheels"]
logger.info(f"Found the following buildkite pipeline settings:\n\n"
f" frequency = {settings['frequency']}\n"
f" test_name_filter = {settings['test_name_filter']}\n"
f" ray_wheels = {settings['ray_wheels']}\n"
f" ray_test_repo = {settings['ray_test_repo']}\n"
f" ray_test_branch = {settings['ray_test_branch']}\n")
filtered_tests = filter_tests(test_collection,
frequency=frequency,
test_name_filter=test_name_filter)
logger.info(f"Found {len(filtered_tests)} tests to run.")
if len(filtered_tests) == 0:
raise ReleaseTestCLIError(
"Empty test collection. The selected frequency or filter did "
"not return any tests to run. Adjust your filters.")
grouped_tests = group_tests(filtered_tests)
group_str = ""
for group, tests in grouped_tests.items():
group_str += f"\n{group}:\n"
for test, smoke in tests:
group_str += f" {test['name']}"
if smoke:
group_str += " [smoke test]"
group_str += "\n"
logger.info(f"Tests to run:\n{group_str}")
# Wait for wheels here so we have them ready before we kick off
# the other workers
ray_wheels_url = find_and_wait_for_ray_wheels_url(
ray_wheels, timeout=DEFAULT_WHEEL_WAIT_TIMEOUT)
logger.info(f"Starting pipeline for Ray wheel: {ray_wheels_url}")
steps = []
for group in sorted(grouped_tests):
tests = grouped_tests[group]
group_steps = []
for test, smoke_test in tests:
step = get_step(test,
smoke_test=smoke_test,
ray_wheels=ray_wheels_url,
env=env)
group_steps.append(step)
group_step = {"group": group, "steps": group_steps}
steps.append(group_step)
steps_str = json.dumps(steps)
print(steps_str)
