def get_step(
test: Test,
report: bool = False,
smoke_test: bool = False,
ray_wheels: Optional[str] = None,
env: Optional[Dict] = None,
priority_val: int = 0,
):
env = env or {}
step = copy.deepcopy(DEFAULT_STEP_TEMPLATE)
cmd = f"./release/run_release_test.sh \"{test['name']}\" "
if report and not bool(int(os.environ.get("NO_REPORT_OVERRIDE", "0"))):
cmd += " --report"
if smoke_test:
cmd += " --smoke-test"
if ray_wheels:
cmd += f" --ray-wheels {ray_wheels}"
step["command"] = cmd
step["env"].update(env)
commit = get_test_env_var("RAY_COMMIT")
branch = get_test_env_var("RAY_BRANCH")
label = commit[:7] if commit else branch
concurrency_group = test.get("concurrency_group", None)
if concurrency_group:
if concurrency_group not in CONCURRENY_GROUPS:
raise ReleaseTestConfigError(
f"Unknown concurrency group: {concurrency_group}")
concurrency_limit = CONCURRENY_GROUPS[concurrency_group]
else:
concurrency_group, concurrency_limit = get_concurrency_group(test)
step["concurrency_group"] = concurrency_group
step["concurrency"] = concurrency_limit
step["priority"] = priority_val
# If a test is not stable, allow to soft fail
stable = test.get("stable", True)
if not stable:
step["soft_fail"] = True
full_label = "[unstable] "
else:
full_label = ""
full_label += test["name"]
if smoke_test:
full_label += " [smoke test] "
full_label += f" ({label})"
step["label"] = full_label
return step
def get_step(
test: Test,
smoke_test: bool = False,
ray_wheels: Optional[str] = None,
env: Optional[Dict] = None,
):
env = env or {}
step = copy.deepcopy(DEFAULT_STEP_TEMPLATE)
cmd = f"./release/run_release_test.sh \"{test['name']}\" --report"
if smoke_test:
cmd += " --smoke-test"
if ray_wheels:
cmd += f" --ray-wheels {ray_wheels}"
step["command"] = cmd
step["env"].update(env)
commit = get_test_env_var("RAY_COMMIT")
branch = get_test_env_var("RAY_BRANCH")
label = commit[:7] if commit else branch
step["label"] = test["name"]
if smoke_test:
step["label"] += " [smoke test] "
step["label"] += f" ({label})"
return step
def get_step(
test: Test,
smoke_test: bool = False,
ray_wheels: Optional[str] = None,
env: Optional[Dict] = None,
priority_val: int = 0,
):
env = env or {}
step = copy.deepcopy(DEFAULT_STEP_TEMPLATE)
cmd = f"./release/run_release_test.sh \"{test['name']}\" --report"
if smoke_test:
cmd += " --smoke-test"
if ray_wheels:
cmd += f" --ray-wheels {ray_wheels}"
step["command"] = cmd
step["env"].update(env)
commit = get_test_env_var("RAY_COMMIT")
branch = get_test_env_var("RAY_BRANCH")
label = commit[:7] if commit else branch
concurrency_group = test.get("concurrency_group", None)
if concurrency_group:
if concurrency_group not in CONCURRENY_GROUPS:
raise ReleaseTestConfigError(
f"Unknown concurrency group: {concurrency_group}")
concurrency_limit = CONCURRENY_GROUPS[concurrency_group]
else:
concurrency_group, concurrency_limit = get_concurrency_group(test)
step["concurrency_group"] = concurrency_group
step["concurrency"] = concurrency_limit
step["priority"] = priority_val
step["label"] = test["name"]
if smoke_test:
step["label"] += " [smoke test] "
step["label"] += f" ({label})"
return step
def report_result(self, test: Test, result: Result):
logger.info("Persisting results to database...")
result_dict = {
"_runtime": result.runtime,
# Keep session url for legacy support
"_session_url": result.cluster_url,
"_cluster_url": result.cluster_url,
"_commit_url": result.wheels_url,
"_stable": result.stable,
}
now = datetime.datetime.utcnow()
rds_data_client = boto3.client("rds-data", region_name="us-west-2")
if "legacy" in test:
test_name = test["legacy"]["test_name"]
test_suite = test["legacy"]["test_suite"]
else:
test_name = test["name"]
test_suite = ""
team = test["team"] or ""
# Branch name
category = get_test_env_var("RAY_BRANCH", "")
status = result.status or "invalid"
last_logs = result.last_logs or ""
if result.results:
result_dict.update(result.results)
artifacts = {}
parameters = [
{
"name": "created_on",
"typeHint": "TIMESTAMP",
"value": {"stringValue": now.strftime("%Y-%m-%d %H:%M:%S")},
},
{"name": "test_suite", "value": {"stringValue": test_suite}},
{"name": "test_name", "value": {"stringValue": test_name}},
{"name": "status", "value": {"stringValue": status}},
{"name": "last_logs", "value": {"stringValue": last_logs}},
{
"name": "results",
"typeHint": "JSON",
"value": {"stringValue": json.dumps(result_dict)},
},
{
"name": "artifacts",
"typeHint": "JSON",
"value": {"stringValue": json.dumps(artifacts)},
},
{"name": "category", "value": {"stringValue": category}},
{"name": "team", "value": {"stringValue": team}},
{"name": "session_url", "value": {"stringValue": result.cluster_url or ""}},
{"name": "commit_url", "value": {"stringValue": result.wheels_url or ""}},
{"name": "runtime", "value": {"doubleValue": result.runtime or -1.0}},
{"name": "stable", "value": {"booleanValue": result.stable}},
{"name": "frequency", "value": {"stringValue": test.get("frequency", "")}},
{"name": "return_code", "value": {"longValue": result.return_code}},
]
columns = [param["name"] for param in parameters]
values = [f":{param['name']}" for param in parameters]
column_str = ", ".join(columns).strip(", ")
value_str = ", ".join(values).strip(", ")
sql = (
f"INSERT INTO {self.database_table} "
f"({column_str}) "
f"VALUES ({value_str})"
)
logger.debug(f"SQL query: {sql}")
# Default boto3 call timeout is 45 seconds.
retry_delay_s = 64
MAX_RDS_RETRY = 3
exponential_backoff_retry(
lambda: rds_data_client.execute_statement(
database=self.database,
parameters=parameters,
secretArn=RELEASE_AWS_DB_SECRET_ARN,
resourceArn=RELEASE_AWS_DB_RESOURCE_ARN,
schema=self.database_table,
sql=sql,
),
retry_exceptions=rds_data_client.exceptions.StatementTimeoutException,
initial_retry_delay_s=retry_delay_s,
max_retries=MAX_RDS_RETRY,
)
logger.info("Result has been persisted to the database")