def wait_for_url(
url, timeout: float = 300.0, check_time: float = 30.0, status_time: float = 60.0
) -> str:
start_time = time.monotonic()
timeout_at = start_time + timeout
next_status = start_time + status_time
logger.info(f"Waiting up to {timeout} seconds until URL is available " f"({url})")
while not url_exists(url):
now = time.monotonic()
if now >= timeout_at:
raise RayWheelsTimeoutError(
f"Time out when waiting for URL to be available: {url}"
)
if now >= next_status:
logger.info(
f"... still waiting for URL {url} "
f"({int(now - start_time)} seconds) ..."
)
next_status += status_time
# Sleep `check_time` sec before next check.
time.sleep(check_time)
logger.info(f"URL is now available: {url}")
return url
def find_ray_wheels_url(ray_wheels: Optional[str] = None) -> str:
if not ray_wheels:
# If no wheels are specified, default to BUILDKITE_COMMIT
commit = os.environ.get("BUILDKITE_COMMIT", None)
if not commit:
raise RayWheelsUnspecifiedError(
"No Ray wheels specified. Pass `--ray-wheels` or set "
"`BUILDKITE_COMMIT` environment variable. "
"Hint: You can use `-ray-wheels master` to fetch "
"the latest available master wheels.")
branch = os.environ.get("BUILDKITE_BRANCH", DEFAULT_BRANCH)
repo_url = os.environ.get("BUILDKITE_REPO", DEFAULT_REPO)
if not re.match(r"\b([a-f0-9]{40})\b", commit):
# commit is symbolic, like HEAD
latest_commits = get_latest_commits(repo_url, branch, ref=commit)
commit = latest_commits[0]
ray_version = get_ray_version(repo_url, commit)
set_test_env_var("RAY_COMMIT", commit)
set_test_env_var("RAY_BRANCH", branch)
set_test_env_var("RAY_VERSION", ray_version)
return get_ray_wheels_url(repo_url, branch, commit, ray_version)
# If this is a URL, return
if ray_wheels.startswith("https://") or ray_wheels.startswith("http://"):
return ray_wheels
# Else, this is either a commit hash, a branch name, or a combination
# with a repo, e.g. ray-project:master or ray-project:<commit>
if ":" in ray_wheels:
# Repo is specified
owner_or_url, commit_or_branch = ray_wheels.split(":")
else:
# Repo is not specified, use ray-project instead
owner_or_url = DEFAULT_GIT_OWNER
commit_or_branch = ray_wheels
# Construct repo URL for cloning
if "https://" in owner_or_url:
# Already is a repo URL
repo_url = owner_or_url
else:
repo_url = REPO_URL_TPL.format(owner=owner_or_url,
package=DEFAULT_GIT_PACKAGE)
# Todo: This is not ideal as branches that mimic a SHA1 hash
# will also match this.
if not re.match(r"\b([a-f0-9]{40})\b", commit_or_branch):
# This is a branch
branch = commit_or_branch
latest_commits = get_latest_commits(repo_url, branch)
# Let's assume the ray version is constant over these commits
# (otherwise just move it into the for loop)
ray_version = get_ray_version(repo_url, latest_commits[0])
for commit in latest_commits:
wheels_url = get_ray_wheels_url(repo_url, branch, commit,
ray_version)
if url_exists(wheels_url):
set_test_env_var("RAY_COMMIT", commit)
return wheels_url
raise RayWheelsNotFoundError(
f"Couldn't find latest available wheels for repo "
f"{repo_url}, branch {branch} (version {ray_version}). "
f"Try again later or check Buildkite logs if wheel builds "
f"failed.")
# Else, this is a commit
commit = commit_or_branch
ray_version = get_ray_version(repo_url, commit)
branch = os.environ.get("BUILDKITE_BRANCH", DEFAULT_BRANCH)
wheels_url = get_ray_wheels_url(repo_url, branch, commit, ray_version)
set_test_env_var("RAY_COMMIT", commit)
set_test_env_var("RAY_BRANCH", branch)
set_test_env_var("RAY_VERSION", ray_version)
return wheels_url