def wait_for_url( url, timeout: float = 300.0, check_time: float = 30.0, status_time: float = 60.0 ) -> str: start_time = time.monotonic() timeout_at = start_time + timeout next_status = start_time + status_time logger.info(f"Waiting up to {timeout} seconds until URL is available " f"({url})") while not url_exists(url): now = time.monotonic() if now >= timeout_at: raise RayWheelsTimeoutError( f"Time out when waiting for URL to be available: {url}" ) if now >= next_status: logger.info( f"... still waiting for URL {url} " f"({int(now - start_time)} seconds) ..." ) next_status += status_time # Sleep `check_time` sec before next check. time.sleep(check_time) logger.info(f"URL is now available: {url}") return url
def find_ray_wheels_url(ray_wheels: Optional[str] = None) -> str: if not ray_wheels: # If no wheels are specified, default to BUILDKITE_COMMIT commit = os.environ.get("BUILDKITE_COMMIT", None) if not commit: raise RayWheelsUnspecifiedError( "No Ray wheels specified. Pass `--ray-wheels` or set " "`BUILDKITE_COMMIT` environment variable. " "Hint: You can use `-ray-wheels master` to fetch " "the latest available master wheels.") branch = os.environ.get("BUILDKITE_BRANCH", DEFAULT_BRANCH) repo_url = os.environ.get("BUILDKITE_REPO", DEFAULT_REPO) if not re.match(r"\b([a-f0-9]{40})\b", commit): # commit is symbolic, like HEAD latest_commits = get_latest_commits(repo_url, branch, ref=commit) commit = latest_commits[0] ray_version = get_ray_version(repo_url, commit) set_test_env_var("RAY_COMMIT", commit) set_test_env_var("RAY_BRANCH", branch) set_test_env_var("RAY_VERSION", ray_version) return get_ray_wheels_url(repo_url, branch, commit, ray_version) # If this is a URL, return if ray_wheels.startswith("https://") or ray_wheels.startswith("http://"): return ray_wheels # Else, this is either a commit hash, a branch name, or a combination # with a repo, e.g. ray-project:master or ray-project:<commit> if ":" in ray_wheels: # Repo is specified owner_or_url, commit_or_branch = ray_wheels.split(":") else: # Repo is not specified, use ray-project instead owner_or_url = DEFAULT_GIT_OWNER commit_or_branch = ray_wheels # Construct repo URL for cloning if "https://" in owner_or_url: # Already is a repo URL repo_url = owner_or_url else: repo_url = REPO_URL_TPL.format(owner=owner_or_url, package=DEFAULT_GIT_PACKAGE) # Todo: This is not ideal as branches that mimic a SHA1 hash # will also match this. if not re.match(r"\b([a-f0-9]{40})\b", commit_or_branch): # This is a branch branch = commit_or_branch latest_commits = get_latest_commits(repo_url, branch) # Let's assume the ray version is constant over these commits # (otherwise just move it into the for loop) ray_version = get_ray_version(repo_url, latest_commits[0]) for commit in latest_commits: wheels_url = get_ray_wheels_url(repo_url, branch, commit, ray_version) if url_exists(wheels_url): set_test_env_var("RAY_COMMIT", commit) return wheels_url raise RayWheelsNotFoundError( f"Couldn't find latest available wheels for repo " f"{repo_url}, branch {branch} (version {ray_version}). " f"Try again later or check Buildkite logs if wheel builds " f"failed.") # Else, this is a commit commit = commit_or_branch ray_version = get_ray_version(repo_url, commit) branch = os.environ.get("BUILDKITE_BRANCH", DEFAULT_BRANCH) wheels_url = get_ray_wheels_url(repo_url, branch, commit, ray_version) set_test_env_var("RAY_COMMIT", commit) set_test_env_var("RAY_BRANCH", branch) set_test_env_var("RAY_VERSION", ray_version) return wheels_url