def read_bytes_from_url(url, gcs_fs):
"""Reads the contents of a URL and returns them as a bytes object."""
if is_file_url(url):
path = path_from_url(url)
return path.read_bytes()
elif is_gcs_url(url):
return gcs_fs.cat_file(url)
else:
raise AssertionError(f"Unexpected scheme in URL: {url}")
def read_bytes_from_url(url):
"""Reads the contents of a URL and returns them as a bytes object."""
if is_file_url(url):
path = path_from_url(url)
return path.read_bytes()
elif is_gcs_url(url):
gcs_client = get_gcs_client_without_warnings()
bucket_name, object_name = bucket_and_object_names_from_gs_url(url)
bucket = gcs_client.get_bucket(bucket_name)
blob = bucket.get_blob(object_name)
return blob.download_as_string()
else:
raise AssertionError(f"Unexpected scheme in URL: {url}")
def put(self, str_path, url, recursive):
# We only use put for dirs in Bionic with recursive=True.
assert recursive
path = path_from_url(str_path)
assert path.is_dir()
if url.endswith("/"):
url = url[:-1]
for file_path in path.glob("**/*"):
if not file_path.is_file():
continue
file_path_str = str(file_path)
sub_path_str = file_path_str.replace(str_path, "")
self.put_file(file_path_str, url + sub_path_str)