def put_block_list(sas_url, block_list, headers):
"""
Performs an Azure `Put Block List` operation
(https://docs.microsoft.com/en-us/rest/api/storageservices/put-block-list)
:param sas_url: A shared access signature URL referring to the Azure Block Blob
to which the specified data should be staged.
:param block_list: A list of uncommitted base64-encoded string block IDs to commit. For
more information, see
https://docs.microsoft.com/en-us/rest/api/storageservices/put-block-list.
:param headers: Headers to include in the Put Block request body.
"""
request_url = _append_query_parameters(sas_url, {"comp": "blocklist"})
data = _build_block_list_xml(block_list)
request_headers = {}
for name, value in headers.items():
if _is_valid_put_block_list_header(name):
request_headers[name] = value
else:
_logger.debug(
"Removed unsupported '%s' header for Put Block List operation",
name)
with rest_utils.cloud_storage_http_request(
"put", request_url, data, headers=request_headers) as response:
response.raise_for_status()
def put_block(sas_url, block_id, data, headers):
"""
Performs an Azure `Put Block` operation
(https://docs.microsoft.com/en-us/rest/api/storageservices/put-block)
:param sas_url: A shared access signature URL referring to the Azure Block Blob
to which the specified data should be staged.
:param block_id: A base64-encoded string identifying the block.
:param data: Data to include in the Put Block request body.
:param headers: Additional headers to include in the Put Block request body
(the `x-ms-blob-type` header is always included automatically).
"""
request_url = _append_query_parameters(sas_url, {
"comp": "block",
"blockid": block_id
})
request_headers = deepcopy(_PUT_BLOCK_HEADERS)
for name, value in headers.items():
if _is_valid_put_block_header(name):
request_headers[name] = value
else:
_logger.debug(
"Removed unsupported '%s' header for Put Block operation",
name)
with rest_utils.cloud_storage_http_request(
"put", request_url, data, headers=request_headers) as response:
response.raise_for_status()
def _signed_url_upload_file(self, credentials, local_file):
try:
headers = self._extract_headers_from_credentials(credentials.headers)
signed_write_uri = credentials.signed_uri
# Putting an empty file in a request by reading file bytes gives 501 error.
if os.stat(local_file).st_size == 0:
with rest_utils.cloud_storage_http_request(
"put", signed_write_uri, data="", headers=headers
) as response:
response.raise_for_status()
else:
with open(local_file, "rb") as file:
with rest_utils.cloud_storage_http_request(
"put", signed_write_uri, data=file, headers=headers
) as response:
response.raise_for_status()
except Exception as err:
raise MlflowException(err)
def download_file_using_http_uri(http_uri, download_path, chunk_size=100000000):
"""
Downloads a file specified using the `http_uri` to a local `download_path`. This function
uses a `chunk_size` to ensure an OOM error is not raised a large file is downloaded.
Note : This function is meant to download files using presigned urls from various cloud
providers.
"""
with cloud_storage_http_request("get", http_uri, stream=True) as response:
augmented_raise_for_status(response)
with open(download_path, "wb") as output_file:
for chunk in response.iter_content(chunk_size=chunk_size):
if not chunk:
break
output_file.write(chunk)