def put_block_list(sas_url, block_list, headers): """ Performs an Azure `Put Block List` operation (https://docs.microsoft.com/en-us/rest/api/storageservices/put-block-list) :param sas_url: A shared access signature URL referring to the Azure Block Blob to which the specified data should be staged. :param block_list: A list of uncommitted base64-encoded string block IDs to commit. For more information, see https://docs.microsoft.com/en-us/rest/api/storageservices/put-block-list. :param headers: Headers to include in the Put Block request body. """ request_url = _append_query_parameters(sas_url, {"comp": "blocklist"}) data = _build_block_list_xml(block_list) request_headers = {} for name, value in headers.items(): if _is_valid_put_block_list_header(name): request_headers[name] = value else: _logger.debug( "Removed unsupported '%s' header for Put Block List operation", name) with rest_utils.cloud_storage_http_request( "put", request_url, data, headers=request_headers) as response: response.raise_for_status()
def put_block(sas_url, block_id, data, headers): """ Performs an Azure `Put Block` operation (https://docs.microsoft.com/en-us/rest/api/storageservices/put-block) :param sas_url: A shared access signature URL referring to the Azure Block Blob to which the specified data should be staged. :param block_id: A base64-encoded string identifying the block. :param data: Data to include in the Put Block request body. :param headers: Additional headers to include in the Put Block request body (the `x-ms-blob-type` header is always included automatically). """ request_url = _append_query_parameters(sas_url, { "comp": "block", "blockid": block_id }) request_headers = deepcopy(_PUT_BLOCK_HEADERS) for name, value in headers.items(): if _is_valid_put_block_header(name): request_headers[name] = value else: _logger.debug( "Removed unsupported '%s' header for Put Block operation", name) with rest_utils.cloud_storage_http_request( "put", request_url, data, headers=request_headers) as response: response.raise_for_status()
def _signed_url_upload_file(self, credentials, local_file): try: headers = self._extract_headers_from_credentials(credentials.headers) signed_write_uri = credentials.signed_uri # Putting an empty file in a request by reading file bytes gives 501 error. if os.stat(local_file).st_size == 0: with rest_utils.cloud_storage_http_request( "put", signed_write_uri, data="", headers=headers ) as response: response.raise_for_status() else: with open(local_file, "rb") as file: with rest_utils.cloud_storage_http_request( "put", signed_write_uri, data=file, headers=headers ) as response: response.raise_for_status() except Exception as err: raise MlflowException(err)
def download_file_using_http_uri(http_uri, download_path, chunk_size=100000000): """ Downloads a file specified using the `http_uri` to a local `download_path`. This function uses a `chunk_size` to ensure an OOM error is not raised a large file is downloaded. Note : This function is meant to download files using presigned urls from various cloud providers. """ with cloud_storage_http_request("get", http_uri, stream=True) as response: augmented_raise_for_status(response) with open(download_path, "wb") as output_file: for chunk in response.iter_content(chunk_size=chunk_size): if not chunk: break output_file.write(chunk)