def upload_from_file(self, file_obj, rewind=False, size=None, content_type=None, num_retries=6, client=None): """Upload the contents of this blob from a file-like object. The content type of the upload will either be - The value passed in to the function (if any) - The value stored on the current blob - The default value of 'application/octet-stream' .. note:: The effect of uploading to an existing blob depends on the "versioning" and "lifecycle" policies defined on the blob's bucket. In the absence of those policies, upload will overwrite any existing contents. See the `object versioning <https://cloud.google.com/storage/docs/object-versioning>`_ and `lifecycle <https://cloud.google.com/storage/docs/lifecycle>`_ API documents for details. Uploading a file with a `customer-supplied`_ encryption key:: >>> from google.cloud import storage >>> from google.cloud.storage import Blob >>> client = storage.Client(project='my-project') >>> bucket = client.get_bucket('my-bucket') >>> encryption_key = 'aa426195405adee2c8081bb9e7e74b19' >>> blob = Blob('secure-data', bucket, ... encryption_key=encryption_key) >>> with open('my-file', 'rb') as my_file: >>> blob.upload_from_file(my_file) The ``encryption_key`` should be a str or bytes with a length of at least 32. .. _customer-supplied: https://cloud.google.com/storage/docs/\ encryption#customer-supplied :type file_obj: file :param file_obj: A file handle open for reading. :type rewind: bool :param rewind: If True, seek to the beginning of the file handle before writing the file to Cloud Storage. :type size: int :param size: The number of bytes to read from the file handle. If not provided, we'll try to guess the size using :func:`os.fstat`. (If the file handle is not from the filesystem this won't be possible.) :type content_type: str :param content_type: Optional type of content being uploaded. :type num_retries: int :param num_retries: Number of upload retries. Defaults to 6. :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` :param client: Optional. The client to use. If not passed, falls back to the ``client`` stored on the blob's bucket. :raises: :class:`ValueError` if size is not passed in and can not be determined; :class:`google.cloud.exceptions.GoogleCloudError` if the upload response returns an error status. """ client = self._require_client(client) # Use ``_base_connection`` rather ``_connection`` since the current # connection may be a batch. A batch wraps a client's connection, # but does not store the ``http`` object. The rest (API_BASE_URL and # build_api_url) are also defined on the Batch class, but we just # use the wrapped connection since it has all three (http, # API_BASE_URL and build_api_url). connection = client._base_connection content_type = (content_type or self._properties.get('contentType') or 'application/octet-stream') # Rewind the file if desired. if rewind: file_obj.seek(0, os.SEEK_SET) # Get the basic stats about the file. total_bytes = size if total_bytes is None: if hasattr(file_obj, 'fileno'): try: total_bytes = os.fstat(file_obj.fileno()).st_size except (OSError, UnsupportedOperation): pass # Assuming fd is not an actual file (maybe socket). headers = { 'Accept': 'application/json', 'Accept-Encoding': 'gzip, deflate', 'User-Agent': connection.USER_AGENT, } headers.update(_get_encryption_headers(self._encryption_key)) upload = Upload(file_obj, content_type, total_bytes, auto_transfer=False) if self.chunk_size is not None: upload.chunksize = self.chunk_size if total_bytes is None: upload.strategy = RESUMABLE_UPLOAD elif total_bytes is None: raise ValueError('total bytes could not be determined. Please ' 'pass an explicit size, or supply a chunk size ' 'for a streaming transfer.') url_builder = _UrlBuilder(bucket_name=self.bucket.name, object_name=self.name) upload_config = _UploadConfig() # Temporary URL, until we know simple vs. resumable. base_url = connection.API_BASE_URL + '/upload' upload_url = connection.build_api_url(api_base_url=base_url, path=self.bucket.path + '/o') # Use apitools 'Upload' facility. request = Request(upload_url, 'POST', headers) upload.configure_request(upload_config, request, url_builder) query_params = url_builder.query_params base_url = connection.API_BASE_URL + '/upload' request.url = connection.build_api_url(api_base_url=base_url, path=self.bucket.path + '/o', query_params=query_params) upload.initialize_upload(request, connection.http) if upload.strategy == RESUMABLE_UPLOAD: http_response = upload.stream_file(use_chunks=True) else: http_response = make_api_request(connection.http, request, retries=num_retries) self._check_response_error(request, http_response) response_content = http_response.content if not isinstance(response_content, six.string_types): # pragma: NO COVER Python3 response_content = response_content.decode('utf-8') self._set_properties(json.loads(response_content))
def _create_upload(self, client, file_obj=None, size=None, content_type=None, chunk_size=None, strategy=None, extra_headers=None): """Helper for upload methods. Creates a :class:`google.cloud.core.streaming.Upload` object to handle the details of uploading a file to Cloud Storage. :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` :param client: Optional. The client to use. If not passed, falls back to the ``client`` stored on the blob's bucket. :type file_obj: file :param file_obj: A file handle open for reading. :type size: int :param size: The size of the upload, in bytes. :type content_type: str :param content_type: Optional type of content being uploaded. :type chunk_size: int :param chunk_size: The size of each chunk when doing resumable and media uploads. :type strategy: str :param strategy: Either :attr:`google.cloud.core.streaming.transfer.SIMPLE_UPLOAD` or :attr:`google.cloud.core.streaming.transfer.RESUMABLE_UPLOAD`. :type extra_headers: dict :param extra_headers: Additional headers to be sent with the upload initiation request. :rtype: Tuple[google.cloud.core.streaming.Upload, google.cloud.core.streaming.Request, google.cloud.core.streaming.Response] :returns: The Upload object, the upload HTTP request, and the upload initiation response. """ client = self._require_client(client) # Use ``_base_connection`` rather ``_connection`` since the current # connection may be a batch. A batch wraps a client's connection, # but does not store the ``http`` object. The rest (API_BASE_URL and # build_api_url) are also defined on the Batch class, but we just # use the wrapped connection since it has all three (http, # API_BASE_URL and build_api_url). connection = client._base_connection content_type = (content_type or self._properties.get('contentType') or 'application/octet-stream') headers = { 'Accept': 'application/json', 'Accept-Encoding': 'gzip, deflate', 'User-Agent': connection.USER_AGENT, } if extra_headers: headers.update(extra_headers) headers.update(_get_encryption_headers(self._encryption_key)) # Use apitools' Upload functionality upload = Upload(file_obj, content_type, total_size=size, auto_transfer=False) if chunk_size is not None: upload.chunksize = chunk_size if strategy is not None: upload.strategy = RESUMABLE_UPLOAD url_builder = _UrlBuilder(bucket_name=self.bucket.name, object_name=self.name) upload_config = _UploadConfig() # Temporary URL until strategy is determined. base_url = connection.API_BASE_URL + '/upload' upload_url = connection.build_api_url(api_base_url=base_url, path=self.bucket.path + '/o') # Configure the upload request parameters. request = Request(upload_url, 'POST', headers) if self._properties: headers['content-type'] = 'application/json' request.body = json.dumps(self._properties) upload.configure_request(upload_config, request, url_builder) # Configure final URL query_params = url_builder.query_params base_url = connection.API_BASE_URL + '/upload' request.url = connection.build_api_url(api_base_url=base_url, path=self.bucket.path + '/o', query_params=query_params) # Start the upload session response = upload.initialize_upload(request, connection.http) return upload, request, response
def upload_from_file(self, file_obj, rewind=False, size=None, encryption_key=None, content_type=None, num_retries=6, client=None): """Upload the contents of this blob from a file-like object. The content type of the upload will either be - The value passed in to the function (if any) - The value stored on the current blob - The default value of 'application/octet-stream' .. note:: The effect of uploading to an existing blob depends on the "versioning" and "lifecycle" policies defined on the blob's bucket. In the absence of those policies, upload will overwrite any existing contents. See the `object versioning <https://cloud.google.com/storage/docs/object-versioning>`_ and `lifecycle <https://cloud.google.com/storage/docs/lifecycle>`_ API documents for details. Uploading a file with a `customer-supplied`_ encryption key:: >>> from google.cloud import storage >>> from google.cloud.storage import Blob >>> client = storage.Client(project='my-project') >>> bucket = client.get_bucket('my-bucket') >>> encryption_key = 'aa426195405adee2c8081bb9e7e74b19' >>> blob = Blob('secure-data', bucket) >>> with open('my-file', 'rb') as my_file: >>> blob.upload_from_file(my_file, ... encryption_key=encryption_key) The ``encryption_key`` should be a str or bytes with a length of at least 32. .. _customer-supplied: https://cloud.google.com/storage/docs/\ encryption#customer-supplied :type file_obj: file :param file_obj: A file handle open for reading. :type rewind: boolean :param rewind: If True, seek to the beginning of the file handle before writing the file to Cloud Storage. :type size: int :param size: The number of bytes to read from the file handle. If not provided, we'll try to guess the size using :func:`os.fstat`. (If the file handle is not from the filesystem this won't be possible.) :type encryption_key: str or bytes :param encryption_key: Optional 32 byte encryption key for customer-supplied encryption. :type content_type: string or ``NoneType`` :param content_type: Optional type of content being uploaded. :type num_retries: integer :param num_retries: Number of upload retries. Defaults to 6. :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` :param client: Optional. The client to use. If not passed, falls back to the ``client`` stored on the blob's bucket. :raises: :class:`ValueError` if size is not passed in and can not be determined; :class:`google.cloud.exceptions.GoogleCloudError` if the upload response returns an error status. """ client = self._require_client(client) # Use the private ``_connection`` rather than the public # ``.connection``, since the public connection may be a batch. A # batch wraps a client's connection, but does not store the `http` # object. The rest (API_BASE_URL and build_api_url) are also defined # on the Batch class, but we just use the wrapped connection since # it has all three (http, API_BASE_URL and build_api_url). connection = client._connection content_type = (content_type or self._properties.get('contentType') or 'application/octet-stream') # Rewind the file if desired. if rewind: file_obj.seek(0, os.SEEK_SET) # Get the basic stats about the file. total_bytes = size if total_bytes is None: if hasattr(file_obj, 'fileno'): try: total_bytes = os.fstat(file_obj.fileno()).st_size except (OSError, UnsupportedOperation): pass # Assuming fd is not an actual file (maybe socket). headers = { 'Accept': 'application/json', 'Accept-Encoding': 'gzip, deflate', 'User-Agent': connection.USER_AGENT, } if encryption_key: _set_encryption_headers(encryption_key, headers) upload = Upload(file_obj, content_type, total_bytes, auto_transfer=False) if self.chunk_size is not None: upload.chunksize = self.chunk_size if total_bytes is None: upload.strategy = RESUMABLE_UPLOAD elif total_bytes is None: raise ValueError('total bytes could not be determined. Please ' 'pass an explicit size, or supply a chunk size ' 'for a streaming transfer.') url_builder = _UrlBuilder(bucket_name=self.bucket.name, object_name=self.name) upload_config = _UploadConfig() # Temporary URL, until we know simple vs. resumable. base_url = connection.API_BASE_URL + '/upload' upload_url = connection.build_api_url(api_base_url=base_url, path=self.bucket.path + '/o') # Use apitools 'Upload' facility. request = Request(upload_url, 'POST', headers) upload.configure_request(upload_config, request, url_builder) query_params = url_builder.query_params base_url = connection.API_BASE_URL + '/upload' request.url = connection.build_api_url(api_base_url=base_url, path=self.bucket.path + '/o', query_params=query_params) upload.initialize_upload(request, connection.http) if upload.strategy == RESUMABLE_UPLOAD: http_response = upload.stream_file(use_chunks=True) else: http_response = make_api_request(connection.http, request, retries=num_retries) self._check_response_error(request, http_response) response_content = http_response.content if not isinstance(response_content, six.string_types): # pragma: NO COVER Python3 response_content = response_content.decode('utf-8') self._set_properties(json.loads(response_content))