def test_move_files(storage): from_bucket = mock.create_autospec(Bucket) from_bucket.name = 'FROM_BUCKET' to_bucket = mock.create_autospec(Bucket) to_bucket.name = 'TO_BUCKET' blob1 = Blob('doc1.txt', bucket=from_bucket) blob2 = Blob('doc2.txt', bucket=from_bucket) from_bucket.list_blobs.return_value = [blob1, blob2] prefix = 'vibe-messages-final' storage.client.move_files(prefix, from_bucket, to_bucket) from_bucket.copy_blob.assert_any_call(blob1, to_bucket) from_bucket.copy_blob.assert_any_call(blob2, to_bucket) from_bucket.list_blobs.assert_called_once_with(prefix=prefix)
def save(self, name, fobj, max_length=None, blob_object=None): if not blob_object: blob = Blob(name, self.bucket) else: blob = blob_object # force the current file to be at file location 0, to # because that's what google wants # determine the current file's mimetype based on the name content_type = self._determine_content_type(name) fobj.seek(0) if self._is_file_empty(fobj): logging.warning( "Stopping the upload of an empty file: {}".format(name)) return name # set a max-age of 5 if we're uploading to content/databases if self.is_database_file(name): blob.cache_control = 'private, max-age={}, no-transform'.format( CONTENT_DATABASES_MAX_AGE) blob.upload_from_file( fobj, content_type=content_type, ) return name
def _make_one(*args, **kw): from google.cloud.storage.blob import Blob properties = kw.pop('properties', {}) blob = Blob(*args, **kw) blob._properties.update(properties) return blob
def get_blob(self, blob_name, client=None): """Get a blob object by name. This will return None if the blob doesn't exist: .. literalinclude:: storage_snippets.py :start-after: [START get_blob] :end-before: [END get_blob] :type blob_name: str :param blob_name: The name of the blob to retrieve. :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` :param client: Optional. The client to use. If not passed, falls back to the ``client`` stored on the current bucket. :rtype: :class:`google.cloud.storage.blob.Blob` or None :returns: The blob object if it exists, otherwise None. """ client = self._require_client(client) blob = Blob(bucket=self, name=blob_name) try: response = client._connection.api_request(method='GET', path=blob.path, _target_object=blob) # NOTE: We assume response.get('name') matches `blob_name`. blob._set_properties(response) # NOTE: This will not fail immediately in a batch. However, when # Batch.finish() is called, the resulting `NotFound` will be # raised. return blob except NotFound: return None
def blob(self, blob_name, chunk_size=None, encryption_key=None): """Factory constructor for blob object. .. note:: This will not make an HTTP request; it simply instantiates a blob object owned by this bucket. :type blob_name: str :param blob_name: The name of the blob to be instantiated. :type chunk_size: int :param chunk_size: The size of a chunk of data whenever iterating (1 MB). This must be a multiple of 256 KB per the API specification. :type encryption_key: bytes :param encryption_key: Optional 32 byte encryption key for customer-supplied encryption. :rtype: :class:`google.cloud.storage.blob.Blob` :returns: The blob object created. """ return Blob(name=blob_name, bucket=self, chunk_size=chunk_size, encryption_key=encryption_key)
def copy_blob(self, blob, destination_bucket, new_name=None, client=None): """Copy the given blob to the given bucket, optionally with a new name. :type blob: :class:`google.cloud.storage.blob.Blob` :param blob: The blob to be copied. :type destination_bucket: :class:`google.cloud.storage.bucket.Bucket` :param destination_bucket: The bucket into which the blob should be copied. :type new_name: string :param new_name: (optional) the new name for the copied file. :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` :param client: Optional. The client to use. If not passed, falls back to the ``client`` stored on the current bucket. :rtype: :class:`google.cloud.storage.blob.Blob` :returns: The new Blob. """ client = self._require_client(client) if new_name is None: new_name = blob.name new_blob = Blob(bucket=destination_bucket, name=new_name) api_path = blob.path + '/copyTo' + new_blob.path copy_result = client.connection.api_request(method='POST', path=api_path, _target_object=new_blob) new_blob._set_properties(copy_result) return new_blob
def download_blob_to_file(self, blob_or_uri, file_obj, start=None, end=None): """Download the contents of a blob object or blob URI into a file-like object. Args: blob_or_uri (Union[ \ :class:`~google.cloud.storage.blob.Blob`, \ str, \ ]): The blob resource to pass or URI to download. file_obj (file): A file handle to which to write the blob's data. start (int): Optional. The first byte in a range to be downloaded. end (int): Optional. The last byte in a range to be downloaded. Examples: Download a blob using using a blob resource. >>> from google.cloud import storage >>> client = storage.Client() >>> bucket = client.get_bucket('my-bucket-name') >>> blob = storage.Blob('path/to/blob', bucket) >>> with open('file-to-download-to') as file_obj: >>> client.download_blob_to_file(blob, file) # API request. Download a blob using a URI. >>> from google.cloud import storage >>> client = storage.Client() >>> with open('file-to-download-to') as file_obj: >>> client.download_blob_to_file( >>> 'gs://bucket_name/path/to/blob', file) """ try: blob_or_uri.download_to_file(file_obj, client=self, start=start, end=end) except AttributeError: scheme, netloc, path, query, frag = urlsplit(blob_or_uri) if scheme != "gs": raise ValueError("URI scheme must be gs") bucket = Bucket(self, name=netloc) blob_or_uri = Blob(path, bucket) blob_or_uri.download_to_file(file_obj, client=self, start=start, end=end)
def save(self, name, fobj, max_length=None): blob = Blob(name, self.bucket) # force the current file to be at file location 0, to # because that's what google wants fobj.seek(0) blob.upload_from_file(fobj) return name
def upload(file_path): if bucket is None: return name = file_path.replace(os.path.abspath(os.curdir) + '/', '') blob = Blob(name, bucket) blob.upload_from_filename(file_path)
def _upload(bucket: Bucket, src: str, prefix=None): dest = src.replace(os.path.abspath(os.curdir) + '/', '') if prefix: dest = os.path.join(prefix, dest) blob = Blob(dest, bucket) print(f'Uploading:{dest}') blob.upload_from_filename(src)
def get_blob(self, blob_name, client=None, encryption_key=None, **kwargs): """Get a blob object by name. This will return None if the blob doesn't exist: .. literalinclude:: snippets.py :start-after: [START get_blob] :end-before: [END get_blob] If :attr:`user_project` is set, bills the API request to that project. :type blob_name: str :param blob_name: The name of the blob to retrieve. :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` :param client: Optional. The client to use. If not passed, falls back to the ``client`` stored on the current bucket. :type encryption_key: bytes :param encryption_key: Optional 32 byte encryption key for customer-supplied encryption. See https://cloud.google.com/storage/docs/encryption#customer-supplied. :type kwargs: dict :param kwargs: Keyword arguments to pass to the :class:`~google.cloud.storage.blob.Blob` constructor. :rtype: :class:`google.cloud.storage.blob.Blob` or None :returns: The blob object if it exists, otherwise None. """ client = self._require_client(client) query_params = {} if self.user_project is not None: query_params['userProject'] = self.user_project blob = Blob(bucket=self, name=blob_name, encryption_key=encryption_key, **kwargs) try: headers = _get_encryption_headers(encryption_key) response = client._connection.api_request( method='GET', path=blob.path, query_params=query_params, headers=headers, _target_object=blob, ) # NOTE: We assume response.get('name') matches `blob_name`. blob._set_properties(response) # NOTE: This will not fail immediately in a batch. However, when # Batch.finish() is called, the resulting `NotFound` will be # raised. return blob except NotFound: return None
def __init__(self, name, mode, storage): self.name = name self._mode = mode self._storage = storage self.blob = storage.bucket.get_blob(name) if not self.blob and 'w' in mode: self.blob = Blob(self.name, storage.bucket) self._file = None self._is_dirty = False
def upload(file_path): if bucket is None: return # remove prefix /app name = file_path.replace(os.path.abspath(os.curdir) + '/', '') blob = Blob(name, bucket) logging.info('uploading {}'.format(name)) blob.upload_from_filename(file_path)
def download_from_bucket(name, bucket, filename): """Download a GCP blob object given a string filename and a bucket Keyword arguments: name -- full key name of the GCP blob object to download bucket -- bucket to download from filename -- string filename to download into """ return download_blob(blob=Blob(name=name, bucket=bucket), filename=filename)
def upload(file_path): if bucket is None: return name = file_path.replace(os.path.abspath(os.curdir) + '/', '') blob = Blob(name, bucket) try: blob.upload_from_filename(file_path, timeout=300) except Exception as e: print(str(e))
def test_has_file(storage): bucket = mock.create_autospec(Bucket) bucket.name = 'FROM_BUCKET' blob = Blob('doc1.txt', bucket=bucket) bucket.list_blobs.return_value = [blob] assert storage.client.has_file(bucket=bucket) empty_bucket = mock.create_autospec(Bucket) empty_bucket.name = 'FROM_BUCKET' has_file_cond = not CloudStorage.factory('PROJECT').has_file(bucket=empty_bucket) assert has_file_cond
def put_object(self, bucket_name, key, obj, **kwargs): try: bucket = self.client.get_bucket(bucket_name) blob = Blob(key, bucket) blob.upload_from_string(obj) except Exception as e: logging.exception( 'Exception in [GoogleCloudStorage.put_object] with bucket_name {} and key {}' .format(bucket_name, key)) raise e
def copy_blob(self, blob, destination_bucket, new_name=None, client=None, preserve_acl=True): """Copy the given blob to the given bucket, optionally with a new name. If :attr:`user_project` is set, bills the API request to that project. :type blob: :class:`google.cloud.storage.blob.Blob` :param blob: The blob to be copied. :type destination_bucket: :class:`google.cloud.storage.bucket.Bucket` :param destination_bucket: The bucket into which the blob should be copied. :type new_name: str :param new_name: (optional) the new name for the copied file. :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` :param client: Optional. The client to use. If not passed, falls back to the ``client`` stored on the current bucket. :type preserve_acl: bool :param preserve_acl: Optional. Copies ACL from old blob to new blob. Default: True. :rtype: :class:`google.cloud.storage.blob.Blob` :returns: The new Blob. """ client = self._require_client(client) query_params = {} if self.user_project is not None: query_params['userProject'] = self.user_project if new_name is None: new_name = blob.name new_blob = Blob(bucket=destination_bucket, name=new_name) api_path = blob.path + '/copyTo' + new_blob.path copy_result = client._connection.api_request( method='POST', path=api_path, query_params=query_params, _target_object=new_blob, ) if not preserve_acl: new_blob.acl.save(acl={}, client=client) new_blob._set_properties(copy_result) return new_blob
def _item_to_value(self, item): """Convert a JSON blob to the native object. :type item: dict :param item: An item to be converted to a blob. :rtype: :class:`.Blob` :returns: The next blob in the page. """ name = item.get('name') blob = Blob(name, bucket=self._parent.bucket) blob._set_properties(item) return blob
def get_items_from_response(self, response): """Yield :class:`.storage.blob.Blob` items from response. :type response: dict :param response: The JSON API response for a page of blobs. """ self._current_prefixes = tuple(response.get('prefixes', ())) self.prefixes.update(self._current_prefixes) for item in response.get('items', []): name = item.get('name') blob = Blob(name, bucket=self.bucket) blob._set_properties(item) yield blob
def save_file(self, file, filename, folder=None, randomize=False, extensions=None, acl=None, replace=False, headers=None): """ :param filename: local filename :param folder: relative path of sub-folder :param randomize: randomize the filename :param extensions: iterable of allowed extensions, if not default :param acl: ACL policy (if None then uses default) :returns: modified filename """ extensions = extensions or self.extensions if not self.filename_allowed(filename, extensions): raise FileNotAllowed() filename = utils.secure_filename(os.path.basename(filename)) if randomize: filename = utils.random_filename(filename) if folder: filename = folder + "/" + filename content_type, _ = mimetypes.guess_type(filename) content_type = content_type or 'application/octet-stream' blob = self.get_bucket().get_blob(filename) # If the file exist and we explicitely asked not to replace it: ignore it. if blob and not replace: return filename # If the file doesn't exist: create it. if not blob: blob = Blob(filename, self.get_bucket()) blob.cache_control = self.cache_control file.seek(0) acl = acl or self.acl blob.upload_from_file(file, rewind=True, content_type=content_type, predefined_acl=acl) return filename
def generate_signed_url(self, url, expiration_time=30): """ url: GS url "gs://xzy/test/test.pdf" expiration_time: validity of signed url by default 30 mins. """ u = urlparse.urlsplit(url) if u.scheme == 'gs': url = u.path[1:] a = Blob(url, self.__bucket) expiry = (datetime.datetime.now() + datetime.timedelta(minutes=expiration_time)).timetuple() expiry = int(time.mktime(expiry)) return a.generate_signed_url(expiry) else: return "Invalid GS url"
def test_list_blobs(*args, **kwargs): """ Rewrites library function to reads a custom list of paths vs real GCS. https://googleapis.dev/python/storage/latest/_modules/google/cloud/storage/client.html#Client.list_blobs """ bucket_or_name = args[0] prefix = kwargs['prefix'] candidate_path = f'{bucket_or_name}/{prefix}' config_paths = [] for c in config_hierarchy: if c.startswith(candidate_path): fn = '/'.join(c.split('/')[1:]) b = Blob(bucket='dummy', name=fn) config_paths.append(b) return iter(config_paths)
def save(self, name, fobj, max_length=None, blob_object=None): if not blob_object: blob = Blob(name, self.bucket) else: blob = blob_object buffer = None # set a max-age of 5 if we're uploading to content/databases if self.is_database_file(name): blob.cache_control = "private, max-age={}, no-transform".format( CONTENT_DATABASES_MAX_AGE) # Compress the database file so that users can save bandwith and download faster. buffer = BytesIO() compressed = GzipFile(fileobj=buffer, mode="w") compressed.write(fobj.read()) compressed.close() blob.content_encoding = "gzip" fobj = buffer # determine the current file's mimetype based on the name # import determine_content_type lazily in here, so we don't get into an infinite loop with circular dependencies from contentcuration.utils.storage_common import determine_content_type content_type = determine_content_type(name) # force the current file to be at file location 0, to # because that's what google wants fobj.seek(0) if self._is_file_empty(fobj): logging.warning( "Stopping the upload of an empty file: {}".format(name)) return name blob.upload_from_file( fobj, content_type=content_type, ) # Close StringIO object and discard memory buffer if created if buffer: buffer.close() return name
def save(self, name, fobj, max_length=None, blob_object=None): if not blob_object: blob = Blob(name, self.bucket) else: blob = blob_object # force the current file to be at file location 0, to # because that's what google wants # determine the current file's mimetype based on the name content_type = self._determine_content_type(name) fobj.seek(0) blob.upload_from_file( fobj, content_type=content_type, ) return name
def __init__(self, name, mode, bucket): """Create a GoogleCloudFile handler. Args: name: The name of the file within the bucket (string). mode: A string of the file mode to open with (string). bucket: The bucket to load and save to (Bucket). """ self.name = name self.mode = mode self._mimetype = mimetypes.guess_type(name)[0] self._is_dirty = False self._file = None self._blob = bucket.get_blob(name) if not self.blob and "w" in mode: self.blob = Blob(self.name, bucket) elif not self.blob and "r" in mode: message = "{} file was not found." message.format(name) raise NotFound(message=message) self.open(mode)
def _item_to_blob(iterator, item): """Convert a JSON blob to the native object. .. note:: This assumes that the ``bucket`` attribute has been added to the iterator after being created. :type iterator: :class:`~google.cloud.iterator.Iterator` :param iterator: The iterator that has retrieved the item. :type item: dict :param item: An item to be converted to a blob. :rtype: :class:`.Blob` :returns: The next blob in the page. """ name = item.get('name') blob = Blob(name, bucket=iterator.bucket) blob._set_properties(item) return blob
def get_blob(self, blob_name, client=None): """Get a blob object by name. This will return None if the blob doesn't exist:: >>> from google.cloud import storage >>> client = storage.Client() >>> bucket = client.get_bucket('my-bucket') >>> print(bucket.get_blob('/path/to/blob.txt')) <Blob: my-bucket, /path/to/blob.txt> >>> print(bucket.get_blob('/does-not-exist.txt')) None :type blob_name: str :param blob_name: The name of the blob to retrieve. :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` :param client: Optional. The client to use. If not passed, falls back to the ``client`` stored on the current bucket. :rtype: :class:`google.cloud.storage.blob.Blob` or None :returns: The blob object if it exists, otherwise None. """ client = self._require_client(client) blob = Blob(bucket=self, name=blob_name) try: response = client.connection.api_request(method='GET', path=blob.path, _target_object=blob) # NOTE: We assume response.get('name') matches `blob_name`. blob._set_properties(response) # NOTE: This will not fail immediately in a batch. However, when # Batch.finish() is called, the resulting `NotFound` will be # raised. return blob except NotFound: return None
def sign(duration: str, key_file: click.File, resource: str) -> None: """ Generate a signed URL that embeds authentication data so the URL can be used by someone who does not have a Google account. This tool exists to overcome a shortcoming of gsutil signurl that limits expiration to 7 days only. KEY_FILE should be a path to a JSON file containing service account private key. See gsutil signurl --help for details RESOURCE is a GCS location in the form <bucket>/<path> (don't add neither "gs://" nor "http://...") Example: gcs-signurl /tmp/creds.json /foo-bucket/bar-file.txt """ bucket_name, _, path = resource.lstrip("/").partition("/") creds = service_account.Credentials.from_service_account_file( key_file.name) till = datetime.now() + _DurationToTimeDelta(duration) # Ignoring potential warning about end user credentials. # We don't actually do any operations on the client, but # unfortunately the only public API in google-cloud-storage package # requires building client->bucket->blob message = "Your application has authenticated using end user credentials from Google Cloud SDK" with warnings.catch_warnings(): warnings.filterwarnings("ignore", message=message) client = Client() bucket = Bucket(client, bucket_name) blob = Blob(path, bucket) # Not passing version argument - to support compatibility with # google-cloud-storage<=1.14.0. They default to version 2 and hopefully # will not change it anytime soon. signed_url = blob.generate_signed_url(expiration=till, credentials=creds) click.echo(signed_url)
def save(self, name, fobj, max_length=None, blob_object=None): if not blob_object: blob = Blob(name, self.bucket) else: blob = blob_object # force the current file to be at file location 0, to # because that's what google wants # determine the current file's mimetype based on the name content_type = self._determine_content_type(name) fobj.seek(0) if self._is_file_empty(fobj): logging.warning( "Stopping the upload of an empty file: {}".format(name)) return name blob.upload_from_file( fobj, content_type=content_type, ) return name