def test_move_files(storage):
from_bucket = mock.create_autospec(Bucket)
from_bucket.name = 'FROM_BUCKET'
to_bucket = mock.create_autospec(Bucket)
to_bucket.name = 'TO_BUCKET'
blob1 = Blob('doc1.txt', bucket=from_bucket)
blob2 = Blob('doc2.txt', bucket=from_bucket)
from_bucket.list_blobs.return_value = [blob1, blob2]
prefix = 'vibe-messages-final'
storage.client.move_files(prefix, from_bucket, to_bucket)
from_bucket.copy_blob.assert_any_call(blob1, to_bucket)
from_bucket.copy_blob.assert_any_call(blob2, to_bucket)
from_bucket.list_blobs.assert_called_once_with(prefix=prefix)
def save(self, name, fobj, max_length=None, blob_object=None):
if not blob_object:
blob = Blob(name, self.bucket)
else:
blob = blob_object
# force the current file to be at file location 0, to
# because that's what google wants
# determine the current file's mimetype based on the name
content_type = self._determine_content_type(name)
fobj.seek(0)
if self._is_file_empty(fobj):
logging.warning(
"Stopping the upload of an empty file: {}".format(name))
return name
# set a max-age of 5 if we're uploading to content/databases
if self.is_database_file(name):
blob.cache_control = 'private, max-age={}, no-transform'.format(
CONTENT_DATABASES_MAX_AGE)
blob.upload_from_file(
fobj,
content_type=content_type,
)
return name
def _make_one(*args, **kw):
from google.cloud.storage.blob import Blob
properties = kw.pop('properties', {})
blob = Blob(*args, **kw)
blob._properties.update(properties)
return blob
def get_blob(self, blob_name, client=None):
"""Get a blob object by name.
This will return None if the blob doesn't exist:
.. literalinclude:: storage_snippets.py
:start-after: [START get_blob]
:end-before: [END get_blob]
:type blob_name: str
:param blob_name: The name of the blob to retrieve.
:type client: :class:`~google.cloud.storage.client.Client` or
``NoneType``
:param client: Optional. The client to use. If not passed, falls back
to the ``client`` stored on the current bucket.
:rtype: :class:`google.cloud.storage.blob.Blob` or None
:returns: The blob object if it exists, otherwise None.
"""
client = self._require_client(client)
blob = Blob(bucket=self, name=blob_name)
try:
response = client._connection.api_request(method='GET',
path=blob.path,
_target_object=blob)
# NOTE: We assume response.get('name') matches `blob_name`.
blob._set_properties(response)
# NOTE: This will not fail immediately in a batch. However, when
# Batch.finish() is called, the resulting `NotFound` will be
# raised.
return blob
except NotFound:
return None
def blob(self, blob_name, chunk_size=None, encryption_key=None):
"""Factory constructor for blob object.
.. note::
This will not make an HTTP request; it simply instantiates
a blob object owned by this bucket.
:type blob_name: str
:param blob_name: The name of the blob to be instantiated.
:type chunk_size: int
:param chunk_size: The size of a chunk of data whenever iterating
(1 MB). This must be a multiple of 256 KB per the
API specification.
:type encryption_key: bytes
:param encryption_key:
Optional 32 byte encryption key for customer-supplied encryption.
:rtype: :class:`google.cloud.storage.blob.Blob`
:returns: The blob object created.
"""
return Blob(name=blob_name,
bucket=self,
chunk_size=chunk_size,
encryption_key=encryption_key)
def copy_blob(self, blob, destination_bucket, new_name=None, client=None):
"""Copy the given blob to the given bucket, optionally with a new name.
:type blob: :class:`google.cloud.storage.blob.Blob`
:param blob: The blob to be copied.
:type destination_bucket: :class:`google.cloud.storage.bucket.Bucket`
:param destination_bucket: The bucket into which the blob should be
copied.
:type new_name: string
:param new_name: (optional) the new name for the copied file.
:type client: :class:`~google.cloud.storage.client.Client` or
``NoneType``
:param client: Optional. The client to use. If not passed, falls back
to the ``client`` stored on the current bucket.
:rtype: :class:`google.cloud.storage.blob.Blob`
:returns: The new Blob.
"""
client = self._require_client(client)
if new_name is None:
new_name = blob.name
new_blob = Blob(bucket=destination_bucket, name=new_name)
api_path = blob.path + '/copyTo' + new_blob.path
copy_result = client.connection.api_request(method='POST',
path=api_path,
_target_object=new_blob)
new_blob._set_properties(copy_result)
return new_blob
def download_blob_to_file(self,
blob_or_uri,
file_obj,
start=None,
end=None):
"""Download the contents of a blob object or blob URI into a file-like object.
Args:
blob_or_uri (Union[ \
:class:`~google.cloud.storage.blob.Blob`, \
str, \
]):
The blob resource to pass or URI to download.
file_obj (file):
A file handle to which to write the blob's data.
start (int):
Optional. The first byte in a range to be downloaded.
end (int):
Optional. The last byte in a range to be downloaded.
Examples:
Download a blob using using a blob resource.
>>> from google.cloud import storage
>>> client = storage.Client()
>>> bucket = client.get_bucket('my-bucket-name')
>>> blob = storage.Blob('path/to/blob', bucket)
>>> with open('file-to-download-to') as file_obj:
>>> client.download_blob_to_file(blob, file) # API request.
Download a blob using a URI.
>>> from google.cloud import storage
>>> client = storage.Client()
>>> with open('file-to-download-to') as file_obj:
>>> client.download_blob_to_file(
>>> 'gs://bucket_name/path/to/blob', file)
"""
try:
blob_or_uri.download_to_file(file_obj,
client=self,
start=start,
end=end)
except AttributeError:
scheme, netloc, path, query, frag = urlsplit(blob_or_uri)
if scheme != "gs":
raise ValueError("URI scheme must be gs")
bucket = Bucket(self, name=netloc)
blob_or_uri = Blob(path, bucket)
blob_or_uri.download_to_file(file_obj,
client=self,
start=start,
end=end)
def save(self, name, fobj, max_length=None):
blob = Blob(name, self.bucket)
# force the current file to be at file location 0, to
# because that's what google wants
fobj.seek(0)
blob.upload_from_file(fobj)
return name
def upload(file_path):
if bucket is None:
return
name = file_path.replace(os.path.abspath(os.curdir) + '/', '')
blob = Blob(name, bucket)
blob.upload_from_filename(file_path)
def _upload(bucket: Bucket, src: str, prefix=None):
dest = src.replace(os.path.abspath(os.curdir) + '/', '')
if prefix:
dest = os.path.join(prefix, dest)
blob = Blob(dest, bucket)
print(f'Uploading:{dest}')
blob.upload_from_filename(src)
def get_blob(self, blob_name, client=None, encryption_key=None, **kwargs):
"""Get a blob object by name.
This will return None if the blob doesn't exist:
.. literalinclude:: snippets.py
:start-after: [START get_blob]
:end-before: [END get_blob]
If :attr:`user_project` is set, bills the API request to that project.
:type blob_name: str
:param blob_name: The name of the blob to retrieve.
:type client: :class:`~google.cloud.storage.client.Client` or
``NoneType``
:param client: Optional. The client to use. If not passed, falls back
to the ``client`` stored on the current bucket.
:type encryption_key: bytes
:param encryption_key:
Optional 32 byte encryption key for customer-supplied encryption.
See
https://cloud.google.com/storage/docs/encryption#customer-supplied.
:type kwargs: dict
:param kwargs: Keyword arguments to pass to the
:class:`~google.cloud.storage.blob.Blob` constructor.
:rtype: :class:`google.cloud.storage.blob.Blob` or None
:returns: The blob object if it exists, otherwise None.
"""
client = self._require_client(client)
query_params = {}
if self.user_project is not None:
query_params['userProject'] = self.user_project
blob = Blob(bucket=self,
name=blob_name,
encryption_key=encryption_key,
**kwargs)
try:
headers = _get_encryption_headers(encryption_key)
response = client._connection.api_request(
method='GET',
path=blob.path,
query_params=query_params,
headers=headers,
_target_object=blob,
)
# NOTE: We assume response.get('name') matches `blob_name`.
blob._set_properties(response)
# NOTE: This will not fail immediately in a batch. However, when
# Batch.finish() is called, the resulting `NotFound` will be
# raised.
return blob
except NotFound:
return None
def __init__(self, name, mode, storage):
self.name = name
self._mode = mode
self._storage = storage
self.blob = storage.bucket.get_blob(name)
if not self.blob and 'w' in mode:
self.blob = Blob(self.name, storage.bucket)
self._file = None
self._is_dirty = False
def upload(file_path):
if bucket is None:
return
# remove prefix /app
name = file_path.replace(os.path.abspath(os.curdir) + '/', '')
blob = Blob(name, bucket)
logging.info('uploading {}'.format(name))
blob.upload_from_filename(file_path)
def download_from_bucket(name, bucket, filename):
"""Download a GCP blob object given a string filename and a bucket
Keyword arguments:
name -- full key name of the GCP blob object to download
bucket -- bucket to download from
filename -- string filename to download into
"""
return download_blob(blob=Blob(name=name, bucket=bucket),
filename=filename)
def upload(file_path):
if bucket is None:
return
name = file_path.replace(os.path.abspath(os.curdir) + '/', '')
blob = Blob(name, bucket)
try:
blob.upload_from_filename(file_path, timeout=300)
except Exception as e:
print(str(e))
def test_has_file(storage):
bucket = mock.create_autospec(Bucket)
bucket.name = 'FROM_BUCKET'
blob = Blob('doc1.txt', bucket=bucket)
bucket.list_blobs.return_value = [blob]
assert storage.client.has_file(bucket=bucket)
empty_bucket = mock.create_autospec(Bucket)
empty_bucket.name = 'FROM_BUCKET'
has_file_cond = not CloudStorage.factory('PROJECT').has_file(bucket=empty_bucket)
assert has_file_cond
def put_object(self, bucket_name, key, obj, **kwargs):
try:
bucket = self.client.get_bucket(bucket_name)
blob = Blob(key, bucket)
blob.upload_from_string(obj)
except Exception as e:
logging.exception(
'Exception in [GoogleCloudStorage.put_object] with bucket_name {} and key {}'
.format(bucket_name, key))
raise e
def copy_blob(self,
blob,
destination_bucket,
new_name=None,
client=None,
preserve_acl=True):
"""Copy the given blob to the given bucket, optionally with a new name.
If :attr:`user_project` is set, bills the API request to that project.
:type blob: :class:`google.cloud.storage.blob.Blob`
:param blob: The blob to be copied.
:type destination_bucket: :class:`google.cloud.storage.bucket.Bucket`
:param destination_bucket: The bucket into which the blob should be
copied.
:type new_name: str
:param new_name: (optional) the new name for the copied file.
:type client: :class:`~google.cloud.storage.client.Client` or
``NoneType``
:param client: Optional. The client to use. If not passed, falls back
to the ``client`` stored on the current bucket.
:type preserve_acl: bool
:param preserve_acl: Optional. Copies ACL from old blob to new blob.
Default: True.
:rtype: :class:`google.cloud.storage.blob.Blob`
:returns: The new Blob.
"""
client = self._require_client(client)
query_params = {}
if self.user_project is not None:
query_params['userProject'] = self.user_project
if new_name is None:
new_name = blob.name
new_blob = Blob(bucket=destination_bucket, name=new_name)
api_path = blob.path + '/copyTo' + new_blob.path
copy_result = client._connection.api_request(
method='POST',
path=api_path,
query_params=query_params,
_target_object=new_blob,
)
if not preserve_acl:
new_blob.acl.save(acl={}, client=client)
new_blob._set_properties(copy_result)
return new_blob
def _item_to_value(self, item):
"""Convert a JSON blob to the native object.
:type item: dict
:param item: An item to be converted to a blob.
:rtype: :class:`.Blob`
:returns: The next blob in the page.
"""
name = item.get('name')
blob = Blob(name, bucket=self._parent.bucket)
blob._set_properties(item)
return blob
def get_items_from_response(self, response):
"""Yield :class:`.storage.blob.Blob` items from response.
:type response: dict
:param response: The JSON API response for a page of blobs.
"""
self._current_prefixes = tuple(response.get('prefixes', ()))
self.prefixes.update(self._current_prefixes)
for item in response.get('items', []):
name = item.get('name')
blob = Blob(name, bucket=self.bucket)
blob._set_properties(item)
yield blob
def save_file(self,
file,
filename,
folder=None,
randomize=False,
extensions=None,
acl=None,
replace=False,
headers=None):
"""
:param filename: local filename
:param folder: relative path of sub-folder
:param randomize: randomize the filename
:param extensions: iterable of allowed extensions, if not default
:param acl: ACL policy (if None then uses default)
:returns: modified filename
"""
extensions = extensions or self.extensions
if not self.filename_allowed(filename, extensions):
raise FileNotAllowed()
filename = utils.secure_filename(os.path.basename(filename))
if randomize:
filename = utils.random_filename(filename)
if folder:
filename = folder + "/" + filename
content_type, _ = mimetypes.guess_type(filename)
content_type = content_type or 'application/octet-stream'
blob = self.get_bucket().get_blob(filename)
# If the file exist and we explicitely asked not to replace it: ignore it.
if blob and not replace:
return filename
# If the file doesn't exist: create it.
if not blob:
blob = Blob(filename, self.get_bucket())
blob.cache_control = self.cache_control
file.seek(0)
acl = acl or self.acl
blob.upload_from_file(file,
rewind=True,
content_type=content_type,
predefined_acl=acl)
return filename
def generate_signed_url(self, url, expiration_time=30):
"""
url: GS url "gs://xzy/test/test.pdf"
expiration_time: validity of signed url by default 30 mins.
"""
u = urlparse.urlsplit(url)
if u.scheme == 'gs':
url = u.path[1:]
a = Blob(url, self.__bucket)
expiry = (datetime.datetime.now() +
datetime.timedelta(minutes=expiration_time)).timetuple()
expiry = int(time.mktime(expiry))
return a.generate_signed_url(expiry)
else:
return "Invalid GS url"
def test_list_blobs(*args, **kwargs):
""" Rewrites library function to reads a custom list of paths vs real GCS.
https://googleapis.dev/python/storage/latest/_modules/google/cloud/storage/client.html#Client.list_blobs
"""
bucket_or_name = args[0]
prefix = kwargs['prefix']
candidate_path = f'{bucket_or_name}/{prefix}'
config_paths = []
for c in config_hierarchy:
if c.startswith(candidate_path):
fn = '/'.join(c.split('/')[1:])
b = Blob(bucket='dummy', name=fn)
config_paths.append(b)
return iter(config_paths)
def save(self, name, fobj, max_length=None, blob_object=None):
if not blob_object:
blob = Blob(name, self.bucket)
else:
blob = blob_object
buffer = None
# set a max-age of 5 if we're uploading to content/databases
if self.is_database_file(name):
blob.cache_control = "private, max-age={}, no-transform".format(
CONTENT_DATABASES_MAX_AGE)
# Compress the database file so that users can save bandwith and download faster.
buffer = BytesIO()
compressed = GzipFile(fileobj=buffer, mode="w")
compressed.write(fobj.read())
compressed.close()
blob.content_encoding = "gzip"
fobj = buffer
# determine the current file's mimetype based on the name
# import determine_content_type lazily in here, so we don't get into an infinite loop with circular dependencies
from contentcuration.utils.storage_common import determine_content_type
content_type = determine_content_type(name)
# force the current file to be at file location 0, to
# because that's what google wants
fobj.seek(0)
if self._is_file_empty(fobj):
logging.warning(
"Stopping the upload of an empty file: {}".format(name))
return name
blob.upload_from_file(
fobj,
content_type=content_type,
)
# Close StringIO object and discard memory buffer if created
if buffer:
buffer.close()
return name
def save(self, name, fobj, max_length=None, blob_object=None):
if not blob_object:
blob = Blob(name, self.bucket)
else:
blob = blob_object
# force the current file to be at file location 0, to
# because that's what google wants
# determine the current file's mimetype based on the name
content_type = self._determine_content_type(name)
fobj.seek(0)
blob.upload_from_file(
fobj,
content_type=content_type,
)
return name
def __init__(self, name, mode, bucket):
"""Create a GoogleCloudFile handler.
Args:
name: The name of the file within the bucket (string).
mode: A string of the file mode to open with (string).
bucket: The bucket to load and save to (Bucket).
"""
self.name = name
self.mode = mode
self._mimetype = mimetypes.guess_type(name)[0]
self._is_dirty = False
self._file = None
self._blob = bucket.get_blob(name)
if not self.blob and "w" in mode:
self.blob = Blob(self.name, bucket)
elif not self.blob and "r" in mode:
message = "{} file was not found."
message.format(name)
raise NotFound(message=message)
self.open(mode)
def _item_to_blob(iterator, item):
"""Convert a JSON blob to the native object.
.. note::
This assumes that the ``bucket`` attribute has been
added to the iterator after being created.
:type iterator: :class:`~google.cloud.iterator.Iterator`
:param iterator: The iterator that has retrieved the item.
:type item: dict
:param item: An item to be converted to a blob.
:rtype: :class:`.Blob`
:returns: The next blob in the page.
"""
name = item.get('name')
blob = Blob(name, bucket=iterator.bucket)
blob._set_properties(item)
return blob
def get_blob(self, blob_name, client=None):
"""Get a blob object by name.
This will return None if the blob doesn't exist::
>>> from google.cloud import storage
>>> client = storage.Client()
>>> bucket = client.get_bucket('my-bucket')
>>> print(bucket.get_blob('/path/to/blob.txt'))
<Blob: my-bucket, /path/to/blob.txt>
>>> print(bucket.get_blob('/does-not-exist.txt'))
None
:type blob_name: str
:param blob_name: The name of the blob to retrieve.
:type client: :class:`~google.cloud.storage.client.Client` or
``NoneType``
:param client: Optional. The client to use. If not passed, falls back
to the ``client`` stored on the current bucket.
:rtype: :class:`google.cloud.storage.blob.Blob` or None
:returns: The blob object if it exists, otherwise None.
"""
client = self._require_client(client)
blob = Blob(bucket=self, name=blob_name)
try:
response = client.connection.api_request(method='GET',
path=blob.path,
_target_object=blob)
# NOTE: We assume response.get('name') matches `blob_name`.
blob._set_properties(response)
# NOTE: This will not fail immediately in a batch. However, when
# Batch.finish() is called, the resulting `NotFound` will be
# raised.
return blob
except NotFound:
return None
def sign(duration: str, key_file: click.File, resource: str) -> None:
"""
Generate a signed URL that embeds authentication data
so the URL can be used by someone who does not have a Google account.
This tool exists to overcome a shortcoming of gsutil signurl that limits
expiration to 7 days only.
KEY_FILE should be a path to a JSON file containing service account private key.
See gsutil signurl --help for details
RESOURCE is a GCS location in the form <bucket>/<path>
(don't add neither "gs://" nor "http://...")
Example: gcs-signurl /tmp/creds.json /foo-bucket/bar-file.txt
"""
bucket_name, _, path = resource.lstrip("/").partition("/")
creds = service_account.Credentials.from_service_account_file(
key_file.name)
till = datetime.now() + _DurationToTimeDelta(duration)
# Ignoring potential warning about end user credentials.
# We don't actually do any operations on the client, but
# unfortunately the only public API in google-cloud-storage package
# requires building client->bucket->blob
message = "Your application has authenticated using end user credentials from Google Cloud SDK"
with warnings.catch_warnings():
warnings.filterwarnings("ignore", message=message)
client = Client()
bucket = Bucket(client, bucket_name)
blob = Blob(path, bucket)
# Not passing version argument - to support compatibility with
# google-cloud-storage<=1.14.0. They default to version 2 and hopefully
# will not change it anytime soon.
signed_url = blob.generate_signed_url(expiration=till, credentials=creds)
click.echo(signed_url)
def save(self, name, fobj, max_length=None, blob_object=None):
if not blob_object:
blob = Blob(name, self.bucket)
else:
blob = blob_object
# force the current file to be at file location 0, to
# because that's what google wants
# determine the current file's mimetype based on the name
content_type = self._determine_content_type(name)
fobj.seek(0)
if self._is_file_empty(fobj):
logging.warning(
"Stopping the upload of an empty file: {}".format(name))
return name
blob.upload_from_file(
fobj,
content_type=content_type,
)
return name