def save(self, name, fobj, max_length=None, blob_object=None):
if not blob_object:
blob = Blob(name, self.bucket)
else:
blob = blob_object
# force the current file to be at file location 0, to
# because that's what google wants
# determine the current file's mimetype based on the name
content_type = self._determine_content_type(name)
fobj.seek(0)
if self._is_file_empty(fobj):
logging.warning(
"Stopping the upload of an empty file: {}".format(name))
return name
# set a max-age of 5 if we're uploading to content/databases
if self.is_database_file(name):
blob.cache_control = 'private, max-age={}, no-transform'.format(
CONTENT_DATABASES_MAX_AGE)
blob.upload_from_file(
fobj,
content_type=content_type,
)
return name
def save(self, name, fobj, max_length=None):
blob = Blob(name, self.bucket)
# force the current file to be at file location 0, to
# because that's what google wants
fobj.seek(0)
blob.upload_from_file(fobj)
return name
class GoogleCloudFile(File):
def __init__(self, name, mode, storage):
self.name = name
self.mime_type = mimetypes.guess_type(name)[0]
self._mode = mode
self._storage = storage
self.blob = storage.bucket.get_blob(name)
if not self.blob and 'w' in mode:
self.blob = Blob(self.name, storage.bucket)
self._file = None
self._is_dirty = False
@property
def size(self):
return self.blob.size
def _get_file(self):
if self._file is None:
self._file = SpooledTemporaryFile(
max_size=self._storage.max_memory_size,
suffix=".GSStorageFile",
dir=setting("FILE_UPLOAD_TEMP_DIR", None)
)
if 'r' in self._mode:
self._is_dirty = False
self.blob.download_to_file(self._file)
self._file.seek(0)
return self._file
def _set_file(self, value):
self._file = value
file = property(_get_file, _set_file)
def read(self, num_bytes=None):
if 'r' not in self._mode:
raise AttributeError("File was not opened in read mode.")
if num_bytes is None:
num_bytes = -1
return super(GoogleCloudFile, self).read(num_bytes)
def write(self, content):
if 'w' not in self._mode:
raise AttributeError("File was not opened in write mode.")
self._is_dirty = True
return super(GoogleCloudFile, self).write(force_bytes(content))
def close(self):
if self._file is not None:
if self._is_dirty:
self.file.seek(0)
self.blob.upload_from_file(self.file, content_type=self.mime_type)
self._file.close()
self._file = None
class GoogleCloudFile(File):
def __init__(self, name, mode, storage):
self.name = name
self.mime_type = mimetypes.guess_type(name)[0]
self._mode = mode
self._storage = storage
self.blob = storage.bucket.get_blob(name)
if not self.blob and 'w' in mode:
self.blob = Blob(self.name, storage.bucket)
self._file = None
self._is_dirty = False
@property
def size(self):
return self.blob.size
def _get_file(self):
if self._file is None:
self._file = SpooledTemporaryFile(
max_size=self._storage.max_memory_size,
suffix=".GSStorageFile",
dir=setting("FILE_UPLOAD_TEMP_DIR", None)
)
if 'r' in self._mode:
self._is_dirty = False
self.blob.download_to_file(self._file)
self._file.seek(0)
return self._file
def _set_file(self, value):
self._file = value
file = property(_get_file, _set_file)
def read(self, num_bytes=None):
if 'r' not in self._mode:
raise AttributeError("File was not opened in read mode.")
if num_bytes is None:
num_bytes = -1
return super(GoogleCloudFile, self).read(num_bytes)
def write(self, content):
if 'w' not in self._mode:
raise AttributeError("File was not opened in write mode.")
self._is_dirty = True
return super(GoogleCloudFile, self).write(force_bytes(content))
def close(self):
if self._file is not None:
if self._is_dirty:
self.file.seek(0)
self.blob.upload_from_file(self.file, content_type=self.mime_type)
self._file.close()
self._file = None
def save_file(self,
file,
filename,
folder=None,
randomize=False,
extensions=None,
acl=None,
replace=False,
headers=None):
"""
:param filename: local filename
:param folder: relative path of sub-folder
:param randomize: randomize the filename
:param extensions: iterable of allowed extensions, if not default
:param acl: ACL policy (if None then uses default)
:returns: modified filename
"""
extensions = extensions or self.extensions
if not self.filename_allowed(filename, extensions):
raise FileNotAllowed()
filename = utils.secure_filename(os.path.basename(filename))
if randomize:
filename = utils.random_filename(filename)
if folder:
filename = folder + "/" + filename
content_type, _ = mimetypes.guess_type(filename)
content_type = content_type or 'application/octet-stream'
blob = self.get_bucket().get_blob(filename)
# If the file exist and we explicitely asked not to replace it: ignore it.
if blob and not replace:
return filename
# If the file doesn't exist: create it.
if not blob:
blob = Blob(filename, self.get_bucket())
blob.cache_control = self.cache_control
file.seek(0)
acl = acl or self.acl
blob.upload_from_file(file,
rewind=True,
content_type=content_type,
predefined_acl=acl)
return filename
def save(self, name, fobj, max_length=None, blob_object=None):
if not blob_object:
blob = Blob(name, self.bucket)
else:
blob = blob_object
buffer = None
# set a max-age of 5 if we're uploading to content/databases
if self.is_database_file(name):
blob.cache_control = "private, max-age={}, no-transform".format(
CONTENT_DATABASES_MAX_AGE)
# Compress the database file so that users can save bandwith and download faster.
buffer = BytesIO()
compressed = GzipFile(fileobj=buffer, mode="w")
compressed.write(fobj.read())
compressed.close()
blob.content_encoding = "gzip"
fobj = buffer
# determine the current file's mimetype based on the name
# import determine_content_type lazily in here, so we don't get into an infinite loop with circular dependencies
from contentcuration.utils.storage_common import determine_content_type
content_type = determine_content_type(name)
# force the current file to be at file location 0, to
# because that's what google wants
fobj.seek(0)
if self._is_file_empty(fobj):
logging.warning(
"Stopping the upload of an empty file: {}".format(name))
return name
blob.upload_from_file(
fobj,
content_type=content_type,
)
# Close StringIO object and discard memory buffer if created
if buffer:
buffer.close()
return name
def save(self, name, fobj, max_length=None, blob_object=None):
if not blob_object:
blob = Blob(name, self.bucket)
else:
blob = blob_object
# force the current file to be at file location 0, to
# because that's what google wants
# determine the current file's mimetype based on the name
content_type = self._determine_content_type(name)
fobj.seek(0)
blob.upload_from_file(
fobj,
content_type=content_type,
)
return name
def save_file(self, file, filename, folder=None, randomize=False,
extensions=None, acl=None, replace=False, headers=None):
"""
:param filename: local filename
:param folder: relative path of sub-folder
:param randomize: randomize the filename
:param extensions: iterable of allowed extensions, if not default
:param acl: ACL policy (if None then uses default)
:returns: modified filename
"""
extensions = extensions or self.extensions
if not self.filename_allowed(filename, extensions):
raise FileNotAllowed()
filename = utils.secure_filename(
os.path.basename(filename)
)
if randomize:
filename = utils.random_filename(filename)
if folder:
filename = folder + "/" + filename
content_type, _ = mimetypes.guess_type(filename)
content_type = content_type or 'application/octet-stream'
blob = self.get_bucket().get_blob(filename)
# If the file exist and we explicitely asked not to replace it: ignore it.
if blob and not replace:
return filename
# If the file doesn't exist: create it.
if not blob:
blob = Blob(filename, self.get_bucket())
blob.cache_control = self.cache_control
file.seek(0)
acl = acl or self.acl
blob.upload_from_file(file, rewind=True, content_type=content_type, predefined_acl=acl)
return filename
def save(self, name, fobj, max_length=None, blob_object=None):
if not blob_object:
blob = Blob(name, self.bucket)
else:
blob = blob_object
# force the current file to be at file location 0, to
# because that's what google wants
# determine the current file's mimetype based on the name
content_type = self._determine_content_type(name)
fobj.seek(0)
if self._is_file_empty(fobj):
logging.warning("Stopping the upload of an empty file: {}".format(name))
return name
blob.upload_from_file(
fobj,
content_type=content_type,
)
return name
def save(self, name, fobj, max_length=None, blob_object=None):
if not blob_object:
blob = Blob(name, self.bucket)
else:
blob = blob_object
# force the current file to be at file location 0, to
# because that's what google wants
# determine the current file's mimetype based on the name
content_type = self._determine_content_type(name)
fobj.seek(0)
if self._is_file_empty(fobj):
logging.warning(
"Stopping the upload of an empty file: {}".format(name))
return name
blob.upload_from_file(
fobj,
content_type=content_type,
)
return name
class GoogleCloudFile(File):
"""File logic for reading and writing to Google Cloud bucket files."""
def __init__(self, name, mode, bucket):
"""Create a GoogleCloudFile handler.
Args:
name: The name of the file within the bucket (string).
mode: A string of the file mode to open with (string).
bucket: The bucket to load and save to (Bucket).
"""
self.name = name
self.mode = mode
self._mimetype = mimetypes.guess_type(name)[0]
self._is_dirty = False
self._file = None
self._blob = bucket.get_blob(name)
if not self.blob and "w" in mode:
self.blob = Blob(self.name, bucket)
elif not self.blob and "r" in mode:
message = "{} file was not found."
message.format(name)
raise NotFound(message=message)
self.open(mode)
@property
def size(self):
"""Get the size of the file.
Returns:
The size of the file (int).
"""
if self._is_dirty:
if hasattr(self.file, 'size'):
return self.file.size
if hasattr(self.file, 'tell') and hasattr(self.file, 'seek'):
pos = self.file.tell()
self.file.seek(0, os.SEEK_END)
size = self.file.tell()
self.file.seek(pos)
return size
raise AttributeError("Unable to determine the file's size.")
return self.blob.size
def _get_file(self):
"""Get the underlying file object.
Returns:
The underlying file object (stream).
"""
if self._file is None:
self.open()
return self._file
def _set_file(self, value):
"""Set the underlying file object.
Args:
value: The new file object (stream).
"""
self._file = value
file = property(_get_file, _set_file)
def open(self, mode=None):
"""Open or reopen the file.
Args:
mode: The mode to file as, similar to pythons open
function (string).
"""
if mode is not None:
self.mode = mode
self._file = None
self._is_dirty = False
self._file = SpooledTemporaryFile(mode="w+b",
suffix=".GoogleCloudFile")
self.blob.download_to_file(self._file)
if "a" not in self.mode:
self._file.seek(0)
def read(self, num_bytes=None):
"""Read from file.
Args:
num_bytes: The number of bytes to read from the file (int).
Returns:
The bytes read (bytes).
"""
if "r" not in self.mode:
raise AttributeError("File was not opened in read mode.")
if num_bytes is None:
num_bytes = -1
return super(GoogleCloudFile, self).read(num_bytes)
def write(self, content):
"""Write to the file.
Args:
content: Something to write to the file, if not bytes
will attempt to convert (object).
"""
if "w" not in self.mode or "a" not in self.mode:
raise AttributeError("File was not opened in write mode.")
self._is_dirty = True
return super(GoogleCloudFile, self).write(force_bytes(content))
def close(self):
"""Close the file, and save out changes."""
if self._file is not None:
if ("w" in self.mode or "a" in self.mode) and self._is_dirty:
self._file.seek(0)
self.blob.upload_from_file(self._file,
content_type=self.mime_type)
self._file.close()
self._file = None
