class LocallyMirroredS3BotoStorage(S3BotoStorage):
def __init__(self, *args, **kwargs):
super(LocallyMirroredS3BotoStorage, self).__init__(*args, **kwargs)
self.mirror = FileSystemStorage(location=settings.S3_MIRROR_ROOT)
def delete(self, name):
super(LocallyMirroredS3BotoStorage, self).delete(name)
try:
self.mirror.delete(name)
except OSError:
full_path = self.mirror.path(name)
if os.path.exists(full_path):
os.rmdir(full_path)
def exists(self, name):
exists_local = self.mirror.exists(name)
if exists_local:
return True
else:
exists_remote = super(LocallyMirroredS3BotoStorage,
self).exists(name)
if exists_remote:
self.mirror._save(name, ContentFile(""))
return True
return False
def _save(self, name, content):
cleaned_name = super(LocallyMirroredS3BotoStorage,
self)._save(name, content)
self.mirror._save(name, ContentFile(""))
return cleaned_name
class ExhibitStorage(FileSystemStorage):
"""
Assume that incoming paths are of the form <username>/.../...
"""
def __init__(self, *args, **kwargs):
self.__userdata_storage = FileSystemStorage(location=kwargs['location'], base_url=kwargs['base_url'])
def _open(self, name, mode='rb'):
return self.__userdata_storage._open(chunk_path(name) + name, mode)
def _save(self, name, content):
chunk = chunk_path(name)
fullname = chunk + name
if (self.__userdata_storage.exists(fullname)):
self.__userdata_storage.delete(fullname)
result = self.__userdata_storage._save(fullname, content)
return result.partition(chunk)[2]
def exists(self, name):
return self.__userdata_storage.exists(chunk_path(name) + name)
def path(self, name):
return self.__userdata_storage.path(chunk_path(name) + name)
def size(self, name):
return self.__userdata_storage.size(chunk_path(name) + name)
def delete(self, name):
return self.__userdata_storage.delete(chunk_path(name) + name)
def url(self, name):
return self.__userdata_storage.url(name)
def get_available_name(self, name):
return self.__userdata_storage.get_available_n
def _save(self, name, content):
name = FileSystemStorage._save(self, name, content)
# After this line, we will assume that 'name' is available on the
# media servers. This could be wrong if a delete for this file name
# failed at some point in the past.
self.execute(self.upload, name)
return name
def _save(self, name, content):
name = FileSystemStorage._save(self, name, content)
# After this line, we will assume that 'name' is available on the
# media servers. This could be wrong if a delete for this file name
# failed at some point in the past.
self.execute(self.upload, name)
return name
def _save(self, name, content):
name = FileSystemStorage._save(self, name, content)
content.seek(0)
# After this line, we will assume that 'name' is available on the
# media servers. This could be wrong if a delete for this file name
# failed at some point in the past.
DistributedStorageMixin._save(self, name, content)
return name
def _save(self, name, content):
name = FileSystemStorage._save(self, name, content)
content.seek(0)
# After this line, we will assume that 'name' is available on the
# media servers. This could be wrong if a delete for this file name
# failed at some point in the past.
DistributedStorageMixin._save(self, name, content)
return name
def _save(self, name, content):
file_system = FileSystemStorage()
image_filename = file_system._save(name, content)
# here we're dropping the resolution of the image for storage
# and transfer purposes
image = nibabel.load(image_filename)
image_data = numpy.nan_to_num(image.get_data())
scaled_data = resize(image_data, 60, 72, 60)
scaled_data = numpy.around(scaled_data, decimals=3)
# javascript doesn't handle NaN very well
scaled_data = numpy.nan_to_num(scaled_data)
scaled_list_data = scaled_data.tolist()
os.unlink(image_filename)
# for some reason ndarray.max returns an ndarray with 1 member
# which is a numpy.flaot32. this converts all that to a regular python float
max = numpy.ndarray.max(scaled_data).tolist()
min = numpy.ndarray.min(scaled_data).tolist()
json_data = {"data": scaled_list_data, "max": max, "min": min}
javascript_tempfile = File(open(tempfile.mktemp(), "w"))
javascript_tempfile.open("w")
javascript_tempfile.writelines(json.dumps(json_data))
javascript_tempfile.close()
# we re-open the file as file_system._save() expects an open file
javascript_tempfile.open("r")
(root, ext) = os.path.splitext(image_filename)
if ext == ".gz":
(root, original_ext) = os.path.splitext(root)
ext = original_ext + ext
return file_system._save(root + ".json", javascript_tempfile)
def _save(self, name, content):
full_path_dst = self.path(name)
directory = os.path.dirname(full_path_dst)
if not os.path.exists(directory):
os.makedirs(directory)
elif not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
full_path_src = os.path.abspath(content.name)
symlinked = False
# Only symlink if the current platform supports it.
if getattr(os, "symlink", False):
for path in self.symlinkWithin:
if full_path_src.startswith(path):
os.symlink(full_path_src, full_path_dst)
symlinked = True
break
if not symlinked:
FileSystemStorage._save(self, name, content)
return name
def _save(self, name, content):
full_path_dst = self.path(name)
directory = os.path.dirname(full_path_dst)
if not os.path.exists(directory):
os.makedirs(directory)
elif not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
full_path_src = os.path.abspath(content.name)
symlinked = False
# Only symlink if the current platform supports it.
if getattr(os, "symlink", False):
for path in self.symlinkWithin:
if full_path_src.startswith(path):
os.symlink(full_path_src, full_path_dst)
symlinked = True
break
if not symlinked:
FileSystemStorage._save(self, name, content)
return name
class DummyExternalStorage(Storage):
def __init__(self, *args, **kwargs):
self.wrapped = FileSystemStorage(*args, **kwargs)
def path(self, name):
# Overridden to give it the behaviour of the base Storage class
# This is what an external storage backend would have
raise NotImplementedError("This backend doesn't support absolute paths.")
def _open(self, name, mode='rb'):
# Overridden to return a DummyExternalStorageFile instead of a normal
# File object
return DummyExternalStorageFile(open(self.wrapped.path(name), mode))
# Wrap all other functions
def _save(self, name, content):
return self.wrapped._save(name, content)
def delete(self, name):
self.wrapped.delete(name)
def exists(self, name):
return self.wrapped.exists(name)
def listdir(self, path):
return self.wrapped.listdir(path)
def size(self, name):
return self.wrapped.size(name)
def url(self, name):
return self.wrapped.url(name)
def accessed_time(self, name):
return self.wrapped.accessed_time(name)
def created_time(self, name):
return self.wrapped.created_time(name)
def modified_time(self, name):
return self.wrapped.modified_time(name)
class DummyExternalStorage(Storage):
def __init__(self, *args, **kwargs):
self.wrapped = FileSystemStorage(*args, **kwargs)
def path(self, name):
# Overridden to give it the behaviour of the base Storage class
# This is what an external storage backend would have
raise NotImplementedError("This backend doesn't support absolute paths.")
def _open(self, name, mode='rb'):
# Overridden to return a DummyExternalStorageFile instead of a normal
# File object
return DummyExternalStorageFile(open(self.wrapped.path(name), mode))
# Wrap all other functions
def _save(self, name, content):
return self.wrapped._save(name, content)
def delete(self, name):
self.wrapped.delete(name)
def exists(self, name):
return self.wrapped.exists(name)
def listdir(self, path):
return self.wrapped.listdir(path)
def size(self, name):
return self.wrapped.size(name)
def url(self, name):
return self.wrapped.url(name)
def accessed_time(self, name):
return self.wrapped.accessed_time(name)
def created_time(self, name):
return self.wrapped.created_time(name)
def modified_time(self, name):
return self.wrapped.modified_time(name)
class SaveLocal(Storage):
def __init__(self, configs):
self.fs = FileSystemStorage()
self.sets = configs
self.location = self.sets.get('location', lambda name: name)
self.name_uuid_len = self.sets.get('name_uuid_len', None)
def get_alternative_name(self, file_root, file_ext):
return DriverUtils.create_file_name((file_root, file_ext),
self.name_uuid_len)
def get_available_name(self, name, max_length=None):
dir_name, file_name = os.path.split(name)
name = DriverUtils.create_file_name(file_name, self.name_uuid_len)
name = posixpath.join(dir_name, name)
name = self.location(name)
name = DriverUtils.clean_name(name)
return self.fs.get_available_name(name)
def _save(self, name, content):
name = self.fs._save(name, content)
return name
def url(self, name):
return self.fs.url(name)
def path(self, name):
return self.fs.path(name)
def delete(self, name):
try:
os.remove(posixpath.join(settings.MEDIA_ROOT, name))
except OSError as ose:
print(f'File "{ose.filename}" does not exist')
def download(self, name):
file = open(self.path(name), 'rb')
filename = posixpath.basename(name)
return file, filename
def retrieve(self, name):
return self.download(name)
def _save(self, name, content):
"""
RSYNC_HOST_DATA = {
'host': '172.17.0.2',
'media_root': '/var/opt/project/media/'
}
name = orders_media/1_rGSQIX2.png # Это <upload_to>/<file_name>
upload_to на серверах должен быть одинаковым.
"""
name = self._add_prefix(name)
name = FileSystemStorage._save(self, name, content)
abs_path_to_file_from = os.path.join(settings.MEDIA_ROOT, name)
for RSYNC_HOST_CONF in settings.RSYNC_HOSTS:
RSYNC_HOST_TO = RSYNC_HOST_CONF['host']
if RSYNC_HOST_TO == CURRENT_HOST:
continue
RSYNC_MEDIA_ROOT_TO = RSYNC_HOST_CONF['media_root']
abs_path_file_name_to = os.path.join(RSYNC_MEDIA_ROOT_TO, name)
dir_to = os.path.dirname(abs_path_file_name_to) + '/'
remote_dir_to = '%s:%s' % (RSYNC_HOST_TO, dir_to)
dir_from = os.path.dirname(abs_path_to_file_from) + '/'
rsync_task.delay(dir_from, remote_dir_to)
return name
def _save(self, name, content):
# here, you should implement how the file is to be saved
# like on other machines or something, and return the name of the file.
fs = FileSystemStorage()
name = fs._save(name, content)
return name
def _save(self, name, content):
if self.exists(name):
self.delete(name)
return FileSystemStorage._save(self, name, content)
def _save(self, name, content):
full_path = self.path(name)
if os.path.exists(full_path):
os.remove(full_path)
return FileSystemStorage._save(self, name, content)
class LocallyMirroredS3BotoStorage(S3BotoStorage):
def __init__(self, *args, **kwargs):
super(LocallyMirroredS3BotoStorage, self).__init__(*args, **kwargs)
self.mirror = FileSystemStorage(location=settings.S3_MIRROR_ROOT)
def delete(self, name):
super(LocallyMirroredS3BotoStorage, self).delete(name)
try:
self.mirror.delete(name)
except OSError:
full_path = self.mirror.path(name)
if os.path.exists(full_path):
os.rmdir(full_path)
def exists(self, name):
exists_local = self.mirror.exists(name)
if exists_local:
return True
else:
exists_remote = super(LocallyMirroredS3BotoStorage, self).exists(name)
if exists_remote:
self.mirror._save(name, ContentFile(""))
return True
return False
def _save(self, name, content):
cleaned_name = super(LocallyMirroredS3BotoStorage, self)._save(name, content)
self.mirror._save(name, ContentFile(""))
return cleaned_name
