def sync(src_storage, src_path, dest_storage, dest_path):
if is_aws_s3(src_storage) and is_aws_s3(dest_storage):
src_key = safe_join(src_storage.location, src_path)
dest_key = safe_join(dest_storage.location, dest_path)
src_s3_client = get_s3_client(src_storage)
if src_storage.access_key == dest_storage.access_key:
src_s3_client.copy(
{'Bucket': src_storage.bucket_name, 'Key': src_key},
dest_storage.bucket_name, dest_key)
else:
_, tmp_file_path = tempfile.mkstemp()
try:
src_s3_client.download_file(
Bucket=src_storage.bucket_name, Key=src_key,
Filename=tmp_file_path)
dest_s3_client = get_s3_client(dest_storage)
dest_s3_client.upload_file(
Filename=tmp_file_path, Bucket=dest_storage.bucket_name,
Key=dest_key)
except Exception:
raise
finally:
os.remove(tmp_file_path)
else:
# копирование таким способом безопасно даже если файл копируется
# "сам в себя" (т.е. на тот же накопитель и с тем же путём и
# именем), в этом случае внутренние механизмы копирования его не
# трогают, а просто возвращают положительный результат
opened = src_storage.open(src_path)
dest_storage.save(dest_path, opened)
opened.close()
def _path(self, name):
name = _clean_name_dance(name)
try:
return safe_join(self.location, name)
except ValueError:
raise SuspiciousOperation("Attempted access to '%s' denied." %
name)
def rename_if_adding_dup(sender, instance, **kwargs):
if not instance._state.adding:
return
is_dup = default_storage.exists(
safe_join(instance.UPLOADS_ROOT_DIR, str(instance)))
if is_dup:
instance.media.name = instance.rename_dup()
def _normalize_name(self, name):
"""
Normalizes the name so that paths like /path/to/ignored/../something.txt
and ./file.txt work. Note that clean_name adds ./ to some paths so
they need to be fixed here.
"""
return safe_join('', name)
def get_etag(self, path):
normalized_path = safe_join(self.storage.location,
path).replace('\\', '/')
try:
return self.storage.bucket.get_key(normalized_path).etag
except AttributeError:
return None
def get_etag(self, path):
normalized_path = safe_join(self.storage.location,
path).replace('\\', '/')
try:
return self.storage.bucket.Object(normalized_path).e_tag
except:
pass
def _normalize_name(self, name):
"""
Normalizes the name so that paths like /path/to/ignored/../something.txt
and ./file.txt work. Note that clean_name adds ./ to some paths so
they need to be fixed here.
"""
return safe_join('', name)
def _normalize_name(self, name):
"""
Normalizes the name so that paths like /path/to/ignored/../something.txt
work. We check to make sure that the path pointed to is not outside
the directory specified by the LOCATION setting.
"""
try:
return safe_join(self.location, name)
except ValueError:
raise SuspiciousOperation("Attempted access to '%s' denied." % name)
def generic_upload_to(base_path, instance, filename):
"""
Just is possible use the ID as the filename where the table id is a UUIDField
"""
assert (base_path is not None)
ext = filename.split('.')[-1]
filename = '%s.%s' % (instance.id, ext)
return safe_join(base_path, filename)
def _normalize_name(self, name):
"""
Normalizes the name so that paths like /path/to/ignored/../something.txt
and ./file.txt work. Note that clean_name adds ./ to some paths so
they need to be fixed here. We check to make sure that the path pointed
to is not outside the directory specified by the LOCATION setting.
"""
try:
return safe_join(self.location, name)
except ValueError:
raise SuspiciousOperation(f"Attempted access to '{name}' denied.")
def delete_directory(self, path):
"""
Delete all files under a certain path from storage
Many storage backends (S3, Azure storage) don't care about "directories".
The directory effectively doesn't exist if there are no files in it.
However, in these backends, there is no "rmdir" operation so you have to recursively
delete all files.
:param path: the path to the directory to remove
"""
log.debug('Deleting directory %s from media storage', path)
folders, files = self.listdir(self._dirpath(path))
for folder_name in folders:
if folder_name:
# Recursively delete the subdirectory
self.delete_directory(safe_join(path, folder_name))
for filename in files:
if filename:
self.delete(safe_join(path, filename))
def rename_dup(self):
max_length = self._meta.get_field('media').max_length
name, ext = os.path.splitext(self.media.name)
for i in itertools.count(1):
dup_count_str = '_%s' % i
name_truncated = name[:max_length - len(dup_count_str) - len(ext)]
file_path = safe_join(
self.UPLOADS_ROOT_DIR, os.path.dirname(str(self)),
get_valid_filename(name + dup_count_str + ext))
if not default_storage.exists(file_path):
return os.path.basename(file_path)
def _normalize_name(self, name):
"""
Normalizes the name so that paths like /path/to/ignored/../something.txt
work. We check to make sure that the path pointed to is not outside
the directory specified by the LOCATION setting.
"""
try:
return safe_join(self.location, name)
except ValueError:
raise SuspiciousOperation("Attempted access to '%s' denied." %
name)
def delete_directory(self, path):
"""
Delete all files under a certain path from storage
Many storage backends (S3, Azure storage) don't care about "directories".
The directory effectively doesn't exist if there are no files in it.
However, in these backends, there is no "rmdir" operation so you have to recursively
delete all files.
:param path: the path to the directory to remove
"""
if path in ('', '/'):
raise SuspiciousFileOperation('Deleting all storage cannot be right')
log.debug('Deleting directory %s from media storage', path)
folders, files = self.listdir(self._dirpath(path))
for folder_name in folders:
if folder_name:
# Recursively delete the subdirectory
self.delete_directory(safe_join(path, folder_name))
for filename in files:
if filename:
self.delete(safe_join(path, filename))
def get_remote_etag(storage, prefixed_path):
"""
Get etag of path from S3 using boto or boto3.
"""
normalized_path = safe_join(storage.location, prefixed_path).replace(
'\\', '/')
try:
return storage.bucket.get_key(normalized_path).etag
except AttributeError:
pass
try:
return storage.bucket.Object(normalized_path).e_tag
except:
pass
return None
def get_remote_etag(storage, prefixed_path):
"""
Get etag of path from S3 using boto or boto3.
"""
normalized_path = safe_join(storage.location,
prefixed_path).replace('\\', '/')
try:
return storage.bucket.get_key(normalized_path).etag
except AttributeError:
pass
try:
return storage.bucket.Object(normalized_path).e_tag
except:
pass
return None
def _normalize_name(self, name):
"""
Normalizes the name so that paths like /path/to/ignored/../something.txt
and ./file.txt work. Note that clean_name adds ./ to some paths so
they need to be fixed here.
"""
if 'panya-bot.appspot.com' in name:
name = name[22:]
return name
try:
return safe_join(self.location, name)
except Exception:
pass
# raise SuspiciousOperation("Attempted access to '%s' denied." %
# name)
return name
def copy_directory(self, source, destination):
"""
Copy a directory recursively to storage
:param source: the source path on the local disk
:param destination: the destination path in storage
"""
log.debug('Copying source directory %s to media storage at %s', source, destination)
source = Path(source)
for filepath in source.iterdir():
sub_destination = safe_join(destination, filepath.name)
if filepath.is_dir():
# Recursively copy the subdirectory
self.copy_directory(filepath, sub_destination)
elif filepath.is_file():
with filepath.open('rb') as fd:
self.save(sub_destination, fd)
def copy_directory(self, source, destination):
"""
Copy a directory recursively to storage
:param source: the source path on the local disk
:param destination: the destination path in storage
"""
log.debug('Copying source directory %s to media storage at %s', source, destination)
source = Path(source)
for filepath in source.iterdir():
sub_destination = safe_join(destination, filepath.name)
if filepath.is_dir():
# Recursively copy the subdirectory
self.copy_directory(filepath, sub_destination)
elif filepath.is_file():
with filepath.open('rb') as fd:
self.save(sub_destination, fd)
def test_with_dot(self):
path = utils.safe_join("", "path/./somewhere/../other", "..",
".", "to/./somewhere")
self.assertEqual(path, "path/to/somewhere")
def test_base_url_with_slash(self):
path = utils.safe_join("base_url/", "path/to/somewhere")
self.assertEqual(path, "base_url/path/to/somewhere")
def test_normal(self):
path = utils.safe_join("", "path/to/somewhere", "other", "path/to/somewhere")
self.assertEqual(path, "path/to/somewhere/other/path/to/somewhere")
def _path(self, name):
name = _clean_name_dance(name)
try:
return safe_join(self.location, name)
except ValueError:
raise SuspiciousOperation("Attempted access to '%s' denied." % name)
class S3FileInputMixin:
"""FileInput that uses JavaScript to directly upload to Amazon S3."""
needs_multipart_form = False
upload_path = str(
getattr(settings, "S3FILE_UPLOAD_PATH", pathlib.PurePosixPath("tmp", "s3file"))
)
upload_path = safe_join(str(storage.location), upload_path)
expires = settings.SESSION_COOKIE_AGE
@property
def bucket_name(self):
return storage.bucket.name
@property
def client(self):
return storage.connection.meta.client
def build_attrs(self, *args, **kwargs):
attrs = super().build_attrs(*args, **kwargs)
accept = attrs.get("accept")
response = self.client.generate_presigned_post(
self.bucket_name,
str(pathlib.PurePosixPath(self.upload_folder, "${filename}")),
Conditions=self.get_conditions(accept),
ExpiresIn=self.expires,
)
defaults = {
"data-fields-%s" % key: value for key, value in response["fields"].items()
}
defaults["data-url"] = response["url"]
defaults.update(attrs)
try:
defaults["class"] += " s3file"
except KeyError:
defaults["class"] = "s3file"
return defaults
def get_conditions(self, accept):
conditions = [
{"bucket": self.bucket_name},
["starts-with", "$key", str(self.upload_folder)],
{"success_action_status": "201"},
]
if accept and "," not in accept:
top_type, sub_type = accept.split("/", 1)
if sub_type == "*":
conditions.append(["starts-with", "$Content-Type", "%s/" % top_type])
else:
conditions.append({"Content-Type": accept})
else:
conditions.append(["starts-with", "$Content-Type", ""])
return conditions
@cached_property
def upload_folder(self):
return str(
pathlib.PurePosixPath(
self.upload_path,
base64.urlsafe_b64encode(uuid.uuid4().bytes)
.decode("utf-8")
.rstrip("=\n"),
)
) # S3 uses POSIX paths
class Media:
js = ("s3file/js/s3file.js" if settings.DEBUG else "s3file/js/s3file.min.js",)
def test_join_empty_string(self):
path = utils.safe_join('base_url', '')
self.assertEqual(path, 'base_url/')
def test_join_empty_string(self):
path = utils.safe_join('base_url', '')
self.assertEqual(path, 'base_url/')
def test_datetime_isoformat(self):
dt = datetime.datetime(2017, 5, 19, 14, 45, 37, 123456)
path = utils.safe_join('base_url', dt.isoformat())
self.assertEqual(path, 'base_url/2017-05-19T14:45:37.123456')
def test_trailing_slash_multi(self):
"""
Test safe_join with multiple paths that end with a trailing slash.
"""
path = utils.safe_join("base_url/", "path/to/", "somewhere/")
self.assertEqual(path, "base_url/path/to/somewhere/")
def test_suspicious_operation(self):
with self.assertRaises(ValueError):
utils.safe_join("base", "../../../../../../../etc/passwd")
with self.assertRaises(ValueError):
utils.safe_join("base", "/etc/passwd")
def test_base_url_with_slash(self):
path = utils.safe_join("base_url/", "path/to/somewhere")
self.assertEqual(path, "base_url/path/to/somewhere")
def test_with_only_dot(self):
path = utils.safe_join("", ".")
self.assertEqual(path, "")
def test_with_base_url_join_nothing(self):
path = utils.safe_join('base_url')
self.assertEqual(path, 'base_url/')
def test_with_only_dot(self):
path = utils.safe_join("", ".")
self.assertEqual(path, "")
def test_with_base_url_join_nothing(self):
path = utils.safe_join('base_url')
self.assertEqual(path, 'base_url/')
def test_join_nothing(self):
path = utils.safe_join('')
self.assertEqual(path, '')
def _normalize_path(self, prefixed_path):
# type: (str) -> str
path = str(safe_join(self.remote_storage.location, prefixed_path))
return path.replace("\\", "")
def join(self, directory, filepath):
return safe_join(directory, filepath)
def test_trailing_slash_multi(self):
"""
Test safe_join with multiple paths that end with a trailing slash.
"""
path = utils.safe_join("base_url/", "path/to/", "somewhere/")
self.assertEqual(path, "base_url/path/to/somewhere/")
def _normalize_name(self, name):
try:
return safe_join(self.location, name)
except ValueError:
raise SuspiciousOperation("Attempted access to '%s' denied." %
name)
def test_with_base_url_and_dot(self):
path = utils.safe_join('base_url', '.')
self.assertEqual(path, 'base_url/')
def test_with_base_url_and_dot_and_path_and_slash(self):
path = utils.safe_join('base_url', '.', 'path/to/', '.')
self.assertEqual(path, 'base_url/path/to/')
def test_with_base_url_and_dot_and_path_and_slash(self):
path = utils.safe_join('base_url', '.', 'path/to/', '.')
self.assertEqual(path, 'base_url/path/to/')
def test_with_dot(self):
path = utils.safe_join("", "path/./somewhere/../other", "..",
".", "to/./somewhere")
self.assertEqual(path, "path/to/somewhere")
def test_join_nothing(self):
path = utils.safe_join('')
self.assertEqual(path, '')
def test_suspicious_operation(self):
with self.assertRaises(ValueError):
utils.safe_join("base", "../../../../../../../etc/passwd")
with self.assertRaises(ValueError):
utils.safe_join("base", "/etc/passwd")
def test_normal(self):
path = utils.safe_join("", "path/to/somewhere", "other", "path/to/somewhere")
self.assertEqual(path, "path/to/somewhere/other/path/to/somewhere")
def test_datetime_isoformat(self):
dt = datetime.datetime(2017, 5, 19, 14, 45, 37, 123456)
path = utils.safe_join('base_url', dt.isoformat())
self.assertEqual(path, 'base_url/2017-05-19T14:45:37.123456')
def test_with_base_url_and_dot(self):
path = utils.safe_join('base_url', '.')
self.assertEqual(path, 'base_url/')