def sync(src_storage, src_path, dest_storage, dest_path): if is_aws_s3(src_storage) and is_aws_s3(dest_storage): src_key = safe_join(src_storage.location, src_path) dest_key = safe_join(dest_storage.location, dest_path) src_s3_client = get_s3_client(src_storage) if src_storage.access_key == dest_storage.access_key: src_s3_client.copy( {'Bucket': src_storage.bucket_name, 'Key': src_key}, dest_storage.bucket_name, dest_key) else: _, tmp_file_path = tempfile.mkstemp() try: src_s3_client.download_file( Bucket=src_storage.bucket_name, Key=src_key, Filename=tmp_file_path) dest_s3_client = get_s3_client(dest_storage) dest_s3_client.upload_file( Filename=tmp_file_path, Bucket=dest_storage.bucket_name, Key=dest_key) except Exception: raise finally: os.remove(tmp_file_path) else: # копирование таким способом безопасно даже если файл копируется # "сам в себя" (т.е. на тот же накопитель и с тем же путём и # именем), в этом случае внутренние механизмы копирования его не # трогают, а просто возвращают положительный результат opened = src_storage.open(src_path) dest_storage.save(dest_path, opened) opened.close()
def _path(self, name): name = _clean_name_dance(name) try: return safe_join(self.location, name) except ValueError: raise SuspiciousOperation("Attempted access to '%s' denied." % name)
def rename_if_adding_dup(sender, instance, **kwargs): if not instance._state.adding: return is_dup = default_storage.exists( safe_join(instance.UPLOADS_ROOT_DIR, str(instance))) if is_dup: instance.media.name = instance.rename_dup()
def _normalize_name(self, name): """ Normalizes the name so that paths like /path/to/ignored/../something.txt and ./file.txt work. Note that clean_name adds ./ to some paths so they need to be fixed here. """ return safe_join('', name)
def get_etag(self, path): normalized_path = safe_join(self.storage.location, path).replace('\\', '/') try: return self.storage.bucket.get_key(normalized_path).etag except AttributeError: return None
def get_etag(self, path): normalized_path = safe_join(self.storage.location, path).replace('\\', '/') try: return self.storage.bucket.Object(normalized_path).e_tag except: pass
def _normalize_name(self, name): """ Normalizes the name so that paths like /path/to/ignored/../something.txt and ./file.txt work. Note that clean_name adds ./ to some paths so they need to be fixed here. """ return safe_join('', name)
def _normalize_name(self, name): """ Normalizes the name so that paths like /path/to/ignored/../something.txt work. We check to make sure that the path pointed to is not outside the directory specified by the LOCATION setting. """ try: return safe_join(self.location, name) except ValueError: raise SuspiciousOperation("Attempted access to '%s' denied." % name)
def generic_upload_to(base_path, instance, filename): """ Just is possible use the ID as the filename where the table id is a UUIDField """ assert (base_path is not None) ext = filename.split('.')[-1] filename = '%s.%s' % (instance.id, ext) return safe_join(base_path, filename)
def _normalize_name(self, name): """ Normalizes the name so that paths like /path/to/ignored/../something.txt and ./file.txt work. Note that clean_name adds ./ to some paths so they need to be fixed here. We check to make sure that the path pointed to is not outside the directory specified by the LOCATION setting. """ try: return safe_join(self.location, name) except ValueError: raise SuspiciousOperation(f"Attempted access to '{name}' denied.")
def delete_directory(self, path): """ Delete all files under a certain path from storage Many storage backends (S3, Azure storage) don't care about "directories". The directory effectively doesn't exist if there are no files in it. However, in these backends, there is no "rmdir" operation so you have to recursively delete all files. :param path: the path to the directory to remove """ log.debug('Deleting directory %s from media storage', path) folders, files = self.listdir(self._dirpath(path)) for folder_name in folders: if folder_name: # Recursively delete the subdirectory self.delete_directory(safe_join(path, folder_name)) for filename in files: if filename: self.delete(safe_join(path, filename))
def rename_dup(self): max_length = self._meta.get_field('media').max_length name, ext = os.path.splitext(self.media.name) for i in itertools.count(1): dup_count_str = '_%s' % i name_truncated = name[:max_length - len(dup_count_str) - len(ext)] file_path = safe_join( self.UPLOADS_ROOT_DIR, os.path.dirname(str(self)), get_valid_filename(name + dup_count_str + ext)) if not default_storage.exists(file_path): return os.path.basename(file_path)
def _normalize_name(self, name): """ Normalizes the name so that paths like /path/to/ignored/../something.txt work. We check to make sure that the path pointed to is not outside the directory specified by the LOCATION setting. """ try: return safe_join(self.location, name) except ValueError: raise SuspiciousOperation("Attempted access to '%s' denied." % name)
def delete_directory(self, path): """ Delete all files under a certain path from storage Many storage backends (S3, Azure storage) don't care about "directories". The directory effectively doesn't exist if there are no files in it. However, in these backends, there is no "rmdir" operation so you have to recursively delete all files. :param path: the path to the directory to remove """ if path in ('', '/'): raise SuspiciousFileOperation('Deleting all storage cannot be right') log.debug('Deleting directory %s from media storage', path) folders, files = self.listdir(self._dirpath(path)) for folder_name in folders: if folder_name: # Recursively delete the subdirectory self.delete_directory(safe_join(path, folder_name)) for filename in files: if filename: self.delete(safe_join(path, filename))
def get_remote_etag(storage, prefixed_path): """ Get etag of path from S3 using boto or boto3. """ normalized_path = safe_join(storage.location, prefixed_path).replace( '\\', '/') try: return storage.bucket.get_key(normalized_path).etag except AttributeError: pass try: return storage.bucket.Object(normalized_path).e_tag except: pass return None
def get_remote_etag(storage, prefixed_path): """ Get etag of path from S3 using boto or boto3. """ normalized_path = safe_join(storage.location, prefixed_path).replace('\\', '/') try: return storage.bucket.get_key(normalized_path).etag except AttributeError: pass try: return storage.bucket.Object(normalized_path).e_tag except: pass return None
def _normalize_name(self, name): """ Normalizes the name so that paths like /path/to/ignored/../something.txt and ./file.txt work. Note that clean_name adds ./ to some paths so they need to be fixed here. """ if 'panya-bot.appspot.com' in name: name = name[22:] return name try: return safe_join(self.location, name) except Exception: pass # raise SuspiciousOperation("Attempted access to '%s' denied." % # name) return name
def copy_directory(self, source, destination): """ Copy a directory recursively to storage :param source: the source path on the local disk :param destination: the destination path in storage """ log.debug('Copying source directory %s to media storage at %s', source, destination) source = Path(source) for filepath in source.iterdir(): sub_destination = safe_join(destination, filepath.name) if filepath.is_dir(): # Recursively copy the subdirectory self.copy_directory(filepath, sub_destination) elif filepath.is_file(): with filepath.open('rb') as fd: self.save(sub_destination, fd)
def copy_directory(self, source, destination): """ Copy a directory recursively to storage :param source: the source path on the local disk :param destination: the destination path in storage """ log.debug('Copying source directory %s to media storage at %s', source, destination) source = Path(source) for filepath in source.iterdir(): sub_destination = safe_join(destination, filepath.name) if filepath.is_dir(): # Recursively copy the subdirectory self.copy_directory(filepath, sub_destination) elif filepath.is_file(): with filepath.open('rb') as fd: self.save(sub_destination, fd)
def test_with_dot(self): path = utils.safe_join("", "path/./somewhere/../other", "..", ".", "to/./somewhere") self.assertEqual(path, "path/to/somewhere")
def test_base_url_with_slash(self): path = utils.safe_join("base_url/", "path/to/somewhere") self.assertEqual(path, "base_url/path/to/somewhere")
def test_normal(self): path = utils.safe_join("", "path/to/somewhere", "other", "path/to/somewhere") self.assertEqual(path, "path/to/somewhere/other/path/to/somewhere")
def _path(self, name): name = _clean_name_dance(name) try: return safe_join(self.location, name) except ValueError: raise SuspiciousOperation("Attempted access to '%s' denied." % name)
class S3FileInputMixin: """FileInput that uses JavaScript to directly upload to Amazon S3.""" needs_multipart_form = False upload_path = str( getattr(settings, "S3FILE_UPLOAD_PATH", pathlib.PurePosixPath("tmp", "s3file")) ) upload_path = safe_join(str(storage.location), upload_path) expires = settings.SESSION_COOKIE_AGE @property def bucket_name(self): return storage.bucket.name @property def client(self): return storage.connection.meta.client def build_attrs(self, *args, **kwargs): attrs = super().build_attrs(*args, **kwargs) accept = attrs.get("accept") response = self.client.generate_presigned_post( self.bucket_name, str(pathlib.PurePosixPath(self.upload_folder, "${filename}")), Conditions=self.get_conditions(accept), ExpiresIn=self.expires, ) defaults = { "data-fields-%s" % key: value for key, value in response["fields"].items() } defaults["data-url"] = response["url"] defaults.update(attrs) try: defaults["class"] += " s3file" except KeyError: defaults["class"] = "s3file" return defaults def get_conditions(self, accept): conditions = [ {"bucket": self.bucket_name}, ["starts-with", "$key", str(self.upload_folder)], {"success_action_status": "201"}, ] if accept and "," not in accept: top_type, sub_type = accept.split("/", 1) if sub_type == "*": conditions.append(["starts-with", "$Content-Type", "%s/" % top_type]) else: conditions.append({"Content-Type": accept}) else: conditions.append(["starts-with", "$Content-Type", ""]) return conditions @cached_property def upload_folder(self): return str( pathlib.PurePosixPath( self.upload_path, base64.urlsafe_b64encode(uuid.uuid4().bytes) .decode("utf-8") .rstrip("=\n"), ) ) # S3 uses POSIX paths class Media: js = ("s3file/js/s3file.js" if settings.DEBUG else "s3file/js/s3file.min.js",)
def test_join_empty_string(self): path = utils.safe_join('base_url', '') self.assertEqual(path, 'base_url/')
def test_join_empty_string(self): path = utils.safe_join('base_url', '') self.assertEqual(path, 'base_url/')
def test_datetime_isoformat(self): dt = datetime.datetime(2017, 5, 19, 14, 45, 37, 123456) path = utils.safe_join('base_url', dt.isoformat()) self.assertEqual(path, 'base_url/2017-05-19T14:45:37.123456')
def test_trailing_slash_multi(self): """ Test safe_join with multiple paths that end with a trailing slash. """ path = utils.safe_join("base_url/", "path/to/", "somewhere/") self.assertEqual(path, "base_url/path/to/somewhere/")
def test_suspicious_operation(self): with self.assertRaises(ValueError): utils.safe_join("base", "../../../../../../../etc/passwd") with self.assertRaises(ValueError): utils.safe_join("base", "/etc/passwd")
def test_base_url_with_slash(self): path = utils.safe_join("base_url/", "path/to/somewhere") self.assertEqual(path, "base_url/path/to/somewhere")
def test_with_only_dot(self): path = utils.safe_join("", ".") self.assertEqual(path, "")
def test_with_base_url_join_nothing(self): path = utils.safe_join('base_url') self.assertEqual(path, 'base_url/')
def test_with_only_dot(self): path = utils.safe_join("", ".") self.assertEqual(path, "")
def test_with_base_url_join_nothing(self): path = utils.safe_join('base_url') self.assertEqual(path, 'base_url/')
def test_join_nothing(self): path = utils.safe_join('') self.assertEqual(path, '')
def _normalize_path(self, prefixed_path): # type: (str) -> str path = str(safe_join(self.remote_storage.location, prefixed_path)) return path.replace("\\", "")
def join(self, directory, filepath): return safe_join(directory, filepath)
def test_trailing_slash_multi(self): """ Test safe_join with multiple paths that end with a trailing slash. """ path = utils.safe_join("base_url/", "path/to/", "somewhere/") self.assertEqual(path, "base_url/path/to/somewhere/")
def _normalize_name(self, name): try: return safe_join(self.location, name) except ValueError: raise SuspiciousOperation("Attempted access to '%s' denied." % name)
def test_with_base_url_and_dot(self): path = utils.safe_join('base_url', '.') self.assertEqual(path, 'base_url/')
def test_with_base_url_and_dot_and_path_and_slash(self): path = utils.safe_join('base_url', '.', 'path/to/', '.') self.assertEqual(path, 'base_url/path/to/')
def test_with_base_url_and_dot_and_path_and_slash(self): path = utils.safe_join('base_url', '.', 'path/to/', '.') self.assertEqual(path, 'base_url/path/to/')
def test_with_dot(self): path = utils.safe_join("", "path/./somewhere/../other", "..", ".", "to/./somewhere") self.assertEqual(path, "path/to/somewhere")
def test_join_nothing(self): path = utils.safe_join('') self.assertEqual(path, '')
def test_suspicious_operation(self): with self.assertRaises(ValueError): utils.safe_join("base", "../../../../../../../etc/passwd") with self.assertRaises(ValueError): utils.safe_join("base", "/etc/passwd")
def test_normal(self): path = utils.safe_join("", "path/to/somewhere", "other", "path/to/somewhere") self.assertEqual(path, "path/to/somewhere/other/path/to/somewhere")
def test_datetime_isoformat(self): dt = datetime.datetime(2017, 5, 19, 14, 45, 37, 123456) path = utils.safe_join('base_url', dt.isoformat()) self.assertEqual(path, 'base_url/2017-05-19T14:45:37.123456')
def test_with_base_url_and_dot(self): path = utils.safe_join('base_url', '.') self.assertEqual(path, 'base_url/')