def get_available_name(self, name, max_length=None):
"""
Return a filename that's free on the target storage system and
available for new content to be written to.
"""
dir_name, file_name = os.path.split(name)
if '..' in pathlib.PurePath(dir_name).parts:
raise SuspiciousFileOperation(
"Detected path traversal attempt in '%s'" % dir_name)
validate_file_name(file_name)
file_root, file_ext = os.path.splitext(file_name)
# If the filename already exists, generate an alternative filename
# until it doesn't exist.
# Truncate original name if required, so the new filename does not
# exceed the max_length.
while self.exists(name) or (max_length and len(name) > max_length):
# file_ext includes the dot.
name = os.path.join(dir_name,
self.get_alternative_name(file_root, file_ext))
if max_length is None:
continue
# Truncate file_root if max_length exceeded.
truncation = len(name) - max_length
if truncation > 0:
file_root = file_root[:-truncation]
# Entire file_root was truncated in attempt to find an available filename.
if not file_root:
raise SuspiciousFileOperation(
'Storage can not find an available filename for "%s". '
'Please make sure that the corresponding file field '
'allows sufficient "max_length".' % name)
name = os.path.join(
dir_name, self.get_alternative_name(file_root, file_ext))
return name
def safe_join(base, *paths):
"""
Based on django/utils/_os/safe_join
Modified to include dataregistry functionality
"""
final_path = abspath(join(base, *paths))
base_path = abspath(base)
# Validate with MEDIA_ROOT
if (not normcase(final_path).startswith(normcase(base_path + sep))
and normcase(final_path) != normcase(base_path)
and dirname(normcase(base_path)) != normcase(base_path)):
if settings.DATAREGISTRY_MEDIA_ROOT:
# Validate with DATAREGISTRY_MEDIA_ROOT
data_registry_path = abspath(settings.DATAREGISTRY_MEDIA_ROOT)
if settings.DATAREGISTRY_ALLOW_SYMLINKS and os.path.islink(
data_registry_path):
data_registry_path = abspath(
dataregistry_path_resolver(data_registry_path))
if (not normcase(final_path).startswith(
normcase(data_registry_path + sep))
and normcase(final_path) != normcase(data_registry_path)
and dirname(normcase(data_registry_path)) !=
normcase(data_registry_path)):
raise SuspiciousFileOperation(
"The joined path is located outside of the allowed locations"
)
else:
return final_path
raise SuspiciousFileOperation(
f"The joined path ({final_path}) is located outside of the base path component ({base_path})"
)
return final_path
def irods_path_is_allowed(path):
""" paths containing '/../' are suspicious """
if path == "":
raise ValidationError("Empty file paths are not allowed")
if '/../' in path:
raise SuspiciousFileOperation("File paths cannot contain '/../'")
if '/./' in path:
raise SuspiciousFileOperation("File paths cannot contain '/./'")
def validate_form_path(base_path, path):
try:
if path.resolve() < base_path.resolve():
raise SuspiciousFileOperation(
"Staff user is attempting forbidden filesystem traversal.")
except RuntimeError:
raise SuspiciousFileOperation(
"Staff user tried to trick form path validator in an infinite loop"
)
def validate_file_name(name):
if name != os.path.basename(name):
raise SuspiciousFileOperation("File name '%s' includes path elements" %
name)
# Remove potentially dangerous names
if name in {'', '.', '..'}:
raise SuspiciousFileOperation("Could not derive file name from '%s'" %
name)
return name
def _get_available_name(self, name, max_length=None):
# Returns a filename that is available in the storage mechanism,
# possibly taking the provided filename into account.
# The name argument passed to this method will have already cleaned to a filename valid for the storage system, according to the get_valid_name() method described above.
dir_name, file_name = os.path.split(name)
file_root, file_ext = os.path.splitext(file_name)
while self.exists(name) or (max_length and len(name) > max_length):
# file_ext includes the dot.
name = os.path.join(
dir_name,
"%s_%s%s" % (file_root, get_random_string(7), file_ext))
if max_length is None:
continue
# Truncate file_root if max_length exceeded.
truncation = len(name) - max_length
if truncation > 0:
file_root = file_root[:-truncation]
# Entire file_root was truncated in attempt to find an available filename.
if not file_root:
raise SuspiciousFileOperation(
'Storage can not find an available filename for "%s". '
'Please make sure that the corresponding file field '
'allows sufficient "max_length".' % name)
name = os.path.join(
dir_name,
"%s_%s%s" % (file_root, get_random_string(7), file_ext))
return name
def get_available_name(self, name, max_length=None):
"""
Returns a filename that's free on the target storage system, and
available for new content to be written to.
"""
dir_name, file_name = os.path.split(name)
file_root, file_ext = os.path.splitext(file_name)
# If the filename already exists, add an underscore and a random 7
# character alphanumeric string (before the file extension, if one
# exists) to the filename until the generated filename doesn't exist.
# Truncate original name if required, so the new filename does not
# exceed the max_length.
while self.exists(name) or (max_length and len(name) > max_length):
# file_ext includes the dot.
name = os.path.join(
dir_name,
"%s_%s%s" % (file_root, get_random_string(7), file_ext))
if max_length is None:
continue
# Truncate file_root if max_length exceeded.
truncation = len(name) - max_length
if truncation > 0:
file_root = file_root[:-truncation]
# Entire file_root was truncated in attempt to find an available filename.
if not file_root:
raise SuspiciousFileOperation(
'Storage can not find an available filename for "%s". '
'Please make sure that the corresponding file field '
'allows sufficient "max_length".' % name)
name = os.path.join(
dir_name,
"%s_%s%s" % (file_root, get_random_string(7), file_ext))
return name
def safe_join(base, *paths):
"""
Join one or more path components to the base.css path component intelligently.
Return a normalized, absolute version of the final path.
Raise ValueError if the final path isn't located inside of the base.css path
component.
"""
base = force_text(base)
paths = [force_text(p) for p in paths]
final_path = abspath(join(base, *paths))
base_path = abspath(base)
# Ensure final_path starts with base_path (using normcase to ensure we
# don't false-negative on case insensitive operating systems like Windows),
# further, one of the following conditions must be true:
# a) The next character is the path separator (to prevent conditions like
# safe_join("/dir", "/../d"))
# b) The final path must be the same as the base.css path.
# c) The base.css path must be the most root path (meaning either "/" or "C:\\")
if (not normcase(final_path).startswith(normcase(base_path + sep))
and normcase(final_path) != normcase(base_path)
and dirname(normcase(base_path)) != normcase(base_path)):
raise SuspiciousFileOperation(
'The joined path ({}) is located outside of the base.css path '
'component ({})'.format(final_path, base_path))
return final_path
def get_available_name(self, name, max_length=None):
"""
Return a filename that's free on the target storage system and
available for new content to be written to.
Arguments:
name(string): The prospective available name for the resource.
max_length(int): The acceptable length of the filename. Defaults to None.
Returns:
string: The availabl resource name for use.
"""
if max_length is None:
return name
# Truncate name if max_length exceeded.
truncation = len(name) - max_length
if truncation > 0:
name = name[:-truncation]
# Entire name was truncated in attempt to find an available filename.
if not name:
raise SuspiciousFileOperation(
'Storage can not find an available filename for "%s". '
'Please make sure that the corresponding file field '
'allows sufficient "max_length".' % name)
return name
def _find_available_name(self, instance, name):
max_length = self.max_length
dir_name, file_name = os.path.split(name)
file_root, file_ext = os.path.splitext(file_name)
while self._variations_collapsed(instance, name) or (
max_length and len(name) > max_length
):
name = os.path.join(
dir_name, "%s_%s%s" % (file_root, get_random_string(7), file_ext)
)
if max_length is None:
continue
# Truncate file_root if max_length exceeded.
truncation = len(name) - max_length
if truncation > 0:
file_root = file_root[:-truncation]
if not file_root:
raise SuspiciousFileOperation(
'Storage cannot find an available filename for "%s". '
'Please make sure the corresponding file field '
'allows sufficient "max_length".' % name
)
name = os.path.join(
dir_name, "%s_%s%s" % (file_root, get_random_string(7), file_ext)
)
return name
def uploaded_path(self, name):
try:
path = safe_join(self.location, name)
except ValueError:
raise SuspiciousFileOperation("Attempted access to '%s' denied." %
name)
return os.path.normpath(path)
def get_available_name(self, name, max_length=None):
if max_length and len(name) + 15 > max_length:
raise SuspiciousFileOperation(
'Storage can not find an available filename for "%s". '
"Please make sure that the corresponding file field "
'allows sufficient "max_length".' % name)
return name
def get_available_name(self,
bucket,
key,
max_length=32,
force_name_change=False):
file_root, file_ext = os.path.splitext(key)
while (force_name_change or self.key_exists(bucket, key)
or (max_length and len(key) > max_length)):
force_name_change = False
# file_ext includes the dot.
key = "%s_%s%s" % (file_root, get_random_string(7), file_ext)
if max_length is None:
continue
# Truncate file_root if max_length exceeded.
truncation = len(key) - max_length
if truncation > 0:
file_root = file_root[:-truncation]
# Entire file_root was truncated in attempt to find an available filename.
if not file_root:
raise SuspiciousFileOperation(
'Storage can not find an available filename for "%s". '
"Please make sure that the corresponding file field "
'allows sufficient "max_length".' % key)
key = "%s_%s%s" % (file_root, get_random_string(7), file_ext)
return key
def delete(self, request, *args, **kwargs):
print("hahaha", self.get_object(), type(self.get_object()))
soal = self.get_object()
if soal.guru.username.username != request.user.username:
raise SuspiciousFileOperation("Sory, Operasi ini dilarang !")
else:
return super(SoalDeleteView, self).delete(request, *args, **kwargs)
def validate_file_name(name, allow_relative_path=False):
# Remove potentially dangerous names
if os.path.basename(name) in {'', '.', '..'}:
raise SuspiciousFileOperation("Could not derive file name from '%s'" % name)
if allow_relative_path:
# Use PurePosixPath() because this branch is checked only in
# FileField.generate_filename() where all file paths are expected to be
# Unix style (with forward slashes).
path = pathlib.PurePosixPath(name)
if path.is_absolute() or '..' in path.parts:
raise SuspiciousFileOperation(
"Detected path traversal attempt in '%s'" % name
)
elif name != os.path.basename(name):
raise SuspiciousFileOperation("File name '%s' includes path elements" % name)
return name
def generate_filename(self, filename):
"""
Validate the filename by calling get_valid_name() and return a filename
to be passed to the save() method.
"""
# `filename` may include a path as returned by FileField.upload_to.
dirname, filename = os.path.split(filename)
if '..' in pathlib.PurePath(dirname).parts:
raise SuspiciousFileOperation("Detected path traversal attempt in '%s'" % dirname)
return os.path.normpath(os.path.join(dirname, self.get_valid_name(filename)))
def path_traversal_check(unsafe_path, known_safe_path):
known_safe_path = os.path.abspath(known_safe_path)
unsafe_path = os.path.abspath(unsafe_path)
if (os.path.commonprefix([known_safe_path, unsafe_path]) !=
known_safe_path):
raise SuspiciousFileOperation("{} is not safe".format(unsafe_path))
# Passes the check
return unsafe_path
def get_available_name(self, name, max_length=None):
if max_length and len(name) > max_length:
raise SuspiciousFileOperation(
'Storage can not find an available filename for "%s". '
'Please make sure that the corresponding file field '
'allows sufficient "max_length".' % name)
full_path = self.path(name)
if self.exists(full_path):
os.remove(full_path)
return name
def walk(self, top):
if top in ('', '/'):
raise SuspiciousFileOperation(
'Iterating all storage cannot be right')
log.debug('Walking %s in media storage', top)
folders, files = self.listdir(self._dirpath(top))
yield top, folders, files
for folder_name in folders:
if folder_name:
# Recursively walk the subdirectory
yield from self.walk(self.join(top, folder_name))
def get_valid_filename(name):
"""
Return the given string converted to a string that can be used for a clean
filename. Remove leading and trailing spaces; convert other spaces to
underscores; and remove anything that is not an alphanumeric, dash,
underscore, or dot.
>>> get_valid_filename("john's portrait in 2004.jpg")
'johns_portrait_in_2004.jpg'
"""
s = str(name).strip().replace(" ", "_")
s = re.sub(r"(?u)[^-\w.]", "", s)
if s in {"", ".", ".."}:
raise SuspiciousFileOperation("Could not derive file name from '%s'" % name)
return s
def get(self, request, *args, **kwargs):
resource = self.get_object()
# Does a valid associated agreement exist for this resource?
associated_agreement = (Agreement.objects.filter(
resource=resource).valid().order_by('-created').first())
if associated_agreement is None:
raise Http404('Unable to find valid and unhidden agreement.')
# Has the user signed that agreement?
try:
associated_agreement.signature_set.filter(
signatory=self.request.user).get()
except Signature.DoesNotExist:
messages.error(
self.request,
f'You must sign this agreement before accessing files associated with {resource.name}.'
)
# Attach some data to the session so that we can redirect back to this request later
request.session['access_attempt'] = (resource.slug,
self.kwargs['accesspath'])
return redirect(associated_agreement)
# Access is granted, is the access path a file or a directory?
resource_scoped_path = os.path.join(resource.slug,
self.kwargs['accesspath'])
try:
if '..' in resource_scoped_path:
raise SuspiciousFileOperation()
self.path = default_storage.path(resource_scoped_path) # pylint: disable=attribute-defined-outside-init
if not os.path.exists(self.path):
raise Http404('File or directory not found at access path.')
if os.path.isfile(self.path):
FileDownloadEvent.objects.get_or_create_if_no_duplicates_past_5_minutes(
resource, self.kwargs['accesspath'],
request.session.session_key)
return sendfile(request, self.path)
except SuspiciousFileOperation as suspicious_file_operation_error:
raise PermissionDenied('SuspiciousFileOperation on file access.'
) from suspicious_file_operation_error
# Redirect if the file listing doesn't end in a slash
if self.kwargs['accesspath'] != '' and self.kwargs['accesspath'][
-1] != '/':
return redirect(
reverse_lazy(
'resources_access',
args=[resource.slug, self.kwargs['accesspath'] + '/']))
# Render a file listing to the user.
return super().get(request, *args, **kwargs)
def assign_illustration(img_url, post, save=True):
"""
Create an Illustration instance out of preview picture
in media/tmp directory.
:param img_url: relative url for an image in media/tmp directory
:param post: BlogPost which the image is illustrating
:param save: if True saves instance to database
:return: created Illustration instance
"""
if not img_url.startswith(f'{settings.MEDIA_URL}tmp/'):
raise SuspiciousFileOperation(f'Invalid url: {img_url}')
illustration = Illustration(post=post)
illustration.picture.save_existing(img_url, save)
return illustration
def get_upload_to(self, filename):
folder_name = 'original_videos'
filename = self.file.field.storage.get_valid_name(filename)
max_length = self._meta.get_field('file').max_length
# Truncate filename so it fits in the 100 character limit
# https://code.djangoproject.com/ticket/9893
file_path = os.path.join(folder_name, filename)
too_long = len(file_path) - max_length
if too_long > 0:
head, ext = os.path.splitext(filename)
if too_long > len(head) + 1:
raise SuspiciousFileOperation('File name can not be shortened to a safe length')
filename = head[:-too_long] + ext
return os.path.join(folder_name, filename)
def unzip_archive(label, zip_ref):
common_prefix = os.path.commonprefix(zip_ref.namelist())
if not common_prefix:
raise SuspiciousFileOperation(
"The zip archive {} is not packed correctly".format(label))
icon_font_root = app_settings.CMSPLUGIN_CASCADE['icon_font_root']
try:
try:
os.makedirs(icon_font_root)
except os.error:
pass # the directory exists already
temp_folder = tempfile.mkdtemp(prefix='', dir=icon_font_root)
for member in zip_ref.infolist():
zip_ref.extract(member, temp_folder)
font_folder = os.path.join(temp_folder, common_prefix)
# this is specific to fontello
with io.open(os.path.join(font_folder, 'config.json'), 'r') as fh:
config_data = json.load(fh)
except Exception as exc:
shutil.rmtree(temp_folder, ignore_errors=True)
raise SuspiciousFileOperation(
"Can not unzip uploaded archive {}: {}".format(label, exc))
return os.path.relpath(font_folder, icon_font_root), config_data
def get_available_overwrite_name(name, max_length):
if max_length is None or len(name) <= max_length:
return name
# Adapted from Django
dir_name, file_name = os.path.split(name)
file_root, file_ext = os.path.splitext(file_name)
truncation = len(name) - max_length
file_root = file_root[:-truncation]
if not file_root:
raise SuspiciousFileOperation(
'Storage tried to truncate away entire filename "%s". '
'Please make sure that the corresponding file field '
'allows sufficient "max_length".' % name)
return os.path.join(dir_name, "{}{}".format(file_root, file_ext))
def path(self, name):
from django.db import connection
from django.utils._os import safe_join
# FIXME: These imports are inline so that the connection object
# can be mocked in tests
if connection.tenant:
location = safe_join(settings.TENANT_BASE,
connection.tenant.schema_name)
else:
location = self.location
try:
path = safe_join(location, name)
except ValueError:
raise SuspiciousFileOperation("Attempted access to '%s' denied." %
name)
return os.path.normpath(path)
def safe_join(base, path):
base = force_str(base).replace("\\", "/").lstrip("/").rstrip("/") + "/"
path = force_str(path).replace("\\", "/").lstrip("/")
# Ugh... there must be a better way that I can't think of right now
if base == "/":
base = ""
resolved_url = urllib.parse.urljoin(base, path)
resolved_url = re.sub("//+", "/", resolved_url)
if not resolved_url.startswith(base):
raise SuspiciousFileOperation(
'The joined path ({}) is located outside of the base path '
'component ({})'.format(resolved_url, base))
return resolved_url
def sync_directory(self, source, destination):
"""
Sync a directory recursively to storage.
Overwrites files in remote storage with files from ``source`` (no timstamp/hash checking).
Removes files and folders in remote storage that are not present in ``source``.
:param source: the source path on the local disk
:param destination: the destination path in storage
"""
if destination in ('', '/'):
raise SuspiciousFileOperation(
'Syncing all storage cannot be right')
log.debug(
'Syncing to media storage. source=%s destination=%s',
source,
destination,
)
source = Path(source)
copied_files = set()
copied_dirs = set()
for filepath in source.iterdir():
sub_destination = self.join(destination, filepath.name)
if filepath.is_dir():
# Recursively sync the subdirectory
self.sync_directory(filepath, sub_destination)
copied_dirs.add(filepath.name)
elif filepath.is_file():
with filepath.open('rb') as fd:
self.save(sub_destination, fd)
copied_files.add(filepath.name)
# Remove files that are not present in ``source``
dest_folders, dest_files = self.listdir(self._dirpath(destination))
for folder in dest_folders:
if folder not in copied_dirs:
self.delete_directory(self.join(destination, folder))
for filename in dest_files:
if filename not in copied_files:
filepath = self.join(destination, filename)
log.debug('Deleting file from media storage. file=%s',
filepath)
self.delete(filepath)
def get_available_name(self, name, max_length=None):
dir_name, file_name = os.path.split(name)
file_root, file_ext = os.path.splitext(file_name)
while self.exists(name) or (max_length and len(name) > max_length):
name = os.path.join(dir_name,
self.get_alternative_name(file_root, file_ext))
if max_length is None:
continue
truncation = len(name) - max_length
if truncation > 0:
file_root = file_root[:-truncation]
if not file_root:
raise SuspiciousFileOperation(
'Storage can not find an available filename for "%s". '
"Please make sure that the corresponding file field "
'allows sufficient "max_length".' % name)
name = os.path.join(
dir_name, self.get_alternative_name(file_root, file_ext))
return name
def get_available_name(self, name, max_length=None):
"""
Modified version of django.core.files.storage.Storage class's get_available_name() method.
Reference: https://github.com/django/django/blob/master/django/core/files/storage.py .
Added a file duplication marker for the purpose of getting the original file name later.
..warning:: If this is changed, so should be the get_original_file_name's implementation.
"""
dir_name, file_name = os.path.split(name)
file_root, file_ext = os.path.splitext(file_name)
file_duplication_marker = settings.FILE_DUPLICATION_MARKER
# If the filename already exists, the following is done to get a unique filename:
# 1. Add a duplicate marker (this shows that there was file name duplication).
# Append it to original filename.
# 2. Generate a random string of length RANDOM_STRING_LEN, and append it.
# Repeat the random string generation, until a unique name is achieved.
# Truncate original name if required, so the new filename does not
# exceed the max_length.
while self.exists(name) or (max_length and len(name) > max_length):
# file_ext includes the dot.
name = os.path.join(
dir_name, "%s_%s_%s%s" %
(file_root, file_duplication_marker,
get_random_string(self.RANDOM_STRING_LEN), file_ext))
if max_length is None:
continue
# Truncate file_root if max_length exceeded.
truncation = len(name) - max_length
if truncation > 0:
file_root = file_root[:-truncation]
# Entire file_root was truncated in attempt to find an available filename.
if not file_root:
raise SuspiciousFileOperation(
'Storage can not find an available filename for "%s". '
'Please make sure that the corresponding file field '
'allows sufficient "max_length".' % name)
name = os.path.join(
dir_name, "%s_%s_%s%s" %
(file_root, file_duplication_marker,
get_random_string(self.RANDOM_STRING_LEN), file_ext))
return name
