def get_available_name(self, name, max_length=None):
"""
Return a filename that's free on the target storage system and
available for new content to be written to.
"""
dir_name, file_name = os.path.split(name)
file_root, file_ext = os.path.splitext(file_name)
# If the filename already exists, add an underscore and a random 7
# character alphanumeric string (before the file extension, if one
# exists) to the filename until the generated filename doesn't exist.
# Truncate original name if required, so the new filename does not
# exceed the max_length.
while self.exists(name) or (max_length and len(name) > max_length):
# file_ext includes the dot.
name = os.path.join(
dir_name,
"%s_%s%s" % (file_root, get_random_string(7), file_ext))
if max_length is None:
continue
# Truncate file_root if max_length exceeded.
truncation = len(name) - max_length
if truncation > 0:
file_root = file_root[:-truncation]
# Entire file_root was truncated in attempt to find an available filename.
if not file_root:
raise SuspiciousFileOperation(
'Storage can not find an available filename for "%s". '
'Please make sure that the corresponding file field '
'allows sufficient "max_length".' % name)
name = os.path.join(
dir_name,
"%s_%s%s" % (file_root, get_random_string(7), file_ext))
return name
def path(self, name):
try:
path = safe_join(self.location, name)
except ValueError:
raise SuspiciousFileOperation("Attempted access to '%s' denied." %
name)
return os.path.normpath(path)
def get_available_name(self, name, max_length=None):
if max_length and len(name) + 15 > max_length:
raise SuspiciousFileOperation(
'Storage can not find an available filename for "%s". '
"Please make sure that the corresponding file field "
'allows sufficient "max_length".' % name)
return name
def safe_join(base, *paths):
"""
Join one or more path components to the base path component intelligently.
Return a normalized, absolute version of the final path.
Raise ValueError if the final path isn't located inside of the base path
component.
"""
base = force_text(base)
paths = [force_text(p) for p in paths]
final_path = abspath(join(base, *paths))
base_path = abspath(base)
# Ensure final_path starts with base_path (using normcase to ensure we
# don't false-negative on case insensitive operating systems like Windows),
# further, one of the following conditions must be true:
# a) The next character is the path separator (to prevent conditions like
# safe_join("/dir", "/../d"))
# b) The final path must be the same as the base path.
# c) The base path must be the most root path (meaning either "/" or "C:\\")
if (not normcase(final_path).startswith(normcase(base_path + sep))
and normcase(final_path) != normcase(base_path)
and dirname(normcase(base_path)) != normcase(base_path)):
raise SuspiciousFileOperation(
'The joined path ({}) is located outside of the base path '
'component ({})'.format(final_path, base_path))
return final_path
def _find_available_name(self, instance, name):
max_length = self.max_length
dir_name, file_name = os.path.split(name)
file_root, file_ext = os.path.splitext(file_name)
while self._variations_collapsed(instance, name) or (
max_length and len(name) > max_length
):
name = os.path.join(
dir_name, "%s_%s%s" % (file_root, get_random_string(7), file_ext)
)
if max_length is None:
continue
# Truncate file_root if max_length exceeded.
truncation = len(name) - max_length
if truncation > 0:
file_root = file_root[:-truncation]
if not file_root:
raise SuspiciousFileOperation(
'Storage cannot find an available filename for "%s". '
'Please make sure the corresponding file field '
'allows sufficient "max_length".' % name
)
name = os.path.join(
dir_name, "%s_%s%s" % (file_root, get_random_string(7), file_ext)
)
return name
def delete(self, request, *args, **kwargs):
print("hahaha", self.get_object(), type(self.get_object()))
soal = self.get_object()
if soal.guru.username.username != request.user.username:
raise SuspiciousFileOperation("Sory, Operasi ini dilarang !")
else:
return super(SoalDeleteView, self).delete(request, *args, **kwargs)
def validate_file_name(name, allow_relative_path=False):
# Remove potentially dangerous names
if os.path.basename(name) in {'', '.', '..'}:
raise SuspiciousFileOperation("Could not derive file name from '%s'" % name)
if allow_relative_path:
# Use PurePosixPath() because this branch is checked only in
# FileField.generate_filename() where all file paths are expected to be
# Unix style (with forward slashes).
path = pathlib.PurePosixPath(name)
if path.is_absolute() or '..' in path.parts:
raise SuspiciousFileOperation(
"Detected path traversal attempt in '%s'" % name
)
elif name != os.path.basename(name):
raise SuspiciousFileOperation("File name '%s' includes path elements" % name)
return name
def path_traversal_check(unsafe_path, known_safe_path):
known_safe_path = os.path.abspath(known_safe_path)
unsafe_path = os.path.abspath(unsafe_path)
if (os.path.commonprefix([known_safe_path, unsafe_path]) !=
known_safe_path):
raise SuspiciousFileOperation("{} is not safe".format(unsafe_path))
# Passes the check
return unsafe_path
def generate_filename(self, filename):
"""
Validate the filename by calling get_valid_name() and return a filename
to be passed to the save() method.
"""
# `filename` may include a path as returned by FileField.upload_to.
dirname, filename = os.path.split(filename)
if '..' in pathlib.PurePath(dirname).parts:
raise SuspiciousFileOperation("Detected path traversal attempt in '%s'" % dirname)
return os.path.normpath(os.path.join(dirname, self.get_valid_name(filename)))
def get_available_name(self, name, max_length=None):
if max_length and len(name) > max_length:
raise SuspiciousFileOperation(
'Storage can not find an available filename for "%s". '
'Please make sure that the corresponding file field '
'allows sufficient "max_length".' % name)
full_path = self.path(name)
if self.exists(full_path):
os.remove(full_path)
return name
def get_available_name(self, name, max_length=None):
"""
Return a filename that's free on the target storage system and
available for new content to be written to.
"""
name = str(name).replace("\\", "/")
dir_name, file_name = os.path.split(name)
if ".." in pathlib.PurePath(dir_name).parts:
raise SuspiciousFileOperation(
"Detected path traversal attempt in '%s'" % dir_name
)
validate_file_name(file_name)
file_root, file_ext = os.path.splitext(file_name)
# If the filename already exists, generate an alternative filename
# until it doesn't exist.
# Truncate original name if required, so the new filename does not
# exceed the max_length.
while self.exists(name) or (max_length and len(name) > max_length):
# file_ext includes the dot.
name = os.path.join(
dir_name, self.get_alternative_name(file_root, file_ext)
)
if max_length is None:
continue
# Truncate file_root if max_length exceeded.
truncation = len(name) - max_length
if truncation > 0:
file_root = file_root[:-truncation]
# Entire file_root was truncated in attempt to find an
# available filename.
if not file_root:
raise SuspiciousFileOperation(
'Storage can not find an available filename for "%s". '
"Please make sure that the corresponding file field "
'allows sufficient "max_length".' % name
)
name = os.path.join(
dir_name, self.get_alternative_name(file_root, file_ext)
)
return name
def get_valid_filename(name):
"""
Return the given string converted to a string that can be used for a clean
filename. Remove leading and trailing spaces; convert other spaces to
underscores; and remove anything that is not an alphanumeric, dash,
underscore, or dot.
>>> get_valid_filename("john's portrait in 2004.jpg")
'johns_portrait_in_2004.jpg'
"""
s = str(name).strip().replace(" ", "_")
s = re.sub(r"(?u)[^-\w.]", "", s)
if s in {"", ".", ".."}:
raise SuspiciousFileOperation("Could not derive file name from '%s'" % name)
return s
def walk(self, top):
if top in ('', '/'):
raise SuspiciousFileOperation(
'Iterating all storage cannot be right')
log.debug('Walking %s in media storage', top)
folders, files = self.listdir(self._dirpath(top))
yield top, folders, files
for folder_name in folders:
if folder_name:
# Recursively walk the subdirectory
yield from self.walk(self.join(top, folder_name))
def get(self, request, *args, **kwargs):
resource = self.get_object()
# Does a valid associated agreement exist for this resource?
associated_agreement = (Agreement.objects.filter(
resource=resource).valid().order_by('-created').first())
if associated_agreement is None:
raise Http404('Unable to find valid and unhidden agreement.')
# Has the user signed that agreement?
try:
associated_agreement.signature_set.filter(
signatory=self.request.user).get()
except Signature.DoesNotExist:
messages.error(
self.request,
f'You must sign this agreement before accessing files associated with {resource.name}.'
)
# Attach some data to the session so that we can redirect back to this request later
request.session['access_attempt'] = (resource.slug,
self.kwargs['accesspath'])
return redirect(associated_agreement)
# Access is granted, is the access path a file or a directory?
resource_scoped_path = os.path.join(resource.slug,
self.kwargs['accesspath'])
try:
if '..' in resource_scoped_path:
raise SuspiciousFileOperation()
self.path = default_storage.path(resource_scoped_path) # pylint: disable=attribute-defined-outside-init
if not os.path.exists(self.path):
raise Http404('File or directory not found at access path.')
if os.path.isfile(self.path):
FileDownloadEvent.objects.get_or_create_if_no_duplicates_past_5_minutes(
resource, self.kwargs['accesspath'],
request.session.session_key)
return sendfile(request, self.path)
except SuspiciousFileOperation as suspicious_file_operation_error:
raise PermissionDenied('SuspiciousFileOperation on file access.'
) from suspicious_file_operation_error
# Redirect if the file listing doesn't end in a slash
if self.kwargs['accesspath'] != '' and self.kwargs['accesspath'][
-1] != '/':
return redirect(
reverse_lazy(
'resources_access',
args=[resource.slug, self.kwargs['accesspath'] + '/']))
# Render a file listing to the user.
return super().get(request, *args, **kwargs)
def get_upload_to(self, filename):
folder_name = 'original_videos'
filename = self.file.field.storage.get_valid_name(filename)
max_length = self._meta.get_field('file').max_length
# Truncate filename so it fits in the 100 character limit
# https://code.djangoproject.com/ticket/9893
file_path = os.path.join(folder_name, filename)
too_long = len(file_path) - max_length
if too_long > 0:
head, ext = os.path.splitext(filename)
if too_long > len(head) + 1:
raise SuspiciousFileOperation('File name can not be shortened to a safe length')
filename = head[:-too_long] + ext
return os.path.join(folder_name, filename)
def unzip_archive(label, zip_ref):
common_prefix = os.path.commonprefix(zip_ref.namelist())
if not common_prefix:
raise SuspiciousFileOperation(
"The zip archive {} is not packed correctly".format(label))
icon_font_root = app_settings.CMSPLUGIN_CASCADE['icon_font_root']
try:
try:
os.makedirs(icon_font_root)
except os.error:
pass # the directory exists already
temp_folder = tempfile.mkdtemp(prefix='', dir=icon_font_root)
for member in zip_ref.infolist():
zip_ref.extract(member, temp_folder)
font_folder = os.path.join(temp_folder, common_prefix)
# this is specific to fontello
with io.open(os.path.join(font_folder, 'config.json'), 'r') as fh:
config_data = json.load(fh)
except Exception as exc:
shutil.rmtree(temp_folder, ignore_errors=True)
raise SuspiciousFileOperation(
"Can not unzip uploaded archive {}: {}".format(label, exc))
return os.path.relpath(font_folder, icon_font_root), config_data
def assign_illustration(img_url, post, save=True):
"""
Create an Illustration instance out of preview picture
in media/tmp directory.
:param img_url: relative url for an image in media/tmp directory
:param post: BlogPost which the image is illustrating
:param save: if True saves instance to database
:return: created Illustration instance
"""
if not img_url.startswith(f'{settings.MEDIA_URL}tmp/'):
raise SuspiciousFileOperation(f'Invalid url: {img_url}')
illustration = Illustration(post=post)
illustration.picture.save_existing(img_url, save)
return illustration
def get_available_overwrite_name(name, max_length):
if max_length is None or len(name) <= max_length:
return name
# Adapted from Django
dir_name, file_name = os.path.split(name)
file_root, file_ext = os.path.splitext(file_name)
truncation = len(name) - max_length
file_root = file_root[:-truncation]
if not file_root:
raise SuspiciousFileOperation(
'Storage tried to truncate away entire filename "%s". '
'Please make sure that the corresponding file field '
'allows sufficient "max_length".' % name)
return os.path.join(dir_name, "{}{}".format(file_root, file_ext))
def path(self, name):
from django.db import connection
from django.utils._os import safe_join
# FIXME: These imports are inline so that the connection object
# can be mocked in tests
if connection.tenant:
location = safe_join(settings.TENANT_BASE,
connection.tenant.schema_name)
else:
location = self.location
try:
path = safe_join(location, name)
except ValueError:
raise SuspiciousFileOperation("Attempted access to '%s' denied." %
name)
return os.path.normpath(path)
def safe_join(base, path):
base = force_str(base).replace("\\", "/").lstrip("/").rstrip("/") + "/"
path = force_str(path).replace("\\", "/").lstrip("/")
# Ugh... there must be a better way that I can't think of right now
if base == "/":
base = ""
resolved_url = urllib.parse.urljoin(base, path)
resolved_url = re.sub("//+", "/", resolved_url)
if not resolved_url.startswith(base):
raise SuspiciousFileOperation(
'The joined path ({}) is located outside of the base path '
'component ({})'.format(resolved_url, base))
return resolved_url
def sync_directory(self, source, destination):
"""
Sync a directory recursively to storage.
Overwrites files in remote storage with files from ``source`` (no timstamp/hash checking).
Removes files and folders in remote storage that are not present in ``source``.
:param source: the source path on the local disk
:param destination: the destination path in storage
"""
if destination in ('', '/'):
raise SuspiciousFileOperation(
'Syncing all storage cannot be right')
log.debug(
'Syncing to media storage. source=%s destination=%s',
source,
destination,
)
source = Path(source)
copied_files = set()
copied_dirs = set()
for filepath in source.iterdir():
sub_destination = self.join(destination, filepath.name)
if filepath.is_dir():
# Recursively sync the subdirectory
self.sync_directory(filepath, sub_destination)
copied_dirs.add(filepath.name)
elif filepath.is_file():
with filepath.open('rb') as fd:
self.save(sub_destination, fd)
copied_files.add(filepath.name)
# Remove files that are not present in ``source``
dest_folders, dest_files = self.listdir(self._dirpath(destination))
for folder in dest_folders:
if folder not in copied_dirs:
self.delete_directory(self.join(destination, folder))
for filename in dest_files:
if filename not in copied_files:
filepath = self.join(destination, filename)
log.debug('Deleting file from media storage. file=%s',
filepath)
self.delete(filepath)
def get_available_name(self, name, max_length=None):
dir_name, file_name = os.path.split(name)
file_root, file_ext = os.path.splitext(file_name)
while self.exists(name) or (max_length and len(name) > max_length):
name = os.path.join(dir_name,
self.get_alternative_name(file_root, file_ext))
if max_length is None:
continue
truncation = len(name) - max_length
if truncation > 0:
file_root = file_root[:-truncation]
if not file_root:
raise SuspiciousFileOperation(
'Storage can not find an available filename for "%s". '
"Please make sure that the corresponding file field "
'allows sufficient "max_length".' % name)
name = os.path.join(
dir_name, self.get_alternative_name(file_root, file_ext))
return name
def get_available_name(self, name, max_length=None):
"""
Modified version of django.core.files.storage.Storage class's get_available_name() method.
Reference: https://github.com/django/django/blob/master/django/core/files/storage.py .
Added a file duplication marker for the purpose of getting the original file name later.
..warning:: If this is changed, so should be the get_original_file_name's implementation.
"""
dir_name, file_name = os.path.split(name)
file_root, file_ext = os.path.splitext(file_name)
file_duplication_marker = settings.FILE_DUPLICATION_MARKER
# If the filename already exists, the following is done to get a unique filename:
# 1. Add a duplicate marker (this shows that there was file name duplication).
# Append it to original filename.
# 2. Generate a random string of length RANDOM_STRING_LEN, and append it.
# Repeat the random string generation, until a unique name is achieved.
# Truncate original name if required, so the new filename does not
# exceed the max_length.
while self.exists(name) or (max_length and len(name) > max_length):
# file_ext includes the dot.
name = os.path.join(
dir_name, "%s_%s_%s%s" %
(file_root, file_duplication_marker,
get_random_string(self.RANDOM_STRING_LEN), file_ext))
if max_length is None:
continue
# Truncate file_root if max_length exceeded.
truncation = len(name) - max_length
if truncation > 0:
file_root = file_root[:-truncation]
# Entire file_root was truncated in attempt to find an available filename.
if not file_root:
raise SuspiciousFileOperation(
'Storage can not find an available filename for "%s". '
'Please make sure that the corresponding file field '
'allows sufficient "max_length".' % name)
name = os.path.join(
dir_name, "%s_%s_%s%s" %
(file_root, file_duplication_marker,
get_random_string(self.RANDOM_STRING_LEN), file_ext))
return name
def get_available_name(self, name, max_length=None):
dir_name, file_name = os.path.split(name)
file_root, file_ext = os.path.splitext(file_name)
while self.exists(name) or (max_length and len(name) > max_length):
date = datetime.now().strftime("%d-%m-%Y_%H-%M-%S")
name = os.path.join(dir_name,
"%s_%s%s" % (file_root, date, file_ext))
if max_length is None:
continue
truncation = len(name) - max_length
if truncation > 0:
file_root = file_root[:-truncation]
if not file_root:
raise SuspiciousFileOperation(
'Storage can not find an available filename for "%s". '
'Please make sure that the corresponding file field '
'allows sufficient "max_length".' % name)
name = os.path.join(dir_name,
"%s_%s%s" % (file_root, date, file_ext))
return name
def get_available_name(self, bucket, key, max_length=32, force_name_change=False):
file_root, file_ext = os.path.splitext(key)
while force_name_change or self.key_exists(bucket, key) or \
(max_length and len(key) > max_length):
force_name_change = False
# file_ext includes the dot.
key = "%s_%s%s" % (file_root, get_random_string(7), file_ext)
if max_length is None:
continue
# Truncate file_root if max_length exceeded.
truncation = len(key) - max_length
if truncation > 0:
file_root = file_root[:-truncation]
# Entire file_root was truncated in attempt to find an available filename.
if not file_root:
raise SuspiciousFileOperation(
'Storage can not find an available filename for "%s". '
'Please make sure that the corresponding file field '
'allows sufficient "max_length".' % key
)
key = "%s_%s%s" % (file_root, get_random_string(7), file_ext)
return key
def determineSubmissionType(submission_code):
submittedLocation = os.path.join(save_location, 'anonymous',
submission_code)
submittedDirectory = os.listdir(submittedLocation)
for i in submittedDirectory:
if i.startswith('.'):
continue
elif os.path.isdir(os.path.join(submittedLocation, i)):
continue
else:
if i.endswith('.pdb') or i.endswith('.ent'):
return 'PDB'
elif i.endswith('tgz') or i.endswith('tar.gz'):
with tarfile.open(os.path.join(submittedLocation, i)) as tfile:
tfileFileNames = tfile.getnames()
for name in tfileFileNames:
if (name.startswith('/') or '..' in name):
raise SuspiciousFileOperation(
'non-relative file name {} in tar file'.format(
name))
memopsImplementatonPath = os.path.join(
'memops', 'Implementation')
if any(
[memopsImplementatonPath in f
for f in tfileFileNames]):
return 'CCPN'
pdb = any(['.pdb' in f for f in tfileFileNames])
upl = any(['.upl' in f for f in tfileFileNames])
aco = any(['.aco' in f for f in tfileFileNames])
if all((pdb, upl, aco)):
return 'CYANA'
logger.warn(
'Failed to determine project type for {}'.format(submission_code))
return 'unknown'
def delete_directory(self, path):
"""
Delete all files under a certain path from storage
Many storage backends (S3, Azure storage) don't care about "directories".
The directory effectively doesn't exist if there are no files in it.
However, in these backends, there is no "rmdir" operation so you have to recursively
delete all files.
:param path: the path to the directory to remove
"""
if path in ('', '/'):
raise SuspiciousFileOperation('Deleting all storage cannot be right')
log.debug('Deleting directory %s from media storage', path)
folders, files = self.listdir(self._dirpath(path))
for folder_name in folders:
if folder_name:
# Recursively delete the subdirectory
self.delete_directory(safe_join(path, folder_name))
for filename in files:
if filename:
self.delete(safe_join(path, filename))
def get_available_name(self,
name: str,
max_length: Optional[int] = 100) -> str:
"""
Return a filename (based on the name parameter) that is available on Mega.
If a file with name already exists, get_alternative_name() is called
to obtain an alternative name.
https://docs.djangoproject.com/en/3.1/ref/files/storage/#django.core.files.storage.Storage.get_available_name
"""
# TODO
dir_name, file_name = os.path.split(name)
file_root, file_ext = os.path.splitext(file_name)
# If the filename already exists, generate an alternative filename
# until it doesn't exist.
# Truncate original name if required, so the new filename does not
# exceed the max_length.
while self.exists(name) or (max_length and len(name) > max_length):
# file_ext includes the dot.
name = os.path.join(dir_name,
self.get_alternative_name(file_root, file_ext))
if max_length is None:
continue
# Truncate file_root if max_length exceeded.
truncation = len(name) - max_length
if truncation > 0:
file_root = file_root[:-truncation]
# Entire file_root was truncated in attempt to find an available filename.
if not file_root:
raise SuspiciousFileOperation(
'Storage can not find an available filename for "%s". '
'Please make sure that the corresponding file field '
'allows sufficient "max_length".' % name)
name = os.path.join(
dir_name, self.get_alternative_name(file_root, file_ext))
return name
def _get_available_name(self, name, max_length=None):
dir_name, file_name = os.path.split(name)
file_root, file_ext = os.path.splitext(file_name)
while self.exists(name) or (max_length and len(name) > max_length):
# file_ext includes the dot.
name = os.path.join(
dir_name,
"%s_%s%s" % (file_root, get_random_string(7), file_ext))
if max_length is None:
continue
# Truncate file_root if max_length exceeded.
truncation = len(name) - max_length
if truncation > 0:
file_root = file_root[:-truncation]
# Entire file_root was truncated in attempt to find an available filename.
if not file_root:
raise SuspiciousFileOperation(
'Storage can not find an available filename for "%s". '
'Please make sure that the corresponding file field '
'allows sufficient "max_length".' % name)
name = os.path.join(
dir_name,
"%s_%s%s" % (file_root, get_random_string(7), file_ext))
return name
def get_available_name(self, name, max_length=None):
ret = name.replace('\\', '/')
if max_length and len(ret) > max_length:
raise SuspiciousFileOperation()
return ret
