def setUp(self):
global store_upload
# Set storage backend to sftp storage
local_settings.STORAGES_BACKEND = \
'storages.backends.sftpstorage.SFTPStorage'
# Setting up a mock for the storage backend based on examples at
# https://docs.python.org/3.7/library/unittest.mock-examples.html# \
# applying-the-same-patch-to-every-test-method
patcher = patch('storages.backends.sftpstorage.SFTPStorage')
patcher.start()
self.addCleanup(patcher.stop)
# Mock the temporary storage object so that we're not unnecessarily
# saving files to the local file system.
patcher2 = patch('django.core.files.storage.FileSystemStorage')
patcher2.start()
self.addCleanup(patcher2.stop)
# Reinitialse stored upload after we changed the STORAGES_BACKEND
# class name above. This ensures that we're looking at the mocked
# backend class.
import django_drf_filepond.api
self.api = django_drf_filepond.api
django_drf_filepond.api.storage_backend_initialised = False
django_drf_filepond.api._init_storage_backend()
self.mock_storage_backend = django_drf_filepond.api.storage_backend
store_upload = django_drf_filepond.api.store_upload
# Check that we're using a mocked storage backend
self.assertTrue(
isinstance(django_drf_filepond.api.storage_backend, MagicMock),
('The created storage backend should be mocked but it is not of '
'type unittest.mock.MagicMock...'))
# Set up an initial file upload
self.upload_id = _get_file_id()
self.upload_id2 = _get_file_id()
self.file_id = _get_file_id()
self.file_content = ('This is some test file data for an '
'uploaded file.')
self.fn = 'my_test_file.txt'
self.test_target_filename = '/test_storage/testfile.txt'
self.test_target_filename2 = '/test_storage/testfile2.txt'
uploaded_file = MagicMock(spec=SimpleUploadedFile)
uploaded_file.name = self.fn
tu = TemporaryUpload(upload_id=self.upload_id,
file_id=self.file_id,
file=uploaded_file,
upload_name=self.fn,
upload_type=TemporaryUpload.FILE_DATA)
tu.save()
tu2 = TemporaryUpload(upload_id=self.upload_id2,
file_id=self.file_id,
file=uploaded_file,
upload_name=self.fn,
upload_type=TemporaryUpload.FILE_DATA)
tu2.save()
def setUp(self):
self.upload_id = _get_file_id()
self.file_id = _get_file_id()
self.upload_name = 'my_test_file.png'
self.uploader = FilepondChunkedFileUploader()
self.request = MagicMock(spec=Request)
self.request.user = AnonymousUser()
self.request.data = ensure_text(
'This is the test upload chunk data...')
def setUp(self):
self.file_id = _get_file_id()
self.upload_id = _get_file_id()
self.file_name = 'my_uploaded_file.txt'
self.request = MagicMock(spec=Request)
self.request.user = AnonymousUser()
file_obj = MagicMock(spec=InMemoryUploadedFile)
file_obj.name = self.file_name
self.request.data = _setupRequestData({'filepond': ['{}', file_obj]})
self.uploader = FilepondStandardFileUploader()
def setUp(self):
self.file_store_path = getattr(local_settings,
'FILE_STORE_PATH', None)
local_settings.STORAGES_BACKEND = \
'storages.backends.sftpstorage.SFTPStorage'
# Setting up a mock for the storage backend based on examples at
# https://docs.python.org/3.7/library/unittest.mock-examples.html# \
# applying-the-same-patch-to-every-test-method
patcher = patch('storages.backends.sftpstorage.SFTPStorage')
patcher.start()
self.addCleanup(patcher.stop)
# Mock the temporary storage object so that we're not unnecessarily
# saving files to the local file system.
patcher2 = patch('django.core.files.storage.FileSystemStorage')
patcher2.start()
self.addCleanup(patcher2.stop)
# Reinitialse stored upload after we changed the STORAGES_BACKEND
# class name above. This ensures that we're looking at the mocked
# backend class.
import django_drf_filepond.api
django_drf_filepond.api.storage_backend_initialised = False
django_drf_filepond.api._init_storage_backend()
self.mock_storage_backend = django_drf_filepond.api.storage_backend
# Set up an initial file upload
self.upload_id = _get_file_id()
self.file_id = _get_file_id()
self.file_content = (b'This is some test file data for an '
b'uploaded file.')
self.fn = 'my_test_file.txt'
self.test_filename = 'sdf5dua32defh754dhsrr2'
# Set up the mock_storage_backend.open to return the file content
# In relation to the changes for #31, file is now a FileField
# Since only a string is stored to the DB, the creation of the
# StoredUpload object below is fine but we need to mock the FileField
# object returned.
self.mock_storage_backend.open.return_value = BytesIO(
self.file_content)
# Override storage object configured for the FileField in StoredUpload
# at original init time since this happens before setUp is run.
StoredUpload.file.field.storage = self.mock_storage_backend
# Now set up a stored version of this upload
su = StoredUpload(upload_id=self.upload_id,
file=('%s' % (self.fn)),
uploaded=timezone.now())
su.file.storage = self.mock_storage_backend
su.save()
def setUp(self):
# Set up a database containing a mock file record
data = BytesIO()
data.write(os.urandom(16384))
self.base_upload_dir_at_startup = os.path.exists(storage.location)
self.file_id = _get_file_id()
self.upload_id = _get_file_id()
test_file = SimpleUploadedFile(self.file_id, data.read())
self.tu = TemporaryUpload.objects.create(
upload_id=self.upload_id,
file_id=self.file_id,
file=test_file,
upload_name='testfile.txt',
upload_type=TemporaryUpload.FILE_DATA)
self.tu.save()
def test_get_file_id(self):
fid = _get_file_id()
self.assertTrue(isinstance(fid, text_type),
'The file ID must be a string.')
id_format = re.compile('^([%s]){22}$' % (shortuuid.get_alphabet()))
self.assertRegex(fid, id_format, ('The generated ID does not match '
'the defined ID format.'))
def test_upload_id_valid(self):
# _get_file_id is currently used for getting both file+upload IDs
# since the spec for both is the same at present.
upload_id = _get_file_id()
self.assertTrue(
FilepondFileUploader._upload_id_valid(upload_id),
'A valid upload ID has been provided and this test should pass!')
def test_chunk_restart_invalid_id(self):
self._setup_tuc()
new_upload_id = _get_file_id()
res = self.uploader._handle_chunk_restart(self.request, new_upload_id)
res = self._prep_response(res)
self.assertContains(res, 'Invalid upload ID specified.',
status_code=404)
def test_file_id_wrong_data_type(self):
# Test using bytes instead of str
file_id = six.ensure_binary(_get_file_id())
self.assertFalse(
FilepondFileUploader._file_id_valid(file_id),
('The provided file ID is of the wrong data type, this test '
'should fail.'))
def test_patch_invalid_request_content_type(self):
chunk_id = _get_file_id()
req_url = reverse('patch', args=[chunk_id])
LOG.debug('About to run patch test with req_url: %s' % req_url)
response = self.client.patch(req_url, data='Some byte data'.encode(),
content_type='image/png')
self.assertContains(response, 'Unsupported media type',
status_code=415)
def setUp(self):
self.storage_backend = local_settings.STORAGES_BACKEND
# Set storage backend to sftp storage
local_settings.STORAGES_BACKEND = \
'storages.backends.sftpstorage.SFTPStorage'
# Setting up a mock for the storage backend based on examples at
# https://docs.python.org/3.7/library/unittest.mock-examples.html# \
# applying-the-same-patch-to-every-test-method
patcher = patch('storages.backends.sftpstorage.SFTPStorage')
patcher.start()
self.addCleanup(patcher.stop)
# Mock the temporary storage object so that we're not unnecessarily
# saving files to the local file system.
patcher2 = patch('django.core.files.storage.FileSystemStorage')
patcher2.start()
self.addCleanup(patcher2.stop)
# Reinitialse stored upload after we changed the STORAGES_BACKEND
# class name above. This ensures that we're looking at the mocked
# backend class.
import django_drf_filepond.api
self.api = django_drf_filepond.api
django_drf_filepond.api.storage_backend_initialised = False
django_drf_filepond.api._init_storage_backend()
self.mock_storage_backend = django_drf_filepond.api.storage_backend
self.delete_upload = django_drf_filepond.api.delete_stored_upload
# Check that we're using a mocked storage backend
self.assertTrue(
isinstance(django_drf_filepond.api.storage_backend, MagicMock),
('The created storage backend should be mocked but it is not of '
'type unittest.mock.MagicMock...'))
# Set up a "StoredUpload"
self.upload_id = _get_file_id()
self.file_id = _get_file_id()
# self.file_content = ('This is some test file data for an '
# 'uploaded file.')
self.fn = 'my_test_file.txt'
self.test_target_filepath = os.path.join('test_storage', self.fn)
self.su = StoredUpload(upload_id=self.upload_id,
file=self.test_target_filepath,
uploaded=timezone.now(),
stored=timezone.now())
self.su.save()
def post(self, request):
LOG.debug('Filepond API: Process view POST called...')
# Check that the temporary upload directory has been set
if not hasattr(local_settings, 'UPLOAD_TMP'):
return Response(
'The file upload path settings are not '
'configured correctly.',
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
# By default, enforce that the temporary upload location must be a
# sub-directory of the project base directory.
# TODO: Check whether this is necessary - maybe add a security
# parameter that can be disabled to turn off this check if the
# developer wishes?
if ((not (storage.location).startswith(local_settings.BASE_DIR))
and (local_settings.BASE_DIR != os.path.dirname(
django_drf_filepond.__file__))):
if not local_settings.ALLOW_EXTERNAL_UPLOAD_DIR:
return Response(
'The file upload path settings are not '
'configured correctly.',
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
# Check that a relative path is not being used to store the
# upload outside the specified UPLOAD_TMP directory.
if not getattr(local_settings, 'UPLOAD_TMP').startswith(
os.path.abspath(storage.location)):
return Response(
'An invalid storage location has been '
'specified.',
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
# Check that we've received a file and then generate a unique ID
# for it. Also generate a unique UD for the temp upload dir
file_id = _get_file_id()
upload_id = _get_file_id()
try:
uploader = FilepondFileUploader.get_uploader(request)
response = uploader.handle_upload(request, file_id, upload_id)
except ParseError as e:
# Re-raise the ParseError to trigger a 400 response via DRF.
raise e
return response
def test_file_id_clashes(self):
GENERATED_IDS = 5000
LOG.debug('Generating list of many file ids...')
file_ids = [_get_file_id() for _ in range(GENERATED_IDS)]
file_id_set = set(file_ids)
LOG.debug('File id list generated...')
self.assertEqual(len(file_id_set), GENERATED_IDS, 'There were '
'clashes in the generated file IDs!')
def test_upload_id_wrong_data_type(self):
# _get_file_id is currently used for getting both file+upload IDs
# since the spec for both is the same at present. Test using bytes
# instead of str
upload_id = _get_file_id().encode()
self.assertFalse(
FilepondFileUploader._upload_id_valid(upload_id),
('The provided upload ID is of the wrong data type, this test '
'should fail.'))
def test_store_upload_local_direct_missing_store_path(self):
fsp = local_settings.FILE_STORE_PATH
test_dir = '/tmp/%s' % _get_file_id()
local_settings.FILE_STORE_PATH = test_dir
with self.assertRaisesMessage(
FileNotFoundError,
'The local output directory [%s] defined by '
'FILE_STORE_PATH is missing.' % test_dir):
_store_upload_local('/test_storage', 'test_file.txt', None)
local_settings.FILE_STORE_PATH = fsp
def test_patch_valid_request(self, mock_hcu):
chunk_id = _get_file_id()
req_url = reverse('patch', args=[chunk_id])
LOG.debug('About to run patch test with req_url: %s' % req_url)
mock_hcu.return_value = Response(chunk_id, status=status.HTTP_200_OK,
content_type='text/plain')
response = self.client.patch(
req_url, data='Some byte data'.encode(),
content_type='application/offset+octet-stream')
self.assertContains(response, chunk_id, status_code=200)
def setUp(self):
# Set up an initial file upload
self.upload_id = _get_file_id()
self.file_id = _get_file_id()
self.file_content = ('This is some test file data for an '
'uploaded file.')
self.fn = 'my_test_file.txt'
self.test_target_filename = '/test_storage/testfile.txt'
self.su = StoredUpload(upload_id=self.upload_id,
file=self.test_target_filename[1:],
uploaded=timezone.now())
self.su.save()
# Create file
self.file_store_path = getattr(local_settings, 'FILE_STORE_PATH', None)
file_full_path = os.path.join(self.file_store_path,
self.test_target_filename[1:])
if not os.path.exists(os.path.dirname(file_full_path)):
os.mkdir(os.path.dirname(file_full_path))
with open(file_full_path, 'w') as f:
f.write(self.file_content)
def setUp(self):
# Set up an initial file upload
self.upload_id = _get_file_id()
self.file_id = _get_file_id()
self.file_content = ('This is some test file data for an '
'uploaded file.')
self.fn = 'my_test_file.txt'
self.test_filename = 'sdf5dua32defh754dhsrr2'
uploaded_file = SimpleUploadedFile(self.fn,
str.encode(self.file_content))
tu = TemporaryUpload(upload_id=self.upload_id,
file_id=self.file_id,
file=uploaded_file,
upload_name=self.fn,
upload_type=TemporaryUpload.FILE_DATA)
tu.save()
# Now set up a stored version of this upload
su = StoredUpload(upload_id=self.upload_id,
file=('%s' % (self.fn)),
uploaded=tu.uploaded)
su.save()
def test_head_request_valid(self, mock_hcr):
upload_id = _get_file_id()
req_url = reverse('patch', args=[upload_id])
mock_hcr.return_value = Response(
upload_id, status=status.HTTP_200_OK,
headers={'Upload-Offset': '100000'},
content_type='text/plain')
response = self.client.head(req_url)
self.assertEqual(response.status_code, 200)
self.assertTrue(
response.has_header('Upload-Offset'),
'Offset header is missing from chunk restart response.')
self.assertEqual(response['Upload-Offset'], '100000',
'Upload-Offset header has wrong value in response.')
def setUp(self):
# Set up an initial file upload
self.upload_id = _get_file_id()
self.upload_id2 = _get_file_id()
self.file_id = _get_file_id()
self.file_content = ('This is some test file data for an '
'uploaded file.')
self.fn = 'my_test_file.txt'
self.test_target_dirname = 'test_storage/'
self.test_target_filename = '/test_storage/testfile.txt'
self.test_target_filename2 = '/test_storage/testfile2.txt'
uploaded_file = SimpleUploadedFile(self.fn,
str.encode(self.file_content))
tu = TemporaryUpload(upload_id=self.upload_id,
file_id=self.file_id,
file=uploaded_file, upload_name=self.fn,
upload_type=TemporaryUpload.FILE_DATA)
tu.save()
tu2 = TemporaryUpload(upload_id=self.upload_id2,
file_id=self.file_id,
file=uploaded_file, upload_name=self.fn,
upload_type=TemporaryUpload.FILE_DATA)
tu2.save()
def head(self, request):
LOG.debug('Filepond API: Fetch view HEAD called...')
result = self._process_request(request)
if isinstance(result, tuple):
buf, file_id, upload_file_name, content_type = result
elif isinstance(result, Response):
return result
else:
raise ValueError('process_request result is of an unexpected type')
file_size = buf.seek(0, os.SEEK_END)
buf.seek(0)
# The addressing of filepond issue #154
# (https://github.com/pqina/filepond/issues/154) means that fetch
# can now store a file downloaded from a remote URL and return file
# metadata in the header if a HEAD request is received. If we get a
# GET request then the standard approach of proxying the file back
# to the client is used.
upload_id = _get_file_id()
memfile = InMemoryUploadedFile(buf, None, file_id, content_type,
file_size, None)
tu = TemporaryUpload(upload_id=upload_id,
file_id=file_id,
file=memfile,
upload_name=upload_file_name,
upload_type=TemporaryUpload.URL,
uploaded_by=_get_user(request))
tu.save()
response = Response(status=status.HTTP_200_OK)
response['Content-Type'] = content_type
response['Content-Length'] = file_size
response['X-Content-Transfer-Id'] = upload_id
response['Content-Disposition'] = ('inline; filename=%s' %
upload_file_name)
return response
def _process_request(self, request):
LOG.debug('Filepond API: Fetch view GET called...')
'''
Supports retrieving a file on the server side that the user has
specified by calling addFile on the filepond API and passing a
URL to a file.
'''
# Retrieve the target URL from the request query string target
# target parameter, pull the file into temp upload storage and
# return a file object.
# First check we have a URL and parse to check it's valid
target_url = request.query_params.get('target', None)
if not target_url:
raise ParseError('Required query parameter(s) missing.')
# Use Django's URL validator to see if we've been given a valid URL
validator = URLValidator(message=('An invalid URL <%s> has been '
'provided' % (target_url)))
try:
validator(target_url)
except ValidationError as e:
raise ParseError(str(e))
# TODO: SHould we check the headers returned when we request the
# download to see that we're getting a file rather than an HTML page?
# For now this check is enabled on the basis that we assume target
# data file will not be HTML. However, there should be a way to turn
# this off if the client knows that they want to get an HTML file.
# TODO: *** Looks like this use of head can be removed since with
# the new approach of streaming content to a BytesIO object, when
# stream=True, the connection begins by being opened and only
# fetching the headers. We could do this check then.
try:
header = requests.head(target_url, allow_redirects=True)
except ConnectionError as e:
msg = ('Unable to access the requested remote file headers: %s' %
str(e))
LOG.error(msg)
return Response(msg, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
if header.status_code == 404:
raise NotFound('The remote file was not found.')
content_type = header.headers.get('Content-Type', '')
# If the URL has returned URL content but an HTML file was not
# requested then assume that the URL has linked to a download page or
# some sort of error page or similar and raise an error.
if 'html' in content_type.lower() and '.html' not in target_url:
LOG.error('The requested data seems to be in HTML format. '
'Assuming this is not valid data file.')
raise ParseError('Provided URL links to HTML content.')
buf = BytesIO()
upload_file_name = None
try:
with requests.get(target_url, allow_redirects=True,
stream=True) as r:
if 'Content-Disposition' in r.headers:
cd = r.headers['Content-Disposition']
matches = re.findall('filename=(.+)', cd)
if len(matches):
upload_file_name = matches[0]
for chunk in r.iter_content(chunk_size=1048576):
buf.write(chunk)
except ConnectionError as e:
raise NotFound('Unable to access the requested remote file: %s' %
str(e))
file_id = _get_file_id()
# If filename wasn't extracted from Content-Disposition header, get
# from the URL or otherwise set it to the auto-generated file_id
if not upload_file_name:
if not target_url.endswith('/'):
split = target_url.rsplit('/', 1)
upload_file_name = split[1] if len(split) > 1 else split[0]
else:
upload_file_name = file_id
return (buf, file_id, upload_file_name, content_type)
def test_file_id_valid(self):
file_id = _get_file_id()
self.assertTrue(
FilepondFileUploader._file_id_valid(file_id),
'A valid file ID has been provided and this test should pass!')
def test_file_id(self):
file_id = _get_file_id()
self.assertEqual(len(file_id), 22,
'Incorrect length for generated file ID.')
def setUp(self):
self.project = prepare_project(self.task)
self.user = self.project.admin
self.data_path = pathlib.Path(__file__).parent / "data"
self.upload_id = _get_file_id()
