def setUp(self):
self.file_store_path = getattr(local_settings,
'FILE_STORE_PATH', None)
local_settings.STORAGES_BACKEND = \
'storages.backends.sftpstorage.SFTPStorage'
# Setting up a mock for the storage backend based on examples at
# https://docs.python.org/3.7/library/unittest.mock-examples.html# \
# applying-the-same-patch-to-every-test-method
patcher = patch('storages.backends.sftpstorage.SFTPStorage')
patcher.start()
self.addCleanup(patcher.stop)
# Mock the temporary storage object so that we're not unnecessarily
# saving files to the local file system.
patcher2 = patch('django.core.files.storage.FileSystemStorage')
patcher2.start()
self.addCleanup(patcher2.stop)
# Reinitialse stored upload after we changed the STORAGES_BACKEND
# class name above. This ensures that we're looking at the mocked
# backend class.
import django_drf_filepond.api
django_drf_filepond.api.storage_backend_initialised = False
django_drf_filepond.api._init_storage_backend()
self.mock_storage_backend = django_drf_filepond.api.storage_backend
# Set up an initial file upload
self.upload_id = _get_file_id()
self.file_id = _get_file_id()
self.file_content = (b'This is some test file data for an '
b'uploaded file.')
self.fn = 'my_test_file.txt'
self.test_filename = 'sdf5dua32defh754dhsrr2'
# Set up the mock_storage_backend.open to return the file content
# In relation to the changes for #31, file is now a FileField
# Since only a string is stored to the DB, the creation of the
# StoredUpload object below is fine but we need to mock the FileField
# object returned.
self.mock_storage_backend.open.return_value = BytesIO(
self.file_content)
# Override storage object configured for the FileField in StoredUpload
# at original init time since this happens before setUp is run.
StoredUpload.file.field.storage = self.mock_storage_backend
# Now set up a stored version of this upload
su = StoredUpload(upload_id=self.upload_id,
file=('%s' % (self.fn)),
uploaded=timezone.now())
su.file.storage = self.mock_storage_backend
su.save()
def _store_upload_local(destination_file_path, destination_file_name,
temp_upload):
file_path_base = local_settings.FILE_STORE_PATH
# If called via store_upload, this has already been checked but in
# case this is called directly, double check that the store path is set
if not file_path_base or file_path_base == '':
raise ValueError('The FILE_STORE_PATH is not set to a directory.')
# Is this necessary? Checking on every file storage in case the directory
# was removed but not sure that this is really necessary.
if ((not os.path.exists(file_path_base))
or (not os.path.isdir(file_path_base))):
raise FileNotFoundError(
'The local output directory [%s] defined by FILE_STORE_PATH is '
'missing.' % file_path_base)
if destination_file_path.startswith(os.sep):
destination_file_path = destination_file_path[1:]
target_dir = os.path.join(file_path_base, destination_file_path)
target_filename = destination_file_name
# If not filename provided, assume a directory was provided, get the
# file name from temp_upload and use this
if not target_filename:
target_filename = temp_upload.upload_name
destination_file_path = os.path.join(destination_file_path,
target_filename)
# Check we're not about to overwrite anything
target_file_path = os.path.join(target_dir, target_filename)
if os.path.exists(target_file_path):
LOG.error('File with specified name and path <%s> already exists' %
target_file_path)
raise FileExistsError('The specified temporary file cannot be stored'
' to the specified location - file exists.')
su = StoredUpload(upload_id=temp_upload.upload_id,
file=destination_file_path,
uploaded=temp_upload.uploaded,
uploaded_by=temp_upload.uploaded_by)
try:
if not os.path.exists(target_dir):
os.makedirs(target_dir)
shutil.copy2(temp_upload.get_file_path(), target_file_path)
su.save()
temp_upload.delete()
except IOError as e:
LOG.error('Error moving temporary file to permanent storage location')
raise e
return su
def setUp(self):
self.storage_backend = local_settings.STORAGES_BACKEND
# Set storage backend to sftp storage
local_settings.STORAGES_BACKEND = \
'storages.backends.sftpstorage.SFTPStorage'
# Setting up a mock for the storage backend based on examples at
# https://docs.python.org/3.7/library/unittest.mock-examples.html# \
# applying-the-same-patch-to-every-test-method
patcher = patch('storages.backends.sftpstorage.SFTPStorage')
patcher.start()
self.addCleanup(patcher.stop)
# Mock the temporary storage object so that we're not unnecessarily
# saving files to the local file system.
patcher2 = patch('django.core.files.storage.FileSystemStorage')
patcher2.start()
self.addCleanup(patcher2.stop)
# Reinitialse stored upload after we changed the STORAGES_BACKEND
# class name above. This ensures that we're looking at the mocked
# backend class.
import django_drf_filepond.api
self.api = django_drf_filepond.api
django_drf_filepond.api.storage_backend_initialised = False
django_drf_filepond.api._init_storage_backend()
self.mock_storage_backend = django_drf_filepond.api.storage_backend
self.delete_upload = django_drf_filepond.api.delete_stored_upload
# Check that we're using a mocked storage backend
self.assertTrue(
isinstance(django_drf_filepond.api.storage_backend, MagicMock),
('The created storage backend should be mocked but it is not of '
'type unittest.mock.MagicMock...'))
# Set up a "StoredUpload"
self.upload_id = _get_file_id()
self.file_id = _get_file_id()
# self.file_content = ('This is some test file data for an '
# 'uploaded file.')
self.fn = 'my_test_file.txt'
self.test_target_filepath = os.path.join('test_storage', self.fn)
self.su = StoredUpload(upload_id=self.upload_id,
file=self.test_target_filepath,
uploaded=timezone.now(),
stored=timezone.now())
self.su.save()
def setUp(self):
# Set up an initial file upload
self.upload_id = _get_file_id()
self.file_id = _get_file_id()
self.file_content = ('This is some test file data for an '
'uploaded file.')
self.fn = 'my_test_file.txt'
self.test_target_filename = '/test_storage/testfile.txt'
self.su = StoredUpload(upload_id=self.upload_id,
file=self.test_target_filename[1:],
uploaded=timezone.now())
self.su.save()
# Create file
self.file_store_path = getattr(local_settings, 'FILE_STORE_PATH', None)
file_full_path = os.path.join(self.file_store_path,
self.test_target_filename[1:])
if not os.path.exists(os.path.dirname(file_full_path)):
os.mkdir(os.path.dirname(file_full_path))
with open(file_full_path, 'w') as f:
f.write(self.file_content)
def setUp(self):
# Set up an initial file upload
self.upload_id = _get_file_id()
self.file_id = _get_file_id()
self.file_content = ('This is some test file data for an '
'uploaded file.')
self.fn = 'my_test_file.txt'
self.test_filename = 'sdf5dua32defh754dhsrr2'
uploaded_file = SimpleUploadedFile(self.fn,
str.encode(self.file_content))
tu = TemporaryUpload(upload_id=self.upload_id,
file_id=self.file_id,
file=uploaded_file,
upload_name=self.fn,
upload_type=TemporaryUpload.FILE_DATA)
tu.save()
# Now set up a stored version of this upload
su = StoredUpload(upload_id=self.upload_id,
file_path=('%s' % (self.fn)),
uploaded=tu.uploaded)
su.save()
def _store_upload_remote(destination_file_path, destination_file_name,
temp_upload):
# Use the storage backend to write the file to the storage backend
target_filename = destination_file_name
if not target_filename:
target_filename = temp_upload.upload_name
su = None
destination_file = os.path.join(destination_file_path, target_filename)
try:
storage_backend.save(destination_file, temp_upload.file)
su = StoredUpload(upload_id=temp_upload.upload_id,
file_path=destination_file,
uploaded=temp_upload.uploaded)
su.save()
temp_upload.delete()
except Exception as e:
errorMsg = ('Error storing temporary upload to remote storage: [%s]'
% str(e))
LOG.error(errorMsg)
raise e
return su
class ApiGetUploadTestCase(TestCase):
def setUp(self):
# Set up an initial file upload
self.upload_id = _get_file_id()
self.file_id = _get_file_id()
self.file_content = ('This is some test file data for an '
'uploaded file.')
self.fn = 'my_test_file.txt'
self.test_target_filename = '/test_storage/testfile.txt'
self.su = StoredUpload(upload_id=self.upload_id,
file=self.test_target_filename[1:],
uploaded=timezone.now())
self.su.save()
# Create file
self.file_store_path = getattr(local_settings, 'FILE_STORE_PATH', None)
file_full_path = os.path.join(self.file_store_path,
self.test_target_filename[1:])
if not os.path.exists(os.path.dirname(file_full_path)):
os.mkdir(os.path.dirname(file_full_path))
with open(file_full_path, 'w') as f:
f.write(self.file_content)
def test_store_upload_unset_file_store_path(self):
(filename, bytes_io) = get_stored_upload_file_data(self.su)
file_data = bytes_io.read().decode()
self.assertEqual(file_data, self.file_content,
'Returned file content not correct.')
self.assertEqual(filename, os.path.basename(self.test_target_filename),
'Returned file name is not correct.')
def test_get_local_stored_upload_no_filestore(self):
fsp = local_settings.FILE_STORE_PATH
local_settings.FILE_STORE_PATH = None
with self.assertRaisesMessage(
ConfigurationError,
'The file upload settings are not configured correctly.'):
get_stored_upload_file_data(self.su)
local_settings.FILE_STORE_PATH = fsp
def test_get_remote_stored_upload_data(self):
# Set up the mock_storage_backend.open to return the file content
mock_storage_backend = self._setup_mock_storage_backend()
mock_storage_backend.open.return_value = BytesIO(
self.file_content.encode())
mock_storage_backend.exists.return_value = True
(filename, bytes_io) = get_stored_upload_file_data(self.su)
file_data = bytes_io.read().decode()
local_settings.STORAGES_BACKEND = None
django_drf_filepond.api.storage_backend = None
self.assertEqual(file_data, self.file_content,
'Returned file content not correct.')
self.assertEqual(filename, os.path.basename(self.test_target_filename),
'Returned file name is not correct.')
def test_storage_backend_initialised_when_required(self):
mock_storage_backend = self._setup_mock_storage_backend()
mock_storage_backend.open.return_value = BytesIO(
self.file_content.encode())
mock_storage_backend.exists.return_value = True
with patch('django_drf_filepond.api._init_storage_backend') as m:
django_drf_filepond.api.storage_backend_initialised = False
get_stored_upload_file_data(self.su)
local_settings.STORAGES_BACKEND = None
django_drf_filepond.api.storage_backend = None
try:
m.assert_called_once()
# No assert_called_once on Python 3.5
except AttributeError:
self.assertEqual(
m.call_count, 1,
('Expected _init_storage_backend to be called once but '
'it has been called %s times.' % m.call_count))
def test_get_remote_upload_not_on_remote_store(self):
# File store path for remote testing should be ''
file_store_path = ''
mock_storage_backend = self._setup_mock_storage_backend()
mock_storage_backend.exists.return_value = False
file_path = os.path.join(file_store_path, self.su.file.name)
with self.assertRaisesMessage(
FileNotFoundError,
('File [%s] for upload_id [%s] not found on remote '
'file store.' % (file_path, self.su.upload_id))):
get_stored_upload_file_data(self.su)
local_settings.STORAGES_BACKEND = None
django_drf_filepond.api.storage_backend = None
def _setup_mock_storage_backend(self):
# Set storage backend to sftp storage
local_settings.STORAGES_BACKEND = \
'storages.backends.sftpstorage.SFTPStorage'
# Set up mocked objects for storage backend testing
# Setting up a mock for the storage backend based on examples at
# https://docs.python.org/3.7/library/unittest.mock-examples.html# \
# applying-the-same-patch-to-every-test-method
patcher = patch('storages.backends.sftpstorage.SFTPStorage')
patcher.start()
self.addCleanup(patcher.stop)
# Mock the temporary storage object so that we're not unnecessarily
# saving files to the local file system.
patcher2 = patch('django.core.files.storage.FileSystemStorage')
patcher2.start()
self.addCleanup(patcher2.stop)
# Set the backend initialisation flag to false to force re-init
django_drf_filepond.api.storage_backend_initialised = False
django_drf_filepond.api._init_storage_backend()
mock_storage_backend = django_drf_filepond.api.storage_backend
return mock_storage_backend
def tearDown(self):
# Delete stored upload
self.su.delete()
# Delete files
filestore_base = getattr(local_settings, 'FILE_STORE_PATH', None)
test_stored_file = os.path.join(filestore_base,
self.test_target_filename[1:])
test_target_dir = os.path.dirname(test_stored_file)
# Remove test_stored_file
if (os.path.exists(test_stored_file)
and os.path.isfile(test_stored_file)):
LOG.debug('Removing test stored file: <%s>' % test_stored_file)
os.remove(test_stored_file)
# Remove directory
if (os.path.exists(test_target_dir)
and os.path.isdir(test_target_dir)):
LOG.debug('Removing test target dir: <%s>' % test_target_dir)
os.rmdir(test_target_dir)
class ApiDeleteTestCase(TestCase):
def setUp(self):
self.storage_backend = local_settings.STORAGES_BACKEND
# Set storage backend to sftp storage
local_settings.STORAGES_BACKEND = \
'storages.backends.sftpstorage.SFTPStorage'
# Setting up a mock for the storage backend based on examples at
# https://docs.python.org/3.7/library/unittest.mock-examples.html# \
# applying-the-same-patch-to-every-test-method
patcher = patch('storages.backends.sftpstorage.SFTPStorage')
patcher.start()
self.addCleanup(patcher.stop)
# Mock the temporary storage object so that we're not unnecessarily
# saving files to the local file system.
patcher2 = patch('django.core.files.storage.FileSystemStorage')
patcher2.start()
self.addCleanup(patcher2.stop)
# Reinitialse stored upload after we changed the STORAGES_BACKEND
# class name above. This ensures that we're looking at the mocked
# backend class.
import django_drf_filepond.api
self.api = django_drf_filepond.api
django_drf_filepond.api.storage_backend_initialised = False
django_drf_filepond.api._init_storage_backend()
self.mock_storage_backend = django_drf_filepond.api.storage_backend
self.delete_upload = django_drf_filepond.api.delete_stored_upload
# Check that we're using a mocked storage backend
self.assertTrue(
isinstance(django_drf_filepond.api.storage_backend, MagicMock),
('The created storage backend should be mocked but it is not of '
'type unittest.mock.MagicMock...'))
# Set up a "StoredUpload"
self.upload_id = _get_file_id()
self.file_id = _get_file_id()
# self.file_content = ('This is some test file data for an '
# 'uploaded file.')
self.fn = 'my_test_file.txt'
self.test_target_filepath = os.path.join('test_storage', self.fn)
self.su = StoredUpload(upload_id=self.upload_id,
file=self.test_target_filepath,
uploaded=timezone.now(),
stored=timezone.now())
self.su.save()
def test_delete_stored_upload_local(self):
# Make an API call to delete the stored upload and check that delete
# is called on the FileSystemStorage backend.
# Set the storage backend to None to force use of local storage.
local_settings.STORAGES_BACKEND = None
self.api.storage_backend_initialised = False
# Get the target file path that we want to check that delete has been
# called with. This is the file path from the DB plus the base file
# system storage location.
base_store_location = local_settings.FILE_STORE_PATH
file_location = os.path.join(base_store_location,
self.test_target_filepath)
with patch('os.remove') as os_patcher:
with patch('os.path.exists') as exists:
with patch('os.path.isfile') as isfile:
exists.return_value = True
isfile.return_value = True
self.delete_upload(self.upload_id, delete_file=True)
with self.assertRaises(StoredUpload.DoesNotExist):
StoredUpload.objects.get(upload_id=self.upload_id)
os_patcher.assert_called_once_with(file_location)
def test_delete_stored_upload_remote(self):
self.mock_storage_backend.exists.return_value = True
self.delete_upload(self.upload_id, delete_file=True)
self.mock_storage_backend.delete.assert_called_once_with(
self.test_target_filepath)
def test_delete_stored_upload_local_keepfile(self):
# Make an API call to delete the stored upload and check that delete
# is called on the FileSystemStorage backend.
# Set the storage backend to None to force use of local storage.
local_settings.STORAGES_BACKEND = None
self.api.storage_backend_initialised = False
with patch('os.remove') as os_patcher:
with patch('os.path.exists') as exists:
with patch('os.path.isdir') as isdir:
exists.return_value = True
isdir.return_value = True
self.delete_upload(self.upload_id, delete_file=False)
with self.assertRaises(StoredUpload.DoesNotExist):
StoredUpload.objects.get(upload_id=self.upload_id)
os_patcher.assert_not_called()
exists.assert_not_called()
isdir.assert_not_called()
def test_delete_stored_upload_remote_keepfile(self):
self.delete_upload(self.upload_id, delete_file=False)
self.mock_storage_backend.exists.assert_not_called()
self.mock_storage_backend.delete.assert_not_called()
def test_delete_stored_upload_invalid_id(self):
test_id = 'abcdefghijklmnopqrstuv'
('No stored upload found with the specified ID [%s].' % (test_id))
with self.assertRaisesMessage(
StoredUpload.DoesNotExist,
'StoredUpload matching query does not exist.'):
self.delete_upload(test_id)
def test_delete_stored_upload_local_no_filestore_path(self):
fsp = local_settings.FILE_STORE_PATH
local_settings.FILE_STORE_PATH = None
# Need to set storage backend to None and make it reinitialise to
# ensure that we're not using a remote backend for this test.
local_settings.STORAGES_BACKEND = None
self.api.storage_backend_initialised = False
with self.assertRaisesMessage(
ConfigurationError,
'The file upload settings are not configured correctly.'):
self.delete_upload(self.upload_id, delete_file=True)
local_settings.FILE_STORE_PATH = fsp
def test_delete_stored_upload_local_not_exists(self):
# Need to set storage backend to None and make it reinitialise to
# ensure that we're not using a remote backend for this test.
local_settings.STORAGES_BACKEND = None
self.api.storage_backend_initialised = False
with patch('os.path.exists') as exists:
with patch('os.path.isdir') as isdir:
exists.return_value = False
isdir.return_value = True
with self.assertRaisesMessage(
ConfigurationError,
('The file upload settings are not configured '
'correctly.')):
self.delete_upload(self.upload_id, delete_file=True)
def test_delete_stored_upload_local_not_isdir(self):
# Need to set storage backend to None and make it reinitialise to
# ensure that we're not using a remote backend for this test.
local_settings.STORAGES_BACKEND = None
self.api.storage_backend_initialised = False
with patch('os.path.exists') as exists:
with patch('os.path.isdir') as isdir:
exists.return_value = True
isdir.return_value = False
with self.assertRaisesMessage(
ConfigurationError,
('The file upload settings are not configured '
'correctly.')):
self.delete_upload(self.upload_id, delete_file=True)
def test_delete_stored_upload_remote_file_missing(self):
self.mock_storage_backend.exists.return_value = False
with self.assertRaisesMessage(
FileNotFoundError,
('File [%s] for stored upload with id [%s] not found on '
'remote file store.' %
(self.test_target_filepath, self.upload_id))):
self.delete_upload(self.upload_id, delete_file=True)
def test_delete_stored_upload_local_file_missing(self):
local_settings.STORAGES_BACKEND = None
self.api.storage_backend_initialised = False
target_filepath = os.path.join(local_settings.FILE_STORE_PATH,
self.test_target_filepath)
with patch('os.path.exists') as exists:
exists.side_effect = [True, False]
with self.assertRaisesMessage(
FileNotFoundError,
('File [%s] to delete was not found on the local disk' %
(target_filepath))):
self.delete_upload(self.upload_id, delete_file=True)
def test_delete_stored_upload_local_remove_fails(self):
# Make an API call to delete the stored upload and check that delete
# is called on the FileSystemStorage backend.
# Set the storage backend to None to force use of local storage.
local_settings.STORAGES_BACKEND = None
self.api.storage_backend_initialised = False
# Get the target file path that we want to check that delete has been
# called with. This is the file path from the DB plus the base file
# system storage location.
base_store_location = local_settings.FILE_STORE_PATH
file_location = os.path.join(base_store_location,
self.test_target_filepath)
with patch('os.remove') as os_patcher:
with patch('os.path.exists') as exists:
with patch('os.path.isfile') as isfile:
exists.return_value = True
isfile.return_value = True
os_patcher.side_effect = OSError('Error deleting file')
with self.assertRaisesMessage(OSError,
'Error deleting file'):
self.delete_upload(self.upload_id, delete_file=True)
os_patcher.assert_called_once_with(file_location)
def tearDown(self):
local_settings.STORAGES_BACKEND = self.storage_backend
self.su.delete()
self.api.storage_backend_initialised = False
def store_upload(upload_id, destination_file_path):
if ((not hasattr(local_settings, 'FILE_STORE_PATH'))
or (not local_settings.FILE_STORE_PATH)
or (not os.path.exists(local_settings.FILE_STORE_PATH))
or (not os.path.isdir(local_settings.FILE_STORE_PATH))):
raise ImproperlyConfigured('A required setting is missing in your '
'application configuration.')
file_path_base = local_settings.FILE_STORE_PATH
if not file_path_base or file_path_base == '':
raise ValueError('The FILE_STORE_PATH is not set to a directory.')
id_fmt = re.compile('^([%s]){22}$' % (shortuuid.get_alphabet()))
if not id_fmt.match(upload_id):
LOG.error('The provided upload ID <%s> is of an invalid format.' %
upload_id)
raise ValueError('The provided upload ID is of an invalid format.')
if not destination_file_path or destination_file_path == '':
raise ValueError('No destination file path provided.')
if destination_file_path.startswith(os.sep):
destination_file_path = destination_file_path[1:]
try:
tu = TemporaryUpload.objects.get(upload_id=upload_id)
except TemporaryUpload.DoesNotExist:
raise ValueError('Record for the specified upload_id doesn\'t exist')
target_dir = os.path.join(file_path_base,
os.path.dirname(destination_file_path))
if destination_file_path.endswith(os.sep):
# Assume a directory was provided, get the file name from tu and
# add this to the provided path.
destination_file_path += tu.upload_name
target_file_path = os.path.join(file_path_base, destination_file_path)
# Check we're not about to overwrite anything
if os.path.exists(target_file_path):
LOG.error('File with specified name and path <%s> already exists' %
destination_file_path)
raise FileExistsError('The specified temporary file cannot be stored'
' to the specified location - file exists.')
su = StoredUpload(upload_id=tu.upload_id,
file_path=destination_file_path,
uploaded=tu.uploaded)
try:
if not os.path.exists(target_dir):
os.makedirs(target_dir)
shutil.copy2(tu.get_file_path(), target_file_path)
su.save()
tu.delete()
except IOError as e:
LOG.error('Error moving temporary file to permanent storage location')
raise e
return su
