def setUp(self): self.file_store_path = getattr(local_settings, 'FILE_STORE_PATH', None) local_settings.STORAGES_BACKEND = \ 'storages.backends.sftpstorage.SFTPStorage' # Setting up a mock for the storage backend based on examples at # https://docs.python.org/3.7/library/unittest.mock-examples.html# \ # applying-the-same-patch-to-every-test-method patcher = patch('storages.backends.sftpstorage.SFTPStorage') patcher.start() self.addCleanup(patcher.stop) # Mock the temporary storage object so that we're not unnecessarily # saving files to the local file system. patcher2 = patch('django.core.files.storage.FileSystemStorage') patcher2.start() self.addCleanup(patcher2.stop) # Reinitialse stored upload after we changed the STORAGES_BACKEND # class name above. This ensures that we're looking at the mocked # backend class. import django_drf_filepond.api django_drf_filepond.api.storage_backend_initialised = False django_drf_filepond.api._init_storage_backend() self.mock_storage_backend = django_drf_filepond.api.storage_backend # Set up an initial file upload self.upload_id = _get_file_id() self.file_id = _get_file_id() self.file_content = (b'This is some test file data for an ' b'uploaded file.') self.fn = 'my_test_file.txt' self.test_filename = 'sdf5dua32defh754dhsrr2' # Set up the mock_storage_backend.open to return the file content # In relation to the changes for #31, file is now a FileField # Since only a string is stored to the DB, the creation of the # StoredUpload object below is fine but we need to mock the FileField # object returned. self.mock_storage_backend.open.return_value = BytesIO( self.file_content) # Override storage object configured for the FileField in StoredUpload # at original init time since this happens before setUp is run. StoredUpload.file.field.storage = self.mock_storage_backend # Now set up a stored version of this upload su = StoredUpload(upload_id=self.upload_id, file=('%s' % (self.fn)), uploaded=timezone.now()) su.file.storage = self.mock_storage_backend su.save()
def _store_upload_local(destination_file_path, destination_file_name, temp_upload): file_path_base = local_settings.FILE_STORE_PATH # If called via store_upload, this has already been checked but in # case this is called directly, double check that the store path is set if not file_path_base or file_path_base == '': raise ValueError('The FILE_STORE_PATH is not set to a directory.') # Is this necessary? Checking on every file storage in case the directory # was removed but not sure that this is really necessary. if ((not os.path.exists(file_path_base)) or (not os.path.isdir(file_path_base))): raise FileNotFoundError( 'The local output directory [%s] defined by FILE_STORE_PATH is ' 'missing.' % file_path_base) if destination_file_path.startswith(os.sep): destination_file_path = destination_file_path[1:] target_dir = os.path.join(file_path_base, destination_file_path) target_filename = destination_file_name # If not filename provided, assume a directory was provided, get the # file name from temp_upload and use this if not target_filename: target_filename = temp_upload.upload_name destination_file_path = os.path.join(destination_file_path, target_filename) # Check we're not about to overwrite anything target_file_path = os.path.join(target_dir, target_filename) if os.path.exists(target_file_path): LOG.error('File with specified name and path <%s> already exists' % target_file_path) raise FileExistsError('The specified temporary file cannot be stored' ' to the specified location - file exists.') su = StoredUpload(upload_id=temp_upload.upload_id, file=destination_file_path, uploaded=temp_upload.uploaded, uploaded_by=temp_upload.uploaded_by) try: if not os.path.exists(target_dir): os.makedirs(target_dir) shutil.copy2(temp_upload.get_file_path(), target_file_path) su.save() temp_upload.delete() except IOError as e: LOG.error('Error moving temporary file to permanent storage location') raise e return su
def setUp(self): self.storage_backend = local_settings.STORAGES_BACKEND # Set storage backend to sftp storage local_settings.STORAGES_BACKEND = \ 'storages.backends.sftpstorage.SFTPStorage' # Setting up a mock for the storage backend based on examples at # https://docs.python.org/3.7/library/unittest.mock-examples.html# \ # applying-the-same-patch-to-every-test-method patcher = patch('storages.backends.sftpstorage.SFTPStorage') patcher.start() self.addCleanup(patcher.stop) # Mock the temporary storage object so that we're not unnecessarily # saving files to the local file system. patcher2 = patch('django.core.files.storage.FileSystemStorage') patcher2.start() self.addCleanup(patcher2.stop) # Reinitialse stored upload after we changed the STORAGES_BACKEND # class name above. This ensures that we're looking at the mocked # backend class. import django_drf_filepond.api self.api = django_drf_filepond.api django_drf_filepond.api.storage_backend_initialised = False django_drf_filepond.api._init_storage_backend() self.mock_storage_backend = django_drf_filepond.api.storage_backend self.delete_upload = django_drf_filepond.api.delete_stored_upload # Check that we're using a mocked storage backend self.assertTrue( isinstance(django_drf_filepond.api.storage_backend, MagicMock), ('The created storage backend should be mocked but it is not of ' 'type unittest.mock.MagicMock...')) # Set up a "StoredUpload" self.upload_id = _get_file_id() self.file_id = _get_file_id() # self.file_content = ('This is some test file data for an ' # 'uploaded file.') self.fn = 'my_test_file.txt' self.test_target_filepath = os.path.join('test_storage', self.fn) self.su = StoredUpload(upload_id=self.upload_id, file=self.test_target_filepath, uploaded=timezone.now(), stored=timezone.now()) self.su.save()
def setUp(self): # Set up an initial file upload self.upload_id = _get_file_id() self.file_id = _get_file_id() self.file_content = ('This is some test file data for an ' 'uploaded file.') self.fn = 'my_test_file.txt' self.test_target_filename = '/test_storage/testfile.txt' self.su = StoredUpload(upload_id=self.upload_id, file=self.test_target_filename[1:], uploaded=timezone.now()) self.su.save() # Create file self.file_store_path = getattr(local_settings, 'FILE_STORE_PATH', None) file_full_path = os.path.join(self.file_store_path, self.test_target_filename[1:]) if not os.path.exists(os.path.dirname(file_full_path)): os.mkdir(os.path.dirname(file_full_path)) with open(file_full_path, 'w') as f: f.write(self.file_content)
def setUp(self): # Set up an initial file upload self.upload_id = _get_file_id() self.file_id = _get_file_id() self.file_content = ('This is some test file data for an ' 'uploaded file.') self.fn = 'my_test_file.txt' self.test_filename = 'sdf5dua32defh754dhsrr2' uploaded_file = SimpleUploadedFile(self.fn, str.encode(self.file_content)) tu = TemporaryUpload(upload_id=self.upload_id, file_id=self.file_id, file=uploaded_file, upload_name=self.fn, upload_type=TemporaryUpload.FILE_DATA) tu.save() # Now set up a stored version of this upload su = StoredUpload(upload_id=self.upload_id, file_path=('%s' % (self.fn)), uploaded=tu.uploaded) su.save()
def _store_upload_remote(destination_file_path, destination_file_name, temp_upload): # Use the storage backend to write the file to the storage backend target_filename = destination_file_name if not target_filename: target_filename = temp_upload.upload_name su = None destination_file = os.path.join(destination_file_path, target_filename) try: storage_backend.save(destination_file, temp_upload.file) su = StoredUpload(upload_id=temp_upload.upload_id, file_path=destination_file, uploaded=temp_upload.uploaded) su.save() temp_upload.delete() except Exception as e: errorMsg = ('Error storing temporary upload to remote storage: [%s]' % str(e)) LOG.error(errorMsg) raise e return su
class ApiGetUploadTestCase(TestCase): def setUp(self): # Set up an initial file upload self.upload_id = _get_file_id() self.file_id = _get_file_id() self.file_content = ('This is some test file data for an ' 'uploaded file.') self.fn = 'my_test_file.txt' self.test_target_filename = '/test_storage/testfile.txt' self.su = StoredUpload(upload_id=self.upload_id, file=self.test_target_filename[1:], uploaded=timezone.now()) self.su.save() # Create file self.file_store_path = getattr(local_settings, 'FILE_STORE_PATH', None) file_full_path = os.path.join(self.file_store_path, self.test_target_filename[1:]) if not os.path.exists(os.path.dirname(file_full_path)): os.mkdir(os.path.dirname(file_full_path)) with open(file_full_path, 'w') as f: f.write(self.file_content) def test_store_upload_unset_file_store_path(self): (filename, bytes_io) = get_stored_upload_file_data(self.su) file_data = bytes_io.read().decode() self.assertEqual(file_data, self.file_content, 'Returned file content not correct.') self.assertEqual(filename, os.path.basename(self.test_target_filename), 'Returned file name is not correct.') def test_get_local_stored_upload_no_filestore(self): fsp = local_settings.FILE_STORE_PATH local_settings.FILE_STORE_PATH = None with self.assertRaisesMessage( ConfigurationError, 'The file upload settings are not configured correctly.'): get_stored_upload_file_data(self.su) local_settings.FILE_STORE_PATH = fsp def test_get_remote_stored_upload_data(self): # Set up the mock_storage_backend.open to return the file content mock_storage_backend = self._setup_mock_storage_backend() mock_storage_backend.open.return_value = BytesIO( self.file_content.encode()) mock_storage_backend.exists.return_value = True (filename, bytes_io) = get_stored_upload_file_data(self.su) file_data = bytes_io.read().decode() local_settings.STORAGES_BACKEND = None django_drf_filepond.api.storage_backend = None self.assertEqual(file_data, self.file_content, 'Returned file content not correct.') self.assertEqual(filename, os.path.basename(self.test_target_filename), 'Returned file name is not correct.') def test_storage_backend_initialised_when_required(self): mock_storage_backend = self._setup_mock_storage_backend() mock_storage_backend.open.return_value = BytesIO( self.file_content.encode()) mock_storage_backend.exists.return_value = True with patch('django_drf_filepond.api._init_storage_backend') as m: django_drf_filepond.api.storage_backend_initialised = False get_stored_upload_file_data(self.su) local_settings.STORAGES_BACKEND = None django_drf_filepond.api.storage_backend = None try: m.assert_called_once() # No assert_called_once on Python 3.5 except AttributeError: self.assertEqual( m.call_count, 1, ('Expected _init_storage_backend to be called once but ' 'it has been called %s times.' % m.call_count)) def test_get_remote_upload_not_on_remote_store(self): # File store path for remote testing should be '' file_store_path = '' mock_storage_backend = self._setup_mock_storage_backend() mock_storage_backend.exists.return_value = False file_path = os.path.join(file_store_path, self.su.file.name) with self.assertRaisesMessage( FileNotFoundError, ('File [%s] for upload_id [%s] not found on remote ' 'file store.' % (file_path, self.su.upload_id))): get_stored_upload_file_data(self.su) local_settings.STORAGES_BACKEND = None django_drf_filepond.api.storage_backend = None def _setup_mock_storage_backend(self): # Set storage backend to sftp storage local_settings.STORAGES_BACKEND = \ 'storages.backends.sftpstorage.SFTPStorage' # Set up mocked objects for storage backend testing # Setting up a mock for the storage backend based on examples at # https://docs.python.org/3.7/library/unittest.mock-examples.html# \ # applying-the-same-patch-to-every-test-method patcher = patch('storages.backends.sftpstorage.SFTPStorage') patcher.start() self.addCleanup(patcher.stop) # Mock the temporary storage object so that we're not unnecessarily # saving files to the local file system. patcher2 = patch('django.core.files.storage.FileSystemStorage') patcher2.start() self.addCleanup(patcher2.stop) # Set the backend initialisation flag to false to force re-init django_drf_filepond.api.storage_backend_initialised = False django_drf_filepond.api._init_storage_backend() mock_storage_backend = django_drf_filepond.api.storage_backend return mock_storage_backend def tearDown(self): # Delete stored upload self.su.delete() # Delete files filestore_base = getattr(local_settings, 'FILE_STORE_PATH', None) test_stored_file = os.path.join(filestore_base, self.test_target_filename[1:]) test_target_dir = os.path.dirname(test_stored_file) # Remove test_stored_file if (os.path.exists(test_stored_file) and os.path.isfile(test_stored_file)): LOG.debug('Removing test stored file: <%s>' % test_stored_file) os.remove(test_stored_file) # Remove directory if (os.path.exists(test_target_dir) and os.path.isdir(test_target_dir)): LOG.debug('Removing test target dir: <%s>' % test_target_dir) os.rmdir(test_target_dir)
class ApiDeleteTestCase(TestCase): def setUp(self): self.storage_backend = local_settings.STORAGES_BACKEND # Set storage backend to sftp storage local_settings.STORAGES_BACKEND = \ 'storages.backends.sftpstorage.SFTPStorage' # Setting up a mock for the storage backend based on examples at # https://docs.python.org/3.7/library/unittest.mock-examples.html# \ # applying-the-same-patch-to-every-test-method patcher = patch('storages.backends.sftpstorage.SFTPStorage') patcher.start() self.addCleanup(patcher.stop) # Mock the temporary storage object so that we're not unnecessarily # saving files to the local file system. patcher2 = patch('django.core.files.storage.FileSystemStorage') patcher2.start() self.addCleanup(patcher2.stop) # Reinitialse stored upload after we changed the STORAGES_BACKEND # class name above. This ensures that we're looking at the mocked # backend class. import django_drf_filepond.api self.api = django_drf_filepond.api django_drf_filepond.api.storage_backend_initialised = False django_drf_filepond.api._init_storage_backend() self.mock_storage_backend = django_drf_filepond.api.storage_backend self.delete_upload = django_drf_filepond.api.delete_stored_upload # Check that we're using a mocked storage backend self.assertTrue( isinstance(django_drf_filepond.api.storage_backend, MagicMock), ('The created storage backend should be mocked but it is not of ' 'type unittest.mock.MagicMock...')) # Set up a "StoredUpload" self.upload_id = _get_file_id() self.file_id = _get_file_id() # self.file_content = ('This is some test file data for an ' # 'uploaded file.') self.fn = 'my_test_file.txt' self.test_target_filepath = os.path.join('test_storage', self.fn) self.su = StoredUpload(upload_id=self.upload_id, file=self.test_target_filepath, uploaded=timezone.now(), stored=timezone.now()) self.su.save() def test_delete_stored_upload_local(self): # Make an API call to delete the stored upload and check that delete # is called on the FileSystemStorage backend. # Set the storage backend to None to force use of local storage. local_settings.STORAGES_BACKEND = None self.api.storage_backend_initialised = False # Get the target file path that we want to check that delete has been # called with. This is the file path from the DB plus the base file # system storage location. base_store_location = local_settings.FILE_STORE_PATH file_location = os.path.join(base_store_location, self.test_target_filepath) with patch('os.remove') as os_patcher: with patch('os.path.exists') as exists: with patch('os.path.isfile') as isfile: exists.return_value = True isfile.return_value = True self.delete_upload(self.upload_id, delete_file=True) with self.assertRaises(StoredUpload.DoesNotExist): StoredUpload.objects.get(upload_id=self.upload_id) os_patcher.assert_called_once_with(file_location) def test_delete_stored_upload_remote(self): self.mock_storage_backend.exists.return_value = True self.delete_upload(self.upload_id, delete_file=True) self.mock_storage_backend.delete.assert_called_once_with( self.test_target_filepath) def test_delete_stored_upload_local_keepfile(self): # Make an API call to delete the stored upload and check that delete # is called on the FileSystemStorage backend. # Set the storage backend to None to force use of local storage. local_settings.STORAGES_BACKEND = None self.api.storage_backend_initialised = False with patch('os.remove') as os_patcher: with patch('os.path.exists') as exists: with patch('os.path.isdir') as isdir: exists.return_value = True isdir.return_value = True self.delete_upload(self.upload_id, delete_file=False) with self.assertRaises(StoredUpload.DoesNotExist): StoredUpload.objects.get(upload_id=self.upload_id) os_patcher.assert_not_called() exists.assert_not_called() isdir.assert_not_called() def test_delete_stored_upload_remote_keepfile(self): self.delete_upload(self.upload_id, delete_file=False) self.mock_storage_backend.exists.assert_not_called() self.mock_storage_backend.delete.assert_not_called() def test_delete_stored_upload_invalid_id(self): test_id = 'abcdefghijklmnopqrstuv' ('No stored upload found with the specified ID [%s].' % (test_id)) with self.assertRaisesMessage( StoredUpload.DoesNotExist, 'StoredUpload matching query does not exist.'): self.delete_upload(test_id) def test_delete_stored_upload_local_no_filestore_path(self): fsp = local_settings.FILE_STORE_PATH local_settings.FILE_STORE_PATH = None # Need to set storage backend to None and make it reinitialise to # ensure that we're not using a remote backend for this test. local_settings.STORAGES_BACKEND = None self.api.storage_backend_initialised = False with self.assertRaisesMessage( ConfigurationError, 'The file upload settings are not configured correctly.'): self.delete_upload(self.upload_id, delete_file=True) local_settings.FILE_STORE_PATH = fsp def test_delete_stored_upload_local_not_exists(self): # Need to set storage backend to None and make it reinitialise to # ensure that we're not using a remote backend for this test. local_settings.STORAGES_BACKEND = None self.api.storage_backend_initialised = False with patch('os.path.exists') as exists: with patch('os.path.isdir') as isdir: exists.return_value = False isdir.return_value = True with self.assertRaisesMessage( ConfigurationError, ('The file upload settings are not configured ' 'correctly.')): self.delete_upload(self.upload_id, delete_file=True) def test_delete_stored_upload_local_not_isdir(self): # Need to set storage backend to None and make it reinitialise to # ensure that we're not using a remote backend for this test. local_settings.STORAGES_BACKEND = None self.api.storage_backend_initialised = False with patch('os.path.exists') as exists: with patch('os.path.isdir') as isdir: exists.return_value = True isdir.return_value = False with self.assertRaisesMessage( ConfigurationError, ('The file upload settings are not configured ' 'correctly.')): self.delete_upload(self.upload_id, delete_file=True) def test_delete_stored_upload_remote_file_missing(self): self.mock_storage_backend.exists.return_value = False with self.assertRaisesMessage( FileNotFoundError, ('File [%s] for stored upload with id [%s] not found on ' 'remote file store.' % (self.test_target_filepath, self.upload_id))): self.delete_upload(self.upload_id, delete_file=True) def test_delete_stored_upload_local_file_missing(self): local_settings.STORAGES_BACKEND = None self.api.storage_backend_initialised = False target_filepath = os.path.join(local_settings.FILE_STORE_PATH, self.test_target_filepath) with patch('os.path.exists') as exists: exists.side_effect = [True, False] with self.assertRaisesMessage( FileNotFoundError, ('File [%s] to delete was not found on the local disk' % (target_filepath))): self.delete_upload(self.upload_id, delete_file=True) def test_delete_stored_upload_local_remove_fails(self): # Make an API call to delete the stored upload and check that delete # is called on the FileSystemStorage backend. # Set the storage backend to None to force use of local storage. local_settings.STORAGES_BACKEND = None self.api.storage_backend_initialised = False # Get the target file path that we want to check that delete has been # called with. This is the file path from the DB plus the base file # system storage location. base_store_location = local_settings.FILE_STORE_PATH file_location = os.path.join(base_store_location, self.test_target_filepath) with patch('os.remove') as os_patcher: with patch('os.path.exists') as exists: with patch('os.path.isfile') as isfile: exists.return_value = True isfile.return_value = True os_patcher.side_effect = OSError('Error deleting file') with self.assertRaisesMessage(OSError, 'Error deleting file'): self.delete_upload(self.upload_id, delete_file=True) os_patcher.assert_called_once_with(file_location) def tearDown(self): local_settings.STORAGES_BACKEND = self.storage_backend self.su.delete() self.api.storage_backend_initialised = False
def store_upload(upload_id, destination_file_path): if ((not hasattr(local_settings, 'FILE_STORE_PATH')) or (not local_settings.FILE_STORE_PATH) or (not os.path.exists(local_settings.FILE_STORE_PATH)) or (not os.path.isdir(local_settings.FILE_STORE_PATH))): raise ImproperlyConfigured('A required setting is missing in your ' 'application configuration.') file_path_base = local_settings.FILE_STORE_PATH if not file_path_base or file_path_base == '': raise ValueError('The FILE_STORE_PATH is not set to a directory.') id_fmt = re.compile('^([%s]){22}$' % (shortuuid.get_alphabet())) if not id_fmt.match(upload_id): LOG.error('The provided upload ID <%s> is of an invalid format.' % upload_id) raise ValueError('The provided upload ID is of an invalid format.') if not destination_file_path or destination_file_path == '': raise ValueError('No destination file path provided.') if destination_file_path.startswith(os.sep): destination_file_path = destination_file_path[1:] try: tu = TemporaryUpload.objects.get(upload_id=upload_id) except TemporaryUpload.DoesNotExist: raise ValueError('Record for the specified upload_id doesn\'t exist') target_dir = os.path.join(file_path_base, os.path.dirname(destination_file_path)) if destination_file_path.endswith(os.sep): # Assume a directory was provided, get the file name from tu and # add this to the provided path. destination_file_path += tu.upload_name target_file_path = os.path.join(file_path_base, destination_file_path) # Check we're not about to overwrite anything if os.path.exists(target_file_path): LOG.error('File with specified name and path <%s> already exists' % destination_file_path) raise FileExistsError('The specified temporary file cannot be stored' ' to the specified location - file exists.') su = StoredUpload(upload_id=tu.upload_id, file_path=destination_file_path, uploaded=tu.uploaded) try: if not os.path.exists(target_dir): os.makedirs(target_dir) shutil.copy2(tu.get_file_path(), target_file_path) su.save() tu.delete() except IOError as e: LOG.error('Error moving temporary file to permanent storage location') raise e return su