def test_transfer_and_delete(self): """ Make sure the TransferAndDelete task does what it says """ storage = QueuedStorage( local='django.core.files.storage.FileSystemStorage', remote='django.core.files.storage.FileSystemStorage', local_options=dict(location=self.local_dir), remote_options=dict(location=self.remote_dir), task='queued_storage.tasks.TransferAndDelete') field = models.TestModel._meta.get_field('testfile') field.storage = storage obj = models.TestModel() obj.testfile.save(self.test_file_name, File(self.test_file)) obj.save() obj.testfile.storage.result.get() self.assertFalse( path.isfile(path.join(self.local_dir, obj.testfile.name)), "Local file is still available") self.assertTrue( path.isfile(path.join(self.remote_dir, obj.testfile.name)), "Remote file is not available.")
def test_delayed_storage(self): storage = QueuedStorage( local='django.core.files.storage.FileSystemStorage', remote='django.core.files.storage.FileSystemStorage', local_options=dict(location=self.local_dir), remote_options=dict(location=self.remote_dir), delayed=True) field = TestModel._meta.get_field('file') field.storage = storage obj = TestModel(file=File(self.test_file)) obj.save() self.assertIsNone(getattr(obj.file.storage, 'result', None)) self.assertFalse( path.isfile(path.join(self.remote_dir, obj.file.name)), "Remote file should not be transferred automatically.") result = obj.file.storage.transfer(obj.file.name) result.get() self.assertTrue(path.isfile(path.join(self.remote_dir, obj.file.name)), "Remote file is not available.")
def test_storage_init(self): """ Make sure that creating a QueuedStorage object works """ storage = QueuedStorage('django.core.files.storage.FileSystemStorage', 'django.core.files.storage.FileSystemStorage') self.assertIsInstance(storage, QueuedStorage) self.assertEqual(FileSystemStorage, storage.local.__class__) self.assertEqual(FileSystemStorage, storage.remote.__class__)
def test_storage_celery_save(self): """ Make sure it actually works when using Celery as a task queue """ storage = QueuedStorage( local='django.core.files.storage.FileSystemStorage', remote='django.core.files.storage.FileSystemStorage', local_options=dict(location=self.local_dir), remote_options=dict(location=self.remote_dir)) field = models.TestModel._meta.get_field('testfile') field.storage = storage obj = models.TestModel(testfile=File(self.test_file)) obj.save() self.assertTrue(obj.testfile.storage.result.get()) self.assertTrue( path.isfile(path.join(self.local_dir, obj.testfile.name))) self.assertTrue( path.isfile(path.join(self.remote_dir, obj.testfile.name)), "Remote file is not available.") self.assertFalse(storage.using_local(obj.testfile.name)) self.assertTrue(storage.using_remote(obj.testfile.name)) self.assertEqual(self.test_file_name, storage.get_valid_name(self.test_file_name)) self.assertEqual(self.test_file_name, storage.get_available_name(self.test_file_name)) subdir_path = os.path.join('test', self.test_file_name) self.assertTrue(storage.exists(subdir_path)) self.assertEqual(storage.path(self.test_file_name), path.join(self.local_dir, self.test_file_name)) self.assertEqual(storage.listdir('test')[1], [self.test_file_name]) self.assertEqual(storage.size(subdir_path), os.stat(self.test_file_path).st_size) self.assertEqual(storage.url(self.test_file_name), self.test_file_name) self.assertIsInstance(storage.accessed_time(subdir_path), datetime) self.assertIsInstance(storage.created_time(subdir_path), datetime) self.assertIsInstance(storage.modified_time(subdir_path), datetime) subdir_name = 'queued_storage_2.txt' testfile = storage.open(subdir_name, 'w') try: testfile.write('test') finally: testfile.close() self.assertTrue(storage.exists(subdir_name)) storage.delete(subdir_name) self.assertFalse(storage.exists(subdir_name))
class Media(models.Model): """ The Media object captures represents and individual piece of media evidence that can be related to Bulletins. """ #Setup Boto AWS storage access system queued_s3storage = QueuedStorage( 'django.core.files.storage.FileSystemStorage', 'storages.backends.s3boto.S3BotoStorage') TYPE = ( ('Video', 'video'), ('Picture', 'picture'), ('Document', 'document'), ) name_en = models.CharField(max_length=255, blank=True, null=True) name_ar = models.CharField(max_length=255, blank=True, null=True) media_file = models.FileField(upload_to='media', storage=queued_s3storage) media_thumb_file = models.FileField( upload_to='media', #storage=queued_s3storage, null=True) media_type = models.CharField('type', max_length=25, choices=TYPE) media_created = models.DateTimeField(auto_now_add=True) media_file_type = models.CharField(max_length=255, blank=True, null=True) media_created = models.DateTimeField(auto_now_add=True) @property def name(self): return lang_helper(self, 'name') def get_uri(self): """ Return AWS Media file URL. This method is primarily used by Django Haystack when populating the Solr index. """ return self.media_file.url def get_thumb_uri(self): """ Return AWS Media file URL. This method is primarily used by Django Haystack when populating the Solr index. """ if self.media_thumb_file.name: return self.media_thumb_file.url else: return ''
def test_storage_methods(self): """ Make sure that QueuedStorage implements all the methods """ storage = QueuedStorage('django.core.files.storage.FileSystemStorage', 'django.core.files.storage.FileSystemStorage') file_storage = Storage() for attr in dir(file_storage): method = getattr(file_storage, attr) if not callable(method): continue method = getattr(storage, attr, False) self.assertTrue(callable(method), "QueuedStorage has no method '%s'" % attr)
def test_storage_simple_save(self): """ Make sure that saving to remote locations actually works """ storage = QueuedStorage( local='django.core.files.storage.FileSystemStorage', remote='django.core.files.storage.FileSystemStorage', local_options=dict(location=self.local_dir), remote_options=dict(location=self.remote_dir), task='tests.tasks.test_task') field = TestModel._meta.get_field('file') field.storage = storage obj = TestModel(file=File(self.test_file)) obj.save() self.assertTrue(path.isfile(path.join(self.local_dir, obj.file.name))) self.assertTrue(path.isfile(path.join(self.remote_dir, obj.file.name)))
def test_remote_file_field(self): storage = QueuedStorage( local='django.core.files.storage.FileSystemStorage', remote='django.core.files.storage.FileSystemStorage', local_options=dict(location=self.local_dir), remote_options=dict(location=self.remote_dir), delayed=True) field = models.TestModel._meta.get_field('remote') field.storage = storage obj = models.TestModel() obj.remote.save(self.test_file_name, File(self.test_file)) obj.save() self.assertIsNone(getattr(obj.testfile.storage, 'result', None)) result = obj.remote.transfer() self.assertTrue(result) self.assertTrue( path.isfile(path.join(self.remote_dir, obj.remote.name)))
def test_transfer_retried(self): """ Make sure the transfer task is retried correctly. """ storage = QueuedStorage( local='django.core.files.storage.FileSystemStorage', remote='django.core.files.storage.FileSystemStorage', local_options=dict(location=self.local_dir), remote_options=dict(location=self.remote_dir), task='tests.tasks.RetryingTask') field = models.TestModel._meta.get_field('testfile') field.storage = storage self.assertFalse(models.TestModel.retried) obj = models.TestModel() obj.testfile.save(self.test_file_name, File(self.test_file)) obj.save() self.assertTrue(models.TestModel.retried)
def test_transfer_returns_boolean(self): """ Make sure an exception is thrown when the transfer task does not return a boolean. We don't want to confuse Celery. """ storage = QueuedStorage( local='django.core.files.storage.FileSystemStorage', remote='django.core.files.storage.FileSystemStorage', local_options=dict(location=self.local_dir), remote_options=dict(location=self.remote_dir), task='tests.tasks.NoneReturningTask') field = TestModel._meta.get_field('file') field.storage = storage obj = TestModel(file=File(self.test_file)) obj.save() self.assertRaises(ValueError, obj.file.storage.result.get, propagate=True)
def test_storage_cache_key(self): storage = QueuedStorage('django.core.files.storage.FileSystemStorage', 'django.core.files.storage.FileSystemStorage', cache_prefix='test_cache_key') self.assertEqual(storage.cache_prefix, 'test_cache_key')
class Media(models.Model): """ The Media object captures represents and individual piece of media evidence that can be related to Bulletins. """ if settings.QUEUED_STORAGE: #Setup Boto AWS storage access system fstorage = QueuedStorage( 'django.core.files.storage.FileSystemStorage', #note: local 'storages.backends.s3boto.S3BotoStorage', #note: remote task= 'queued_storage.tasks.Transfer', #note: TransferAndDelete would remove the local copy ) else: fstorage = default_storage TYPE = ( ('Video', 'video'), ('Picture', 'picture'), ('Document', 'document'), ) name_en = models.CharField(max_length=255, blank=True, null=True) name_ar = models.CharField(max_length=255, blank=True, null=True) media_file = models.FileField(upload_to='media', storage=fstorage) media_thumb_file = models.FileField(upload_to='media', storage=fstorage, null=True) media_type = models.CharField('type', max_length=25, choices=TYPE) media_created = models.DateTimeField(auto_now_add=True) media_file_type = models.CharField(max_length=255, blank=True, null=True) media_created = models.DateTimeField(auto_now_add=True) #dupe! @property def name(self): return lang_helper(self, 'name') def get_uri(self): """ Return AWS Media file URL. This method is primarily used by Django Haystack when populating the Solr index. """ return self.media_file.url def get_thumb_uri(self): """ Return AWS Media file URL. This method is primarily used by Django Haystack when populating the Solr index. """ if self.media_thumb_file.name: return self.media_thumb_file.url else: return '' class Meta: verbose_name_plural = 'media' def __unicode__(self): return self.name_en