def test_delayed_storage(self):
storage = QueuedStorage(
local='django.core.files.storage.FileSystemStorage',
remote='django.core.files.storage.FileSystemStorage',
local_options=dict(location=self.local_dir),
remote_options=dict(location=self.remote_dir),
delayed=True)
field = TestModel._meta.get_field('file')
field.storage = storage
obj = TestModel(file=File(self.test_file))
obj.save()
self.assertIsNone(getattr(obj.file.storage, 'result', None))
self.assertFalse(
path.isfile(path.join(self.remote_dir, obj.file.name)),
"Remote file should not be transferred automatically.")
result = obj.file.storage.transfer(obj.file.name)
result.get()
self.assertTrue(path.isfile(path.join(self.remote_dir, obj.file.name)),
"Remote file is not available.")
def test_transfer_and_delete(self):
"""
Make sure the TransferAndDelete task does what it says
"""
storage = QueuedStorage(
local='django.core.files.storage.FileSystemStorage',
remote='django.core.files.storage.FileSystemStorage',
local_options=dict(location=self.local_dir),
remote_options=dict(location=self.remote_dir),
task='queued_storage.tasks.TransferAndDelete')
field = models.TestModel._meta.get_field('testfile')
field.storage = storage
obj = models.TestModel()
obj.testfile.save(self.test_file_name, File(self.test_file))
obj.save()
obj.testfile.storage.result.get()
self.assertFalse(
path.isfile(path.join(self.local_dir, obj.testfile.name)),
"Local file is still available")
self.assertTrue(
path.isfile(path.join(self.remote_dir, obj.testfile.name)),
"Remote file is not available.")
def test_storage_init(self):
"""
Make sure that creating a QueuedStorage object works
"""
storage = QueuedStorage('django.core.files.storage.FileSystemStorage',
'django.core.files.storage.FileSystemStorage')
self.assertIsInstance(storage, QueuedStorage)
self.assertEqual(FileSystemStorage, storage.local.__class__)
self.assertEqual(FileSystemStorage, storage.remote.__class__)
class Media(models.Model):
"""
The Media object captures represents and individual piece of
media evidence that can be related to Bulletins.
"""
#Setup Boto AWS storage access system
queued_s3storage = QueuedStorage(
'django.core.files.storage.FileSystemStorage',
'storages.backends.s3boto.S3BotoStorage')
TYPE = (
('Video', 'video'),
('Picture', 'picture'),
('Document', 'document'),
)
name_en = models.CharField(max_length=255, blank=True, null=True)
name_ar = models.CharField(max_length=255, blank=True, null=True)
media_file = models.FileField(upload_to='media', storage=queued_s3storage)
media_thumb_file = models.FileField(
upload_to='media',
#storage=queued_s3storage,
null=True)
media_type = models.CharField('type', max_length=25, choices=TYPE)
media_created = models.DateTimeField(auto_now_add=True)
media_file_type = models.CharField(max_length=255, blank=True, null=True)
media_created = models.DateTimeField(auto_now_add=True)
@property
def name(self):
return lang_helper(self, 'name')
def get_uri(self):
"""
Return AWS Media file URL.
This method is primarily used by Django Haystack
when populating the Solr index.
"""
return self.media_file.url
def get_thumb_uri(self):
"""
Return AWS Media file URL.
This method is primarily used by Django Haystack
when populating the Solr index.
"""
if self.media_thumb_file.name:
return self.media_thumb_file.url
else:
return ''
def test_storage_methods(self):
"""
Make sure that QueuedStorage implements all the methods
"""
storage = QueuedStorage('django.core.files.storage.FileSystemStorage',
'django.core.files.storage.FileSystemStorage')
file_storage = Storage()
for attr in dir(file_storage):
method = getattr(file_storage, attr)
if not callable(method):
continue
method = getattr(storage, attr, False)
self.assertTrue(callable(method),
"QueuedStorage has no method '%s'" % attr)
def test_storage_simple_save(self):
"""
Make sure that saving to remote locations actually works
"""
storage = QueuedStorage(
local='django.core.files.storage.FileSystemStorage',
remote='django.core.files.storage.FileSystemStorage',
local_options=dict(location=self.local_dir),
remote_options=dict(location=self.remote_dir),
task='tests.tasks.test_task')
field = TestModel._meta.get_field('file')
field.storage = storage
obj = TestModel(file=File(self.test_file))
obj.save()
self.assertTrue(path.isfile(path.join(self.local_dir, obj.file.name)))
self.assertTrue(path.isfile(path.join(self.remote_dir, obj.file.name)))
def test_remote_file_field(self):
storage = QueuedStorage(
local='django.core.files.storage.FileSystemStorage',
remote='django.core.files.storage.FileSystemStorage',
local_options=dict(location=self.local_dir),
remote_options=dict(location=self.remote_dir),
delayed=True)
field = models.TestModel._meta.get_field('remote')
field.storage = storage
obj = models.TestModel()
obj.remote.save(self.test_file_name, File(self.test_file))
obj.save()
self.assertIsNone(getattr(obj.testfile.storage, 'result', None))
result = obj.remote.transfer()
self.assertTrue(result)
self.assertTrue(
path.isfile(path.join(self.remote_dir, obj.remote.name)))
def test_transfer_retried(self):
"""
Make sure the transfer task is retried correctly.
"""
storage = QueuedStorage(
local='django.core.files.storage.FileSystemStorage',
remote='django.core.files.storage.FileSystemStorage',
local_options=dict(location=self.local_dir),
remote_options=dict(location=self.remote_dir),
task='tests.tasks.RetryingTask')
field = models.TestModel._meta.get_field('testfile')
field.storage = storage
self.assertFalse(models.TestModel.retried)
obj = models.TestModel()
obj.testfile.save(self.test_file_name, File(self.test_file))
obj.save()
self.assertTrue(models.TestModel.retried)
def test_transfer_returns_boolean(self):
"""
Make sure an exception is thrown when the transfer task does not return
a boolean. We don't want to confuse Celery.
"""
storage = QueuedStorage(
local='django.core.files.storage.FileSystemStorage',
remote='django.core.files.storage.FileSystemStorage',
local_options=dict(location=self.local_dir),
remote_options=dict(location=self.remote_dir),
task='tests.tasks.NoneReturningTask')
field = TestModel._meta.get_field('file')
field.storage = storage
obj = TestModel(file=File(self.test_file))
obj.save()
self.assertRaises(ValueError,
obj.file.storage.result.get,
propagate=True)
def test_storage_celery_save(self):
"""
Make sure it actually works when using Celery as a task queue
"""
storage = QueuedStorage(
local='django.core.files.storage.FileSystemStorage',
remote='django.core.files.storage.FileSystemStorage',
local_options=dict(location=self.local_dir),
remote_options=dict(location=self.remote_dir))
field = TestModel._meta.get_field('file')
field.storage = storage
obj = TestModel(file=File(self.test_file))
obj.save()
self.assertTrue(obj.file.storage.result.get())
self.assertTrue(path.isfile(path.join(self.local_dir, obj.file.name)))
self.assertTrue(
path.isfile(path.join(self.remote_dir, obj.file.name)),
"Remote file is not available.")
self.assertFalse(storage.using_local(obj.file.name))
self.assertTrue(storage.using_remote(obj.file.name))
self.assertEqual(self.test_file_name,
storage.get_valid_name(self.test_file_name))
self.assertEqual(self.test_file_name,
storage.get_available_name(self.test_file_name))
subdir_path = os.path.join('test', self.test_file_name)
self.assertTrue(storage.exists(subdir_path))
self.assertEqual(storage.path(self.test_file_name),
path.join(self.local_dir, self.test_file_name))
self.assertEqual(storage.listdir('test')[1], [self.test_file_name])
self.assertEqual(storage.size(subdir_path),
os.stat(self.test_file_path).st_size)
self.assertEqual(storage.url(self.test_file_name), self.test_file_name)
self.assertIsInstance(storage.accessed_time(subdir_path), datetime)
self.assertIsInstance(storage.created_time(subdir_path), datetime)
self.assertIsInstance(storage.modified_time(subdir_path), datetime)
subdir_name = 'queued_storage_2.txt'
testfile = storage.open(subdir_name, 'w')
try:
testfile.write('test')
finally:
testfile.close()
self.assertTrue(storage.exists(subdir_name))
storage.delete(subdir_name)
self.assertFalse(storage.exists(subdir_name))
def test_storage_celery_save(self):
"""
Make sure it actually works when using Celery as a task queue
"""
storage = QueuedStorage(
local='django.core.files.storage.FileSystemStorage',
remote='django.core.files.storage.FileSystemStorage',
local_options=dict(location=self.local_dir),
remote_options=dict(location=self.remote_dir))
field = TestModel._meta.get_field('file')
field.storage = storage
obj = TestModel(file=File(self.test_file))
obj.save()
self.assertTrue(obj.file.storage.result.get())
self.assertTrue(path.isfile(path.join(self.local_dir, obj.file.name)))
self.assertTrue(path.isfile(path.join(self.remote_dir, obj.file.name)),
"Remote file is not available.")
self.assertFalse(storage.using_local(obj.file.name))
self.assertTrue(storage.using_remote(obj.file.name))
self.assertEqual(self.test_file_name,
storage.get_valid_name(self.test_file_name))
self.assertEqual(self.test_file_name,
storage.get_available_name(self.test_file_name))
subdir_path = os.path.join('test', self.test_file_name)
self.assertTrue(storage.exists(subdir_path))
self.assertEqual(storage.path(self.test_file_name),
path.join(self.local_dir, self.test_file_name))
self.assertEqual(storage.listdir('test')[1], [self.test_file_name])
self.assertEqual(storage.size(subdir_path),
os.stat(self.test_file_path).st_size)
self.assertEqual(storage.url(self.test_file_name), self.test_file_name)
self.assertIsInstance(storage.accessed_time(subdir_path), datetime)
self.assertIsInstance(storage.created_time(subdir_path), datetime)
self.assertIsInstance(storage.modified_time(subdir_path), datetime)
subdir_name = 'queued_storage_2.txt'
testfile = storage.open(subdir_name, 'w')
try:
testfile.write('test')
finally:
testfile.close()
self.assertTrue(storage.exists(subdir_name))
storage.delete(subdir_name)
self.assertFalse(storage.exists(subdir_name))
def test_storage_cache_key(self):
storage = QueuedStorage('django.core.files.storage.FileSystemStorage',
'django.core.files.storage.FileSystemStorage',
cache_prefix='test_cache_key')
self.assertEqual(storage.cache_prefix, 'test_cache_key')
class Media(models.Model):
"""
The Media object captures represents and individual piece of
media evidence that can be related to Bulletins.
"""
if settings.QUEUED_STORAGE:
#Setup Boto AWS storage access system
fstorage = QueuedStorage(
'django.core.files.storage.FileSystemStorage', #note: local
'storages.backends.s3boto.S3BotoStorage', #note: remote
task=
'queued_storage.tasks.Transfer', #note: TransferAndDelete would remove the local copy
)
else:
fstorage = default_storage
TYPE = (
('Video', 'video'),
('Picture', 'picture'),
('Document', 'document'),
)
name_en = models.CharField(max_length=255, blank=True, null=True)
name_ar = models.CharField(max_length=255, blank=True, null=True)
media_file = models.FileField(upload_to='media', storage=fstorage)
media_thumb_file = models.FileField(upload_to='media',
storage=fstorage,
null=True)
media_type = models.CharField('type', max_length=25, choices=TYPE)
media_created = models.DateTimeField(auto_now_add=True)
media_file_type = models.CharField(max_length=255, blank=True, null=True)
media_created = models.DateTimeField(auto_now_add=True) #dupe!
@property
def name(self):
return lang_helper(self, 'name')
def get_uri(self):
"""
Return AWS Media file URL.
This method is primarily used by Django Haystack
when populating the Solr index.
"""
return self.media_file.url
def get_thumb_uri(self):
"""
Return AWS Media file URL.
This method is primarily used by Django Haystack
when populating the Solr index.
"""
if self.media_thumb_file.name:
return self.media_thumb_file.url
else:
return ''
class Meta:
verbose_name_plural = 'media'
def __unicode__(self):
return self.name_en