def test_to_python_returns_LargeObjectFile_when_value_long(self):
oid = int(randint(1, 100))
field = LargeObjectField()
# South normally substitutes a FakeModel here, but with a baseline
# schema, we can skip the migration that creates LargeObjectField.
self.patch(field, "model")
obj_file = field.to_python(oid)
self.assertEqual(oid, obj_file.oid)
class LargeFile(CleanSave, TimestampedModel):
"""Files that are stored in the large object storage.
Only unique files are stored in the database, as only one sha256 value
can exist per file. This provides data deduplication on the file level.
Currently only used by `BootResourceFile`. This speeds up the import
process by only saving unique files.
:ivar sha256: Calculated SHA256 value of `content`.
:ivar size: Current size of `content`.
:ivar total_size: Final size of `content`. The data might currently
be saving, so total_size could be larger than `size`. `size` should
never be larger than `total_size`.
:ivar content: File data.
"""
class Meta(DefaultMeta):
"""Needed for South to recognize this model."""
objects = FileStorageManager()
sha256 = CharField(max_length=64, unique=True, editable=False)
size = BigIntegerField(default=0, editable=False)
total_size = BigIntegerField(editable=False)
# content is stored directly in the database, in the large object storage.
# Max file storage size is 4TB.
content = LargeObjectField()
def __str__(self):
return "<LargeFile size=%d sha256=%s>" % (self.total_size, self.sha256)
@property
def progress(self):
"""Percentage of `content` saved."""
if self.size <= 0:
# Handle division of zero
return 0
return self.total_size / float(self.size)
@property
def complete(self):
"""`content` has been completely saved."""
return self.total_size == self.size
@property
def valid(self):
"""All content has been written and stored SHA256 value is the same
as the calculated SHA256 value stored in the database.
Note: Depending on the size of the file, this can take some time.
"""
if not self.complete:
return False
sha256 = hashlib.sha256()
with self.content.open("rb") as stream:
for data in stream:
sha256.update(data)
hexdigest = sha256.hexdigest()
return hexdigest == self.sha256
def delete(self, *args, **kwargs):
"""Delete this object.
Important: You must remove your reference to this object or
it will not delete. Object will only be deleted if no other objects are
referencing this object.
"""
links = [
f.get_accessor_name() for f in self._meta.get_fields()
if ((f.one_to_many or f.one_to_one) and f.auto_created
and not f.concrete)
]
for link in links:
if getattr(self, link).exists():
return
super().delete(*args, **kwargs)
def test_to_python_raises_error_when_not_valid_type(self):
field = LargeObjectField()
self.assertRaises(AssertionError, field.to_python,
factory.make_string())
def test_to_python_returns_None_when_value_None(self):
field = LargeObjectField()
self.assertEqual(None, field.to_python(None))
def test_to_python_returns_value_when_value_LargeObjectFile(self):
field = LargeObjectField()
obj_file = LargeObjectFile()
self.assertEqual(obj_file, field.to_python(obj_file))
def test_get_db_prep_value_raises_error_when_not_LargeObjectFile(self):
field = LargeObjectField()
self.assertRaises(AssertionError, field.get_db_prep_value,
factory.make_string())
def test_get_db_prep_value_raises_error_when_oid_less_than_zero(self):
oid = randint(-100, 0)
field = LargeObjectField()
obj_file = LargeObjectFile()
obj_file.oid = oid
self.assertRaises(AssertionError, field.get_db_prep_value, obj_file)
def test_get_db_prep_value_returns_oid_when_value_LargeObjectFile(self):
oid = randint(1, 100)
field = LargeObjectField()
obj_file = LargeObjectFile()
obj_file.oid = oid
self.assertEqual(oid, field.get_db_prep_value(obj_file))
def test_get_db_prep_value_returns_None_when_value_None(self):
field = LargeObjectField()
self.assertEqual(None, field.get_db_prep_value(None))
class LargeObjectFieldModel(Model):
name = CharField(max_length=255, unique=False)
large_object = LargeObjectField(block_size=10)
