class Artifact(Model):
"""
A file associated with a piece of content.
When calling `save()` on an Artifact, if the file is not stored in Django's storage backend, it
is moved into place then.
Artifact is compatible with Django's `bulk_create()` method.
Fields:
file (models.FileField): The stored file. This field should be set using an absolute path to
a temporary file. It also accepts `class:django.core.files.File`.
size (models.IntegerField): The size of the file in bytes.
md5 (models.CharField): The MD5 checksum of the file.
sha1 (models.CharField): The SHA-1 checksum of the file.
sha224 (models.CharField): The SHA-224 checksum of the file.
sha256 (models.CharField): The SHA-256 checksum of the file.
sha384 (models.CharField): The SHA-384 checksum of the file.
sha512 (models.CharField): The SHA-512 checksum of the file.
"""
def storage_path(self, name):
"""
Callable used by FileField to determine where the uploaded file should be stored.
Args:
name (str): Original name of uploaded file. It is ignored by this method because the
sha256 checksum is used to determine a file path instead.
"""
return storage.get_artifact_path(self.sha256)
file = fields.ArtifactFileField(null=False,
upload_to=storage_path,
max_length=255)
size = models.BigIntegerField(null=False)
md5 = models.CharField(max_length=32,
null=False,
unique=False,
db_index=True)
sha1 = models.CharField(max_length=40,
null=False,
unique=False,
db_index=True)
sha224 = models.CharField(max_length=56,
null=False,
unique=False,
db_index=True)
sha256 = models.CharField(max_length=64,
null=False,
unique=True,
db_index=True)
sha384 = models.CharField(max_length=96,
null=False,
unique=True,
db_index=True)
sha512 = models.CharField(max_length=128,
null=False,
unique=True,
db_index=True)
objects = BulkCreateManager()
# All digest fields ordered by algorithm strength.
DIGEST_FIELDS = (
'sha512',
'sha384',
'sha256',
'sha224',
'sha1',
'md5',
)
# Reliable digest fields ordered by algorithm strength.
RELIABLE_DIGEST_FIELDS = DIGEST_FIELDS[:-3]
def q(self):
if not self._state.adding:
return models.Q(pk=self.pk)
for digest_name in self.DIGEST_FIELDS:
digest_value = getattr(self, digest_name)
if digest_value:
return models.Q(**{digest_name: digest_value})
return models.Q()
def is_equal(self, other):
"""
Is equal by matching digest.
Args:
other (pulpcore.app.models.Artifact): A artifact to match.
Returns:
bool: True when equal.
"""
for field in Artifact.RELIABLE_DIGEST_FIELDS:
digest = getattr(self, field)
if not digest:
continue
if digest == getattr(other, field):
return True
return False
def save(self, *args, **kwargs):
"""
Saves Artifact model and closes the file associated with the Artifact
Args:
args (list): list of positional arguments for Model.save()
kwargs (dict): dictionary of keyword arguments to pass to Model.save()
"""
try:
super().save(*args, **kwargs)
self.file.close()
except Exception:
self.file.close()
raise
def delete(self, *args, **kwargs):
"""
Deletes Artifact model and the file associated with the Artifact
Args:
args (list): list of positional arguments for Model.delete()
kwargs (dict): dictionary of keyword arguments to pass to Model.delete()
"""
super().delete(*args, **kwargs)
self.file.delete(save=False)
@staticmethod
def init_and_validate(file, expected_digests=None, expected_size=None):
"""
Initialize an in-memory Artifact from a file, and validate digest and size info.
This accepts both a path to a file on-disk or a
:class:`~pulpcore.app.files.PulpTemporaryUploadedFile`.
Args:
file (:class:`~pulpcore.app.files.PulpTemporaryUploadedFile` or str): The
PulpTemporaryUploadedFile to create the Artifact from or a string with the full path
to the file on disk.
expected_digests (dict): Keyed on the algorithm name provided by hashlib and stores the
value of the expected digest. e.g. {'md5': '912ec803b2ce49e4a541068d495ab570'}
expected_size (int): The number of bytes the download is expected to have.
Raises:
:class:`~pulpcore.exceptions.DigestValidationError`: When any of the ``expected_digest``
values don't match the digest of the data
:class:`~pulpcore.exceptions.SizeValidationError`: When the ``expected_size`` value
doesn't match the size of the data
Returns:
An in-memory, unsaved :class:`~pulpcore.plugin.models.Artifact`
"""
if isinstance(file, str):
with open(file, 'rb') as f:
hashers = {n: hashlib.new(n) for n in Artifact.DIGEST_FIELDS}
size = 0
while True:
chunk = f.read(1048576) # 1 megabyte
if not chunk:
break
for algorithm in hashers.values():
algorithm.update(chunk)
size = size + len(chunk)
else:
size = file.size
hashers = file.hashers
if expected_size:
if size != expected_size:
raise SizeValidationError()
if expected_digests:
for algorithm, expected_digest in expected_digests.items():
if expected_digest != hashers[algorithm].hexdigest():
raise DigestValidationError()
attributes = {'size': size, 'file': file}
for algorithm in Artifact.DIGEST_FIELDS:
attributes[algorithm] = hashers[algorithm].hexdigest()
return Artifact(**attributes)
class Artifact(HandleTempFilesMixin, BaseModel):
"""
A file associated with a piece of content.
When calling `save()` on an Artifact, if the file is not stored in Django's storage backend, it
is moved into place then.
Artifact is compatible with Django's `bulk_create()` method.
Fields:
file (pulpcore.app.models.fields.ArtifactFileField): The stored file. This field should
be set using an absolute path to a temporary file.
It also accepts `class:django.core.files.File`.
size (models.BigIntegerField): The size of the file in bytes.
md5 (models.CharField): The MD5 checksum of the file.
sha1 (models.CharField): The SHA-1 checksum of the file.
sha224 (models.CharField): The SHA-224 checksum of the file.
sha256 (models.CharField): The SHA-256 checksum of the file (REQUIRED).
sha384 (models.CharField): The SHA-384 checksum of the file.
sha512 (models.CharField): The SHA-512 checksum of the file.
"""
def storage_path(self, name):
"""
Callable used by FileField to determine where the uploaded file should be stored.
Args:
name (str): Original name of uploaded file. It is ignored by this method because the
sha256 checksum is used to determine a file path instead.
"""
return storage.get_artifact_path(self.sha256)
file = fields.ArtifactFileField(null=False, upload_to=storage_path, max_length=255)
size = models.BigIntegerField(null=False)
md5 = models.CharField(max_length=32, null=True, unique=False, db_index=True)
sha1 = models.CharField(max_length=40, null=True, unique=False, db_index=True)
sha224 = models.CharField(max_length=56, null=True, unique=False, db_index=True)
sha256 = models.CharField(max_length=64, null=False, unique=True, db_index=True)
sha384 = models.CharField(max_length=96, null=True, unique=True, db_index=True)
sha512 = models.CharField(max_length=128, null=True, unique=True, db_index=True)
objects = BulkCreateManager()
# All available digest fields ordered by algorithm strength.
DIGEST_FIELDS = _DIGEST_FIELDS
# All available digest fields ordered by relative frequency
# (Better average-case performance in some algorithms with fallback)
COMMON_DIGEST_FIELDS = _COMMON_DIGEST_FIELDS
# Available, reliable digest fields ordered by algorithm strength.
RELIABLE_DIGEST_FIELDS = _RELIABLE_DIGEST_FIELDS
# Digest-fields that are NOT ALLOWED
FORBIDDEN_DIGESTS = _FORBIDDEN_DIGESTS
@hook(BEFORE_SAVE)
def before_save(self):
"""
Pre-save hook that validates checksum rules prior to allowing an Artifact to be saved.
An Artifact with any checksums from the FORBIDDEN set will fail to save while raising
an UnsupportedDigestValidationError exception.
Similarly, any checksums in DIGEST_FIELDS that is NOT set will raise a
MissingDigestValidationError exception.
Raises:
:class: `~pulpcore.exceptions.UnsupportedDigestValidationError`: When any of the
keys on FORBIDDEN_DIGESTS are set for the Artifact
:class: `~pulpcore.exceptions.MissingDigestValidationError`: When any of the
keys on DIGEST_FIELDS are found to be missing from the Artifact
"""
bad_keys = [k for k in self.FORBIDDEN_DIGESTS if getattr(self, k)]
if bad_keys:
raise UnsupportedDigestValidationError(
_("Checksum algorithms {} are forbidden for this Pulp instance.").format(bad_keys)
)
missing_keys = [k for k in self.DIGEST_FIELDS if not getattr(self, k)]
if missing_keys:
raise MissingDigestValidationError(
_("Missing required checksum algorithms {}.").format(missing_keys)
)
def q(self):
if not self._state.adding:
return models.Q(pk=self.pk)
for digest_name in self.DIGEST_FIELDS:
digest_value = getattr(self, digest_name)
if digest_value:
return models.Q(**{digest_name: digest_value})
return models.Q()
def is_equal(self, other):
"""
Is equal by matching digest.
Args:
other (pulpcore.app.models.Artifact): A artifact to match.
Returns:
bool: True when equal.
"""
for field in self.RELIABLE_DIGEST_FIELDS:
digest = getattr(self, field)
if not digest:
continue
if digest == getattr(other, field):
return True
return False
@staticmethod
def init_and_validate(file, expected_digests=None, expected_size=None):
"""
Initialize an in-memory Artifact from a file, and validate digest and size info.
This accepts both a path to a file on-disk or a
:class:`~pulpcore.app.files.PulpTemporaryUploadedFile`.
Args:
file (:class:`~pulpcore.app.files.PulpTemporaryUploadedFile` or str): The
PulpTemporaryUploadedFile to create the Artifact from or a string with the full path
to the file on disk.
expected_digests (dict): Keyed on the algorithm name provided by hashlib and stores the
value of the expected digest. e.g. {'md5': '912ec803b2ce49e4a541068d495ab570'}
expected_size (int): The number of bytes the download is expected to have.
Raises:
:class:`~pulpcore.exceptions.DigestValidationError`: When any of the ``expected_digest``
values don't match the digest of the data
:class:`~pulpcore.exceptions.SizeValidationError`: When the ``expected_size`` value
doesn't match the size of the data
:class:`~pulpcore.exceptions.UnsupportedDigestValidationError`: When any of the
``expected_digest`` algorithms aren't in the ALLOWED_CONTENT_CHECKSUMS list
Returns:
An in-memory, unsaved :class:`~pulpcore.plugin.models.Artifact`
"""
if isinstance(file, str):
with open(file, "rb") as f:
hashers = {n: pulp_hashlib.new(n) for n in Artifact.DIGEST_FIELDS}
size = 0
while True:
chunk = f.read(1048576) # 1 megabyte
if not chunk:
break
for algorithm in hashers.values():
algorithm.update(chunk)
size = size + len(chunk)
else:
size = file.size
hashers = file.hashers
if expected_size:
if size != expected_size:
raise SizeValidationError()
if expected_digests:
for algorithm, expected_digest in expected_digests.items():
if algorithm not in hashers:
raise UnsupportedDigestValidationError(
_("Checksum algorithm {} forbidden for this Pulp instance.").format(
algorithm
)
)
if expected_digest != hashers[algorithm].hexdigest():
raise DigestValidationError()
attributes = {"size": size, "file": file}
for algorithm in Artifact.DIGEST_FIELDS:
attributes[algorithm] = hashers[algorithm].hexdigest()
return Artifact(**attributes)
@classmethod
def from_pulp_temporary_file(cls, temp_file):
"""
Creates an Artifact from PulpTemporaryFile.
Returns:
An saved :class:`~pulpcore.plugin.models.Artifact`
"""
artifact_file = default_storage.open(temp_file.file.name)
with tempfile.NamedTemporaryFile("wb") as new_file:
shutil.copyfileobj(artifact_file, new_file)
new_file.flush()
artifact = cls.init_and_validate(new_file.name)
artifact.save()
temp_file.delete()
return artifact
class PulpTemporaryFile(HandleTempFilesMixin, BaseModel):
"""
A temporary file saved to the storage backend.
Commonly used to pass data to one or more tasks.
Fields:
file (pulpcore.app.models.fields.ArtifactFileField): The stored file. This field should
be set using an absolute path to a temporary file.
It also accepts `class:django.core.files.File`.
"""
def storage_path(self, name):
"""
Callable used by FileField to determine where the uploaded file should be stored.
Args:
name (str): Original name of uploaded file. It is ignored by this method because the
pulp_id is used to determine a file path instead.
"""
return storage.get_temp_file_path(self.pulp_id)
file = fields.ArtifactFileField(null=False, upload_to=storage_path, max_length=255)
@staticmethod
def init_and_validate(file, expected_digests=None, expected_size=None):
"""
Initialize an in-memory PulpTemporaryFile from a file, and validate digest and size info.
This accepts both a path to a file on-disk or a
:class:`~pulpcore.app.files.PulpTemporaryUploadedFile`.
Args:
file (:class:`~pulpcore.app.files.PulpTemporaryUploadedFile` or str): The
PulpTemporaryUploadedFile to create the PulpTemporaryFile from or a string with the
full path to the file on disk.
expected_digests (dict): Keyed on the algorithm name provided by hashlib and stores the
value of the expected digest. e.g. {'md5': '912ec803b2ce49e4a541068d495ab570'}
expected_size (int): The number of bytes the download is expected to have.
Raises:
:class:`~pulpcore.exceptions.DigestValidationError`: When any of the ``expected_digest``
values don't match the digest of the data
:class:`~pulpcore.exceptions.SizeValidationError`: When the ``expected_size`` value
doesn't match the size of the data
:class:`~pulpcore.exceptions.UnsupportedDigestValidationError`: When any of the
``expected_digest`` algorithms aren't in the ALLOWED_CONTENT_CHECKSUMS list
Returns:
An in-memory, unsaved :class:`~pulpcore.plugin.models.PulpTemporaryFile`
"""
if not expected_digests and not expected_size:
return PulpTemporaryFile(file=file)
if isinstance(file, str):
with open(file, "rb") as f:
hashers = {n: pulp_hashlib.new(n) for n in expected_digests.keys()}
size = 0
while True:
chunk = f.read(1048576) # 1 megabyte
if not chunk:
break
for algorithm in hashers.values():
algorithm.update(chunk)
size = size + len(chunk)
else:
size = file.size
hashers = file.hashers
if expected_size:
if size != expected_size:
raise SizeValidationError()
if expected_digests:
for algorithm, expected_digest in expected_digests.items():
if algorithm not in hashers:
raise UnsupportedDigestValidationError(
_("Checksum algorithm {} forbidden for this Pulp instance.").format(
algorithm
)
)
if expected_digest != hashers[algorithm].hexdigest():
raise DigestValidationError()
return PulpTemporaryFile(file=file)
class Artifact(Model):
"""
A file associated with a piece of content.
When calling `save()` on an Artifact, if the file is not stored in Django's storage backend, it
is moved into place then.
Artifact is compatible with Django's `bulk_create()` method.
Fields:
file (models.FileField): The stored file. This field should be set using an absolute path to
a temporary file. It also accepts `class:django.core.files.File`.
size (models.IntegerField): The size of the file in bytes.
md5 (models.CharField): The MD5 checksum of the file.
sha1 (models.CharField): The SHA-1 checksum of the file.
sha224 (models.CharField): The SHA-224 checksum of the file.
sha256 (models.CharField): The SHA-256 checksum of the file.
sha384 (models.CharField): The SHA-384 checksum of the file.
sha512 (models.CharField): The SHA-512 checksum of the file.
"""
def storage_path(self, name):
"""
Callable used by FileField to determine where the uploaded file should be stored.
Args:
name (str): Original name of uploaded file. It is ignored by this method because the
sha256 checksum is used to determine a file path instead.
"""
return storage.get_artifact_path(self.sha256)
file = fields.ArtifactFileField(blank=False, null=False, upload_to=storage_path, max_length=255)
size = models.IntegerField(blank=False, null=False)
md5 = models.CharField(max_length=32, blank=False, null=False, unique=False, db_index=True)
sha1 = models.CharField(max_length=40, blank=False, null=False, unique=False, db_index=True)
sha224 = models.CharField(max_length=56, blank=False, null=False, unique=False, db_index=True)
sha256 = models.CharField(max_length=64, blank=False, null=False, unique=True, db_index=True)
sha384 = models.CharField(max_length=96, blank=False, null=False, unique=True, db_index=True)
sha512 = models.CharField(max_length=128, blank=False, null=False, unique=True, db_index=True)
# All digest fields ordered by algorithm strength.
DIGEST_FIELDS = (
'sha512',
'sha384',
'sha256',
'sha224',
'sha1',
'md5',
)
# Reliable digest fields ordered by algorithm strength.
RELIABLE_DIGEST_FIELDS = DIGEST_FIELDS[:-3]
def is_equal(self, other):
"""
Is equal by matching digest.
Args:
other (pulpcore.app.models.Artifact): A artifact to match.
Returns:
bool: True when equal.
"""
for field in Artifact.RELIABLE_DIGEST_FIELDS:
digest = getattr(self, field)
if not digest:
continue
if digest == getattr(other, field):
return True
return False
def save(self, *args, **kwargs):
"""
Saves Artifact model and closes the file associated with the Artifact
Args:
args (list): list of positional arguments for Model.save()
kwargs (dict): dictionary of keyword arguments to pass to Model.save()
"""
try:
super().save(*args, **kwargs)
self.file.close()
except Exception:
self.file.close()
raise
def delete(self, *args, **kwargs):
"""
Deletes Artifact model and the file associated with the Artifact
Args:
args (list): list of positional arguments for Model.delete()
kwargs (dict): dictionary of keyword arguments to pass to Model.delete()
"""
super().delete(*args, **kwargs)
self.file.delete(save=False)
