class Artifact(Model): """ A file associated with a piece of content. When calling `save()` on an Artifact, if the file is not stored in Django's storage backend, it is moved into place then. Artifact is compatible with Django's `bulk_create()` method. Fields: file (models.FileField): The stored file. This field should be set using an absolute path to a temporary file. It also accepts `class:django.core.files.File`. size (models.IntegerField): The size of the file in bytes. md5 (models.CharField): The MD5 checksum of the file. sha1 (models.CharField): The SHA-1 checksum of the file. sha224 (models.CharField): The SHA-224 checksum of the file. sha256 (models.CharField): The SHA-256 checksum of the file. sha384 (models.CharField): The SHA-384 checksum of the file. sha512 (models.CharField): The SHA-512 checksum of the file. """ def storage_path(self, name): """ Callable used by FileField to determine where the uploaded file should be stored. Args: name (str): Original name of uploaded file. It is ignored by this method because the sha256 checksum is used to determine a file path instead. """ return storage.get_artifact_path(self.sha256) file = fields.ArtifactFileField(null=False, upload_to=storage_path, max_length=255) size = models.BigIntegerField(null=False) md5 = models.CharField(max_length=32, null=False, unique=False, db_index=True) sha1 = models.CharField(max_length=40, null=False, unique=False, db_index=True) sha224 = models.CharField(max_length=56, null=False, unique=False, db_index=True) sha256 = models.CharField(max_length=64, null=False, unique=True, db_index=True) sha384 = models.CharField(max_length=96, null=False, unique=True, db_index=True) sha512 = models.CharField(max_length=128, null=False, unique=True, db_index=True) objects = BulkCreateManager() # All digest fields ordered by algorithm strength. DIGEST_FIELDS = ( 'sha512', 'sha384', 'sha256', 'sha224', 'sha1', 'md5', ) # Reliable digest fields ordered by algorithm strength. RELIABLE_DIGEST_FIELDS = DIGEST_FIELDS[:-3] def q(self): if not self._state.adding: return models.Q(pk=self.pk) for digest_name in self.DIGEST_FIELDS: digest_value = getattr(self, digest_name) if digest_value: return models.Q(**{digest_name: digest_value}) return models.Q() def is_equal(self, other): """ Is equal by matching digest. Args: other (pulpcore.app.models.Artifact): A artifact to match. Returns: bool: True when equal. """ for field in Artifact.RELIABLE_DIGEST_FIELDS: digest = getattr(self, field) if not digest: continue if digest == getattr(other, field): return True return False def save(self, *args, **kwargs): """ Saves Artifact model and closes the file associated with the Artifact Args: args (list): list of positional arguments for Model.save() kwargs (dict): dictionary of keyword arguments to pass to Model.save() """ try: super().save(*args, **kwargs) self.file.close() except Exception: self.file.close() raise def delete(self, *args, **kwargs): """ Deletes Artifact model and the file associated with the Artifact Args: args (list): list of positional arguments for Model.delete() kwargs (dict): dictionary of keyword arguments to pass to Model.delete() """ super().delete(*args, **kwargs) self.file.delete(save=False) @staticmethod def init_and_validate(file, expected_digests=None, expected_size=None): """ Initialize an in-memory Artifact from a file, and validate digest and size info. This accepts both a path to a file on-disk or a :class:`~pulpcore.app.files.PulpTemporaryUploadedFile`. Args: file (:class:`~pulpcore.app.files.PulpTemporaryUploadedFile` or str): The PulpTemporaryUploadedFile to create the Artifact from or a string with the full path to the file on disk. expected_digests (dict): Keyed on the algorithm name provided by hashlib and stores the value of the expected digest. e.g. {'md5': '912ec803b2ce49e4a541068d495ab570'} expected_size (int): The number of bytes the download is expected to have. Raises: :class:`~pulpcore.exceptions.DigestValidationError`: When any of the ``expected_digest`` values don't match the digest of the data :class:`~pulpcore.exceptions.SizeValidationError`: When the ``expected_size`` value doesn't match the size of the data Returns: An in-memory, unsaved :class:`~pulpcore.plugin.models.Artifact` """ if isinstance(file, str): with open(file, 'rb') as f: hashers = {n: hashlib.new(n) for n in Artifact.DIGEST_FIELDS} size = 0 while True: chunk = f.read(1048576) # 1 megabyte if not chunk: break for algorithm in hashers.values(): algorithm.update(chunk) size = size + len(chunk) else: size = file.size hashers = file.hashers if expected_size: if size != expected_size: raise SizeValidationError() if expected_digests: for algorithm, expected_digest in expected_digests.items(): if expected_digest != hashers[algorithm].hexdigest(): raise DigestValidationError() attributes = {'size': size, 'file': file} for algorithm in Artifact.DIGEST_FIELDS: attributes[algorithm] = hashers[algorithm].hexdigest() return Artifact(**attributes)
class Artifact(HandleTempFilesMixin, BaseModel): """ A file associated with a piece of content. When calling `save()` on an Artifact, if the file is not stored in Django's storage backend, it is moved into place then. Artifact is compatible with Django's `bulk_create()` method. Fields: file (pulpcore.app.models.fields.ArtifactFileField): The stored file. This field should be set using an absolute path to a temporary file. It also accepts `class:django.core.files.File`. size (models.BigIntegerField): The size of the file in bytes. md5 (models.CharField): The MD5 checksum of the file. sha1 (models.CharField): The SHA-1 checksum of the file. sha224 (models.CharField): The SHA-224 checksum of the file. sha256 (models.CharField): The SHA-256 checksum of the file (REQUIRED). sha384 (models.CharField): The SHA-384 checksum of the file. sha512 (models.CharField): The SHA-512 checksum of the file. """ def storage_path(self, name): """ Callable used by FileField to determine where the uploaded file should be stored. Args: name (str): Original name of uploaded file. It is ignored by this method because the sha256 checksum is used to determine a file path instead. """ return storage.get_artifact_path(self.sha256) file = fields.ArtifactFileField(null=False, upload_to=storage_path, max_length=255) size = models.BigIntegerField(null=False) md5 = models.CharField(max_length=32, null=True, unique=False, db_index=True) sha1 = models.CharField(max_length=40, null=True, unique=False, db_index=True) sha224 = models.CharField(max_length=56, null=True, unique=False, db_index=True) sha256 = models.CharField(max_length=64, null=False, unique=True, db_index=True) sha384 = models.CharField(max_length=96, null=True, unique=True, db_index=True) sha512 = models.CharField(max_length=128, null=True, unique=True, db_index=True) objects = BulkCreateManager() # All available digest fields ordered by algorithm strength. DIGEST_FIELDS = _DIGEST_FIELDS # All available digest fields ordered by relative frequency # (Better average-case performance in some algorithms with fallback) COMMON_DIGEST_FIELDS = _COMMON_DIGEST_FIELDS # Available, reliable digest fields ordered by algorithm strength. RELIABLE_DIGEST_FIELDS = _RELIABLE_DIGEST_FIELDS # Digest-fields that are NOT ALLOWED FORBIDDEN_DIGESTS = _FORBIDDEN_DIGESTS @hook(BEFORE_SAVE) def before_save(self): """ Pre-save hook that validates checksum rules prior to allowing an Artifact to be saved. An Artifact with any checksums from the FORBIDDEN set will fail to save while raising an UnsupportedDigestValidationError exception. Similarly, any checksums in DIGEST_FIELDS that is NOT set will raise a MissingDigestValidationError exception. Raises: :class: `~pulpcore.exceptions.UnsupportedDigestValidationError`: When any of the keys on FORBIDDEN_DIGESTS are set for the Artifact :class: `~pulpcore.exceptions.MissingDigestValidationError`: When any of the keys on DIGEST_FIELDS are found to be missing from the Artifact """ bad_keys = [k for k in self.FORBIDDEN_DIGESTS if getattr(self, k)] if bad_keys: raise UnsupportedDigestValidationError( _("Checksum algorithms {} are forbidden for this Pulp instance.").format(bad_keys) ) missing_keys = [k for k in self.DIGEST_FIELDS if not getattr(self, k)] if missing_keys: raise MissingDigestValidationError( _("Missing required checksum algorithms {}.").format(missing_keys) ) def q(self): if not self._state.adding: return models.Q(pk=self.pk) for digest_name in self.DIGEST_FIELDS: digest_value = getattr(self, digest_name) if digest_value: return models.Q(**{digest_name: digest_value}) return models.Q() def is_equal(self, other): """ Is equal by matching digest. Args: other (pulpcore.app.models.Artifact): A artifact to match. Returns: bool: True when equal. """ for field in self.RELIABLE_DIGEST_FIELDS: digest = getattr(self, field) if not digest: continue if digest == getattr(other, field): return True return False @staticmethod def init_and_validate(file, expected_digests=None, expected_size=None): """ Initialize an in-memory Artifact from a file, and validate digest and size info. This accepts both a path to a file on-disk or a :class:`~pulpcore.app.files.PulpTemporaryUploadedFile`. Args: file (:class:`~pulpcore.app.files.PulpTemporaryUploadedFile` or str): The PulpTemporaryUploadedFile to create the Artifact from or a string with the full path to the file on disk. expected_digests (dict): Keyed on the algorithm name provided by hashlib and stores the value of the expected digest. e.g. {'md5': '912ec803b2ce49e4a541068d495ab570'} expected_size (int): The number of bytes the download is expected to have. Raises: :class:`~pulpcore.exceptions.DigestValidationError`: When any of the ``expected_digest`` values don't match the digest of the data :class:`~pulpcore.exceptions.SizeValidationError`: When the ``expected_size`` value doesn't match the size of the data :class:`~pulpcore.exceptions.UnsupportedDigestValidationError`: When any of the ``expected_digest`` algorithms aren't in the ALLOWED_CONTENT_CHECKSUMS list Returns: An in-memory, unsaved :class:`~pulpcore.plugin.models.Artifact` """ if isinstance(file, str): with open(file, "rb") as f: hashers = {n: pulp_hashlib.new(n) for n in Artifact.DIGEST_FIELDS} size = 0 while True: chunk = f.read(1048576) # 1 megabyte if not chunk: break for algorithm in hashers.values(): algorithm.update(chunk) size = size + len(chunk) else: size = file.size hashers = file.hashers if expected_size: if size != expected_size: raise SizeValidationError() if expected_digests: for algorithm, expected_digest in expected_digests.items(): if algorithm not in hashers: raise UnsupportedDigestValidationError( _("Checksum algorithm {} forbidden for this Pulp instance.").format( algorithm ) ) if expected_digest != hashers[algorithm].hexdigest(): raise DigestValidationError() attributes = {"size": size, "file": file} for algorithm in Artifact.DIGEST_FIELDS: attributes[algorithm] = hashers[algorithm].hexdigest() return Artifact(**attributes) @classmethod def from_pulp_temporary_file(cls, temp_file): """ Creates an Artifact from PulpTemporaryFile. Returns: An saved :class:`~pulpcore.plugin.models.Artifact` """ artifact_file = default_storage.open(temp_file.file.name) with tempfile.NamedTemporaryFile("wb") as new_file: shutil.copyfileobj(artifact_file, new_file) new_file.flush() artifact = cls.init_and_validate(new_file.name) artifact.save() temp_file.delete() return artifact
class PulpTemporaryFile(HandleTempFilesMixin, BaseModel): """ A temporary file saved to the storage backend. Commonly used to pass data to one or more tasks. Fields: file (pulpcore.app.models.fields.ArtifactFileField): The stored file. This field should be set using an absolute path to a temporary file. It also accepts `class:django.core.files.File`. """ def storage_path(self, name): """ Callable used by FileField to determine where the uploaded file should be stored. Args: name (str): Original name of uploaded file. It is ignored by this method because the pulp_id is used to determine a file path instead. """ return storage.get_temp_file_path(self.pulp_id) file = fields.ArtifactFileField(null=False, upload_to=storage_path, max_length=255) @staticmethod def init_and_validate(file, expected_digests=None, expected_size=None): """ Initialize an in-memory PulpTemporaryFile from a file, and validate digest and size info. This accepts both a path to a file on-disk or a :class:`~pulpcore.app.files.PulpTemporaryUploadedFile`. Args: file (:class:`~pulpcore.app.files.PulpTemporaryUploadedFile` or str): The PulpTemporaryUploadedFile to create the PulpTemporaryFile from or a string with the full path to the file on disk. expected_digests (dict): Keyed on the algorithm name provided by hashlib and stores the value of the expected digest. e.g. {'md5': '912ec803b2ce49e4a541068d495ab570'} expected_size (int): The number of bytes the download is expected to have. Raises: :class:`~pulpcore.exceptions.DigestValidationError`: When any of the ``expected_digest`` values don't match the digest of the data :class:`~pulpcore.exceptions.SizeValidationError`: When the ``expected_size`` value doesn't match the size of the data :class:`~pulpcore.exceptions.UnsupportedDigestValidationError`: When any of the ``expected_digest`` algorithms aren't in the ALLOWED_CONTENT_CHECKSUMS list Returns: An in-memory, unsaved :class:`~pulpcore.plugin.models.PulpTemporaryFile` """ if not expected_digests and not expected_size: return PulpTemporaryFile(file=file) if isinstance(file, str): with open(file, "rb") as f: hashers = {n: pulp_hashlib.new(n) for n in expected_digests.keys()} size = 0 while True: chunk = f.read(1048576) # 1 megabyte if not chunk: break for algorithm in hashers.values(): algorithm.update(chunk) size = size + len(chunk) else: size = file.size hashers = file.hashers if expected_size: if size != expected_size: raise SizeValidationError() if expected_digests: for algorithm, expected_digest in expected_digests.items(): if algorithm not in hashers: raise UnsupportedDigestValidationError( _("Checksum algorithm {} forbidden for this Pulp instance.").format( algorithm ) ) if expected_digest != hashers[algorithm].hexdigest(): raise DigestValidationError() return PulpTemporaryFile(file=file)
class Artifact(Model): """ A file associated with a piece of content. When calling `save()` on an Artifact, if the file is not stored in Django's storage backend, it is moved into place then. Artifact is compatible with Django's `bulk_create()` method. Fields: file (models.FileField): The stored file. This field should be set using an absolute path to a temporary file. It also accepts `class:django.core.files.File`. size (models.IntegerField): The size of the file in bytes. md5 (models.CharField): The MD5 checksum of the file. sha1 (models.CharField): The SHA-1 checksum of the file. sha224 (models.CharField): The SHA-224 checksum of the file. sha256 (models.CharField): The SHA-256 checksum of the file. sha384 (models.CharField): The SHA-384 checksum of the file. sha512 (models.CharField): The SHA-512 checksum of the file. """ def storage_path(self, name): """ Callable used by FileField to determine where the uploaded file should be stored. Args: name (str): Original name of uploaded file. It is ignored by this method because the sha256 checksum is used to determine a file path instead. """ return storage.get_artifact_path(self.sha256) file = fields.ArtifactFileField(blank=False, null=False, upload_to=storage_path, max_length=255) size = models.IntegerField(blank=False, null=False) md5 = models.CharField(max_length=32, blank=False, null=False, unique=False, db_index=True) sha1 = models.CharField(max_length=40, blank=False, null=False, unique=False, db_index=True) sha224 = models.CharField(max_length=56, blank=False, null=False, unique=False, db_index=True) sha256 = models.CharField(max_length=64, blank=False, null=False, unique=True, db_index=True) sha384 = models.CharField(max_length=96, blank=False, null=False, unique=True, db_index=True) sha512 = models.CharField(max_length=128, blank=False, null=False, unique=True, db_index=True) # All digest fields ordered by algorithm strength. DIGEST_FIELDS = ( 'sha512', 'sha384', 'sha256', 'sha224', 'sha1', 'md5', ) # Reliable digest fields ordered by algorithm strength. RELIABLE_DIGEST_FIELDS = DIGEST_FIELDS[:-3] def is_equal(self, other): """ Is equal by matching digest. Args: other (pulpcore.app.models.Artifact): A artifact to match. Returns: bool: True when equal. """ for field in Artifact.RELIABLE_DIGEST_FIELDS: digest = getattr(self, field) if not digest: continue if digest == getattr(other, field): return True return False def save(self, *args, **kwargs): """ Saves Artifact model and closes the file associated with the Artifact Args: args (list): list of positional arguments for Model.save() kwargs (dict): dictionary of keyword arguments to pass to Model.save() """ try: super().save(*args, **kwargs) self.file.close() except Exception: self.file.close() raise def delete(self, *args, **kwargs): """ Deletes Artifact model and the file associated with the Artifact Args: args (list): list of positional arguments for Model.delete() kwargs (dict): dictionary of keyword arguments to pass to Model.delete() """ super().delete(*args, **kwargs) self.file.delete(save=False)