def init_and_validate(file, expected_digests=None, expected_size=None):
"""
Initialize an in-memory PulpTemporaryFile from a file, and validate digest and size info.
This accepts both a path to a file on-disk or a
:class:`~pulpcore.app.files.PulpTemporaryUploadedFile`.
Args:
file (:class:`~pulpcore.app.files.PulpTemporaryUploadedFile` or str): The
PulpTemporaryUploadedFile to create the PulpTemporaryFile from or a string with the
full path to the file on disk.
expected_digests (dict): Keyed on the algorithm name provided by hashlib and stores the
value of the expected digest. e.g. {'md5': '912ec803b2ce49e4a541068d495ab570'}
expected_size (int): The number of bytes the download is expected to have.
Raises:
:class:`~pulpcore.exceptions.DigestValidationError`: When any of the ``expected_digest``
values don't match the digest of the data
:class:`~pulpcore.exceptions.SizeValidationError`: When the ``expected_size`` value
doesn't match the size of the data
:class:`~pulpcore.exceptions.UnsupportedDigestValidationError`: When any of the
``expected_digest`` algorithms aren't in the ALLOWED_CONTENT_CHECKSUMS list
Returns:
An in-memory, unsaved :class:`~pulpcore.plugin.models.PulpTemporaryFile`
"""
if not expected_digests and not expected_size:
return PulpTemporaryFile(file=file)
if isinstance(file, str):
with open(file, "rb") as f:
hashers = {n: pulp_hashlib.new(n) for n in expected_digests.keys()}
size = 0
while True:
chunk = f.read(1048576) # 1 megabyte
if not chunk:
break
for algorithm in hashers.values():
algorithm.update(chunk)
size = size + len(chunk)
else:
size = file.size
hashers = file.hashers
if expected_size:
if size != expected_size:
raise SizeValidationError()
if expected_digests:
for algorithm, expected_digest in expected_digests.items():
if algorithm not in hashers:
raise UnsupportedDigestValidationError(
_("Checksum algorithm {} forbidden for this Pulp instance.").format(
algorithm
)
)
if expected_digest != hashers[algorithm].hexdigest():
raise DigestValidationError()
return PulpTemporaryFile(file=file)
def before_save(self):
"""
Pre-save hook that validates checksum rules prior to allowing an Artifact to be saved.
An Artifact with any checksums from the FORBIDDEN set will fail to save while raising
an UnsupportedDigestValidationError exception.
Similarly, any checksums in DIGEST_FIELDS that is NOT set will raise a
MissingDigestValidationError exception.
Raises:
:class: `~pulpcore.exceptions.UnsupportedDigestValidationError`: When any of the
keys on FORBIDDEN_DIGESTS are set for the Artifact
:class: `~pulpcore.exceptions.MissingDigestValidationError`: When any of the
keys on DIGEST_FIELDS are found to be missing from the Artifact
"""
bad_keys = [k for k in self.FORBIDDEN_DIGESTS if getattr(self, k)]
if bad_keys:
raise UnsupportedDigestValidationError(
_("Checksum algorithms {} are forbidden for this Pulp instance.").format(bad_keys)
)
missing_keys = [k for k in self.DIGEST_FIELDS if not getattr(self, k)]
if missing_keys:
raise MissingDigestValidationError(
_("Missing required checksum algorithms {}.").format(missing_keys)
)
def __init__(
self,
url,
custom_file_object=None,
expected_digests=None,
expected_size=None,
semaphore=None,
*args,
**kwargs,
):
"""
Create a BaseDownloader object. This is expected to be called by all subclasses.
Args:
url (str): The url to download.
custom_file_object (file object): An open, writable file object that downloaded data
can be written to by
:meth:`~pulpcore.plugin.download.BaseDownloader.handle_data`.
expected_digests (dict): Keyed on the algorithm name provided by hashlib and stores the
value of the expected digest. e.g. {'md5': '912ec803b2ce49e4a541068d495ab570'}
expected_size (int): The number of bytes the download is expected to have.
semaphore (asyncio.Semaphore): A semaphore the downloader must acquire before running.
Useful for limiting the number of outstanding downloaders in various ways.
"""
if custom_file_object:
deprecation_logger.warn(
"The 'custom_file_object' argument to 'BaseDownloader' is"
"deprecated and will be removed in pulpcore==3.20; stop using it."
)
self.url = url
self._writer = custom_file_object
self.path = None
self.expected_digests = expected_digests
self.expected_size = expected_size
if semaphore:
self.semaphore = semaphore
else:
self.semaphore = asyncio.Semaphore(
) # This will always be acquired
self._digests = {}
self._size = 0
if self.expected_digests:
if not set(self.expected_digests).intersection(
set(Artifact.DIGEST_FIELDS)):
raise UnsupportedDigestValidationError(
_("Content at the url {} does not contain at least one trusted hasher which"
" is specified in 'ALLOWED_CONTENT_CHECKSUMS' setting.").
format(self.url))
def validate_checksums(self):
"""Validate if RemoteArtifact has allowed checksum or no checksum at all."""
if not any([
checksum_type for checksum_type in ALL_KNOWN_CONTENT_CHECKSUMS
if getattr(self, checksum_type, False)
]):
return
if not any([
checksum_type for checksum_type in Artifact.DIGEST_FIELDS
if getattr(self, checksum_type, False)
]):
raise UnsupportedDigestValidationError(
_("On-demand content located at the url {} contains forbidden checksum type,"
"thus cannot be synced."
"You can allow checksum type with 'ALLOWED_CONTENT_CHECKSUMS' setting."
).format(self.url))