def validate_file(file_path, checksum_algorithm, checksum):
"""
Attempts to validate the checksum of file referenced by the catalog entry. If
the checksum and checksum algorithm is not available, this method simply checks
that the file exists.
:param file_path: Absolute path to the file to validate.
:type file_path: str
:param checksum_algorithm: Algorithm used to generate the provided checksum.
:type checksum_algorithm: str
:param checksum: The expected checksum to verify against.
:type checksum: str
:raises IOError: If self.path is not a file.
:raises InvalidChecksumType: If the checksum algorithm is not supported by
pulp.plugins.utils.verification.
:raises VerificationException: If the calculated checksum does not match the
one provided in the report.
"""
if checksum_algorithm and checksum:
with open(file_path) as f:
verify_checksum(f, checksum_algorithm, checksum)
else:
if not os.path.isfile(file_path):
raise IOError(
_("The path '{path}' does not exist").format(
path=file_path))
def _verify_checksum(self, unit, report):
"""
Verifies the checksum of the given unit if the sync is configured to do so.
If the verification fails, the error is noted in this instance's progress
report and the error is re-raised.
:param unit: domain model instance of the package that was downloaded
:type unit: pulp_rpm.plugins.db.models.RpmBase
:param report: report handed to this listener by the downloader
:type report: nectar.report.DownloadReport
:raises verification.VerificationException: if the checksum of the content is incorrect
"""
if not self.sync.config.get(importer_constants.KEY_VALIDATE):
return
try:
with open(report.destination) as fp:
verification.verify_checksum(fp, unit.checksumtype, unit.checksum)
except verification.VerificationException, e:
error_report = {
constants.NAME: unit.name,
constants.ERROR_CODE: constants.ERROR_CHECKSUM_VERIFICATION,
constants.CHECKSUM_TYPE: unit.checksumtype,
constants.ERROR_KEY_CHECKSUM_EXPECTED: unit.checksum,
constants.ERROR_KEY_CHECKSUM_ACTUAL: e[0]
}
self.sync.progress_report['content'].failure(unit, error_report)
raise
def validate_file(file_path, checksum_algorithm, checksum):
"""
Attempts to validate the checksum of file referenced by the catalog entry. If
the checksum and checksum algorithm is not available, this method simply checks
that the file exists.
:param file_path: Absolute path to the file to validate.
:type file_path: str
:param checksum_algorithm: Algorithm used to generate the provided checksum.
:type checksum_algorithm: str
:param checksum: The expected checksum to verify against.
:type checksum: str
:raises IOError: If self.path is not a file.
:raises InvalidChecksumType: If the checksum algorithm is not supported by
pulp.plugins.utils.verification.
:raises VerificationException: If the calculated checksum does not match the
one provided in the report.
"""
if checksum_algorithm and checksum:
with open(file_path) as f:
verify_checksum(f, checksum_algorithm, checksum)
else:
if not os.path.isfile(file_path):
raise IOError(_("The path '{path}' does not exist").format(path=file_path))
def _verify_checksum(self, unit, report):
"""
Verifies the checksum of the given unit if the sync is configured to do so.
If the verification fails, the error is noted in this instance's progress
report and the error is re-raised.
:param unit: domain model instance of the package that was downloaded
:type unit: pulp_rpm.plugins.db.models.RpmBase
:param report: report handed to this listener by the downloader
:type report: nectar.report.DownloadReport
:raises verification.VerificationException: if the checksum of the content is incorrect
"""
if not self.sync.config.get(importer_constants.KEY_VALIDATE):
return
try:
with open(report.destination) as fp:
verification.verify_checksum(fp, unit.checksumtype,
unit.checksum)
except verification.VerificationException, e:
error_report = {
constants.NAME: unit.name,
constants.ERROR_CODE: constants.ERROR_CHECKSUM_VERIFICATION,
constants.CHECKSUM_TYPE: unit.checksumtype,
constants.ERROR_KEY_CHECKSUM_EXPECTED: unit.checksum,
constants.ERROR_KEY_CHECKSUM_ACTUAL: e[0]
}
self.sync.progress_report['content'].failure(unit, error_report)
raise
def test_checksum_sha(self):
# Setup (sha is an alias for sha1)
test_file = StringIO('Test Data')
expected_checksum = 'cae99c6102aa3596ff9b86c73881154e340c2ea8'
# Test - Should not raise an exception
verification.verify_checksum(test_file, verification.TYPE_SHA, expected_checksum)
def test_checksum_sha256(self):
# Setup
test_file = StringIO('Test data')
expected_checksum = 'e27c8214be8b7cf5bccc7c08247e3cb0c1514a48ee1f63197fe4ef3ef51d7e6f'
# Test - Should not raise an exception
verification.verify_checksum(test_file, verification.TYPE_SHA256, expected_checksum)
def test_checksum_sha256(self):
# Setup
test_file = StringIO('Test data')
expected_checksum = 'e27c8214be8b7cf5bccc7c08247e3cb0c1514a48ee1f63197fe4ef3ef51d7e6f'
# Test - Should not raise an exception
verification.verify_checksum(test_file, verification.TYPE_SHA256, expected_checksum)
def test_checksum_sha(self):
# Setup (sha is an alias for sha1)
test_file = StringIO('Test Data')
expected_checksum = 'cae99c6102aa3596ff9b86c73881154e340c2ea8'
# Test - Should not raise an exception
verification.verify_checksum(test_file, verification.TYPE_SHA, expected_checksum)
def _verify_checksum(self, model, report):
"""
Verifies the checksum of the given unit if the sync is configured to do so. If the verification
fails, the error is noted in this instance's progress report and the error is re-raised.
:param model: domain model instance of the package that was downloaded
:type model: pulp_rpm.common.models.RPM
:param report: report handed to this listener by the downloader
:type report: nectar.report.DownloadReport
:raises verification.VerificationException: if the checksum of the content is incorrect
"""
if not self.sync_call_config.get(importer_constants.KEY_VALIDATE):
return
try:
with open(report.destination) as dest_file:
verification.verify_checksum(dest_file, model.unit_key['checksumtype'],
model.unit_key['checksum'])
except verification.VerificationException, e:
error_report = {
constants.NAME: model.unit_key['name'],
constants.ERROR_CODE: constants.ERROR_CHECKSUM_VERIFICATION,
constants.CHECKSUM_TYPE: model.unit_key['checksumtype'],
constants.ERROR_KEY_CHECKSUM_EXPECTED: model.unit_key['checksum'],
constants.ERROR_KEY_CHECKSUM_ACTUAL: e[0]
}
self.progress_report['content'].failure(model, error_report)
raise
def process_main(self, item=None):
"""
Save blobs and manifest to repository
:param item: A Docker manifest or blob unit
:type : pulp_docker.plugins.models.Blob or pulp_docker.plugins.models.Manifest
:return: None
"""
if isinstance(item, models.Blob):
checksum_type, _, checksum = item.digest.rpartition(':')
if not checksum_type:
# Never assume. But oh well
checksum_type = "sha256"
blob_src_path = os.path.join(self.get_working_dir(),
checksum + '.tar')
try:
fobj = open(blob_src_path)
except IOError:
raise PulpCodedValidationException(
error_code=error_codes.DKR1018,
layer=os.path.basename(blob_src_path))
try:
verification.verify_checksum(fobj, checksum_type, checksum)
except verification.VerificationException:
raise PulpCodedValidationException(
error_code=error_codes.DKR1017,
checksum_type=checksum_type,
checksum=checksum)
fobj.close()
blob_dest_path = os.path.join(self.get_working_dir(), item.digest)
os.rename(blob_src_path, blob_dest_path)
item.set_storage_path(item.digest)
try:
item.save_and_import_content(
os.path.join(self.get_working_dir(), item.digest))
except NotUniqueError:
item = item.__class__.objects.get(**item.unit_key)
repository.associate_single_unit(self.get_repo().repo_obj, item)
if isinstance(item, models.Manifest):
self.parent.uploaded_unit = item
def _handle_package(repo, type_id, unit_key, metadata, file_path, conduit, config):
"""
Handles the upload for an RPM or SRPM.
This inspects the package contents to determine field values. The unit_key
and metadata fields overwrite field values determined through package inspection.
:param repo: The repository to import the package into
:type repo: pulp.server.db.model.Repository
:param type_id: The type_id of the package being uploaded
:type type_id: str
:param unit_key: A dictionary of fields to overwrite introspected field values
:type unit_key: dict
:param metadata: A dictionary of fields to overwrite introspected field values, or None
:type metadata: dict or None
:param file_path: The path to the uploaded package
:type file_path: str
:param conduit: provides access to relevant Pulp functionality
:type conduit: pulp.plugins.conduits.upload.UploadConduit
:param config: plugin configuration for the repository
:type config: pulp.plugins.config.PluginCallConfiguration
:raises PulpCodedException PLP1005: if the checksum type from the user is not recognized
:raises PulpCodedException PLP1013: if the checksum value from the user does not validate
"""
try:
rpm_data = _extract_rpm_data(type_id, file_path)
except:
_LOGGER.exception('Error extracting RPM metadata for [%s]' % file_path)
raise
model_class = plugin_api.get_unit_model_by_id(type_id)
update_fields_inbound(model_class, unit_key or {})
update_fields_inbound(model_class, metadata or {})
# set checksum and checksumtype
if metadata:
checksumtype = metadata.pop('checksumtype', verification.TYPE_SHA256)
rpm_data['checksumtype'] = verification.sanitize_checksum_type(checksumtype)
if 'checksum' in metadata:
rpm_data['checksum'] = metadata.pop('checksum')
try:
with open(file_path) as dest_file:
verification.verify_checksum(dest_file, rpm_data['checksumtype'],
rpm_data['checksum'])
except verification.VerificationException:
raise PulpCodedException(error_code=platform_errors.PLP1013)
else:
rpm_data['checksum'] = _calculate_checksum(rpm_data['checksumtype'], file_path)
else:
rpm_data['checksumtype'] = verification.TYPE_SHA256
rpm_data['checksum'] = _calculate_checksum(rpm_data['checksumtype'], file_path)
# Update the RPM-extracted data with anything additional the user specified.
# Allow the user-specified values to override the extracted ones.
rpm_data.update(metadata or {})
rpm_data.update(unit_key or {})
# Validate the user specified data by instantiating the model
try:
unit = model_class(**rpm_data)
except TypeError:
raise ModelInstantiationError()
# Extract/adjust the repodata snippets
unit.repodata = rpm_parse.get_package_xml(file_path, sumtype=unit.checksumtype)
_update_provides_requires(unit)
_update_location(unit)
# check if the unit has duplicate nevra
purge.remove_unit_duplicate_nevra(unit, repo)
unit.set_storage_path(os.path.basename(file_path))
try:
unit.save_and_import_content(file_path)
except NotUniqueError:
unit = unit.__class__.objects.filter(**unit.unit_key).first()
repo_controller.associate_single_unit(repo, unit)
def _handle_package(repo, type_id, unit_key, metadata, file_path, conduit, config):
"""
Handles the upload for an RPM or SRPM.
This inspects the package contents to determine field values. The unit_key
and metadata fields overwrite field values determined through package inspection.
:param repo: The repository to import the package into
:type repo: pulp.server.db.model.Repository
:param type_id: The type_id of the package being uploaded
:type type_id: str
:param unit_key: A dictionary of fields to overwrite introspected field values
:type unit_key: dict
:param metadata: A dictionary of fields to overwrite introspected field values, or None
:type metadata: dict or None
:param file_path: The path to the uploaded package
:type file_path: str
:param conduit: provides access to relevant Pulp functionality
:type conduit: pulp.plugins.conduits.upload.UploadConduit
:param config: plugin configuration for the repository
:type config: pulp.plugins.config.PluginCallConfiguration
:raises PulpCodedException PLP1005: if the checksum type from the user is not recognized
:raises PulpCodedException PLP1013: if the checksum value from the user does not validate
"""
try:
rpm_data = _extract_rpm_data(type_id, file_path)
except:
_LOGGER.exception('Error extracting RPM metadata for [%s]' % file_path)
raise
model_class = plugin_api.get_unit_model_by_id(type_id)
update_fields_inbound(model_class, unit_key or {})
update_fields_inbound(model_class, metadata or {})
# set checksum and checksumtype
if metadata:
checksumtype = metadata.pop('checksumtype', verification.TYPE_SHA256)
rpm_data['checksumtype'] = verification.sanitize_checksum_type(checksumtype)
if 'checksum' in metadata:
rpm_data['checksum'] = metadata.pop('checksum')
try:
with open(file_path) as dest_file:
verification.verify_checksum(dest_file, rpm_data['checksumtype'],
rpm_data['checksum'])
except verification.VerificationException:
raise PulpCodedException(error_code=platform_errors.PLP1013)
else:
rpm_data['checksum'] = _calculate_checksum(rpm_data['checksumtype'], file_path)
else:
rpm_data['checksumtype'] = verification.TYPE_SHA256
rpm_data['checksum'] = _calculate_checksum(rpm_data['checksumtype'], file_path)
# Update the RPM-extracted data with anything additional the user specified.
# Allow the user-specified values to override the extracted ones.
rpm_data.update(metadata or {})
rpm_data.update(unit_key or {})
# Validate the user specified data by instantiating the model
try:
unit = model_class(**rpm_data)
except TypeError:
raise ModelInstantiationError()
# Extract the repodata snippets
unit.repodata = rpm_parse.get_package_xml(file_path, sumtype=unit.checksumtype)
_update_provides_requires(unit)
# check if the unit has duplicate nevra
purge.remove_unit_duplicate_nevra(unit, repo)
unit.set_storage_path(os.path.basename(file_path))
unit.save_and_import_content(file_path)
repo_controller.associate_single_unit(repo, unit)
