def _verify_checksum(self, unit, report): """ Verifies the checksum of the given unit if the sync is configured to do so. If the verification fails, the error is noted in this instance's progress report and the error is re-raised. :param unit: domain model instance of the package that was downloaded :type unit: pulp_rpm.plugins.db.models.NonMetadataPackage :param report: report handed to this listener by the downloader :type report: nectar.report.DownloadReport :raises verification.VerificationException: if the checksum of the content is incorrect """ if not self.sync.config.get(importer_constants.KEY_VALIDATE): return with open(report.destination) as fp: sums = util.calculate_checksums(fp, [util.TYPE_MD5, util.TYPE_SHA1, util.TYPE_SHA256]) if sums[unit.checksumtype] != unit.checksum: error_report = { constants.NAME: unit.name, constants.ERROR_CODE: constants.ERROR_CHECKSUM_VERIFICATION, constants.CHECKSUM_TYPE: unit.checksumtype, constants.ERROR_KEY_CHECKSUM_EXPECTED: unit.checksum, constants.ERROR_KEY_CHECKSUM_ACTUAL: sums[unit.checksumtype] } self.sync.progress_report['content'].failure(unit, error_report) # I don't know why the argument is the calculated sum, but that's the pre-existing # behavior in pulp.server.util.verify_checksum raise verification.VerificationException(sums[unit.checksumtype]) else: # The unit will be saved later in the workflow, after the file is moved into place. unit.checksums.update(sums)
def migrate(*args, **kwargs): """ Add seperate fields for md5sum, sha1, sha256 """ warnings_encountered = False deb_collection = connection.get_collection('units_deb') deb_count = deb_collection.count() with migration_utils.MigrationProgressLog('Deb Package', deb_count) as progress_log: for deb_package in deb_collection.find({}).batch_size(100): storage_path = deb_package['_storage_path'] package_id = deb_package['_id'] if not os.path.exists(storage_path): warnings_encountered = True msg = 'deb package file corresponding to db_unit with _id = {}\n'\ 'was not found at _storage_path = {}.\n'\ 'The unit was not migrated!'.format(package_id, storage_path) _logger.warn(msg) continue with open(storage_path, 'r') as file_handle: checksums = server_utils.calculate_checksums( file_handle, CHECKSUM_TYPES) new_fields = { 'md5sum': checksums[server_utils.TYPE_MD5], 'sha1': checksums[server_utils.TYPE_SHA1], 'sha256': checksums[server_utils.TYPE_SHA256], } if checksums[ deb_package['checksumtype']] != deb_package['checksum']: raise Exception( 'New checksum does not match existing checksum for\n' '_id = {}\nfile = {}'.format(package_id, storage_path)) deb_collection.update_one( {'_id': package_id}, {'$set': new_fields}, ) progress_log.progress() if warnings_encountered: msg = 'Warnings were encountered during the db migration!\n'\ 'Check the logs for more information, and consider deleting broken units.' _logger.warn(msg)
def validate(self, unit, *args): """Check the unit. :param unit: the unit to check :type unit: dict :param args: unused :returns: None :raises: MISSING_ERROR/CHECKSUM_ERROR """ checksumtype, expected_checksum, _storage_path = self.get_unit_attributes( unit, 'checksumtype', 'checksum', '_storage_path') try: with open(_storage_path, 'rb') as fd: checksums = util.calculate_checksums(fd, [checksumtype]) except (IOError, OSError): raise MISSING_ERROR if checksums[checksumtype] != expected_checksum: raise CHECKSUM_ERROR
def verify_checksum(file_object, checksum_type, checksum_value): """ Returns whether or not the checksum of the contents of the given file-like object match the expectation. :param file_object: file-like object to verify :type file_object: file-like object :param checksum_type: type of checksum to calculate; must be one of the TYPE_* constants in this module :type checksum_type: str :param checksum_value: expected checksum to verify against :type checksum_value: str :raises ValueError: if the checksum_type isn't one of the TYPE_* constants """ calculated_sum = calculate_checksums(file_object, [checksum_type])[checksum_type] if calculated_sum != checksum_value: raise VerificationException(calculated_sum)
def _verify_checksum(self, unit, location): """ Verifies the checksum of the given unit if the sync is configured to do so. If the verification fails, the error is noted in this instance's progress report and the error is re-raised. :param unit: domain model instance of the package that was downloaded :type unit: pulp_rpm.plugins.db.models.NonMetadataPackage :param location: location of the unit that needs to be verified :type location: str :raises verification.VerificationException: if the checksum of the content is incorrect """ if not self.sync.config.get(importer_constants.KEY_VALIDATE): return with open(location) as fp: sums = util.calculate_checksums( fp, [util.TYPE_MD5, util.TYPE_SHA1, util.TYPE_SHA256]) if sums[unit.checksumtype] != unit.checksum: error_report = { constants.NAME: unit.name, constants.ERROR_CODE: constants.ERROR_CHECKSUM_VERIFICATION, constants.CHECKSUM_TYPE: unit.checksumtype, constants.ERROR_KEY_CHECKSUM_EXPECTED: unit.checksum, constants.ERROR_KEY_CHECKSUM_ACTUAL: sums[unit.checksumtype] } self.sync.progress_report['content'].failure(unit, error_report) # I don't know why the argument is the calculated sum, but that's the pre-existing # behavior in pulp.server.util.verify_checksum raise verification.VerificationException(sums[unit.checksumtype]) else: # The unit will be saved later in the workflow, after the file is moved into place. unit.checksums.update(sums)
def _compute_checksum(cls, fobj): cstype = util.TYPE_SHA256 return util.calculate_checksums(fobj, [cstype])[cstype]
def test_one(self): ret = util.calculate_checksums(self.f, ['sha256']) self.assertEqual(ret['sha256'], self.sha256_sum) self.assertTrue(len(ret), 1)
def test_with_data(self): ret = util.calculate_checksums(self.f, ['sha1', 'sha256']) self.assertEqual(ret['sha1'], self.sha1_sum) self.assertEqual(ret['sha256'], self.sha256_sum)
def _handle_package(repo, type_id, unit_key, metadata, file_path, conduit, config): """ Handles the upload for an RPM, SRPM or DRPM. This inspects the package contents to determine field values. The unit_key and metadata fields overwrite field values determined through package inspection. :param repo: The repository to import the package into :type repo: pulp.server.db.model.Repository :param type_id: The type_id of the package being uploaded :type type_id: str :param unit_key: A dictionary of fields to overwrite introspected field values :type unit_key: dict :param metadata: A dictionary of fields to overwrite introspected field values, or None :type metadata: dict or None :param file_path: The path to the uploaded package :type file_path: str :param conduit: provides access to relevant Pulp functionality :type conduit: pulp.plugins.conduits.upload.UploadConduit :param config: plugin configuration for the repository :type config: pulp.plugins.config.PluginCallConfiguration :raises PulpCodedException PLP1005: if the checksum type from the user is not recognized :raises PulpCodedException PLP1013: if the checksum value from the user does not validate """ try: if type_id == models.DRPM._content_type_id.default: rpm_data = _extract_drpm_data(file_path) else: rpm_data = _extract_rpm_data(type_id, file_path) except: _LOGGER.exception('Error extracting RPM metadata for [%s]' % file_path) raise # metadata can be None metadata = metadata or {} model_class = plugin_api.get_unit_model_by_id(type_id) update_fields_inbound(model_class, unit_key or {}) update_fields_inbound(model_class, metadata or {}) with open(file_path) as fp: sums = util.calculate_checksums(fp, models.RpmBase.DEFAULT_CHECKSUM_TYPES) # validate checksum if possible if metadata.get('checksum'): checksumtype = metadata.pop('checksum_type', util.TYPE_SHA256) checksumtype = util.sanitize_checksum_type(checksumtype) if checksumtype not in sums: raise PulpCodedException(error_code=error_codes.RPM1009, checksumtype=checksumtype) if metadata['checksum'] != sums[checksumtype]: raise PulpCodedException(error_code=platform_errors.PLP1013) _LOGGER.debug(_('Upload checksum matches.')) # Save all uploaded RPMs with sha256 in the unit key, since we can now publish with other # types, regardless of what is in the unit key. rpm_data['checksumtype'] = util.TYPE_SHA256 rpm_data['checksum'] = sums[util.TYPE_SHA256] # keep all available checksum values on the model rpm_data['checksums'] = sums # Update the RPM-extracted data with anything additional the user specified. # Allow the user-specified values to override the extracted ones. rpm_data.update(metadata or {}) rpm_data.update(unit_key or {}) # Validate the user specified data by instantiating the model try: unit = model_class(**rpm_data) except TypeError: raise ModelInstantiationError() if type_id != models.DRPM._content_type_id.default: # Extract/adjust the repodata snippets repodata = rpm_parse.get_package_xml(file_path, sumtype=unit.checksumtype) _update_provides_requires(unit, repodata) _update_files(unit, repodata) unit.modify_xml(repodata) # check if the unit has duplicate nevra purge.remove_unit_duplicate_nevra(unit, repo) unit.set_storage_path(os.path.basename(file_path)) try: unit.save_and_import_content(file_path) except NotUniqueError: unit = unit.__class__.objects.filter(**unit.unit_key).first() if rpm_parse.signature_enabled(config): rpm_parse.filter_signature(unit, config) repo_controller.associate_single_unit(repo, unit)
def _handle_package(repo, type_id, unit_key, metadata, file_path, conduit, config): """ Handles the upload for an RPM or SRPM. This inspects the package contents to determine field values. The unit_key and metadata fields overwrite field values determined through package inspection. :param repo: The repository to import the package into :type repo: pulp.server.db.model.Repository :param type_id: The type_id of the package being uploaded :type type_id: str :param unit_key: A dictionary of fields to overwrite introspected field values :type unit_key: dict :param metadata: A dictionary of fields to overwrite introspected field values, or None :type metadata: dict or None :param file_path: The path to the uploaded package :type file_path: str :param conduit: provides access to relevant Pulp functionality :type conduit: pulp.plugins.conduits.upload.UploadConduit :param config: plugin configuration for the repository :type config: pulp.plugins.config.PluginCallConfiguration :raises PulpCodedException PLP1005: if the checksum type from the user is not recognized :raises PulpCodedException PLP1013: if the checksum value from the user does not validate """ try: rpm_data = _extract_rpm_data(type_id, file_path) except: _LOGGER.exception('Error extracting RPM metadata for [%s]' % file_path) raise # metadata can be None metadata = metadata or {} model_class = plugin_api.get_unit_model_by_id(type_id) update_fields_inbound(model_class, unit_key or {}) update_fields_inbound(model_class, metadata or {}) with open(file_path) as fp: sums = util.calculate_checksums(fp, models.RpmBase.DEFAULT_CHECKSUM_TYPES) # validate checksum if possible if metadata.get('checksum'): checksumtype = metadata.pop('checksum_type', util.TYPE_SHA256) checksumtype = util.sanitize_checksum_type(checksumtype) if checksumtype not in sums: raise PulpCodedException(error_code=error_codes.RPM1009, checksumtype=checksumtype) if metadata['checksum'] != sums[checksumtype]: raise PulpCodedException(error_code=platform_errors.PLP1013) _LOGGER.debug(_('Upload checksum matches.')) # Save all uploaded RPMs with sha256 in the unit key, since we can now publish with other # types, regardless of what is in the unit key. rpm_data['checksumtype'] = util.TYPE_SHA256 rpm_data['checksum'] = sums[util.TYPE_SHA256] # keep all available checksum values on the model rpm_data['checksums'] = sums # Update the RPM-extracted data with anything additional the user specified. # Allow the user-specified values to override the extracted ones. rpm_data.update(metadata or {}) rpm_data.update(unit_key or {}) # Validate the user specified data by instantiating the model try: unit = model_class(**rpm_data) except TypeError: raise ModelInstantiationError() # Extract/adjust the repodata snippets unit.repodata = rpm_parse.get_package_xml(file_path, sumtype=unit.checksumtype) _update_provides_requires(unit) unit.modify_xml() # check if the unit has duplicate nevra purge.remove_unit_duplicate_nevra(unit, repo) unit.set_storage_path(os.path.basename(file_path)) try: unit.save_and_import_content(file_path) except NotUniqueError: unit = unit.__class__.objects.filter(**unit.unit_key).first() repo_controller.associate_single_unit(repo, unit)
def _handle_package(repo, type_id, unit_key, metadata, file_path, conduit, config): """ Handles the upload for an RPM, SRPM or DRPM. This inspects the package contents to determine field values. The unit_key and metadata fields overwrite field values determined through package inspection. :param repo: The repository to import the package into :type repo: pulp.server.db.model.Repository :param type_id: The type_id of the package being uploaded :type type_id: str :param unit_key: A dictionary of fields to overwrite introspected field values :type unit_key: dict :param metadata: A dictionary of fields to overwrite introspected field values, or None :type metadata: dict or None :param file_path: The path to the uploaded package :type file_path: str :param conduit: provides access to relevant Pulp functionality :type conduit: pulp.plugins.conduits.upload.UploadConduit :param config: plugin configuration for the repository :type config: pulp.plugins.config.PluginCallConfiguration :raises PulpCodedException PLP1005: if the checksum type from the user is not recognized :raises PulpCodedException PLP1013: if the checksum value from the user does not validate """ try: if type_id == models.DRPM._content_type_id.default: unit = models.DRPM(**_extract_drpm_data(file_path)) else: repodata = rpm_parse.get_package_xml(file_path, sumtype=util.TYPE_SHA256) package_xml = (utils.fake_xml_element( repodata['primary'], constants.COMMON_NAMESPACE).find(primary.PACKAGE_TAG)) unit = primary.process_package_element(package_xml) except Exception: raise PulpCodedException(error_codes.RPM1016) # metadata can be None metadata = metadata or {} model_class = plugin_api.get_unit_model_by_id(type_id) update_fields_inbound(model_class, unit_key or {}) update_fields_inbound(model_class, metadata or {}) with open(file_path) as fp: sums = util.calculate_checksums(fp, models.RpmBase.DEFAULT_CHECKSUM_TYPES) # validate checksum if possible if metadata.get('checksum'): checksumtype = metadata.pop('checksum_type', util.TYPE_SHA256) checksumtype = util.sanitize_checksum_type(checksumtype) if checksumtype not in sums: raise PulpCodedException(error_code=error_codes.RPM1009, checksumtype=checksumtype) if metadata['checksum'] != sums[checksumtype]: raise PulpCodedException(error_code=platform_errors.PLP1013) _LOGGER.debug(_('Upload checksum matches.')) # Save all uploaded RPMs with sha256 in the unit key, since we can now publish with other # types, regardless of what is in the unit key. unit.checksumtype = util.TYPE_SHA256 unit.checksum = sums[util.TYPE_SHA256] # keep all available checksum values on the model unit.checksums = sums # Update the RPM-extracted data with anything additional the user specified. # Allow the user-specified values to override the extracted ones. for key, value in metadata.items(): setattr(unit, key, value) for key, value in unit_key.items(): setattr(unit, key, value) if type_id != models.DRPM._content_type_id.default: # Extract/adjust the repodata snippets unit.signing_key = rpm_parse.package_signature( rpm_parse.package_headers(file_path)) # construct filename from metadata (BZ #1101168) if type_id == models.SRPM._content_type_id.default: rpm_basefilename = "%s-%s-%s.src.rpm" % (unit.name, unit.version, unit.release) else: rpm_basefilename = "%s-%s-%s.%s.rpm" % (unit.name, unit.version, unit.release, unit.arch) unit.relativepath = rpm_basefilename unit.filename = rpm_basefilename _update_files(unit, repodata) unit.modify_xml(repodata) # check if the unit has duplicate nevra purge.remove_unit_duplicate_nevra(unit, repo) unit.set_storage_path(os.path.basename(file_path)) try: unit.save_and_import_content(file_path) except TypeError: raise ModelInstantiationError() except NotUniqueError: unit = unit.__class__.objects.filter(**unit.unit_key).first() if rpm_parse.signature_enabled(config): rpm_parse.filter_signature(unit, config) repo_controller.associate_single_unit(repo, unit)