def process_distribution(feed, tmp_dir, nectar_config, model, report): """ Get the pulp_distribution.xml file from the server and if it exists download all the files it references to add them to the distribution unit. :param feed: URL to the repository :type feed: str :param tmp_dir: full path to the temporary directory being used :type tmp_dir: str :param nectar_config: download config to be used by nectar :type nectar_config: nectar.config.DownloaderConfig :param model: :type model: :param report: :type report: :return: list of file dictionaries :rtype: list of dict """ # Get the Distribution file result = get_distribution_file(feed, tmp_dir, nectar_config) files = [] # If there is a Distribution file - parse it and add all files to the file_list if result: xsd = os.path.join(constants.USR_SHARE_DIR, 'pulp_distribution.xsd') schema_doc = ET.parse(xsd) xmlschema = ET.XMLSchema(schema_doc) try: tree = ET.parse(result) xmlschema.assertValid(tree) except Exception, e: raise PulpCodedValidationException(validation_exceptions=[ PulpCodedValidationException(error_code=error_codes.RPM1001, feed=feed, validation_exceptions=[e]) ]) model.metadata[ constants. CONFIG_KEY_DISTRIBUTION_XML_FILE] = constants.DISTRIBUTION_XML # parse the distribution file and add all the files to the download request root = tree.getroot() for file_element in root.findall('file'): relative_path = file_element.text files.append({ 'relativepath': relative_path, 'checksum': None, 'checksumtype': None, }) # Add the distribution file to the list of files files.append({ 'relativepath': constants.DISTRIBUTION_XML, 'checksum': None, 'checksumtype': None, })
def process_main(self, item=None): """ Update tags based on the parent metadata :param item: Not used by this step :type item: None """ md = self.parent.metadata tag = md.get('name') if tag is None: raise PulpCodedValidationException(error_code=error_codes.DKR1019, field='name') # https://pulp.plan.io/issues/3250 - use manifest_digest if available digest = md.get('manifest_digest', md.get('digest')) if digest is None: raise PulpCodedValidationException(error_code=error_codes.DKR1019, field='manifest_digest') pulp_user_metadata = md.get('pulp_user_metadata') repo_id = self.parent.repo.id manifest_type_id = models.Manifest._content_type_id.default repo_manifest_ids = repository.get_associated_unit_ids( repo_id, manifest_type_id) # check if there is manifest with such id within the queried repo # since we don't know if the provided digest is of an image manifest or manifest list # we need to try both. manifests = models.Manifest.objects.filter(digest=digest, id__in=repo_manifest_ids) manifest_type = constants.MANIFEST_IMAGE_TYPE if manifests.count() == 0: manifest_list_type_id = models.ManifestList._content_type_id.default repo_manifest_list_ids = repository.get_associated_unit_ids( repo_id, manifest_list_type_id) manifests = models.ManifestList.objects.filter( digest=digest, id__in=repo_manifest_list_ids) manifest_type = constants.MANIFEST_LIST_TYPE if manifests.count() == 0: raise PulpCodedValidationException( error_code=error_codes.DKR1010, digest=digest, repo_id=repo_id) new_tag = models.Tag.objects.tag_manifest( repo_id=self.parent.repo.id, tag_name=tag, manifest_digest=digest, schema_version=manifests[0].schema_version, manifest_type=manifest_type, pulp_user_metadata=pulp_user_metadata) if new_tag: repository.associate_single_unit(self.parent.repo.repo_obj, new_tag) self.parent.uploaded_unit = new_tag
def validate_importer_config(repo_id, importer_type_id, importer_config): """ Validate an importer configuration. This validates that the repository and importer type exist as these are both required to validate the configuration. :param repo_id: identifies the repo :type repo_id: str :param importer_type_id: identifies the type of importer being added; must correspond to an importer loaded at server startup :type importer_type_id: str :param importer_config: configuration values for the importer; may be None :type importer_config: dict """ repo_coll = Repo.get_collection() repo = repo_coll.find_one({'id': repo_id}) if repo is None: raise MissingResource(repo_id) if not plugin_api.is_valid_importer(importer_type_id): raise PulpCodedValidationException( error_code=error_codes.PLP1008, importer_type_id=importer_type_id) importer_instance, plugin_config = plugin_api.get_importer_by_id( importer_type_id) # Convention is that a value of None means unset. Remove any keys that # are explicitly set to None so the plugin will default them. if importer_config is not None: clean_config = dict([(k, v) for k, v in importer_config.items() if v is not None]) else: clean_config = None # Let the importer plugin verify the configuration call_config = PluginCallConfiguration(plugin_config, clean_config) transfer_repo = common_utils.to_transfer_repo(repo) transfer_repo.working_dir = common_utils.importer_working_dir( importer_type_id, repo_id) result = importer_instance.validate_config(transfer_repo, call_config) # For backward compatibility with plugins that don't yet return the tuple if isinstance(result, bool): valid_config = result message = None else: valid_config, message = result if not valid_config: raise PulpCodedValidationException(validation_errors=message)
def process_distribution(self, tmp_dir): """ Get the pulp_distribution.xml file from the server and if it exists download all the files it references to add them to the distribution unit. :param tmp_dir: The absolute path to the temporary directory :type tmp_dir: str :return: A list of file dictionaries :rtype: list """ # Get the Distribution file result = self.get_distribution_file(tmp_dir) files = [] # If there is a Distribution file - parse it and add all files to the file_list if result: xsd = os.path.join(constants.USR_SHARE_DIR, 'pulp_distribution.xsd') schema_doc = ET.parse(xsd) xmlschema = ET.XMLSchema(schema_doc) try: tree = ET.parse(result) xmlschema.assertValid(tree) except Exception, e: raise PulpCodedValidationException(validation_exceptions=[ PulpCodedValidationException( error_code=error_codes.RPM1001, feed=self.feed, validation_exceptions=[e]) ]) # This is broken and best I can tell - not used. # model.metadata[constants.CONFIG_KEY_DISTRIBUTION_XML_FILE] = \ # constants.DISTRIBUTION_XML # parse the distribution file and add all the files to the download request root = tree.getroot() for file_element in root.findall('file'): relative_path = file_element.text files.append({ RELATIVE_PATH: relative_path, CHECKSUM: None, CHECKSUM_TYPE: None, }) # Add the distribution file to the list of files files.append({ RELATIVE_PATH: constants.DISTRIBUTION_XML, CHECKSUM: None, CHECKSUM_TYPE: None, })
def process_distribution(self, tmp_dir): """ Get the pulp_distribution.xml file from the server and if it exists download all the files it references to add them to the distribution unit. :param tmp_dir: The absolute path to the temporary directory :type tmp_dir: str :return: A tuple that contains the list of file dictionaries and the absolute path to the distribution file, or None if no PULP_DISTRIBUTION.xml was found. :rtype: (list, basestring) """ # Get the Distribution file result = self.get_distribution_file(tmp_dir) files = [] # If there is a Distribution file - parse it and add all files to the file_list if result: xsd = os.path.join(constants.USR_SHARE_DIR, 'pulp_distribution.xsd') schema_doc = ET.parse(xsd) xmlschema = ET.XMLSchema(schema_doc) try: tree = ET.parse(result) xmlschema.assertValid(tree) except Exception, e: raise PulpCodedValidationException(validation_exceptions=[ PulpCodedValidationException( error_code=error_codes.RPM1001, feed=self.feed, validation_exceptions=[e]) ]) # parse the distribution file and add all the files to the download request root = tree.getroot() for file_element in root.findall('file'): relative_path = file_element.text if not relative_path.startswith('repodata/'): files.append({ RELATIVE_PATH: relative_path, CHECKSUM: None, CHECKSUM_TYPE: None, }) # Add the distribution file to the list of files files.append({ RELATIVE_PATH: constants.DISTRIBUTION_XML, CHECKSUM: None, CHECKSUM_TYPE: None, })
def errata_format_to_datetime(datetime_str, msg): """ Convert known errata date-time formats to datetime object. Expected formats are: - '%Y-%m-%d %H:%M:%S' - '%Y-%m-%d %H:%M:%S UTC' :param datetime_str: date and time in errata specific format :type datetime_str: str :param msg: additional error message in case of exception :type msg: str :return: parsed date and time :rtype: datetime.datetime :raises ValueError: if the date and time are in unknown format """ strptime_pattern = '%Y-%m-%d %H:%M:%S' datetime_str = datetime_str.strip() if datetime_str.endswith(' UTC'): datetime_str = datetime_str[:-4] try: datetime_obj = datetime.datetime.strptime(datetime_str, strptime_pattern) except ValueError: raise PulpCodedValidationException(error_code=error_codes.RPM1007, details=msg, expected_format=strptime_pattern) return datetime_obj
def test_validates_error_codes_not_present(self): mock_method = Mock(side_effect=PulpCodedValidationException( validation_exceptions=[PulpCodedException()])) self.assertRaises( AssertionError, util.assert_validation_exception, mock_method, error_codes=[error_codes.PLP0001, error_codes.PLP0012])
def validate_config(config): """ Validate a configuration :param config: Pulp configuration for the distributor :type config: pulp.plugins.config.PluginCallConfiguration :raises: PulpCodedValidationException if any validations failed :return: (True, None) :rtype: tuple """ errors = [] server_url = config.get(constants.CONFIG_KEY_REDIRECT_URL) if server_url: parsed = urlparse(server_url) if not parsed.scheme: errors.append( PulpCodedValidationException( error_code=error_codes.OST1001, field=constants.CONFIG_KEY_REDIRECT_URL, url=server_url)) if not parsed.netloc: errors.append( PulpCodedValidationException( error_code=error_codes.OST1002, field=constants.CONFIG_KEY_REDIRECT_URL, url=server_url)) if not parsed.path: errors.append( PulpCodedValidationException( error_code=error_codes.OST1003, field=constants.CONFIG_KEY_REDIRECT_URL, url=server_url)) protected = config.get(constants.CONFIG_KEY_PROTECTED) if protected: protected_parsed = config.get_boolean(constants.CONFIG_KEY_PROTECTED) if protected_parsed is None: errors.append( PulpCodedValidationException( error_code=error_codes.OST1004, field=constants.CONFIG_KEY_PROTECTED, value=protected)) if errors: raise PulpCodedValidationException(validation_exceptions=errors) return True, None
def wrapper(*args, **kwargs): request = args[1] if allow_empty and not request.body: request.body_as_json = {} return func(*args, **kwargs) try: request_json = json.loads(request.body) except ValueError: raise PulpCodedValidationException( error_code=error_codes.PLP1009) else: if not (json_type is None or isinstance(request_json, json_type)): raise PulpCodedValidationException( error_code=error_codes.PLP1015, data_type=json_type.__name__) request.body_as_json = _ensure_input_encoding(request_json) return func(*args, **kwargs)
def process_main(self, item=None): """ Save blobs and manifest to repository :param item: A Docker manifest or blob unit :type : pulp_docker.plugins.models.Blob or pulp_docker.plugins.models.Manifest :return: None """ if isinstance(item, models.Blob): checksum_type, _, checksum = item.digest.rpartition(':') if not checksum_type: # Never assume. But oh well checksum_type = "sha256" blob_src_path = os.path.join(self.get_working_dir(), checksum + '.tar') try: fobj = open(blob_src_path) except IOError: raise PulpCodedValidationException( error_code=error_codes.DKR1018, layer=os.path.basename(blob_src_path)) try: verification.verify_checksum(fobj, checksum_type, checksum) except verification.VerificationException: raise PulpCodedValidationException( error_code=error_codes.DKR1017, checksum_type=checksum_type, checksum=checksum) fobj.close() blob_dest_path = os.path.join(self.get_working_dir(), item.digest) os.rename(blob_src_path, blob_dest_path) item.set_storage_path(item.digest) try: item.save_and_import_content( os.path.join(self.get_working_dir(), item.digest)) except NotUniqueError: item = item.__class__.objects.get(**item.unit_key) repository.associate_single_unit(self.get_repo().repo_obj, item) if isinstance(item, models.Manifest): self.parent.uploaded_unit = item
def wrapper(*args, **kwargs): request = args[1] if allow_empty and not request.body: request.body_as_json = {} return func(*args, **kwargs) try: request_json = json.loads(request.body) except ValueError: raise PulpCodedValidationException(error_code=error_codes.PLP1009) else: request.body_as_json = _ensure_input_encoding(request_json) return func(*args, **kwargs)
def process_main(self, item=None): """ Update tags based on the parent metadata :param item: Not used by this step :type item: None """ tag = self.parent.metadata['name'] digest = self.parent.metadata['digest'] repo_id = self.parent.repo.id manifest_type_id = models.Manifest._content_type_id.default repo_manifest_ids = repository.get_associated_unit_ids( repo_id, manifest_type_id) # check if there is manifest with such id within the queried repo # since we don't know if the provided digest is of an image manifest or manifest list # we need to try both. manifests = models.Manifest.objects.filter(digest=digest, id__in=repo_manifest_ids) manifest_type = constants.MANIFEST_IMAGE_TYPE if manifests.count() == 0: manifest_list_type_id = models.ManifestList._content_type_id.default repo_manifest_list_ids = repository.get_associated_unit_ids( repo_id, manifest_list_type_id) manifests = models.ManifestList.objects.filter( digest=digest, id__in=repo_manifest_list_ids) manifest_type = constants.MANIFEST_LIST_TYPE if manifests.count() == 0: raise PulpCodedValidationException( error_code=error_codes.DKR1010, digest=digest, repo_id=repo_id) new_tag = models.Tag.objects.tag_manifest( repo_id=self.parent.repo.id, tag_name=tag, manifest_digest=digest, schema_version=manifests[0].schema_version, manifest_type=manifest_type) if new_tag: repository.associate_single_unit(self.parent.repo.repo_obj, new_tag)
def process_main(self): _logger.debug(self.description) for unit_key in self.parent.step_get_local_units.units_to_download: hash_key = get_key_hash(unit_key) file_name = self.parent.deb_data[hash_key]['file_name'] storage_path = generate_internal_storage_path(file_name) dest_dir = os.path.join(self.working_dir, storage_path) # validate the size of the file downloaded file_size = int(self.parent.deb_data[hash_key]['file_size']) if file_size != os.stat(dest_dir).st_size: raise PulpCodedValidationException( error_code=error_codes.DEB1001, file_name=file_name) unit = self.get_conduit().init_unit(constants.DEB_TYPE_ID, unit_key, {'file_name': file_name}, storage_path) shutil.move(dest_dir, unit.storage_path) self.get_conduit().save_unit(unit)
def process_main(self): """ Validate the uploaded manifest list json, then import content unit into repository. """ with open(self.parent.file_path, 'r') as uploaded_file: manifest_list = uploaded_file.read() models.ManifestList.check_json(manifest_list) digest = models.UnitMixin.calculate_digest(manifest_list) manifest_list_instance = models.ManifestList.from_json( manifest_list, digest) transfer_repo = self.get_repo() # Ensure that all referenced manifests are already in repository. manifest_digests = set(manifest_list_instance.manifests) qs = models.Manifest.objects.filter( digest__in=sorted(manifest_digests)).only('id', 'digest') known_manifests = dict( (manifest['digest'], manifest['id']) for manifest in qs) unit_qs = pulp_models.RepositoryContentUnit.objects.filter( repo_id=transfer_repo.id, unit_id__in=known_manifests.values()).values_list('unit_id') unit_ids_in_repo = list(unit_qs) missing_manifest_digests = [] for manifest_digest in manifest_digests: unit_id = known_manifests.get(manifest_digest) if not unit_id or unit_id not in unit_ids_in_repo: missing_manifest_digests.append(manifest_digest) if missing_manifest_digests: raise PulpCodedValidationException( error_code=error_codes.DKR1013, digests=missing_manifest_digests) manifest_list_instance.set_storage_path(digest) try: manifest_list_instance.save_and_import_content( self.parent.file_path) except NotUniqueError: manifest_list_instance = models.ManifestList.objects.get( **manifest_list_instance.unit_key) repository.associate_single_unit(transfer_repo.repo_obj, manifest_list_instance) self.parent.uploaded_unit = manifest_list_instance
def check_json(manifest_list_json): """ Check the structure of a manifest list json file. This function is a sanity check to make sure the JSON contains the correct structure. It does not validate with the database. :param manifest_list_json: A JSON document describing a ManifestList object as defined by the Docker v2, Schema 2 Manifest List documentation. :type manifest_list_json: basestring :raises PulpCodedValidationException: DKR1011 if manifest_list_json is invalid JSON :raises PulpCodedValidationException: DKR1012 if Manifest List has an invalid mediaType :raises PulpCodedValidationException: DKR1014 if any of the listed Manifests contain invalid mediaType :raises PulpCodedValidationException: DKR1015 if Manifest List does not have all required fields. :raises PulpCodedValidationException: DKR1016 if any Image Manifest in the list does not have all required fields. """ try: manifest_list = json.loads(manifest_list_json) except ValueError: raise PulpCodedValidationException(error_code=error_codes.DKR1011) for field in MANIFEST_LIST_REQUIRED_FIELDS: if field not in manifest_list: raise PulpCodedValidationException(error_code=error_codes.DKR1015, field=field) if manifest_list['mediaType'] != constants.MEDIATYPE_MANIFEST_LIST: raise PulpCodedValidationException(error_code=error_codes.DKR1012, media_type=manifest_list['mediaType']) for image_manifest_dict in manifest_list['manifests']: for field in IMAGE_MANIFEST_REQUIRED_FIELDS: if field not in image_manifest_dict: raise PulpCodedValidationException(error_code=error_codes.DKR1016, field=field) for field in IMAGE_MANIFEST_REQUIRED_PLATFORM_SUBFIELDS: if field not in image_manifest_dict['platform']: subfield = "platform.{field}".format(field=field) raise PulpCodedValidationException(error_code=error_codes.DKR1016, field=subfield) if image_manifest_dict['mediaType'] not in [constants.MEDIATYPE_MANIFEST_S2, constants.MEDIATYPE_MANIFEST_S1, constants.MEDIATYPE_SIGNED_MANIFEST_S1]: raise PulpCodedValidationException(error_code=error_codes.DKR1014, digest=image_manifest_dict['digest'])
def __init__(self, metadata_file_path, checksum_type=None): """ :param metadata_file_path: full path to metadata file to be generated :type metadata_file_path: str :param checksum_type: checksum type to be used to generate and prepend checksum to the file names of files. If checksum_type is None, no checksum is added to the filename :type checksum_type: str or None """ self.metadata_file_path = metadata_file_path self.metadata_file_handle = None self.checksum_type = checksum_type self.checksum = None if self.checksum_type is not None: checksum_function = CHECKSUM_FUNCTIONS.get(checksum_type) if not checksum_function: raise PulpCodedValidationException( [PulpCodedException(error_codes.PLP1005, checksum_type=checksum_type)]) self.checksum_constructor = checksum_function
def validate_postdistributor(repo, config): """ Validates that the postdistributor_id is set and is valid for this repositotry. :param repo: metadata describing the repository to which the configuration applies :type repo: pulp.plugins.model.Repository :param config: Pulp configuration for the distributor :type config: pulp.plugins.config.PluginCallConfiguration :return: tuple comprised of a boolean indicating whether validation succeeded or failed and a list of errors (if any) :rtype: (bool, list of strings) or (bool, None) :raises: PulpCodedValidationException if postdistributor_id is not defined or 404 if the distributor_id is not associated with the repo """ postdistributor = config.flatten().get("postdistributor_id", None) if postdistributor: Distributor.objects.get_or_404(repo_id=repo.id, distributor_id=postdistributor) return True, None else: raise PulpCodedValidationException(error_code=error_codes.DKR1009)
def validate_config(config, repo): """ Validate a configuration :param config: Pulp configuration for the distributor :type config: pulp.plugins.config.PluginCallConfiguration :param repo: metadata describing the repository to which the configuration applies :type repo: pulp.plugins.model.Repository :raises: PulpCodedValidationException if any validations failed """ errors = [] server_url = config.get(constants.CONFIG_KEY_REDIRECT_URL) if server_url: parsed = urlparse(server_url) if not parsed.scheme: errors.append( PulpCodedValidationException( error_code=error_codes.DKR1001, field=constants.CONFIG_KEY_REDIRECT_URL, url=server_url)) if not parsed.netloc: errors.append( PulpCodedValidationException( error_code=error_codes.DKR1002, field=constants.CONFIG_KEY_REDIRECT_URL, url=server_url)) if not parsed.path: errors.append( PulpCodedValidationException( error_code=error_codes.DKR1003, field=constants.CONFIG_KEY_REDIRECT_URL, url=server_url)) protected = config.get(constants.CONFIG_KEY_PROTECTED) if protected: protected_parsed = config.get_boolean(constants.CONFIG_KEY_PROTECTED) if protected_parsed is None: errors.append( PulpCodedValidationException( error_code=error_codes.DKR1004, field=constants.CONFIG_KEY_PROTECTED, value=protected)) # Check that the repo_registry is valid repo_registry_id = config.get(constants.CONFIG_KEY_REPO_REGISTRY_ID) if repo_registry_id and not _is_valid_repo_registry_id(repo_registry_id): errors.append( PulpCodedValidationException( error_code=error_codes.DKR1005, field=constants.CONFIG_KEY_REPO_REGISTRY_ID, value=repo_registry_id)) # If the repo_registry_id is not specified, this value defaults to the # repo id, so we need to validate that. elif not repo_registry_id and not _is_valid_repo_registry_id(repo.id): errors.append( PulpCodedValidationException( error_code=error_codes.DKR1006, field=constants.CONFIG_KEY_REPO_REGISTRY_ID, value=repo.id)) if errors: raise PulpCodedValidationException(validation_exceptions=errors) return True, None
def test_calls_method_with_kwargs(self): mock_method = Mock(side_effect=PulpCodedValidationException()) util.assert_validation_exception(mock_method, [], baz="qux") mock_method.assert_called_once_with(baz="qux")
def _extract_rpm_data(type_id, rpm_filename): """ Extract a dict of information for a given RPM or SRPM. :param type_id: The type of the unit that is being generated :type type_id: str :param rpm_filename: full path to the package to analyze :type rpm_filename: str :return: dict of data about the package :rtype: dict """ rpm_data = dict() # Read the RPM header attributes for use later ts = rpm.TransactionSet() ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES) fd = os.open(rpm_filename, os.O_RDONLY) try: headers = ts.hdrFromFdno(fd) os.close(fd) except rpm.error: # Raised if the headers cannot be read os.close(fd) raise for k in ['name', 'version', 'release', 'epoch']: rpm_data[k] = headers[k] if rpm_data['epoch'] is not None: rpm_data['epoch'] = str(rpm_data['epoch']) else: rpm_data['epoch'] = str(0) if headers['sourcepackage']: if RPMTAG_NOSOURCE in headers.keys(): rpm_data['arch'] = 'nosrc' else: rpm_data['arch'] = 'src' else: rpm_data['arch'] = headers['arch'] # construct filename from metadata (BZ #1101168) if headers[rpm.RPMTAG_SOURCEPACKAGE]: if type_id != models.SRPM._content_type_id.default: raise PulpCodedValidationException(error_code=error_codes.RPM1002) rpm_basefilename = "%s-%s-%s.src.rpm" % (headers['name'], headers['version'], headers['release']) else: if type_id != models.RPM._content_type_id.default: raise PulpCodedValidationException(error_code=error_codes.RPM1003) rpm_basefilename = "%s-%s-%s.%s.rpm" % (headers['name'], headers['version'], headers['release'], headers['arch']) rpm_data['relativepath'] = rpm_basefilename rpm_data['filename'] = rpm_basefilename # This format is, and has always been, incorrect. As of the new yum importer, the # plugin will generate these from the XML snippet because the API into RPM headers # is atrocious. This is the end game for this functionality anyway, moving all of # that metadata derivation into the plugin, so this is just a first step. # I'm leaving these in and commented to show how not to do it. # rpm_data['requires'] = [(r,) for r in headers['requires']] # rpm_data['provides'] = [(p,) for p in headers['provides']] rpm_data['buildhost'] = headers['buildhost'] rpm_data['license'] = headers['license'] rpm_data['vendor'] = headers['vendor'] rpm_data['description'] = headers['description'] rpm_data['build_time'] = headers[rpm.RPMTAG_BUILDTIME] # Use the mtime of the file to match what is in the generated xml from # rpm_parse.get_package_xml(..) file_stat = os.stat(rpm_filename) rpm_data['time'] = file_stat[stat.ST_MTIME] return rpm_data
def test_calls_method_with_child_exceptions(self): mock_method = Mock(side_effect=PulpCodedValidationException( validation_exceptions=[PulpCodedException()])) util.assert_validation_exception(mock_method, error_codes=[error_codes.PLP0001])
def test_error_codes_no_child_exceptions(self): mock_method = Mock(side_effect=PulpCodedValidationException()) self.assertRaises(AssertionError, util.assert_validation_exception, mock_method, error_codes=[error_codes.PLP1002])
def _generate_rpm_data(type_id, rpm_filename, user_metadata=None): """ For the given RPM, analyzes its metadata to generate the appropriate unit key and metadata fields, returning both to the caller. :param type_id: The type of the unit that is being generated :type type_id: str :param rpm_filename: full path to the RPM to analyze :type rpm_filename: str :param user_metadata: user supplied metadata about the unit. This is optional. :type user_metadata: dict :return: tuple of unit key and unit metadata for the RPM :rtype: tuple """ # Expected metadata fields: # "vendor", "description", "buildhost", "license", "vendor", "requires", "provides", # "relativepath", "filename" # # Expected unit key fields: # "name", "epoch", "version", "release", "arch", "checksumtype", "checksum" unit_key = dict() metadata = dict() # Read the RPM header attributes for use later ts = rpm.TransactionSet() ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES) fd = os.open(rpm_filename, os.O_RDONLY) try: headers = ts.hdrFromFdno(fd) os.close(fd) except rpm.error: # Raised if the headers cannot be read os.close(fd) raise # -- Unit Key ----------------------- # Checksum if user_metadata and user_metadata.get('checksum_type'): user_checksum_type = user_metadata.get('checksum_type') user_checksum_type = verification.sanitize_checksum_type( user_checksum_type) unit_key['checksumtype'] = user_checksum_type else: unit_key['checksumtype'] = verification.TYPE_SHA256 unit_key['checksum'] = _calculate_checksum(unit_key['checksumtype'], rpm_filename) # Name, Version, Release, Epoch for k in ['name', 'version', 'release', 'epoch']: unit_key[k] = headers[k] # Epoch munging if unit_key['epoch'] is None: unit_key['epoch'] = str(0) else: unit_key['epoch'] = str(unit_key['epoch']) # Arch if headers['sourcepackage']: if RPMTAG_NOSOURCE in headers.keys(): unit_key['arch'] = 'nosrc' else: unit_key['arch'] = 'src' else: unit_key['arch'] = headers['arch'] # -- Unit Metadata ------------------ # construct filename from metadata (BZ #1101168) if headers[rpm.RPMTAG_SOURCEPACKAGE]: if type_id != models.SRPM.TYPE: raise PulpCodedValidationException(error_code=error_codes.RPM1002) rpm_basefilename = "%s-%s-%s.src.rpm" % ( headers['name'], headers['version'], headers['release']) else: if type_id != models.RPM.TYPE: raise PulpCodedValidationException(error_code=error_codes.RPM1003) rpm_basefilename = "%s-%s-%s.%s.rpm" % ( headers['name'], headers['version'], headers['release'], headers['arch']) metadata['relativepath'] = rpm_basefilename metadata['filename'] = rpm_basefilename # This format is, and has always been, incorrect. As of the new yum importer, the # plugin will generate these from the XML snippet because the API into RPM headers # is atrocious. This is the end game for this functionality anyway, moving all of # that metadata derivation into the plugin, so this is just a first step. # I'm leaving these in and commented to show how not to do it. # metadata['requires'] = [(r,) for r in headers['requires']] # metadata['provides'] = [(p,) for p in headers['provides']] metadata['buildhost'] = headers['buildhost'] metadata['license'] = headers['license'] metadata['vendor'] = headers['vendor'] metadata['description'] = headers['description'] metadata['build_time'] = headers[rpm.RPMTAG_BUILDTIME] # Use the mtime of the file to match what is in the generated xml from # rpm_parse.get_package_xml(..) file_stat = os.stat(rpm_filename) metadata['time'] = file_stat[stat.ST_MTIME] return unit_key, metadata
def test_child_exceptions_no_error_codes(self): mock_method = Mock(side_effect=PulpCodedValidationException( validation_exceptions=[PulpCodedException()])) self.assertRaises(AssertionError, util.assert_validation_exception, mock_method, [])