def generate_dbs(self): """ For repo data files that contain data we need to access later for each unit in the repo, generate a local db file that gives us quick read access to each unit's data. """ for filename, tag, process_func in ( (filelists.METADATA_FILE_NAME, filelists.PACKAGE_TAG, filelists.process_package_element), (other.METADATA_FILE_NAME, other.PACKAGE_TAG, other.process_package_element), ): xml_file_handle = self.get_metadata_file_handle(filename) try: generator = package_list_generator(xml_file_handle, tag) db_filename = os.path.join(self.dst_dir, '%s.db' % filename) # always a New file, and open with Fast writing mode. db_file_handle = gdbm.open(db_filename, 'nf') try: for element in generator: utils.strip_ns(element) raw_xml = utils.element_to_raw_xml(element) unit_key, _ = process_func(element) db_key = self.generate_db_key(unit_key) db_file_handle[db_key] = raw_xml db_file_handle.sync() finally: db_file_handle.close() finally: xml_file_handle.close() self.dbs[filename] = db_filename
def _update_provides_requires(unit): """ Determines the provides and requires fields based on the RPM's XML snippet and updates the model instance. :param unit: the unit being added to Pulp; the metadata attribute must already have a key called 'repodata' :type unit: pulp.plugins.model.Unit """ try: # make a guess at the encoding codec = "UTF-8" unit.metadata["repodata"]["primary"].encode(codec) except UnicodeEncodeError: # best second guess we have, and it will never fail due to the nature # of the encoding. codec = "ISO-8859-1" unit.metadata["repodata"]["primary"].encode(codec) fake_xml = FAKE_XML % {"encoding": codec, "xml": unit.metadata["repodata"]["primary"]} fake_element = ET.fromstring(fake_xml.encode(codec)) utils.strip_ns(fake_element) primary_element = fake_element.find("package") format_element = primary_element.find("format") provides_element = format_element.find("provides") requires_element = format_element.find("requires") unit.metadata["provides"] = ( map(primary._process_rpm_entry_element, provides_element.findall("entry")) if provides_element else [] ) unit.metadata["requires"] = ( map(primary._process_rpm_entry_element, requires_element.findall("entry")) if requires_element else [] )
def _migrate_collection(type_id): collection = types_db.type_units_collection(type_id) for package in collection.find(): # grab the raw XML and parse it into the elements we'll need later try: # make a guess at the encoding codec = 'UTF-8' package['repodata']['primary'].encode(codec) except UnicodeEncodeError: # best second guess we have, and it will never fail due to the nature # of the encoding. codec = 'ISO-8859-1' package['repodata']['primary'].encode(codec) fake_xml = FAKE_XML % {'encoding': codec, 'xml': package['repodata']['primary']} fake_element = ET.fromstring(fake_xml.encode(codec)) utils.strip_ns(fake_element) primary_element = fake_element.find('package') format_element = primary_element.find('format') provides_element = format_element.find('provides') requires_element = format_element.find('requires') # add these attributes, which we previously didn't track in the DB. package['size'] = int(primary_element.find('size').attrib['package']) if type_id == 'rpm': package['sourcerpm'] = format_element.find('sourcerpm').text package['summary'] = primary_element.find('summary').text # re-parse provides and requires. The format changed from 2.1, and the # 2.1 upload workflow was known to produce invalid data for these fields package['provides'] = map(primary._process_rpm_entry_element, provides_element.findall('entry')) if provides_element else [] package['requires'] = map(primary._process_rpm_entry_element, requires_element.findall('entry')) if requires_element else [] collection.save(package)
def generate_dbs(self): """ For repo data files that contain data we need to access later for each unit in the repo, generate a local db file that gives us quick read access to each unit's data. """ for filename, tag, process_func in ( (filelists.METADATA_FILE_NAME, filelists.PACKAGE_TAG, filelists.process_package_element), (other.METADATA_FILE_NAME, other.PACKAGE_TAG, other.process_package_element), ): xml_file_handle = self.get_metadata_file_handle(filename) try: generator = package_list_generator(xml_file_handle, tag) db_filename = os.path.join(self.dst_dir, "%s.db" % filename) # always a New file, and open with Fast writing mode. db_file_handle = gdbm.open(db_filename, "nf") try: for element in generator: utils.strip_ns(element) raw_xml = utils.element_to_raw_xml(element) unit_key, _ = process_func(element) db_key = self.generate_db_key(unit_key) db_file_handle[db_key] = raw_xml db_file_handle.sync() finally: db_file_handle.close() finally: xml_file_handle.close() self.dbs[filename] = db_filename
def _update_provides_requires(unit): """ Determines the provides and requires fields based on the RPM's XML snippet and updates the model instance. :param unit: the unit being added to Pulp; the metadata attribute must already have a key called 'repodata' :type unit: subclass of pulp.server.db.model.ContentUnit """ try: # make a guess at the encoding codec = 'UTF-8' unit.repodata['primary'].encode(codec) except UnicodeEncodeError: # best second guess we have, and it will never fail due to the nature # of the encoding. codec = 'ISO-8859-1' unit.repodata['primary'].encode(codec) fake_xml = FAKE_XML % {'encoding': codec, 'xml': unit.repodata['primary']} fake_element = ET.fromstring(fake_xml.encode(codec)) utils.strip_ns(fake_element) primary_element = fake_element.find('package') format_element = primary_element.find('format') provides_element = format_element.find('provides') requires_element = format_element.find('requires') unit.provides = map(primary._process_rpm_entry_element, provides_element.findall('entry')) if provides_element else [] unit.requires = map(primary._process_rpm_entry_element, requires_element.findall('entry')) if requires_element else []
def _update_provides_requires(unit): """ Determines the provides and requires fields based on the RPM's XML snippet and updates the model instance. :param unit: the unit being added to Pulp; the metadata attribute must already have a key called 'repodata' :type unit: pulp.plugins.model.Unit """ try: # make a guess at the encoding codec = 'UTF-8' unit.metadata['repodata']['primary'].encode(codec) except UnicodeEncodeError: # best second guess we have, and it will never fail due to the nature # of the encoding. codec = 'ISO-8859-1' unit.metadata['repodata']['primary'].encode(codec) fake_xml = FAKE_XML % {'encoding': codec, 'xml': unit.metadata['repodata']['primary']} fake_element = ET.fromstring(fake_xml.encode(codec)) utils.strip_ns(fake_element) primary_element = fake_element.find('package') format_element = primary_element.find('format') provides_element = format_element.find('provides') requires_element = format_element.find('requires') unit.metadata['provides'] = map(primary._process_rpm_entry_element, provides_element.findall('entry')) if provides_element else [] unit.metadata['requires'] = map(primary._process_rpm_entry_element, requires_element.findall('entry')) if requires_element else []
def generate_dbs(self): """ For repo data files that contain data we need to access later for each unit in the repo, generate a local db file that gives us quick read access to each unit's data. """ for filename, tag, process_func in ( (filelists.METADATA_FILE_NAME, filelists.PACKAGE_TAG, filelists.process_package_element), (other.METADATA_FILE_NAME, other.PACKAGE_TAG, other.process_package_element), ): with contextlib.closing(self.get_metadata_file_handle(filename)) as xml_file_handle: generator = package_list_generator(xml_file_handle, tag) db_filename = os.path.join(self.dst_dir, '%s.db' % filename) # always a New file, and open with Fast writing mode. with contextlib.closing(gdbm.open(db_filename, 'nf')) as db_file_handle: for element in generator: utils.strip_ns(element) element.attrib['pkgid'] = models.RpmBase.PKGID_TEMPLATE raw_xml = utils.element_to_raw_xml(element) unit_key, _ = process_func(element) db_key = self.generate_db_key(unit_key) db_file_handle[db_key] = raw_xml db_file_handle.sync() self.dbs[filename] = db_filename
def generate_dbs(self): """ For repo data files that contain data we need to access later for each unit in the repo, generate a local db file that gives us quick read access to each unit's data. :raises PulpCodedException: if there is some inconsistency in metadata """ package_count = {} for filename, tag, process_func in ( (filelists.METADATA_FILE_NAME, filelists.PACKAGE_TAG, filelists.process_package_element), (other.METADATA_FILE_NAME, other.PACKAGE_TAG, other.process_package_element), ): with contextlib.closing(self.get_metadata_file_handle(filename)) as xml_file_handle: generator = package_list_generator(xml_file_handle, tag) db_filename = os.path.join(self.dst_dir, '%s.db' % filename) # always a New file, and open with Fast writing mode. with contextlib.closing(gdbm.open(db_filename, 'nf')) as db_file_handle: for element in generator: utils.strip_ns(element) element.attrib['pkgid'] = models.RpmBase.PKGID_TEMPLATE raw_xml = utils.element_to_raw_xml(element) unit_key, _ = process_func(element) db_key = self.generate_db_key(unit_key) db_file_handle[db_key] = raw_xml db_file_handle.sync() package_count[filename] = len(db_file_handle) self.dbs[filename] = db_filename if package_count[filelists.METADATA_FILE_NAME] != package_count[other.METADATA_FILE_NAME]: reason = ('metadata is specified for different set of packages in filelists.xml' ' and in other.xml') raise PulpCodedException(error_code=error_codes.RPM1015, reason=reason) self.rpm_count = package_count[filelists.METADATA_FILE_NAME]
def _update_provides_requires(model): """ Determines the provides and requires fields based on the RPM's XML snippet and updates the model instance. :param model: an RPM model to which providesand requires fields should be added. The model's metadata must already include the 'repodata' attribute. :type model: pulp_rpm.common.models.RPM """ try: # make a guess at the encoding codec = 'UTF-8' model.metadata['repodata']['primary'].encode(codec) except UnicodeEncodeError: # best second guess we have, and it will never fail due to the nature # of the encoding. codec = 'ISO-8859-1' model.metadata['repodata']['primary'].encode(codec) fake_xml = FAKE_XML % {'encoding': codec, 'xml': model.metadata['repodata']['primary']} fake_element = ET.fromstring(fake_xml.encode(codec)) utils.strip_ns(fake_element) primary_element = fake_element.find('package') format_element = primary_element.find('format') provides_element = format_element.find('provides') requires_element = format_element.find('requires') model.metadata['provides'] = map(primary._process_rpm_entry_element, provides_element.findall('entry')) if provides_element else [] model.metadata['requires'] = map(primary._process_rpm_entry_element, requires_element.findall('entry')) if requires_element else []
def test_other(self): utils.strip_ns(self.other_element) raw_xml = utils.element_to_raw_xml(self.other_element) self.assertTrue(raw_xml.startswith('<package ')) self.assertTrue(raw_xml.find('<version ') >= 0) self.assertEqual(raw_xml.count('<changelog '), 10) self.assertEqual(raw_xml.count('author="Doug Ledford'), 7) # fromstring just to make sure this is valid ET.fromstring(raw_xml)
def test_other(self): utils.strip_ns(self.other_element) raw_xml = utils.element_to_raw_xml(self.other_element) self.assertTrue(raw_xml.startswith('<package ')) self.assertTrue(raw_xml.find('<version ') >= 0) self.assertEqual(raw_xml.count('<changelog '), 10) self.assertEqual(raw_xml.count('author="Doug Ledford'), 7) # fromstring just to make sure this is valid ET.fromstring(raw_xml)
def test_filelists(self): utils.strip_ns(self.filelists_element) raw_xml = utils.element_to_raw_xml(self.filelists_element) self.assertTrue(raw_xml.startswith('<package ')) self.assertTrue(raw_xml.find('<version ') >= 0) self.assertTrue(raw_xml.find('name="opensm-libs"') >= 0) self.assertTrue(raw_xml.find('<file>/usr/lib64/libosmcomp.so.3</file>') >= 0) self.assertEqual(raw_xml.count('<file>'), 6) # fromstring just to make sure this is valid ET.fromstring(raw_xml)
def test_filelists(self): utils.strip_ns(self.filelists_element) raw_xml = utils.element_to_raw_xml(self.filelists_element) self.assertTrue(raw_xml.startswith('<package ')) self.assertTrue(raw_xml.find('<version ') >= 0) self.assertTrue(raw_xml.find('name="opensm-libs"') >= 0) self.assertTrue(raw_xml.find('<file>/usr/lib64/libosmcomp.so.3</file>') >= 0) self.assertEqual(raw_xml.count('<file>'), 6) # fromstring just to make sure this is valid ET.fromstring(raw_xml)
def _update_provides_requires(unit): """ Determines the provides and requires fields based on the RPM's XML snippet and updates the model instance. :param unit: the unit being added to Pulp; the metadata attribute must already have a key called 'repodata' :type unit: subclass of pulp.server.db.model.ContentUnit """ fake_element = utils.fake_xml_element(unit.repodata['primary']) utils.strip_ns(fake_element) primary_element = fake_element.find('package') format_element = primary_element.find('format') provides_element = format_element.find('provides') requires_element = format_element.find('requires') unit.provides = map(primary._process_rpm_entry_element, provides_element.findall('entry')) if provides_element else [] unit.requires = map(primary._process_rpm_entry_element, requires_element.findall('entry')) if requires_element else []
def _update_provides_requires(unit): """ Determines the provides and requires fields based on the RPM's XML snippet and updates the model instance. :param unit: the unit being added to Pulp; the metadata attribute must already have a key called 'repodata' :type unit: subclass of pulp.server.db.model.ContentUnit """ fake_element = _fake_xml_element(unit.repodata['primary']) utils.strip_ns(fake_element) primary_element = fake_element.find('package') format_element = primary_element.find('format') provides_element = format_element.find('provides') requires_element = format_element.find('requires') unit.provides = map(primary._process_rpm_entry_element, provides_element.findall('entry')) if provides_element else [] unit.requires = map(primary._process_rpm_entry_element, requires_element.findall('entry')) if requires_element else []
def generate_dbs(self): """ For repo data files that contain data we need to access later for each unit in the repo, generate a local db file that gives us quick read access to each unit's data. :raises PulpCodedException: if there is some inconsistency in metadata """ package_count = {} for filename, tag, process_func in ( (filelists.METADATA_FILE_NAME, filelists.PACKAGE_TAG, filelists.process_package_element), (other.METADATA_FILE_NAME, other.PACKAGE_TAG, other.process_package_element), ): with contextlib.closing(self.get_metadata_file_handle( filename)) as xml_file_handle: generator = package_list_generator(xml_file_handle, tag) db_filename = os.path.join(self.dst_dir, '%s.db' % filename) # always a New file, and open with Fast writing mode. with contextlib.closing(gdbm.open(db_filename, 'nf')) as db_file_handle: for element in generator: utils.strip_ns(element) element.attrib['pkgid'] = models.RpmBase.PKGID_TEMPLATE raw_xml = utils.element_to_raw_xml(element) unit_key, _ = process_func(element) db_key = self.generate_db_key(unit_key) db_file_handle[db_key] = raw_xml db_file_handle.sync() package_count[filename] = len(db_file_handle) self.dbs[filename] = db_filename if package_count[filelists.METADATA_FILE_NAME] != package_count[ other.METADATA_FILE_NAME]: reason = ( 'metadata is specified for different set of packages in filelists.xml' ' and in other.xml') raise PulpCodedException(error_code=error_codes.RPM1015, reason=reason) self.rpm_count = package_count[filelists.METADATA_FILE_NAME]
def _migrate_collection(type_id): collection = types_db.type_units_collection(type_id) for package in collection.find(): # grab the raw XML and parse it into the elements we'll need later try: # make a guess at the encoding codec = 'UTF-8' package['repodata']['primary'].encode(codec) except UnicodeEncodeError: # best second guess we have, and it will never fail due to the nature # of the encoding. codec = 'ISO-8859-1' package['repodata']['primary'].encode(codec) fake_xml = FAKE_XML % { 'encoding': codec, 'xml': package['repodata']['primary'] } fake_element = ET.fromstring(fake_xml.encode(codec)) utils.strip_ns(fake_element) primary_element = fake_element.find('package') format_element = primary_element.find('format') provides_element = format_element.find('provides') requires_element = format_element.find('requires') # add these attributes, which we previously didn't track in the DB. package['size'] = int(primary_element.find('size').attrib['package']) if type_id == RPM.TYPE: package['sourcerpm'] = format_element.find('sourcerpm').text package['summary'] = primary_element.find('summary').text # re-parse provides and requires. The format changed from 2.1, and the # 2.1 upload workflow was known to produce invalid data for these fields package['provides'] = map( primary._process_rpm_entry_element, provides_element.findall('entry')) if provides_element else [] package['requires'] = map( primary._process_rpm_entry_element, requires_element.findall('entry')) if requires_element else [] collection.save(package, safe=True)
def test_strip_all(self): utils.strip_ns(self.element) self._check_all_elements(self.element)
def test_strip_common_ns_only(self): utils.strip_ns(self.element, primary.COMMON_SPEC_URL) self.assertTrue(self._check_for_one_ns(self.element, primary.RPM_SPEC_URL))
def test_strip_common_ns_only(self): utils.strip_ns(self.element, primary.COMMON_SPEC_URL) self.assertTrue(self._check_for_one_ns(self.element, primary.RPM_SPEC_URL))
def test_strip_all(self): utils.strip_ns(self.element) self._check_all_elements(self.element)