def generate_dbs(self):
"""
For repo data files that contain data we need to access later for each
unit in the repo, generate a local db file that gives us quick read
access to each unit's data.
"""
for filename, tag, process_func in (
(filelists.METADATA_FILE_NAME, filelists.PACKAGE_TAG,
filelists.process_package_element),
(other.METADATA_FILE_NAME, other.PACKAGE_TAG,
other.process_package_element),
):
xml_file_handle = self.get_metadata_file_handle(filename)
try:
generator = package_list_generator(xml_file_handle, tag)
db_filename = os.path.join(self.dst_dir, '%s.db' % filename)
# always a New file, and open with Fast writing mode.
db_file_handle = gdbm.open(db_filename, 'nf')
try:
for element in generator:
utils.strip_ns(element)
raw_xml = utils.element_to_raw_xml(element)
unit_key, _ = process_func(element)
db_key = self.generate_db_key(unit_key)
db_file_handle[db_key] = raw_xml
db_file_handle.sync()
finally:
db_file_handle.close()
finally:
xml_file_handle.close()
self.dbs[filename] = db_filename
def _update_provides_requires(unit):
"""
Determines the provides and requires fields based on the RPM's XML snippet and updates
the model instance.
:param unit: the unit being added to Pulp; the metadata attribute must already have
a key called 'repodata'
:type unit: pulp.plugins.model.Unit
"""
try:
# make a guess at the encoding
codec = "UTF-8"
unit.metadata["repodata"]["primary"].encode(codec)
except UnicodeEncodeError:
# best second guess we have, and it will never fail due to the nature
# of the encoding.
codec = "ISO-8859-1"
unit.metadata["repodata"]["primary"].encode(codec)
fake_xml = FAKE_XML % {"encoding": codec, "xml": unit.metadata["repodata"]["primary"]}
fake_element = ET.fromstring(fake_xml.encode(codec))
utils.strip_ns(fake_element)
primary_element = fake_element.find("package")
format_element = primary_element.find("format")
provides_element = format_element.find("provides")
requires_element = format_element.find("requires")
unit.metadata["provides"] = (
map(primary._process_rpm_entry_element, provides_element.findall("entry")) if provides_element else []
)
unit.metadata["requires"] = (
map(primary._process_rpm_entry_element, requires_element.findall("entry")) if requires_element else []
)
def _migrate_collection(type_id):
collection = types_db.type_units_collection(type_id)
for package in collection.find():
# grab the raw XML and parse it into the elements we'll need later
try:
# make a guess at the encoding
codec = 'UTF-8'
package['repodata']['primary'].encode(codec)
except UnicodeEncodeError:
# best second guess we have, and it will never fail due to the nature
# of the encoding.
codec = 'ISO-8859-1'
package['repodata']['primary'].encode(codec)
fake_xml = FAKE_XML % {'encoding': codec, 'xml': package['repodata']['primary']}
fake_element = ET.fromstring(fake_xml.encode(codec))
utils.strip_ns(fake_element)
primary_element = fake_element.find('package')
format_element = primary_element.find('format')
provides_element = format_element.find('provides')
requires_element = format_element.find('requires')
# add these attributes, which we previously didn't track in the DB.
package['size'] = int(primary_element.find('size').attrib['package'])
if type_id == 'rpm':
package['sourcerpm'] = format_element.find('sourcerpm').text
package['summary'] = primary_element.find('summary').text
# re-parse provides and requires. The format changed from 2.1, and the
# 2.1 upload workflow was known to produce invalid data for these fields
package['provides'] = map(primary._process_rpm_entry_element,
provides_element.findall('entry')) if provides_element else []
package['requires'] = map(primary._process_rpm_entry_element,
requires_element.findall('entry')) if requires_element else []
collection.save(package)
def generate_dbs(self):
"""
For repo data files that contain data we need to access later for each
unit in the repo, generate a local db file that gives us quick read
access to each unit's data.
"""
for filename, tag, process_func in (
(filelists.METADATA_FILE_NAME, filelists.PACKAGE_TAG, filelists.process_package_element),
(other.METADATA_FILE_NAME, other.PACKAGE_TAG, other.process_package_element),
):
xml_file_handle = self.get_metadata_file_handle(filename)
try:
generator = package_list_generator(xml_file_handle, tag)
db_filename = os.path.join(self.dst_dir, "%s.db" % filename)
# always a New file, and open with Fast writing mode.
db_file_handle = gdbm.open(db_filename, "nf")
try:
for element in generator:
utils.strip_ns(element)
raw_xml = utils.element_to_raw_xml(element)
unit_key, _ = process_func(element)
db_key = self.generate_db_key(unit_key)
db_file_handle[db_key] = raw_xml
db_file_handle.sync()
finally:
db_file_handle.close()
finally:
xml_file_handle.close()
self.dbs[filename] = db_filename
def _update_provides_requires(unit):
"""
Determines the provides and requires fields based on the RPM's XML snippet and updates
the model instance.
:param unit: the unit being added to Pulp; the metadata attribute must already have
a key called 'repodata'
:type unit: subclass of pulp.server.db.model.ContentUnit
"""
try:
# make a guess at the encoding
codec = 'UTF-8'
unit.repodata['primary'].encode(codec)
except UnicodeEncodeError:
# best second guess we have, and it will never fail due to the nature
# of the encoding.
codec = 'ISO-8859-1'
unit.repodata['primary'].encode(codec)
fake_xml = FAKE_XML % {'encoding': codec, 'xml': unit.repodata['primary']}
fake_element = ET.fromstring(fake_xml.encode(codec))
utils.strip_ns(fake_element)
primary_element = fake_element.find('package')
format_element = primary_element.find('format')
provides_element = format_element.find('provides')
requires_element = format_element.find('requires')
unit.provides = map(primary._process_rpm_entry_element,
provides_element.findall('entry')) if provides_element else []
unit.requires = map(primary._process_rpm_entry_element,
requires_element.findall('entry')) if requires_element else []
def _update_provides_requires(unit):
"""
Determines the provides and requires fields based on the RPM's XML snippet and updates
the model instance.
:param unit: the unit being added to Pulp; the metadata attribute must already have
a key called 'repodata'
:type unit: pulp.plugins.model.Unit
"""
try:
# make a guess at the encoding
codec = 'UTF-8'
unit.metadata['repodata']['primary'].encode(codec)
except UnicodeEncodeError:
# best second guess we have, and it will never fail due to the nature
# of the encoding.
codec = 'ISO-8859-1'
unit.metadata['repodata']['primary'].encode(codec)
fake_xml = FAKE_XML % {'encoding': codec, 'xml': unit.metadata['repodata']['primary']}
fake_element = ET.fromstring(fake_xml.encode(codec))
utils.strip_ns(fake_element)
primary_element = fake_element.find('package')
format_element = primary_element.find('format')
provides_element = format_element.find('provides')
requires_element = format_element.find('requires')
unit.metadata['provides'] = map(primary._process_rpm_entry_element,
provides_element.findall('entry')) if provides_element else []
unit.metadata['requires'] = map(primary._process_rpm_entry_element,
requires_element.findall('entry')) if requires_element else []
def generate_dbs(self):
"""
For repo data files that contain data we need to access later for each
unit in the repo, generate a local db file that gives us quick read
access to each unit's data.
"""
for filename, tag, process_func in (
(filelists.METADATA_FILE_NAME,
filelists.PACKAGE_TAG, filelists.process_package_element),
(other.METADATA_FILE_NAME, other.PACKAGE_TAG, other.process_package_element),
):
with contextlib.closing(self.get_metadata_file_handle(filename)) as xml_file_handle:
generator = package_list_generator(xml_file_handle, tag)
db_filename = os.path.join(self.dst_dir, '%s.db' % filename)
# always a New file, and open with Fast writing mode.
with contextlib.closing(gdbm.open(db_filename, 'nf')) as db_file_handle:
for element in generator:
utils.strip_ns(element)
element.attrib['pkgid'] = models.RpmBase.PKGID_TEMPLATE
raw_xml = utils.element_to_raw_xml(element)
unit_key, _ = process_func(element)
db_key = self.generate_db_key(unit_key)
db_file_handle[db_key] = raw_xml
db_file_handle.sync()
self.dbs[filename] = db_filename
def generate_dbs(self):
"""
For repo data files that contain data we need to access later for each
unit in the repo, generate a local db file that gives us quick read
access to each unit's data.
:raises PulpCodedException: if there is some inconsistency in metadata
"""
package_count = {}
for filename, tag, process_func in (
(filelists.METADATA_FILE_NAME,
filelists.PACKAGE_TAG, filelists.process_package_element),
(other.METADATA_FILE_NAME, other.PACKAGE_TAG, other.process_package_element),
):
with contextlib.closing(self.get_metadata_file_handle(filename)) as xml_file_handle:
generator = package_list_generator(xml_file_handle, tag)
db_filename = os.path.join(self.dst_dir, '%s.db' % filename)
# always a New file, and open with Fast writing mode.
with contextlib.closing(gdbm.open(db_filename, 'nf')) as db_file_handle:
for element in generator:
utils.strip_ns(element)
element.attrib['pkgid'] = models.RpmBase.PKGID_TEMPLATE
raw_xml = utils.element_to_raw_xml(element)
unit_key, _ = process_func(element)
db_key = self.generate_db_key(unit_key)
db_file_handle[db_key] = raw_xml
db_file_handle.sync()
package_count[filename] = len(db_file_handle)
self.dbs[filename] = db_filename
if package_count[filelists.METADATA_FILE_NAME] != package_count[other.METADATA_FILE_NAME]:
reason = ('metadata is specified for different set of packages in filelists.xml'
' and in other.xml')
raise PulpCodedException(error_code=error_codes.RPM1015, reason=reason)
self.rpm_count = package_count[filelists.METADATA_FILE_NAME]
def _update_provides_requires(model):
"""
Determines the provides and requires fields based on the RPM's XML snippet and updates
the model instance.
:param model: an RPM model to which providesand requires fields should be
added. The model's metadata must already include the
'repodata' attribute.
:type model: pulp_rpm.common.models.RPM
"""
try:
# make a guess at the encoding
codec = 'UTF-8'
model.metadata['repodata']['primary'].encode(codec)
except UnicodeEncodeError:
# best second guess we have, and it will never fail due to the nature
# of the encoding.
codec = 'ISO-8859-1'
model.metadata['repodata']['primary'].encode(codec)
fake_xml = FAKE_XML % {'encoding': codec, 'xml': model.metadata['repodata']['primary']}
fake_element = ET.fromstring(fake_xml.encode(codec))
utils.strip_ns(fake_element)
primary_element = fake_element.find('package')
format_element = primary_element.find('format')
provides_element = format_element.find('provides')
requires_element = format_element.find('requires')
model.metadata['provides'] = map(primary._process_rpm_entry_element,
provides_element.findall('entry')) if provides_element else []
model.metadata['requires'] = map(primary._process_rpm_entry_element,
requires_element.findall('entry')) if requires_element else []
def test_other(self):
utils.strip_ns(self.other_element)
raw_xml = utils.element_to_raw_xml(self.other_element)
self.assertTrue(raw_xml.startswith('<package '))
self.assertTrue(raw_xml.find('<version ') >= 0)
self.assertEqual(raw_xml.count('<changelog '), 10)
self.assertEqual(raw_xml.count('author="Doug Ledford'), 7)
# fromstring just to make sure this is valid
ET.fromstring(raw_xml)
def test_other(self):
utils.strip_ns(self.other_element)
raw_xml = utils.element_to_raw_xml(self.other_element)
self.assertTrue(raw_xml.startswith('<package '))
self.assertTrue(raw_xml.find('<version ') >= 0)
self.assertEqual(raw_xml.count('<changelog '), 10)
self.assertEqual(raw_xml.count('author="Doug Ledford'), 7)
# fromstring just to make sure this is valid
ET.fromstring(raw_xml)
def test_filelists(self):
utils.strip_ns(self.filelists_element)
raw_xml = utils.element_to_raw_xml(self.filelists_element)
self.assertTrue(raw_xml.startswith('<package '))
self.assertTrue(raw_xml.find('<version ') >= 0)
self.assertTrue(raw_xml.find('name="opensm-libs"') >= 0)
self.assertTrue(raw_xml.find('<file>/usr/lib64/libosmcomp.so.3</file>') >= 0)
self.assertEqual(raw_xml.count('<file>'), 6)
# fromstring just to make sure this is valid
ET.fromstring(raw_xml)
def test_filelists(self):
utils.strip_ns(self.filelists_element)
raw_xml = utils.element_to_raw_xml(self.filelists_element)
self.assertTrue(raw_xml.startswith('<package '))
self.assertTrue(raw_xml.find('<version ') >= 0)
self.assertTrue(raw_xml.find('name="opensm-libs"') >= 0)
self.assertTrue(raw_xml.find('<file>/usr/lib64/libosmcomp.so.3</file>') >= 0)
self.assertEqual(raw_xml.count('<file>'), 6)
# fromstring just to make sure this is valid
ET.fromstring(raw_xml)
def _update_provides_requires(unit):
"""
Determines the provides and requires fields based on the RPM's XML snippet and updates
the model instance.
:param unit: the unit being added to Pulp; the metadata attribute must already have
a key called 'repodata'
:type unit: subclass of pulp.server.db.model.ContentUnit
"""
fake_element = utils.fake_xml_element(unit.repodata['primary'])
utils.strip_ns(fake_element)
primary_element = fake_element.find('package')
format_element = primary_element.find('format')
provides_element = format_element.find('provides')
requires_element = format_element.find('requires')
unit.provides = map(primary._process_rpm_entry_element,
provides_element.findall('entry')) if provides_element else []
unit.requires = map(primary._process_rpm_entry_element,
requires_element.findall('entry')) if requires_element else []
def _update_provides_requires(unit):
"""
Determines the provides and requires fields based on the RPM's XML snippet and updates
the model instance.
:param unit: the unit being added to Pulp; the metadata attribute must already have
a key called 'repodata'
:type unit: subclass of pulp.server.db.model.ContentUnit
"""
fake_element = _fake_xml_element(unit.repodata['primary'])
utils.strip_ns(fake_element)
primary_element = fake_element.find('package')
format_element = primary_element.find('format')
provides_element = format_element.find('provides')
requires_element = format_element.find('requires')
unit.provides = map(primary._process_rpm_entry_element,
provides_element.findall('entry')) if provides_element else []
unit.requires = map(primary._process_rpm_entry_element,
requires_element.findall('entry')) if requires_element else []
def generate_dbs(self):
"""
For repo data files that contain data we need to access later for each
unit in the repo, generate a local db file that gives us quick read
access to each unit's data.
:raises PulpCodedException: if there is some inconsistency in metadata
"""
package_count = {}
for filename, tag, process_func in (
(filelists.METADATA_FILE_NAME, filelists.PACKAGE_TAG,
filelists.process_package_element),
(other.METADATA_FILE_NAME, other.PACKAGE_TAG,
other.process_package_element),
):
with contextlib.closing(self.get_metadata_file_handle(
filename)) as xml_file_handle:
generator = package_list_generator(xml_file_handle, tag)
db_filename = os.path.join(self.dst_dir, '%s.db' % filename)
# always a New file, and open with Fast writing mode.
with contextlib.closing(gdbm.open(db_filename,
'nf')) as db_file_handle:
for element in generator:
utils.strip_ns(element)
element.attrib['pkgid'] = models.RpmBase.PKGID_TEMPLATE
raw_xml = utils.element_to_raw_xml(element)
unit_key, _ = process_func(element)
db_key = self.generate_db_key(unit_key)
db_file_handle[db_key] = raw_xml
db_file_handle.sync()
package_count[filename] = len(db_file_handle)
self.dbs[filename] = db_filename
if package_count[filelists.METADATA_FILE_NAME] != package_count[
other.METADATA_FILE_NAME]:
reason = (
'metadata is specified for different set of packages in filelists.xml'
' and in other.xml')
raise PulpCodedException(error_code=error_codes.RPM1015,
reason=reason)
self.rpm_count = package_count[filelists.METADATA_FILE_NAME]
def _migrate_collection(type_id):
collection = types_db.type_units_collection(type_id)
for package in collection.find():
# grab the raw XML and parse it into the elements we'll need later
try:
# make a guess at the encoding
codec = 'UTF-8'
package['repodata']['primary'].encode(codec)
except UnicodeEncodeError:
# best second guess we have, and it will never fail due to the nature
# of the encoding.
codec = 'ISO-8859-1'
package['repodata']['primary'].encode(codec)
fake_xml = FAKE_XML % {
'encoding': codec,
'xml': package['repodata']['primary']
}
fake_element = ET.fromstring(fake_xml.encode(codec))
utils.strip_ns(fake_element)
primary_element = fake_element.find('package')
format_element = primary_element.find('format')
provides_element = format_element.find('provides')
requires_element = format_element.find('requires')
# add these attributes, which we previously didn't track in the DB.
package['size'] = int(primary_element.find('size').attrib['package'])
if type_id == RPM.TYPE:
package['sourcerpm'] = format_element.find('sourcerpm').text
package['summary'] = primary_element.find('summary').text
# re-parse provides and requires. The format changed from 2.1, and the
# 2.1 upload workflow was known to produce invalid data for these fields
package['provides'] = map(
primary._process_rpm_entry_element,
provides_element.findall('entry')) if provides_element else []
package['requires'] = map(
primary._process_rpm_entry_element,
requires_element.findall('entry')) if requires_element else []
collection.save(package, safe=True)
def test_strip_all(self):
utils.strip_ns(self.element)
self._check_all_elements(self.element)
def test_strip_common_ns_only(self):
utils.strip_ns(self.element, primary.COMMON_SPEC_URL)
self.assertTrue(self._check_for_one_ns(self.element, primary.RPM_SPEC_URL))
def test_strip_common_ns_only(self):
utils.strip_ns(self.element, primary.COMMON_SPEC_URL)
self.assertTrue(self._check_for_one_ns(self.element, primary.RPM_SPEC_URL))
def test_strip_all(self):
utils.strip_ns(self.element)
self._check_all_elements(self.element)
