def rpm_search_dicts(self):
ret = []
for collection in self.pkglist:
for package in collection.get('packages', []):
if len(package.get('sum') or []) == 2:
checksum = package['sum'][1]
checksumtype = verification.sanitize_checksum_type(package['sum'][0])
elif 'sums' in package and 'type' in package:
# these are the field names we get from an erratum upload.
# I have no idea why they are different.
checksum = package['sums']
checksumtype = verification.sanitize_checksum_type(package['type'])
else:
checksum = None
checksumtype = None
rpm = RPM(name=package['name'], epoch=package['epoch'],
version=package['version'], release=package['release'],
arch=package['arch'], checksum=checksum,
checksumtype=checksumtype)
unit_key = rpm.unit_key
for key in ['checksum', 'checksumtype']:
if unit_key[key] is None:
del unit_key[key]
ret.append(unit_key)
return ret
def rpm_search_dicts(self):
ret = []
for collection in self.pkglist:
for package in collection.get('packages', []):
if len(package.get('sum') or []) == 2:
checksum = package['sum'][1]
checksumtype = verification.sanitize_checksum_type(
package['sum'][0])
elif 'sums' in package and 'type' in package:
# these are the field names we get from an erratum upload.
# I have no idea why they are different.
checksum = package['sums']
checksumtype = verification.sanitize_checksum_type(
package['type'])
else:
checksum = None
checksumtype = None
rpm = RPM(name=package['name'],
epoch=package['epoch'],
version=package['version'],
release=package['release'],
arch=package['arch'],
checksum=checksum,
checksumtype=checksumtype)
unit_key = rpm.unit_key
for key in ['checksum', 'checksumtype']:
if unit_key[key] is None:
del unit_key[key]
ret.append(unit_key)
return ret
def rpm_search_dicts(self):
ret = []
for collection in self.pkglist:
for package in collection.get("packages", []):
if len(package.get("sum") or []) == 2:
checksum = package["sum"][1]
checksumtype = verification.sanitize_checksum_type(package["sum"][0])
elif "sums" in package and "type" in package:
# these are the field names we get from an erratum upload.
# I have no idea why they are different.
checksum = package["sums"]
checksumtype = verification.sanitize_checksum_type(package["type"])
else:
checksum = None
checksumtype = None
rpm = RPM(
name=package["name"],
epoch=package["epoch"],
version=package["version"],
release=package["release"],
arch=package["arch"],
checksum=checksum,
checksumtype=checksumtype,
)
unit_key = rpm.unit_key
for key in ["checksum", "checksumtype"]:
if unit_key[key] is None:
del unit_key[key]
ret.append(unit_key)
return ret
def process_repomd_data_element(data_element):
"""
Process the data elements of the repomd.xml file.
This returns a file information dictionary with the following keys:
* `name`: name of the element
* `relative_path`: the path of the metadata file, relative to the repository URL
* `checksum`: dictionary of `algorithm` and `hex_digest` keys and values
* `size`: size of the metadata file, in bytes
* `timestamp`: unix timestamp of the file's creation, as a float
* `open_checksum`: optional checksum dictionary of uncompressed metadata file
* `open_size`: optional size of the uncompressed metadata file, in bytes
:param data_element: XML data element parsed from the repomd.xml file
:return: file_info dictionary
:rtype: dict
"""
file_info = deepcopy(FILE_INFO_SKEL)
file_info['name'] = data_element.attrib['type']
location_element = data_element.find(LOCATION_TAG)
if location_element is not None:
file_info['relative_path'] = location_element.attrib['href']
checksum_element = data_element.find(CHECKSUM_TAG)
if checksum_element is not None:
checksum_type = verification.sanitize_checksum_type(
checksum_element.attrib['type'])
file_info['checksum']['algorithm'] = checksum_type
file_info['checksum']['hex_digest'] = checksum_element.text
size_element = data_element.find(SIZE_TAG)
if size_element is not None:
file_info['size'] = int(size_element.text)
timestamp_element = data_element.find(TIMESTAMP_TAG)
if timestamp_element is not None:
file_info['timestamp'] = float(timestamp_element.text)
open_checksum_element = data_element.find(OPEN_CHECKSUM_TAG)
if open_checksum_element is not None:
checksum_type = verification.sanitize_checksum_type(
open_checksum_element.attrib['type'])
file_info['open_checksum']['algorithm'] = checksum_type
file_info['open_checksum']['hex_digest'] = open_checksum_element.text
open_size_element = data_element.find(OPEN_SIZE_TAG)
if open_size_element is not None:
file_info['open_size'] = int(open_size_element.text)
for child in data_element.getchildren():
child.clear()
data_element.clear()
return file_info
def process_repomd_data_element(data_element):
"""
Process the data elements of the repomd.xml file.
This returns a file information dictionary with the following keys:
* `name`: name of the element
* `relative_path`: the path of the metadata file, relative to the repository URL
* `checksum`: dictionary of `algorithm` and `hex_digest` keys and values
* `size`: size of the metadata file, in bytes
* `timestamp`: unix timestamp of the file's creation, as a float
* `open_checksum`: optional checksum dictionary of uncompressed metadata file
* `open_size`: optional size of the uncompressed metadata file, in bytes
:param data_element: XML data element parsed from the repomd.xml file
:return: file_info dictionary
:rtype: dict
"""
file_info = deepcopy(FILE_INFO_SKEL)
file_info['name'] = data_element.attrib['type']
location_element = data_element.find(LOCATION_TAG)
if location_element is not None:
file_info['relative_path'] = location_element.attrib['href']
checksum_element = data_element.find(CHECKSUM_TAG)
if checksum_element is not None:
checksum_type = verification.sanitize_checksum_type(checksum_element.attrib['type'])
file_info['checksum']['algorithm'] = checksum_type
file_info['checksum']['hex_digest'] = checksum_element.text
size_element = data_element.find(SIZE_TAG)
if size_element is not None:
file_info['size'] = int(size_element.text)
timestamp_element = data_element.find(TIMESTAMP_TAG)
if timestamp_element is not None:
file_info['timestamp'] = float(timestamp_element.text)
open_checksum_element = data_element.find(OPEN_CHECKSUM_TAG)
if open_checksum_element is not None:
checksum_type = verification.sanitize_checksum_type(open_checksum_element.attrib['type'])
file_info['open_checksum']['algorithm'] = checksum_type
file_info['open_checksum']['hex_digest'] = open_checksum_element.text
open_size_element = data_element.find(OPEN_SIZE_TAG)
if open_size_element is not None:
file_info['open_size'] = int(open_size_element.text)
for child in data_element.getchildren():
child.clear()
data_element.clear()
return file_info
def validate(self, value):
"""
Validates that value is a checksumtype known to pulp platform
:param value: The value to validate
:type value: basestring
:return: None
"""
super(ChecksumTypeStringField, self).validate(value)
verification.sanitize_checksum_type(value)
def validate(self, value):
"""
Validates that value is a checksumtype known to pulp platform
:param value: The value to validate
:type value: basestring
:return: None
"""
super(ChecksumTypeStringField, self).validate(value)
verification.sanitize_checksum_type(value)
def process_download_reports(self, reports):
"""
Once downloading is complete, add information about each file to this
model instance. This is required before saving the new unit.
:param reports: list of successful download reports
:type reports: list(pulp.common.download.report.DownloadReport)
"""
# TODO: maybe this shouldn't be in common
metadata_files = self.metadata.setdefault('files', [])
for report in reports:
# the following data model is mostly intended to match what the
# previous importer generated.
metadata_files.append({
'checksum': report.data['checksum'],
'checksumtype': verification.sanitize_checksum_type(report.data['checksumtype']),
'downloadurl': report.url,
'filename': os.path.basename(report.data['relativepath']),
'fileName': os.path.basename(report.data['relativepath']),
'item_type': self.TYPE,
'pkgpath': os.path.join(
constants.DISTRIBUTION_STORAGE_PATH,
self.id,
os.path.dirname(report.data['relativepath']),
),
'relativepath': report.data['relativepath'],
'savepath': report.destination,
'size': report.total_bytes,
})
def test_nothing_necessary(self):
"""
Assert that the method doesn't change the checksum_type when it's not needed.
"""
checksum_type = verification.sanitize_checksum_type('sha512')
self.assertEqual(checksum_type, 'sha512')
def test_none(self):
"""
Assert correct behavior when the checksum_type is None.
"""
checksum_type = verification.sanitize_checksum_type(None)
self.assertEqual(checksum_type, None)
def process_package_element(element):
"""
Process one XML block from prestodelta.xml and return a models.DRPM instance
:param element: object representing one "DRPM" block from the XML file
:type element: xml.etree.ElementTree.Element
:return: models.DRPM instance for the XML block
:rtype: pulp_rpm.plugins.db.models.DRPM
"""
delta = element.find('delta')
filename = delta.find('filename')
sequence = delta.find('sequence')
size = delta.find('size')
checksum = delta.find('checksum')
checksum_type = verification.sanitize_checksum_type(checksum.attrib['type'])
return models.DRPM.from_package_info({
'type': 'drpm',
'new_package': element.attrib['name'],
'epoch': element.attrib['epoch'],
'version': element.attrib['version'],
'release': element.attrib['release'],
'arch': element.attrib['arch'],
'oldepoch': delta.attrib['oldepoch'],
'oldversion': delta.attrib['oldversion'],
'oldrelease': delta.attrib['oldrelease'],
'filename': filename.text,
'sequence': sequence.text,
'size': int(size.text),
'checksum': checksum.text,
'checksumtype': checksum_type,
})
def process_package_element(element):
"""
Process one XML block from prestodelta.xml and return a models.DRPM instance
:param element: object representing one "DRPM" block from the XML file
:type element: xml.etree.ElementTree.Element
:return: models.DRPM instance for the XML block
:rtype: pulp_rpm.plugins.db.models.DRPM
"""
delta = element.find('delta')
filename = delta.find('filename')
sequence = delta.find('sequence')
size = delta.find('size')
checksum = delta.find('checksum')
checksum_type = verification.sanitize_checksum_type(checksum.attrib['type'])
return models.DRPM.from_package_info({
'type': 'drpm',
'new_package': element.attrib['name'],
'epoch': element.attrib['epoch'],
'version': element.attrib['version'],
'release': element.attrib['release'],
'arch': element.attrib['arch'],
'oldepoch': delta.attrib['oldepoch'],
'oldversion': delta.attrib['oldversion'],
'oldrelease': delta.attrib['oldrelease'],
'filename': filename.text,
'sequence': sequence.text,
'size': int(size.text),
'checksum': checksum.text,
'checksumtype': checksum_type,
})
def import_unknown_metadata_files(self, metadata_files):
"""
Import metadata files whose type is not known to us. These are any files
that we are not already parsing.
:param metadata_files: object containing access to all metadata files
:type metadata_files: pulp_rpm.plugins.importers.yum.repomd.metadata.MetadataFiles
"""
for metadata_type, file_info in metadata_files.metadata.iteritems():
if metadata_type not in metadata_files.KNOWN_TYPES:
file_path = file_info['local_path']
checksum_type = file_info['checksum']['algorithm']
checksum_type = verification.sanitize_checksum_type(checksum_type)
checksum = file_info['checksum']['hex_digest']
# Find an existing model
model = models.YumMetadataFile.objects.filter(
data_type=metadata_type,
repo_id=self.repo.repo_id).first()
# If an existing model, use that
if model:
model.checksum = checksum
model.checksum_type = checksum_type
else:
# Else, create a new mode
model = models.YumMetadataFile(
data_type=metadata_type,
repo_id=self.repo.repo_id,
checksum=checksum,
checksum_type=checksum_type)
model.set_storage_path(os.path.basename(file_path))
model.save_and_import_content(file_path)
# associate/re-associate model to the repo
repo_controller.associate_single_unit(self.repo, model)
def test_nothing_necessary(self):
"""
Assert that the method doesn't change the checksum_type when it's not needed.
"""
checksum_type = verification.sanitize_checksum_type('sha512')
self.assertEqual(checksum_type, 'sha512')
def test_none(self):
"""
Assert correct behavior when the checksum_type is None.
"""
checksum_type = verification.sanitize_checksum_type(None)
self.assertEqual(checksum_type, None)
def process_package_element(element):
"""
Process one XML block from prestodelta.xml and return a models.DRPM instance
:param element: object representing one "DRPM" block from the XML file
:type element: xml.etree.ElementTree.Element
:return: models.DRPM instance for the XML block
:rtype: pulp_rpm.plugins.db.models.DRPM
"""
delta = element.find("delta")
filename = delta.find("filename")
sequence = delta.find("sequence")
size = delta.find("size")
checksum = delta.find("checksum")
checksum_type = verification.sanitize_checksum_type(checksum.attrib["type"])
return models.DRPM.from_package_info(
{
"type": "drpm",
"new_package": element.attrib["name"],
"epoch": element.attrib["epoch"],
"version": element.attrib["version"],
"release": element.attrib["release"],
"arch": element.attrib["arch"],
"oldepoch": delta.attrib["oldepoch"],
"oldversion": delta.attrib["oldversion"],
"oldrelease": delta.attrib["oldrelease"],
"filename": filename.text,
"sequence": sequence.text,
"size": int(size.text),
"checksum": checksum.text,
"checksumtype": checksum_type,
}
)
def test_SHA256_to_sha256(self):
"""
Assert that "SHA256" is converted to "sha256".
"""
checksum_type = verification.sanitize_checksum_type('SHA256')
self.assertEqual(checksum_type, 'sha256')
def test_ShA_to_sha1(self):
"""
Assert that "ShA" is converted to "sha1".
"""
checksum_type = verification.sanitize_checksum_type('ShA')
self.assertEqual(checksum_type, 'sha1')
def process_package_element(element):
"""
Process one XML block from prestodelta.xml and return a models.DRPM instance
:param element: object representing one "DRPM" block from the XML file
:type element: xml.etree.ElementTree.Element
:return: models.DRPM instance for the XML block
:rtype: pulp_rpm.plugins.db.models.DRPM
"""
delta = element.find('delta')
filename = delta.find('filename')
sequence = delta.find('sequence')
size = delta.find('size')
checksum = delta.find('checksum')
checksum_type = verification.sanitize_checksum_type(checksum.attrib['type'])
return models.DRPM(
new_package=element.attrib['name'],
epoch=element.attrib['epoch'],
version=element.attrib['version'],
release=element.attrib['release'],
arch=element.attrib['arch'],
oldepoch=delta.attrib['oldepoch'],
oldversion=delta.attrib['oldversion'],
oldrelease=delta.attrib['oldrelease'],
filename=filename.text,
sequence=sequence.text,
size=int(size.text),
checksum=checksum.text,
checksumtype=checksum_type)
def import_unknown_metadata_files(self, metadata_files):
"""
Import metadata files whose type is not known to us. These are any files
that we are not already parsing.
:param metadata_files: object containing access to all metadata files
:type metadata_files: pulp_rpm.plugins.importers.yum.repomd.metadata.MetadataFiles
"""
for metadata_type, file_info in metadata_files.metadata.iteritems():
if metadata_type not in metadata_files.KNOWN_TYPES:
checksum_type = file_info['checksum']['algorithm']
checksum_type = verification.sanitize_checksum_type(
checksum_type)
unit_metadata = {
'checksum': file_info['checksum']['hex_digest'],
'checksum_type': checksum_type,
}
model = models.YumMetadataFile(metadata_type,
self.sync_conduit.repo_id,
unit_metadata)
relative_path = os.path.join(
model.relative_dir,
os.path.basename(file_info['local_path']))
unit = self.sync_conduit.init_unit(models.YumMetadataFile.TYPE,
model.unit_key,
model.metadata,
relative_path)
shutil.copyfile(file_info['local_path'], unit.storage_path)
self.sync_conduit.save_unit(unit)
def import_unknown_metadata_files(self, metadata_files):
"""
Import metadata files whose type is not known to us. These are any files
that we are not already parsing.
:param metadata_files: object containing access to all metadata files
:type metadata_files: pulp_rpm.plugins.importers.yum.repomd.metadata.MetadataFiles
"""
for metadata_type, file_info in metadata_files.metadata.iteritems():
if metadata_type not in metadata_files.KNOWN_TYPES:
file_path = file_info['local_path']
checksum_type = file_info['checksum']['algorithm']
checksum_type = verification.sanitize_checksum_type(
checksum_type)
checksum = file_info['checksum']['hex_digest']
# Find an existing model
model = models.YumMetadataFile.objects.filter(
data_type=metadata_type,
repo_id=self.repo.repo_id).first()
# If an existing model, use that
if model:
model.checksum = checksum
model.checksum_type = checksum_type
else:
# Else, create a new mode
model = models.YumMetadataFile(data_type=metadata_type,
repo_id=self.repo.repo_id,
checksum=checksum,
checksum_type=checksum_type)
model.set_storage_path(os.path.basename(file_path))
model.save_and_import_content(file_path)
# associate/re-associate model to the repo
repo_controller.associate_single_unit(self.repo, model)
def process_download_reports(self, reports):
"""
Once downloading is complete, add information about each file to this
model instance. This is required before saving the new unit.
:param reports: list of successful download reports
:type reports: list(pulp.common.download.report.DownloadReport)
"""
if not isinstance(self.files, list):
self.files = []
for report in reports:
# the following data model is mostly intended to match what the
# previous importer generated.
self.files.append({
'checksum': report.data['checksum'],
'checksumtype': verification.sanitize_checksum_type(report.data['checksumtype']),
'downloadurl': report.url,
'filename': os.path.basename(report.data['relativepath']),
'fileName': os.path.basename(report.data['relativepath']),
'item_type': "distribution",
'pkgpath': os.path.join(
self._storage_path, os.path.dirname(report.data['relativepath']),
),
'relativepath': report.data['relativepath'],
'savepath': report.destination,
'size': report.total_bytes,
})
def import_unknown_metadata_files(self, metadata_files):
"""
Import metadata files whose type is not known to us. These are any files
that we are not already parsing.
:param metadata_files: object containing access to all metadata files
:type metadata_files: pulp_rpm.plugins.importers.yum.repomd.metadata.MetadataFiles
"""
for metadata_type, file_info in metadata_files.metadata.iteritems():
if metadata_type not in metadata_files.KNOWN_TYPES:
checksum_type = file_info['checksum']['algorithm']
checksum_type = verification.sanitize_checksum_type(checksum_type)
unit_metadata = {
'checksum': file_info['checksum']['hex_digest'],
'checksum_type': checksum_type,
}
model = models.YumMetadataFile(metadata_type,
self.sync_conduit.repo_id,
unit_metadata)
relative_path = os.path.join(model.relative_dir,
os.path.basename(file_info['local_path']))
unit = self.sync_conduit.init_unit(models.YumMetadataFile.TYPE, model.unit_key,
model.metadata, relative_path)
shutil.copyfile(file_info['local_path'], unit.storage_path)
self.sync_conduit.save_unit(unit)
def _migrate_rpmlike_units(unit_type):
"""
This function performs the migration on RPMs, DRPMs, and SRPMs. These all have the same schema
when it comes to checksumtype, so they can be treated the same way.
:param unit_type: The unit_type_id, as found in pulp_rpm.common.ids.
:type unit_type: basestring
"""
repos = connection.get_collection('repos')
repo_content_units = connection.get_collection('repo_content_units')
unit_collection = connection.get_collection('units_%s' % unit_type)
for unit in unit_collection.find():
try:
sanitized_type = verification.sanitize_checksum_type(unit['checksumtype'])
if sanitized_type != unit['checksumtype']:
# Let's see if we can get away with changing its checksumtype to the sanitized
# value. If this works, we won't have to do anything else.
unit_collection.update({'_id': unit['_id']},
{'$set': {'checksumtype': sanitized_type}})
except errors.DuplicateKeyError:
# Looks like there is already an identical unit with the sanitized checksum type. This
# means we need to remove the current unit, but first we will need to change any
# references to this unit to point to the other.
conflicting_unit = unit_collection.find_one(
{'name': unit['name'], 'epoch': unit['epoch'], 'version': unit['version'],
'release': unit['release'], 'arch': unit['arch'], 'checksum': unit['checksum'],
'checksumtype': sanitized_type})
for rcu in repo_content_units.find({'unit_type_id': unit_type, 'unit_id': unit['_id']}):
# Now we must either switch the rcu from pointing to unit to pointing to
# conflicting_unit, or delete the rcu if there is already one in the same repo.
try:
msg = _('Updating %(repo_id)s to contain %(type)s %(conflicting)s instead of '
'%(old_id)s.')
msg = msg % {'repo_id': rcu['repo_id'], 'type': unit_type,
'conflicting': conflicting_unit['_id'], 'old_id': unit['_id']}
_logger.debug(msg)
repo_content_units.update({'_id': rcu['_id']},
{'$set': {'unit_id': conflicting_unit['_id']}})
except errors.DuplicateKeyError:
# We will delete this RepoContentUnit since the sha1 RPM is already in the
# repository.
msg = _('Removing %(type)s %(old_id)s from repo %(repo_id)s since it conflicts '
'with %(conflicting)s.')
msg = msg % {'repo_id': rcu['repo_id'], 'type': unit_type,
'conflicting': conflicting_unit['_id'], 'old_id': unit['_id']}
_logger.debug(msg)
repo_content_units.remove({'_id': rcu['_id']})
# In this case, we now need to decrement the repository's "content_unit_counts"
# for this unit_type by one, since we removed a unit from a repository.
repos.update(
{'id': rcu['repo_id']},
{'$inc': {'content_unit_counts.%s' % unit_type: -1}})
# Now that we have removed or altered all references to the "sha" Unit, we need to
# remove it since it is a duplicate.
unit_collection.remove({'_id': unit['_id']})
def _migrate_yum_metadata_files():
"""
Migrate each YumMetadataFile to use the new sanitized checksum_type. This is mostly similar to
_migrate_rpmlike_units, except that the checksum type field name is checksum_type instead of
checksumtype, and there can't be any collisions since the checksum type isn't part of this
unit's unit_key. This means we don't have to worry about the repo_content_units table.
"""
collection = connection.get_collection('units_yum_repo_metadata_file')
for unit in collection.find():
sanitized_type = verification.sanitize_checksum_type(unit['checksum_type'])
if sanitized_type != unit['checksum_type']:
collection.update({'_id': unit['_id']},
{'$set': {'checksum_type': sanitized_type}})
def _migrate_yum_metadata_files():
"""
Migrate each YumMetadataFile to use the new sanitized checksum_type. This is mostly similar to
_migrate_rpmlike_units, except that the checksum type field name is checksum_type instead of
checksumtype, and there can't be any collisions since the checksum type isn't part of this
unit's unit_key. This means we don't have to worry about the repo_content_units table.
"""
collection = connection.get_collection('units_yum_repo_metadata_file')
for unit in collection.find():
sanitized_type = verification.sanitize_checksum_type(
unit['checksum_type'])
if sanitized_type != unit['checksum_type']:
collection.update({'_id': unit['_id']},
{'$set': {
'checksum_type': sanitized_type
}})
def update_unit_files(unit, files):
"""
Update the *files* list on the unit.
:param unit: A distribution model object.
:type unit: pulp_rpm.plugins.db.models.Distribution
:param files: List of distribution files.
:type files: list
"""
_list = []
if not isinstance(unit.files, list):
_list = list(unit.files)
for _file in files:
_list.append({
RELATIVE_PATH: _file[RELATIVE_PATH],
CHECKSUM: _file[CHECKSUM],
CHECKSUM_TYPE: verification.sanitize_checksum_type(_file[CHECKSUM_TYPE])})
unit.files = _list
def process_successful_download_reports(unit, reports):
"""
Once downloading is complete, add information about each file to this
model instance. This is required before saving the new unit.
:param reports: list of successful pulp.common.download.report.DownloadReport
:type reports: list
"""
files = []
if not isinstance(unit.files, list):
files = list(unit.files)
for report in reports:
_file = report.data
files.append({
RELATIVE_PATH: _file[RELATIVE_PATH],
CHECKSUM: _file[CHECKSUM],
CHECKSUM_TYPE: verification.sanitize_checksum_type(_file[CHECKSUM_TYPE])})
unit.files = files
def update_unit_files(unit, files):
"""
Update the *files* list on the unit.
:param unit: A distribution model object.
:type unit: pulp_rpm.plugins.db.models.Distribution
:param files: List of distribution files.
:type files: list
"""
_list = []
if not isinstance(unit.files, list):
_list = list(unit.files)
for _file in files:
if _file[CHECKSUM_TYPE] is not None:
_file[CHECKSUM_TYPE] = verification.sanitize_checksum_type(_file[CHECKSUM_TYPE])
_list.append({
RELATIVE_PATH: _file[RELATIVE_PATH],
CHECKSUM: _file[CHECKSUM],
CHECKSUM_TYPE: _file[CHECKSUM_TYPE]})
unit.files = _list
def save_default_metadata_checksum_on_repo(self, metadata_files):
"""
Determine the default checksum that should be used for metadata files and save it in
the repo scratchpad.
There is no good way to order a preference on the checksum type so the first one
found is used.
:param metadata_files: object containing access to all metadata files
:type metadata_files: pulp_rpm.plugins.importers.yum.repomd.metadata.MetadataFiles
"""
checksum_type = None
for metadata_item in metadata_files.metadata.iteritems():
if 'checksum' in metadata_item[1]:
checksum_type = metadata_item[1]['checksum']['algorithm']
break
if checksum_type:
checksum_type = verification.sanitize_checksum_type(checksum_type)
scratchpad = self.conduit.get_repo_scratchpad()
scratchpad[constants.SCRATCHPAD_DEFAULT_METADATA_CHECKSUM] = checksum_type
self.conduit.set_repo_scratchpad(scratchpad)
def _migrate_errata():
"""
Visit each errata and check its references RPMs for the checksum type, sanitizing it if
necessary. Since these sums aren't part of the unit key for erratum, this will not cause any
collisions. The erratum also do not reference RPMs by unit_id, but by unit_key, so this is
important.
"""
errata = connection.get_collection('units_erratum')
for erratum in errata.find():
changed = False
pkglist = erratum.get('pkglist', [])
for collection in pkglist:
for package in collection.get('packages', []):
if 'sum' in package and package['sum']:
sanitized_type = verification.sanitize_checksum_type(package['sum'][0])
if sanitized_type != package['sum'][0]:
package['sum'][0] = sanitized_type
changed = True
if changed:
errata.update({'_id': erratum['_id']},
{'$set': {'pkglist': pkglist}})
def save_default_metadata_checksum_on_repo(self, metadata_files):
"""
Determine the default checksum that should be used for metadata files and save it in
the repo scratchpad.
There is no good way to order a preference on the checksum type so the first one
found is used.
:param metadata_files: object containing access to all metadata files
:type metadata_files: pulp_rpm.plugins.importers.yum.repomd.metadata.MetadataFiles
"""
checksum_type = None
for metadata_item in metadata_files.metadata.iteritems():
if 'checksum' in metadata_item[1]:
checksum_type = metadata_item[1]['checksum']['algorithm']
break
if checksum_type:
checksum_type = verification.sanitize_checksum_type(checksum_type)
scratchpad = self.conduit.get_repo_scratchpad()
scratchpad[constants.SCRATCHPAD_DEFAULT_METADATA_CHECKSUM] = checksum_type
self.conduit.set_repo_scratchpad(scratchpad)
def process_successful_download_reports(unit, reports):
"""
Once downloading is complete, add information about each file to this
model instance. This is required before saving the new unit.
:param reports: list of successful pulp.common.download.report.DownloadReport
:type reports: list
"""
files = []
if not isinstance(unit.files, list):
files = list(unit.files)
for report in reports:
_file = report.data
files.append({
RELATIVE_PATH:
_file[RELATIVE_PATH],
CHECKSUM:
_file[CHECKSUM],
CHECKSUM_TYPE:
verification.sanitize_checksum_type(_file[CHECKSUM_TYPE])
})
unit.files = files
def _migrate_errata():
"""
Visit each errata and check its references RPMs for the checksum type, sanitizing it if
necessary. Since these sums aren't part of the unit key for erratum, this will not cause any
collisions. The erratum also do not reference RPMs by unit_id, but by unit_key, so this is
important.
"""
errata = connection.get_collection('units_erratum')
for erratum in errata.find():
changed = False
pkglist = erratum.get('pkglist', [])
for collection in pkglist:
for package in collection.get('packages', []):
if 'sum' in package and package['sum']:
sanitized_type = verification.sanitize_checksum_type(
package['sum'][0])
if sanitized_type != package['sum'][0]:
package['sum'][0] = sanitized_type
changed = True
if changed:
errata.update({'_id': erratum['_id']},
{'$set': {
'pkglist': pkglist
}})
def parse_treefile(path):
"""
The treefile seems to be approximately in INI format, which can be read
by the standard library's ConfigParser.
:param path: full path to the treefile
:return: instance of Distribution model, and a list of dicts
describing the distribution's files
:rtype: (pulp_rpm.plugins.db.models.Distribution, list of dict)
"""
parser = ConfigParser.RawConfigParser()
# the default implementation of this method makes all option names lowercase,
# which we don't want. This is the suggested solution in the python.org docs.
parser.optionxform = str
with open(path) as open_file:
try:
parser.readfp(open_file)
except ConfigParser.ParsingError:
# wouldn't need this if ParsingError subclassed ValueError.
raise ValueError('could not parse treeinfo file')
# apparently the 'variant' is optional. for example, it does not appear
# in the RHEL 5.9 treeinfo file. This is how the previous importer
# handled that.
try:
variant = parser.get(SECTION_GENERAL, 'variant')
except ConfigParser.NoOptionError:
variant = None
try:
packagedir = parser.get(SECTION_GENERAL, KEY_PACKAGEDIR)
except ConfigParser.NoOptionError:
packagedir = None
try:
model = models.Distribution(
parser.get(SECTION_GENERAL, 'family'),
variant,
parser.get(SECTION_GENERAL, 'version'),
parser.get(SECTION_GENERAL, 'arch'),
metadata={
KEY_PACKAGEDIR: packagedir,
KEY_TIMESTAMP: float(parser.get(SECTION_GENERAL, KEY_TIMESTAMP)),
}
)
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
raise ValueError('invalid treefile: could not find unit key components')
files = {}
# this section is likely to have all the files we care about listed with
# checksums. But, it might not. Other sections checked below will only add
# files to the "files" dict if they are not already present. For those cases,
# there will not be checksums available.
if parser.has_section(SECTION_CHECKSUMS):
for item in parser.items(SECTION_CHECKSUMS):
relativepath = item[0]
checksumtype, checksum = item[1].split(':')
checksumtype = verification.sanitize_checksum_type(checksumtype)
files[relativepath] = {
'relativepath': relativepath,
'checksum': checksum,
'checksumtype': checksumtype
}
for section_name in parser.sections():
if section_name.startswith('images-') or section_name == SECTION_STAGE2:
for item in parser.items(section_name):
if item[1] not in files:
relativepath = item[1]
files[relativepath] = {
'relativepath': relativepath,
'checksum': None,
'checksumtype': None,
}
return model, files.values()
def process_package_element(package_element):
"""
Process a parsed primary.xml package element into a package information
dictionary.
:param package_element: parsed primary.xml package element
:return: package information dictionary
:rtype: pulp_rpm.plugins.db.models.RPM
"""
# NOTE the use of deepcopy relies on cpython's very sensible policy of never
# duplicating string literals, this may not hold up in other implementations
# the python interpreter.
package_info = dict()
name_element = package_element.find(NAME_TAG)
if name_element is not None:
package_info['name'] = name_element.text
arch_element = package_element.find(ARCH_TAG)
if arch_element is not None:
package_info['arch'] = arch_element.text
version_element = package_element.find(VERSION_TAG)
if version_element is not None:
package_info['version'] = version_element.attrib['ver']
package_info['release'] = version_element.attrib.get('rel', None)
package_info['epoch'] = version_element.attrib.get('epoch', None)
checksum_element = package_element.find(CHECKSUM_TAG)
if checksum_element is not None:
checksum_type = verification.sanitize_checksum_type(checksum_element.attrib['type'])
package_info['checksumtype'] = checksum_type
package_info['checksum'] = checksum_element.text
summary_element = package_element.find(SUMMARY_TAG)
if summary_element is not None:
package_info['summary'] = summary_element.text
description_element = package_element.find(DESCRIPTION_TAG)
if description_element is not None:
package_info['description'] = description_element.text
url_element = package_element.find(URL_TAG)
if url_element is not None:
package_info['url'] = url_element.text
time_element = package_element.find(TIME_TAG)
if time_element is not None:
package_info['time'] = int(time_element.attrib['file'])
package_info['build_time'] = int(time_element.attrib['build'])
size_element = package_element.find(SIZE_TAG)
if size_element is not None:
package_info['size'] = int(size_element.attrib['package'])
location_element = package_element.find(LOCATION_TAG)
if location_element is not None:
href = location_element.attrib['href']
base_url = None
for attribute, value in location_element.items():
if attribute == 'base' or attribute.endswith('}base'):
base_url = value
package_info['base_url'] = base_url
filename = os.path.basename(href)
package_info['relativepath'] = href
package_info['filename'] = filename
# we don't make any attempt to preserve the original directory structure
# this element will end up being converted back to XML and stuffed into
# the DB on the unit object, so this is our chance to modify it.
location_element.attrib['href'] = filename
format_element = package_element.find(FORMAT_TAG)
package_info.update(_process_format_element(format_element))
if package_info['arch'].lower() == 'src':
model = models.SRPM(**package_info)
else:
model = models.RPM(**package_info)
# add the raw XML so it can be saved in the database later
rpm_namespace = utils.Namespace('rpm', RPM_SPEC_URL)
model.raw_xml = utils.element_to_raw_xml(package_element, [rpm_namespace], COMMON_SPEC_URL)
return model
def _generate_rpm_data(type_id, rpm_filename, user_metadata=None):
"""
For the given RPM, analyzes its metadata to generate the appropriate unit
key and metadata fields, returning both to the caller.
:param type_id: The type of the unit that is being generated
:type type_id: str
:param rpm_filename: full path to the RPM to analyze
:type rpm_filename: str
:param user_metadata: user supplied metadata about the unit. This is optional.
:type user_metadata: dict
:return: tuple of unit key and unit metadata for the RPM
:rtype: tuple
"""
# Expected metadata fields:
# "vendor", "description", "buildhost", "license", "vendor", "requires", "provides",
# "relativepath", "filename"
#
# Expected unit key fields:
# "name", "epoch", "version", "release", "arch", "checksumtype", "checksum"
unit_key = dict()
metadata = dict()
# Read the RPM header attributes for use later
ts = rpm.TransactionSet()
ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES)
fd = os.open(rpm_filename, os.O_RDONLY)
try:
headers = ts.hdrFromFdno(fd)
os.close(fd)
except rpm.error:
# Raised if the headers cannot be read
os.close(fd)
raise
# -- Unit Key -----------------------
# Checksum
if user_metadata and user_metadata.get('checksum_type'):
user_checksum_type = user_metadata.get('checksum_type')
user_checksum_type = verification.sanitize_checksum_type(
user_checksum_type)
unit_key['checksumtype'] = user_checksum_type
else:
unit_key['checksumtype'] = verification.TYPE_SHA256
unit_key['checksum'] = _calculate_checksum(unit_key['checksumtype'],
rpm_filename)
# Name, Version, Release, Epoch
for k in ['name', 'version', 'release', 'epoch']:
unit_key[k] = headers[k]
# Epoch munging
if unit_key['epoch'] is None:
unit_key['epoch'] = str(0)
else:
unit_key['epoch'] = str(unit_key['epoch'])
# Arch
if headers['sourcepackage']:
if RPMTAG_NOSOURCE in headers.keys():
unit_key['arch'] = 'nosrc'
else:
unit_key['arch'] = 'src'
else:
unit_key['arch'] = headers['arch']
# -- Unit Metadata ------------------
# construct filename from metadata (BZ #1101168)
if headers[rpm.RPMTAG_SOURCEPACKAGE]:
if type_id != models.SRPM.TYPE:
raise PulpCodedValidationException(error_code=error_codes.RPM1002)
rpm_basefilename = "%s-%s-%s.src.rpm" % (
headers['name'], headers['version'], headers['release'])
else:
if type_id != models.RPM.TYPE:
raise PulpCodedValidationException(error_code=error_codes.RPM1003)
rpm_basefilename = "%s-%s-%s.%s.rpm" % (
headers['name'], headers['version'], headers['release'],
headers['arch'])
metadata['relativepath'] = rpm_basefilename
metadata['filename'] = rpm_basefilename
# This format is, and has always been, incorrect. As of the new yum importer, the
# plugin will generate these from the XML snippet because the API into RPM headers
# is atrocious. This is the end game for this functionality anyway, moving all of
# that metadata derivation into the plugin, so this is just a first step.
# I'm leaving these in and commented to show how not to do it.
# metadata['requires'] = [(r,) for r in headers['requires']]
# metadata['provides'] = [(p,) for p in headers['provides']]
metadata['buildhost'] = headers['buildhost']
metadata['license'] = headers['license']
metadata['vendor'] = headers['vendor']
metadata['description'] = headers['description']
metadata['build_time'] = headers[rpm.RPMTAG_BUILDTIME]
# Use the mtime of the file to match what is in the generated xml from
# rpm_parse.get_package_xml(..)
file_stat = os.stat(rpm_filename)
metadata['time'] = file_stat[stat.ST_MTIME]
return unit_key, metadata
def __init__(self, *args, **kwargs):
if kwargs.get('checksumtype') is not None:
kwargs['checksumtype'] = verification.sanitize_checksum_type(kwargs['checksumtype'])
super(NonMetadataPackage, self).__init__(*args, **kwargs)
def _generate_rpm_data(type_id, rpm_filename, user_metadata=None):
"""
For the given RPM, analyzes its metadata to generate the appropriate unit
key and metadata fields, returning both to the caller.
:param type_id: The type of the unit that is being generated
:type type_id: str
:param rpm_filename: full path to the RPM to analyze
:type rpm_filename: str
:param user_metadata: user supplied metadata about the unit. This is optional.
:type user_metadata: dict
:return: tuple of unit key and unit metadata for the RPM
:rtype: tuple
"""
# Expected metadata fields:
# "vendor", "description", "buildhost", "license", "vendor", "requires", "provides",
# "relativepath", "filename"
#
# Expected unit key fields:
# "name", "epoch", "version", "release", "arch", "checksumtype", "checksum"
unit_key = dict()
metadata = dict()
# Read the RPM header attributes for use later
ts = rpm.TransactionSet()
ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES)
fd = os.open(rpm_filename, os.O_RDONLY)
try:
headers = ts.hdrFromFdno(fd)
os.close(fd)
except rpm.error:
# Raised if the headers cannot be read
os.close(fd)
raise
# -- Unit Key -----------------------
# Checksum
if user_metadata and user_metadata.get('checksum_type'):
user_checksum_type = user_metadata.get('checksum_type')
user_checksum_type = verification.sanitize_checksum_type(user_checksum_type)
unit_key['checksumtype'] = user_checksum_type
else:
unit_key['checksumtype'] = verification.TYPE_SHA256
unit_key['checksum'] = _calculate_checksum(unit_key['checksumtype'], rpm_filename)
# Name, Version, Release, Epoch
for k in ['name', 'version', 'release', 'epoch']:
unit_key[k] = headers[k]
# Epoch munging
if unit_key['epoch'] is None:
unit_key['epoch'] = str(0)
else:
unit_key['epoch'] = str(unit_key['epoch'])
# Arch
if headers['sourcepackage']:
if RPMTAG_NOSOURCE in headers.keys():
unit_key['arch'] = 'nosrc'
else:
unit_key['arch'] = 'src'
else:
unit_key['arch'] = headers['arch']
# -- Unit Metadata ------------------
# construct filename from metadata (BZ #1101168)
if headers[rpm.RPMTAG_SOURCEPACKAGE]:
if type_id != models.SRPM.TYPE:
raise PulpCodedValidationException(error_code=error_codes.RPM1002)
rpm_basefilename = "%s-%s-%s.src.rpm" % (headers['name'],
headers['version'],
headers['release'])
else:
if type_id != models.RPM.TYPE:
raise PulpCodedValidationException(error_code=error_codes.RPM1003)
rpm_basefilename = "%s-%s-%s.%s.rpm" % (headers['name'],
headers['version'],
headers['release'],
headers['arch'])
metadata['relativepath'] = rpm_basefilename
metadata['filename'] = rpm_basefilename
# This format is, and has always been, incorrect. As of the new yum importer, the
# plugin will generate these from the XML snippet because the API into RPM headers
# is atrocious. This is the end game for this functionality anyway, moving all of
# that metadata derivation into the plugin, so this is just a first step.
# I'm leaving these in and commented to show how not to do it.
# metadata['requires'] = [(r,) for r in headers['requires']]
# metadata['provides'] = [(p,) for p in headers['provides']]
metadata['buildhost'] = headers['buildhost']
metadata['license'] = headers['license']
metadata['vendor'] = headers['vendor']
metadata['description'] = headers['description']
return unit_key, metadata
def _migrate_rpmlike_units(unit_type):
"""
This function performs the migration on RPMs, DRPMs, and SRPMs. These all have the same schema
when it comes to checksumtype, so they can be treated the same way.
:param unit_type: The unit_type_id, as found in pulp_rpm.common.ids.
:type unit_type: basestring
"""
repos = connection.get_collection('repos')
repo_content_units = connection.get_collection('repo_content_units')
unit_collection = connection.get_collection('units_%s' % unit_type)
for unit in unit_collection.find():
try:
sanitized_type = verification.sanitize_checksum_type(
unit['checksumtype'])
if sanitized_type != unit['checksumtype']:
# Let's see if we can get away with changing its checksumtype to the sanitized
# value. If this works, we won't have to do anything else.
unit_collection.update(
{'_id': unit['_id']},
{'$set': {
'checksumtype': sanitized_type
}})
except errors.DuplicateKeyError:
# Looks like there is already an identical unit with the sanitized checksum type. This
# means we need to remove the current unit, but first we will need to change any
# references to this unit to point to the other.
conflicting_unit = unit_collection.find_one({
'name':
unit['name'],
'epoch':
unit['epoch'],
'version':
unit['version'],
'release':
unit['release'],
'arch':
unit['arch'],
'checksum':
unit['checksum'],
'checksumtype':
sanitized_type
})
for rcu in repo_content_units.find({
'unit_type_id': unit_type,
'unit_id': unit['_id']
}):
# Now we must either switch the rcu from pointing to unit to pointing to
# conflicting_unit, or delete the rcu if there is already one in the same repo.
try:
msg = _(
'Updating %(repo_id)s to contain %(type)s %(conflicting)s instead of '
'%(old_id)s.')
msg = msg % {
'repo_id': rcu['repo_id'],
'type': unit_type,
'conflicting': conflicting_unit['_id'],
'old_id': unit['_id']
}
_logger.debug(msg)
repo_content_units.update(
{'_id': rcu['_id']},
{'$set': {
'unit_id': conflicting_unit['_id']
}})
except errors.DuplicateKeyError:
# We will delete this RepoContentUnit since the sha1 RPM is already in the
# repository.
msg = _(
'Removing %(type)s %(old_id)s from repo %(repo_id)s since it conflicts '
'with %(conflicting)s.')
msg = msg % {
'repo_id': rcu['repo_id'],
'type': unit_type,
'conflicting': conflicting_unit['_id'],
'old_id': unit['_id']
}
_logger.debug(msg)
repo_content_units.remove({'_id': rcu['_id']})
# In this case, we now need to decrement the repository's "content_unit_counts"
# for this unit_type by one, since we removed a unit from a repository.
repos.update(
{'id': rcu['repo_id']},
{'$inc': {
'content_unit_counts.%s' % unit_type: -1
}})
# Now that we have removed or altered all references to the "sha" Unit, we need to
# remove it since it is a duplicate.
unit_collection.remove({'_id': unit['_id']})
def __init__(self, *args, **kwargs):
if "checksumtype" in kwargs:
kwargs["checksumtype"] = verification.sanitize_checksum_type(kwargs["checksumtype"])
super(NonMetadataPackage, self).__init__(*args, **kwargs)
def _handle_package(repo, type_id, unit_key, metadata, file_path, conduit, config):
"""
Handles the upload for an RPM or SRPM.
This inspects the package contents to determine field values. The unit_key
and metadata fields overwrite field values determined through package inspection.
:param repo: The repository to import the package into
:type repo: pulp.server.db.model.Repository
:param type_id: The type_id of the package being uploaded
:type type_id: str
:param unit_key: A dictionary of fields to overwrite introspected field values
:type unit_key: dict
:param metadata: A dictionary of fields to overwrite introspected field values, or None
:type metadata: dict or None
:param file_path: The path to the uploaded package
:type file_path: str
:param conduit: provides access to relevant Pulp functionality
:type conduit: pulp.plugins.conduits.upload.UploadConduit
:param config: plugin configuration for the repository
:type config: pulp.plugins.config.PluginCallConfiguration
:raises PulpCodedException PLP1005: if the checksum type from the user is not recognized
:raises PulpCodedException PLP1013: if the checksum value from the user does not validate
"""
try:
rpm_data = _extract_rpm_data(type_id, file_path)
except:
_LOGGER.exception('Error extracting RPM metadata for [%s]' % file_path)
raise
model_class = plugin_api.get_unit_model_by_id(type_id)
update_fields_inbound(model_class, unit_key or {})
update_fields_inbound(model_class, metadata or {})
# set checksum and checksumtype
if metadata:
checksumtype = metadata.pop('checksumtype', verification.TYPE_SHA256)
rpm_data['checksumtype'] = verification.sanitize_checksum_type(checksumtype)
if 'checksum' in metadata:
rpm_data['checksum'] = metadata.pop('checksum')
try:
with open(file_path) as dest_file:
verification.verify_checksum(dest_file, rpm_data['checksumtype'],
rpm_data['checksum'])
except verification.VerificationException:
raise PulpCodedException(error_code=platform_errors.PLP1013)
else:
rpm_data['checksum'] = _calculate_checksum(rpm_data['checksumtype'], file_path)
else:
rpm_data['checksumtype'] = verification.TYPE_SHA256
rpm_data['checksum'] = _calculate_checksum(rpm_data['checksumtype'], file_path)
# Update the RPM-extracted data with anything additional the user specified.
# Allow the user-specified values to override the extracted ones.
rpm_data.update(metadata or {})
rpm_data.update(unit_key or {})
# Validate the user specified data by instantiating the model
try:
unit = model_class(**rpm_data)
except TypeError:
raise ModelInstantiationError()
# Extract/adjust the repodata snippets
unit.repodata = rpm_parse.get_package_xml(file_path, sumtype=unit.checksumtype)
_update_provides_requires(unit)
_update_location(unit)
# check if the unit has duplicate nevra
purge.remove_unit_duplicate_nevra(unit, repo)
unit.set_storage_path(os.path.basename(file_path))
try:
unit.save_and_import_content(file_path)
except NotUniqueError:
unit = unit.__class__.objects.filter(**unit.unit_key).first()
repo_controller.associate_single_unit(repo, unit)
def parse_treefile(path):
"""
The treefile seems to be approximately in INI format, which can be read
by the standard library's ConfigParser.
:param path: full path to the treefile
:return: instance of Distribution model, and a list of dicts
describing the distribution's files
:rtype: (pulp_rpm.plugins.db.models.Distribution, list of dict)
"""
parser = ConfigParser.RawConfigParser()
# the default implementation of this method makes all option names lowercase,
# which we don't want. This is the suggested solution in the python.org docs.
parser.optionxform = str
with open(path) as open_file:
try:
parser.readfp(open_file)
except ConfigParser.ParsingError:
# wouldn't need this if ParsingError subclassed ValueError.
raise ValueError('could not parse treeinfo file')
# apparently the 'variant' is optional. for example, it does not appear
# in the RHEL 5.9 treeinfo file. This is how the previous importer
# handled that.
try:
variant = parser.get(SECTION_GENERAL, 'variant')
except ConfigParser.NoOptionError:
variant = None
try:
packagedir = parser.get(SECTION_GENERAL, KEY_PACKAGEDIR)
except ConfigParser.NoOptionError:
packagedir = None
try:
model = models.Distribution(parser.get(SECTION_GENERAL, 'family'),
variant,
parser.get(SECTION_GENERAL, 'version'),
parser.get(SECTION_GENERAL, 'arch'),
metadata={
KEY_PACKAGEDIR:
packagedir,
KEY_TIMESTAMP:
float(
parser.get(SECTION_GENERAL,
KEY_TIMESTAMP)),
})
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
raise ValueError(
'invalid treefile: could not find unit key components')
files = {}
# this section is likely to have all the files we care about listed with
# checksums. But, it might not. Other sections checked below will only add
# files to the "files" dict if they are not already present. For those cases,
# there will not be checksums available.
if parser.has_section(SECTION_CHECKSUMS):
for item in parser.items(SECTION_CHECKSUMS):
relativepath = item[0]
checksumtype, checksum = item[1].split(':')
checksumtype = verification.sanitize_checksum_type(checksumtype)
files[relativepath] = {
'relativepath': relativepath,
'checksum': checksum,
'checksumtype': checksumtype
}
for section_name in parser.sections():
if section_name.startswith(
'images-') or section_name == SECTION_STAGE2:
for item in parser.items(section_name):
if item[1] not in files:
relativepath = item[1]
files[relativepath] = {
'relativepath': relativepath,
'checksum': None,
'checksumtype': None,
}
return model, files.values()
def __init__(self, *args, **kwargs):
if 'checksumtype' in kwargs:
kwargs['checksumtype'] = verification.sanitize_checksum_type(
kwargs['checksumtype'])
super(NonMetadataPackage, self).__init__(*args, **kwargs)
def __init__(self, name, epoch, version, release, arch, checksumtype, checksum, metadata):
checksumtype = verification.sanitize_checksum_type(checksumtype)
Package.__init__(self, locals())
self.raw_xml = ''
def _handle_package(repo, type_id, unit_key, metadata, file_path, conduit, config):
"""
Handles the upload for an RPM or SRPM.
This inspects the package contents to determine field values. The unit_key
and metadata fields overwrite field values determined through package inspection.
:param repo: The repository to import the package into
:type repo: pulp.server.db.model.Repository
:param type_id: The type_id of the package being uploaded
:type type_id: str
:param unit_key: A dictionary of fields to overwrite introspected field values
:type unit_key: dict
:param metadata: A dictionary of fields to overwrite introspected field values, or None
:type metadata: dict or None
:param file_path: The path to the uploaded package
:type file_path: str
:param conduit: provides access to relevant Pulp functionality
:type conduit: pulp.plugins.conduits.upload.UploadConduit
:param config: plugin configuration for the repository
:type config: pulp.plugins.config.PluginCallConfiguration
:raises PulpCodedException PLP1005: if the checksum type from the user is not recognized
:raises PulpCodedException PLP1013: if the checksum value from the user does not validate
"""
try:
rpm_data = _extract_rpm_data(type_id, file_path)
except:
_LOGGER.exception('Error extracting RPM metadata for [%s]' % file_path)
raise
model_class = plugin_api.get_unit_model_by_id(type_id)
update_fields_inbound(model_class, unit_key or {})
update_fields_inbound(model_class, metadata or {})
# set checksum and checksumtype
if metadata:
checksumtype = metadata.pop('checksumtype', verification.TYPE_SHA256)
rpm_data['checksumtype'] = verification.sanitize_checksum_type(checksumtype)
if 'checksum' in metadata:
rpm_data['checksum'] = metadata.pop('checksum')
try:
with open(file_path) as dest_file:
verification.verify_checksum(dest_file, rpm_data['checksumtype'],
rpm_data['checksum'])
except verification.VerificationException:
raise PulpCodedException(error_code=platform_errors.PLP1013)
else:
rpm_data['checksum'] = _calculate_checksum(rpm_data['checksumtype'], file_path)
else:
rpm_data['checksumtype'] = verification.TYPE_SHA256
rpm_data['checksum'] = _calculate_checksum(rpm_data['checksumtype'], file_path)
# Update the RPM-extracted data with anything additional the user specified.
# Allow the user-specified values to override the extracted ones.
rpm_data.update(metadata or {})
rpm_data.update(unit_key or {})
# Validate the user specified data by instantiating the model
try:
unit = model_class(**rpm_data)
except TypeError:
raise ModelInstantiationError()
# Extract the repodata snippets
unit.repodata = rpm_parse.get_package_xml(file_path, sumtype=unit.checksumtype)
_update_provides_requires(unit)
# check if the unit has duplicate nevra
purge.remove_unit_duplicate_nevra(unit, repo)
unit.set_storage_path(os.path.basename(file_path))
unit.save_and_import_content(file_path)
repo_controller.associate_single_unit(repo, unit)
