def verify_download(missing_rpms, new_rpms, new_units, verify_options={}):
"""
Will verify that intended items have been downloaded.
Items not downloaded will be removed from passed in dicts
@param missing_rpms dict of rpms determined to be missing from repo prior to sync
@type missing_rpms {}
@param new_rpms dict of rpms determined to be new to repo prior to sync
@type new_rpms {}
@param new_units
@type new_units {key:pulp.server.content.plugins.model.Unit}
@return dict of rpms which have not been downloaded
@rtype {}
"""
not_synced = {}
for key in new_rpms.keys():
rpm = new_rpms[key]
rpm_path = os.path.join(rpm["pkgpath"], rpm["filename"])
if not util.verify_exists(rpm_path, rpm['checksum'], rpm['checksumtype'], rpm['size'], verify_options):
not_synced[key] = rpm
del new_rpms[key]
for key in missing_rpms.keys():
rpm = missing_rpms[key]
rpm_path = os.path.join(rpm["pkgpath"], rpm["filename"])
if not util.verify_exists(rpm_path, rpm['checksum'], rpm['checksumtype'], rpm['size'], verify_options):
not_synced[key] = rpm
del missing_rpms[key]
for key in not_synced:
del new_units[key]
return not_synced
def get_missing_distros_and_units(available_distro, existing_distro_units, verify_options={}):
"""
@param available_distro dict of available distributions
@type available_distro {}
@param existing_distro_units dict of existing Units
@type existing_distro_units {key:pulp.server.content.plugins.model.Unit}
@return a tuple of 2 dictionaries. First dict is of missing distros files, second dict is of missing units
@rtype ({}, {})
"""
missing_distro = {}
missing_units = {}
missing_distro_files = {}
for key in available_distro:
if key in existing_distro_units:
missing_distro_files[key] = []
for ksfile in existing_distro_units[key].metadata.get("files"):
distro_file_path = os.path.join(existing_distro_units[key].storage_path, ksfile["fileName"])
if not util.verify_exists(distro_file_path, ksfile['checksum'],
ksfile['checksumtype'], verify_options):
_LOG.debug("Missing an existing unit: %s. Will add to resync." % distro_file_path)
# Adjust storage path to match intended location
# Grinder will use this 'pkgpath' to write the file
ksfile["pkgpath"] = os.path.dirname(distro_file_path)
missing_distro_files[key].append(ksfile)
missing_distro[key] = available_distro[key]
missing_units[key] = existing_distro_units[key]
return missing_distro_files, missing_units
def get_missing_rpms_and_units(available_rpms, existing_units, verify_options={}):
"""
@param available_rpms dict of available rpms
@type available_rpms {}
@param existing_units dict of existing Units
@type existing_units {key:pulp.server.content.plugins.model.Unit}
@return a tuple of 2 dictionaries. First dict is of missing rpms, second dict is of missing units
@rtype ({}, {})
"""
missing_rpms = {}
missing_units = {}
for key in available_rpms:
if key in existing_units:
rpm_path = existing_units[key].storage_path
if not util.verify_exists(rpm_path, existing_units[key].unit_key.get('checksum'),
existing_units[key].unit_key.get('checksumtype'), verify_options):
_LOG.debug("Missing an existing unit: %s. Will add to resync." % (rpm_path))
missing_rpms[key] = available_rpms[key]
missing_units[key] = existing_units[key]
# Adjust storage path to match intended location
# Grinder will use this 'pkgpath' to write the file
missing_rpms[key]["pkgpath"] = os.path.dirname(missing_units[key].storage_path)
return missing_rpms, missing_units
def test_verify_options(self):
def side_effect(path):
# no-op to override file removal
pass
util.cleanup_file = mock.Mock()
util.cleanup_file = side_effect
test_pkg_path = os.path.join(self.data_dir, "test_repo", "pulp-test-package-0.2.1-1.fc11.x86_64.rpm")
verify_options = dict(checksum=True, size=True)
size = 2216
checksum = "4dbde07b4a8eab57e42ed0c9203083f1d61e0b13935d1a569193ed8efc9ecfd7"
checksum_type = "sha256"
exists = util.verify_exists(test_pkg_path, checksum, checksum_type, size, verify_options)
self.assertTrue(exists)
# check invalid size
size = 1232
t_exists = util.verify_exists(test_pkg_path, checksum, checksum_type, size, verify_options)
self.assertFalse(t_exists)
# check None size
size = None
exists = util.verify_exists(test_pkg_path, checksum, checksum_type, size, verify_options)
self.assertTrue(exists)
# check invalid checksum
checksum = "test_value"
exists = util.verify_exists(test_pkg_path, checksum, checksum_type, size, verify_options)
self.assertFalse(exists)
# skip size/checksum checks
verify_options = dict(checksum=False, size=False)
exists = util.verify_exists(test_pkg_path, checksum, checksum_type, size, verify_options)
self.assertTrue(exists)
# invalid path
test_pkg_fake_path = os.path.join(self.data_dir, "test_fake_repo", "pulp-test-package-0.2.1-1.fc11.x86_64.rpm")
exists = util.verify_exists(test_pkg_fake_path, checksum, checksum_type, size, verify_options)
self.assertFalse(exists)
