def copy_rpms_by_name(names, source_repo, dest_repo, import_conduit,
copy_deps):
"""
Copy RPMs from source repo to destination repo by name
:param names: iterable of RPM names
:type names: iterable of basestring
:param source_repo: The repository we are copying units from.
:type source_repo: pulp.server.db.model.Repository
:param dest_repo: The repository we are copying units to
:type dest_repo: pulp.server.db.model.Repository
:param import_conduit: import conduit passed to the Importer
:type import_conduit: pulp.plugins.conduits.unit_import.ImportUnitConduit
:return: set of pulp.plugins.model.Unit that were copied
:rtype: set
"""
name_q = mongoengine.Q(name__in=names)
type_q = mongoengine.Q(unit_type_id=ids.TYPE_ID_RPM)
units = repo_controller.find_repo_content_units(
source_repo,
units_q=name_q,
repo_content_unit_q=type_q,
unit_fields=models.RPM.unit_key_fields,
yield_content_unit=True)
return copy_rpms(units, source_repo, dest_repo, import_conduit, copy_deps)
def _do_import_modules(self, metadata):
"""
Actual logic of the import. This method will do a best effort per module;
if an individual module fails it will be recorded and the import will
continue. This method will only raise an exception in an extreme case
where it cannot react and continue.
"""
downloader = self._create_downloader()
self.downloader = downloader
# Ease module lookup
metadata_modules_by_key = dict([(m.unit_key_str, m)
for m in metadata.modules])
# Collect information about the repository's modules before changing it
existing_module_ids_by_key = {}
modules = repo_controller.find_repo_content_units(
self.repo.repo_obj,
unit_fields=Module.unit_key_fields,
yield_content_unit=True)
for module in modules:
existing_module_ids_by_key[module.unit_key_str] = module.id
new_unit_keys = self._resolve_new_units(
existing_module_ids_by_key.keys(), metadata_modules_by_key.keys())
# Once we know how many things need to be processed, we can update the progress report
self.progress_report.modules_total_count = len(new_unit_keys)
self.progress_report.modules_finished_count = 0
self.progress_report.modules_error_count = 0
self.progress_report.update_progress()
# Add new units
for key in new_unit_keys:
if self._canceled:
break
module = metadata_modules_by_key[key]
try:
self._add_new_module(downloader, module)
self.progress_report.modules_finished_count += 1
except Exception as e:
self.progress_report.add_failed_module(module, e,
sys.exc_info()[2])
self.progress_report.update_progress()
# Remove missing units if the configuration indicates to do so
if self._should_remove_missing():
remove_unit_keys = self._resolve_remove_units(
existing_module_ids_by_key.keys(),
metadata_modules_by_key.keys())
doomed_ids = [
existing_module_ids_by_key[key] for key in remove_unit_keys
]
doomed_module_iterator = Module.objects.in_bulk(
doomed_ids).itervalues()
repo_controller.disassociate_units(self.repo,
doomed_module_iterator)
self.downloader = None
def publish_repo(self, repo, publish_conduit, config):
"""
Publish the repository.
:param repo: metadata describing the repo
:type repo: pulp.plugins.model.Repository
:param publish_conduit: The conduit for publishing a repo
:type publish_conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit
:param config: plugin configuration
:type config: pulp.plugins.config.PluginConfiguration
:param config_conduit: Configuration Conduit;
:type config_conduit: pulp.plugins.conduits.repo_validate.RepoConfigConduit
:return: report describing the publish operation
:rtype: pulp.plugins.model.PublishReport
"""
progress_report = FilePublishProgressReport(publish_conduit)
_logger.info(_('Beginning publish for repository <%(repo)s>') % {'repo': repo.id})
try:
progress_report.state = progress_report.STATE_IN_PROGRESS
repo_model = repo.repo_obj
units = repo_controller.find_repo_content_units(repo_model, yield_content_unit=True)
# Set up an empty build_dir
working_dir = common_utils.get_working_directory()
build_dir = os.path.join(working_dir, BUILD_DIRNAME)
os.makedirs(build_dir)
self.initialize_metadata(build_dir)
try:
# process each unit
for unit in units:
links_to_create = self.get_paths_for_unit(unit)
self._symlink_unit(build_dir, unit, links_to_create)
self.publish_metadata_for_unit(unit)
finally:
# Finalize the processing
self.finalize_metadata()
# Let's unpublish, and then republish
self.unpublish_repo(repo, config)
hosting_locations = self.get_hosting_locations(repo_model, config)
for location in hosting_locations:
shutil.copytree(build_dir, location, symlinks=True)
self.post_repo_publish(repo_model, config)
# Report that we are done
progress_report.state = progress_report.STATE_COMPLETE
return progress_report.build_final_report()
except Exception, e:
_logger.exception(e)
# Something failed. Let's put an error message on the report
progress_report.error_message = str(e)
progress_report.traceback = traceback.format_exc()
progress_report.state = progress_report.STATE_FAILED
report = progress_report.build_final_report()
return report
def get_existing_units(search_dicts, unit_class, repo):
"""
Get units from the given repository that match the search terms. The unit instances will only
have their unit key fields populated.
:param search_dicts: iterable of dictionaries that should be used to search units
:type search_dicts: iterable
:param unit_class: subclass representing the type of unit to search for
:type unit_class: pulp_rpm.plugins.db.models.Package
:param repo: repository to search in
:type repo: pulp.server.db.model.Repository
:return: generator of unit_class instances with only their unit key fields populated
:rtype: generator
"""
unit_fields = unit_class.unit_key_fields
for segment in paginate(search_dicts):
unit_filters = {'$or': list(segment)}
units_q = mongoengine.Q(__raw__=unit_filters)
association_q = mongoengine.Q(unit_type_id=unit_class._content_type_id.default)
for result in repo_controller.find_repo_content_units(repo, units_q=units_q,
repo_content_unit_q=association_q,
unit_fields=unit_fields,
yield_content_unit=True):
yield result
def get_existing_units(search_dicts, unit_class, repo):
"""
Get units from the given repository that match the search terms. The unit instances will only
have their unit key fields populated.
:param search_dicts: iterable of dictionaries that should be used to search units
:type search_dicts: iterable
:param unit_class: subclass representing the type of unit to search for
:type unit_class: pulp_rpm.plugins.db.models.Package
:param repo: repository to search in
:type repo: pulp.server.db.model.Repository
:return: generator of unit_class instances with only their unit key fields populated
:rtype: generator
"""
unit_fields = unit_class.unit_key_fields
for segment in paginate(search_dicts):
unit_filters = {'$or': list(segment)}
units_q = mongoengine.Q(__raw__=unit_filters)
association_q = mongoengine.Q(unit_type_id=unit_class._content_type_id.default)
for result in repo_controller.find_repo_content_units(repo, units_q=units_q,
repo_content_unit_q=association_q,
unit_fields=unit_fields,
yield_content_unit=True):
yield result
def test_limit(self, mock_get_model, mock_demo_objects, mock_rcu_objects):
"""
Test that limits are applied properly to the results
"""
repo = MagicMock(repo_id='foo')
rcu_list = []
unit_list = []
for i in range(10):
unit_id = 'bar_%i' % i
unit_key = 'key_%i' % i
rcu = model.RepositoryContentUnit(repo_id='foo',
unit_type_id='demo_model',
unit_id=unit_id)
rcu_list.append(rcu)
unit_list.append(DemoModel(id=unit_id, key_field=unit_key))
mock_rcu_objects.return_value = rcu_list
mock_get_model.return_value = DemoModel
mock_demo_objects.return_value = unit_list
result = list(repo_controller.find_repo_content_units(repo, limit=5))
self.assertEquals(5, len(result))
self.assertEquals(result[0].unit_id, 'bar_0')
self.assertEquals(result[4].unit_id, 'bar_4')
def test_repo_content_units_query(self, mock_rcu_objects):
"""
Test the query parameters for the RepositoryContentUnit
"""
repo = MagicMock(repo_id='foo')
rcu_filter = mongoengine.Q(unit_type_id='demo_model')
list(repo_controller.find_repo_content_units(repo, repo_content_unit_q=rcu_filter))
self.assertEquals(mock_rcu_objects.call_args[1]['repo_id'], 'foo')
self.assertEquals(mock_rcu_objects.call_args[1]['q_obj'], rcu_filter)
def _filter_missing_isos(self, manifest, download_deferred):
"""
Use the sync_conduit and the manifest to determine which ISOs are at the feed_url
that are not in our local store, as well as which ISOs are in our local store that are not
available at the feed_url.
:param manifest: An ISOManifest describing the ISOs that are available at the
feed_url that we are synchronizing with
:type manifest: pulp_rpm.plugins.db.models.ISOManifest
:param download_deferred: indicates downloading is deferred (or not).
:type download_deferred: bool
:return: A 3-tuple. The first element of the tuple is a list of ISOs that we should
retrieve from the feed_url. The second element of the tuple is a list of
Units that are available locally already, but are not currently associated
with the repository. The third element of the tuple is a list of Units that
represent the ISOs that we have in our local repo that were not found in
the remote repo.
:rtype: tuple
"""
# A list of all the ISOs we have in Pulp
existing_units = models.ISO.objects()
existing_units_by_key = dict([(unit.unit_key_str, unit)
for unit in existing_units if not download_deferred and
os.path.isfile(unit.storage_path)])
existing_units.rewind()
existing_unit_keys = set([unit.unit_key_str
for unit in existing_units if not download_deferred and
os.path.isfile(unit.storage_path)])
# A list of units currently associated with the repository
existing_repo_units = repo_controller.find_repo_content_units(
self.sync_conduit.repo, yield_content_unit=True)
existing_repo_units = list(existing_repo_units)
existing_repo_units_by_key = dict([(unit.unit_key_str, unit)
for unit in existing_repo_units])
existing_repo_unit_keys = set([unit.unit_key_str
for unit in existing_repo_units])
# A list of the ISOs in the remote repository
available_isos_by_key = dict([(iso.unit_key_str, iso) for iso in manifest])
available_iso_keys = set([iso.unit_key_str for iso in manifest])
# Content that is available locally and just needs to be associated with the repository
local_available_iso_keys = set([iso for iso in available_iso_keys
if iso in existing_unit_keys])
local_available_iso_keys = local_available_iso_keys - existing_repo_unit_keys
local_available_units = [existing_units_by_key[k] for k in local_available_iso_keys]
# Content that is missing locally and must be downloaded
local_missing_iso_keys = list(available_iso_keys - existing_unit_keys)
local_missing_isos = [available_isos_by_key[k] for k in local_missing_iso_keys]
# Content that is missing from the remote repository that is present locally
remote_missing_unit_keys = list(existing_repo_unit_keys - available_iso_keys)
remote_missing_units = [existing_repo_units_by_key[k] for k in remote_missing_unit_keys]
return local_missing_isos, local_available_units, remote_missing_units
def _filter_missing_isos(self, manifest, download_deferred):
"""
Use the sync_conduit and the manifest to determine which ISOs are at the feed_url
that are not in our local store, as well as which ISOs are in our local store that are not
available at the feed_url.
:param manifest: An ISOManifest describing the ISOs that are available at the
feed_url that we are synchronizing with
:type manifest: pulp_rpm.plugins.db.models.ISOManifest
:param download_deferred: indicates downloading is deferred (or not).
:type download_deferred: bool
:return: A 3-tuple. The first element of the tuple is a list of ISOs that we should
retrieve from the feed_url. The second element of the tuple is a list of
Units that are available locally already, but are not currently associated
with the repository. The third element of the tuple is a list of Units that
represent the ISOs that we have in our local repo that were not found in
the remote repo.
:rtype: tuple
"""
# A list of all the ISOs we have in Pulp
existing_units = models.ISO.objects()
existing_units_by_key = dict([(unit.unit_key_str, unit)
for unit in existing_units if not download_deferred and
os.path.isfile(unit.storage_path)])
existing_units.rewind()
existing_unit_keys = set([unit.unit_key_str
for unit in existing_units if not download_deferred and
os.path.isfile(unit.storage_path)])
# A list of units currently associated with the repository
existing_repo_units = repo_controller.find_repo_content_units(
self.sync_conduit.repo, yield_content_unit=True)
existing_repo_units = list(existing_repo_units)
existing_repo_units_by_key = dict([(unit.unit_key_str, unit)
for unit in existing_repo_units])
existing_repo_unit_keys = set([unit.unit_key_str
for unit in existing_repo_units])
# A list of the ISOs in the remote repository
available_isos_by_key = dict([(iso.unit_key_str, iso) for iso in manifest])
available_iso_keys = set([iso.unit_key_str for iso in manifest])
# Content that is available locally and just needs to be associated with the repository
local_available_iso_keys = set([iso for iso in available_iso_keys
if iso in existing_unit_keys])
local_available_iso_keys = local_available_iso_keys - existing_repo_unit_keys
local_available_units = [existing_units_by_key[k] for k in local_available_iso_keys]
# Content that is missing locally and must be downloaded
local_missing_iso_keys = list(available_iso_keys - existing_unit_keys)
local_missing_isos = [available_isos_by_key[k] for k in local_missing_iso_keys]
# Content that is missing from the remote repository that is present locally
remote_missing_unit_keys = list(existing_repo_unit_keys - available_iso_keys)
remote_missing_units = [existing_repo_units_by_key[k] for k in remote_missing_unit_keys]
return local_missing_isos, local_available_units, remote_missing_units
def _retrieve_repo_modules(self):
"""
Retrieves all modules in the repository.
:return: list of modules in the repository; empty list if there are none
:rtype: list of pulp_puppet.plugins.db.models.Module objects
"""
modules_generator = find_repo_content_units(self.repo, yield_content_unit=True)
modules = list(modules_generator)
return modules
def _retrieve_repo_modules(self):
"""
Retrieves all modules in the repository.
:return: list of modules in the repository; empty list if there are none
:rtype: list of pulp_puppet.plugins.db.models.Module objects
"""
modules_generator = find_repo_content_units(self.repo,
yield_content_unit=True)
modules = list(modules_generator)
return modules
def publish_repo(self, repo, publish_conduit, config):
"""
Publish the repository by "installing" each puppet module into the given
destination directory. This effectively means extracting each module's
tarball in that directory.
:param repo: plugin repository object
:type repo: pulp.plugins.model.Repository
:param publish_conduit: provides access to relevant Pulp functionality
:type publish_conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit
:param config: plugin configuration
:type config: pulp.plugins.config.PluginConfiguration
:return: report describing the publish run
:rtype: pulp.plugins.model.PublishReport
"""
# get dir from config
destination = config.get(constants.CONFIG_INSTALL_PATH)
subdir = config.get(constants.CONFIG_SUBDIR)
if not destination:
return publish_conduit.build_failure_report(
_('install path not provided'), self.detail_report.report)
if subdir:
destination = os.path.join(destination, subdir)
units = list(
repo_controller.find_repo_content_units(repo.repo_obj,
yield_content_unit=True))
duplicate_units = self._find_duplicate_names(units)
if duplicate_units:
for unit in duplicate_units:
self.detail_report.error(
unit.unit_key,
'another unit in this repo also has this name')
return publish_conduit.build_failure_report(
_('duplicate unit names'), self.detail_report.report)
# check for unsafe paths in tarballs, and fail early if problems are found
self._check_for_unsafe_archive_paths(units, destination)
if self.detail_report.has_errors:
return publish_conduit.build_failure_report(
'failed', self.detail_report.report)
# ensure the destination directory exists
try:
mkdir(destination)
temporarydestination = self._create_temporary_destination_directory(
destination)
except OSError, e:
return publish_conduit.build_failure_report(
_('failed to create destination directory: %s') % str(e),
self.detail_report.report)
def _do_import_modules(self, metadata):
"""
Actual logic of the import. This method will do a best effort per module;
if an individual module fails it will be recorded and the import will
continue. This method will only raise an exception in an extreme case
where it cannot react and continue.
"""
downloader = self._create_downloader()
self.downloader = downloader
# Ease module lookup
metadata_modules_by_key = dict([(m.unit_key_as_named_tuple, m) for m in metadata.modules])
# Collect information about the repository's modules before changing it
existing_module_ids_by_key = {}
modules = repo_controller.find_repo_content_units(
self.repo.repo_obj, unit_fields=Module.unit_key_fields, yield_content_unit=True)
for module in modules:
existing_module_ids_by_key[module.unit_key_as_named_tuple] = module.id
new_unit_keys = self._resolve_new_units(existing_module_ids_by_key.keys(),
metadata_modules_by_key.keys())
# Once we know how many things need to be processed, we can update the progress report
self.progress_report.modules_total_count = len(new_unit_keys)
self.progress_report.modules_finished_count = 0
self.progress_report.modules_error_count = 0
self.progress_report.update_progress()
# Add new units
for key in new_unit_keys:
if self._canceled:
break
module = metadata_modules_by_key[key]
try:
self._add_new_module(downloader, module)
self.progress_report.modules_finished_count += 1
except Exception as e:
self.progress_report.add_failed_module(module, e, sys.exc_info()[2])
self.progress_report.update_progress()
# Remove missing units if the configuration indicates to do so
if self._should_remove_missing():
remove_unit_keys = self._resolve_remove_units(existing_module_ids_by_key.keys(),
metadata_modules_by_key.keys())
doomed_ids = [existing_module_ids_by_key[key] for key in remove_unit_keys]
doomed_module_iterator = Module.objects.in_bulk(doomed_ids).itervalues()
repo_controller.disassociate_units(self.repo.repo_obj, doomed_module_iterator)
self.downloader = None
def get_iterator(self):
"""
This method returns a generator to loop over items.
The items created by this generator will be iterated over by the process_main method.
:return: a list or other iterable
:rtype: iterator of pulp_docker.plugins.db.models.DockerImage
"""
return repo_controller.find_repo_content_units(
self.get_repo(),
repo_content_unit_q=Q(unit_type_id=constants.IMAGE_TYPE_ID),
yield_content_unit=True)
def install_units(self, consumer, units, options, config, conduit):
"""
Inspect the options, and if constants.WHOLE_REPO_ID has a non-False
value, replace the list of units with a list of all units in the given
repository. Omits version numbers, which allows the install tool to
automatically choose the most recent version of each.
:param consumer: A consumer.
:type consumer: pulp.plugins.model.Consumer
:param units: A list of content units to be installed.
:type units: list of: { type_id:<str>, unit_key:<dict> }
:param options: Install options; based on unit type.
:type options: dict
:param config: plugin configuration
:type config: pulp.plugins.config.PluginCallConfiguration
:param conduit: provides access to relevant Pulp functionality
:type conduit: pulp.plugins.conduits.profiler.ProfilerConduit
:return: The translated units
:rtype: list of: {'type_id': <str>, unit_key: {'author': <author>, 'name': <name>}
"""
repo_id = options.get(constants.REPO_ID_OPTION)
self._inject_forge_settings(options)
if options.get(constants.WHOLE_REPO_OPTION) and repo_id:
msg = _('installing whole repo %(repo_id)s on %(consumer_id)s')
msg_dict = {'repo_id': repo_id, 'consumer_id': consumer.id}
_LOGGER.debug(msg, msg_dict)
repo = Repository.objects.get(repo_id=repo_id)
units = find_repo_content_units(repo, yield_content_unit=True)
unit_key_dict = {}
for unit in units:
fullname = '%s/%s' % (unit.author, unit.name)
unit_key_dict[fullname] = {
'unit_key': {
'author': unit.author,
'name': unit.name
},
'type_id': constants.TYPE_PUPPET_MODULE
}
return unit_key_dict.values()
else:
return units
def _get_units(self):
"""
Get the collection of units to be published.
The collection contains only the newest unit for each branch.
:return: An iterable of units to publish.
:rtype: iterable
"""
units = {}
query = Q(unit_type_id=constants.OSTREE_TYPE_ID)
associations = find_repo_content_units(
self.get_repo(),
repo_content_unit_q=query)
for unit in sorted([a.unit for a in associations], key=lambda u: u.created):
units[unit.branch] = unit
return units.values()
def install_units(self, consumer, units, options, config, conduit):
"""
Inspect the options, and if constants.WHOLE_REPO_ID has a non-False
value, replace the list of units with a list of all units in the given
repository. Omits version numbers, which allows the install tool to
automatically choose the most recent version of each.
:param consumer: A consumer.
:type consumer: pulp.plugins.model.Consumer
:param units: A list of content units to be installed.
:type units: list of: { type_id:<str>, unit_key:<dict> }
:param options: Install options; based on unit type.
:type options: dict
:param config: plugin configuration
:type config: pulp.plugins.config.PluginCallConfiguration
:param conduit: provides access to relevant Pulp functionality
:type conduit: pulp.plugins.conduits.profiler.ProfilerConduit
:return: The translated units
:rtype: list of: {'type_id': <str>, unit_key: {'author': <author>, 'name': <name>}
"""
repo_id = options.get(constants.REPO_ID_OPTION)
self._inject_forge_settings(options)
if options.get(constants.WHOLE_REPO_OPTION) and repo_id:
msg = _('installing whole repo %(repo_id)s on %(consumer_id)s')
msg_dict = {'repo_id': repo_id, 'consumer_id': consumer.id}
_LOGGER.debug(msg, msg_dict)
repo = Repository.objects.get(repo_id=repo_id)
units = find_repo_content_units(repo, yield_content_unit=True)
unit_key_dict = {}
for unit in units:
fullname = '%s/%s' % (unit.author, unit.name)
unit_key_dict[fullname] = {
'unit_key': {'author': unit.author, 'name': unit.name},
'type_id': constants.TYPE_PUPPET_MODULE
}
return unit_key_dict.values()
else:
return units
def publish_repo(self, repo, publish_conduit, config):
"""
Publish the repository by "installing" each puppet module into the given
destination directory. This effectively means extracting each module's
tarball in that directory.
:param repo: plugin repository object
:type repo: pulp.plugins.model.Repository
:param publish_conduit: provides access to relevant Pulp functionality
:type publish_conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit
:param config: plugin configuration
:type config: pulp.plugins.config.PluginConfiguration
:return: report describing the publish run
:rtype: pulp.plugins.model.PublishReport
"""
# get dir from config
destination = config.get(constants.CONFIG_INSTALL_PATH)
subdir = config.get(constants.CONFIG_SUBDIR)
if not destination:
return publish_conduit.build_failure_report(_('install path not provided'),
self.detail_report.report)
if subdir:
destination = os.path.join(destination, subdir)
units = list(repo_controller.find_repo_content_units(repo.repo_obj,
yield_content_unit=True))
duplicate_units = self._find_duplicate_names(units)
if duplicate_units:
for unit in duplicate_units:
self.detail_report.error(unit.unit_key,
'another unit in this repo also has this name')
return publish_conduit.build_failure_report(_('duplicate unit names'),
self.detail_report.report)
# check for unsafe paths in tarballs, and fail early if problems are found
self._check_for_unsafe_archive_paths(units, destination)
if self.detail_report.has_errors:
return publish_conduit.build_failure_report('failed', self.detail_report.report)
# ensure the destination directory exists
try:
mkdir(destination)
temporarydestination = self._create_temporary_destination_directory(destination)
except OSError, e:
return publish_conduit.build_failure_report(
_('failed to create destination directory: %s') % str(e), self.detail_report.report)
def _build_source_with_provides(self):
"""
Get a list of all available packages with their "Provides" info.
Note that the 'provides' metadata will be flattened via _trim_provides().
:return: list of pulp_rpm.plugins.db.models.RPM
:rtype: list
"""
fields = list(models.RPM.unit_key_fields)
fields.extend(['provides', 'version_sort_index', 'release_sort_index'])
units = repo_controller.find_repo_content_units(
repository=self.source_repo,
repo_content_unit_q=mongoengine.Q(unit_type_id=ids.TYPE_ID_RPM),
unit_fields=fields, yield_content_unit=True
)
return [self._trim_provides(unit) for unit in units]
def _build_source_with_provides(self):
"""
Get a list of all available packages with their "Provides" info.
Note that the 'provides' metadata will be flattened via _trim_provides().
:return: list of pulp_rpm.plugins.db.models.RPM
:rtype: list
"""
fields = list(models.RPM.unit_key_fields)
fields.extend(['provides', 'version_sort_index', 'release_sort_index'])
units = repo_controller.find_repo_content_units(
repository=self.source_repo,
repo_content_unit_q=mongoengine.Q(unit_type_id=ids.TYPE_ID_RPM),
unit_fields=fields,
yield_content_unit=True)
return [self._trim_provides(unit) for unit in units]
def _unit_generator(self, fields):
"""
Yields RPM content units in the current source repo with the specified fields
Note that the 'provides' metadata will be flattened via _trim_provides().
:param fields: list of fields to include in the yielded units
:type fields: list
:return: iterable of pulp_rpm.plugins.db.models.RPM
:rtype: generator
"""
# integration point with repo_controller, ideal for mocking in testing
return repo_controller.find_repo_content_units(
repository=self.source_repo,
repo_content_unit_q=mongoengine.Q(unit_type_id=ids.TYPE_ID_RPM),
unit_fields=fields, yield_content_unit=True
)
def _unit_generator(self, fields):
"""
Yields RPM content units in the current source repo with the specified fields
Note that the 'provides' metadata will be flattened via _trim_provides().
:param fields: list of fields to include in the yielded units
:type fields: list
:return: iterable of pulp_rpm.plugins.db.models.RPM
:rtype: generator
"""
# integration point with repo_controller, ideal for mocking in testing
return repo_controller.find_repo_content_units(
repository=self.source_repo,
repo_content_unit_q=mongoengine.Q(unit_type_id=ids.TYPE_ID_RPM),
unit_fields=fields, yield_content_unit=True
)
def _associate_unit(self, repo, unit):
"""
Associate an iso unit with a repository but first check if there's already any with the same
name and if so, remove them.
:param repo: An ISO repository that is being synced
:type repo: pulp.server.db.model.Repository
:param unit: An ISO unit to associate with repo
:type unit: pulp_rpm.plugins.db.models.ISO
"""
if not self.repo_units:
# store the existing repo units to prevent querying mongo multiple times
self.repo_units = repo_controller.find_repo_content_units(repo, yield_content_unit=True)
units_to_remove = [iso for iso in self.repo_units if iso['name'] == unit['name']]
repo_controller.disassociate_units(repo, units_to_remove)
repo_controller.associate_single_unit(repo, unit)
def _units_from_criteria(source_repo, criteria):
"""
Given a criteria, return an iterator of units
:param source_repo: repository to look for units in
:type source_repo: pulp.server.db.model.Repository
:param criteria: criteria object to use for the search parameters
:type criteria: pulp.server.db.model.criteria.UnitAssociationCriteria
:return: generator of pulp.server.db.model.ContentUnit instances
:rtype: generator
"""
association_q = mongoengine.Q(__raw__=criteria.association_spec)
if criteria.type_ids:
association_q &= mongoengine.Q(unit_type_id__in=criteria.type_ids)
unit_type_ids = criteria.type_ids
else:
# don't need to limit the association_q by content type here
# since filtering for every content_type seen in the repo is
# achieved by limiting the query to the repo
unit_type_ids = repo_controller.get_repo_unit_type_ids(
source_repo.repo_id)
# base unit_q, works as-is for non-mongoengine unit types
unit_q = mongoengine.Q(__raw__=criteria.unit_spec)
# for mongoengine unit types, use the unit model's serializer to translate the unit spec
# for each possible content unit type as determined by the search criteria
for unit_type_id in unit_type_ids:
serializer = units_controller.get_model_serializer_for_type(
unit_type_id)
if serializer:
unit_spec_t = serializer.translate_filters(
serializer.model, criteria.unit_spec)
# after translating the fields, this spec is only good for this unit type
unit_spec_t['_content_type_id'] = unit_type_id
unit_q |= mongoengine.Q(__raw__=unit_spec_t)
return repo_controller.find_repo_content_units(
repository=source_repo,
repo_content_unit_q=association_q,
units_q=unit_q,
unit_fields=criteria['unit_fields'],
yield_content_unit=True)
def copy_units(import_conduit, units):
"""
Copies puppet modules from one repo into another. There is nothing that
the importer needs to do; it maintains no state in the working directory
so the process is to simply tell Pulp to import each unit specified.
"""
# Determine which units are being copied
if units is None:
repo = Repository.objects.get(repo_id=import_conduit.source_repo_id)
units = find_repo_content_units(repo, yield_content_unit=True)
# Associate to the new repository
units_to_return = []
for u in units:
units_to_return.append(u)
import_conduit.associate_unit(u)
return units_to_return
def _units_from_criteria(source_repo, criteria):
"""
Given a criteria, return an iterator of units
:param source_repo: repository to look for units in
:type source_repo: pulp.server.db.model.Repository
:param criteria: criteria object to use for the search parameters
:type criteria: pulp.server.db.model.criteria.UnitAssociationCriteria
:return: generator of pulp.server.db.model.ContentUnit instances
:rtype: generator
"""
association_q = mongoengine.Q(__raw__=criteria.association_spec)
if criteria.type_ids:
association_q &= mongoengine.Q(unit_type_id__in=criteria.type_ids)
unit_q = mongoengine.Q(__raw__=criteria.unit_spec)
return repo_controller.find_repo_content_units(
repository=source_repo, repo_content_unit_q=association_q, units_q=unit_q, yield_content_unit=True
)
def _units_from_criteria(source_repo, criteria):
"""
Given a criteria, return an iterator of units
:param source_repo: repository to look for units in
:type source_repo: pulp.server.db.model.Repository
:param criteria: criteria object to use for the search parameters
:type criteria: pulp.server.db.model.criteria.UnitAssociationCriteria
:return: generator of pulp.server.db.model.ContentUnit instances
:rtype: generator
"""
association_q = mongoengine.Q(__raw__=criteria.association_spec)
if criteria.type_ids:
association_q &= mongoengine.Q(unit_type_id__in=criteria.type_ids)
unit_type_ids = criteria.type_ids
else:
# don't need to limit the association_q by content type here
# since filtering for every content_type seen in the repo is
# achieved by limiting the query to the repo
unit_type_ids = repo_controller.get_repo_unit_type_ids(source_repo.repo_id)
# base unit_q, works as-is for non-mongoengine unit types
unit_q = mongoengine.Q(__raw__=criteria.unit_spec)
# for mongoengine unit types, use the unit model's serializer to translate the unit spec
# for each possible content unit type as determined by the search criteria
for unit_type_id in unit_type_ids:
serializer = units_controller.get_model_serializer_for_type(unit_type_id)
if serializer:
unit_spec_t = serializer.translate_filters(serializer.model, criteria.unit_spec)
# after translating the fields, this spec is only good for this unit type
unit_spec_t['_content_type_id'] = unit_type_id
unit_q |= mongoengine.Q(__raw__=unit_spec_t)
return repo_controller.find_repo_content_units(
repository=source_repo,
repo_content_unit_q=association_q,
units_q=unit_q,
unit_fields=criteria['unit_fields'],
yield_content_unit=True)
def _units_from_criteria(source_repo, criteria):
"""
Given a criteria, return an iterator of units
:param source_repo: repository to look for units in
:type source_repo: pulp.server.db.model.Repository
:param criteria: criteria object to use for the search parameters
:type criteria: pulp.server.db.model.criteria.UnitAssociationCriteria
:return: generator of pulp.server.db.model.ContentUnit instances
:rtype: generator
"""
association_q = mongoengine.Q(__raw__=criteria.association_spec)
if criteria.type_ids:
association_q &= mongoengine.Q(unit_type_id__in=criteria.type_ids)
unit_q = mongoengine.Q(__raw__=criteria.unit_spec)
return repo_controller.find_repo_content_units(
repository=source_repo,
repo_content_unit_q=association_q,
units_q=unit_q,
yield_content_unit=True)
def remove_unit_duplicate_nevra(unit, repo):
"""
Removes units from the repo that have same NEVRA, ignoring the checksum
and checksum type.
:param unit: The unit whose NEVRA should be removed
:type unit: ContentUnit
:param repo: the repo from which units will be unassociated
:type repo: pulp.server.db.model.Repository
"""
nevra_filters = unit.unit_key.copy()
del nevra_filters['checksum']
del nevra_filters['checksumtype']
Q_filters = [Q(**{key: value}) for key, value in nevra_filters.iteritems()]
Q_nevra_filter = reduce(operator.and_, Q_filters)
Q_type_filter = Q(unit_type_id=unit._content_type_id)
unit_iterator = repo_controller.find_repo_content_units(repo,
repo_content_unit_q=Q_type_filter,
units_q=Q_nevra_filter,
yield_content_unit=True)
repo_controller.disassociate_units(repo, unit_iterator)
def _associate_unit(self, repo, unit):
"""
Associate an iso unit with a repository but first check if there's already any with the same
name and if so, remove them.
:param repo: An ISO repository that is being synced
:type repo: pulp.server.db.model.Repository
:param unit: An ISO unit to associate with repo
:type unit: pulp_rpm.plugins.db.models.ISO
"""
if not self.repo_units:
# store the existing repo units to prevent querying mongo multiple times
self.repo_units = list(
repo_controller.find_repo_content_units(
repo, yield_content_unit=True))
units_to_remove = [
iso for iso in self.repo_units if iso['name'] == unit['name']
]
repo_controller.disassociate_units(repo, units_to_remove)
repo_controller.associate_single_unit(repo, unit)
def test_content_units_query_test(self, mock_get_model, mock_demo_objects, mock_rcu_objects):
"""
Test the query parameters for the ContentUnit
"""
repo = MagicMock(repo_id='foo')
test_unit = DemoModel(id='bar', key_field='baz')
test_rcu = model.RepositoryContentUnit(repo_id='foo',
unit_type_id='demo_model',
unit_id='bar')
mock_rcu_objects.return_value = [test_rcu]
u_filter = mongoengine.Q(key_field='baz')
u_fields = ['key_field']
mock_get_model.return_value = DemoModel
mock_demo_objects.return_value.only.return_value = [test_unit]
result = list(repo_controller.find_repo_content_units(repo, units_q=u_filter,
unit_fields=u_fields))
mock_demo_objects.return_value.only.assert_called_once_with(['key_field'])
# validate that the repo content unit was returned and that the unit is attached
self.assertEquals(result, [test_rcu])
self.assertEquals(result[0].unit, test_unit)
def copy_rpms_by_name(names, source_repo, dest_repo, import_conduit, copy_deps):
"""
Copy RPMs from source repo to destination repo by name
:param names: iterable of RPM names
:type names: iterable of basestring
:param source_repo: The repository we are copying units from.
:type source_repo: pulp.server.db.model.Repository
:param dest_repo: The repository we are copying units to
:type dest_repo: pulp.server.db.model.Repository
:param import_conduit: import conduit passed to the Importer
:type import_conduit: pulp.plugins.conduits.unit_import.ImportUnitConduit
:return: set of pulp.plugins.model.Unit that were copied
:rtype: set
"""
name_q = mongoengine.Q(name__in=names)
type_q = mongoengine.Q(unit_type_id=ids.TYPE_ID_RPM)
units = repo_controller.find_repo_content_units(source_repo, units_q=name_q,
repo_content_unit_q=type_q,
unit_fields=models.RPM.unit_key_fields,
yield_content_unit=True)
return copy_rpms(units, source_repo, dest_repo, import_conduit, copy_deps)
def _run(self, tmp_dir):
"""
Look for a distribution in the target repo and sync it if found
:param tmp_dir: The absolute path to the temporary directory
:type tmp_dir: str
"""
treeinfo_path = self.get_treefile(tmp_dir)
if not treeinfo_path:
_logger.debug(_('No treeinfo found'))
return
try:
unit, files = self.parse_treeinfo_file(treeinfo_path)
except ValueError:
_logger.error(_('could not parse treeinfo'))
self.progress_report['state'] = constants.STATE_FAILED
return
existing_units = repo_controller.find_repo_content_units(
self.repo,
repo_content_unit_q=Q(unit_type_id=ids.TYPE_ID_DISTRO),
yield_content_unit=True)
existing_units = list(existing_units)
# determine missing units
missing_units = repo_controller.missing_unit_count(self.repo.repo_id)
# Continue only when the distribution has changed.
if len(existing_units) == 1 and \
self.existing_distribution_is_current(existing_units[0], unit) and \
(self.download_deferred or not missing_units):
_logger.info(_('upstream distribution unchanged; skipping'))
return
# Process the distribution
dist_files, pulp_dist_xml_path = self.process_distribution(tmp_dir)
files.extend(dist_files)
self.update_unit_files(unit, files)
# Download distribution files
if not self.download_deferred:
try:
downloaded = self.download_files(tmp_dir, files)
except DownloadFailed:
# All files must be downloaded to continue.
return
else:
unit.downloaded = False
downloaded = []
# Save the unit.
unit.save()
# Update deferred downloading catalog
self.update_catalog_entries(unit, files)
# The treeinfo and PULP_DISTRIBTION.xml files are always imported into platform
# # storage regardless of the download policy
unit.safe_import_content(treeinfo_path,
os.path.basename(treeinfo_path))
if pulp_dist_xml_path is not None:
unit.safe_import_content(pulp_dist_xml_path,
os.path.basename(pulp_dist_xml_path))
# The downloaded files are imported into platform storage.
if downloaded:
for destination, location in downloaded:
unit.safe_import_content(destination, location)
else:
if not unit.downloaded:
unit.downloaded = True
unit.save()
# Associate the unit.
repo_controller.associate_single_unit(self.repo, unit)
# find any old distribution units and remove them. See BZ #1150714
for existing_unit in existing_units:
if existing_unit == unit:
continue
msg = _('Removing out-of-date distribution unit {k} for repo {r}')
_logger.info(
msg.format(k=existing_unit.unit_key, r=self.repo.repo_id))
qs = RepositoryContentUnit.objects.filter(
repo_id=self.repo.repo_id, unit_id=existing_unit.id)
qs.delete()
def associate(source_repo, dest_repo, import_conduit, config, units=None):
"""
This is the primary method to call when a copy operation is desired. This
gets called directly by the Importer
Certain variables are set to "None" as the method progresses so that they
may be garbage collected.
:param source_repo: source repo
:type source_repo: pulp.server.db.model.Repository
:param dest_repo: destination repo
:type dest_repo: pulp.server.db.model.Repository
:param import_conduit: import conduit passed to the Importer
:type import_conduit: pulp.plugins.conduits.unit_import.ImportUnitConduit
:param config: config object for the distributor
:type config: pulp.plugins.config.PluginCallConfiguration
:param units: iterable of ContentUnit objects to copy
:type units: iterable
:return: List of associated units.
"""
if units is None:
# this might use a lot of RAM since RPMs tend to have lots of metadata
# TODO: so we should probably do something about that
units = repo_controller.find_repo_content_units(
source_repo, yield_content_unit=True)
# get config items that we care about
recursive = config.get(constants.CONFIG_RECURSIVE)
if recursive is None:
recursive = False
associated_units = set(
[_associate_unit(dest_repo, unit) for unit in units])
# allow garbage collection
units = None
associated_units |= copy_rpms(
(unit for unit in associated_units if isinstance(unit, models.RPM)),
source_repo, dest_repo, import_conduit, recursive)
# return here if we shouldn't get child units
if not recursive:
return list(associated_units)
group_ids, rpm_names, rpm_search_dicts = identify_children_to_copy(
associated_units)
# ------ get group children of the categories ------
for page in paginate(group_ids):
group_units = models.PackageGroup.objects.filter(
repo_id=source_repo.repo_id, package_group_id__in=page)
if group_units.count() > 0:
associated_units |= set(
associate(source_repo, dest_repo, import_conduit, config,
group_units))
# ------ get RPM children of errata ------
wanted_rpms = get_rpms_to_copy_by_key(rpm_search_dicts, import_conduit,
source_repo)
rpm_search_dicts = None
rpms_to_copy = filter_available_rpms(wanted_rpms, import_conduit,
source_repo)
associated_units |= copy_rpms(rpms_to_copy, source_repo, dest_repo,
import_conduit, recursive)
rpms_to_copy = None
# ------ get RPM children of groups ------
names_to_copy = get_rpms_to_copy_by_name(rpm_names, import_conduit,
dest_repo)
associated_units |= copy_rpms_by_name(names_to_copy, source_repo,
dest_repo, import_conduit, recursive)
return list(associated_units)
def associate(source_repo, dest_repo, import_conduit, config, units=None):
"""
This is the primary method to call when a copy operation is desired. This
gets called directly by the Importer
Certain variables are set to "None" as the method progresses so that they
may be garbage collected.
:param source_repo: source repo
:type source_repo: pulp.server.db.model.Repository
:param dest_repo: destination repo
:type dest_repo: pulp.server.db.model.Repository
:param import_conduit: import conduit passed to the Importer
:type import_conduit: pulp.plugins.conduits.unit_import.ImportUnitConduit
:param config: config object for the distributor
:type config: pulp.plugins.config.PluginCallConfiguration
:param units: iterable of ContentUnit objects to copy
:type units: iterable
:return: List of associated units.
"""
if units is None:
# this might use a lot of RAM since RPMs tend to have lots of metadata
# TODO: so we should probably do something about that
units = repo_controller.find_repo_content_units(source_repo, yield_content_unit=True)
# get config items that we care about
recursive = config.get(constants.CONFIG_RECURSIVE)
if recursive is None:
recursive = False
associated_units = set([_associate_unit(dest_repo, unit) for unit in units])
# allow garbage collection
units = None
associated_units |= copy_rpms(
(unit for unit in associated_units if isinstance(unit, models.RPM)),
source_repo, dest_repo, import_conduit, recursive)
# return here if we shouldn't get child units
if not recursive:
return list(associated_units)
group_ids, rpm_names, rpm_search_dicts = identify_children_to_copy(associated_units)
# ------ get group children of the categories ------
for page in paginate(group_ids):
group_units = models.PackageGroup.objects.filter(repo_id=source_repo.repo_id,
package_group_id__in=page)
if group_units.count() > 0:
associated_units |= set(
associate(source_repo, dest_repo, import_conduit, config, group_units))
# ------ get RPM children of errata ------
wanted_rpms = get_rpms_to_copy_by_key(rpm_search_dicts, import_conduit, source_repo)
rpm_search_dicts = None
rpms_to_copy = filter_available_rpms(wanted_rpms, import_conduit, source_repo)
associated_units |= copy_rpms(rpms_to_copy, source_repo, dest_repo, import_conduit, recursive)
rpms_to_copy = None
# ------ get RPM children of groups ------
names_to_copy = get_rpms_to_copy_by_name(rpm_names, import_conduit, dest_repo)
associated_units |= copy_rpms_by_name(names_to_copy, source_repo, dest_repo,
import_conduit, recursive)
return list(associated_units)
def publish_repo(self, repo, publish_conduit, config):
"""
Publish the repository.
:param repo: metadata describing the repo
:type repo: pulp.plugins.model.Repository
:param publish_conduit: The conduit for publishing a repo
:type publish_conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit
:param config: plugin configuration
:type config: pulp.plugins.config.PluginConfiguration
:param config_conduit: Configuration Conduit;
:type config_conduit: pulp.plugins.conduits.repo_validate.RepoConfigConduit
:return: report describing the publish operation
:rtype: pulp.plugins.model.PublishReport
"""
progress_report = FilePublishProgressReport(publish_conduit)
_logger.info(
_('Beginning publish for repository <%(repo)s>') %
{'repo': repo.id})
try:
progress_report.state = progress_report.STATE_IN_PROGRESS
repo_model = repo.repo_obj
units = repo_controller.find_repo_content_units(
repo_model, yield_content_unit=True)
# Set up an empty build_dir
working_dir = common_utils.get_working_directory()
build_dir = os.path.join(working_dir, BUILD_DIRNAME)
misc.mkdir(build_dir)
self.initialize_metadata(build_dir)
try:
# process each unit
for unit in units:
links_to_create = self.get_paths_for_unit(unit)
self._symlink_unit(build_dir, unit, links_to_create)
self.publish_metadata_for_unit(unit)
finally:
# Finalize the processing
self.finalize_metadata()
# Let's unpublish, and then republish
self.unpublish_repo(repo, config)
hosting_locations = self.get_hosting_locations(repo_model, config)
for location in hosting_locations:
shutil.copytree(build_dir, location, symlinks=True)
self.post_repo_publish(repo_model, config)
# Report that we are done
progress_report.state = progress_report.STATE_COMPLETE
return progress_report.build_final_report()
except Exception, e:
_logger.exception(e)
# Something failed. Let's put an error message on the report
progress_report.error_message = str(e)
progress_report.traceback = traceback.format_exc()
progress_report.state = progress_report.STATE_FAILED
report = progress_report.build_final_report()
return report
class ISOImporter(Importer):
"""
All methods that are missing docstrings are documented in the Importer superclass.
"""
def import_units(self,
source_repo,
dest_repo,
import_conduit,
config,
units=None):
"""
Import content units into the given repository. This method will be
called in a number of different situations:
* A user is attempting to copy a content unit from one repository
into the repository that uses this importer
* A user is attempting to add an orphaned unit into a repository.
This call has two options for handling the requested units:
* Associate the given units with the destination repository. This will
link the repository with the existing unit directly; changes to the
unit will be reflected in all repositories that reference it.
* Create a new unit and save it to the repository. This would act as
a deep copy of sorts, creating a unique unit in the database. Keep
in mind that the unit key must change in order for the unit to
be considered different than the supplied one.
The APIs for both approaches are similar to those in the sync conduit.
In the case of a simple association, the init_unit step can be skipped
and save_unit simply called on each specified unit.
The units argument is optional. If None, all units in the source
repository should be imported. The conduit is used to query for those
units. If specified, only the units indicated should be imported (this
is the case where the caller passed a filter to Pulp).
:param source_repo: metadata describing the repository containing the
units to import
:type source_repo: pulp.plugins.model.Repository
:param dest_repo: metadata describing the repository to import units
into
:type dest_repo: pulp.plugins.model.Repository
:param import_conduit: provides access to relevant Pulp functionality
:type import_conduit: pulp.plugins.conduits.unit_import.ImportUnitConduit
:param config: plugin configuration
:type config: pulp.plugins.config.PluginCallConfiguration
:param units: optional list of pre-filtered units to import
:type units: list of pulp.plugins.model.Unit
:return: list of Unit instances that were saved to the destination repository
:rtype: list
"""
if units is None:
criteria = UnitAssociationCriteria(type_ids=[ids.TYPE_ID_ISO])
units = import_conduit.get_source_units(criteria=criteria)
units = list(units)
for u in units:
import_conduit.associate_unit(u)
return units
@classmethod
def metadata(cls):
return {
'id': ids.TYPE_ID_IMPORTER_ISO,
'display_name': 'ISO Importer',
'types': [ids.TYPE_ID_ISO]
}
def sync_repo(self, transfer_repo, sync_conduit, config):
sync_conduit.repo = transfer_repo.repo_obj
if config.get(importer_constants.KEY_FEED) is None:
raise ValueError('Repository without feed cannot be synchronized')
self.iso_sync = sync.ISOSyncRun(sync_conduit, config)
report = self.iso_sync.perform_sync()
self.iso_sync = None
return report
def upload_unit(self, transfer_repo, type_id, unit_key, metadata,
file_path, conduit, config):
"""
Handles the creation and association of an ISO.
:param repo: The repository to import the package into
:type repo: pulp.server.db.model.Repository
:param type_id: The type_id of the package being uploaded
:type type_id: str
:param unit_key: A dictionary of fields to overwrite introspected field values
:type unit_key: dict
:param metadata: A dictionary of fields to overwrite introspected field values, or None
:type metadata: dict or None
:param file_path: The path to the uploaded package
:type file_path: str
:param conduit: provides access to relevant Pulp functionality
:type conduit: pulp.plugins.conduits.upload.UploadConduit
:param config: plugin configuration for the repository
:type config: pulp.plugins.config.PluginCallConfiguration
"""
qs = models.ISO.objects.filter(**unit_key)
if qs:
# iso with this key already exists, use it
iso = qs.first()
else:
# this is a new ISO, create it
iso = models.ISO(**unit_key)
validate = config.get_boolean(importer_constants.KEY_VALIDATE)
validate = validate if validate is not None else constants.CONFIG_VALIDATE_DEFAULT
try:
# Let's validate the ISO. This will raise a
# ValueError if the ISO does not validate correctly.
iso.validate_iso(file_path, full_validation=validate)
except ValueError, e:
return {
'success_flag': False,
'summary': e.message,
'details': None
}
try:
iso.save_and_import_content(file_path)
except NotUniqueError:
iso = iso.__class__.objects.get(**iso.unit_key)
# remove any existing units with the same name
units = repo_controller.find_repo_content_units(
transfer_repo.repo_obj,
units_q=Q(name=iso['name']),
yield_content_unit=True)
repo_controller.disassociate_units(transfer_repo.repo_obj, units)
repo_controller.associate_single_unit(transfer_repo.repo_obj, iso)
return {'success_flag': True, 'summary': None, 'details': None}
def associate_from_repo(source_repo_id,
dest_repo_id,
criteria=None,
import_config_override=None):
"""
Creates associations in a repository based on the contents of a source
repository. Units from the source repository can be filtered by
specifying a criteria object.
The destination repository must have an importer that can support
the types of units being associated. This is done by analyzing the
unit list and the importer metadata and takes place before the
destination repository is called.
Pulp does not actually perform the associations as part of this call.
The unit list is determined and passed to the destination repository's
importer. It is the job of the importer to make the associate calls
back into Pulp where applicable.
If criteria is None, the effect of this call is to copy the source
repository's associations into the destination repository.
:param source_repo_id: identifies the source repository
:type source_repo_id: str
:param dest_repo_id: identifies the destination repository
:type dest_repo_id: str
:param criteria: optional; if specified, will filter the units retrieved from
the source repository
:type criteria: pulp.server.db.model.criteria.UnitAssociationCriteria
:param import_config_override: optional config containing values to use for this import only
:type import_config_override: dict
:return: dict with key 'units_successful' whose
value is a list of unit keys that were copied.
units that were associated by this operation
:rtype: dict
:raise MissingResource: if either of the specified repositories don't exist
"""
importer_manager = manager_factory.repo_importer_manager()
source_repo = model.Repository.objects.get_repo_or_missing_resource(
source_repo_id)
dest_repo = model.Repository.objects.get_repo_or_missing_resource(
dest_repo_id)
# This will raise MissingResource if there isn't one, which is the
# behavior we want this method to exhibit, so just let it bubble up.
dest_repo_importer = importer_manager.get_importer(dest_repo_id)
source_repo_importer = importer_manager.get_importer(source_repo_id)
# The docs are incorrect on the list_importer_types call; it actually
# returns a dict with the types under key "types" for some reason.
supported_type_ids = set(
plugin_api.list_importer_types(
dest_repo_importer['importer_type_id'])['types'])
# Get the unit types from the repo source repo
source_repo_unit_types = set(source_repo.content_unit_counts.keys())
# Now we can make sure the destination repository's importer is capable
# of importing either the selected units or all of the units
if not source_repo_unit_types.issubset(supported_type_ids):
raise exceptions.PulpCodedException(
error_code=error_codes.PLP0000,
message=
'The the target importer does not support the types from the source'
)
transfer_units = None
# If criteria is specified, retrieve the list of units now
if criteria is not None:
# if all source types have been converted to mongo - search via new style
if source_repo_unit_types.issubset(
set(plugin_api.list_unit_models())):
association_q = mongoengine.Q(
__raw__=criteria.association_spec)
unit_q = mongoengine.Q(__raw__=criteria.unit_spec)
transfer_units = repo_controller.find_repo_content_units(
repository=source_repo,
repo_content_unit_q=association_q,
units_q=unit_q,
yield_content_unit=True)
else:
# else, search via old style
associate_us = load_associated_units(source_repo_id, criteria)
# If units were supposed to be filtered but none matched, we're done
if len(associate_us) == 0:
# Return an empty list to indicate nothing was copied
return {'units_successful': []}
# Convert all of the units into the plugin standard representation if
# a filter was specified
transfer_units = None
if associate_us is not None:
transfer_units = create_transfer_units(associate_us)
# Convert the two repos into the plugin API model
transfer_dest_repo = dest_repo.to_transfer_repo()
transfer_source_repo = source_repo.to_transfer_repo()
# Invoke the importer
importer_instance, plugin_config = plugin_api.get_importer_by_id(
dest_repo_importer['importer_type_id'])
call_config = PluginCallConfiguration(plugin_config,
dest_repo_importer['config'],
import_config_override)
conduit = ImportUnitConduit(source_repo_id, dest_repo_id,
source_repo_importer['id'],
dest_repo_importer['id'])
try:
copied_units = importer_instance.import_units(transfer_source_repo,
transfer_dest_repo,
conduit,
call_config,
units=transfer_units)
unit_ids = [u.to_id_dict() for u in copied_units]
return {'units_successful': unit_ids}
except Exception:
msg = _(
'Exception from importer [%(i)s] while importing units into repository [%(r)s]'
)
msg_dict = {
'i': dest_repo_importer['importer_type_id'],
'r': dest_repo_id
}
logger.exception(msg % msg_dict)
raise exceptions.PulpExecutionException(), None, sys.exc_info()[2]
def _run(self, tmp_dir):
"""
Look for a distribution in the target repo and sync it if found
:param tmp_dir: The absolute path to the temporary directory
:type tmp_dir: str
"""
treeinfo_path = self.get_treefile(tmp_dir)
if not treeinfo_path:
_logger.debug(_('No treeinfo found'))
return
try:
unit, files = self.parse_treeinfo_file(treeinfo_path)
except ValueError:
_logger.error(_('could not parse treeinfo'))
self.progress_report['state'] = constants.STATE_FAILED
return
existing_units = repo_controller.find_repo_content_units(
self.repo,
repo_content_unit_q=Q(unit_type_id=ids.TYPE_ID_DISTRO),
yield_content_unit=True)
existing_units = list(existing_units)
# Continue only when the distribution has changed.
if len(existing_units) == 1 and \
self.existing_distribution_is_current(existing_units[0], unit):
_logger.debug(_('upstream distribution unchanged; skipping'))
return
# Process the distribution
dist_files = self.process_distribution(tmp_dir)
files.extend(dist_files)
self.update_unit_files(unit, files)
# Download distribution files
if not self.download_deferred:
try:
downloaded = self.download_files(tmp_dir, files)
except DownloadFailed:
# All files must be downloaded to continue.
return
else:
unit.downloaded = False
downloaded = []
# Save the unit.
unit.save()
# Update deferred downloading catalog
self.update_catalog_entries(unit, files)
# The treeinfo file is always imported into platform
# # storage regardless of the download policy
unit.safe_import_content(treeinfo_path, os.path.basename(treeinfo_path))
# The downloaded files are imported into platform storage.
for destination, location in downloaded:
unit.safe_import_content(destination, location)
# Associate the unit.
repo_controller.associate_single_unit(self.repo, unit)
# find any old distribution units and remove them. See BZ #1150714
for existing_unit in existing_units:
if existing_unit == unit:
continue
msg = _('Removing out-of-date distribution unit {k} for repo {r}')
_logger.info(msg.format(k=existing_unit.unit_key, r=self.repo.repo_id))
qs = RepositoryContentUnit.objects.filter(
repo_id=self.repo.repo_id,
unit_id=existing_unit.id)
qs.delete()
def sync(repo, sync_conduit, feed, working_dir, nectar_config, report, progress_callback):
"""
Look for a distribution in the target repo and sync it if found
:param repo: The repository that is the target of the sync
:type repo: pulp.server.db.model.Repository
:param sync_conduit: conduit provided by the platform
:type sync_conduit: pulp.plugins.conduits.repo_sync.RepoSyncConduit
:param feed: URL of the yum repo being sync'd
:type feed: basestring
:param working_dir: full path to the directory to which files
should be downloaded
:type working_dir: basestring
:param nectar_config: download config to be used by nectar
:type nectar_config: nectar.config.DownloaderConfig
:param report: progress report object
:type report: pulp_rpm.plugins.importers.yum.report.DistributionReport
:param progress_callback: function that takes no arguments but induces
the current progress report to be sent.
"""
tmp_dir = tempfile.mkdtemp(dir=working_dir)
try:
treefile_path = get_treefile(feed, tmp_dir, nectar_config)
if not treefile_path:
_LOGGER.debug('no treefile found')
return
try:
model, files = parse_treefile(treefile_path)
except ValueError:
_LOGGER.error('could not parse treefile')
report['state'] = constants.STATE_FAILED
return
existing_units = repo_controller.find_repo_content_units(
repo, repo_content_unit_q=mongoengine.Q(unit_type_id=ids.TYPE_ID_DISTRO),
yield_content_unit=True)
existing_units = list(existing_units)
# skip this whole process if the upstream treeinfo file hasn't changed
if len(existing_units) == 1 and existing_distribution_is_current(existing_units[0], model):
_LOGGER.debug('upstream distribution unchanged; skipping')
return
# Get any errors
dist_files = process_distribution(feed, tmp_dir, nectar_config, model, report)
files.extend(dist_files)
report.set_initial_values(len(files))
listener = DistroFileListener(report, progress_callback)
downloader = nectar_factory.create_downloader(feed, nectar_config, listener)
_LOGGER.debug('downloading distribution files')
downloader.download(file_to_download_request(f, feed, tmp_dir) for f in files)
if len(listener.failed_reports) == 0:
model.set_content(tmp_dir)
model.save()
# The save sets the content path, which is needed to generate the download_reports
# Long term this should be done by a serializer
model.process_download_reports(listener.succeeded_reports)
model.save()
repo_controller.associate_single_unit(repo, model)
# find any old distribution units and remove them. See BZ #1150714
for existing_unit in existing_units:
if existing_unit != model:
_LOGGER.info("Removing out-of-date distribution unit %s for repo %s" %
(existing_unit.unit_key, sync_conduit.repo_id))
platform_models.RepositoryContentUnit.objects(
repo_id=sync_conduit.repo_id,
unit_type_id=models.Distribution._content_type_id.default,
unit_id=existing_unit.id).delete()
else:
_LOGGER.error('some distro file downloads failed')
report['state'] = constants.STATE_FAILED
report['error_details'] = [(fail.url, fail.error_report) for fail in
listener.failed_reports]
return
finally:
shutil.rmtree(tmp_dir, ignore_errors=True)
def associate_from_repo(source_repo_id, dest_repo_id, criteria=None,
import_config_override=None):
"""
Creates associations in a repository based on the contents of a source
repository. Units from the source repository can be filtered by
specifying a criteria object.
The destination repository must have an importer that can support
the types of units being associated. This is done by analyzing the
unit list and the importer metadata and takes place before the
destination repository is called.
Pulp does not actually perform the associations as part of this call.
The unit list is determined and passed to the destination repository's
importer. It is the job of the importer to make the associate calls
back into Pulp where applicable.
If criteria is None, the effect of this call is to copy the source
repository's associations into the destination repository.
:param source_repo_id: identifies the source repository
:type source_repo_id: str
:param dest_repo_id: identifies the destination repository
:type dest_repo_id: str
:param criteria: optional; if specified, will filter the units retrieved from
the source repository
:type criteria: pulp.server.db.model.criteria.UnitAssociationCriteria
:param import_config_override: optional config containing values to use for this import only
:type import_config_override: dict
:return: dict with key 'units_successful' whose
value is a list of unit keys that were copied.
units that were associated by this operation
:rtype: dict
:raise MissingResource: if either of the specified repositories don't exist
"""
importer_manager = manager_factory.repo_importer_manager()
source_repo = model.Repository.objects.get_repo_or_missing_resource(source_repo_id)
dest_repo = model.Repository.objects.get_repo_or_missing_resource(dest_repo_id)
# This will raise MissingResource if there isn't one, which is the
# behavior we want this method to exhibit, so just let it bubble up.
dest_repo_importer = importer_manager.get_importer(dest_repo_id)
source_repo_importer = importer_manager.get_importer(source_repo_id)
# The docs are incorrect on the list_importer_types call; it actually
# returns a dict with the types under key "types" for some reason.
supported_type_ids = set(plugin_api.list_importer_types(
dest_repo_importer['importer_type_id'])['types'])
# Get the unit types from the repo source repo
source_repo_unit_types = set(source_repo.content_unit_counts.keys())
# Now we can make sure the destination repository's importer is capable
# of importing either the selected units or all of the units
if not source_repo_unit_types.issubset(supported_type_ids):
raise exceptions.PulpCodedException(
error_code=error_codes.PLP0000,
message='The the target importer does not support the types from the source')
transfer_units = None
# If criteria is specified, retrieve the list of units now
if criteria is not None:
# if all source types have been converted to mongo - search via new style
if source_repo_unit_types.issubset(set(plugin_api.list_unit_models())):
association_q = mongoengine.Q(__raw__=criteria.association_spec)
unit_q = mongoengine.Q(__raw__=criteria.unit_spec)
transfer_units = repo_controller.find_repo_content_units(
repository=source_repo,
repo_content_unit_q=association_q,
units_q=unit_q,
yield_content_unit=True)
else:
# else, search via old style
associate_us = load_associated_units(source_repo_id, criteria)
# If units were supposed to be filtered but none matched, we're done
if len(associate_us) == 0:
# Return an empty list to indicate nothing was copied
return {'units_successful': []}
# Convert all of the units into the plugin standard representation if
# a filter was specified
transfer_units = None
if associate_us is not None:
associated_unit_type_ids = calculate_associated_type_ids(source_repo_id,
associate_us)
transfer_units = create_transfer_units(associate_us, associated_unit_type_ids)
# Convert the two repos into the plugin API model
transfer_dest_repo = dest_repo.to_transfer_repo()
transfer_source_repo = source_repo.to_transfer_repo()
# Invoke the importer
importer_instance, plugin_config = plugin_api.get_importer_by_id(
dest_repo_importer['importer_type_id'])
call_config = PluginCallConfiguration(plugin_config, dest_repo_importer['config'],
import_config_override)
conduit = ImportUnitConduit(
source_repo_id, dest_repo_id, source_repo_importer['id'], dest_repo_importer['id'])
try:
copied_units = importer_instance.import_units(
transfer_source_repo, transfer_dest_repo, conduit, call_config,
units=transfer_units)
unit_ids = [u.to_id_dict() for u in copied_units]
return {'units_successful': unit_ids}
except Exception:
msg = _('Exception from importer [%(i)s] while importing units into repository [%(r)s]')
msg_dict = {'i': dest_repo_importer['importer_type_id'], 'r': dest_repo_id}
logger.exception(msg % msg_dict)
raise exceptions.PulpExecutionException(), None, sys.exc_info()[2]
