def add_v2_steps(self, repo, conduit, config):
"""
Add v2 sync steps.
:param repo: repository to sync
:type repo: pulp.plugins.model.Repository
:param conduit: sync conduit to use
:type conduit: pulp.plugins.conduits.repo_sync.RepoSyncConduit
:param config: config object for the sync
:type config: pulp.plugins.config.PluginCallConfiguration
"""
self.add_child(
DownloadManifestsStep(repo=repo, conduit=conduit, config=config))
# save these steps so their "units_to_download" attributes can be accessed later. We want
# them to be separate steps because we have already downloaded all the Manifests but should
# only save the new ones, while needing to go download the missing Blobs. Thus they must be
# handled separately.
self.step_get_local_manifests = publish_step.GetLocalUnitsStep(
importer_type=constants.IMPORTER_TYPE_ID,
available_units=self.available_manifests)
self.step_get_local_blobs = publish_step.GetLocalUnitsStep(
importer_type=constants.IMPORTER_TYPE_ID,
available_units=self.available_blobs)
self.add_child(self.step_get_local_manifests)
self.add_child(self.step_get_local_blobs)
self.add_child(
AuthDownloadStep(step_type=constants.SYNC_STEP_DOWNLOAD,
downloads=self.generate_download_requests(),
repo=self.repo,
config=self.config,
description=_('Downloading remote files')))
self.add_child(SaveUnitsStep())
self.save_tags_step = SaveTagsStep()
self.add_child(self.save_tags_step)
def test_uses_passed_available_units_when_requested(self, mock_find_units, mock_associate):
"""
Assert that if the step is constructed with available_units passed in, the step's parent's
available_units attribute is ignored.
"""
demo_1 = self.DemoModel(key_field='a')
demo_2 = self.DemoModel(key_field='b')
demo_3 = self.DemoModel(key_field='c')
self.parent.available_units = [demo_1, demo_2]
available_units = [demo_1, demo_2, demo_3]
fake_repo = Repository(id='fake-repo', repo_obj='fake_repo')
step = publish_step.GetLocalUnitsStep('fake_importer_type', repo=fake_repo,
available_units=available_units)
step.parent = self.parent
step.conduit = MagicMock()
existing_demo = self.DemoModel(key_field='b', id='foo')
mock_find_units.return_value = [existing_demo]
step.process_main()
# The parent step's available units only has demo 1 and 2, so this asserts that that is
# being ignored and the correct available_units is being used instead.
mock_find_units.assert_called_once_with((demo_1, demo_2, demo_3))
# the one that exists is associated
mock_associate.assert_called_once_with('fake_repo', existing_demo)
# the two that do not exist yet are added to the download list
self.assertEqual(step.units_to_download, [demo_1, demo_3])
def _import_modules(self, module_paths):
"""
Import the puppet modules (tarballs) at the specified paths. This will also handle
removing any modules in the local repository if they are no longer present on remote
repository and the 'remove_missing' config value is True.
:param module_paths: A list of paths to puppet module files.
:type module_paths: list
"""
existing_module_ids_by_key = {}
for module in Module.objects.only(*Module.unit_key_fields).all():
existing_module_ids_by_key[module.unit_key_str] = module.id
remote_paths = {}
list_of_modules = []
for module_path in module_paths:
puppet_manifest = self._extract_metadata(module_path)
module = Module.from_metadata(puppet_manifest)
remote_paths[module.unit_key_str] = module_path
list_of_modules.append(module)
pub_step = publish_step.GetLocalUnitsStep(
constants.IMPORTER_TYPE_ID,
available_units=list_of_modules,
repo=self.repo)
pub_step.process_main()
self.report.modules_total_count = len(pub_step.units_to_download)
for module in pub_step.units_to_download:
remote_path = remote_paths[module.unit_key_str]
if self.canceled:
return
_logger.debug(IMPORT_MODULE, dict(mod=remote_path))
module.set_storage_path(os.path.basename(remote_path))
try:
module.save_and_import_content(remote_path)
except NotUniqueError:
module = module.__class__.objects.get(**module.unit_key)
repo_controller.associate_single_unit(self.repo.repo_obj, module)
self.report.modules_finished_count += 1
self.report.update_progress()
# Write the report, making sure we don't overwrite a failure in _fetch_modules
if self.report.modules_state not in constants.COMPLETE_STATES:
self.report.modules_state = constants.STATE_SUCCESS
self.report.modules_execution_time = time(
) - self.started_fetch_modules
self.report.update_progress()
remove_missing = self.config.get_boolean(
constants.CONFIG_REMOVE_MISSING)
if remove_missing is None:
remove_missing = constants.DEFAULT_REMOVE_MISSING
if remove_missing:
self._remove_missing(existing_module_ids_by_key,
remote_paths.keys())
def setUp(self):
super(TestGetLocalUnitsStep, self).setUp()
self.parent = MagicMock()
self.step = publish_step.GetLocalUnitsStep('fake_importer_type', repo='fake_repo')
self.step.parent = self.parent
self.step.conduit = MagicMock()
self.parent.available_units = []
def test___init___default_available_units(self, find_units, associate):
"""
Assert that the __init__() method correctly defaults to a value of None for the
available_units parameter.
"""
step = publish_step.GetLocalUnitsStep('fake_importer_type', repo='fake_repo')
self.assertEqual(step.available_units, None)
def test___init___with_available_units(self, find_units, associate):
"""
Assert that the __init__() method allows the user to override the value of None for the
available_units parameter.
"""
available_units = ['unit1', 'unit2']
step = publish_step.GetLocalUnitsStep('fake_importer_type', repo='fake_repo',
available_units=available_units)
self.assertEqual(step.available_units, available_units)
def __init__(self, repo, conduit, config):
"""
:param repo: repository to sync
:type repo: pulp.plugins.model.Repository
:param conduit: sync conduit to use
:type conduit: pulp.plugins.conduits.repo_sync.RepoSyncConduit
:param config: config object for the sync
:type config: pulp.plugins.config.PluginCallConfiguration
"""
super(RepoSync, self).__init__(step_type=constants.SYNC_STEP,
repo=repo,
conduit=conduit,
config=config)
self.description = _('Syncing Repository')
self.apt_repo_meta = None
# https://pulp.plan.io/issues/2765 should remove the need to hardcode
# the dist/component here
self.feed_url = self.get_config().get('feed').strip(
'/') + '/dists/stable/'
self.release_file = os.path.join(self.get_working_dir(), "Release")
self.available_units = None
rel_url = urlparse.urljoin(self.feed_url, 'Release')
_logger.info("Downloading %s", rel_url)
self.add_child(
publish_step.DownloadStep(
constants.SYNC_STEP_RELEASE_DOWNLOAD,
plugin_type=ids.TYPE_ID_IMPORTER,
description=_('Retrieving metadata: release file'),
downloads=[DownloadRequest(rel_url, self.release_file)]))
self.add_child(ParseReleaseStep(constants.SYNC_STEP_RELEASE_PARSE))
self.step_download_Packages = publish_step.DownloadStep(
constants.SYNC_STEP_PACKAGES_DOWNLOAD,
plugin_type=ids.TYPE_ID_IMPORTER,
description=_('Retrieving metadata: Packages files'))
self.add_child(self.step_download_Packages)
self.add_child(ParsePackagesStep(constants.SYNC_STEP_PACKAGES_PARSE))
self.step_local_units = publish_step.GetLocalUnitsStep(
importer_type=ids.TYPE_ID_IMPORTER)
self.add_child(self.step_local_units)
self.add_child(
CreateRequestsUnitsToDownload(
constants.SYNC_STEP_UNITS_DOWNLOAD_REQUESTS))
self.step_download_units = publish_step.DownloadStep(
constants.SYNC_STEP_UNITS_DOWNLOAD,
plugin_type=ids.TYPE_ID_IMPORTER,
description=_('Retrieving units'))
self.add_child(self.step_download_units)
self.add_child(SaveDownloadedUnits(constants.SYNC_STEP_SAVE))
def add_v1_steps(self, repo, config):
"""
Add v1 sync steps.
:param repo: repository to sync
:type repo: pulp.plugins.model.Repository
:param config: config object for the sync
:type config: pulp.plugins.config.PluginCallConfiguration
"""
self.add_child(v1_sync.GetMetadataStep())
# save this step so its "units_to_download" attribute can be accessed later
self.v1_step_get_local_units = publish_step.GetLocalUnitsStep(
constants.IMPORTER_TYPE_ID, available_units=self.v1_available_units)
self.v1_step_get_local_units.step_id = constants.SYNC_STEP_GET_LOCAL_V1
self.add_child(self.v1_step_get_local_units)
self.add_child(publish_step.DownloadStep(
constants.SYNC_STEP_DOWNLOAD_V1, downloads=self.v1_generate_download_requests(),
repo=repo, config=config, description=_('Downloading remote files')))
self.add_child(v1_sync.SaveImages())
def test_empty_available_units(self, mock_find_units, mock_associate):
"""
Assert that if the step is constructed with the an empty available_units passed in, the
step's parent's available_units attribute is ignored.
"""
demo_1 = self.DemoModel(key_field='a')
self.parent.available_units = [demo_1]
fake_repo = Repository(id='fake-repo', repo_obj='fake_repo')
step = publish_step.GetLocalUnitsStep('fake_importer_type', repo=fake_repo,
available_units=[])
step.parent = self.parent
step.conduit = MagicMock()
existing_demo = self.DemoModel(key_field='b', id='foo')
mock_find_units.return_value = [existing_demo]
step.process_main()
# The parent step's available units only has demo 1 and 2, so this asserts that that is
# being ignored and the correct available_units is being used instead.
self.assertEqual(mock_find_units.call_count, 0)
def __init__(self, repo, conduit, config):
"""
:param repo: repository to sync
:type repo: pulp.plugins.model.Repository
:param conduit: sync conduit to use
:type conduit: pulp.plugins.conduits.repo_sync.RepoSyncConduit
:param config: config object for the sync
:type config: pulp.plugins.config.PluginCallConfiguration
"""
super(RepoSync, self).__init__(step_type=constants.SYNC_STEP,
repo=repo,
conduit=conduit,
config=config)
self.description = _('Syncing Repository')
self.feed_url = self.get_config().get('feed').strip('/')
self.releases = self.get_config().get('releases', 'stable').split(',')
self.architectures = split_or_none(
self.get_config().get('architectures'))
self.components = split_or_none(self.get_config().get('components'))
self.unit_relative_urls = {}
self.available_units = None
# dicts with release names as keys to multiplex variables
self.apt_repo_meta = {}
self.release_units = {}
self.release_files = {
release: os.path.join(self.get_working_dir(), release, 'Release')
for release in self.releases
}
self.feed_urls = {
release: urlparse.urljoin(self.feed_url + '/',
'/'.join(['dists', release]))
for release in self.releases
}
self.release_urls = {
release: urlparse.urljoin(self.feed_urls[release] + '/', 'Release')
for release in self.releases
}
self.packages_urls = {}
# double dicts with release/component as keys
self.component_units = defaultdict(dict)
self.component_packages = defaultdict(dict)
for release in self.releases:
misc.mkdir(os.path.dirname(self.release_files[release]))
_logger.info("Downloading %s", self.release_urls[release])
# defining lifecycle
# metadata
self.add_child(
publish_step.DownloadStep(
constants.SYNC_STEP_RELEASE_DOWNLOAD,
plugin_type=ids.TYPE_ID_IMPORTER,
description=_('Retrieving metadata: release file(s)'),
downloads=[
DownloadRequest(self.release_urls[release],
self.release_files[release])
for release in self.releases
]))
self.add_child(ParseReleaseStep(constants.SYNC_STEP_RELEASE_PARSE))
self.step_download_Packages = publish_step.DownloadStep(
constants.SYNC_STEP_PACKAGES_DOWNLOAD,
plugin_type=ids.TYPE_ID_IMPORTER,
description=_('Retrieving metadata: Packages files'))
self.add_child(self.step_download_Packages)
self.add_child(ParsePackagesStep(constants.SYNC_STEP_PACKAGES_PARSE))
# packages
self.step_local_units = publish_step.GetLocalUnitsStep(
importer_type=ids.TYPE_ID_IMPORTER)
self.add_child(self.step_local_units)
self.add_child(
CreateRequestsUnitsToDownload(
constants.SYNC_STEP_UNITS_DOWNLOAD_REQUESTS))
self.step_download_units = publish_step.DownloadStep(
constants.SYNC_STEP_UNITS_DOWNLOAD,
plugin_type=ids.TYPE_ID_IMPORTER,
description=_('Retrieving units'))
self.add_child(self.step_download_units)
self.add_child(SaveDownloadedUnits(constants.SYNC_STEP_SAVE))
# metadata
self.add_child(SaveMetadataStep(constants.SYNC_STEP_SAVE_META))