def __init__(self, repo, conduit, config):
"""
:param repo: repository to sync
:type repo: pulp.plugins.model.Repository
:param conduit: sync conduit to use
:type conduit: pulp.plugins.conduits.repo_sync.RepoSyncConduit
:param config: config object for the sync
:type config: pulp.plugins.config.PluginCallConfiguration
"""
super(RepoSync, self).__init__(step_type=constants.SYNC_STEP,
repo=repo,
conduit=conduit,
config=config)
self.description = _('Syncing Repository')
self.apt_repo_meta = None
# https://pulp.plan.io/issues/2765 should remove the need to hardcode
# the dist/component here
self.feed_url = self.get_config().get('feed').strip(
'/') + '/dists/stable/'
self.release_file = os.path.join(self.get_working_dir(), "Release")
self.available_units = None
rel_url = urlparse.urljoin(self.feed_url, 'Release')
_logger.info("Downloading %s", rel_url)
self.add_child(
publish_step.DownloadStep(
constants.SYNC_STEP_RELEASE_DOWNLOAD,
plugin_type=ids.TYPE_ID_IMPORTER,
description=_('Retrieving metadata: release file'),
downloads=[DownloadRequest(rel_url, self.release_file)]))
self.add_child(ParseReleaseStep(constants.SYNC_STEP_RELEASE_PARSE))
self.step_download_Packages = publish_step.DownloadStep(
constants.SYNC_STEP_PACKAGES_DOWNLOAD,
plugin_type=ids.TYPE_ID_IMPORTER,
description=_('Retrieving metadata: Packages files'))
self.add_child(self.step_download_Packages)
self.add_child(ParsePackagesStep(constants.SYNC_STEP_PACKAGES_PARSE))
self.step_local_units = publish_step.GetLocalUnitsStep(
importer_type=ids.TYPE_ID_IMPORTER)
self.add_child(self.step_local_units)
self.add_child(
CreateRequestsUnitsToDownload(
constants.SYNC_STEP_UNITS_DOWNLOAD_REQUESTS))
self.step_download_units = publish_step.DownloadStep(
constants.SYNC_STEP_UNITS_DOWNLOAD,
plugin_type=ids.TYPE_ID_IMPORTER,
description=_('Retrieving units'))
self.add_child(self.step_download_units)
self.add_child(SaveDownloadedUnits(constants.SYNC_STEP_SAVE))
def test_process_main(self):
mock_downloader = Mock()
mock_downloads = ['fake', 'downloads']
dlstep = publish_step.DownloadStep('fake-step', downloads=mock_downloads)
dlstep.downloader = mock_downloader
dlstep.process_main()
mock_downloader.download.assert_called_once_with(['fake', 'downloads'])
def setUp(self):
conf_dict = {
importer_constants.KEY_FEED: 'http://fake.com/file_feed/',
importer_constants.KEY_MAX_SPEED: 500.0,
importer_constants.KEY_MAX_DOWNLOADS: 5,
importer_constants.KEY_SSL_VALIDATION: False,
importer_constants.KEY_SSL_CLIENT_CERT: "Trust me, I'm who I say I am.",
importer_constants.KEY_SSL_CLIENT_KEY: "Secret Key",
importer_constants.KEY_SSL_CA_CERT: "Uh, I guess that's the right server.",
importer_constants.KEY_PROXY_HOST: 'proxy.com',
importer_constants.KEY_PROXY_PORT: 1234,
importer_constants.KEY_PROXY_USER: "******",
importer_constants.KEY_PROXY_PASS: '******',
importer_constants.KEY_VALIDATE: False,
}
self.real_config = self.get_basic_config(**conf_dict)
self.real_conduit = self.get_sync_conduit()
self.mock_repo = Mock()
self.mock_conduit = Mock()
self.mock_config = Mock()
self.mock_working_dir = Mock()
self.dlstep = publish_step.DownloadStep(
"fake_download", repo=self.mock_repo, conduit=self.mock_conduit,
config=self.mock_config, working_dir=self.mock_working_dir,
plugin_type="fake plugin", description='foo')
def test_cancel(self):
dlstep = publish_step.DownloadStep('fake-step')
dlstep.parent = MagicMock()
dlstep.initialize()
dlstep.cancel()
self.assertTrue(dlstep.downloader.is_canceled)
def test_download_succeeded(self):
dlstep = publish_step.DownloadStep('fake-step')
mock_report = Mock()
mock_report_progress = Mock()
dlstep.report_progress = mock_report_progress
dlstep.download_succeeded(mock_report)
self.assertEquals(dlstep.progress_successes, 1)
# assert report_progress was called with no args
mock_report_progress.assert_called_once_with()
def test_downloads_property(self):
generator = (DownloadRequest(url, '/a/b/c') for url in ['http://pulpproject.org'])
dlstep = publish_step.DownloadStep('fake-step', downloads=generator)
downloads = dlstep.downloads
self.assertTrue(isinstance(downloads, list))
self.assertEqual(len(downloads), 1)
self.assertTrue(isinstance(downloads[0], DownloadRequest))
def add_v1_steps(self, repo, config):
"""
Add v1 sync steps.
:param repo: repository to sync
:type repo: pulp.plugins.model.Repository
:param config: config object for the sync
:type config: pulp.plugins.config.PluginCallConfiguration
"""
self.add_child(v1_sync.GetMetadataStep())
# save this step so its "units_to_download" attribute can be accessed later
self.v1_step_get_local_units = publish_step.GetLocalUnitsStep(
constants.IMPORTER_TYPE_ID, available_units=self.v1_available_units)
self.v1_step_get_local_units.step_id = constants.SYNC_STEP_GET_LOCAL_V1
self.add_child(self.v1_step_get_local_units)
self.add_child(publish_step.DownloadStep(
constants.SYNC_STEP_DOWNLOAD_V1, downloads=self.v1_generate_download_requests(),
repo=repo, config=config, description=_('Downloading remote files')))
self.add_child(v1_sync.SaveImages())
def test_get_total(self):
mock_downloads = ['fake', 'downloads']
dlstep = publish_step.DownloadStep('fake-step', downloads=mock_downloads)
self.assertEquals(dlstep.get_total(), 2)
def __init__(self, repo, conduit, config):
"""
:param repo: repository to sync
:type repo: pulp.plugins.model.Repository
:param conduit: sync conduit to use
:type conduit: pulp.plugins.conduits.repo_sync.RepoSyncConduit
:param config: config object for the sync
:type config: pulp.plugins.config.PluginCallConfiguration
"""
super(RepoSync, self).__init__(step_type=constants.SYNC_STEP,
repo=repo,
conduit=conduit,
config=config)
self.description = _('Syncing Repository')
self.feed_url = self.get_config().get('feed').strip('/')
self.releases = self.get_config().get('releases', 'stable').split(',')
self.architectures = split_or_none(
self.get_config().get('architectures'))
self.components = split_or_none(self.get_config().get('components'))
self.unit_relative_urls = {}
self.available_units = None
# dicts with release names as keys to multiplex variables
self.apt_repo_meta = {}
self.release_units = {}
self.release_files = {
release: os.path.join(self.get_working_dir(), release, 'Release')
for release in self.releases
}
self.feed_urls = {
release: urlparse.urljoin(self.feed_url + '/',
'/'.join(['dists', release]))
for release in self.releases
}
self.release_urls = {
release: urlparse.urljoin(self.feed_urls[release] + '/', 'Release')
for release in self.releases
}
self.packages_urls = {}
# double dicts with release/component as keys
self.component_units = defaultdict(dict)
self.component_packages = defaultdict(dict)
for release in self.releases:
misc.mkdir(os.path.dirname(self.release_files[release]))
_logger.info("Downloading %s", self.release_urls[release])
# defining lifecycle
# metadata
self.add_child(
publish_step.DownloadStep(
constants.SYNC_STEP_RELEASE_DOWNLOAD,
plugin_type=ids.TYPE_ID_IMPORTER,
description=_('Retrieving metadata: release file(s)'),
downloads=[
DownloadRequest(self.release_urls[release],
self.release_files[release])
for release in self.releases
]))
self.add_child(ParseReleaseStep(constants.SYNC_STEP_RELEASE_PARSE))
self.step_download_Packages = publish_step.DownloadStep(
constants.SYNC_STEP_PACKAGES_DOWNLOAD,
plugin_type=ids.TYPE_ID_IMPORTER,
description=_('Retrieving metadata: Packages files'))
self.add_child(self.step_download_Packages)
self.add_child(ParsePackagesStep(constants.SYNC_STEP_PACKAGES_PARSE))
# packages
self.step_local_units = publish_step.GetLocalUnitsStep(
importer_type=ids.TYPE_ID_IMPORTER)
self.add_child(self.step_local_units)
self.add_child(
CreateRequestsUnitsToDownload(
constants.SYNC_STEP_UNITS_DOWNLOAD_REQUESTS))
self.step_download_units = publish_step.DownloadStep(
constants.SYNC_STEP_UNITS_DOWNLOAD,
plugin_type=ids.TYPE_ID_IMPORTER,
description=_('Retrieving units'))
self.add_child(self.step_download_units)
self.add_child(SaveDownloadedUnits(constants.SYNC_STEP_SAVE))
# metadata
self.add_child(SaveMetadataStep(constants.SYNC_STEP_SAVE_META))