def __init__(self, repo, conduit, config): """ :param repo: repository to sync :type repo: pulp.plugins.model.Repository :param conduit: sync conduit to use :type conduit: pulp.plugins.conduits.repo_sync.RepoSyncConduit :param config: config object for the sync :type config: pulp.plugins.config.PluginCallConfiguration """ super(RepoSync, self).__init__(step_type=constants.SYNC_STEP, repo=repo, conduit=conduit, config=config) self.description = _('Syncing Repository') self.apt_repo_meta = None # https://pulp.plan.io/issues/2765 should remove the need to hardcode # the dist/component here self.feed_url = self.get_config().get('feed').strip( '/') + '/dists/stable/' self.release_file = os.path.join(self.get_working_dir(), "Release") self.available_units = None rel_url = urlparse.urljoin(self.feed_url, 'Release') _logger.info("Downloading %s", rel_url) self.add_child( publish_step.DownloadStep( constants.SYNC_STEP_RELEASE_DOWNLOAD, plugin_type=ids.TYPE_ID_IMPORTER, description=_('Retrieving metadata: release file'), downloads=[DownloadRequest(rel_url, self.release_file)])) self.add_child(ParseReleaseStep(constants.SYNC_STEP_RELEASE_PARSE)) self.step_download_Packages = publish_step.DownloadStep( constants.SYNC_STEP_PACKAGES_DOWNLOAD, plugin_type=ids.TYPE_ID_IMPORTER, description=_('Retrieving metadata: Packages files')) self.add_child(self.step_download_Packages) self.add_child(ParsePackagesStep(constants.SYNC_STEP_PACKAGES_PARSE)) self.step_local_units = publish_step.GetLocalUnitsStep( importer_type=ids.TYPE_ID_IMPORTER) self.add_child(self.step_local_units) self.add_child( CreateRequestsUnitsToDownload( constants.SYNC_STEP_UNITS_DOWNLOAD_REQUESTS)) self.step_download_units = publish_step.DownloadStep( constants.SYNC_STEP_UNITS_DOWNLOAD, plugin_type=ids.TYPE_ID_IMPORTER, description=_('Retrieving units')) self.add_child(self.step_download_units) self.add_child(SaveDownloadedUnits(constants.SYNC_STEP_SAVE))
def test_process_main(self): mock_downloader = Mock() mock_downloads = ['fake', 'downloads'] dlstep = publish_step.DownloadStep('fake-step', downloads=mock_downloads) dlstep.downloader = mock_downloader dlstep.process_main() mock_downloader.download.assert_called_once_with(['fake', 'downloads'])
def setUp(self): conf_dict = { importer_constants.KEY_FEED: 'http://fake.com/file_feed/', importer_constants.KEY_MAX_SPEED: 500.0, importer_constants.KEY_MAX_DOWNLOADS: 5, importer_constants.KEY_SSL_VALIDATION: False, importer_constants.KEY_SSL_CLIENT_CERT: "Trust me, I'm who I say I am.", importer_constants.KEY_SSL_CLIENT_KEY: "Secret Key", importer_constants.KEY_SSL_CA_CERT: "Uh, I guess that's the right server.", importer_constants.KEY_PROXY_HOST: 'proxy.com', importer_constants.KEY_PROXY_PORT: 1234, importer_constants.KEY_PROXY_USER: "******", importer_constants.KEY_PROXY_PASS: '******', importer_constants.KEY_VALIDATE: False, } self.real_config = self.get_basic_config(**conf_dict) self.real_conduit = self.get_sync_conduit() self.mock_repo = Mock() self.mock_conduit = Mock() self.mock_config = Mock() self.mock_working_dir = Mock() self.dlstep = publish_step.DownloadStep( "fake_download", repo=self.mock_repo, conduit=self.mock_conduit, config=self.mock_config, working_dir=self.mock_working_dir, plugin_type="fake plugin", description='foo')
def test_cancel(self): dlstep = publish_step.DownloadStep('fake-step') dlstep.parent = MagicMock() dlstep.initialize() dlstep.cancel() self.assertTrue(dlstep.downloader.is_canceled)
def test_download_succeeded(self): dlstep = publish_step.DownloadStep('fake-step') mock_report = Mock() mock_report_progress = Mock() dlstep.report_progress = mock_report_progress dlstep.download_succeeded(mock_report) self.assertEquals(dlstep.progress_successes, 1) # assert report_progress was called with no args mock_report_progress.assert_called_once_with()
def test_downloads_property(self): generator = (DownloadRequest(url, '/a/b/c') for url in ['http://pulpproject.org']) dlstep = publish_step.DownloadStep('fake-step', downloads=generator) downloads = dlstep.downloads self.assertTrue(isinstance(downloads, list)) self.assertEqual(len(downloads), 1) self.assertTrue(isinstance(downloads[0], DownloadRequest))
def add_v1_steps(self, repo, config): """ Add v1 sync steps. :param repo: repository to sync :type repo: pulp.plugins.model.Repository :param config: config object for the sync :type config: pulp.plugins.config.PluginCallConfiguration """ self.add_child(v1_sync.GetMetadataStep()) # save this step so its "units_to_download" attribute can be accessed later self.v1_step_get_local_units = publish_step.GetLocalUnitsStep( constants.IMPORTER_TYPE_ID, available_units=self.v1_available_units) self.v1_step_get_local_units.step_id = constants.SYNC_STEP_GET_LOCAL_V1 self.add_child(self.v1_step_get_local_units) self.add_child(publish_step.DownloadStep( constants.SYNC_STEP_DOWNLOAD_V1, downloads=self.v1_generate_download_requests(), repo=repo, config=config, description=_('Downloading remote files'))) self.add_child(v1_sync.SaveImages())
def test_get_total(self): mock_downloads = ['fake', 'downloads'] dlstep = publish_step.DownloadStep('fake-step', downloads=mock_downloads) self.assertEquals(dlstep.get_total(), 2)
def __init__(self, repo, conduit, config): """ :param repo: repository to sync :type repo: pulp.plugins.model.Repository :param conduit: sync conduit to use :type conduit: pulp.plugins.conduits.repo_sync.RepoSyncConduit :param config: config object for the sync :type config: pulp.plugins.config.PluginCallConfiguration """ super(RepoSync, self).__init__(step_type=constants.SYNC_STEP, repo=repo, conduit=conduit, config=config) self.description = _('Syncing Repository') self.feed_url = self.get_config().get('feed').strip('/') self.releases = self.get_config().get('releases', 'stable').split(',') self.architectures = split_or_none( self.get_config().get('architectures')) self.components = split_or_none(self.get_config().get('components')) self.unit_relative_urls = {} self.available_units = None # dicts with release names as keys to multiplex variables self.apt_repo_meta = {} self.release_units = {} self.release_files = { release: os.path.join(self.get_working_dir(), release, 'Release') for release in self.releases } self.feed_urls = { release: urlparse.urljoin(self.feed_url + '/', '/'.join(['dists', release])) for release in self.releases } self.release_urls = { release: urlparse.urljoin(self.feed_urls[release] + '/', 'Release') for release in self.releases } self.packages_urls = {} # double dicts with release/component as keys self.component_units = defaultdict(dict) self.component_packages = defaultdict(dict) for release in self.releases: misc.mkdir(os.path.dirname(self.release_files[release])) _logger.info("Downloading %s", self.release_urls[release]) # defining lifecycle # metadata self.add_child( publish_step.DownloadStep( constants.SYNC_STEP_RELEASE_DOWNLOAD, plugin_type=ids.TYPE_ID_IMPORTER, description=_('Retrieving metadata: release file(s)'), downloads=[ DownloadRequest(self.release_urls[release], self.release_files[release]) for release in self.releases ])) self.add_child(ParseReleaseStep(constants.SYNC_STEP_RELEASE_PARSE)) self.step_download_Packages = publish_step.DownloadStep( constants.SYNC_STEP_PACKAGES_DOWNLOAD, plugin_type=ids.TYPE_ID_IMPORTER, description=_('Retrieving metadata: Packages files')) self.add_child(self.step_download_Packages) self.add_child(ParsePackagesStep(constants.SYNC_STEP_PACKAGES_PARSE)) # packages self.step_local_units = publish_step.GetLocalUnitsStep( importer_type=ids.TYPE_ID_IMPORTER) self.add_child(self.step_local_units) self.add_child( CreateRequestsUnitsToDownload( constants.SYNC_STEP_UNITS_DOWNLOAD_REQUESTS)) self.step_download_units = publish_step.DownloadStep( constants.SYNC_STEP_UNITS_DOWNLOAD, plugin_type=ids.TYPE_ID_IMPORTER, description=_('Retrieving units')) self.add_child(self.step_download_units) self.add_child(SaveDownloadedUnits(constants.SYNC_STEP_SAVE)) # metadata self.add_child(SaveMetadataStep(constants.SYNC_STEP_SAVE_META))