def test__process_block(self):
mock_downloader = Mock()
mock_downloads = ['fake', 'downloads']
dlstep = DownloadStep('fake-step', downloads=mock_downloads)
dlstep.downloader = mock_downloader
dlstep._process_block()
mock_downloader.download.assert_called_once_with(['fake', 'downloads'])
def setUp(self):
conf_dict = {
importer_constants.KEY_FEED: 'http://fake.com/file_feed/',
importer_constants.KEY_MAX_SPEED: 500.0,
importer_constants.KEY_MAX_DOWNLOADS: 5,
importer_constants.KEY_SSL_VALIDATION: False,
importer_constants.KEY_SSL_CLIENT_CERT: "Trust me, I'm who I say I am.",
importer_constants.KEY_SSL_CLIENT_KEY: "Secret Key",
importer_constants.KEY_SSL_CA_CERT: "Uh, I guess that's the right server.",
importer_constants.KEY_PROXY_HOST: 'proxy.com',
importer_constants.KEY_PROXY_PORT: 1234,
importer_constants.KEY_PROXY_USER: "******",
importer_constants.KEY_PROXY_PASS: '******',
importer_constants.KEY_VALIDATE: False,
}
self.real_config = self.get_basic_config(**conf_dict)
self.real_conduit = self.get_sync_conduit()
self.mock_repo = Mock()
self.mock_conduit = Mock()
self.mock_config = Mock()
self.mock_working_dir = Mock()
self.dlstep = DownloadStep("fake_download", repo=self.mock_repo, conduit=self.mock_conduit,
config=self.mock_config, working_dir=self.mock_working_dir,
plugin_type="fake plugin", description='foo')
def test_download_failed(self):
dlstep = DownloadStep('fake-step')
mock_report = Mock()
mock_report_progress = Mock()
dlstep.report_progress = mock_report_progress
dlstep.download_failed(mock_report)
self.assertEquals(dlstep.progress_failures, 1)
# assert report_progress was called with no args
mock_report_progress.assert_called_once_with()
def test_downloads_property(self):
generator = (DownloadRequest(url, '/a/b/c') for url in ['http://pulpproject.org'])
dlstep = DownloadStep('fake-step', downloads=generator)
downloads = dlstep.downloads
self.assertTrue(isinstance(downloads, list))
self.assertEqual(len(downloads), 1)
self.assertTrue(isinstance(downloads[0], DownloadRequest))
def test_cancel(self):
dlstep = DownloadStep('fake-step')
dlstep.parent = MagicMock()
dlstep.initialize()
dlstep.cancel()
self.assertTrue(dlstep.downloader.is_canceled)
def __init__(self, repo=None, conduit=None, config=None, working_dir=None):
"""
:param repo: repository to sync
:type repo: pulp.plugins.model.Repository
:param conduit: sync conduit to use
:type conduit: pulp.plugins.conduits.repo_sync.RepoSyncConduit
:param config: config object for the sync
:type config: pulp.plugins.config.PluginCallConfiguration
:param working_dir: full path to the directory in which transient files
should be stored before being moved into long-term
storage. This should be deleted by the caller after
step processing is complete.
:type working_dir: basestring
"""
super(SyncStep,
self).__init__(constants.SYNC_STEP_MAIN, repo, conduit, config,
working_dir, constants.IMPORTER_TYPE_ID)
self.description = _('Syncing Docker Repository')
# Unit keys, populated by GetMetadataStep
self.available_units = []
# populated by GetMetadataStep
self.tags = {}
self.validate(config)
download_config = nectar_config.importer_config_to_nectar_config(
config.flatten())
upstream_name = config.get(constants.CONFIG_KEY_UPSTREAM_NAME)
url = config.get(importer_constants.KEY_FEED)
# create a Repository object to interact with
self.index_repository = Repository(upstream_name, download_config, url,
working_dir)
self.add_child(GetMetadataStep(working_dir=working_dir))
# save this step so its "units_to_download" attribute can be accessed later
self.step_get_local_units = GetLocalImagesStep(
constants.IMPORTER_TYPE_ID, constants.IMAGE_TYPE_ID, ['image_id'],
working_dir)
self.add_child(self.step_get_local_units)
self.add_child(
DownloadStep(constants.SYNC_STEP_DOWNLOAD,
downloads=self.generate_download_requests(),
repo=repo,
config=config,
working_dir=working_dir,
description=_('Downloading remote files')))
self.add_child(SaveUnits(working_dir))
def __init__(self, **kwargs):
"""
:param repo: repository to sync
:type repo: pulp.plugins.model.Repository
:param conduit: sync conduit to use
:type conduit: pulp.plugins.conduits.repo_sync.RepoSyncConduit
:param config: config object for the sync
:type config: pulp.plugins.config.PluginCallConfiguration
:param working_dir: full path to the directory in which transient files
should be stored before being moved into long-term
storage. This should be deleted by the caller after
step processing is complete.
:type working_dir: basestring
"""
super(SyncStep,
self).__init__(constants.IMPORT_STEP_MAIN,
plugin_type=constants.WEB_IMPORTER_TYPE_ID,
**kwargs)
self.description = _('Syncing Repository')
# Unit keys, populated by GetMetadataStep
self.available_units = []
# config = self.get_config()
working_dir = self.get_working_dir()
self.deb_data = {}
# repo = self.get_repo()
# create a Repository object to interact with
self.add_child(GetMetadataStep())
self.step_get_local_units = GetLocalUnitsStepDeb()
self.add_child(self.step_get_local_units)
self.add_child(
DownloadStep(constants.SYNC_STEP_DOWNLOAD,
downloads=self.generate_download_requests(),
repo=kwargs["repo"],
config=kwargs["config"],
working_dir=kwargs["working_dir"],
description=_('Downloading remote files')))
self.add_child(SaveUnits(working_dir))
class DownloadStepTests(unittest.TestCase):
TYPE_ID_FOO = 'foo'
def get_basic_config(*arg, **kwargs):
plugin_config = {"num_retries": 0, "retry_delay": 0}
repo_plugin_config = {}
for key in kwargs:
repo_plugin_config[key] = kwargs[key]
config = PluginCallConfiguration(plugin_config,
repo_plugin_config=repo_plugin_config)
return config
def get_sync_conduit(type_id=None, existing_units=None, pkg_dir=None):
def build_failure_report(summary, details):
return SyncReport(False, sync_conduit._added_count, sync_conduit._updated_count,
sync_conduit._removed_count, summary, details)
def build_success_report(summary, details):
return SyncReport(True, sync_conduit._added_count, sync_conduit._updated_count,
sync_conduit._removed_count, summary, details)
def side_effect(type_id, key, metadata, rel_path):
if rel_path and pkg_dir:
rel_path = os.path.join(pkg_dir, rel_path)
if not os.path.exists(os.path.dirname(rel_path)):
os.makedirs(os.path.dirname(rel_path))
unit = Unit(type_id, key, metadata, rel_path)
return unit
def get_units(criteria=None):
ret_val = []
if existing_units:
for u in existing_units:
if criteria:
if u.type_id in criteria.type_ids:
ret_val.append(u)
else:
ret_val.append(u)
return ret_val
def search_all_units(type_id, criteria):
ret_val = []
if existing_units:
for u in existing_units:
if u.type_id == type_id:
if u.unit_key['id'] == criteria['filters']['id']:
ret_val.append(u)
return ret_val
sync_conduit = Mock(spec=RepoSyncConduit)
sync_conduit._added_count = sync_conduit._updated_count = sync_conduit._removed_count = 0
sync_conduit.init_unit.side_effect = side_effect
sync_conduit.get_units.side_effect = get_units
sync_conduit.save_unit = Mock()
sync_conduit.search_all_units.side_effect = search_all_units
sync_conduit.build_failure_report = MagicMock(side_effect=build_failure_report)
sync_conduit.build_success_report = MagicMock(side_effect=build_success_report)
sync_conduit.set_progress = MagicMock()
return sync_conduit
def setUp(self):
conf_dict = {
importer_constants.KEY_FEED: 'http://fake.com/file_feed/',
importer_constants.KEY_MAX_SPEED: 500.0,
importer_constants.KEY_MAX_DOWNLOADS: 5,
importer_constants.KEY_SSL_VALIDATION: False,
importer_constants.KEY_SSL_CLIENT_CERT: "Trust me, I'm who I say I am.",
importer_constants.KEY_SSL_CLIENT_KEY: "Secret Key",
importer_constants.KEY_SSL_CA_CERT: "Uh, I guess that's the right server.",
importer_constants.KEY_PROXY_HOST: 'proxy.com',
importer_constants.KEY_PROXY_PORT: 1234,
importer_constants.KEY_PROXY_USER: "******",
importer_constants.KEY_PROXY_PASS: '******',
importer_constants.KEY_VALIDATE: False,
}
self.real_config = self.get_basic_config(**conf_dict)
self.real_conduit = self.get_sync_conduit()
self.mock_repo = Mock()
self.mock_conduit = Mock()
self.mock_config = Mock()
self.mock_working_dir = Mock()
self.dlstep = DownloadStep("fake_download", repo=self.mock_repo, conduit=self.mock_conduit,
config=self.mock_config, working_dir=self.mock_working_dir,
plugin_type="fake plugin", description='foo')
def test_init(self):
self.assertEquals(self.dlstep.get_repo(), self.mock_repo)
self.assertEquals(self.dlstep.get_conduit(), self.mock_conduit)
self.assertEquals(self.dlstep.get_config(), self.mock_config)
self.assertEquals(self.dlstep.get_working_dir(), self.mock_working_dir)
self.assertEquals(self.dlstep.get_plugin_type(), "fake plugin")
self.assertEqual(self.dlstep.description, 'foo')
def test_initalize(self):
# override mock config with real config dict
self.dlstep.config = self.real_config
self.dlstep.conduit = self.get_sync_conduit()
self.dlstep.initialize()
# Now let's assert that all the right things happened during initialization
self.assertEqual(self.dlstep._repo_url, 'http://fake.com/file_feed/')
# Validation of downloads should be disabled by default
self.assertEqual(self.dlstep._validate_downloads, False)
# Inspect the downloader
downloader = self.dlstep.downloader
# The dlstep should be the event listener for the downloader
self.assertEqual(downloader.event_listener, self.dlstep)
# Inspect the downloader config
expected_downloader_config = {
'max_speed': 500.0,
'max_concurrent': 5,
'ssl_client_cert': "Trust me, I'm who I say I am.",
'ssl_client_key': 'Secret Key',
'ssl_ca_cert': "Uh, I guess that's the right server.",
'ssl_validation': False,
'proxy_url': 'proxy.com',
'proxy_port': 1234,
'proxy_username': '******',
'proxy_password': '******'}
for key, value in expected_downloader_config.items():
self.assertEquals(getattr(downloader.config, key), value)
def test__init___with_feed_lacking_trailing_slash(self):
"""
tests https://bugzilla.redhat.com/show_bug.cgi?id=949004
"""
slash_config = self.get_basic_config(
**{importer_constants.KEY_FEED: 'http://fake.com/no_trailing_slash'})
# override mock config with real config dict
self.dlstep.config = slash_config
self.dlstep.initialize()
# Humorously enough, the _repo_url attribute named no_trailing_slash
# should now have a trailing slash
self.assertEqual(self.dlstep._repo_url, 'http://fake.com/no_trailing_slash/')
def test__init___file_downloader(self):
slash_config = self.get_basic_config(
**{importer_constants.KEY_FEED: 'file:///some/path/'})
# override mock config with real config dict
self.dlstep.config = slash_config
self.dlstep.initialize()
self.assertTrue(isinstance(self.dlstep.downloader, LocalFileDownloader))
def test__init___ssl_validation(self):
"""
Make sure the SSL validation is on by default.
"""
# It should default to True
self.dlstep.config = self.get_basic_config(
**{importer_constants.KEY_FEED: 'http://fake.com/iso_feed/'})
self.dlstep.initialize()
self.assertEqual(self.dlstep.downloader.config.ssl_validation, True)
# It should be possible to explicitly set it to False
self.dlstep.config = self.get_basic_config(
**{importer_constants.KEY_FEED: 'http://fake.com/iso_feed/',
importer_constants.KEY_SSL_VALIDATION: False})
self.dlstep.initialize()
self.assertEqual(self.dlstep.downloader.config.ssl_validation, False)
# It should be possible to explicitly set it to True
self.dlstep.config = self.get_basic_config(
**{importer_constants.KEY_FEED: 'http://fake.com/iso_feed/',
importer_constants.KEY_SSL_VALIDATION: True})
self.dlstep.initialize()
self.assertEqual(self.dlstep.downloader.config.ssl_validation, True)
def test__get_total(self):
mock_downloads = ['fake', 'downloads']
dlstep = DownloadStep('fake-step', downloads=mock_downloads)
self.assertEquals(dlstep._get_total(), 2)
def test__process_block(self):
mock_downloader = Mock()
mock_downloads = ['fake', 'downloads']
dlstep = DownloadStep('fake-step', downloads=mock_downloads)
dlstep.downloader = mock_downloader
dlstep._process_block()
mock_downloader.download.assert_called_once_with(['fake', 'downloads'])
def test_download_succeeded(self):
dlstep = DownloadStep('fake-step')
mock_report = Mock()
mock_report_progress = Mock()
dlstep.report_progress = mock_report_progress
dlstep.download_succeeded(mock_report)
self.assertEquals(dlstep.progress_successes, 1)
# assert report_progress was called with no args
mock_report_progress.assert_called_once_with()
def test_download_failed(self):
dlstep = DownloadStep('fake-step')
mock_report = Mock()
mock_report_progress = Mock()
dlstep.report_progress = mock_report_progress
dlstep.download_failed(mock_report)
self.assertEquals(dlstep.progress_failures, 1)
# assert report_progress was called with no args
mock_report_progress.assert_called_once_with()
def test_downloads_property(self):
generator = (DownloadRequest(url, '/a/b/c') for url in ['http://pulpproject.org'])
dlstep = DownloadStep('fake-step', downloads=generator)
downloads = dlstep.downloads
self.assertTrue(isinstance(downloads, list))
self.assertEqual(len(downloads), 1)
self.assertTrue(isinstance(downloads[0], DownloadRequest))
def test_cancel(self):
dlstep = DownloadStep('fake-step')
dlstep.parent = MagicMock()
dlstep.initialize()
dlstep.cancel()
self.assertTrue(dlstep.downloader.is_canceled)
def test__get_total(self):
mock_downloads = ['fake', 'downloads']
dlstep = DownloadStep('fake-step', downloads=mock_downloads)
self.assertEquals(dlstep._get_total(), 2)
