def test_fetch_manifest_failed(self, mock_download):
feed_url = 'http://host/root/'
conduit = Mock()
config = {constants.CONFIG_FEED: feed_url}
failed_report = Mock()
failed_report.error_msg = 'just up and failed'
mock_download.return_value = [], [failed_report]
# test
method = SynchronizeWithDirectory(conduit, config)
method.report = Mock()
manifest = method._fetch_manifest()
# validation
mock_download.assert_called_with([
(urljoin(feed_url, constants.MANIFEST_FILENAME), ANY)
])
self.assertTrue(manifest is None)
self.assertTrue(method.report.update_progress.called)
self.assertEqual(method.report.metadata_state, constants.STATE_FAILED)
self.assertEqual(method.report.metadata_error_message,
failed_report.error_msg)
self.assertTrue(method.report.metadata_execution_time > 0)
def test_fetch_manifest(self, mock_download, mock_get_value):
feed_url = 'http://host/root/'
mock_repo = Mock()
conduit = Mock()
config = {constants.CONFIG_FEED: feed_url}
succeeded_report = Mock()
mock_download.return_value = [succeeded_report], []
mock_get_value.return_value = 'A,B,C\nD,E,F\n'
# test
method = SynchronizeWithDirectory(mock_repo, conduit, config)
method.report = Mock()
manifest = method._fetch_manifest()
# validation
mock_download.assert_called_with([(urljoin(feed_url, constants.MANIFEST_FILENAME), ANY)])
self.assertEqual(manifest, [('A', 'B', 'C'), ('D', 'E', 'F')])
self.assertTrue(method.report.update_progress.called)
self.assertEqual(method.report.metadata_state, constants.STATE_SUCCESS)
self.assertEqual(method.report.metadata_query_finished_count, 1)
self.assertEqual(method.report.metadata_query_total_count, 1)
self.assertEqual(method.report.metadata_current_query, None)
self.assertTrue(method.report.metadata_execution_time > 0)
def test_import_modules_failed(self):
"""
Test that when there was some failure in a previous step, _import_modules does not
overwrite the failed state
"""
config = {}
mock_conduit = Mock()
mock_inventory = Mock()
method = SynchronizeWithDirectory(mock_conduit, config)
method.started_fetch_modules = 0
method._extract_metadata = Mock(return_value={'name': 'j-p', 'author': 'J', 'version': '1.1'})
method.report = Mock()
method.report.modules_total_count = 1
method.report.modules_finished_count = 0
method.report.modules_state = constants.STATE_FAILED
# test
imported_modules = method._import_modules(mock_inventory, ['/path1'])
# validation
self.assertEquals(constants.STATE_FAILED, method.report.modules_state)
self.assertEquals(1, method.report.update_progress.call_count)
self.assertEquals(0, method.report.modules_total_count)
self.assertEquals(0, method.report.modules_finished_count)
self.assertEquals([], imported_modules)
def test_add_module(self, mock_shutil):
module_path = '/tmp/mod.tar.gz'
feed_url = 'http://host/root/PULP_MANAFEST'
unit_key = {'name': 'puppet-module'}
unit_metadata = {'A': 1, 'B': 2}
unit = Mock()
unit.storage_path = '/tmp/%s' % uuid4()
mock_conduit = Mock()
mock_conduit.init_unit = Mock(return_value=unit)
config = {constants.CONFIG_FEED: feed_url}
mock_module = Mock()
mock_module.unit_key = Mock(return_value=unit_key)
mock_module.unit_metadata = Mock(return_value=unit_metadata)
mock_module.filename = Mock(return_value='puppet-module')
# test
method = SynchronizeWithDirectory(mock_conduit, config)
method._add_module(module_path, mock_module)
# validation
mock_conduit.init_unit.assert_called_with(constants.TYPE_PUPPET_MODULE,
unit_key, unit_metadata,
mock_module.filename())
mock_shutil.copy.assert_called_with(module_path, unit.storage_path)
def test_fetch_manifest_failed(self, mock_download):
feed_url = 'http://host/root/'
mock_repo = Mock()
conduit = Mock()
config = {constants.CONFIG_FEED: feed_url}
failed_report = Mock()
failed_report.error_msg = 'just up and failed'
mock_download.return_value = [], [failed_report]
# test
method = SynchronizeWithDirectory(mock_repo, conduit, config)
method.report = Mock()
manifest = method._fetch_manifest()
# validation
mock_download.assert_called_with([(urljoin(feed_url, constants.MANIFEST_FILENAME), ANY)])
self.assertTrue(manifest is None)
self.assertTrue(method.report.update_progress.called)
self.assertEqual(method.report.metadata_state, constants.STATE_FAILED)
self.assertEqual(method.report.metadata_error_message, failed_report.error_msg)
self.assertTrue(method.report.metadata_execution_time > 0)
def test_add_module_not_copied(self, mock_shutil):
module_path = '/tmp/mod.tar.gz'
feed_url = 'http://host/root/PULP_MANAFEST'
unit_key = {'name': 'puppet-module'}
unit_metadata = {'A': 1, 'B': 2}
unit = Mock()
unit.storage_path = os.path.join(os.getcwd(), __file__)
mock_conduit = Mock()
mock_conduit.init_unit = Mock(return_value=unit)
config = {constants.CONFIG_FEED: feed_url}
mock_module = Mock()
mock_module.unit_key = Mock(return_value=unit_key)
mock_module.unit_metadata = Mock(return_value=unit_metadata)
mock_module.filename = Mock(return_value='puppet-module')
# test
method = SynchronizeWithDirectory(mock_conduit, config)
method._add_module(module_path, mock_module)
# validation
self.assertFalse(mock_shutil.copy.called)
def test_fetch_manifest(self, mock_download, mock_get_value):
feed_url = 'http://host/root/'
conduit = Mock()
config = {constants.CONFIG_FEED: feed_url}
succeeded_report = Mock()
mock_download.return_value = [succeeded_report], []
mock_get_value.return_value = 'A,B,C\nD,E,F\n'
# test
method = SynchronizeWithDirectory(conduit, config)
method.report = Mock()
manifest = method._fetch_manifest()
# validation
mock_download.assert_called_with([
(urljoin(feed_url, constants.MANIFEST_FILENAME), ANY)
])
self.assertEqual(manifest, [('A', 'B', 'C'), ('D', 'E', 'F')])
self.assertTrue(method.report.update_progress.called)
self.assertEqual(method.report.metadata_state, constants.STATE_SUCCESS)
self.assertEqual(method.report.metadata_query_finished_count, 1)
self.assertEqual(method.report.metadata_query_total_count, 1)
self.assertEqual(method.report.metadata_current_query, None)
self.assertTrue(method.report.metadata_execution_time > 0)
def test_add_module_not_copied(self, mock_shutil):
module_path = '/tmp/mod.tar.gz'
feed_url = 'http://host/root/PULP_MANAFEST'
unit_key = {'name': 'puppet-module'}
unit_metadata = {'A': 1, 'B': 2}
unit = Mock()
unit.storage_path = os.path.join(os.getcwd(), __file__)
mock_conduit = Mock()
mock_conduit.init_unit = Mock(return_value=unit)
config = {constants.CONFIG_FEED: feed_url}
mock_module = Mock()
mock_module.unit_key = Mock(return_value=unit_key)
mock_module.unit_metadata = Mock(return_value=unit_metadata)
mock_module.filename = Mock(return_value='puppet-module')
# test
method = SynchronizeWithDirectory(mock_conduit, config)
method._add_module(module_path, mock_module)
# validation
self.assertFalse(mock_shutil.copy.called)
def test_import_modules(self, mock_extract, mock_add, mock_remove_missing):
# These manifests represent the parsed metadata.json file. These contain a 'name'
# field, where we retrieve both the unit key's 'name' and 'author' field.
manifest = [{'name': 'john-pulp1', 'author': 'Johnathon', 'version': '1.0'}]
mock_extract.side_effect = manifest
module_paths = ['/tmp/module_1']
mock_pulp1, mock_pulp2 = (Mock(), Mock())
mock_pulp1.unit_key = {'name': 'pulp1', 'author': 'john', 'version': '1.0'}
mock_pulp2.unit_key = {'name': 'pulp2', 'author': 'john', 'version': '2.0'}
conduit = Mock()
conduit.get_units.return_value = [mock_pulp1, mock_pulp2]
config = Mock()
config.get_boolean.return_value = True
# test
method = SynchronizeWithDirectory(conduit, config)
method.started_fetch_modules = 10
method.report = Mock()
method.report.modules_total_count = 2
method.report.modules_finished_count = 0
method._import_modules(module_paths)
# validation
config.get_boolean.assert_called_once_with(constants.CONFIG_REMOVE_MISSING)
mock_remove_missing.assert_called_once_with([mock_pulp1, mock_pulp2], [mock_pulp1.unit_key])
def test_add_module(self, mock_shutil):
module_path = '/tmp/mod.tar.gz'
feed_url = 'http://host/root/PULP_MANAFEST'
unit_key = {'name': 'puppet-module'}
unit_metadata = {'A': 1, 'B': 2}
unit = Mock()
unit.storage_path = '/tmp/%s' % uuid4()
mock_conduit = Mock()
mock_conduit.init_unit = Mock(return_value=unit)
config = {constants.CONFIG_FEED: feed_url}
mock_module = Mock()
mock_module.unit_key = Mock(return_value=unit_key)
mock_module.unit_metadata = Mock(return_value=unit_metadata)
mock_module.filename = Mock(return_value='puppet-module')
# test
method = SynchronizeWithDirectory(mock_conduit, config)
method._add_module(module_path, mock_module)
# validation
mock_conduit.init_unit.assert_called_with(
constants.TYPE_PUPPET_MODULE, unit_key, unit_metadata, mock_module.filename())
mock_shutil.copy.assert_called_with(module_path, unit.storage_path)
def test_remove_missing(self):
"""
Test that when there are units to remove, the conduit is called correctly.
"""
mock_unit = Mock()
mock_conduit = Mock()
method = SynchronizeWithDirectory(mock_conduit, {})
method._remove_missing([mock_unit], [])
mock_conduit.remove_unit.assert_called_once_with(mock_unit)
def test_remove_missing(self):
"""
Test that when there are units to remove, the conduit is called correctly.
"""
mock_unit = Mock()
mock_conduit = Mock()
method = SynchronizeWithDirectory(mock_conduit, {})
method._remove_missing([mock_unit], [])
mock_conduit.remove_unit.assert_called_once_with(mock_unit)
def test_remove_missing_canceled(self):
"""
Test that when the sync is canceled, no units are removed.
"""
mock_unit = Mock()
mock_conduit = Mock()
method = SynchronizeWithDirectory(mock_conduit, {})
method.canceled = True
method._remove_missing([mock_unit], [])
self.assertEqual(0, mock_conduit.remove_unit.call_count)
def test_remove_missing_canceled(self):
"""
Test that when the sync is canceled, no units are removed.
"""
mock_unit = Mock()
mock_conduit = Mock()
method = SynchronizeWithDirectory(mock_conduit, {})
method.canceled = True
method._remove_missing([mock_unit], [])
self.assertEqual(0, mock_conduit.remove_unit.call_count)
def test_feed_url(self):
feed_url = 'http://abc.com/repository'
conduit = Mock()
config = {constants.CONFIG_FEED: feed_url}
# testing
method = SynchronizeWithDirectory(conduit, config)
# validation
self.assertEqual(method.feed_url(), feed_url + '/')
def test_cancel(self):
conduit = Mock()
config = {}
# testing
method = SynchronizeWithDirectory(conduit, config)
method.cancel()
# validation
self.assertTrue(method.canceled)
def test_import_modules_cancelled(self, mock_extract):
config = {}
mock_conduit = Mock()
mock_conduit.get_units.return_value = []
# test
method = SynchronizeWithDirectory(mock_conduit, config)
method.canceled = True
method._import_modules(['/path1', '/path2'])
# validation
self.assertFalse(mock_extract.called)
def test_import_modules_cancelled(self, mock_extract):
config = {}
mock_conduit = Mock()
mock_conduit.get_units.return_value = []
# test
method = SynchronizeWithDirectory(mock_conduit, config)
method.canceled = True
method._import_modules(['/path1', '/path2'])
# validation
self.assertFalse(mock_extract.called)
def test_cancel(self):
conduit = Mock()
config = {}
# testing
method = SynchronizeWithDirectory(conduit, config)
method.cancel()
# validation
self.assertTrue(method.canceled)
def test_feed_url(self):
feed_url = 'http://abc.com/repository'
conduit = Mock()
config = {constants.CONFIG_FEED: feed_url}
# testing
method = SynchronizeWithDirectory(conduit, config)
# validation
self.assertEqual(method.feed_url(), feed_url + '/')
def test_download(self, mock_listener, mock_nectar_config, mock_downloader_mapping):
mock_nectar_config.return_value = Mock()
mock_http_downloader = Mock()
mock_http_class = Mock(return_value=mock_http_downloader)
mock_downloader_mapping.__getitem__.return_value = mock_http_class
mock_repo = Mock()
conduit = Mock()
config = Mock()
config.get = Mock(side_effect={constants.CONFIG_FEED: 'http://host/root/PULP_MANAFEST'})
config.flatten = Mock(return_value={})
urls = [
('http://host/root/path_1', '/tmp/path_1'),
('http://host/root/path_2', '/tmp/path_1'),
]
report = namedtuple('Report', ['url', 'destination', 'error_msg'])
_listener = Mock()
_listener.succeeded_reports = [report(urls[0][0], urls[0][1], None)]
_listener.failed_reports = [report(urls[1][0], urls[1][1], 'File Not Found')]
mock_listener.return_value = _listener
# test
method = SynchronizeWithDirectory(mock_repo, conduit, config)
succeeded_reports, failed_reports = method._download(urls)
# validation
method.config.flatten.assert_called_with()
mock_nectar_config.assert_called_with(method.config.flatten())
self.assertTrue(mock_http_downloader.download.called)
self.assertEqual(mock_http_downloader.download.call_count, 1)
self.assertEqual(mock_http_downloader.download.call_args[0][0][0].url, urls[0][0])
self.assertEqual(mock_http_downloader.download.call_args[0][0][0].destination, urls[0][1])
self.assertEqual(mock_http_downloader.download.call_args[0][0][1].url, urls[1][0])
self.assertEqual(mock_http_downloader.download.call_args[0][0][0].destination, urls[1][1])
self.assertTrue(isinstance(succeeded_reports, list))
self.assertEqual(len(succeeded_reports), 1)
self.assertEqual(succeeded_reports[0].url, urls[0][0])
self.assertEqual(succeeded_reports[0].destination, urls[0][1])
self.assertTrue(isinstance(succeeded_reports, list))
self.assertTrue(isinstance(failed_reports, list))
self.assertEqual(len(failed_reports), 1)
self.assertEqual(failed_reports[0].url, urls[1][0])
self.assertEqual(failed_reports[0].destination, urls[1][1])
self.assertTrue(isinstance(failed_reports, list))
def test_import_modules(self, mock_extract, mock_add):
feed_url = 'http://host/root/PULP_MANAFEST'
conduit = Mock()
config = {constants.CONFIG_FEED: feed_url}
mock_inventory = Mock()
mock_inventory.already_associated.side_effect = [False, True, False]
# These manifests represent the parsed metadata.json file. These contain a 'name'
# field, where we retrieve both the unit key's 'name' and 'author' field.
manifests = [
{'name': 'john-pulp1', 'author': 'Johnathon', 'version': '1.0'},
{'name': 'john-pulp2', 'author': 'Johnathon', 'version': '2.0'},
{'name': 'john/pulp3', 'author': 'Johnathon', 'version': '3.0'},
]
mock_extract.side_effect = manifests
unit_keys = [
{'name': 'pulp1', 'author': 'john', 'version': '1.0'},
{'name': 'pulp2', 'author': 'john', 'version': '2.0'},
{'name': 'pulp3', 'author': 'john', 'version': '3.0'},
]
module_paths = [
'/tmp/module_1',
'/tmp/module_2',
'/tmp/module_3',
]
# test
method = SynchronizeWithDirectory(conduit, config)
method.started_fetch_modules = 10
method.report = Mock()
method.report.modules_total_count = 3
method.report.modules_finished_count = 0
imported_modules = method._import_modules(mock_inventory, module_paths)
# validation
mock_add.assert_any_with(module_paths[0], ANY)
mock_add.assert_any_with(module_paths[2], ANY)
# should only be modules 1 and 3. 2 already associated.
self.assertEqual(len(imported_modules), 2)
self.assertEqual(imported_modules[0], unit_keys[0])
self.assertEqual(imported_modules[1], unit_keys[2])
# Check that the progress reporting was called as expected
self.assertEquals(3, method.report.update_progress.call_count)
self.assertEquals(2, method.report.modules_finished_count)
self.assertEquals(2, method.report.modules_total_count)
def test_import_modules(self, mock_extract, mock_add, mock_remove_missing):
# These manifests represent the parsed metadata.json file. These contain a 'name'
# field, where we retrieve both the unit key's 'name' and 'author' field.
manifests = [
{'name': 'john-pulp1', 'author': 'Johnathon', 'version': '1.0'},
{'name': 'john-pulp2', 'author': 'Johnathon', 'version': '2.0'},
{'name': 'john/pulp3', 'author': 'Johnathon', 'version': '3.0'},
]
mock_extract.side_effect = manifests
unit_keys = [
{'name': 'pulp1', 'author': 'john', 'version': '1.0'},
{'name': 'pulp2', 'author': 'john', 'version': '2.0'},
{'name': 'pulp3', 'author': 'john', 'version': '3.0'},
]
module_paths = [
'/tmp/module_1',
'/tmp/module_2',
'/tmp/module_3',
]
mock_pulp2 = Mock()
mock_pulp2.unit_key = unit_keys[1]
conduit = Mock()
conduit.get_units.return_value = [mock_pulp2]
config = Mock()
config.get_boolean.return_value = False
# test
method = SynchronizeWithDirectory(conduit, config)
method.started_fetch_modules = 10
method.report = Mock()
method.report.modules_total_count = 3
method.report.modules_finished_count = 0
method._import_modules(module_paths)
# validation
self.assertEqual(3, mock_extract.call_count)
self.assertEqual(mock_extract.mock_calls[0][1][0], module_paths[0])
self.assertEqual(mock_extract.mock_calls[1][1][0], module_paths[1])
self.assertEqual(mock_extract.mock_calls[2][1][0], module_paths[2])
self.assertEqual(2, mock_add.call_count)
self.assertEqual(mock_add.mock_calls[0][1][0], module_paths[0])
self.assertEqual(mock_add.mock_calls[1][1][0], module_paths[2])
config.get_boolean.assert_called_once_with(constants.CONFIG_REMOVE_MISSING)
self.assertEqual(0, mock_remove_missing.call_count)
# Check that the progress reporting was called as expected
self.assertEquals(3, method.report.update_progress.call_count)
self.assertEquals(2, method.report.modules_finished_count)
self.assertEquals(2, method.report.modules_total_count)
def test_import_modules_cancelled(self, mock_extract):
config = {}
mock_conduit = Mock()
mock_inventory = Mock()
# test
method = SynchronizeWithDirectory(mock_conduit, config)
method.canceled = True
imported_modules = method._import_modules(mock_inventory, ['/path1', '/path2'])
# validation
self.assertFalse(mock_extract.called)
self.assertEqual(imported_modules, [])
def test_import(self, mock_associated):
test_dir = os.path.dirname(__file__)
mock_associated.return_value = set()
unit = Mock()
unit.storage_path = self.tmp_dir
config = PluginCallConfiguration(
{},
{constants.CONFIG_FEED: 'file://%s/../data/simple/' % test_dir,
constants.CONFIG_REMOVE_MISSING: True})
conduit = Mock()
conduit.init_unit = Mock(return_value=unit)
repository = Mock()
repository.working_dir = self.tmp_dir
# test
method = SynchronizeWithDirectory(conduit, config)
method(repository)
# validation
conduit.save_unit.assert_called_once_with(unit)
def test_call(self, mock_mkdtemp, mock_rmtree, mock_fetch_manifest,
mock_fetch_modules, mock_import_modules,
mock_remove_missing):
mock_fetch_manifest.return_value = 'manifest_destiny'
mock_fetch_modules.return_value = 'some modules'
conduit = Mock()
config = {constants.CONFIG_FEED: 'http://host/root/PULP_MANAFEST'}
repository = Mock()
repository.working_dir = 'working'
mock_mkdtemp.return_value = '/abc'
# testing
method = SynchronizeWithDirectory(conduit, config)
report = method(repository)
# validation
self.assertEqual(1, mock_fetch_manifest.call_count)
mock_fetch_modules.assert_called_once_with('manifest_destiny')
mock_import_modules.assert_called_once_with('some modules')
self.assertEqual(0, mock_remove_missing.call_count)
self.assertFalse(method.canceled)
self.assertTrue(isinstance(method.report, SyncProgressReport))
self.assertTrue(isinstance(report, SyncProgressReport))
mock_mkdtemp.assert_called_with(dir=repository.working_dir)
mock_rmtree.assert_called_with(
os.path.join(repository.working_dir, mock_mkdtemp()))
def test_import_modules_failed(self, *mocks):
"""
Test that when there was some failure in a previous step, _import_modules does not
overwrite the failed state
"""
mock_conduit = Mock()
mock_conduit.get_units.return_value = []
method = SynchronizeWithDirectory(mock_conduit, Mock())
method.started_fetch_modules = 0
metadata = {'name': 'j-p', 'author': 'J', 'version': '1.1'}
method._extract_metadata = Mock(return_value=metadata)
method.report = Mock()
method.report.modules_total_count = 1
method.report.modules_finished_count = 0
method.report.modules_state = constants.STATE_FAILED
# test
method._import_modules(['/path1'])
# validation
self.assertEquals(constants.STATE_FAILED, method.report.modules_state)
self.assertEquals(2, method.report.update_progress.call_count)
self.assertEquals(1, method.report.modules_total_count)
self.assertEquals(1, method.report.modules_finished_count)
def test_purge_unwanted_modules_not_requested(self):
mock_conduit = Mock()
config = Mock()
config.get_boolean = Mock(side_effect={constants.CONFIG_REMOVE_MISSING: False}.get)
mock_inventory = Mock()
# test
method = SynchronizeWithDirectory(mock_conduit, config)
method._purge_unwanted_modules(mock_inventory, [])
# validation
self.assertFalse(mock_inventory.unwanted_modules.called)
self.assertFalse(mock_conduit.remove_unit.called)
self.assertFalse(mock_conduit.remove_unit.called)
def test_purge_unwanted_modules_default(self):
mock_conduit = Mock()
config = Mock()
config.get_boolean = Mock(side_effect={}.get)
mock_inventory = Mock()
# test
method = SynchronizeWithDirectory(mock_conduit, config)
method._purge_unwanted_modules(mock_inventory, [])
# validation
self.assertFalse(mock_inventory.unwanted_modules.called)
self.assertFalse(mock_conduit.remove_unit.called)
self.assertFalse(mock_conduit.remove_unit.called)
def test_purge_unwanted_modules_canceled(self):
mock_conduit = Mock()
config = Mock()
config.get_boolean = Mock(side_effect={constants.CONFIG_REMOVE_MISSING: True}.get)
mock_inventory = Mock()
mock_inventory.unwanted_modules = Mock(return_value=['A', 'B'])
# test
method = SynchronizeWithDirectory(mock_conduit, config)
method.canceled = True
method._purge_unwanted_modules(mock_inventory, [])
# validation
self.assertFalse(mock_conduit.remove_unit.called)
self.assertFalse(mock_conduit.remove_unit.called)
def test_fetch_modules_failures(self, mock_download):
tmp_dir = '/tmp/puppet-testing'
feed_url = 'http://host/root/'
mock_repo = Mock()
conduit = Mock()
config = {constants.CONFIG_FEED: feed_url}
manifest = [('path1', 'AA', 10), ('path2', 'BB', 20)]
report_1 = Mock()
report_1.destination = os.path.join(tmp_dir, manifest[0][0])
report_2 = Mock()
report_2.destination = os.path.join(tmp_dir, manifest[1][0])
report_2.error_msg = 'it just dont work'
mock_download.return_value = [report_1], [report_2]
# test
method = SynchronizeWithDirectory(mock_repo, conduit, config)
method.report = Mock()
method.tmp_dir = '/tmp/puppet-testing'
module_paths = method._fetch_modules(manifest)
# validation
url_1 = os.path.join(feed_url, manifest[0][0])
url_2 = os.path.join(feed_url, manifest[1][0])
mock_download.assert_any_with([(url_1, report_1.destination)])
mock_download.assert_any_with([(url_2, report_2.destination)])
self.assertEqual(len(module_paths), 1)
self.assertEqual(module_paths[0], report_1.destination)
self.assertTrue(method.report.update_progress.called)
self.assertEqual(method.report.modules_state, constants.STATE_FAILED)
self.assertEqual(method.report.modules_error_count, 1)
self.assertEqual(len(method.report.modules_individual_errors), 1)
self.assertEqual(method.report.modules_individual_errors[0],
report_2.error_msg)
def test_constructor(self):
conduit = Mock()
config = {}
# testing
method = SynchronizeWithDirectory(conduit, config)
# validation
self.assertEqual(method.conduit, conduit)
self.assertEqual(method.config, config)
def test_fetch_modules_failures(self, mock_download):
tmp_dir = '/tmp/puppet-testing'
feed_url = 'http://host/root/'
mock_repo = Mock()
conduit = Mock()
config = {constants.CONFIG_FEED: feed_url}
manifest = [('path1', 'AA', 10), ('path2', 'BB', 20)]
report_1 = Mock()
report_1.destination = os.path.join(tmp_dir, manifest[0][0])
report_2 = Mock()
report_2.destination = os.path.join(tmp_dir, manifest[1][0])
report_2.error_msg = 'it just dont work'
mock_download.return_value = [report_1], [report_2]
# test
method = SynchronizeWithDirectory(mock_repo, conduit, config)
method.report = Mock()
method.tmp_dir = '/tmp/puppet-testing'
module_paths = method._fetch_modules(manifest)
# validation
url_1 = os.path.join(feed_url, manifest[0][0])
url_2 = os.path.join(feed_url, manifest[1][0])
mock_download.assert_any_with([(url_1, report_1.destination)])
mock_download.assert_any_with([(url_2, report_2.destination)])
self.assertEqual(len(module_paths), 1)
self.assertEqual(module_paths[0], report_1.destination)
self.assertTrue(method.report.update_progress.called)
self.assertEqual(method.report.modules_state, constants.STATE_FAILED)
self.assertEqual(method.report.modules_error_count, 1)
self.assertEqual(len(method.report.modules_individual_errors), 1)
self.assertEqual(method.report.modules_individual_errors[0], report_2.error_msg)
def test_extract_metadata(self, *mocks):
mock_mkdtemp, mock_tarfile, mock_json, mock_shutil, mock_open = mocks
Member = namedtuple('Member', ['name'])
module_path = '/build/modules/puppet-module.tar.gz'
mock_mkdtemp.return_value = '/tmp/xx'
members = [
Member(name='puppet-module'),
Member(name='puppet-module/manifests'),
Member(name='puppet-module/spec'),
Member(name='puppet-module/templates'),
Member(name='puppet-module/tests'),
Member(name='puppet-module/CHANGELOG'),
Member(name='puppet-module/%s' %
constants.MODULE_METADATA_FILENAME),
Member(name='puppet-module/README'),
]
tarball = Mock()
tarball.getmembers = Mock(return_value=members)
mock_tarfile.open = Mock(return_value=tarball)
mock_fp = Mock()
mock_fp.__enter__ = Mock(return_value=mock_fp)
mock_fp.__exit__ = Mock()
mock_open.return_value = mock_fp
mock_json.load.return_value = '12345'
# test
SynchronizeWithDirectory._extract_metadata(module_path)
# validation
mock_mkdtemp.assert_called_with(dir=os.path.dirname(module_path))
mock_tarfile.open.assert_called_with(module_path)
tarball.getmembers.assert_called_with()
tarball.extract.assert_called_with(members[6], mock_mkdtemp())
mock_open.assert_called_with(
os.path.join(mock_mkdtemp(), members[6].name))
mock_json.load.assert_called_with(mock_fp)
mock_shutil.rmtree.assert_called_with(mock_mkdtemp())
def test_fetch_modules(self, mock_download):
tmp_dir = '/tmp/puppet-testing'
feed_url = 'http://host/root/'
mock_repo = Mock()
conduit = Mock()
config = {constants.CONFIG_FEED: feed_url}
manifest = [('path1', 'AA', 10), ('path2', 'BB', 20)]
report_1 = Mock()
report_1.destination = os.path.join(tmp_dir, manifest[0][0])
report_2 = Mock()
report_2.destination = os.path.join(tmp_dir, manifest[1][0])
mock_download.return_value = [report_1, report_2], []
# test
method = SynchronizeWithDirectory(mock_repo, conduit, config)
method.report = Mock()
method.tmp_dir = '/tmp/puppet-testing'
module_paths = method._fetch_modules(manifest)
# validation
url_1 = os.path.join(feed_url, manifest[0][0])
url_2 = os.path.join(feed_url, manifest[1][0])
mock_download.assert_any_with([(url_1, report_1.destination)])
mock_download.assert_any_with([(url_2, report_2.destination)])
self.assertEqual(len(module_paths), 2)
self.assertEqual(module_paths[0], report_1.destination)
self.assertEqual(module_paths[1], report_2.destination)
# Assert the progress report was updated and the report is still in the running state.
# The _import_modules method must be called to complete the task.
self.assertTrue(method.report.update_progress.called)
self.assertEqual(method.report.modules_state, constants.STATE_RUNNING)
def test_fetch_modules(self, mock_download):
tmp_dir = '/tmp/puppet-testing'
feed_url = 'http://host/root/'
mock_repo = Mock()
conduit = Mock()
config = {constants.CONFIG_FEED: feed_url}
manifest = [('path1', 'AA', 10), ('path2', 'BB', 20)]
report_1 = Mock()
report_1.destination = os.path.join(tmp_dir, manifest[0][0])
report_2 = Mock()
report_2.destination = os.path.join(tmp_dir, manifest[1][0])
mock_download.return_value = [report_1, report_2], []
# test
method = SynchronizeWithDirectory(mock_repo, conduit, config)
method.report = Mock()
method.tmp_dir = '/tmp/puppet-testing'
module_paths = method._fetch_modules(manifest)
# validation
url_1 = os.path.join(feed_url, manifest[0][0])
url_2 = os.path.join(feed_url, manifest[1][0])
mock_download.assert_any_with([(url_1, report_1.destination)])
mock_download.assert_any_with([(url_2, report_2.destination)])
self.assertEqual(len(module_paths), 2)
self.assertEqual(module_paths[0], report_1.destination)
self.assertEqual(module_paths[1], report_2.destination)
# Assert the progress report was updated and the report is still in the running state.
# The _import_modules method must be called to complete the task.
self.assertTrue(method.report.update_progress.called)
self.assertEqual(method.report.modules_state, constants.STATE_RUNNING)
def test_extract_metadata(self, *mocks):
mock_mkdtemp, mock_tarfile, mock_json, mock_shutil, mock_open = mocks
Member = namedtuple('Member', ['name'])
module_path = '/build/modules/puppet-module.tar.gz'
mock_mkdtemp.return_value = '/tmp/xx'
members = [
Member(name='puppet-module'),
Member(name='puppet-module/manifests'),
Member(name='puppet-module/spec'),
Member(name='puppet-module/templates'),
Member(name='puppet-module/tests'),
Member(name='puppet-module/CHANGELOG'),
Member(name='puppet-module/%s' % constants.MODULE_METADATA_FILENAME),
Member(name='puppet-module/README'),
]
tarball = Mock()
tarball.getmembers = Mock(return_value=members)
mock_tarfile.open = Mock(return_value=tarball)
mock_fp = Mock()
mock_fp.__enter__ = Mock(return_value=mock_fp)
mock_fp.__exit__ = Mock()
mock_open.return_value = mock_fp
mock_json.load.return_value = '12345'
# test
SynchronizeWithDirectory._extract_metadata(module_path)
# validation
mock_mkdtemp.assert_called_with(dir=os.path.dirname(module_path))
mock_tarfile.open.assert_called_with(module_path)
tarball.getmembers.assert_called_with()
tarball.extract.assert_called_with(members[6], mock_mkdtemp())
mock_open.assert_called_with(os.path.join(mock_mkdtemp(), members[6].name))
mock_json.load.assert_called_with(mock_fp)
mock_shutil.rmtree.assert_called_with(mock_mkdtemp())
def test_purge_unwanted_modules(self):
imported_modules = [{'A': 1}, {'B': 2}]
unwanted_modules = [{'A': 3}, {'B': 4}]
mock_conduit = Mock()
config = Mock()
config.get_boolean = Mock(side_effect={constants.CONFIG_REMOVE_MISSING: True}.get)
mock_inventory = Mock()
mock_inventory.unwanted_modules = Mock(return_value=unwanted_modules)
# test
method = SynchronizeWithDirectory(mock_conduit, config)
method._purge_unwanted_modules(mock_inventory, imported_modules)
# validation
mock_inventory.unwanted_modules.assert_called_with(imported_modules)
mock_conduit.remove_unit.assert_any_with(unwanted_modules[0])
mock_conduit.remove_unit.assert_any_with(unwanted_modules[1])
self.assertEqual(mock_conduit.remove_unit.call_count, 2)
def test_import_modules(self, mock_extract, mock_add):
feed_url = 'http://host/root/PULP_MANAFEST'
conduit = Mock()
config = {constants.CONFIG_FEED: feed_url}
mock_inventory = Mock()
mock_inventory.already_associated.side_effect = [False, True, False]
manifests = [
{'name': 'pulp1', 'author': 'john', 'version': '1.0'},
{'name': 'pulp2', 'author': 'john', 'version': '2.0'},
{'name': 'pulp3', 'author': 'john', 'version': '3.0'},
]
mock_extract.side_effect = manifests
module_paths = [
'/tmp/module_1',
'/tmp/module_2',
'/tmp/module_3',
]
# test
method = SynchronizeWithDirectory(conduit, config)
imported_modules = method._import_modules(mock_inventory, module_paths)
# validation
mock_add.assert_any_with(module_paths[0], ANY)
mock_add.assert_any_with(module_paths[2], ANY)
# should only be modules 1 and 3. 2 already associated.
self.assertEqual(len(imported_modules), 2)
self.assertEqual(imported_modules[0], manifests[0])
self.assertEqual(imported_modules[1], manifests[2])
def test_run_fetch_manifest_failed(self, *mocks):
config = {}
conduit = Mock()
inventory = Mock()
_purge_unwanted_modules = mocks[0]
_import_modules = mocks[1]
_import_modules.return_value = []
_fetch_modules = mocks[2]
_fetch_modules.return_value = []
_fetch_manifest = mocks[3]
_fetch_manifest.return_value = None
# testing
method = SynchronizeWithDirectory(conduit, config)
method._run(inventory)
# validation
_fetch_manifest.assert_called_with()
self.assertFalse(_fetch_modules.called)
self.assertFalse(_import_modules.called)
self.assertFalse(_purge_unwanted_modules.called)
def test_run(self, *mocks):
config = {}
conduit = Mock()
inventory = Mock()
_purge_unwanted_modules = mocks[0]
_import_modules = mocks[1]
_import_modules.return_value = []
_fetch_modules = mocks[2]
_fetch_modules.return_value = []
_fetch_manifest = mocks[3]
_fetch_manifest.return_value = 'path,sum,size'
# testing
method = SynchronizeWithDirectory(conduit, config)
method._run(inventory)
# validation
_fetch_manifest.assert_called_with()
_fetch_modules.assert_called_with(_fetch_manifest())
_import_modules.assert_called_with(inventory, _fetch_modules())
_purge_unwanted_modules.assert_called_with(inventory, _import_modules())
def sync_repo(self, repo, sync_conduit, config):
self.sync_cancelled = False
# Supports two methods of synchronization.
# 1. Synchronize with a directory containing a pulp manifest and puppet modules.
# 2. Synchronize with Puppet Forge.
# When the feed URL references a PULP_MANIFEST, the directory synchronization
# method is used. Otherwise, the puppet forge synchronization method is used.
# synchronize with a directory
self.sync_method = SynchronizeWithDirectory(repo, sync_conduit, config)
report = self.sync_method()
# When fetching the PULP_MANIFEST is not successful, it's assumed that the
# feed points to a puppet forge instance and the synchronization is retried
# using puppet forge method.
if report.metadata_state == constants.STATE_FAILED:
self.sync_method = SynchronizeWithPuppetForge(repo, sync_conduit, config)
report = self.sync_method()
self.sync_method = None
return report.build_final_report()
def test_call_no_manifest(self, mock_mkdtemp, mock_rmtree,
mock_fetch_manifest, *mocks):
mock_fetch_manifest.return_value = None
mock_repo = Mock()
conduit = Mock()
config = {constants.CONFIG_FEED: 'http://host/root/PULP_MANAFEST'}
repository = Mock()
repository.working_dir = 'working'
mock_mkdtemp.return_value = '/abc'
# testing
method = SynchronizeWithDirectory(mock_repo, conduit, config)
report = method()
# validation
self.assertEqual(1, mock_fetch_manifest.call_count)
self.assertEqual(0, mocks[0].call_count)
self.assertEqual(0, mocks[1].call_count)
self.assertFalse(method.canceled)
self.assertTrue(isinstance(method.report, SyncProgressReport))
self.assertTrue(isinstance(report, SyncProgressReport))
mock_mkdtemp.assert_called_with(dir=mock_repo.working_dir)
mock_rmtree.assert_called_with(
os.path.join(repository.working_dir, mock_mkdtemp()))
def test_import_modules(self, mock_extract, mock_add, mock_remove_missing):
# These manifests represent the parsed metadata.json file. These contain a 'name'
# field, where we retrieve both the unit key's 'name' and 'author' field.
manifest = [{
'name': 'john-pulp1',
'author': 'Johnathon',
'version': '1.0'
}]
mock_extract.side_effect = manifest
module_paths = ['/tmp/module_1']
mock_pulp1, mock_pulp2 = (Mock(), Mock())
mock_pulp1.unit_key = {
'name': 'pulp1',
'author': 'john',
'version': '1.0'
}
mock_pulp2.unit_key = {
'name': 'pulp2',
'author': 'john',
'version': '2.0'
}
conduit = Mock()
conduit.get_units.return_value = [mock_pulp1, mock_pulp2]
config = Mock()
config.get_boolean.return_value = True
# test
method = SynchronizeWithDirectory(conduit, config)
method.started_fetch_modules = 10
method.report = Mock()
method.report.modules_total_count = 2
method.report.modules_finished_count = 0
method._import_modules(module_paths)
# validation
config.get_boolean.assert_called_once_with(
constants.CONFIG_REMOVE_MISSING)
mock_remove_missing.assert_called_once_with([mock_pulp1, mock_pulp2],
[mock_pulp1.unit_key])
def test_import_modules(self, mock_extract, mock_add, mock_remove_missing):
# These manifests represent the parsed metadata.json file. These contain a 'name'
# field, where we retrieve both the unit key's 'name' and 'author' field.
manifests = [
{
'name': 'john-pulp1',
'author': 'Johnathon',
'version': '1.0'
},
{
'name': 'john-pulp2',
'author': 'Johnathon',
'version': '2.0'
},
{
'name': 'john/pulp3',
'author': 'Johnathon',
'version': '3.0'
},
]
mock_extract.side_effect = manifests
unit_keys = [
{
'name': 'pulp1',
'author': 'john',
'version': '1.0'
},
{
'name': 'pulp2',
'author': 'john',
'version': '2.0'
},
{
'name': 'pulp3',
'author': 'john',
'version': '3.0'
},
]
module_paths = [
'/tmp/module_1',
'/tmp/module_2',
'/tmp/module_3',
]
mock_pulp2 = Mock()
mock_pulp2.unit_key = unit_keys[1]
conduit = Mock()
conduit.get_units.return_value = [mock_pulp2]
config = Mock()
config.get_boolean.return_value = False
# test
method = SynchronizeWithDirectory(conduit, config)
method.started_fetch_modules = 10
method.report = Mock()
method.report.modules_total_count = 3
method.report.modules_finished_count = 0
method._import_modules(module_paths)
# validation
self.assertEqual(3, mock_extract.call_count)
self.assertEqual(mock_extract.mock_calls[0][1][0], module_paths[0])
self.assertEqual(mock_extract.mock_calls[1][1][0], module_paths[1])
self.assertEqual(mock_extract.mock_calls[2][1][0], module_paths[2])
self.assertEqual(2, mock_add.call_count)
self.assertEqual(mock_add.mock_calls[0][1][0], module_paths[0])
self.assertEqual(mock_add.mock_calls[1][1][0], module_paths[2])
config.get_boolean.assert_called_once_with(
constants.CONFIG_REMOVE_MISSING)
self.assertEqual(0, mock_remove_missing.call_count)
# Check that the progress reporting was called as expected
self.assertEquals(3, method.report.update_progress.call_count)
self.assertEquals(2, method.report.modules_finished_count)
self.assertEquals(2, method.report.modules_total_count)
def test_download(self, mock_listener, mock_nectar_config,
mock_downloader_mapping):
mock_nectar_config.return_value = Mock()
mock_http_downloader = Mock()
mock_http_class = Mock(return_value=mock_http_downloader)
mock_downloader_mapping.__getitem__.return_value = mock_http_class
conduit = Mock()
config = Mock()
config.get = Mock(
side_effect={
constants.CONFIG_FEED: 'http://host/root/PULP_MANAFEST'
})
config.flatten = Mock(return_value={})
urls = [
('http://host/root/path_1', '/tmp/path_1'),
('http://host/root/path_2', '/tmp/path_1'),
]
report = namedtuple('Report', ['url', 'destination', 'error_msg'])
_listener = Mock()
_listener.succeeded_reports = [report(urls[0][0], urls[0][1], None)]
_listener.failed_reports = [
report(urls[1][0], urls[1][1], 'File Not Found')
]
mock_listener.return_value = _listener
# test
method = SynchronizeWithDirectory(conduit, config)
succeeded_reports, failed_reports = method._download(urls)
# validation
method.config.flatten.assert_called_with()
mock_nectar_config.assert_called_with(method.config.flatten())
self.assertTrue(mock_http_downloader.download.called)
self.assertEqual(mock_http_downloader.download.call_count, 1)
self.assertEqual(mock_http_downloader.download.call_args[0][0][0].url,
urls[0][0])
self.assertEqual(
mock_http_downloader.download.call_args[0][0][0].destination,
urls[0][1])
self.assertEqual(mock_http_downloader.download.call_args[0][0][1].url,
urls[1][0])
self.assertEqual(
mock_http_downloader.download.call_args[0][0][0].destination,
urls[1][1])
self.assertTrue(isinstance(succeeded_reports, list))
self.assertEqual(len(succeeded_reports), 1)
self.assertEqual(succeeded_reports[0].url, urls[0][0])
self.assertEqual(succeeded_reports[0].destination, urls[0][1])
self.assertTrue(isinstance(succeeded_reports, list))
self.assertTrue(isinstance(failed_reports, list))
self.assertEqual(len(failed_reports), 1)
self.assertEqual(failed_reports[0].url, urls[1][0])
self.assertEqual(failed_reports[0].destination, urls[1][1])
self.assertTrue(isinstance(failed_reports, list))