def setUp(self, mock_repo_qs, mock_imp_qs, mock_remove):
super(RepoSyncConduitTests, self).setUp()
mock_plugins.install()
types_database.update_database([TYPE_1_DEF, TYPE_2_DEF])
self.association_manager = association_manager.RepoUnitAssociationManager()
self.association_query_manager = association_query_manager.RepoUnitAssociationQueryManager()
self.content_manager = content_manager.ContentManager()
self.query_manager = query_manager.ContentQueryManager()
self.conduit = RepoSyncConduit('repo-1', 'test-importer', 'abc123')
importer_controller.set_importer('repo-1', 'mock-importer', {})
def setUp(self):
super(RepoSyncConduitTests, self).setUp()
mock_plugins.install()
types_database.update_database([TYPE_1_DEF, TYPE_2_DEF])
self.repo_manager = repo_manager.RepoManager()
self.importer_manager = importer_manager.RepoImporterManager()
self.sync_manager = sync_manager.RepoSyncManager()
self.association_manager = association_manager.RepoUnitAssociationManager()
self.association_query_manager = association_query_manager.RepoUnitAssociationQueryManager()
self.content_manager = content_manager.ContentManager()
self.query_manager = query_manager.ContentQueryManager()
self.repo_manager.create_repo('repo-1')
self.conduit = RepoSyncConduit('repo-1', 'test-importer', 'importer', 'importer-id')
class RepoSyncConduitTests(base.PulpServerTests):
def clean(self):
super(RepoSyncConduitTests, self).clean()
model.Repository.objects.delete()
model.Importer.objects.delete()
RepoContentUnit.get_collection().remove()
@mock.patch('pulp.server.controllers.importer.remove_importer')
@mock.patch('pulp.server.controllers.importer.model.Repository.objects')
@mock.patch('pulp.server.controllers.importer.model.Importer.objects')
def setUp(self, mock_repo_qs, mock_imp_qs, mock_remove):
super(RepoSyncConduitTests, self).setUp()
mock_plugins.install()
types_database.update_database([TYPE_1_DEF, TYPE_2_DEF])
self.association_manager = association_manager.RepoUnitAssociationManager()
self.association_query_manager = association_query_manager.RepoUnitAssociationQueryManager()
self.content_manager = content_manager.ContentManager()
self.query_manager = query_manager.ContentQueryManager()
self.conduit = RepoSyncConduit('repo-1', 'test-importer', 'abc123')
importer_controller.set_importer('repo-1', 'mock-importer', {})
def tearDown(self):
super(RepoSyncConduitTests, self).tearDown()
types_database.clean()
mock_plugins.reset()
def test_str(self):
"""
Makes sure the __str__ implementation does not raise an error.
"""
str(self.conduit)
def test_get_remove_unit(self):
"""
Tests retrieving units through the conduit and removing them.
"""
model.Repository(repo_id='repo-1').save()
unit_1_key = {'key-1': 'unit_1'}
unit_1_metadata = {'meta_1': 'value_1'}
unit_1 = self.conduit.init_unit(TYPE_1_DEF.id, unit_1_key, unit_1_metadata, '/foo/bar')
self.conduit.save_unit(unit_1)
# Test - get_units
units = self.conduit.get_units()
# Verify returned units
self.assertEqual(1, len(units))
self.assertEqual(unit_1_key, units[0].unit_key)
self.assertTrue(units[0].id is not None)
# Test - remove_units
self.conduit.remove_unit(units[0])
# Verify repo association removed in the database
associated_units = list(RepoContentUnit.get_collection().find({'repo_id': 'repo-1'}))
self.assertEqual(0, len(associated_units))
# Verify the unit itself is still in the database
db_unit = self.query_manager.get_content_unit_by_id(TYPE_1_DEF.id, unit_1.id)
self.assertTrue(db_unit is not None)
def test_build_reports(self):
"""
Tests that the conduit correctly inserts the count values into the report.
"""
model.Repository(repo_id='repo-1').save()
# Created - 10
for i in range(0, 10):
unit_key = {'key-1': 'unit_%d' % i}
unit = self.conduit.init_unit(TYPE_1_DEF.id, unit_key, {}, '/foo/bar')
self.conduit.save_unit(unit)
# Removed - 1
doomed = self.conduit.get_units()[0]
self.conduit.remove_unit(doomed)
# Updated - 1
update_me = self.conduit.init_unit(TYPE_1_DEF.id, {'key-1': 'unit_5'}, {}, '/foo/bar')
self.conduit.save_unit(update_me)
# Test
success_report = self.conduit.build_success_report('summary', 'details')
failure_report = self.conduit.build_failure_report('summary', 'details')
cancel_report = self.conduit.build_cancel_report('summary', 'details')
# Verify
self.assertEqual(success_report.success_flag, True)
self.assertEqual(success_report.canceled_flag, False)
self.assertEqual(failure_report.success_flag, False)
self.assertEqual(failure_report.canceled_flag, False)
self.assertEqual(cancel_report.success_flag, False)
self.assertEqual(cancel_report.canceled_flag, True)
for r in (success_report, failure_report, cancel_report):
self.assertTrue(isinstance(r, SyncReport))
self.assertEqual(10, r.added_count)
self.assertEqual(1, r.removed_count)
self.assertEqual(1, r.updated_count)
self.assertEqual('summary', r.summary)
self.assertEqual('details', r.details)
def test_remove_unit_with_error(self):
# Setup
self.conduit._association_manager = mock.Mock()
self.conduit._association_manager.unassociate_unit_by_id.side_effect = Exception()
# Test
self.assertRaises(ImporterConduitException, self.conduit.remove_unit, None)
def test_associate_existing(self):
mock_am = mock.Mock()
self.conduit._association_manager = mock_am
self.conduit._content_query_manager = mock.Mock()
mock_unit_key = {'some_key': 123}
mock_id = mock.Mock()
self.conduit._content_query_manager.get_content_unit_ids.return_value = [mock_id]
self.conduit.associate_existing('fake-type', [mock_unit_key])
mock_am.associate_all_by_ids.assert_called_once_with('repo-1', 'fake-type', [mock_id])
class RepoSyncConduitTests(base.PulpServerTests):
def clean(self):
super(RepoSyncConduitTests, self).clean()
types_database.clean()
mock_plugins.reset()
RepoContentUnit.get_collection().remove()
Repo.get_collection().remove()
def setUp(self):
super(RepoSyncConduitTests, self).setUp()
mock_plugins.install()
types_database.update_database([TYPE_1_DEF, TYPE_2_DEF])
self.repo_manager = repo_manager.RepoManager()
self.importer_manager = importer_manager.RepoImporterManager()
self.sync_manager = sync_manager.RepoSyncManager()
self.association_manager = association_manager.RepoUnitAssociationManager()
self.association_query_manager = association_query_manager.RepoUnitAssociationQueryManager()
self.content_manager = content_manager.ContentManager()
self.query_manager = query_manager.ContentQueryManager()
self.repo_manager.create_repo('repo-1')
self.conduit = RepoSyncConduit('repo-1', 'test-importer', 'importer', 'importer-id')
def test_str(self):
"""
Makes sure the __str__ implementation does not raise an error.
"""
str(self.conduit)
def test_get_remove_unit(self):
"""
Tests retrieving units through the conduit and removing them.
"""
# Setup
unit_1_key = {'key-1' : 'unit_1'}
unit_1_metadata = {'meta_1' : 'value_1'}
unit_1 = self.conduit.init_unit(TYPE_1_DEF.id, unit_1_key, unit_1_metadata, '/foo/bar')
self.conduit.save_unit(unit_1)
# Test - get_units
units = self.conduit.get_units()
# Verify returned units
self.assertEqual(1, len(units))
self.assertEqual(unit_1_key, units[0].unit_key)
self.assertTrue(units[0].id is not None)
# Test - remove_units
self.conduit.remove_unit(units[0])
# Verify repo association removed in the database
associated_units = list(RepoContentUnit.get_collection().find({'repo_id' : 'repo-1'}))
self.assertEqual(0, len(associated_units))
# Verify the unit itself is still in the database
db_unit = self.query_manager.get_content_unit_by_id(TYPE_1_DEF.id, unit_1.id)
self.assertTrue(db_unit is not None)
def test_build_reports(self):
"""
Tests that the conduit correctly inserts the count values into the report.
"""
# Setup
# Created - 10
for i in range(0, 10):
unit_key = {'key-1' : 'unit_%d' % i}
unit = self.conduit.init_unit(TYPE_1_DEF.id, unit_key, {}, '/foo/bar')
self.conduit.save_unit(unit)
# Removed - 1
doomed = self.conduit.get_units()[0]
self.conduit.remove_unit(doomed)
# Updated - 1
update_me = self.conduit.init_unit(TYPE_1_DEF.id, {'key-1' : 'unit_5'}, {}, '/foo/bar')
self.conduit.save_unit(update_me)
# Test
success_report = self.conduit.build_success_report('summary', 'details')
failure_report = self.conduit.build_failure_report('summary', 'details')
# Verify
self.assertEqual(success_report.success_flag, True)
self.assertEqual(failure_report.success_flag, False)
for r in (success_report, failure_report):
self.assertTrue(isinstance(r, SyncReport))
self.assertEqual(10, r.added_count)
self.assertEqual(1, r.removed_count)
self.assertEqual(1, r.updated_count)
self.assertEqual('summary', r.summary)
self.assertEqual('details', r.details)
def test_remove_unit_with_error(self):
# Setup
self.conduit._association_manager = mock.Mock()
self.conduit._association_manager.unassociate_unit_by_id.side_effect = Exception()
# Test
self.assertRaises(ImporterConduitException, self.conduit.remove_unit, None)
class RepoSyncConduitTests(base.PulpServerTests):
def clean(self):
super(RepoSyncConduitTests, self).clean()
RepoContentUnit.get_collection().remove()
Repo.get_collection().remove()
def setUp(self):
super(RepoSyncConduitTests, self).setUp()
mock_plugins.install()
types_database.update_database([TYPE_1_DEF, TYPE_2_DEF])
self.repo_manager = repo_manager.RepoManager()
self.importer_manager = importer_manager.RepoImporterManager()
self.sync_manager = sync_manager.RepoSyncManager()
self.association_manager = association_manager.RepoUnitAssociationManager(
)
self.association_query_manager = association_query_manager.RepoUnitAssociationQueryManager(
)
self.content_manager = content_manager.ContentManager()
self.query_manager = query_manager.ContentQueryManager()
self.repo_manager.create_repo('repo-1')
self.importer_manager.set_importer('repo-1', 'mock-importer', {})
self.conduit = RepoSyncConduit('repo-1', 'test-importer', 'importer',
'importer-id')
def tearDown(self):
super(RepoSyncConduitTests, self).tearDown()
types_database.clean()
mock_plugins.reset()
def test_str(self):
"""
Makes sure the __str__ implementation does not raise an error.
"""
str(self.conduit)
def test_get_remove_unit(self):
"""
Tests retrieving units through the conduit and removing them.
"""
# Setup
unit_1_key = {'key-1': 'unit_1'}
unit_1_metadata = {'meta_1': 'value_1'}
unit_1 = self.conduit.init_unit(TYPE_1_DEF.id, unit_1_key,
unit_1_metadata, '/foo/bar')
self.conduit.save_unit(unit_1)
# Test - get_units
units = self.conduit.get_units()
# Verify returned units
self.assertEqual(1, len(units))
self.assertEqual(unit_1_key, units[0].unit_key)
self.assertTrue(units[0].id is not None)
# Test - remove_units
self.conduit.remove_unit(units[0])
# Verify repo association removed in the database
associated_units = list(RepoContentUnit.get_collection().find(
{'repo_id': 'repo-1'}))
self.assertEqual(0, len(associated_units))
# Verify the unit itself is still in the database
db_unit = self.query_manager.get_content_unit_by_id(
TYPE_1_DEF.id, unit_1.id)
self.assertTrue(db_unit is not None)
def test_build_reports(self):
"""
Tests that the conduit correctly inserts the count values into the report.
"""
# Setup
# Created - 10
for i in range(0, 10):
unit_key = {'key-1': 'unit_%d' % i}
unit = self.conduit.init_unit(TYPE_1_DEF.id, unit_key, {},
'/foo/bar')
self.conduit.save_unit(unit)
# Removed - 1
doomed = self.conduit.get_units()[0]
self.conduit.remove_unit(doomed)
# Updated - 1
update_me = self.conduit.init_unit(TYPE_1_DEF.id, {'key-1': 'unit_5'},
{}, '/foo/bar')
self.conduit.save_unit(update_me)
# Test
success_report = self.conduit.build_success_report(
'summary', 'details')
failure_report = self.conduit.build_failure_report(
'summary', 'details')
cancel_report = self.conduit.build_cancel_report('summary', 'details')
# Verify
self.assertEqual(success_report.success_flag, True)
self.assertEqual(success_report.canceled_flag, False)
self.assertEqual(failure_report.success_flag, False)
self.assertEqual(failure_report.canceled_flag, False)
self.assertEqual(cancel_report.success_flag, False)
self.assertEqual(cancel_report.canceled_flag, True)
for r in (success_report, failure_report, cancel_report):
self.assertTrue(isinstance(r, SyncReport))
self.assertEqual(10, r.added_count)
self.assertEqual(1, r.removed_count)
self.assertEqual(1, r.updated_count)
self.assertEqual('summary', r.summary)
self.assertEqual('details', r.details)
def test_remove_unit_with_error(self):
# Setup
self.conduit._association_manager = mock.Mock()
self.conduit._association_manager.unassociate_unit_by_id.side_effect = Exception(
)
# Test
self.assertRaises(ImporterConduitException, self.conduit.remove_unit,
None)
def sync(repo_id, sync_config_override=None):
"""
Performs a synchronize operation on the given repository.
The given repo must have an importer configured. The identity of the
importer is not a parameter to this call; if multiple importers are
eventually supported this will have to change to indicate which
importer to use.
This method is intentionally limited to synchronizing a single repo.
Performing multiple repository syncs concurrently will require a more
global view of the server and must be handled outside the scope of this
class.
@param repo_id: identifies the repo to sync
@type repo_id: str
@param sync_config_override: optional config containing values to use
for this sync only
@type sync_config_override: dict
@return: The synchronization report.
@rtype: L{pulp.server.plugins.model.SyncReport}
@raise MissingResource: if repo_id does not refer to a valid repo
@raise OperationFailed: if the given repo does not have an importer set
"""
repo_coll = Repo.get_collection()
# Validation
repo = repo_coll.find_one({'id': repo_id})
if repo is None:
raise MissingResource(repo_id)
importer_instance, importer_config = RepoSyncManager._get_importer_instance_and_config(
repo_id)
if importer_instance is None:
raise MissingResource(repo_id)
importer_manager = manager_factory.repo_importer_manager()
repo_importer = importer_manager.get_importer(repo_id)
# Assemble the data needed for the sync
conduit = RepoSyncConduit(repo_id, repo_importer['id'], RepoContentUnit.OWNER_TYPE_IMPORTER,
repo_importer['id'])
call_config = PluginCallConfiguration(importer_config, repo_importer['config'],
sync_config_override)
transfer_repo = common_utils.to_transfer_repo(repo)
transfer_repo.working_dir = common_utils.importer_working_dir(
repo_importer['importer_type_id'], repo_id, mkdir=True)
# Fire an events around the call
fire_manager = manager_factory.event_fire_manager()
fire_manager.fire_repo_sync_started(repo_id)
sync_result = RepoSyncManager._do_sync(repo, importer_instance, transfer_repo, conduit,
call_config)
fire_manager.fire_repo_sync_finished(sync_result)
if sync_result['result'] == RepoSyncResult.RESULT_FAILED:
raise PulpExecutionException(_('Importer indicated a failed response'))
repo_publish_manager = manager_factory.repo_publish_manager()
auto_distributors = repo_publish_manager.auto_distributors(repo_id)
spawned_tasks = []
for distributor in auto_distributors:
distributor_id = distributor['id']
spawned_tasks.append(
repo_publish_manager.queue_publish(repo_id, distributor_id).task_id)
return TaskResult(sync_result, spawned_tasks=spawned_tasks)
def sync(repo_id, sync_config_override=None):
"""
Performs a synchronize operation on the given repository and triggers publishs for distributors
with autopublish enabled.
The given repo must have an importer configured. This method is intentionally limited to
synchronizing a single repo. Performing multiple repository syncs concurrently will require a
more global view of the server and must be handled outside the scope of this class.
:param repo_id: identifies the repo to sync
:type repo_id: str
:param sync_config_override: optional config containing values to use for this sync only
:type sync_config_override: dict
:return: TaskResult containing sync results and a list of spawned tasks
:rtype: pulp.server.async.tasks.TaskResult
:raise pulp_exceptions.MissingResource: if specified repo does not exist, or it does not have
an importer and associated plugin
:raise pulp_exceptions.PulpExecutionException: if the task fails.
"""
repo_obj = model.Repository.objects.get_repo_or_missing_resource(repo_id)
transfer_repo = repo_obj.to_transfer_repo()
importer_collection = RepoImporter.get_collection()
repo_importer = importer_collection.find_one({'repo_id': repo_obj.repo_id})
if repo_importer is None:
raise pulp_exceptions.MissingResource(repository=repo_id)
try:
importer, imp_config = plugin_api.get_importer_by_id(
repo_importer['importer_type_id'])
except plugin_exceptions.PluginNotFound:
raise pulp_exceptions.MissingResource(repository=repo_id)
call_config = PluginCallConfiguration(imp_config, repo_importer['config'],
sync_config_override)
transfer_repo.working_dir = common_utils.get_working_directory()
conduit = RepoSyncConduit(repo_id, repo_importer['id'])
sync_result_collection = RepoSyncResult.get_collection()
# Fire an events around the call
fire_manager = manager_factory.event_fire_manager()
fire_manager.fire_repo_sync_started(repo_id)
# Perform the sync
sync_start_timestamp = _now_timestamp()
sync_result = None
try:
# Replace the Importer's sync_repo() method with our register_sigterm_handler decorator,
# which will set up cancel_sync_repo() as the target for the signal handler
sync_repo = register_sigterm_handler(importer.sync_repo,
importer.cancel_sync_repo)
sync_report = sync_repo(transfer_repo, conduit, call_config)
except Exception, e:
sync_end_timestamp = _now_timestamp()
sync_result = RepoSyncResult.error_result(
repo_obj.repo_id, repo_importer['id'],
repo_importer['importer_type_id'], sync_start_timestamp,
sync_end_timestamp, e,
sys.exc_info()[2])
raise
