def setUp(self):
self.metadata_files = metadata.MetadataFiles('http://pulpproject.org',
'/a/b/c',
DownloaderConfig())
self.repo = Repository('repo1')
self.config = PluginCallConfiguration({}, {})
self.conduit = RepoSyncConduit(self.repo.id, 'yum_importer', 'abc123')
def sync(repo_id, sync_config_override=None, scheduled_call_id=None):
"""
Performs a synchronize operation on the given repository and triggers publishes for
distributors with auto-publish enabled.
The given repo must have an importer configured. This method is intentionally limited to
synchronizing a single repo. Performing multiple repository syncs concurrently will require a
more global view of the server and must be handled outside the scope of this class.
:param repo_id: identifies the repo to sync
:type repo_id: str
:param sync_config_override: optional config containing values to use for this sync only
:type sync_config_override: dict
:param scheduled_call_id: id of scheduled call that dispatched this task
:type scheduled_call_id: str
:return: TaskResult containing sync results and a list of spawned tasks
:rtype: pulp.server.async.tasks.TaskResult
:raise pulp_exceptions.MissingResource: if specified repo does not exist, or it does not have
an importer and associated plugin
:raise pulp_exceptions.PulpExecutionException: if the task fails.
"""
repo_obj = model.Repository.objects.get_repo_or_missing_resource(repo_id)
transfer_repo = repo_obj.to_transfer_repo()
repo_importer = model.Importer.objects.get_or_404(repo_id=repo_id)
try:
importer, imp_config = plugin_api.get_importer_by_id(repo_importer.importer_type_id)
except plugin_exceptions.PluginNotFound:
raise pulp_exceptions.MissingResource(repository=repo_id)
call_config = PluginCallConfiguration(imp_config, repo_importer.config, sync_config_override)
transfer_repo.working_dir = common_utils.get_working_directory()
conduit = RepoSyncConduit(repo_id, repo_importer.importer_type_id, repo_importer.id)
sync_result_collection = RepoSyncResult.get_collection()
# Fire an events around the call
fire_manager = manager_factory.event_fire_manager()
fire_manager.fire_repo_sync_started(repo_id)
# Perform the sync
sync_start_timestamp = _now_timestamp()
sync_result = None
try:
# Replace the Importer's sync_repo() method with our register_sigterm_handler decorator,
# which will set up cancel_sync_repo() as the target for the signal handler
sync_repo = register_sigterm_handler(importer.sync_repo, importer.cancel_sync_repo)
sync_report = sync_repo(transfer_repo, conduit, call_config)
except Exception, e:
sync_end_timestamp = _now_timestamp()
sync_result = RepoSyncResult.error_result(
repo_obj.repo_id, repo_importer['id'], repo_importer['importer_type_id'],
sync_start_timestamp, sync_end_timestamp, e, sys.exc_info()[2])
raise
def setUp(self, mock_repo_qs, mock_imp_qs, mock_remove):
super(RepoSyncConduitTests, self).setUp()
mock_plugins.install()
types_database.update_database([TYPE_1_DEF, TYPE_2_DEF])
self.association_manager = association_manager.RepoUnitAssociationManager()
self.association_query_manager = association_query_manager.RepoUnitAssociationQueryManager()
self.content_manager = content_manager.ContentManager()
self.query_manager = query_manager.ContentQueryManager()
self.conduit = RepoSyncConduit('repo-1', 'test-importer', 'abc123')
importer_controller.set_importer('repo-1', 'mock-importer', {})
def test_import_modified_units(self, mock_get_working, *mocks):
# Setup
self.populate()
mock_get_working.return_value = self.temp_dir
max_concurrency = 5
max_bandwidth = 12345
with mock_config.patch({'server': {'storage_dir': self.parentfs}}):
dist = NodesHttpDistributor()
working_dir = os.path.join(self.childfs, 'working_dir')
os.makedirs(working_dir)
repo = Repository(self.REPO_ID, working_dir)
cfg = self.dist_conf()
conduit = RepoPublishConduit(self.REPO_ID,
constants.HTTP_DISTRIBUTOR)
dist.publish_repo(repo, conduit, cfg)
# make the published unit have a newer _last_updated.
collection = connection.get_collection(
unit_db.unit_collection_name(self.UNIT_TYPE_ID))
# N=0 (no file)
unit = collection.find_one({'N': 0})
unit['age'] = 84 # this will be updated back to 42.
unit['_last_updated'] -= 1
unit['_storage_path'] = None
collection.update({'N': 0}, unit)
# N=1
unit = collection.find_one({'N': 1})
unit['age'] = 85 # this will be updated back to 42.
unit['_last_updated'] -= 1
collection.update({'N': 1}, unit)
# Test
importer = NodesHttpImporter()
publisher = dist.publisher(repo, cfg)
manifest_url = pathlib.url_join(publisher.base_url,
publisher.manifest_path())
configuration = {
constants.MANIFEST_URL_KEYWORD: manifest_url,
constants.STRATEGY_KEYWORD: constants.MIRROR_STRATEGY,
importer_constants.KEY_MAX_DOWNLOADS: max_concurrency,
importer_constants.KEY_MAX_SPEED: max_bandwidth,
}
configuration = PluginCallConfiguration(configuration, {})
conduit = RepoSyncConduit(self.REPO_ID, constants.HTTP_IMPORTER,
Mock())
with mock_config.patch({'server': {'storage_dir': self.childfs}}):
with patch('pulp_node.constants.CONTENT_PATH', self.parentfs):
importer.sync_repo(repo, conduit, configuration)
# Verify
unit = collection.find_one({'N': 0})
self.assertEqual(unit['age'], 42)
unit = collection.find_one({'N': 1})
self.assertEqual(unit['age'], 42)
def test_import_unit_files_already_exist_size_mismatch(
self, mock_get_working, *mocks):
# Setup
self.populate()
mock_get_working.return_value = self.temp_dir
with mock_config.patch({'server': {'storage_dir': self.parentfs}}):
dist = NodesHttpDistributor()
working_dir = os.path.join(self.childfs, 'working_dir')
os.makedirs(working_dir)
repo = Repository(self.REPO_ID, working_dir)
cfg = self.dist_conf()
conduit = RepoPublishConduit(self.REPO_ID,
constants.HTTP_DISTRIBUTOR)
dist.publish_repo(repo, conduit, cfg)
model.Distributor.objects.delete()
RepoContentUnit.get_collection().remove()
unit_db.clean()
self.define_plugins()
parent_content = os.path.join(self.parentfs, 'content')
child_content = os.path.join(self.childfs, 'content')
shutil.copytree(parent_content, child_content)
for fn in os.listdir(child_content):
path = os.path.join(child_content, fn)
if os.path.isdir(path):
continue
with open(path, 'w') as fp:
fp.truncate()
# Test
importer = NodesHttpImporter()
publisher = dist.publisher(repo, cfg)
manifest_url = pathlib.url_join(publisher.base_url,
publisher.manifest_path())
configuration = {
constants.MANIFEST_URL_KEYWORD: manifest_url,
constants.STRATEGY_KEYWORD: constants.MIRROR_STRATEGY,
}
configuration = PluginCallConfiguration(configuration, {})
conduit = RepoSyncConduit(self.REPO_ID, constants.HTTP_IMPORTER,
Mock())
with mock_config.patch({'server': {'storage_dir': self.childfs}}):
with patch('pulp_node.constants.CONTENT_PATH', self.parentfs):
importer.sync_repo(repo, conduit, configuration)
# Verify
units = conduit.get_units()
self.assertEquals(len(units), self.NUM_UNITS)
mock_importer_config_to_nectar_config = mocks[0]
mock_importer_config_to_nectar_config.assert_called_with(
configuration.flatten())
def setUp(self):
super(RepoSyncConduitTests, self).setUp()
mock_plugins.install()
types_database.update_database([TYPE_1_DEF, TYPE_2_DEF])
self.repo_manager = repo_manager.RepoManager()
self.importer_manager = importer_manager.RepoImporterManager()
self.sync_manager = sync_manager.RepoSyncManager()
self.association_manager = association_manager.RepoUnitAssociationManager()
self.association_query_manager = association_query_manager.RepoUnitAssociationQueryManager()
self.content_manager = content_manager.ContentManager()
self.query_manager = query_manager.ContentQueryManager()
self.repo_manager.create_repo('repo-1')
self.importer_manager.set_importer('repo-1', 'mock-importer', {})
self.conduit = RepoSyncConduit('repo-1', 'test-importer', 'importer', 'importer-id')
def test_import_cached_manifest_matched(self, mock_get_working, mock_fetch,
*unused):
# Setup
self.populate()
mock_get_working.return_value = self.temp_dir
with mock_config.patch({'server': {'storage_dir': self.parentfs}}):
dist = NodesHttpDistributor()
working_dir = os.path.join(self.childfs, 'working_dir')
os.makedirs(working_dir)
repo = Repository(self.REPO_ID, working_dir)
configuration = self.dist_conf()
conduit = RepoPublishConduit(self.REPO_ID,
constants.HTTP_DISTRIBUTOR)
dist.publish_repo(repo, conduit, configuration)
model.Distributor.objects.delete()
RepoContentUnit.get_collection().remove()
unit_db.clean()
self.define_plugins()
publisher = dist.publisher(repo, configuration)
manifest_path = publisher.manifest_path()
units_path = os.path.join(os.path.dirname(manifest_path),
UNITS_FILE_NAME)
manifest = Manifest(manifest_path)
manifest.read()
shutil.copy(manifest_path,
os.path.join(working_dir, MANIFEST_FILE_NAME))
shutil.copy(units_path, os.path.join(working_dir, UNITS_FILE_NAME))
# Test
importer = NodesHttpImporter()
manifest_url = pathlib.url_join(publisher.base_url, manifest_path)
configuration = {
constants.MANIFEST_URL_KEYWORD: manifest_url,
constants.STRATEGY_KEYWORD: constants.MIRROR_STRATEGY,
}
configuration = PluginCallConfiguration(configuration, {})
conduit = RepoSyncConduit(self.REPO_ID, constants.HTTP_IMPORTER,
Mock())
with mock_config.patch({'server': {'storage_dir': self.childfs}}):
with patch('pulp_node.constants.CONTENT_PATH', self.parentfs):
importer.sync_repo(repo, conduit, configuration)
# Verify
units = conduit.get_units()
self.assertEquals(len(units), self.NUM_UNITS)
self.assertFalse(mock_fetch.called)
def test_import(self, *mocks):
# Setup
self.populate()
max_concurrency = 5
max_bandwidth = 12345
with mock_config.patch({'server': {'storage_dir': self.parentfs}}):
dist = NodesHttpDistributor()
working_dir = os.path.join(self.childfs, 'working_dir')
os.makedirs(working_dir)
repo = Repository(self.REPO_ID, working_dir)
cfg = self.dist_conf()
conduit = RepoPublishConduit(self.REPO_ID,
constants.HTTP_DISTRIBUTOR)
dist.publish_repo(repo, conduit, cfg)
model.Repository.drop_collection()
RepoDistributor.get_collection().remove()
RepoContentUnit.get_collection().remove()
unit_db.clean()
self.define_plugins()
# Test
importer = NodesHttpImporter()
publisher = dist.publisher(repo, cfg)
manifest_url = pathlib.url_join(publisher.base_url,
publisher.manifest_path())
configuration = {
constants.MANIFEST_URL_KEYWORD: manifest_url,
constants.STRATEGY_KEYWORD: constants.MIRROR_STRATEGY,
importer_constants.KEY_MAX_DOWNLOADS: max_concurrency,
importer_constants.KEY_MAX_SPEED: max_bandwidth,
}
configuration = PluginCallConfiguration(configuration, {})
conduit = RepoSyncConduit(self.REPO_ID, constants.HTTP_IMPORTER)
with mock_config.patch({'server': {'storage_dir': self.childfs}}):
importer.sync_repo(repo, conduit, configuration)
# Verify
units = conduit.get_units()
self.assertEquals(len(units), self.NUM_UNITS)
mock_importer_config_to_nectar_config = mocks[0]
mock_importer_config_to_nectar_config.assert_called_with(
configuration.flatten())
def test_import_cached_manifest_units_invalid(self, *unused):
# Setup
self.populate()
with mock_config.patch({'server': {'storage_dir': self.parentfs}}):
dist = NodesHttpDistributor()
working_dir = os.path.join(self.childfs, 'working_dir')
os.makedirs(working_dir)
repo = Repository(self.REPO_ID, working_dir)
configuration = self.dist_conf()
conduit = RepoPublishConduit(self.REPO_ID,
constants.HTTP_DISTRIBUTOR)
dist.publish_repo(repo, conduit, configuration)
model.Repository.drop_collection()
RepoDistributor.get_collection().remove()
RepoContentUnit.get_collection().remove()
unit_db.clean()
self.define_plugins()
publisher = dist.publisher(repo, configuration)
manifest_path = publisher.manifest_path()
manifest = Manifest(manifest_path)
manifest.read()
shutil.copy(manifest_path,
os.path.join(working_dir, MANIFEST_FILE_NAME))
with open(os.path.join(working_dir, UNITS_FILE_NAME), 'w+') as fp:
fp.write('invalid-units')
# Test
importer = NodesHttpImporter()
manifest_url = pathlib.url_join(publisher.base_url, manifest_path)
configuration = {
constants.MANIFEST_URL_KEYWORD: manifest_url,
constants.STRATEGY_KEYWORD: constants.MIRROR_STRATEGY,
}
configuration = PluginCallConfiguration(configuration, {})
conduit = RepoSyncConduit(self.REPO_ID, constants.HTTP_IMPORTER)
with mock_config.patch({'server': {'storage_dir': self.childfs}}):
importer.sync_repo(repo, conduit, configuration)
# Verify
units = conduit.get_units()
self.assertEquals(len(units), self.NUM_UNITS)
def test_import_modified_units(self, *mocks):
# Setup
self.populate()
max_concurrency = 5
max_bandwidth = 12345
pulp_conf.set('server', 'storage_dir', self.parentfs)
dist = NodesHttpDistributor()
working_dir = os.path.join(self.childfs, 'working_dir')
os.makedirs(working_dir)
repo = Repository(self.REPO_ID, working_dir)
cfg = self.dist_conf()
conduit = RepoPublishConduit(self.REPO_ID, constants.HTTP_DISTRIBUTOR)
dist.publish_repo(repo, conduit, cfg)
# make the published unit have a newer _last_updated.
collection = connection.get_collection(
unit_db.unit_collection_name(self.UNIT_TYPE_ID))
unit = collection.find_one({'N': 0})
unit['age'] = 84
unit['_last_updated'] -= 1
collection.update({'N': 0}, unit, safe=True)
# Test
importer = NodesHttpImporter()
publisher = dist.publisher(repo, cfg)
manifest_url = pathlib.url_join(publisher.base_url,
publisher.manifest_path())
configuration = {
constants.MANIFEST_URL_KEYWORD: manifest_url,
constants.STRATEGY_KEYWORD: constants.MIRROR_STRATEGY,
importer_constants.KEY_MAX_DOWNLOADS: max_concurrency,
importer_constants.KEY_MAX_SPEED: max_bandwidth,
}
configuration = PluginCallConfiguration(configuration, {})
conduit = RepoSyncConduit(self.REPO_ID, constants.HTTP_IMPORTER,
RepoContentUnit.OWNER_TYPE_IMPORTER,
constants.HTTP_IMPORTER)
pulp_conf.set('server', 'storage_dir', self.childfs)
importer.sync_repo(repo, conduit, configuration)
# Verify
unit = collection.find_one({'N': 0})
self.assertEqual(unit['age'], 42)
def test_import_unit_files_already_exist(self, *mocks):
# Setup
self.populate()
pulp_conf.set('server', 'storage_dir', self.parentfs)
dist = NodesHttpDistributor()
working_dir = os.path.join(self.childfs, 'working_dir')
os.makedirs(working_dir)
repo = Repository(self.REPO_ID, working_dir)
cfg = self.dist_conf()
conduit = RepoPublishConduit(self.REPO_ID, constants.HTTP_DISTRIBUTOR)
dist.publish_repo(repo, conduit, cfg)
Repo.get_collection().remove()
RepoDistributor.get_collection().remove()
RepoContentUnit.get_collection().remove()
unit_db.clean()
self.define_plugins()
parent_content = os.path.join(self.parentfs, 'content')
child_content = os.path.join(self.childfs, 'content')
shutil.copytree(parent_content, child_content)
# Test
importer = NodesHttpImporter()
publisher = dist.publisher(repo, cfg)
manifest_url = pathlib.url_join(publisher.base_url,
publisher.manifest_path())
configuration = {
constants.MANIFEST_URL_KEYWORD: manifest_url,
constants.STRATEGY_KEYWORD: constants.MIRROR_STRATEGY,
}
configuration = PluginCallConfiguration(configuration, {})
conduit = RepoSyncConduit(self.REPO_ID, constants.HTTP_IMPORTER,
RepoContentUnit.OWNER_TYPE_IMPORTER,
constants.HTTP_IMPORTER)
pulp_conf.set('server', 'storage_dir', self.childfs)
importer.sync_repo(repo, conduit, configuration)
# Verify
units = conduit.get_units()
self.assertEquals(len(units), self.NUM_UNITS)
mock_importer_config_to_nectar_config = mocks[0]
mock_importer_config_to_nectar_config.assert_called_with(
configuration.flatten())
def sync(self, repo_id, sync_config_override=None):
"""
Performs a synchronize operation on the given repository.
The given repo must have an importer configured. The identity of the
importer is not a parameter to this call; if multiple importers are
eventually supported this will have to change to indicate which
importer to use.
This method is intentionally limited to synchronizing a single repo.
Performing multiple repository syncs concurrently will require a more
global view of the server and must be handled outside the scope of this
class.
@param repo_id: identifies the repo to sync
@type repo_id: str
@param sync_config_override: optional config containing values to use
for this sync only
@type sync_config_override: dict
@return: The synchronization report.
@rtype: L{pulp.server.plugins.model.SyncReport}
@raise MissingResource: if repo_id does not refer to a valid repo
@raise OperationFailed: if the given repo does not have an importer set
"""
repo_coll = Repo.get_collection()
# Validation
repo = repo_coll.find_one({'id' : repo_id})
if repo is None:
raise MissingResource(repo_id)
importer_instance, importer_config = self._get_importer_instance_and_config(repo_id)
if importer_instance is None:
raise MissingResource(repo_id)
dispatch_context = dispatch_factory.context()
dispatch_context.set_cancel_control_hook(importer_instance.cancel_sync_repo)
importer_manager = manager_factory.repo_importer_manager()
repo_importer = importer_manager.get_importer(repo_id)
# Assemble the data needed for the sync
conduit = RepoSyncConduit(repo_id, repo_importer['id'], RepoContentUnit.OWNER_TYPE_IMPORTER, repo_importer['id'])
call_config = PluginCallConfiguration(importer_config, repo_importer['config'], sync_config_override)
transfer_repo = common_utils.to_transfer_repo(repo)
transfer_repo.working_dir = common_utils.importer_working_dir(repo_importer['importer_type_id'], repo_id, mkdir=True)
# Fire an events around the call
fire_manager = manager_factory.event_fire_manager()
fire_manager.fire_repo_sync_started(repo_id)
sync_result = self._do_sync(repo, importer_instance, transfer_repo, conduit, call_config)
fire_manager.fire_repo_sync_finished(sync_result)
dispatch_context.clear_cancel_control_hook()
if sync_result['result'] == RepoSyncResult.RESULT_FAILED:
raise PulpExecutionException(_('Importer indicated a failed response'))
return sync_result
def sync(repo_id, sync_config_override=None):
"""
Performs a synchronize operation on the given repository.
The given repo must have an importer configured. The identity of the
importer is not a parameter to this call; if multiple importers are
eventually supported this will have to change to indicate which
importer to use.
This method is intentionally limited to synchronizing a single repo.
Performing multiple repository syncs concurrently will require a more
global view of the server and must be handled outside the scope of this
class.
@param repo_id: identifies the repo to sync
@type repo_id: str
@param sync_config_override: optional config containing values to use
for this sync only
@type sync_config_override: dict
@return: The synchronization report.
@rtype: L{pulp.server.plugins.model.SyncReport}
@raise MissingResource: if repo_id does not refer to a valid repo
@raise OperationFailed: if the given repo does not have an importer set
"""
repo_coll = Repo.get_collection()
# Validation
repo = repo_coll.find_one({'id': repo_id})
if repo is None:
raise MissingResource(repo_id)
importer_instance, importer_config = RepoSyncManager._get_importer_instance_and_config(
repo_id)
if importer_instance is None:
raise MissingResource(repo_id)
importer_manager = manager_factory.repo_importer_manager()
repo_importer = importer_manager.get_importer(repo_id)
# Assemble the data needed for the sync
conduit = RepoSyncConduit(repo_id, repo_importer['id'])
call_config = PluginCallConfiguration(importer_config,
repo_importer['config'],
sync_config_override)
transfer_repo = common_utils.to_transfer_repo(repo)
transfer_repo.working_dir = common_utils.get_working_directory()
# Fire an events around the call
fire_manager = manager_factory.event_fire_manager()
fire_manager.fire_repo_sync_started(repo_id)
sync_result = RepoSyncManager._do_sync(repo, importer_instance,
transfer_repo, conduit,
call_config)
fire_manager.fire_repo_sync_finished(sync_result)
if sync_result['result'] == RepoSyncResult.RESULT_FAILED:
raise PulpExecutionException(
_('Importer indicated a failed response'))
repo_publish_manager = manager_factory.repo_publish_manager()
auto_distributors = repo_publish_manager.auto_distributors(repo_id)
spawned_tasks = []
for distributor in auto_distributors:
distributor_id = distributor['id']
spawned_tasks.append(
repo_publish_manager.queue_publish(repo_id,
distributor_id).task_id)
return TaskResult(sync_result, spawned_tasks=spawned_tasks)
