def test_resolve_dependencies_by_unit_no_importer(self):
# Setup
manager_factory.repo_manager().create_repo('empty')
# Test
self.assertRaises(MissingResource, self.manager.resolve_dependencies_by_units, 'empty', [],
{})
def test_resolve_dependencies_by_unit_no_importer(self):
# Setup
manager_factory.repo_manager().create_repo('empty')
# Test
self.assertRaises(MissingResource,
self.manager.resolve_dependencies_by_units, 'empty',
[], {})
def unassociate_all_by_ids(self, repo_id, unit_type_id, unit_id_list, owner_type, owner_id):
"""
Removes the association between a repo and a number of units. Only the
association made by the given owner will be removed. It is possible the
repo will still have a manually created association will for the unit.
@param repo_id: identifies the repo
@type repo_id: str
@param unit_type_id: identifies the type of units being removed
@type unit_type_id: str
@param unit_id_list: list of unique identifiers for units within the given type
@type unit_id_list: list of str
@param owner_type: category of the caller who created the association;
must be one of the OWNER_* variables in this module
@type owner_type: str
@param owner_id: identifies the caller who created the association, either
the importer ID or user login
@type owner_id: str
"""
spec = {'repo_id' : repo_id,
'unit_type_id' : unit_type_id,
'unit_id' : {'$in' : unit_id_list},
'owner_type' : owner_type,
'owner_id' : owner_id,
}
unit_coll = RepoContentUnit.get_collection()
unit_coll.remove(spec, safe=True)
unique_count = sum(1 for unit_id in unit_id_list
if not self.association_exists(repo_id, unit_id, unit_type_id))
# update the count of associated units on the repo object
if unique_count:
manager_factory.repo_manager().update_unit_count(
repo_id, -unique_count)
try:
repo_query_manager = manager_factory.repo_query_manager()
repo = repo_query_manager.get_repository(repo_id)
content_query_manager = manager_factory.content_query_manager()
content_units = content_query_manager.get_multiple_units_by_ids(unit_type_id, unit_id_list)
importer_manager = manager_factory.repo_importer_manager()
importer = importer_manager.get_importer(repo_id)
importer.remove_units(repo, content_units)
except:
_LOG.exception('Exception informing importer for [%s] of unassociation' % repo_id)
def setUp(self):
super(DependencyManagerTests, self).setUp()
mock_plugins.install()
database.update_database([TYPE_1_DEF])
self.repo_id = 'dep-repo'
self.manager = manager_factory.dependency_manager()
manager_factory.repo_manager().create_repo(self.repo_id)
manager_factory.repo_importer_manager().set_importer(self.repo_id, 'mock-importer', {})
def preserve_custom_metadata_on_repo_scratchpad():
"""
Lookups all the yum based repos in pulp; grabs any custom metadata
and set the the data on repo scratchpad.
"""
factory.initialize()
repos = factory.repo_query_manager().find_with_importer_type(
"yum_importer")
if not repos:
_log.debug("No repos found to perform db migrate")
return
repo_ids = [repo['id'] for repo in repos]
for repo_id in repo_ids:
_log.debug("Processing repo %s" % repo_id)
repo_scratchpad = factory.repo_manager().get_repo_scratchpad(repo_id)
if "repodata" in repo_scratchpad and repo_scratchpad["repodata"]:
# repo scratchpad already has repodata, skip migration
_log.debug(
"repo [%s] scratchpad already has repodata, skip migration" %
repo_id)
continue
repo_working_dir = importer_working_dir('yum_importer', repo_id)
importer_repodata_dir = os.path.join(repo_working_dir, repo_id,
"repodata")
repomd_xml_path = os.path.join(importer_repodata_dir, "repomd.xml")
if not os.path.exists(repomd_xml_path):
# repodata doesn't exist on filesystem cannot lookup custom data, continue to next
continue
ftypes = util.get_repomd_filetypes(repomd_xml_path)
base_ftypes = [
'primary', 'primary_db', 'filelists_db', 'filelists', 'other',
'other_db', 'group', 'group_gz', 'updateinfo', 'updateinfo_db'
]
for ftype in ftypes:
if ftype in base_ftypes:
# no need to process these again
continue
filetype_path = os.path.join(
importer_repodata_dir,
os.path.basename(
util.get_repomd_filetype_path(repomd_xml_path, ftype)))
if filetype_path.endswith('.gz'):
# if file is gzipped, decompress
data = gzip.open(filetype_path).read().decode(
"utf-8", "replace")
else:
data = open(filetype_path).read().decode("utf-8", "replace")
repo_scratchpad["repodata"].update({ftype: data})
# set the custom metadata on scratchpad
factory.repo_manager().set_repo_scratchpad(repo_id, repo_scratchpad)
_log.info("Updated repo [%s] scratchpad with new custom repodata" %
repo_id)
def setUp(self):
super(DependencyManagerTests, self).setUp()
mock_plugins.install()
database.update_database([TYPE_1_DEF])
self.repo_id = 'dep-repo'
self.manager = manager_factory.dependency_manager()
manager_factory.repo_manager().create_repo(self.repo_id)
manager_factory.repo_importer_manager().set_importer(
self.repo_id, 'mock-importer', {})
def test_post_with_repos(self):
# Setup
manager_factory.repo_manager().create_repo('add-me')
data = {'id': 'with-repos', 'repo_ids': ['add-me']}
# Test
status, body = self.post('/v2/repo_groups/', data)
# Verify
self.assertEqual(201, status)
found = RepoGroup.get_collection().find_one({'id': data['id']})
self.assertEqual(found['repo_ids'], data['repo_ids'])
def test_post_with_repos(self):
# Setup
manager_factory.repo_manager().create_repo("add-me")
data = {"id": "with-repos", "repo_ids": ["add-me"]}
# Test
status, body = self.post("/v2/repo_groups/", data)
# Verify
self.assertEqual(201, status)
found = RepoGroup.get_collection().find_one({"id": data["id"]})
self.assertEqual(found["repo_ids"], data["repo_ids"])
def associate_all_by_ids(self, repo_id, unit_type_id, unit_id_list,
owner_type, owner_id):
"""
Creates multiple associations between the given repo and content units.
See associate_unit_by_id for semantics.
@param repo_id: identifies the repo
@type repo_id: str
@param unit_type_id: identifies the type of unit being added
@type unit_type_id: str
@param unit_id_list: list or generator of unique identifiers for units within the given type
@type unit_id_list: list or generator of str
@param owner_type: category of the caller making the association;
must be one of the OWNER_* variables in this module
@type owner_type: str
@param owner_id: identifies the caller making the association, either
the importer ID or user login
@type owner_id: str
:return: number of new units added to the repo
:rtype: int
@raise InvalidType: if the given owner type is not of the valid enumeration
"""
# There may be a way to batch this in mongo which would be ideal for a
# bulk operation like this. But for deadline purposes, this call will
# simply loop and call the single method.
unique_count = 0
for unit_id in unit_id_list:
if not RepoUnitAssociationManager.association_exists(
repo_id, unit_id, unit_type_id):
unique_count += 1
self.associate_unit_by_id(repo_id, unit_type_id, unit_id,
owner_type, owner_id, False)
# update the count of associated units on the repo object
if unique_count:
manager_factory.repo_manager().update_unit_count(
repo_id, unit_type_id, unique_count)
# update the timestamp for when the units were added to the repo
manager_factory.repo_manager().update_last_unit_added(repo_id)
return unique_count
def remove_units(self, repo, units, config):
"""
Removes content units from the given repository.
This method also removes the tags associated with images in the repository.
This call will not result in the unit being deleted from Pulp itself.
:param repo: metadata describing the repository
:type repo: pulp.plugins.model.Repository
:param units: list of objects describing the units to import in
this call
:type units: list of pulp.plugins.model.AssociatedUnit
:param config: plugin configuration
:type config: pulp.plugins.config.PluginCallConfiguration
"""
repo_manager = manager_factory.repo_manager()
scratchpad = repo_manager.get_repo_scratchpad(repo.id)
tags = scratchpad.get(u'tags', [])
unit_ids = set([unit.unit_key[u'image_id'] for unit in units])
for tag_dict in tags[:]:
if tag_dict[constants.IMAGE_ID_KEY] in unit_ids:
tags.remove(tag_dict)
scratchpad[u'tags'] = tags
repo_manager.set_repo_scratchpad(repo.id, scratchpad)
def repo_delete_itinerary(repo_id):
"""
Get the itinerary for deleting a repository.
1. Delete the repository on the sever.
2. Unbind any bound consumers.
@param repo_id: A repository ID.
@type repo_id: str
@return: A list of call_requests known as an itinerary.
@rtype list
"""
call_requests = []
# delete repository
manager = managers.repo_manager()
resources = {dispatch_constants.RESOURCE_REPOSITORY_TYPE: {repo_id: dispatch_constants.RESOURCE_DELETE_OPERATION}}
tags = [resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE, repo_id), action_tag("delete")]
delete_request = CallRequest(manager.delete_repo, [repo_id], resources=resources, tags=tags, archive=True)
call_requests.append(delete_request)
# append unbind itineraries foreach bound consumer
options = {}
manager = managers.consumer_bind_manager()
for bind in manager.find_by_repo(repo_id):
unbind_requests = unbind_itinerary(bind["consumer_id"], bind["repo_id"], bind["distributor_id"], options)
if unbind_requests:
unbind_requests[0].depends_on(delete_request.id)
call_requests.extend(unbind_requests)
return call_requests
def PUT(self, id):
parameters = self.params()
delta = parameters.get('delta', None)
importer_config = parameters.get('importer_config', None)
distributor_configs = parameters.get('distributor_configs', None)
repo_manager = manager_factory.repo_manager()
tags = [
resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE, id),
action_tag('update')
]
call_request = CallRequest(
repo_manager.update_repo_and_plugins, [id, delta], {
'importer_config': importer_config,
'distributor_configs': distributor_configs
},
tags=tags,
archive=True,
kwarg_blacklist=['importer_config', 'distributor_configs'])
call_request.updates_resource(
dispatch_constants.RESOURCE_REPOSITORY_TYPE, id)
repo = execution.execute(call_request)
repo.update(serialization.link.current_link_obj())
return self.ok(repo)
def delete(repo_id):
"""
Get the itinerary for deleting a repository.
1. Delete the repository on the sever.
2. Unbind any bound consumers.
:param repo_id: A repository ID.
:type repo_id: str
:return: A TaskRequest object with the details of any errors or spawned tasks
:rtype TaskRequest
"""
# delete repository
manager = managers.repo_manager()
manager.delete_repo(repo_id)
# append unbind itineraries foreach bound consumer
options = {}
manager = managers.consumer_bind_manager()
additional_tasks = []
errors = []
for bind in manager.find_by_repo(repo_id):
try:
report = consumer.unbind(bind['consumer_id'],
bind['repo_id'],
bind['distributor_id'],
options)
if report:
additional_tasks.extend(report.spawned_tasks)
except Exception, e:
errors.append(e)
def remove_units(self, repo, units, config):
"""
Removes content units from the given repository.
This method also removes the tags associated with images in the repository.
This call will not result in the unit being deleted from Pulp itself.
:param repo: metadata describing the repository
:type repo: pulp.plugins.model.Repository
:param units: list of objects describing the units to import in
this call
:type units: list of pulp.plugins.model.AssociatedUnit
:param config: plugin configuration
:type config: pulp.plugins.config.PluginCallConfiguration
"""
repo_manager = manager_factory.repo_manager()
scratchpad = repo_manager.get_repo_scratchpad(repo.id)
tags = scratchpad.get(u'tags', [])
unit_ids = set([unit.unit_key[u'image_id'] for unit in units])
for tag_dict in tags[:]:
if tag_dict[constants.IMAGE_ID_KEY] in unit_ids:
tags.remove(tag_dict)
scratchpad[u'tags'] = tags
repo_manager.set_repo_scratchpad(repo.id, scratchpad)
def post(self, request):
"""
Create a new repo. 'id' field in body is required.
:param request: WSGI request object
:type request: django.core.handlers.wsgi.WSGIRequest
:return: Response containing a serialized dict for the created repo.
:rtype : django.http.HttpResponse
"""
# Pull the repo data out of the request body (validation will occur
# in the manager)
repo_data = request.body_as_json
repo_id = repo_data.get('id', None)
display_name = repo_data.get('display_name', None)
description = repo_data.get('description', None)
notes = repo_data.get('notes', None)
importer_type_id = repo_data.get('importer_type_id', None)
importer_repo_plugin_config = repo_data.get('importer_config', None)
distributors = repo_data.get('distributors', None)
# Creation
repo_manager = manager_factory.repo_manager()
args = [repo_id, display_name, description, notes]
kwargs = {'importer_type_id': importer_type_id,
'importer_repo_plugin_config': importer_repo_plugin_config,
'distributor_list': distributors}
repo = repo_manager.create_and_configure_repo(*args, **kwargs)
repo['_href'] = reverse('repo_resource', kwargs={'repo_id': repo_id})
response = generate_json_response_with_pulp_encoder(repo)
return generate_redirect_response(response, repo['_href'])
def DELETE(self, id):
repo_manager = manager_factory.repo_manager()
resources = {dispatch_constants.RESOURCE_REPOSITORY_TYPE: {id: dispatch_constants.RESOURCE_DELETE_OPERATION}}
tags = [resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE, id), action_tag("delete")]
call_request = CallRequest(repo_manager.delete_repo, [id], resources=resources, tags=tags, archive=True)
return execution.execute_ok(self, call_request)
def setUp(self):
base.PulpServerTests.setUp(self)
mock_plugins.install()
self.upload_manager = manager_factory.content_upload_manager()
self.repo_manager = manager_factory.repo_manager()
self.importer_manager = manager_factory.repo_importer_manager()
def test_syntactic_sugar_methods(self):
"""
Tests the syntactic sugar methods for retrieving specific managers.
"""
# Setup
factory.initialize()
# Test
self.assertTrue(isinstance(factory.authentication_manager(), AuthenticationManager))
self.assertTrue(isinstance(factory.cert_generation_manager(), CertGenerationManager))
self.assertTrue(isinstance(factory.certificate_manager(), CertificateManager))
self.assertTrue(isinstance(factory.password_manager(), PasswordManager))
self.assertTrue(isinstance(factory.permission_manager(), PermissionManager))
self.assertTrue(isinstance(factory.permission_query_manager(), PermissionQueryManager))
self.assertTrue(isinstance(factory.role_manager(), RoleManager))
self.assertTrue(isinstance(factory.role_query_manager(), RoleQueryManager))
self.assertTrue(isinstance(factory.user_manager(), UserManager))
self.assertTrue(isinstance(factory.user_query_manager(), UserQueryManager))
self.assertTrue(isinstance(factory.repo_manager(), RepoManager))
self.assertTrue(isinstance(factory.repo_unit_association_manager(), RepoUnitAssociationManager))
self.assertTrue(isinstance(factory.repo_publish_manager(), RepoPublishManager))
self.assertTrue(isinstance(factory.repo_query_manager(), RepoQueryManager))
self.assertTrue(isinstance(factory.repo_sync_manager(), RepoSyncManager))
self.assertTrue(isinstance(factory.content_manager(), ContentManager))
self.assertTrue(isinstance(factory.content_query_manager(), ContentQueryManager))
self.assertTrue(isinstance(factory.content_upload_manager(), ContentUploadManager))
self.assertTrue(isinstance(factory.consumer_manager(), ConsumerManager))
self.assertTrue(isinstance(factory.topic_publish_manager(), TopicPublishManager))
def POST(self):
# Pull the repo data out of the request body (validation will occur
# in the manager)
repo_data = self.params()
id = repo_data.get("id", None)
display_name = repo_data.get("display_name", None)
description = repo_data.get("description", None)
notes = repo_data.get("notes", None)
importer_type_id = repo_data.get("importer_type_id", None)
importer_repo_plugin_config = repo_data.get("importer_config", None)
distributors = repo_data.get("distributors", None)
# Creation
repo_manager = manager_factory.repo_manager()
args = [id, display_name, description, notes]
kwargs = {
"importer_type_id": importer_type_id,
"importer_repo_plugin_config": importer_repo_plugin_config,
"distributor_list": distributors,
}
repo = repo_manager.create_and_configure_repo(*args, **kwargs)
repo.update(serialization.link.child_link_obj(id))
return self.created(id, repo)
def test_get_with_bindings(self):
"""
Test consumer with bindings.
"""
# Setup
manager = factory.repo_manager()
repo = manager.create_repo(self.REPO_ID)
manager = factory.repo_distributor_manager()
manager.add_distributor(self.REPO_ID, self.DISTRIBUTOR_TYPE_ID, {}, True, distributor_id=self.DISTRIBUTOR_ID)
manager = factory.consumer_manager()
manager.register(self.CONSUMER_ID)
manager = factory.consumer_bind_manager()
bind = manager.bind(self.CONSUMER_ID, self.REPO_ID, self.DISTRIBUTOR_ID)
# Test
params = {"bindings": True}
path = "/v2/consumers/%s/" % self.CONSUMER_ID
status, body = self.get(path, params=params)
# Verify
self.assertEqual(200, status)
self.assertEqual(self.CONSUMER_ID, body["id"])
self.assertTrue("_href" in body)
self.assertTrue(body["_href"].endswith(path))
self.assertTrue("bindings" in body)
bindings = body["bindings"]
self.assertEquals(len(bindings), 1)
self.assertEquals(bindings[0], self.REPO_ID)
def setUp(self):
base.PulpServerTests.setUp(self)
mock_plugins.install()
self.upload_manager = manager_factory.content_upload_manager()
self.repo_manager = manager_factory.repo_manager()
self.importer_manager = manager_factory.repo_importer_manager()
def put(self, request, repo_id):
"""
Update a repository. This call will return synchronously unless a distributor is updated.
:param request: WSGI request object
:type request: django.core.handlers.wsgi.WSGIRequest
:param repo_id: id of repository to be updated
:type repo_id: str
:return: Response containing a serialized dict for the updated repo.
:rtype : django.http.HttpResponse
:raises pulp_exceptions.OperationPostponed: if a distributor is updated, dispatch a task
"""
delta = request.body_as_json.get('delta', None)
importer_config = request.body_as_json.get('importer_config', None)
distributor_configs = request.body_as_json.get('distributor_configs', None)
repo_manager = manager_factory.repo_manager()
task_result = repo_manager.update_repo_and_plugins(repo_id, delta, importer_config,
distributor_configs)
repo = task_result.return_value
repo['_href'] = reverse('repo_resource', kwargs={'repo_id': repo_id})
_convert_repo_dates_to_strings(repo)
# Tasks are spawned if a distributor is updated, raise that as a result
if task_result.spawned_tasks:
raise pulp_exceptions.OperationPostponed(task_result)
result = task_result.serialize()
return generate_json_response_with_pulp_encoder(result)
def test_get_with_bindings(self):
"""
Test consumer with bindings.
"""
# Setup
manager = factory.repo_manager()
repo = manager.create_repo(self.REPO_ID)
manager = factory.repo_distributor_manager()
manager.add_distributor(
self.REPO_ID,
self.DISTRIBUTOR_TYPE_ID,
{},
True,
distributor_id=self.DISTRIBUTOR_ID)
manager = factory.consumer_manager()
manager.register(self.CONSUMER_ID)
manager = factory.consumer_bind_manager()
bind = manager.bind(self.CONSUMER_ID, self.REPO_ID, self.DISTRIBUTOR_ID)
# Test
params = {'bindings':True}
path = '/v2/consumers/%s/' % self.CONSUMER_ID
status, body = self.get(path, params=params)
# Verify
self.assertEqual(200, status)
self.assertEqual(self.CONSUMER_ID, body['id'])
self.assertTrue('_href' in body)
self.assertTrue(body['_href'].endswith(path))
self.assertTrue('bindings' in body)
bindings = body['bindings']
self.assertEquals(len(bindings), 1)
self.assertEquals(bindings[0]['repo_id'], self.REPO_ID)
self.assertEquals(bindings[0]['distributor_id'], self.DISTRIBUTOR_ID)
self.assertEquals(bindings[0]['consumer_actions'], [])
def delete(repo_id):
"""
Get the itinerary for deleting a repository.
1. Delete the repository on the sever.
2. Unbind any bound consumers.
:param repo_id: A repository ID.
:type repo_id: str
:return: A TaskRequest object with the details of any errors or spawned tasks
:rtype TaskRequest
"""
# delete repository
manager = managers.repo_manager()
manager.delete_repo(repo_id)
# append unbind itineraries foreach bound consumer
options = {}
manager = managers.consumer_bind_manager()
additional_tasks = []
errors = []
for bind in manager.find_by_repo(repo_id):
try:
report = consumer.unbind(bind["consumer_id"], bind["repo_id"], bind["distributor_id"], options)
if report:
additional_tasks.extend(report.spawned_tasks)
except Exception, e:
errors.append(e)
def setUp(self):
super(RepoConfigConduitTests, self).setUp()
mock_plugins.install()
manager_factory.initialize()
self.repo_manager = manager_factory.repo_manager()
self.distributor_manager = manager_factory.repo_distributor_manager()
# Populate the database with a repo with units
self.repo_manager.create_repo('repo-1')
self.distributor_manager.add_distributor('repo-1', 'mock-distributor', {"relative_url": "/a/bc/d"},
True, distributor_id='dist-1')
self.distributor_manager.add_distributor('repo-1', 'mock-distributor', {"relative_url": "/a/c"},
True, distributor_id='dist-2')
self.repo_manager.create_repo('repo-2')
self.distributor_manager.add_distributor('repo-2', 'mock-distributor', {"relative_url": "/a/bc/e"},
True, distributor_id='dist-3')
self.repo_manager.create_repo('repo-3')
self.distributor_manager.add_distributor('repo-3', 'mock-distributor', {},
True, distributor_id='dist-4')
self.repo_manager.create_repo('repo-4')
self.distributor_manager.add_distributor('repo-4', 'mock-distributor', {"relative_url": "/repo-5"},
True, distributor_id='dist-5')
self.conduit = RepoConfigConduit('rpm')
def test_post(self, _reserve_resource, mock_apply_async):
# Setup
task_id = str(uuid.uuid4())
mock_apply_async.return_value = AsyncResult(task_id)
_reserve_resource.return_value = ReservedResourceApplyAsync()
upload_id = self.upload_manager.initialize_upload()
self.upload_manager.save_data(upload_id, 0, 'string data')
repo_manager = manager_factory.repo_manager()
repo_manager.create_repo('repo-upload')
importer_manager = manager_factory.repo_importer_manager()
importer_manager.set_importer('repo-upload', 'dummy-importer', {})
# Test
body = {
'upload_id' : upload_id,
'unit_type_id' : 'dummy-type',
'unit_key' : {'name' : 'foo'},
'unit_metadata' : {'stuff' : 'bar'},
}
status, body = self.post('/v2/repositories/repo-upload/actions/import_upload/', body)
# Verify
self.assertEqual(202, status)
assert_body_matches_async_task(body, mock_apply_async.return_value)
exepcted_call_args = ['repo-upload', 'dummy-type',
{'name': 'foo'}, {'stuff': 'bar'},
upload_id]
self.assertEqual(exepcted_call_args, mock_apply_async.call_args[0][0])
def POST(self):
# Pull the repo data out of the request body (validation will occur
# in the manager)
repo_data = self.params()
id = repo_data.get('id', None)
display_name = repo_data.get('display_name', None)
description = repo_data.get('description', None)
notes = repo_data.get('notes', None)
importer_type_id = repo_data.get('importer_type_id', None)
importer_repo_plugin_config = repo_data.get('importer_config', None)
distributors = repo_data.get('distributors', None)
# Creation
repo_manager = manager_factory.repo_manager()
args = [id, display_name, description, notes]
kwargs = {
'importer_type_id': importer_type_id,
'importer_repo_plugin_config': importer_repo_plugin_config,
'distributor_list': distributors
}
repo = repo_manager.create_and_configure_repo(*args, **kwargs)
repo.update(serialization.link.child_link_obj(id))
return self.created(id, repo)
def POST(self):
# Pull the repo data out of the request body (validation will occur
# in the manager)
repo_data = self.params()
id = repo_data.get('id', None)
display_name = repo_data.get('display_name', None)
description = repo_data.get('description', None)
notes = repo_data.get('notes', None)
importer_type_id = repo_data.get('importer_type_id', None)
importer_repo_plugin_config = repo_data.get('importer_config', None)
distributors = repo_data.get('distributors', None)
# Creation
repo_manager = manager_factory.repo_manager()
resources = {dispatch_constants.RESOURCE_REPOSITORY_TYPE: {id: dispatch_constants.RESOURCE_CREATE_OPERATION}}
args = [id, display_name, description, notes, importer_type_id, importer_repo_plugin_config, distributors]
weight = pulp_config.config.getint('tasks', 'create_weight')
tags = [resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE, id),
action_tag('create')]
call_request = CallRequest(repo_manager.create_and_configure_repo,
args,
resources=resources,
weight=weight,
tags=tags)
repo = execution.execute_sync(call_request)
repo.update(serialization.link.child_link_obj(id))
return self.created(id, repo)
def test_post_with_override_config(self, mock_get_worker_for_reservation, mock_uuid,
mock_apply_async):
# Setup
uuid_list = [uuid.uuid4() for i in range(10)]
mock_uuid.uuid4.side_effect = copy.deepcopy(uuid_list)
expected_async_result = AsyncResult(str(uuid_list[0]))
mock_get_worker_for_reservation.return_value = Worker('some_queue', datetime.datetime.now())
upload_id = self.upload_manager.initialize_upload()
self.upload_manager.save_data(upload_id, 0, 'string data')
repo_manager = manager_factory.repo_manager()
repo_manager.create_repo('repo-upload')
importer_manager = manager_factory.repo_importer_manager()
importer_manager.set_importer('repo-upload', 'dummy-importer', {})
# Test
test_override_config = {'key1': 'value1', 'key2': 'value2'}
body = {
'upload_id' : upload_id,
'unit_type_id' : 'dummy-type',
'unit_key' : {'name' : 'foo'},
'unit_metadata' : {'stuff' : 'bar'},
'override_config': test_override_config,
}
status, body = self.post('/v2/repositories/repo-upload/actions/import_upload/', body)
# Verify
self.assertEqual(202, status)
assert_body_matches_async_task(body, expected_async_result)
exepcted_call_args = ['repo-upload', 'dummy-type',
{'name': 'foo'}, {'stuff': 'bar'},
upload_id, test_override_config]
self.assertEqual(exepcted_call_args, mock_apply_async.call_args[0][0])
def setUp(self):
super(RepoConfigConduitTests, self).setUp()
mock_plugins.install()
manager_factory.initialize()
self.repo_manager = manager_factory.repo_manager()
self.distributor_manager = manager_factory.repo_distributor_manager()
# Populate the database with a repo with units
self.repo_manager.create_repo('repo-1')
self.distributor_manager.add_distributor(
'repo-1', 'mock-distributor', {"relative_url": "/a/bc/d"}, True,
distributor_id='dist-1')
self.distributor_manager.add_distributor(
'repo-1', 'mock-distributor', {"relative_url": "/a/c"}, True, distributor_id='dist-2')
self.repo_manager.create_repo('repo-2')
self.distributor_manager.add_distributor(
'repo-2', 'mock-distributor', {"relative_url": "/a/bc/e"}, True,
distributor_id='dist-3')
self.repo_manager.create_repo('repo-3')
self.distributor_manager.add_distributor('repo-3', 'mock-distributor', {},
True, distributor_id='dist-4')
self.repo_manager.create_repo('repo-4')
self.distributor_manager.add_distributor(
'repo-4', 'mock-distributor', {"relative_url": "repo-5"}, True, distributor_id='dist-5')
self.repo_manager.create_repo('repo-5')
self.distributor_manager.add_distributor(
'repo-5', 'mock-distributor', {"relative_url": "a/bcd/e"}, True,
distributor_id='dist-1')
self.repo_manager.create_repo('repo-6')
self.distributor_manager.add_distributor(
'repo-6', 'mock-distributor', {"relative_url": "a/bcde/f/"}, True,
distributor_id='dist-1')
self.conduit = RepoConfigConduit('rpm')
def test_syntactic_sugar_methods(self):
"""
Tests the syntactic sugar methods for retrieving specific managers.
"""
# Setup
factory.initialize()
# Test
self.assertTrue(isinstance(factory.authentication_manager(), AuthenticationManager))
self.assertTrue(isinstance(factory.cert_generation_manager(), CertGenerationManager))
self.assertTrue(isinstance(factory.certificate_manager(), CertificateManager))
self.assertTrue(isinstance(factory.password_manager(), PasswordManager))
self.assertTrue(isinstance(factory.permission_manager(), PermissionManager))
self.assertTrue(isinstance(factory.permission_query_manager(), PermissionQueryManager))
self.assertTrue(isinstance(factory.role_manager(), RoleManager))
self.assertTrue(isinstance(factory.role_query_manager(), RoleQueryManager))
self.assertTrue(isinstance(factory.user_manager(), UserManager))
self.assertTrue(isinstance(factory.user_query_manager(), UserQueryManager))
self.assertTrue(isinstance(factory.repo_manager(), RepoManager))
self.assertTrue(isinstance(factory.repo_unit_association_manager(),
RepoUnitAssociationManager))
self.assertTrue(isinstance(factory.repo_publish_manager(), RepoPublishManager))
self.assertTrue(isinstance(factory.repo_query_manager(), RepoQueryManager))
self.assertTrue(isinstance(factory.repo_sync_manager(), RepoSyncManager))
self.assertTrue(isinstance(factory.content_manager(), ContentManager))
self.assertTrue(isinstance(factory.content_query_manager(), ContentQueryManager))
self.assertTrue(isinstance(factory.content_upload_manager(), ContentUploadManager))
self.assertTrue(isinstance(factory.consumer_manager(), ConsumerManager))
self.assertTrue(isinstance(factory.topic_publish_manager(), TopicPublishManager))
def test_post(self):
# Setup
upload_id = self.upload_manager.initialize_upload()
self.upload_manager.save_data(upload_id, 0, 'string data')
repo_manager = manager_factory.repo_manager()
repo_manager.create_repo('repo-upload')
importer_manager = manager_factory.repo_importer_manager()
importer_manager.set_importer('repo-upload', 'dummy-importer', {})
# Test
body = {
'upload_id': upload_id,
'unit_type_id': 'dummy-type',
'unit_key': {
'name': 'foo'
},
'unit_metadata': {
'stuff': 'bar'
},
}
status, body = self.post(
'/v2/repositories/repo-upload/actions/import_upload/', body)
# Verify
self.assertEqual(200, status)
def setUp(self):
super(RepoQueryManagerTests, self).setUp()
mock_plugins.install()
self.repo_manager = manager_factory.repo_manager()
self.importer_manager = manager_factory.repo_importer_manager()
self.distributor_manager = manager_factory.repo_distributor_manager()
self.query_manager = manager_factory.repo_query_manager()
def test_associate_by_id_does_not_call_update_unit_count(self):
"""
This would be the case when doing a bulk update.
"""
self.manager.associate_unit_by_id(
'repo-1', 'type-1', 'unit-1', OWNER_TYPE_USER, 'admin', False)
mock_manager = manager_factory.repo_manager()
self.assertFalse(mock_manager.update_unit_count.called)
def populate(self):
config = {"key1": "value1", "key2": None}
manager = factory.repo_manager()
repo = manager.create_repo(self.REPO_ID)
manager = factory.repo_distributor_manager()
manager.add_distributor(self.REPO_ID, "mock-distributor", config, True, distributor_id=self.DISTRIBUTOR_ID)
manager = factory.consumer_manager()
manager.register(self.CONSUMER_ID)
def associate_all_by_ids(self, repo_id, unit_type_id, unit_id_list, owner_type, owner_id):
"""
Creates multiple associations between the given repo and content units.
See associate_unit_by_id for semantics.
@param repo_id: identifies the repo
@type repo_id: str
@param unit_type_id: identifies the type of unit being added
@type unit_type_id: str
@param unit_id_list: list or generator of unique identifiers for units within the given type
@type unit_id_list: list or generator of str
@param owner_type: category of the caller making the association;
must be one of the OWNER_* variables in this module
@type owner_type: str
@param owner_id: identifies the caller making the association, either
the importer ID or user login
@type owner_id: str
:return: number of new units added to the repo
:rtype: int
@raise InvalidType: if the given owner type is not of the valid enumeration
"""
# There may be a way to batch this in mongo which would be ideal for a
# bulk operation like this. But for deadline purposes, this call will
# simply loop and call the single method.
unique_count = 0
for unit_id in unit_id_list:
if not RepoUnitAssociationManager.association_exists(repo_id, unit_id, unit_type_id):
unique_count += 1
self.associate_unit_by_id(repo_id, unit_type_id, unit_id, owner_type, owner_id, False)
# update the count of associated units on the repo object
if unique_count:
manager_factory.repo_manager().update_unit_count(
repo_id, unit_type_id, unique_count)
# update the timestamp for when the units were added to the repo
manager_factory.repo_manager().update_last_unit_added(repo_id)
return unique_count
def setUp(self):
super(RepoQueryManagerTests, self).setUp()
mock_plugins.install()
self.repo_manager = manager_factory.repo_manager()
self.importer_manager = manager_factory.repo_importer_manager()
self.distributor_manager = manager_factory.repo_distributor_manager()
self.query_manager = manager_factory.repo_query_manager()
def test_post_with_repos(self):
# Setup
manager_factory.repo_manager().create_repo('add-me')
data = {
'id' : 'with-repos',
'repo_ids' : ['add-me']
}
# Test
status, body = self.post('/v2/repo_groups/', data)
# Verify
self.assertEqual(201, status)
found = RepoGroup.get_collection().find_one({'id' : data['id']})
self.assertEqual(found['repo_ids'], data['repo_ids'])
def test_unassociate_by_id_calls_update_unit_count(self):
self.manager.associate_unit_by_id(
'repo-1', 'type-1', 'unit-1', OWNER_TYPE_USER, 'admin')
mock_manager = manager_factory.repo_manager()
mock_manager.reset_mock()
self.manager.unassociate_unit_by_id(
'repo-1', 'type-1', 'unit-1', OWNER_TYPE_USER, 'admin')
mock_manager.update_unit_count.assert_called_once_with('repo-1', -1)
def test_unassociate_by_id_does_not_call_update_unit_count(self):
self.manager.associate_unit_by_id(
'repo-1', 'type-1', 'unit-1', OWNER_TYPE_USER, 'admin')
mock_manager = manager_factory.repo_manager()
mock_manager.reset_mock()
self.manager.unassociate_unit_by_id(
'repo-1', 'type-1', 'unit-1', OWNER_TYPE_USER, 'admin', False)
self.assertFalse(mock_manager.update_unit_count.called)
def populate(self):
manager = factory.repo_manager()
repo = manager.create_repo(self.REPO_ID)
manager = factory.repo_distributor_manager()
manager.add_distributor(self.REPO_ID, self.DISTRIBUTOR_TYPE_ID, {}, True, distributor_id=self.DISTRIBUTOR_ID)
mock_plugins.MOCK_DISTRIBUTOR.create_consumer_payload.return_value = self.PAYLOAD
manager = factory.consumer_manager()
manager.register(self.CONSUMER_ID)
def unassociate_by_criteria(repo_id, criteria, owner_type, owner_id, notify_plugins=True):
"""
Unassociate units that are matched by the given criteria.
:param repo_id: identifies the repo
:type repo_id: str
:param criteria:
:param owner_type: category of the caller who created the association
:type owner_type: str
:param owner_id: identifies the call who created the association
:type owner_id: str
:param notify_plugins: if true, relevant plugins will be informed of the removal
:type notify_plugins: bool
"""
association_query_manager = manager_factory.repo_unit_association_query_manager()
unassociate_units = association_query_manager.get_units(repo_id, criteria=criteria)
if len(unassociate_units) == 0:
return {}
unit_map = {} # maps unit_type_id to a list of unit_ids
for unit in unassociate_units:
id_list = unit_map.setdefault(unit['unit_type_id'], [])
id_list.append(unit['unit_id'])
collection = RepoContentUnit.get_collection()
repo_manager = manager_factory.repo_manager()
for unit_type_id, unit_ids in unit_map.items():
spec = {'repo_id': repo_id,
'unit_type_id': unit_type_id,
'unit_id': {'$in': unit_ids}
}
collection.remove(spec, safe=True)
unique_count = sum(
1 for unit_id in unit_ids if not RepoUnitAssociationManager.association_exists(
repo_id, unit_id, unit_type_id))
if not unique_count:
continue
repo_manager.update_unit_count(repo_id, unit_type_id, -unique_count)
repo_manager.update_last_unit_removed(repo_id)
# Convert the units into transfer units. This happens regardless of whether or not
# the plugin will be notified as it's used to generate the return result,
unit_type_ids = calculate_associated_type_ids(repo_id, unassociate_units)
transfer_units = create_transfer_units(unassociate_units, unit_type_ids)
if notify_plugins:
remove_from_importer(repo_id, transfer_units)
# Match the return type/format as copy
serializable_units = [u.to_id_dict() for u in transfer_units]
return {'units_successful': serializable_units}
def test_associate_all_by_ids_calls_update_unit_count(self):
IDS = ('foo', 'bar', 'baz')
self.manager.associate_all_by_ids(
'repo-1', 'type-1', IDS, OWNER_TYPE_USER, 'admin')
mock_manager = manager_factory.repo_manager()
mock_manager.update_unit_count.assert_called_once_with(
'repo-1', len(IDS))
def associate_unit_by_id(self, repo_id, unit_type_id, unit_id, update_repo_metadata=True):
"""
Creates an association between the given repository and content unit.
If there is already an association between the given repo and content
unit where all other metadata matches the input to this method,
this call has no effect.
Both repo and unit must exist in the database prior to this call,
however this call will not verify that for performance reasons. Care
should be taken by the caller to preserve the data integrity.
@param repo_id: identifies the repo
@type repo_id: str
@param unit_type_id: identifies the type of unit being added
@type unit_type_id: str
@param unit_id: uniquely identifies the unit within the given type
@type unit_id: str
@param update_repo_metadata: if True, updates the unit association count
after the new association is made. The last
unit added field will also be updated. Set this
to False when doing bulk associations, and
make one call to update the count at the end.
defaults to True
@type update_repo_metadata: bool
@raise InvalidType: if the given owner type is not of the valid enumeration
"""
# If the association already exists, no need to do anything else
spec = {'repo_id': repo_id,
'unit_id': unit_id,
'unit_type_id': unit_type_id}
existing_association = RepoContentUnit.get_collection().find_one(spec)
if existing_association is not None:
return
similar_exists = False
if update_repo_metadata:
similar_exists = RepoUnitAssociationManager.association_exists(repo_id, unit_id,
unit_type_id)
# Create the database entry
association = RepoContentUnit(repo_id, unit_id, unit_type_id)
RepoContentUnit.get_collection().save(association, safe=True)
manager = manager_factory.repo_manager()
# update the count of associated units on the repo object
if update_repo_metadata and not similar_exists:
manager.update_unit_count(repo_id, unit_type_id, 1)
# update the record for the last added field
manager.update_last_unit_added(repo_id)
def unassociate_by_criteria(repo_id, criteria, owner_type, owner_id, notify_plugins=True):
"""
Unassociate units that are matched by the given criteria.
:param repo_id: identifies the repo
:type repo_id: str
:param criteria:
:param owner_type: category of the caller who created the association
:type owner_type: str
:param owner_id: identifies the call who created the association
:type owner_id: str
:param notify_plugins: if true, relevant plugins will be informed of the removal
:type notify_plugins: bool
"""
association_query_manager = manager_factory.repo_unit_association_query_manager()
unassociate_units = association_query_manager.get_units(repo_id, criteria=criteria)
if len(unassociate_units) == 0:
return {}
unit_map = {} # maps unit_type_id to a list of unit_ids
for unit in unassociate_units:
id_list = unit_map.setdefault(unit['unit_type_id'], [])
id_list.append(unit['unit_id'])
collection = RepoContentUnit.get_collection()
repo_manager = manager_factory.repo_manager()
for unit_type_id, unit_ids in unit_map.items():
spec = {'repo_id': repo_id,
'unit_type_id': unit_type_id,
'unit_id': {'$in': unit_ids}
}
collection.remove(spec, safe=True)
unique_count = sum(
1 for unit_id in unit_ids if not RepoUnitAssociationManager.association_exists(
repo_id, unit_id, unit_type_id))
if not unique_count:
continue
repo_manager.update_unit_count(repo_id, unit_type_id, -unique_count)
# Convert the units into transfer units. This happens regardless of whether or not
# the plugin will be notified as it's used to generate the return result,
unit_type_ids = calculate_associated_type_ids(repo_id, unassociate_units)
transfer_units = create_transfer_units(unassociate_units, unit_type_ids)
if notify_plugins:
remove_from_importer(repo_id, transfer_units)
# Match the return type/format as copy
serializable_units = [u.to_id_dict() for u in transfer_units]
return {'units_successful': serializable_units}
def populate_repository(self):
config = {'key1': 'value1', 'key2': None}
manager = factory.repo_manager()
manager.create_repo(self.REPO_ID)
manager = factory.repo_distributor_manager()
manager.add_distributor(self.REPO_ID,
'mock-distributor',
config,
True,
distributor_id=self.DISTRIBUTOR_ID)
def populate(self):
manager = factory.repo_manager()
manager.create_repo(self.REPO_ID)
manager = factory.repo_distributor_manager()
manager.add_distributor(self.REPO_ID,
self.DISTRIBUTOR_TYPE_ID, {},
True,
distributor_id=self.DISTRIBUTOR_ID)
mock_plugins.MOCK_DISTRIBUTOR.create_consumer_payload.return_value = self.PAYLOAD
manager = factory.consumer_manager()
manager.register(self.CONSUMER_ID)
def populate(self):
# make content/ dir.
os.makedirs(os.path.join(self.parentfs, 'content'))
pulp_conf.set('server', 'storage_dir', self.parentfs)
# create repo
manager = managers.repo_manager()
manager.create_repo(
self.REPO_ID, display_name=REPO_NAME, description=REPO_DESCRIPTION, notes=REPO_NOTES)
manager.set_repo_scratchpad(self.REPO_ID, REPO_SCRATCHPAD)
# add units
units = self.add_units(0, self.NUM_UNITS)
self.units = units
def update_tags(repo_id, new_tags):
"""
Gets the current scratchpad's tags and updates them with the new_tags
:param repo_id: unique ID of a repository
:type repo_id: basestring
:param new_tags: dictionary of tag:image_id
:type new_tags: dict
"""
repo_manager = factory.repo_manager()
scratchpad = repo_manager.get_repo_scratchpad(repo_id)
new_tags = tags.generate_updated_tags(scratchpad, new_tags)
repo_manager.update_repo_scratchpad(repo_id, {'tags': new_tags})
def populate(self):
manager = factory.repo_manager()
manager.create_repo(self.REPO_ID)
manager = factory.repo_distributor_manager()
manager.add_distributor(self.REPO_ID,
self.DISTRIBUTOR_TYPE_ID, {},
True,
distributor_id=self.DISTRIBUTOR_ID)
manager = factory.consumer_manager()
manager.register(self.CONSUMER_ID)
manager = factory.consumer_bind_manager()
manager.bind(self.CONSUMER_ID, self.REPO_ID, self.DISTRIBUTOR_ID,
self.NOTIFY_AGENT, self.BINDING_CONFIG)
def setUp(self):
super(RepoPublishConduitTests, self).setUp()
mock_plugins.install()
manager_factory.initialize()
self.repo_manager = manager_factory.repo_manager()
self.distributor_manager = manager_factory.repo_distributor_manager()
# Populate the database with a repo with units
self.repo_manager.create_repo('repo-1')
self.distributor_manager.add_distributor('repo-1', 'mock-distributor', {}, True, distributor_id='dist-1')
self.conduit = RepoPublishConduit('repo-1', 'dist-1')
def test_delete_repo(self):
group_id = 'delete_from_me'
repo = self._create_repo('delete_me')
self.manager.create_repo_group(group_id, repo_ids=[repo['id']])
group = self.collection.find_one({'id': group_id})
self.assertTrue(repo['id'] in group['repo_ids'])
repo_manager = managers_factory.repo_manager()
repo_manager.delete_repo(repo['id'])
group = self.collection.find_one({'id': group_id})
self.assertFalse(repo['id'] in group['repo_ids'])
def populate(self):
config = {'key1': 'value1', 'key2': None}
manager = factory.repo_manager()
repo = manager.create_repo(self.REPO_ID)
manager = factory.repo_distributor_manager()
manager.add_distributor(self.REPO_ID,
'mock-distributor',
config,
True,
distributor_id=self.DISTRIBUTOR_ID)
manager = factory.consumer_manager()
for consumer_id in self.ALL_CONSUMERS:
manager.register(consumer_id)
def update_repo_scratchpad(self, scratchpad):
"""
Update the repository scratchpad with the specified key-value pairs.
New keys are added; existing keys are updated.
:param scratchpad: a dict used to update the scratchpad.
"""
try:
manager = manager_factory.repo_manager()
manager.update_repo_scratchpad(self.repo_id, scratchpad)
except Exception, e:
msg = _('Error updating repository scratchpad for repo [%(r)s]') % {'r': self.repo_id}
logger.exception(msg)
raise self.exception_class(e), None, sys.exc_info()[2]
def get_repo_scratchpad(self, repo_id):
"""
Returns the repository-level scratchpad for the indicated repository.
@raise ImporterConduitException: wraps any exception that may occur
in the Pulp server
"""
try:
repo_manager = manager_factory.repo_manager()
value = repo_manager.get_repo_scratchpad(repo_id)
return value
except Exception, e:
logger.exception(_('Error getting repository scratchpad for repo [%(r)s]') % {'r' : repo_id})
raise self.exception_class(e), None, sys.exc_info()[2]
def get_repo_scratchpad(self):
"""
Returns the repository-level scratchpad for this repository. The
repository-level scratchpad can be seen and edited by all importers
and distributors on the repository. Care should be taken to not destroy
any data set by another plugin. This may be used to communicate between
importers, distributors and profilers relevant data for the repository.
"""
try:
repo_manager = manager_factory.repo_manager()
value = repo_manager.get_repo_scratchpad(self.repo_id)
return value
except Exception, e:
logger.exception(_('Error getting repository scratchpad for repo [%(r)s]') % {'r' : self.repo_id})
raise self.exception_class(e), None, sys.exc_info()[2]
def POST(self):
# Pull the repo data out of the request body (validation will occur
# in the manager)
repo_data = self.params()
id = repo_data.get('id', None)
display_name = repo_data.get('display_name', None)
description = repo_data.get('description', None)
notes = repo_data.get('notes', None)
importer_type_id = repo_data.get('importer_type_id', None)
importer_repo_plugin_config = repo_data.get('importer_config', None)
distributors = repo_data.get('distributors', None)
# Creation
repo_manager = manager_factory.repo_manager()
resources = {
dispatch_constants.RESOURCE_REPOSITORY_TYPE: {
id: dispatch_constants.RESOURCE_CREATE_OPERATION
}
}
args = [id, display_name, description, notes]
kwargs = {
'importer_type_id': importer_type_id,
'importer_repo_plugin_config': importer_repo_plugin_config,
'distributor_list': distributors
}
weight = pulp_config.config.getint('tasks', 'create_weight')
tags = [
resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE, id),
action_tag('create')
]
call_request = CallRequest(repo_manager.create_and_configure_repo,
args,
kwargs,
resources=resources,
weight=weight,
tags=tags,
kwarg_blacklist=[
'importer_repo_plugin_config',
'distributor_list'
])
repo = execution.execute_sync(call_request)
repo.update(serialization.link.child_link_obj(id))
return self.created(id, repo)
def populate(self):
manager = managers.consumer_manager()
for consumer_id in CONSUMER_IDS:
manager.register(consumer_id)
manager = managers.consumer_group_manager()
manager.create_consumer_group(GROUP_ID)
for consumer_id in CONSUMER_IDS:
criteria = Criteria(filters={'id': consumer_id}, fields=['id'])
manager.associate(GROUP_ID, criteria)
manager = managers.repo_manager()
manager.create_repo(REPO_ID)
manager = managers.repo_distributor_manager()
manager.add_distributor(REPO_ID,
DISTRIBUTOR_TYPE_ID, {},
True,
distributor_id=DISTRIBUTOR_ID)
