def test_resolve_dependencies_by_unit_no_importer(self): # Setup manager_factory.repo_manager().create_repo('empty') # Test self.assertRaises(MissingResource, self.manager.resolve_dependencies_by_units, 'empty', [], {})
def unassociate_all_by_ids(self, repo_id, unit_type_id, unit_id_list, owner_type, owner_id): """ Removes the association between a repo and a number of units. Only the association made by the given owner will be removed. It is possible the repo will still have a manually created association will for the unit. @param repo_id: identifies the repo @type repo_id: str @param unit_type_id: identifies the type of units being removed @type unit_type_id: str @param unit_id_list: list of unique identifiers for units within the given type @type unit_id_list: list of str @param owner_type: category of the caller who created the association; must be one of the OWNER_* variables in this module @type owner_type: str @param owner_id: identifies the caller who created the association, either the importer ID or user login @type owner_id: str """ spec = {'repo_id' : repo_id, 'unit_type_id' : unit_type_id, 'unit_id' : {'$in' : unit_id_list}, 'owner_type' : owner_type, 'owner_id' : owner_id, } unit_coll = RepoContentUnit.get_collection() unit_coll.remove(spec, safe=True) unique_count = sum(1 for unit_id in unit_id_list if not self.association_exists(repo_id, unit_id, unit_type_id)) # update the count of associated units on the repo object if unique_count: manager_factory.repo_manager().update_unit_count( repo_id, -unique_count) try: repo_query_manager = manager_factory.repo_query_manager() repo = repo_query_manager.get_repository(repo_id) content_query_manager = manager_factory.content_query_manager() content_units = content_query_manager.get_multiple_units_by_ids(unit_type_id, unit_id_list) importer_manager = manager_factory.repo_importer_manager() importer = importer_manager.get_importer(repo_id) importer.remove_units(repo, content_units) except: _LOG.exception('Exception informing importer for [%s] of unassociation' % repo_id)
def setUp(self): super(DependencyManagerTests, self).setUp() mock_plugins.install() database.update_database([TYPE_1_DEF]) self.repo_id = 'dep-repo' self.manager = manager_factory.dependency_manager() manager_factory.repo_manager().create_repo(self.repo_id) manager_factory.repo_importer_manager().set_importer(self.repo_id, 'mock-importer', {})
def preserve_custom_metadata_on_repo_scratchpad(): """ Lookups all the yum based repos in pulp; grabs any custom metadata and set the the data on repo scratchpad. """ factory.initialize() repos = factory.repo_query_manager().find_with_importer_type( "yum_importer") if not repos: _log.debug("No repos found to perform db migrate") return repo_ids = [repo['id'] for repo in repos] for repo_id in repo_ids: _log.debug("Processing repo %s" % repo_id) repo_scratchpad = factory.repo_manager().get_repo_scratchpad(repo_id) if "repodata" in repo_scratchpad and repo_scratchpad["repodata"]: # repo scratchpad already has repodata, skip migration _log.debug( "repo [%s] scratchpad already has repodata, skip migration" % repo_id) continue repo_working_dir = importer_working_dir('yum_importer', repo_id) importer_repodata_dir = os.path.join(repo_working_dir, repo_id, "repodata") repomd_xml_path = os.path.join(importer_repodata_dir, "repomd.xml") if not os.path.exists(repomd_xml_path): # repodata doesn't exist on filesystem cannot lookup custom data, continue to next continue ftypes = util.get_repomd_filetypes(repomd_xml_path) base_ftypes = [ 'primary', 'primary_db', 'filelists_db', 'filelists', 'other', 'other_db', 'group', 'group_gz', 'updateinfo', 'updateinfo_db' ] for ftype in ftypes: if ftype in base_ftypes: # no need to process these again continue filetype_path = os.path.join( importer_repodata_dir, os.path.basename( util.get_repomd_filetype_path(repomd_xml_path, ftype))) if filetype_path.endswith('.gz'): # if file is gzipped, decompress data = gzip.open(filetype_path).read().decode( "utf-8", "replace") else: data = open(filetype_path).read().decode("utf-8", "replace") repo_scratchpad["repodata"].update({ftype: data}) # set the custom metadata on scratchpad factory.repo_manager().set_repo_scratchpad(repo_id, repo_scratchpad) _log.info("Updated repo [%s] scratchpad with new custom repodata" % repo_id)
def setUp(self): super(DependencyManagerTests, self).setUp() mock_plugins.install() database.update_database([TYPE_1_DEF]) self.repo_id = 'dep-repo' self.manager = manager_factory.dependency_manager() manager_factory.repo_manager().create_repo(self.repo_id) manager_factory.repo_importer_manager().set_importer( self.repo_id, 'mock-importer', {})
def test_post_with_repos(self): # Setup manager_factory.repo_manager().create_repo('add-me') data = {'id': 'with-repos', 'repo_ids': ['add-me']} # Test status, body = self.post('/v2/repo_groups/', data) # Verify self.assertEqual(201, status) found = RepoGroup.get_collection().find_one({'id': data['id']}) self.assertEqual(found['repo_ids'], data['repo_ids'])
def test_post_with_repos(self): # Setup manager_factory.repo_manager().create_repo("add-me") data = {"id": "with-repos", "repo_ids": ["add-me"]} # Test status, body = self.post("/v2/repo_groups/", data) # Verify self.assertEqual(201, status) found = RepoGroup.get_collection().find_one({"id": data["id"]}) self.assertEqual(found["repo_ids"], data["repo_ids"])
def associate_all_by_ids(self, repo_id, unit_type_id, unit_id_list, owner_type, owner_id): """ Creates multiple associations between the given repo and content units. See associate_unit_by_id for semantics. @param repo_id: identifies the repo @type repo_id: str @param unit_type_id: identifies the type of unit being added @type unit_type_id: str @param unit_id_list: list or generator of unique identifiers for units within the given type @type unit_id_list: list or generator of str @param owner_type: category of the caller making the association; must be one of the OWNER_* variables in this module @type owner_type: str @param owner_id: identifies the caller making the association, either the importer ID or user login @type owner_id: str :return: number of new units added to the repo :rtype: int @raise InvalidType: if the given owner type is not of the valid enumeration """ # There may be a way to batch this in mongo which would be ideal for a # bulk operation like this. But for deadline purposes, this call will # simply loop and call the single method. unique_count = 0 for unit_id in unit_id_list: if not RepoUnitAssociationManager.association_exists( repo_id, unit_id, unit_type_id): unique_count += 1 self.associate_unit_by_id(repo_id, unit_type_id, unit_id, owner_type, owner_id, False) # update the count of associated units on the repo object if unique_count: manager_factory.repo_manager().update_unit_count( repo_id, unit_type_id, unique_count) # update the timestamp for when the units were added to the repo manager_factory.repo_manager().update_last_unit_added(repo_id) return unique_count
def remove_units(self, repo, units, config): """ Removes content units from the given repository. This method also removes the tags associated with images in the repository. This call will not result in the unit being deleted from Pulp itself. :param repo: metadata describing the repository :type repo: pulp.plugins.model.Repository :param units: list of objects describing the units to import in this call :type units: list of pulp.plugins.model.AssociatedUnit :param config: plugin configuration :type config: pulp.plugins.config.PluginCallConfiguration """ repo_manager = manager_factory.repo_manager() scratchpad = repo_manager.get_repo_scratchpad(repo.id) tags = scratchpad.get(u'tags', []) unit_ids = set([unit.unit_key[u'image_id'] for unit in units]) for tag_dict in tags[:]: if tag_dict[constants.IMAGE_ID_KEY] in unit_ids: tags.remove(tag_dict) scratchpad[u'tags'] = tags repo_manager.set_repo_scratchpad(repo.id, scratchpad)
def repo_delete_itinerary(repo_id): """ Get the itinerary for deleting a repository. 1. Delete the repository on the sever. 2. Unbind any bound consumers. @param repo_id: A repository ID. @type repo_id: str @return: A list of call_requests known as an itinerary. @rtype list """ call_requests = [] # delete repository manager = managers.repo_manager() resources = {dispatch_constants.RESOURCE_REPOSITORY_TYPE: {repo_id: dispatch_constants.RESOURCE_DELETE_OPERATION}} tags = [resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE, repo_id), action_tag("delete")] delete_request = CallRequest(manager.delete_repo, [repo_id], resources=resources, tags=tags, archive=True) call_requests.append(delete_request) # append unbind itineraries foreach bound consumer options = {} manager = managers.consumer_bind_manager() for bind in manager.find_by_repo(repo_id): unbind_requests = unbind_itinerary(bind["consumer_id"], bind["repo_id"], bind["distributor_id"], options) if unbind_requests: unbind_requests[0].depends_on(delete_request.id) call_requests.extend(unbind_requests) return call_requests
def PUT(self, id): parameters = self.params() delta = parameters.get('delta', None) importer_config = parameters.get('importer_config', None) distributor_configs = parameters.get('distributor_configs', None) repo_manager = manager_factory.repo_manager() tags = [ resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE, id), action_tag('update') ] call_request = CallRequest( repo_manager.update_repo_and_plugins, [id, delta], { 'importer_config': importer_config, 'distributor_configs': distributor_configs }, tags=tags, archive=True, kwarg_blacklist=['importer_config', 'distributor_configs']) call_request.updates_resource( dispatch_constants.RESOURCE_REPOSITORY_TYPE, id) repo = execution.execute(call_request) repo.update(serialization.link.current_link_obj()) return self.ok(repo)
def delete(repo_id): """ Get the itinerary for deleting a repository. 1. Delete the repository on the sever. 2. Unbind any bound consumers. :param repo_id: A repository ID. :type repo_id: str :return: A TaskRequest object with the details of any errors or spawned tasks :rtype TaskRequest """ # delete repository manager = managers.repo_manager() manager.delete_repo(repo_id) # append unbind itineraries foreach bound consumer options = {} manager = managers.consumer_bind_manager() additional_tasks = [] errors = [] for bind in manager.find_by_repo(repo_id): try: report = consumer.unbind(bind['consumer_id'], bind['repo_id'], bind['distributor_id'], options) if report: additional_tasks.extend(report.spawned_tasks) except Exception, e: errors.append(e)
def post(self, request): """ Create a new repo. 'id' field in body is required. :param request: WSGI request object :type request: django.core.handlers.wsgi.WSGIRequest :return: Response containing a serialized dict for the created repo. :rtype : django.http.HttpResponse """ # Pull the repo data out of the request body (validation will occur # in the manager) repo_data = request.body_as_json repo_id = repo_data.get('id', None) display_name = repo_data.get('display_name', None) description = repo_data.get('description', None) notes = repo_data.get('notes', None) importer_type_id = repo_data.get('importer_type_id', None) importer_repo_plugin_config = repo_data.get('importer_config', None) distributors = repo_data.get('distributors', None) # Creation repo_manager = manager_factory.repo_manager() args = [repo_id, display_name, description, notes] kwargs = {'importer_type_id': importer_type_id, 'importer_repo_plugin_config': importer_repo_plugin_config, 'distributor_list': distributors} repo = repo_manager.create_and_configure_repo(*args, **kwargs) repo['_href'] = reverse('repo_resource', kwargs={'repo_id': repo_id}) response = generate_json_response_with_pulp_encoder(repo) return generate_redirect_response(response, repo['_href'])
def DELETE(self, id): repo_manager = manager_factory.repo_manager() resources = {dispatch_constants.RESOURCE_REPOSITORY_TYPE: {id: dispatch_constants.RESOURCE_DELETE_OPERATION}} tags = [resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE, id), action_tag("delete")] call_request = CallRequest(repo_manager.delete_repo, [id], resources=resources, tags=tags, archive=True) return execution.execute_ok(self, call_request)
def setUp(self): base.PulpServerTests.setUp(self) mock_plugins.install() self.upload_manager = manager_factory.content_upload_manager() self.repo_manager = manager_factory.repo_manager() self.importer_manager = manager_factory.repo_importer_manager()
def test_syntactic_sugar_methods(self): """ Tests the syntactic sugar methods for retrieving specific managers. """ # Setup factory.initialize() # Test self.assertTrue(isinstance(factory.authentication_manager(), AuthenticationManager)) self.assertTrue(isinstance(factory.cert_generation_manager(), CertGenerationManager)) self.assertTrue(isinstance(factory.certificate_manager(), CertificateManager)) self.assertTrue(isinstance(factory.password_manager(), PasswordManager)) self.assertTrue(isinstance(factory.permission_manager(), PermissionManager)) self.assertTrue(isinstance(factory.permission_query_manager(), PermissionQueryManager)) self.assertTrue(isinstance(factory.role_manager(), RoleManager)) self.assertTrue(isinstance(factory.role_query_manager(), RoleQueryManager)) self.assertTrue(isinstance(factory.user_manager(), UserManager)) self.assertTrue(isinstance(factory.user_query_manager(), UserQueryManager)) self.assertTrue(isinstance(factory.repo_manager(), RepoManager)) self.assertTrue(isinstance(factory.repo_unit_association_manager(), RepoUnitAssociationManager)) self.assertTrue(isinstance(factory.repo_publish_manager(), RepoPublishManager)) self.assertTrue(isinstance(factory.repo_query_manager(), RepoQueryManager)) self.assertTrue(isinstance(factory.repo_sync_manager(), RepoSyncManager)) self.assertTrue(isinstance(factory.content_manager(), ContentManager)) self.assertTrue(isinstance(factory.content_query_manager(), ContentQueryManager)) self.assertTrue(isinstance(factory.content_upload_manager(), ContentUploadManager)) self.assertTrue(isinstance(factory.consumer_manager(), ConsumerManager)) self.assertTrue(isinstance(factory.topic_publish_manager(), TopicPublishManager))
def POST(self): # Pull the repo data out of the request body (validation will occur # in the manager) repo_data = self.params() id = repo_data.get("id", None) display_name = repo_data.get("display_name", None) description = repo_data.get("description", None) notes = repo_data.get("notes", None) importer_type_id = repo_data.get("importer_type_id", None) importer_repo_plugin_config = repo_data.get("importer_config", None) distributors = repo_data.get("distributors", None) # Creation repo_manager = manager_factory.repo_manager() args = [id, display_name, description, notes] kwargs = { "importer_type_id": importer_type_id, "importer_repo_plugin_config": importer_repo_plugin_config, "distributor_list": distributors, } repo = repo_manager.create_and_configure_repo(*args, **kwargs) repo.update(serialization.link.child_link_obj(id)) return self.created(id, repo)
def test_get_with_bindings(self): """ Test consumer with bindings. """ # Setup manager = factory.repo_manager() repo = manager.create_repo(self.REPO_ID) manager = factory.repo_distributor_manager() manager.add_distributor(self.REPO_ID, self.DISTRIBUTOR_TYPE_ID, {}, True, distributor_id=self.DISTRIBUTOR_ID) manager = factory.consumer_manager() manager.register(self.CONSUMER_ID) manager = factory.consumer_bind_manager() bind = manager.bind(self.CONSUMER_ID, self.REPO_ID, self.DISTRIBUTOR_ID) # Test params = {"bindings": True} path = "/v2/consumers/%s/" % self.CONSUMER_ID status, body = self.get(path, params=params) # Verify self.assertEqual(200, status) self.assertEqual(self.CONSUMER_ID, body["id"]) self.assertTrue("_href" in body) self.assertTrue(body["_href"].endswith(path)) self.assertTrue("bindings" in body) bindings = body["bindings"] self.assertEquals(len(bindings), 1) self.assertEquals(bindings[0], self.REPO_ID)
def put(self, request, repo_id): """ Update a repository. This call will return synchronously unless a distributor is updated. :param request: WSGI request object :type request: django.core.handlers.wsgi.WSGIRequest :param repo_id: id of repository to be updated :type repo_id: str :return: Response containing a serialized dict for the updated repo. :rtype : django.http.HttpResponse :raises pulp_exceptions.OperationPostponed: if a distributor is updated, dispatch a task """ delta = request.body_as_json.get('delta', None) importer_config = request.body_as_json.get('importer_config', None) distributor_configs = request.body_as_json.get('distributor_configs', None) repo_manager = manager_factory.repo_manager() task_result = repo_manager.update_repo_and_plugins(repo_id, delta, importer_config, distributor_configs) repo = task_result.return_value repo['_href'] = reverse('repo_resource', kwargs={'repo_id': repo_id}) _convert_repo_dates_to_strings(repo) # Tasks are spawned if a distributor is updated, raise that as a result if task_result.spawned_tasks: raise pulp_exceptions.OperationPostponed(task_result) result = task_result.serialize() return generate_json_response_with_pulp_encoder(result)
def test_get_with_bindings(self): """ Test consumer with bindings. """ # Setup manager = factory.repo_manager() repo = manager.create_repo(self.REPO_ID) manager = factory.repo_distributor_manager() manager.add_distributor( self.REPO_ID, self.DISTRIBUTOR_TYPE_ID, {}, True, distributor_id=self.DISTRIBUTOR_ID) manager = factory.consumer_manager() manager.register(self.CONSUMER_ID) manager = factory.consumer_bind_manager() bind = manager.bind(self.CONSUMER_ID, self.REPO_ID, self.DISTRIBUTOR_ID) # Test params = {'bindings':True} path = '/v2/consumers/%s/' % self.CONSUMER_ID status, body = self.get(path, params=params) # Verify self.assertEqual(200, status) self.assertEqual(self.CONSUMER_ID, body['id']) self.assertTrue('_href' in body) self.assertTrue(body['_href'].endswith(path)) self.assertTrue('bindings' in body) bindings = body['bindings'] self.assertEquals(len(bindings), 1) self.assertEquals(bindings[0]['repo_id'], self.REPO_ID) self.assertEquals(bindings[0]['distributor_id'], self.DISTRIBUTOR_ID) self.assertEquals(bindings[0]['consumer_actions'], [])
def delete(repo_id): """ Get the itinerary for deleting a repository. 1. Delete the repository on the sever. 2. Unbind any bound consumers. :param repo_id: A repository ID. :type repo_id: str :return: A TaskRequest object with the details of any errors or spawned tasks :rtype TaskRequest """ # delete repository manager = managers.repo_manager() manager.delete_repo(repo_id) # append unbind itineraries foreach bound consumer options = {} manager = managers.consumer_bind_manager() additional_tasks = [] errors = [] for bind in manager.find_by_repo(repo_id): try: report = consumer.unbind(bind["consumer_id"], bind["repo_id"], bind["distributor_id"], options) if report: additional_tasks.extend(report.spawned_tasks) except Exception, e: errors.append(e)
def setUp(self): super(RepoConfigConduitTests, self).setUp() mock_plugins.install() manager_factory.initialize() self.repo_manager = manager_factory.repo_manager() self.distributor_manager = manager_factory.repo_distributor_manager() # Populate the database with a repo with units self.repo_manager.create_repo('repo-1') self.distributor_manager.add_distributor('repo-1', 'mock-distributor', {"relative_url": "/a/bc/d"}, True, distributor_id='dist-1') self.distributor_manager.add_distributor('repo-1', 'mock-distributor', {"relative_url": "/a/c"}, True, distributor_id='dist-2') self.repo_manager.create_repo('repo-2') self.distributor_manager.add_distributor('repo-2', 'mock-distributor', {"relative_url": "/a/bc/e"}, True, distributor_id='dist-3') self.repo_manager.create_repo('repo-3') self.distributor_manager.add_distributor('repo-3', 'mock-distributor', {}, True, distributor_id='dist-4') self.repo_manager.create_repo('repo-4') self.distributor_manager.add_distributor('repo-4', 'mock-distributor', {"relative_url": "/repo-5"}, True, distributor_id='dist-5') self.conduit = RepoConfigConduit('rpm')
def test_post(self, _reserve_resource, mock_apply_async): # Setup task_id = str(uuid.uuid4()) mock_apply_async.return_value = AsyncResult(task_id) _reserve_resource.return_value = ReservedResourceApplyAsync() upload_id = self.upload_manager.initialize_upload() self.upload_manager.save_data(upload_id, 0, 'string data') repo_manager = manager_factory.repo_manager() repo_manager.create_repo('repo-upload') importer_manager = manager_factory.repo_importer_manager() importer_manager.set_importer('repo-upload', 'dummy-importer', {}) # Test body = { 'upload_id' : upload_id, 'unit_type_id' : 'dummy-type', 'unit_key' : {'name' : 'foo'}, 'unit_metadata' : {'stuff' : 'bar'}, } status, body = self.post('/v2/repositories/repo-upload/actions/import_upload/', body) # Verify self.assertEqual(202, status) assert_body_matches_async_task(body, mock_apply_async.return_value) exepcted_call_args = ['repo-upload', 'dummy-type', {'name': 'foo'}, {'stuff': 'bar'}, upload_id] self.assertEqual(exepcted_call_args, mock_apply_async.call_args[0][0])
def POST(self): # Pull the repo data out of the request body (validation will occur # in the manager) repo_data = self.params() id = repo_data.get('id', None) display_name = repo_data.get('display_name', None) description = repo_data.get('description', None) notes = repo_data.get('notes', None) importer_type_id = repo_data.get('importer_type_id', None) importer_repo_plugin_config = repo_data.get('importer_config', None) distributors = repo_data.get('distributors', None) # Creation repo_manager = manager_factory.repo_manager() args = [id, display_name, description, notes] kwargs = { 'importer_type_id': importer_type_id, 'importer_repo_plugin_config': importer_repo_plugin_config, 'distributor_list': distributors } repo = repo_manager.create_and_configure_repo(*args, **kwargs) repo.update(serialization.link.child_link_obj(id)) return self.created(id, repo)
def POST(self): # Pull the repo data out of the request body (validation will occur # in the manager) repo_data = self.params() id = repo_data.get('id', None) display_name = repo_data.get('display_name', None) description = repo_data.get('description', None) notes = repo_data.get('notes', None) importer_type_id = repo_data.get('importer_type_id', None) importer_repo_plugin_config = repo_data.get('importer_config', None) distributors = repo_data.get('distributors', None) # Creation repo_manager = manager_factory.repo_manager() resources = {dispatch_constants.RESOURCE_REPOSITORY_TYPE: {id: dispatch_constants.RESOURCE_CREATE_OPERATION}} args = [id, display_name, description, notes, importer_type_id, importer_repo_plugin_config, distributors] weight = pulp_config.config.getint('tasks', 'create_weight') tags = [resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE, id), action_tag('create')] call_request = CallRequest(repo_manager.create_and_configure_repo, args, resources=resources, weight=weight, tags=tags) repo = execution.execute_sync(call_request) repo.update(serialization.link.child_link_obj(id)) return self.created(id, repo)
def test_post_with_override_config(self, mock_get_worker_for_reservation, mock_uuid, mock_apply_async): # Setup uuid_list = [uuid.uuid4() for i in range(10)] mock_uuid.uuid4.side_effect = copy.deepcopy(uuid_list) expected_async_result = AsyncResult(str(uuid_list[0])) mock_get_worker_for_reservation.return_value = Worker('some_queue', datetime.datetime.now()) upload_id = self.upload_manager.initialize_upload() self.upload_manager.save_data(upload_id, 0, 'string data') repo_manager = manager_factory.repo_manager() repo_manager.create_repo('repo-upload') importer_manager = manager_factory.repo_importer_manager() importer_manager.set_importer('repo-upload', 'dummy-importer', {}) # Test test_override_config = {'key1': 'value1', 'key2': 'value2'} body = { 'upload_id' : upload_id, 'unit_type_id' : 'dummy-type', 'unit_key' : {'name' : 'foo'}, 'unit_metadata' : {'stuff' : 'bar'}, 'override_config': test_override_config, } status, body = self.post('/v2/repositories/repo-upload/actions/import_upload/', body) # Verify self.assertEqual(202, status) assert_body_matches_async_task(body, expected_async_result) exepcted_call_args = ['repo-upload', 'dummy-type', {'name': 'foo'}, {'stuff': 'bar'}, upload_id, test_override_config] self.assertEqual(exepcted_call_args, mock_apply_async.call_args[0][0])
def setUp(self): super(RepoConfigConduitTests, self).setUp() mock_plugins.install() manager_factory.initialize() self.repo_manager = manager_factory.repo_manager() self.distributor_manager = manager_factory.repo_distributor_manager() # Populate the database with a repo with units self.repo_manager.create_repo('repo-1') self.distributor_manager.add_distributor( 'repo-1', 'mock-distributor', {"relative_url": "/a/bc/d"}, True, distributor_id='dist-1') self.distributor_manager.add_distributor( 'repo-1', 'mock-distributor', {"relative_url": "/a/c"}, True, distributor_id='dist-2') self.repo_manager.create_repo('repo-2') self.distributor_manager.add_distributor( 'repo-2', 'mock-distributor', {"relative_url": "/a/bc/e"}, True, distributor_id='dist-3') self.repo_manager.create_repo('repo-3') self.distributor_manager.add_distributor('repo-3', 'mock-distributor', {}, True, distributor_id='dist-4') self.repo_manager.create_repo('repo-4') self.distributor_manager.add_distributor( 'repo-4', 'mock-distributor', {"relative_url": "repo-5"}, True, distributor_id='dist-5') self.repo_manager.create_repo('repo-5') self.distributor_manager.add_distributor( 'repo-5', 'mock-distributor', {"relative_url": "a/bcd/e"}, True, distributor_id='dist-1') self.repo_manager.create_repo('repo-6') self.distributor_manager.add_distributor( 'repo-6', 'mock-distributor', {"relative_url": "a/bcde/f/"}, True, distributor_id='dist-1') self.conduit = RepoConfigConduit('rpm')
def test_post(self): # Setup upload_id = self.upload_manager.initialize_upload() self.upload_manager.save_data(upload_id, 0, 'string data') repo_manager = manager_factory.repo_manager() repo_manager.create_repo('repo-upload') importer_manager = manager_factory.repo_importer_manager() importer_manager.set_importer('repo-upload', 'dummy-importer', {}) # Test body = { 'upload_id': upload_id, 'unit_type_id': 'dummy-type', 'unit_key': { 'name': 'foo' }, 'unit_metadata': { 'stuff': 'bar' }, } status, body = self.post( '/v2/repositories/repo-upload/actions/import_upload/', body) # Verify self.assertEqual(200, status)
def setUp(self): super(RepoQueryManagerTests, self).setUp() mock_plugins.install() self.repo_manager = manager_factory.repo_manager() self.importer_manager = manager_factory.repo_importer_manager() self.distributor_manager = manager_factory.repo_distributor_manager() self.query_manager = manager_factory.repo_query_manager()
def test_associate_by_id_does_not_call_update_unit_count(self): """ This would be the case when doing a bulk update. """ self.manager.associate_unit_by_id( 'repo-1', 'type-1', 'unit-1', OWNER_TYPE_USER, 'admin', False) mock_manager = manager_factory.repo_manager() self.assertFalse(mock_manager.update_unit_count.called)
def populate(self): config = {"key1": "value1", "key2": None} manager = factory.repo_manager() repo = manager.create_repo(self.REPO_ID) manager = factory.repo_distributor_manager() manager.add_distributor(self.REPO_ID, "mock-distributor", config, True, distributor_id=self.DISTRIBUTOR_ID) manager = factory.consumer_manager() manager.register(self.CONSUMER_ID)
def associate_all_by_ids(self, repo_id, unit_type_id, unit_id_list, owner_type, owner_id): """ Creates multiple associations between the given repo and content units. See associate_unit_by_id for semantics. @param repo_id: identifies the repo @type repo_id: str @param unit_type_id: identifies the type of unit being added @type unit_type_id: str @param unit_id_list: list or generator of unique identifiers for units within the given type @type unit_id_list: list or generator of str @param owner_type: category of the caller making the association; must be one of the OWNER_* variables in this module @type owner_type: str @param owner_id: identifies the caller making the association, either the importer ID or user login @type owner_id: str :return: number of new units added to the repo :rtype: int @raise InvalidType: if the given owner type is not of the valid enumeration """ # There may be a way to batch this in mongo which would be ideal for a # bulk operation like this. But for deadline purposes, this call will # simply loop and call the single method. unique_count = 0 for unit_id in unit_id_list: if not RepoUnitAssociationManager.association_exists(repo_id, unit_id, unit_type_id): unique_count += 1 self.associate_unit_by_id(repo_id, unit_type_id, unit_id, owner_type, owner_id, False) # update the count of associated units on the repo object if unique_count: manager_factory.repo_manager().update_unit_count( repo_id, unit_type_id, unique_count) # update the timestamp for when the units were added to the repo manager_factory.repo_manager().update_last_unit_added(repo_id) return unique_count
def test_post_with_repos(self): # Setup manager_factory.repo_manager().create_repo('add-me') data = { 'id' : 'with-repos', 'repo_ids' : ['add-me'] } # Test status, body = self.post('/v2/repo_groups/', data) # Verify self.assertEqual(201, status) found = RepoGroup.get_collection().find_one({'id' : data['id']}) self.assertEqual(found['repo_ids'], data['repo_ids'])
def test_unassociate_by_id_calls_update_unit_count(self): self.manager.associate_unit_by_id( 'repo-1', 'type-1', 'unit-1', OWNER_TYPE_USER, 'admin') mock_manager = manager_factory.repo_manager() mock_manager.reset_mock() self.manager.unassociate_unit_by_id( 'repo-1', 'type-1', 'unit-1', OWNER_TYPE_USER, 'admin') mock_manager.update_unit_count.assert_called_once_with('repo-1', -1)
def test_unassociate_by_id_does_not_call_update_unit_count(self): self.manager.associate_unit_by_id( 'repo-1', 'type-1', 'unit-1', OWNER_TYPE_USER, 'admin') mock_manager = manager_factory.repo_manager() mock_manager.reset_mock() self.manager.unassociate_unit_by_id( 'repo-1', 'type-1', 'unit-1', OWNER_TYPE_USER, 'admin', False) self.assertFalse(mock_manager.update_unit_count.called)
def populate(self): manager = factory.repo_manager() repo = manager.create_repo(self.REPO_ID) manager = factory.repo_distributor_manager() manager.add_distributor(self.REPO_ID, self.DISTRIBUTOR_TYPE_ID, {}, True, distributor_id=self.DISTRIBUTOR_ID) mock_plugins.MOCK_DISTRIBUTOR.create_consumer_payload.return_value = self.PAYLOAD manager = factory.consumer_manager() manager.register(self.CONSUMER_ID)
def unassociate_by_criteria(repo_id, criteria, owner_type, owner_id, notify_plugins=True): """ Unassociate units that are matched by the given criteria. :param repo_id: identifies the repo :type repo_id: str :param criteria: :param owner_type: category of the caller who created the association :type owner_type: str :param owner_id: identifies the call who created the association :type owner_id: str :param notify_plugins: if true, relevant plugins will be informed of the removal :type notify_plugins: bool """ association_query_manager = manager_factory.repo_unit_association_query_manager() unassociate_units = association_query_manager.get_units(repo_id, criteria=criteria) if len(unassociate_units) == 0: return {} unit_map = {} # maps unit_type_id to a list of unit_ids for unit in unassociate_units: id_list = unit_map.setdefault(unit['unit_type_id'], []) id_list.append(unit['unit_id']) collection = RepoContentUnit.get_collection() repo_manager = manager_factory.repo_manager() for unit_type_id, unit_ids in unit_map.items(): spec = {'repo_id': repo_id, 'unit_type_id': unit_type_id, 'unit_id': {'$in': unit_ids} } collection.remove(spec, safe=True) unique_count = sum( 1 for unit_id in unit_ids if not RepoUnitAssociationManager.association_exists( repo_id, unit_id, unit_type_id)) if not unique_count: continue repo_manager.update_unit_count(repo_id, unit_type_id, -unique_count) repo_manager.update_last_unit_removed(repo_id) # Convert the units into transfer units. This happens regardless of whether or not # the plugin will be notified as it's used to generate the return result, unit_type_ids = calculate_associated_type_ids(repo_id, unassociate_units) transfer_units = create_transfer_units(unassociate_units, unit_type_ids) if notify_plugins: remove_from_importer(repo_id, transfer_units) # Match the return type/format as copy serializable_units = [u.to_id_dict() for u in transfer_units] return {'units_successful': serializable_units}
def test_associate_all_by_ids_calls_update_unit_count(self): IDS = ('foo', 'bar', 'baz') self.manager.associate_all_by_ids( 'repo-1', 'type-1', IDS, OWNER_TYPE_USER, 'admin') mock_manager = manager_factory.repo_manager() mock_manager.update_unit_count.assert_called_once_with( 'repo-1', len(IDS))
def associate_unit_by_id(self, repo_id, unit_type_id, unit_id, update_repo_metadata=True): """ Creates an association between the given repository and content unit. If there is already an association between the given repo and content unit where all other metadata matches the input to this method, this call has no effect. Both repo and unit must exist in the database prior to this call, however this call will not verify that for performance reasons. Care should be taken by the caller to preserve the data integrity. @param repo_id: identifies the repo @type repo_id: str @param unit_type_id: identifies the type of unit being added @type unit_type_id: str @param unit_id: uniquely identifies the unit within the given type @type unit_id: str @param update_repo_metadata: if True, updates the unit association count after the new association is made. The last unit added field will also be updated. Set this to False when doing bulk associations, and make one call to update the count at the end. defaults to True @type update_repo_metadata: bool @raise InvalidType: if the given owner type is not of the valid enumeration """ # If the association already exists, no need to do anything else spec = {'repo_id': repo_id, 'unit_id': unit_id, 'unit_type_id': unit_type_id} existing_association = RepoContentUnit.get_collection().find_one(spec) if existing_association is not None: return similar_exists = False if update_repo_metadata: similar_exists = RepoUnitAssociationManager.association_exists(repo_id, unit_id, unit_type_id) # Create the database entry association = RepoContentUnit(repo_id, unit_id, unit_type_id) RepoContentUnit.get_collection().save(association, safe=True) manager = manager_factory.repo_manager() # update the count of associated units on the repo object if update_repo_metadata and not similar_exists: manager.update_unit_count(repo_id, unit_type_id, 1) # update the record for the last added field manager.update_last_unit_added(repo_id)
def unassociate_by_criteria(repo_id, criteria, owner_type, owner_id, notify_plugins=True): """ Unassociate units that are matched by the given criteria. :param repo_id: identifies the repo :type repo_id: str :param criteria: :param owner_type: category of the caller who created the association :type owner_type: str :param owner_id: identifies the call who created the association :type owner_id: str :param notify_plugins: if true, relevant plugins will be informed of the removal :type notify_plugins: bool """ association_query_manager = manager_factory.repo_unit_association_query_manager() unassociate_units = association_query_manager.get_units(repo_id, criteria=criteria) if len(unassociate_units) == 0: return {} unit_map = {} # maps unit_type_id to a list of unit_ids for unit in unassociate_units: id_list = unit_map.setdefault(unit['unit_type_id'], []) id_list.append(unit['unit_id']) collection = RepoContentUnit.get_collection() repo_manager = manager_factory.repo_manager() for unit_type_id, unit_ids in unit_map.items(): spec = {'repo_id': repo_id, 'unit_type_id': unit_type_id, 'unit_id': {'$in': unit_ids} } collection.remove(spec, safe=True) unique_count = sum( 1 for unit_id in unit_ids if not RepoUnitAssociationManager.association_exists( repo_id, unit_id, unit_type_id)) if not unique_count: continue repo_manager.update_unit_count(repo_id, unit_type_id, -unique_count) # Convert the units into transfer units. This happens regardless of whether or not # the plugin will be notified as it's used to generate the return result, unit_type_ids = calculate_associated_type_ids(repo_id, unassociate_units) transfer_units = create_transfer_units(unassociate_units, unit_type_ids) if notify_plugins: remove_from_importer(repo_id, transfer_units) # Match the return type/format as copy serializable_units = [u.to_id_dict() for u in transfer_units] return {'units_successful': serializable_units}
def populate_repository(self): config = {'key1': 'value1', 'key2': None} manager = factory.repo_manager() manager.create_repo(self.REPO_ID) manager = factory.repo_distributor_manager() manager.add_distributor(self.REPO_ID, 'mock-distributor', config, True, distributor_id=self.DISTRIBUTOR_ID)
def populate(self): manager = factory.repo_manager() manager.create_repo(self.REPO_ID) manager = factory.repo_distributor_manager() manager.add_distributor(self.REPO_ID, self.DISTRIBUTOR_TYPE_ID, {}, True, distributor_id=self.DISTRIBUTOR_ID) mock_plugins.MOCK_DISTRIBUTOR.create_consumer_payload.return_value = self.PAYLOAD manager = factory.consumer_manager() manager.register(self.CONSUMER_ID)
def populate(self): # make content/ dir. os.makedirs(os.path.join(self.parentfs, 'content')) pulp_conf.set('server', 'storage_dir', self.parentfs) # create repo manager = managers.repo_manager() manager.create_repo( self.REPO_ID, display_name=REPO_NAME, description=REPO_DESCRIPTION, notes=REPO_NOTES) manager.set_repo_scratchpad(self.REPO_ID, REPO_SCRATCHPAD) # add units units = self.add_units(0, self.NUM_UNITS) self.units = units
def update_tags(repo_id, new_tags): """ Gets the current scratchpad's tags and updates them with the new_tags :param repo_id: unique ID of a repository :type repo_id: basestring :param new_tags: dictionary of tag:image_id :type new_tags: dict """ repo_manager = factory.repo_manager() scratchpad = repo_manager.get_repo_scratchpad(repo_id) new_tags = tags.generate_updated_tags(scratchpad, new_tags) repo_manager.update_repo_scratchpad(repo_id, {'tags': new_tags})
def populate(self): manager = factory.repo_manager() manager.create_repo(self.REPO_ID) manager = factory.repo_distributor_manager() manager.add_distributor(self.REPO_ID, self.DISTRIBUTOR_TYPE_ID, {}, True, distributor_id=self.DISTRIBUTOR_ID) manager = factory.consumer_manager() manager.register(self.CONSUMER_ID) manager = factory.consumer_bind_manager() manager.bind(self.CONSUMER_ID, self.REPO_ID, self.DISTRIBUTOR_ID, self.NOTIFY_AGENT, self.BINDING_CONFIG)
def setUp(self): super(RepoPublishConduitTests, self).setUp() mock_plugins.install() manager_factory.initialize() self.repo_manager = manager_factory.repo_manager() self.distributor_manager = manager_factory.repo_distributor_manager() # Populate the database with a repo with units self.repo_manager.create_repo('repo-1') self.distributor_manager.add_distributor('repo-1', 'mock-distributor', {}, True, distributor_id='dist-1') self.conduit = RepoPublishConduit('repo-1', 'dist-1')
def test_delete_repo(self): group_id = 'delete_from_me' repo = self._create_repo('delete_me') self.manager.create_repo_group(group_id, repo_ids=[repo['id']]) group = self.collection.find_one({'id': group_id}) self.assertTrue(repo['id'] in group['repo_ids']) repo_manager = managers_factory.repo_manager() repo_manager.delete_repo(repo['id']) group = self.collection.find_one({'id': group_id}) self.assertFalse(repo['id'] in group['repo_ids'])
def populate(self): config = {'key1': 'value1', 'key2': None} manager = factory.repo_manager() repo = manager.create_repo(self.REPO_ID) manager = factory.repo_distributor_manager() manager.add_distributor(self.REPO_ID, 'mock-distributor', config, True, distributor_id=self.DISTRIBUTOR_ID) manager = factory.consumer_manager() for consumer_id in self.ALL_CONSUMERS: manager.register(consumer_id)
def update_repo_scratchpad(self, scratchpad): """ Update the repository scratchpad with the specified key-value pairs. New keys are added; existing keys are updated. :param scratchpad: a dict used to update the scratchpad. """ try: manager = manager_factory.repo_manager() manager.update_repo_scratchpad(self.repo_id, scratchpad) except Exception, e: msg = _('Error updating repository scratchpad for repo [%(r)s]') % {'r': self.repo_id} logger.exception(msg) raise self.exception_class(e), None, sys.exc_info()[2]
def get_repo_scratchpad(self, repo_id): """ Returns the repository-level scratchpad for the indicated repository. @raise ImporterConduitException: wraps any exception that may occur in the Pulp server """ try: repo_manager = manager_factory.repo_manager() value = repo_manager.get_repo_scratchpad(repo_id) return value except Exception, e: logger.exception(_('Error getting repository scratchpad for repo [%(r)s]') % {'r' : repo_id}) raise self.exception_class(e), None, sys.exc_info()[2]
def get_repo_scratchpad(self): """ Returns the repository-level scratchpad for this repository. The repository-level scratchpad can be seen and edited by all importers and distributors on the repository. Care should be taken to not destroy any data set by another plugin. This may be used to communicate between importers, distributors and profilers relevant data for the repository. """ try: repo_manager = manager_factory.repo_manager() value = repo_manager.get_repo_scratchpad(self.repo_id) return value except Exception, e: logger.exception(_('Error getting repository scratchpad for repo [%(r)s]') % {'r' : self.repo_id}) raise self.exception_class(e), None, sys.exc_info()[2]
def POST(self): # Pull the repo data out of the request body (validation will occur # in the manager) repo_data = self.params() id = repo_data.get('id', None) display_name = repo_data.get('display_name', None) description = repo_data.get('description', None) notes = repo_data.get('notes', None) importer_type_id = repo_data.get('importer_type_id', None) importer_repo_plugin_config = repo_data.get('importer_config', None) distributors = repo_data.get('distributors', None) # Creation repo_manager = manager_factory.repo_manager() resources = { dispatch_constants.RESOURCE_REPOSITORY_TYPE: { id: dispatch_constants.RESOURCE_CREATE_OPERATION } } args = [id, display_name, description, notes] kwargs = { 'importer_type_id': importer_type_id, 'importer_repo_plugin_config': importer_repo_plugin_config, 'distributor_list': distributors } weight = pulp_config.config.getint('tasks', 'create_weight') tags = [ resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE, id), action_tag('create') ] call_request = CallRequest(repo_manager.create_and_configure_repo, args, kwargs, resources=resources, weight=weight, tags=tags, kwarg_blacklist=[ 'importer_repo_plugin_config', 'distributor_list' ]) repo = execution.execute_sync(call_request) repo.update(serialization.link.child_link_obj(id)) return self.created(id, repo)
def populate(self): manager = managers.consumer_manager() for consumer_id in CONSUMER_IDS: manager.register(consumer_id) manager = managers.consumer_group_manager() manager.create_consumer_group(GROUP_ID) for consumer_id in CONSUMER_IDS: criteria = Criteria(filters={'id': consumer_id}, fields=['id']) manager.associate(GROUP_ID, criteria) manager = managers.repo_manager() manager.create_repo(REPO_ID) manager = managers.repo_distributor_manager() manager.add_distributor(REPO_ID, DISTRIBUTOR_TYPE_ID, {}, True, distributor_id=DISTRIBUTOR_ID)