def test_resolve_dependencies_by_criteria(self): # Setup report = 'dep report' mock_plugins.MOCK_IMPORTER.resolve_dependencies.return_value = report unit_id_1 = manager_factory.content_manager().add_content_unit( 'type-1', None, {'key-1': 'unit-id-1'}) unit_id_2 = manager_factory.content_manager().add_content_unit( 'type-1', None, {'key-1': 'dep-1'}) association_manager = manager_factory.repo_unit_association_manager() association_manager.associate_unit_by_id(self.repo_id, 'type-1', unit_id_1) association_manager.associate_unit_by_id(self.repo_id, 'type-1', unit_id_2) criteria = UnitAssociationCriteria(type_ids=['type-1'], unit_filters={'key-1': 'unit-id-1'}) # Test result = self.manager.resolve_dependencies_by_criteria( self.repo_id, criteria, {}) # Verify self.assertEqual(report, result) self.assertEqual( 1, mock_plugins.MOCK_IMPORTER.resolve_dependencies.call_count) args = mock_plugins.MOCK_IMPORTER.resolve_dependencies.call_args[0] self.assertEqual(1, len(args[1]))
def POST(self, dest_repo_id): # Params params = self.params() source_repo_id = params.get('source_repo_id', None) overrides = params.get('override_config', None) if source_repo_id is None: raise exceptions.MissingValue(['source_repo_id']) criteria = params.get('criteria', None) if criteria is not None: try: criteria = UnitAssociationCriteria.from_client_input(criteria) except: _LOG.exception('Error parsing association criteria [%s]' % criteria) raise exceptions.PulpDataException(), None, sys.exc_info()[2] association_manager = manager_factory.repo_unit_association_manager() resources = {dispatch_constants.RESOURCE_REPOSITORY_TYPE: {source_repo_id: dispatch_constants.RESOURCE_READ_OPERATION, dest_repo_id: dispatch_constants.RESOURCE_UPDATE_OPERATION}} tags = [resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE, dest_repo_id), resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE, source_repo_id), action_tag('associate')] call_request = CallRequest(association_manager.associate_from_repo, [source_repo_id, dest_repo_id], {'criteria': criteria, 'import_config_override': overrides}, resources=resources, tags=tags, archive=True) return execution.execute_async(self, call_request)
def save_unit(self, unit): """ Performs two distinct steps on the Pulp server: - Creates or updates Pulp's knowledge of the content unit. - Associates the unit to the repository being synchronized. If a unit with the provided unit key already exists, it is updated with the attributes on the passed-in unit. A reference to the provided unit is returned from this call. This call will populate the unit's id field with the UUID for the unit. :param unit: unit object returned from the init_unit call :type unit: Unit :return: object reference to the provided unit, its state updated from the call :rtype: Unit """ try: association_manager = manager_factory.repo_unit_association_manager() # Save or update the unit pulp_unit = common_utils.to_pulp_unit(unit) unit.id = self._update_unit(unit, pulp_unit) # Associate it with the repo association_manager.associate_unit_by_id( self.repo_id, unit.type_id, unit.id) return unit except Exception, e: _logger.exception(_('Content unit association failed [%s]' % str(unit))) raise ImporterConduitException(e), None, sys.exc_info()[2]
def test_resolve_dependencies_by_criteria(self): # Setup report = 'dep report' mock_plugins.MOCK_IMPORTER.resolve_dependencies.return_value = report unit_id_1 = manager_factory.content_manager().add_content_unit('type-1', None, {'key-1': 'unit-id-1'}) unit_id_2 = manager_factory.content_manager().add_content_unit('type-1', None, {'key-1': 'dep-1'}) association_manager = manager_factory.repo_unit_association_manager() association_manager.associate_unit_by_id(self.repo_id, 'type-1', unit_id_1) association_manager.associate_unit_by_id(self.repo_id, 'type-1', unit_id_2) criteria = UnitAssociationCriteria(type_ids=['type-1'], unit_filters={'key-1': 'unit-id-1'}) # Test result = self.manager.resolve_dependencies_by_criteria(self.repo_id, criteria, {}) # Verify self.assertEqual(report, result) self.assertEqual(1, mock_plugins.MOCK_IMPORTER.resolve_dependencies.call_count) args = mock_plugins.MOCK_IMPORTER.resolve_dependencies.call_args[0] self.assertEqual(1, len(args[1]))
def __init__(self, source_repo_id, dest_repo_id, source_importer_id, dest_importer_id): """ :param source_repo_id: ID of the repository from which units are being copied :type source_repo_id: str :param dest_repo_id: ID of the repository into which units are being copied :type dest_repo_id: str :param source_importer_id: ID of the importer on the source repository :type source_importer_id: str :param dest_importer_id: ID of the importer on the destination repository :type dest_importer_id: str """ ImporterScratchPadMixin.__init__(self, dest_repo_id, dest_importer_id) RepoScratchPadMixin.__init__(self, dest_repo_id, ImporterConduitException) SearchUnitsMixin.__init__(self, ImporterConduitException) AddUnitMixin.__init__(self, dest_repo_id, dest_importer_id) self.source_repo_id = source_repo_id self.dest_repo_id = dest_repo_id self.source_importer_id = source_importer_id self.dest_importer_id = dest_importer_id self.__association_manager = manager_factory.repo_unit_association_manager() self.__association_query_manager = manager_factory.repo_unit_association_query_manager() self.__importer_manager = manager_factory.repo_importer_manager()
def test_resolve_dependencies_by_unit(self): # Setup report = 'dep report' mock_plugins.MOCK_IMPORTER.resolve_dependencies.return_value = report unit_id_1 = manager_factory.content_manager().add_content_unit('type-1', None, {'key-1' : 'v1'}) unit_id_2 = manager_factory.content_manager().add_content_unit('type-1', None, {'key-1' : 'v2'}) association_manager = manager_factory.repo_unit_association_manager() association_manager.associate_unit_by_id(self.repo_id, 'type-1', unit_id_1, 'user', 'admin') association_manager.associate_unit_by_id(self.repo_id, 'type-1', unit_id_2, 'user', 'admin') # Test result = self.manager.resolve_dependencies_by_units(self.repo_id, [], {}) # Verify self.assertEqual(result, report) self.assertEqual(1, mock_plugins.MOCK_IMPORTER.resolve_dependencies.call_count) args = mock_plugins.MOCK_IMPORTER.resolve_dependencies.call_args[0] self.assertEqual(args[0].id, self.repo_id) self.assertEqual(len(args[1]), 0) self.assertTrue(isinstance(args[2], DependencyResolutionConduit)) self.assertTrue(isinstance(args[3], PluginCallConfiguration))
def POST(self, repo_id): params = self.params() criteria = params.get('criteria', None) if criteria is not None: try: criteria = UnitAssociationCriteria.from_client_input(criteria) except: _LOG.error('Error parsing unassociation criteria [%s]' % criteria) raise exceptions.PulpDataException(), None, sys.exc_info()[2] association_manager = manager_factory.repo_unit_association_manager() tags = [ resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE, repo_id), action_tag('unassociate') ] call_request = CallRequest( association_manager.unassociate_by_criteria, [ repo_id, criteria, RepoContentUnit.OWNER_TYPE_USER, manager_factory.principal_manager().get_principal()['login'] ], tags=tags, archive=True) call_request.updates_resource( dispatch_constants.RESOURCE_REPOSITORY_TYPE, repo_id) return execution.execute_async(self, call_request)
def __init__(self, source_repo_id, dest_repo_id, source_importer_id, dest_importer_id, association_owner_type, association_owner_id): """ :param source_repo_id: ID of the repository from which units are being copied :type source_repo_id: str :param dest_repo_id: ID of the repository into which units are being copied :type dest_repo_id: str :param source_importer_id: ID of the importer on the source repository :type source_importer_id: str :param dest_importer_id: ID of the importer on the destination repository :type dest_importer_id: str :param association_owner_type: distinguishes the owner when creating an association through this conduit :type association_owner_type: str :param association_owner_id: specific ID of the owner when creating an association through this conduit :type association_owner_id: str """ ImporterScratchPadMixin.__init__(self, dest_repo_id, dest_importer_id) RepoScratchPadMixin.__init__(self, dest_repo_id, ImporterConduitException) SearchUnitsMixin.__init__(self, ImporterConduitException) AddUnitMixin.__init__(self, dest_repo_id, dest_importer_id, association_owner_type, association_owner_id) self.source_repo_id = source_repo_id self.dest_repo_id = dest_repo_id self.source_importer_id = source_importer_id self.dest_importer_id = dest_importer_id self.association_owner_type = association_owner_type self.association_owner_id = association_owner_id self.__association_manager = manager_factory.repo_unit_association_manager() self.__association_query_manager = manager_factory.repo_unit_association_query_manager() self.__importer_manager = manager_factory.repo_importer_manager()
def add_units(self, begin, end): units = [] storage_dir = os.path.join( pulp_conf.config.get('server', 'storage_dir'), 'content') if not os.path.exists(storage_dir): os.makedirs(storage_dir) for n in range(begin, end): unit_id = self.UNIT_ID % n unit = dict(self.UNIT_KEY) unit.update(self.UNIT_METADATA) unit['N'] = n # add unit file storage_path = os.path.join(storage_dir, '.'.join( (unit_id, self.UNIT_TYPE_ID))) if n % 2 == 0: # even numbered has file associated unit['_storage_path'] = storage_path if n == 0: # 1st one is a directory of files os.makedirs(storage_path) dist_path = os.path.join(os.path.dirname(__file__), 'data/distribution.tar') tb = tarfile.open(dist_path) tb.extractall(path=storage_path) tb.close() else: with open(storage_path, 'w+') as fp: fp.write(unit_id) # add unit manager = managers.content_manager() manager.add_content_unit(self.UNIT_TYPE_ID, unit_id, unit) manager = managers.repo_unit_association_manager() # associate unit manager.associate_unit_by_id(self.REPO_ID, self.UNIT_TYPE_ID, unit_id) units.append(unit) return units
def test_syntactic_sugar_methods(self): """ Tests the syntactic sugar methods for retrieving specific managers. """ # Setup factory.initialize() # Test self.assertTrue(isinstance(factory.authentication_manager(), AuthenticationManager)) self.assertTrue(isinstance(factory.cert_generation_manager(), CertGenerationManager)) self.assertTrue(isinstance(factory.certificate_manager(), CertificateManager)) self.assertTrue(isinstance(factory.password_manager(), PasswordManager)) self.assertTrue(isinstance(factory.permission_manager(), PermissionManager)) self.assertTrue(isinstance(factory.permission_query_manager(), PermissionQueryManager)) self.assertTrue(isinstance(factory.role_manager(), RoleManager)) self.assertTrue(isinstance(factory.role_query_manager(), RoleQueryManager)) self.assertTrue(isinstance(factory.user_manager(), UserManager)) self.assertTrue(isinstance(factory.user_query_manager(), UserQueryManager)) self.assertTrue(isinstance(factory.repo_manager(), RepoManager)) self.assertTrue(isinstance(factory.repo_unit_association_manager(), RepoUnitAssociationManager)) self.assertTrue(isinstance(factory.repo_publish_manager(), RepoPublishManager)) self.assertTrue(isinstance(factory.repo_query_manager(), RepoQueryManager)) self.assertTrue(isinstance(factory.repo_sync_manager(), RepoSyncManager)) self.assertTrue(isinstance(factory.content_manager(), ContentManager)) self.assertTrue(isinstance(factory.content_query_manager(), ContentQueryManager)) self.assertTrue(isinstance(factory.content_upload_manager(), ContentUploadManager)) self.assertTrue(isinstance(factory.consumer_manager(), ConsumerManager)) self.assertTrue(isinstance(factory.topic_publish_manager(), TopicPublishManager))
def test_resolve_dependencies_by_unit(self): # Setup report = 'dep report' mock_plugins.MOCK_IMPORTER.resolve_dependencies.return_value = report unit_id_1 = manager_factory.content_manager().add_content_unit( 'type-1', None, {'key-1': 'v1'}) unit_id_2 = manager_factory.content_manager().add_content_unit( 'type-1', None, {'key-1': 'v2'}) association_manager = manager_factory.repo_unit_association_manager() association_manager.associate_unit_by_id(self.repo_id, 'type-1', unit_id_1) association_manager.associate_unit_by_id(self.repo_id, 'type-1', unit_id_2) # Test result = self.manager.resolve_dependencies_by_units( self.repo_id, [], {}) # Verify self.assertEqual(result, report) self.assertEqual( 1, mock_plugins.MOCK_IMPORTER.resolve_dependencies.call_count) args = mock_plugins.MOCK_IMPORTER.resolve_dependencies.call_args[0] self.assertEqual(args[0].id, self.repo_id) self.assertEqual(len(args[1]), 0) self.assertTrue(isinstance(args[2], DependencyResolutionConduit)) self.assertTrue(isinstance(args[3], PluginCallConfiguration))
def test_syntactic_sugar_methods(self): """ Tests the syntactic sugar methods for retrieving specific managers. """ # Setup factory.initialize() # Test self.assertTrue(isinstance(factory.authentication_manager(), AuthenticationManager)) self.assertTrue(isinstance(factory.cert_generation_manager(), CertGenerationManager)) self.assertTrue(isinstance(factory.certificate_manager(), CertificateManager)) self.assertTrue(isinstance(factory.password_manager(), PasswordManager)) self.assertTrue(isinstance(factory.permission_manager(), PermissionManager)) self.assertTrue(isinstance(factory.permission_query_manager(), PermissionQueryManager)) self.assertTrue(isinstance(factory.role_manager(), RoleManager)) self.assertTrue(isinstance(factory.role_query_manager(), RoleQueryManager)) self.assertTrue(isinstance(factory.user_manager(), UserManager)) self.assertTrue(isinstance(factory.user_query_manager(), UserQueryManager)) self.assertTrue(isinstance(factory.repo_manager(), RepoManager)) self.assertTrue(isinstance(factory.repo_unit_association_manager(), RepoUnitAssociationManager)) self.assertTrue(isinstance(factory.repo_publish_manager(), RepoPublishManager)) self.assertTrue(isinstance(factory.repo_query_manager(), RepoQueryManager)) self.assertTrue(isinstance(factory.repo_sync_manager(), RepoSyncManager)) self.assertTrue(isinstance(factory.content_manager(), ContentManager)) self.assertTrue(isinstance(factory.content_query_manager(), ContentQueryManager)) self.assertTrue(isinstance(factory.content_upload_manager(), ContentUploadManager)) self.assertTrue(isinstance(factory.consumer_manager(), ConsumerManager)) self.assertTrue(isinstance(factory.topic_publish_manager(), TopicPublishManager))
def __init__(self, source_repo_id, dest_repo_id, source_importer_id, dest_importer_id): """ :param source_repo_id: ID of the repository from which units are being copied :type source_repo_id: str :param dest_repo_id: ID of the repository into which units are being copied :type dest_repo_id: str :param source_importer_id: ID of the importer on the source repository :type source_importer_id: str :param dest_importer_id: ID of the importer on the destination repository :type dest_importer_id: str """ ImporterScratchPadMixin.__init__(self, dest_repo_id, dest_importer_id) RepoScratchPadMixin.__init__(self, dest_repo_id, ImporterConduitException) SearchUnitsMixin.__init__(self, ImporterConduitException) AddUnitMixin.__init__(self, dest_repo_id, dest_importer_id) self.source_repo_id = source_repo_id self.dest_repo_id = dest_repo_id self.source_importer_id = source_importer_id self.dest_importer_id = dest_importer_id self.__association_manager = manager_factory.repo_unit_association_manager( ) self.__association_query_manager = manager_factory.repo_unit_association_query_manager( ) self.__importer_manager = manager_factory.repo_importer_manager()
def _purge_unlinked_manifests(repo, manifest_list): # Find manifest digests referenced by removed manifest lists (orphaned) orphaned = set() for image_man in manifest_list.manifests: orphaned.add(image_man) if manifest_list.amd64_digest: orphaned.add(manifest_list.amd64_digest) if not orphaned: # nothing orphaned return # Find manifest digests still referenced by other manifest lists (adopted) adopted = set() criteria = UnitAssociationCriteria( type_ids=[constants.MANIFEST_LIST_TYPE_ID], unit_filters={'digest': { '$ne': manifest_list.digest }}) for man_list in unit_association.RepoUnitAssociationManager._units_from_criteria( repo, criteria): for image_man in man_list.manifests: adopted.add(image_man) if man_list.amd64_digest: adopted.add(man_list.amd64_digest) # Remove unreferenced manifests orphaned = orphaned.difference(adopted) if not orphaned: # all adopted return # Check if those manifests have tags, tagged manifests cannot be removed criteria = UnitAssociationCriteria(type_ids=[constants.TAG_TYPE_ID], unit_filters={ 'manifest_digest': { '$in': list(orphaned) }, 'manifest_type': constants.MANIFEST_IMAGE_TYPE }) for tag in unit_association.RepoUnitAssociationManager._units_from_criteria( repo, criteria): orphaned.remove(tag.manifest_digest) unit_filter = {'digest': {'$in': sorted(orphaned)}} criteria = UnitAssociationCriteria( type_ids=[constants.MANIFEST_TYPE_ID], unit_filters=unit_filter) manager = manager_factory.repo_unit_association_manager() manager.unassociate_by_criteria(repo_id=repo.repo_id, criteria=criteria, notify_plugins=False) for manifest in models.Manifest.objects.filter( digest__in=sorted(orphaned)): DockerImporter._purge_unlinked_blobs(repo, manifest)
def __init__(self, repo_id, importer_id, association_owner_type, association_owner_id): RepoScratchPadMixin.__init__(self, repo_id, ImporterConduitException) ImporterScratchPadMixin.__init__(self, repo_id, importer_id) AddUnitMixin.__init__(self, repo_id, importer_id, association_owner_type, association_owner_id) SingleRepoUnitsMixin.__init__(self, repo_id, ImporterConduitException) StatusMixin.__init__(self, importer_id, ImporterConduitException) self._association_manager = manager_factory.repo_unit_association_manager() self._removed_count = 0
def populate_units(self, key, typedef): for i in range(1, 10): unit_id = 'unit-%s' % self.UNIT_ID md = {key: str(i)} manager = factory.content_manager() manager.add_content_unit(typedef.id, unit_id, md) manager = factory.repo_unit_association_manager() manager.associate_unit_by_id(self.REPO_ID, typedef.id, unit_id, OWNER_TYPE_IMPORTER, 'test-importer') self.UNIT_ID += 1
def _purge_orphaned_blobs(repo, units): """ Purge blobs associated with removed manifests when no longer referenced by any remaining manifests. :param repo: The affected repository. :type repo: pulp.plugins.model.Repository :param units: List of removed units. :type units: list of: pulp.plugins.model.AssociatedUnit """ # Find blob digests referenced by removed manifests (orphaned) orphaned = set() for unit in units: if unit.type_id != constants.MANIFEST_TYPE_ID: continue manifest = unit for layer in manifest.metadata['fs_layers']: digest = layer['blobSum'] orphaned.add(digest) # Find blob digests still referenced by other manifests (adopted) if not orphaned: # nothing orphaned return adopted = set() manager = manager_factory.repo_unit_association_query_manager() for manifest in manager.get_units_by_type(repo.id, constants.MANIFEST_TYPE_ID): for layer in manifest.metadata['fs_layers']: digest = layer['blobSum'] adopted.add(digest) # Remove unreferenced blobs orphaned = orphaned.difference(adopted) if not orphaned: # all adopted return unit_filter = { 'digest': { '$in': sorted(orphaned) } } criteria = UnitAssociationCriteria( type_ids=[constants.BLOB_TYPE_ID], unit_filters=unit_filter) manager = manager_factory.repo_unit_association_manager() manager.unassociate_by_criteria( repo_id=repo.id, criteria=criteria, owner_type='', # unused owner_id='', # unused notify_plugins=False)
def __init__(self, repo_id, importer_id, association_owner_type, association_owner_id): RepoScratchPadMixin.__init__(self, repo_id, ImporterConduitException) ImporterScratchPadMixin.__init__(self, repo_id, importer_id) AddUnitMixin.__init__(self, repo_id, importer_id, association_owner_type, association_owner_id) SingleRepoUnitsMixin.__init__(self, repo_id, ImporterConduitException) StatusMixin.__init__(self, importer_id, ImporterConduitException) SearchUnitsMixin.__init__(self, ImporterConduitException) self._association_manager = manager_factory.repo_unit_association_manager() self._removed_count = 0
def populate_units(self, key, typedef, additional_key=None): for i in range(1, 10): unit_id = 'unit-%s' % self.UNIT_ID md = {key: str(i)} if additional_key: md[additional_key] = str(i) manager = factory.content_manager() manager.add_content_unit(typedef.id, unit_id, md) manager = factory.repo_unit_association_manager() manager.associate_unit_by_id(self.REPO_ID, typedef.id, unit_id) self.UNIT_ID += 1
def POST(self, dest_repo_id): # Params params = self.params() source_repo_id = params.get('source_repo_id', None) overrides = params.get('override_config', None) if source_repo_id is None: raise exceptions.MissingValue(['source_repo_id']) # A 404 only applies to things in the URL, so the destination repo # check allows the MissingResource to bubble up, but if the source # repo doesn't exist, it's considered bad data. repo_query_manager = manager_factory.repo_query_manager() repo_query_manager.get_repository(dest_repo_id) try: repo_query_manager.get_repository(source_repo_id) except exceptions.MissingResource: raise exceptions.InvalidValue(['source_repo_id']) criteria = params.get('criteria', None) if criteria is not None: try: criteria = UnitAssociationCriteria.from_client_input(criteria) except: _LOG.error('Error parsing association criteria [%s]' % criteria) raise exceptions.PulpDataException(), None, sys.exc_info()[2] association_manager = manager_factory.repo_unit_association_manager() tags = [ resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE, dest_repo_id), resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE, source_repo_id), action_tag('associate') ] call_request = CallRequest( association_manager.associate_from_repo, [source_repo_id, dest_repo_id], { 'criteria': criteria, 'import_config_override': overrides }, tags=tags, archive=True, kwarg_blacklist=['criteria', 'import_config_override']) call_request.reads_resource( dispatch_constants.RESOURCE_REPOSITORY_TYPE, source_repo_id) call_request.updates_resource( dispatch_constants.RESOURCE_REPOSITORY_TYPE, dest_repo_id) return execution.execute_async(self, call_request)
def __init__(self, repo_id, importer_id, importer_object_id): RepoScratchPadMixin.__init__(self, repo_id, ImporterConduitException) ImporterScratchPadMixin.__init__(self, repo_id, importer_id) AddUnitMixin.__init__(self, repo_id, importer_id) SingleRepoUnitsMixin.__init__(self, repo_id, ImporterConduitException) StatusMixin.__init__(self, importer_id, ImporterConduitException) SearchUnitsMixin.__init__(self, ImporterConduitException) self.importer_object_id = importer_object_id self._association_manager = manager_factory.repo_unit_association_manager() self._content_query_manager = manager_factory.content_query_manager() self._removed_count = 0
def __init__(self, repo_id, importer_id, importer_object_id): RepoScratchPadMixin.__init__(self, repo_id, ImporterConduitException) ImporterScratchPadMixin.__init__(self, repo_id, importer_id) AddUnitMixin.__init__(self, repo_id, importer_id) SingleRepoUnitsMixin.__init__(self, repo_id, ImporterConduitException) StatusMixin.__init__(self, importer_id, ImporterConduitException) SearchUnitsMixin.__init__(self, ImporterConduitException) self.importer_object_id = importer_object_id self._association_manager = manager_factory.repo_unit_association_manager() self._content_query_manager = manager_factory.content_query_manager() self._removed_count = 0
def __init__(self, source_repo_id, dest_repo_id, source_importer_id, dest_importer_id): ImporterScratchPadMixin.__init__(self, dest_repo_id, dest_importer_id) RepoScratchPadMixin.__init__(self, dest_repo_id, ImporterConduitException) self.source_repo_id = source_repo_id self.dest_repo_id = dest_repo_id self.source_importer_id = source_importer_id self.dest_importer_id = dest_importer_id self.__association_manager = manager_factory.repo_unit_association_manager() self.__association_query_manager = manager_factory.repo_unit_association_query_manager() self.__importer_manager = manager_factory.repo_importer_manager()
def populate_units(self, key, typedef): for i in range(1,10): unit_id = 'unit-%s' % self.UNIT_ID md = {key:str(i)} manager = factory.content_manager() manager.add_content_unit(typedef.id, unit_id, md) manager = factory.repo_unit_association_manager() manager.associate_unit_by_id( self.REPO_ID, typedef.id, unit_id, OWNER_TYPE_IMPORTER, 'test-importer') self.UNIT_ID += 1
def populate_units(self, key, typedef, additional_key=None): for i in range(1, 10): unit_id = 'unit-%s' % self.UNIT_ID md = {key: str(i)} if additional_key: md[additional_key] = str(i) manager = factory.content_manager() manager.add_content_unit(typedef.id, unit_id, md) manager = factory.repo_unit_association_manager() manager.associate_unit_by_id( self.REPO_ID, typedef.id, unit_id) self.UNIT_ID += 1
def test_syntactic_sugar_methods(self): """ Tests the syntactic sugar methods for retrieving specific managers. """ # Test self.assertTrue(isinstance(factory.repo_manager(), RepoManager)) self.assertTrue(isinstance(factory.repo_unit_association_manager(), RepoUnitAssociationManager)) self.assertTrue(isinstance(factory.repo_publish_manager(), RepoPublishManager)) self.assertTrue(isinstance(factory.repo_query_manager(), RepoQueryManager)) self.assertTrue(isinstance(factory.repo_sync_manager(), RepoSyncManager)) self.assertTrue(isinstance(factory.content_manager(), ContentManager)) self.assertTrue(isinstance(factory.content_query_manager(), ContentQueryManager)) self.assertTrue(isinstance(factory.content_upload_manager(), ContentUploadManager)) self.assertTrue(isinstance(factory.consumer_manager(), ConsumerManager))
def populate(self): # make content/ dir. os.makedirs(os.path.join(self.parentfs, 'content')) pulp_conf.set('server', 'storage_dir', self.parentfs) # create repo manager = managers.repo_manager() manager.create_repo(self.REPO_ID) # add units units = [] for n in range(0, self.NUM_UNITS): unit_id = self.UNIT_ID % n unit = dict(self.UNIT_METADATA) unit['N'] = n # add unit file storage_dir = pulp_conf.get('server', 'storage_dir') storage_path = \ os.path.join(storage_dir, 'content', '.'.join((unit_id, self.UNIT_TYPE_ID))) unit['_storage_path'] = storage_path fp = open(storage_path, 'w+') fp.write(unit_id) fp.close() # add unit manager = managers.content_manager() manager.add_content_unit( self.UNIT_TYPE_ID, unit_id, unit) manager = managers.repo_unit_association_manager() # associate unit manager.associate_unit_by_id( self.REPO_ID, self.UNIT_TYPE_ID, unit_id, RepoContentUnit.OWNER_TYPE_IMPORTER, constants.HTTP_IMPORTER) units.append(unit) # CA self.units = units path = os.path.join(self.parentfs, 'ca.crt') fp = open(path, 'w+') fp.write(self.CA_CERT) fp.close() # client cert path = os.path.join(self.parentfs, 'local.crt') fp = open(path, 'w+') fp.write(self.CLIENT_CERT) fp.close()
def save_unit(self, unit): """ Performs two distinct steps on the Pulp server: - Creates or updates Pulp's knowledge of the content unit. - Associates the unit to the repository being synchronized. This call is idempotent. If the unit already exists or the association already exists, this call will have no effect. A reference to the provided unit is returned from this call. This call will populate the unit's id field with the UUID for the unit. @param unit: unit object returned from the init_unit call @type unit: L{Unit} @return: object reference to the provided unit, its state updated from the call @rtype: L{Unit} """ try: content_query_manager = manager_factory.content_query_manager() content_manager = manager_factory.content_manager() association_manager = manager_factory.repo_unit_association_manager( ) # Save or update the unit pulp_unit = common_utils.to_pulp_unit(unit) try: existing_unit = content_query_manager.get_content_unit_by_keys_dict( unit.type_id, unit.unit_key) unit.id = existing_unit['_id'] content_manager.update_content_unit(unit.type_id, unit.id, pulp_unit) self._updated_count += 1 except MissingResource: unit.id = content_manager.add_content_unit( unit.type_id, None, pulp_unit) self._added_count += 1 # Associate it with the repo association_manager.associate_unit_by_id( self.repo_id, unit.type_id, unit.id, self.association_owner_type, self.association_owner_id) return unit except Exception, e: _LOG.exception( _('Content unit association failed [%s]' % str(unit))) raise ImporterConduitException(e), None, sys.exc_info()[2]
def _purge_unlinked_blobs(repo, manifest): """ Purge blobs associated with the given Manifests when removing it would leave them no longer referenced by any remaining Manifests. :param repo: The affected repository. :type repo: pulp.server.db.model.Repository :param units: List of removed units. :type units: list of: pulp.plugins.model.AssociatedUnit """ # Find blob digests referenced by removed manifests (orphaned) orphaned = set() map((lambda layer: orphaned.add(layer.blob_sum)), manifest.fs_layers) # in manifest schema version 2 there is an additional blob layer called config_layer if manifest.config_layer: orphaned.add(manifest.config_layer) if not orphaned: # nothing orphaned return # Find blob digests still referenced by other manifests (adopted) adopted = set() criteria = UnitAssociationCriteria( type_ids=[constants.MANIFEST_TYPE_ID], unit_filters={'digest': { '$ne': manifest.digest }}) for manifest in unit_association.RepoUnitAssociationManager._units_from_criteria( repo, criteria): map((lambda layer: adopted.add(layer.blob_sum)), manifest.fs_layers) if manifest.config_layer: adopted.add(manifest.config_layer) # Remove unreferenced blobs orphaned = orphaned.difference(adopted) if not orphaned: # all adopted return unit_filter = {'digest': {'$in': sorted(orphaned)}} criteria = UnitAssociationCriteria(type_ids=[constants.BLOB_TYPE_ID], unit_filters=unit_filter) manager = manager_factory.repo_unit_association_manager() manager.unassociate_by_criteria(repo_id=repo.repo_id, criteria=criteria, notify_plugins=False)
def add_units(num_units=10): units = [] n = 0 for type_id in ALL_TYPES: for x in range(0, num_units): unit_id = create_unit_id(type_id, n) unit = dict(UNIT_METADATA) unit['N'] = n unit['_storage_path'] = create_storage_path(unit_id) manager = managers.content_manager() manager.add_content_unit(type_id, unit_id, unit) manager = managers.repo_unit_association_manager() # associate unit manager.associate_unit_by_id(REPO_ID, type_id, unit_id) units.append(unit) n += 1 return units
def add_units(num_units=10): units = [] n = 0 for type_id in ALL_TYPES: for x in range(0, num_units): unit_id = create_unit_id(type_id, n) unit = dict(UNIT_METADATA) unit['N'] = n unit['_storage_path'] = create_storage_path(unit_id) manager = managers.content_manager() manager.add_content_unit(type_id, unit_id, unit) manager = managers.repo_unit_association_manager() # associate unit with mock.patch('pulp.server.managers.repo.unit_association.repo_controller'): manager.associate_unit_by_id(REPO_ID, type_id, unit_id) units.append(unit) n += 1 return units
def _purge_unlinked_blobs(repo, manifest): """ Purge blobs associated with the given Manifests when removing it would leave them no longer referenced by any remaining Manifests. :param repo: The affected repository. :type repo: pulp.server.db.model.Repository :param units: List of removed units. :type units: list of: pulp.plugins.model.AssociatedUnit """ # Find blob digests referenced by removed manifests (orphaned) orphaned = set() map((lambda layer: orphaned.add(layer.blob_sum)), manifest.fs_layers) if not orphaned: # nothing orphaned return # Find blob digests still referenced by other manifests (adopted) adopted = set() criteria = UnitAssociationCriteria(type_ids=[constants.MANIFEST_TYPE_ID], unit_filters={'digest__ne': manifest.digest}) for manifest in unit_association.RepoUnitAssociationManager._units_from_criteria( repo, criteria): map((lambda layer: adopted.add(layer.blob_sum)), manifest.fs_layers) # Remove unreferenced blobs orphaned = orphaned.difference(adopted) if not orphaned: # all adopted return unit_filter = { 'digest': { '$in': sorted(orphaned) } } criteria = UnitAssociationCriteria( type_ids=[constants.BLOB_TYPE_ID], unit_filters=unit_filter) manager = manager_factory.repo_unit_association_manager() manager.unassociate_by_criteria( repo_id=repo.repo_id, criteria=criteria, notify_plugins=False)
def POST(self, dest_repo_id): # Params params = self.params() source_repo_id = params.get('source_repo_id', None) overrides = params.get('override_config', None) if source_repo_id is None: raise exceptions.MissingValue(['source_repo_id']) # A 404 only applies to things in the URL, so the destination repo # check allows the MissingResource to bubble up, but if the source # repo doesn't exist, it's considered bad data. repo_query_manager = manager_factory.repo_query_manager() repo_query_manager.get_repository(dest_repo_id) try: repo_query_manager.get_repository(source_repo_id) except exceptions.MissingResource: raise exceptions.InvalidValue(['source_repo_id']) criteria = params.get('criteria', None) if criteria is not None: try: criteria = UnitAssociationCriteria.from_client_input(criteria) except: _LOG.error('Error parsing association criteria [%s]' % criteria) raise exceptions.PulpDataException(), None, sys.exc_info()[2] association_manager = manager_factory.repo_unit_association_manager() resources = {dispatch_constants.RESOURCE_REPOSITORY_TYPE: {source_repo_id: dispatch_constants.RESOURCE_READ_OPERATION, dest_repo_id: dispatch_constants.RESOURCE_UPDATE_OPERATION}} tags = [resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE, dest_repo_id), resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE, source_repo_id), action_tag('associate')] call_request = CallRequest(association_manager.associate_from_repo, [source_repo_id, dest_repo_id], {'criteria': criteria, 'import_config_override': overrides}, resources=resources, tags=tags, archive=True) return execution.execute_async(self, call_request)
def save_unit(self, unit): """ Performs two distinct steps on the Pulp server: - Creates or updates Pulp's knowledge of the content unit. - Associates the unit to the repository being synchronized. If a unit with the provided unit key already exists, it is updated with the attributes on the passed-in unit. A reference to the provided unit is returned from this call. This call will populate the unit's id field with the UUID for the unit. @param unit: unit object returned from the init_unit call @type unit: L{Unit} @return: object reference to the provided unit, its state updated from the call @rtype: L{Unit} """ try: content_query_manager = manager_factory.content_query_manager() content_manager = manager_factory.content_manager() association_manager = manager_factory.repo_unit_association_manager() # Save or update the unit pulp_unit = common_utils.to_pulp_unit(unit) try: existing_unit = content_query_manager.get_content_unit_by_keys_dict(unit.type_id, unit.unit_key) unit.id = existing_unit['_id'] content_manager.update_content_unit(unit.type_id, unit.id, pulp_unit) self._updated_count += 1 except MissingResource: unit.id = content_manager.add_content_unit(unit.type_id, None, pulp_unit) self._added_count += 1 # Associate it with the repo association_manager.associate_unit_by_id(self.repo_id, unit.type_id, unit.id, self.association_owner_type, self.association_owner_id) return unit except Exception, e: _LOG.exception(_('Content unit association failed [%s]' % str(unit))) raise ImporterConduitException(e), None, sys.exc_info()[2]
def __init__(self, source_repo_id, dest_repo_id, source_importer_id, dest_importer_id, association_owner_type, association_owner_id): """ :param source_repo_id: ID of the repository from which units are being copied :type source_repo_id: str :param dest_repo_id: ID of the repository into which units are being copied :type dest_repo_id: str :param source_importer_id: ID of the importer on the source repository :type source_importer_id: str :param dest_importer_id: ID of the importer on the destination repository :type dest_importer_id: str :param association_owner_type: distinguishes the owner when creating an association through this conduit :type association_owner_type: str :param association_owner_id: specific ID of the owner when creating an association through this conduit :type association_owner_id: str """ ImporterScratchPadMixin.__init__(self, dest_repo_id, dest_importer_id) RepoScratchPadMixin.__init__(self, dest_repo_id, ImporterConduitException) SearchUnitsMixin.__init__(self, ImporterConduitException) AddUnitMixin.__init__(self, dest_repo_id, dest_importer_id, association_owner_type, association_owner_id) self.source_repo_id = source_repo_id self.dest_repo_id = dest_repo_id self.source_importer_id = source_importer_id self.dest_importer_id = dest_importer_id self.association_owner_type = association_owner_type self.association_owner_id = association_owner_id self.__association_manager = manager_factory.repo_unit_association_manager( ) self.__association_query_manager = manager_factory.repo_unit_association_query_manager( ) self.__importer_manager = manager_factory.repo_importer_manager()
def add_units(num_units=10): units = [] n = 0 for type_id in ALL_TYPES: for x in range(0, num_units): unit_id = create_unit_id(type_id, n) unit = dict(UNIT_METADATA) unit['N'] = n unit['_storage_path'] = create_storage_path(unit_id) manager = managers.content_manager() manager.add_content_unit(type_id, unit_id, unit) manager = managers.repo_unit_association_manager() # associate unit manager.associate_unit_by_id( REPO_ID, type_id, unit_id, RepoContentUnit.OWNER_TYPE_IMPORTER, constants.HTTP_IMPORTER) units.append(unit) n += 1 return units
def add_units(self, begin, end): units = [] storage_dir = os.path.join(pulp_conf.get('server', 'storage_dir'), 'content') if not os.path.exists(storage_dir): os.makedirs(storage_dir) for n in range(begin, end): unit_id = self.UNIT_ID % n unit = dict(self.UNIT_KEY) unit.update(self.UNIT_METADATA) unit['N'] = n # add unit file storage_path = os.path.join(storage_dir, '.'.join((unit_id, self.UNIT_TYPE_ID))) if n % 2 == 0: # even numbered has file associated unit['_storage_path'] = storage_path if n == 0: # 1st one is a directory of files os.makedirs(storage_path) dist_path = os.path.join(os.path.dirname(__file__), 'data/distribution.tar') tb = tarfile.open(dist_path) tb.extractall(path=storage_path) tb.close() else: with open(storage_path, 'w+') as fp: fp.write(unit_id) # add unit manager = managers.content_manager() manager.add_content_unit( self.UNIT_TYPE_ID, unit_id, unit) manager = managers.repo_unit_association_manager() # associate unit manager.associate_unit_by_id( self.REPO_ID, self.UNIT_TYPE_ID, unit_id, RepoContentUnit.OWNER_TYPE_IMPORTER, constants.HTTP_IMPORTER) units.append(unit) return units
def _purge_unlinked_tags(repo, manifest): """ Purge Tags associated with the given Manifest (image or list) in the repository. We don't want to leave Tags that reference Manifests (image or lists) that no longer exist. :param repo: The affected repository. :type repo: pulp.server.db.model.Repository :param manifest: The Manifest(image or list) that is being removed :type manifest: pulp_docker.plugins.models.Manifest/ManifestList """ # Find Tag objects that reference the removed Manifest. We can remove any such Tags from # the repository, and from Pulp as well (since Tag objects are repository specific). unit_filter = {'manifest_digest': manifest.digest} criteria = UnitAssociationCriteria(type_ids=[constants.TAG_TYPE_ID], unit_filters=unit_filter) manager = manager_factory.repo_unit_association_manager() manager.unassociate_by_criteria(repo_id=repo.repo_id, criteria=criteria, notify_plugins=False) # Finally, we can remove the Tag objects from Pulp entirely, since Tags are repository # specific. models.Tag.objects.filter(repo_id=repo.repo_id, manifest_digest=manifest.digest).delete()
def POST(self, repo_id): params = self.params() criteria = params.get('criteria', None) if criteria is not None: try: criteria = UnitAssociationCriteria.from_client_input(criteria) except: _LOG.exception('Error parsing unassociation criteria [%s]' % criteria) raise exceptions.PulpDataException(), None, sys.exc_info()[2] association_manager = manager_factory.repo_unit_association_manager() tags = [resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE, repo_id), action_tag('unassociate')] call_request = CallRequest(association_manager.unassociate_by_criteria, [repo_id, criteria, RepoContentUnit.OWNER_TYPE_USER, manager_factory.principal_manager().get_principal()['login']], tags=tags, archive=True) call_request.updates_resource(dispatch_constants.RESOURCE_REPOSITORY_TYPE, repo_id) return execution.execute_async(self, call_request)
def _purge_unlinked_tags(repo, manifest): """ Purge Tags associated with the given Manifest in the repository. We don't want to leave Tags that reference Manifests that no longer exist. :param repo: The affected repository. :type repo: pulp.server.db.model.Repository :param manifest: The Manifest that is being removed :type manifest: pulp_docker.plugins.models.Manifest """ # Find Tag objects that reference the removed Manifest. We can remove any such Tags from # the repository, and from Pulp as well (since Tag objects are repository specific). unit_filter = {'manifest_digest': manifest.digest} criteria = UnitAssociationCriteria( type_ids=[constants.TAG_TYPE_ID], unit_filters=unit_filter) manager = manager_factory.repo_unit_association_manager() manager.unassociate_by_criteria( repo_id=repo.repo_id, criteria=criteria, notify_plugins=False) # Finally, we can remove the Tag objects from Pulp entirely, since Tags are repository # specific. models.Tag.objects.filter(repo_id=repo.repo_id, manifest_digest=manifest.digest).delete()
def associate_units(self): manager = managers.repo_unit_association_manager() # RPMs collection = database.type_units_collection(self.TYPE_ID) for unit in collection.find(): manager.associate_unit_by_id( self.REPO_ID, self.TYPE_ID, unit['_id'], RepoContentUnit.OWNER_TYPE_IMPORTER, 'stuffed', False ) # ERRATA collection = database.type_units_collection(self.ERRATA_TYPE_ID) for unit in collection.find(): manager.associate_unit_by_id( self.REPO_ID, self.ERRATA_TYPE_ID, unit['_id'], RepoContentUnit.OWNER_TYPE_IMPORTER, 'stuffed', False )
def associate_unit(mongo_id, to_repo_id, type_id, mock_update_last): manager = factory.repo_unit_association_manager() manager.associate_unit_by_id(to_repo_id, type_id, mongo_id, update_repo_metadata=True)
def associate_unit(mongo_id, to_repo_id, type_id, mock_update_last): manager = factory.repo_unit_association_manager() manager.associate_unit_by_id(to_repo_id, type_id, mongo_id, update_repo_metadata=True)
import copy import logging import pymongo.errors from pulp.server.managers import factory from pulp.server.managers.repo.cud import RepoContentUnit from pulp.server.managers.repo.unit_association_query import UnitAssociationCriteria from pulp_rpm.common import ids _log = logging.getLogger('pulp') # Initialize plugin loader api and other managers factory.initialize() ass_query_mgr = factory.repo_unit_association_query_manager() ass_mgr = factory.repo_unit_association_manager() content_mgr = factory.content_manager() repo_mgr = factory.repo_manager() def _get_repos(): """ Lookups all the yum based repos in pulp. @return a list of repoids """ repos = factory.repo_query_manager().find_with_importer_type( "yum_importer") if not repos: _log.debug("No repos found to perform db migrate") return [] repo_ids = [repo['id'] for repo in repos]
def associate_unit(mongo_id, to_repo_id, type_id): manager = factory.repo_unit_association_manager() manager.associate_unit_by_id(to_repo_id, type_id, mongo_id, 'importer', 'yum_importer', update_repo_metadata=True)
import logging from pulp.plugins.types import database as types_db from pulp.server.managers import factory from pulp.server.managers.repo.cud import RepoContentUnit from pulp.server.managers.repo.unit_association_query import UnitAssociationCriteria import pymongo.errors from pulp_rpm.common import ids _log = logging.getLogger('pulp') # Initialize plugin loader api and other managers factory.initialize() ass_query_mgr = factory.repo_unit_association_query_manager() ass_mgr = factory.repo_unit_association_manager() content_mgr = factory.content_manager() repo_mgr = factory.repo_manager() def _get_repos(): """ Lookups all the yum based repos in pulp. @return a list of repoids """ repos = factory.repo_query_manager().find_with_importer_type("yum_importer") if not repos: _log.debug("No repos found to perform db migrate") return [] repo_ids = [repo['id'] for repo in repos] return repo_ids