def test_resolve_dependencies_by_criteria(self):
# Setup
report = 'dep report'
mock_plugins.MOCK_IMPORTER.resolve_dependencies.return_value = report
unit_id_1 = manager_factory.content_manager().add_content_unit(
'type-1', None, {'key-1': 'unit-id-1'})
unit_id_2 = manager_factory.content_manager().add_content_unit(
'type-1', None, {'key-1': 'dep-1'})
association_manager = manager_factory.repo_unit_association_manager()
association_manager.associate_unit_by_id(self.repo_id, 'type-1',
unit_id_1)
association_manager.associate_unit_by_id(self.repo_id, 'type-1',
unit_id_2)
criteria = UnitAssociationCriteria(type_ids=['type-1'],
unit_filters={'key-1': 'unit-id-1'})
# Test
result = self.manager.resolve_dependencies_by_criteria(
self.repo_id, criteria, {})
# Verify
self.assertEqual(report, result)
self.assertEqual(
1, mock_plugins.MOCK_IMPORTER.resolve_dependencies.call_count)
args = mock_plugins.MOCK_IMPORTER.resolve_dependencies.call_args[0]
self.assertEqual(1, len(args[1]))
def POST(self, dest_repo_id):
# Params
params = self.params()
source_repo_id = params.get('source_repo_id', None)
overrides = params.get('override_config', None)
if source_repo_id is None:
raise exceptions.MissingValue(['source_repo_id'])
criteria = params.get('criteria', None)
if criteria is not None:
try:
criteria = UnitAssociationCriteria.from_client_input(criteria)
except:
_LOG.exception('Error parsing association criteria [%s]' % criteria)
raise exceptions.PulpDataException(), None, sys.exc_info()[2]
association_manager = manager_factory.repo_unit_association_manager()
resources = {dispatch_constants.RESOURCE_REPOSITORY_TYPE: {source_repo_id: dispatch_constants.RESOURCE_READ_OPERATION,
dest_repo_id: dispatch_constants.RESOURCE_UPDATE_OPERATION}}
tags = [resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE, dest_repo_id),
resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE, source_repo_id),
action_tag('associate')]
call_request = CallRequest(association_manager.associate_from_repo,
[source_repo_id, dest_repo_id],
{'criteria': criteria, 'import_config_override': overrides},
resources=resources,
tags=tags,
archive=True)
return execution.execute_async(self, call_request)
def save_unit(self, unit):
"""
Performs two distinct steps on the Pulp server:
- Creates or updates Pulp's knowledge of the content unit.
- Associates the unit to the repository being synchronized.
If a unit with the provided unit key already exists, it is updated with
the attributes on the passed-in unit.
A reference to the provided unit is returned from this call. This call
will populate the unit's id field with the UUID for the unit.
:param unit: unit object returned from the init_unit call
:type unit: Unit
:return: object reference to the provided unit, its state updated from the call
:rtype: Unit
"""
try:
association_manager = manager_factory.repo_unit_association_manager()
# Save or update the unit
pulp_unit = common_utils.to_pulp_unit(unit)
unit.id = self._update_unit(unit, pulp_unit)
# Associate it with the repo
association_manager.associate_unit_by_id(
self.repo_id, unit.type_id, unit.id)
return unit
except Exception, e:
_logger.exception(_('Content unit association failed [%s]' % str(unit)))
raise ImporterConduitException(e), None, sys.exc_info()[2]
def test_resolve_dependencies_by_criteria(self):
# Setup
report = 'dep report'
mock_plugins.MOCK_IMPORTER.resolve_dependencies.return_value = report
unit_id_1 = manager_factory.content_manager().add_content_unit('type-1', None,
{'key-1': 'unit-id-1'})
unit_id_2 = manager_factory.content_manager().add_content_unit('type-1', None,
{'key-1': 'dep-1'})
association_manager = manager_factory.repo_unit_association_manager()
association_manager.associate_unit_by_id(self.repo_id, 'type-1', unit_id_1)
association_manager.associate_unit_by_id(self.repo_id, 'type-1', unit_id_2)
criteria = UnitAssociationCriteria(type_ids=['type-1'], unit_filters={'key-1': 'unit-id-1'})
# Test
result = self.manager.resolve_dependencies_by_criteria(self.repo_id, criteria, {})
# Verify
self.assertEqual(report, result)
self.assertEqual(1, mock_plugins.MOCK_IMPORTER.resolve_dependencies.call_count)
args = mock_plugins.MOCK_IMPORTER.resolve_dependencies.call_args[0]
self.assertEqual(1, len(args[1]))
def __init__(self, source_repo_id, dest_repo_id, source_importer_id, dest_importer_id):
"""
:param source_repo_id: ID of the repository from which units are being copied
:type source_repo_id: str
:param dest_repo_id: ID of the repository into which units are being copied
:type dest_repo_id: str
:param source_importer_id: ID of the importer on the source repository
:type source_importer_id: str
:param dest_importer_id: ID of the importer on the destination repository
:type dest_importer_id: str
"""
ImporterScratchPadMixin.__init__(self, dest_repo_id, dest_importer_id)
RepoScratchPadMixin.__init__(self, dest_repo_id, ImporterConduitException)
SearchUnitsMixin.__init__(self, ImporterConduitException)
AddUnitMixin.__init__(self, dest_repo_id, dest_importer_id)
self.source_repo_id = source_repo_id
self.dest_repo_id = dest_repo_id
self.source_importer_id = source_importer_id
self.dest_importer_id = dest_importer_id
self.__association_manager = manager_factory.repo_unit_association_manager()
self.__association_query_manager = manager_factory.repo_unit_association_query_manager()
self.__importer_manager = manager_factory.repo_importer_manager()
def test_resolve_dependencies_by_unit(self):
# Setup
report = 'dep report'
mock_plugins.MOCK_IMPORTER.resolve_dependencies.return_value = report
unit_id_1 = manager_factory.content_manager().add_content_unit('type-1', None, {'key-1' : 'v1'})
unit_id_2 = manager_factory.content_manager().add_content_unit('type-1', None, {'key-1' : 'v2'})
association_manager = manager_factory.repo_unit_association_manager()
association_manager.associate_unit_by_id(self.repo_id, 'type-1', unit_id_1, 'user', 'admin')
association_manager.associate_unit_by_id(self.repo_id, 'type-1', unit_id_2, 'user', 'admin')
# Test
result = self.manager.resolve_dependencies_by_units(self.repo_id, [], {})
# Verify
self.assertEqual(result, report)
self.assertEqual(1, mock_plugins.MOCK_IMPORTER.resolve_dependencies.call_count)
args = mock_plugins.MOCK_IMPORTER.resolve_dependencies.call_args[0]
self.assertEqual(args[0].id, self.repo_id)
self.assertEqual(len(args[1]), 0)
self.assertTrue(isinstance(args[2], DependencyResolutionConduit))
self.assertTrue(isinstance(args[3], PluginCallConfiguration))
def POST(self, repo_id):
params = self.params()
criteria = params.get('criteria', None)
if criteria is not None:
try:
criteria = UnitAssociationCriteria.from_client_input(criteria)
except:
_LOG.error('Error parsing unassociation criteria [%s]' %
criteria)
raise exceptions.PulpDataException(), None, sys.exc_info()[2]
association_manager = manager_factory.repo_unit_association_manager()
tags = [
resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE, repo_id),
action_tag('unassociate')
]
call_request = CallRequest(
association_manager.unassociate_by_criteria, [
repo_id, criteria, RepoContentUnit.OWNER_TYPE_USER,
manager_factory.principal_manager().get_principal()['login']
],
tags=tags,
archive=True)
call_request.updates_resource(
dispatch_constants.RESOURCE_REPOSITORY_TYPE, repo_id)
return execution.execute_async(self, call_request)
def __init__(self, source_repo_id, dest_repo_id, source_importer_id, dest_importer_id,
association_owner_type, association_owner_id):
"""
:param source_repo_id: ID of the repository from which units are being copied
:type source_repo_id: str
:param dest_repo_id: ID of the repository into which units are being copied
:type dest_repo_id: str
:param source_importer_id: ID of the importer on the source repository
:type source_importer_id: str
:param dest_importer_id: ID of the importer on the destination repository
:type dest_importer_id: str
:param association_owner_type: distinguishes the owner when creating an
association through this conduit
:type association_owner_type: str
:param association_owner_id: specific ID of the owner when creating an
association through this conduit
:type association_owner_id: str
"""
ImporterScratchPadMixin.__init__(self, dest_repo_id, dest_importer_id)
RepoScratchPadMixin.__init__(self, dest_repo_id, ImporterConduitException)
SearchUnitsMixin.__init__(self, ImporterConduitException)
AddUnitMixin.__init__(self, dest_repo_id, dest_importer_id, association_owner_type, association_owner_id)
self.source_repo_id = source_repo_id
self.dest_repo_id = dest_repo_id
self.source_importer_id = source_importer_id
self.dest_importer_id = dest_importer_id
self.association_owner_type = association_owner_type
self.association_owner_id = association_owner_id
self.__association_manager = manager_factory.repo_unit_association_manager()
self.__association_query_manager = manager_factory.repo_unit_association_query_manager()
self.__importer_manager = manager_factory.repo_importer_manager()
def add_units(self, begin, end):
units = []
storage_dir = os.path.join(
pulp_conf.config.get('server', 'storage_dir'), 'content')
if not os.path.exists(storage_dir):
os.makedirs(storage_dir)
for n in range(begin, end):
unit_id = self.UNIT_ID % n
unit = dict(self.UNIT_KEY)
unit.update(self.UNIT_METADATA)
unit['N'] = n
# add unit file
storage_path = os.path.join(storage_dir, '.'.join(
(unit_id, self.UNIT_TYPE_ID)))
if n % 2 == 0: # even numbered has file associated
unit['_storage_path'] = storage_path
if n == 0: # 1st one is a directory of files
os.makedirs(storage_path)
dist_path = os.path.join(os.path.dirname(__file__),
'data/distribution.tar')
tb = tarfile.open(dist_path)
tb.extractall(path=storage_path)
tb.close()
else:
with open(storage_path, 'w+') as fp:
fp.write(unit_id)
# add unit
manager = managers.content_manager()
manager.add_content_unit(self.UNIT_TYPE_ID, unit_id, unit)
manager = managers.repo_unit_association_manager()
# associate unit
manager.associate_unit_by_id(self.REPO_ID, self.UNIT_TYPE_ID,
unit_id)
units.append(unit)
return units
def test_syntactic_sugar_methods(self):
"""
Tests the syntactic sugar methods for retrieving specific managers.
"""
# Setup
factory.initialize()
# Test
self.assertTrue(isinstance(factory.authentication_manager(), AuthenticationManager))
self.assertTrue(isinstance(factory.cert_generation_manager(), CertGenerationManager))
self.assertTrue(isinstance(factory.certificate_manager(), CertificateManager))
self.assertTrue(isinstance(factory.password_manager(), PasswordManager))
self.assertTrue(isinstance(factory.permission_manager(), PermissionManager))
self.assertTrue(isinstance(factory.permission_query_manager(), PermissionQueryManager))
self.assertTrue(isinstance(factory.role_manager(), RoleManager))
self.assertTrue(isinstance(factory.role_query_manager(), RoleQueryManager))
self.assertTrue(isinstance(factory.user_manager(), UserManager))
self.assertTrue(isinstance(factory.user_query_manager(), UserQueryManager))
self.assertTrue(isinstance(factory.repo_manager(), RepoManager))
self.assertTrue(isinstance(factory.repo_unit_association_manager(),
RepoUnitAssociationManager))
self.assertTrue(isinstance(factory.repo_publish_manager(), RepoPublishManager))
self.assertTrue(isinstance(factory.repo_query_manager(), RepoQueryManager))
self.assertTrue(isinstance(factory.repo_sync_manager(), RepoSyncManager))
self.assertTrue(isinstance(factory.content_manager(), ContentManager))
self.assertTrue(isinstance(factory.content_query_manager(), ContentQueryManager))
self.assertTrue(isinstance(factory.content_upload_manager(), ContentUploadManager))
self.assertTrue(isinstance(factory.consumer_manager(), ConsumerManager))
self.assertTrue(isinstance(factory.topic_publish_manager(), TopicPublishManager))
def test_resolve_dependencies_by_unit(self):
# Setup
report = 'dep report'
mock_plugins.MOCK_IMPORTER.resolve_dependencies.return_value = report
unit_id_1 = manager_factory.content_manager().add_content_unit(
'type-1', None, {'key-1': 'v1'})
unit_id_2 = manager_factory.content_manager().add_content_unit(
'type-1', None, {'key-1': 'v2'})
association_manager = manager_factory.repo_unit_association_manager()
association_manager.associate_unit_by_id(self.repo_id, 'type-1',
unit_id_1)
association_manager.associate_unit_by_id(self.repo_id, 'type-1',
unit_id_2)
# Test
result = self.manager.resolve_dependencies_by_units(
self.repo_id, [], {})
# Verify
self.assertEqual(result, report)
self.assertEqual(
1, mock_plugins.MOCK_IMPORTER.resolve_dependencies.call_count)
args = mock_plugins.MOCK_IMPORTER.resolve_dependencies.call_args[0]
self.assertEqual(args[0].id, self.repo_id)
self.assertEqual(len(args[1]), 0)
self.assertTrue(isinstance(args[2], DependencyResolutionConduit))
self.assertTrue(isinstance(args[3], PluginCallConfiguration))
def test_syntactic_sugar_methods(self):
"""
Tests the syntactic sugar methods for retrieving specific managers.
"""
# Setup
factory.initialize()
# Test
self.assertTrue(isinstance(factory.authentication_manager(), AuthenticationManager))
self.assertTrue(isinstance(factory.cert_generation_manager(), CertGenerationManager))
self.assertTrue(isinstance(factory.certificate_manager(), CertificateManager))
self.assertTrue(isinstance(factory.password_manager(), PasswordManager))
self.assertTrue(isinstance(factory.permission_manager(), PermissionManager))
self.assertTrue(isinstance(factory.permission_query_manager(), PermissionQueryManager))
self.assertTrue(isinstance(factory.role_manager(), RoleManager))
self.assertTrue(isinstance(factory.role_query_manager(), RoleQueryManager))
self.assertTrue(isinstance(factory.user_manager(), UserManager))
self.assertTrue(isinstance(factory.user_query_manager(), UserQueryManager))
self.assertTrue(isinstance(factory.repo_manager(), RepoManager))
self.assertTrue(isinstance(factory.repo_unit_association_manager(), RepoUnitAssociationManager))
self.assertTrue(isinstance(factory.repo_publish_manager(), RepoPublishManager))
self.assertTrue(isinstance(factory.repo_query_manager(), RepoQueryManager))
self.assertTrue(isinstance(factory.repo_sync_manager(), RepoSyncManager))
self.assertTrue(isinstance(factory.content_manager(), ContentManager))
self.assertTrue(isinstance(factory.content_query_manager(), ContentQueryManager))
self.assertTrue(isinstance(factory.content_upload_manager(), ContentUploadManager))
self.assertTrue(isinstance(factory.consumer_manager(), ConsumerManager))
self.assertTrue(isinstance(factory.topic_publish_manager(), TopicPublishManager))
def __init__(self, source_repo_id, dest_repo_id, source_importer_id,
dest_importer_id):
"""
:param source_repo_id: ID of the repository from which units are being copied
:type source_repo_id: str
:param dest_repo_id: ID of the repository into which units are being copied
:type dest_repo_id: str
:param source_importer_id: ID of the importer on the source repository
:type source_importer_id: str
:param dest_importer_id: ID of the importer on the destination repository
:type dest_importer_id: str
"""
ImporterScratchPadMixin.__init__(self, dest_repo_id, dest_importer_id)
RepoScratchPadMixin.__init__(self, dest_repo_id,
ImporterConduitException)
SearchUnitsMixin.__init__(self, ImporterConduitException)
AddUnitMixin.__init__(self, dest_repo_id, dest_importer_id)
self.source_repo_id = source_repo_id
self.dest_repo_id = dest_repo_id
self.source_importer_id = source_importer_id
self.dest_importer_id = dest_importer_id
self.__association_manager = manager_factory.repo_unit_association_manager(
)
self.__association_query_manager = manager_factory.repo_unit_association_query_manager(
)
self.__importer_manager = manager_factory.repo_importer_manager()
def _purge_unlinked_manifests(repo, manifest_list):
# Find manifest digests referenced by removed manifest lists (orphaned)
orphaned = set()
for image_man in manifest_list.manifests:
orphaned.add(image_man)
if manifest_list.amd64_digest:
orphaned.add(manifest_list.amd64_digest)
if not orphaned:
# nothing orphaned
return
# Find manifest digests still referenced by other manifest lists (adopted)
adopted = set()
criteria = UnitAssociationCriteria(
type_ids=[constants.MANIFEST_LIST_TYPE_ID],
unit_filters={'digest': {
'$ne': manifest_list.digest
}})
for man_list in unit_association.RepoUnitAssociationManager._units_from_criteria(
repo, criteria):
for image_man in man_list.manifests:
adopted.add(image_man)
if man_list.amd64_digest:
adopted.add(man_list.amd64_digest)
# Remove unreferenced manifests
orphaned = orphaned.difference(adopted)
if not orphaned:
# all adopted
return
# Check if those manifests have tags, tagged manifests cannot be removed
criteria = UnitAssociationCriteria(type_ids=[constants.TAG_TYPE_ID],
unit_filters={
'manifest_digest': {
'$in': list(orphaned)
},
'manifest_type':
constants.MANIFEST_IMAGE_TYPE
})
for tag in unit_association.RepoUnitAssociationManager._units_from_criteria(
repo, criteria):
orphaned.remove(tag.manifest_digest)
unit_filter = {'digest': {'$in': sorted(orphaned)}}
criteria = UnitAssociationCriteria(
type_ids=[constants.MANIFEST_TYPE_ID], unit_filters=unit_filter)
manager = manager_factory.repo_unit_association_manager()
manager.unassociate_by_criteria(repo_id=repo.repo_id,
criteria=criteria,
notify_plugins=False)
for manifest in models.Manifest.objects.filter(
digest__in=sorted(orphaned)):
DockerImporter._purge_unlinked_blobs(repo, manifest)
def __init__(self, repo_id, importer_id, association_owner_type, association_owner_id):
RepoScratchPadMixin.__init__(self, repo_id, ImporterConduitException)
ImporterScratchPadMixin.__init__(self, repo_id, importer_id)
AddUnitMixin.__init__(self, repo_id, importer_id, association_owner_type, association_owner_id)
SingleRepoUnitsMixin.__init__(self, repo_id, ImporterConduitException)
StatusMixin.__init__(self, importer_id, ImporterConduitException)
self._association_manager = manager_factory.repo_unit_association_manager()
self._removed_count = 0
def populate_units(self, key, typedef):
for i in range(1, 10):
unit_id = 'unit-%s' % self.UNIT_ID
md = {key: str(i)}
manager = factory.content_manager()
manager.add_content_unit(typedef.id, unit_id, md)
manager = factory.repo_unit_association_manager()
manager.associate_unit_by_id(self.REPO_ID, typedef.id, unit_id,
OWNER_TYPE_IMPORTER, 'test-importer')
self.UNIT_ID += 1
def _purge_orphaned_blobs(repo, units):
"""
Purge blobs associated with removed manifests when no longer
referenced by any remaining manifests.
:param repo: The affected repository.
:type repo: pulp.plugins.model.Repository
:param units: List of removed units.
:type units: list of: pulp.plugins.model.AssociatedUnit
"""
# Find blob digests referenced by removed manifests (orphaned)
orphaned = set()
for unit in units:
if unit.type_id != constants.MANIFEST_TYPE_ID:
continue
manifest = unit
for layer in manifest.metadata['fs_layers']:
digest = layer['blobSum']
orphaned.add(digest)
# Find blob digests still referenced by other manifests (adopted)
if not orphaned:
# nothing orphaned
return
adopted = set()
manager = manager_factory.repo_unit_association_query_manager()
for manifest in manager.get_units_by_type(repo.id, constants.MANIFEST_TYPE_ID):
for layer in manifest.metadata['fs_layers']:
digest = layer['blobSum']
adopted.add(digest)
# Remove unreferenced blobs
orphaned = orphaned.difference(adopted)
if not orphaned:
# all adopted
return
unit_filter = {
'digest': {
'$in': sorted(orphaned)
}
}
criteria = UnitAssociationCriteria(
type_ids=[constants.BLOB_TYPE_ID],
unit_filters=unit_filter)
manager = manager_factory.repo_unit_association_manager()
manager.unassociate_by_criteria(
repo_id=repo.id,
criteria=criteria,
owner_type='', # unused
owner_id='', # unused
notify_plugins=False)
def __init__(self, repo_id, importer_id, association_owner_type, association_owner_id):
RepoScratchPadMixin.__init__(self, repo_id, ImporterConduitException)
ImporterScratchPadMixin.__init__(self, repo_id, importer_id)
AddUnitMixin.__init__(self, repo_id, importer_id, association_owner_type, association_owner_id)
SingleRepoUnitsMixin.__init__(self, repo_id, ImporterConduitException)
StatusMixin.__init__(self, importer_id, ImporterConduitException)
SearchUnitsMixin.__init__(self, ImporterConduitException)
self._association_manager = manager_factory.repo_unit_association_manager()
self._removed_count = 0
def populate_units(self, key, typedef, additional_key=None):
for i in range(1, 10):
unit_id = 'unit-%s' % self.UNIT_ID
md = {key: str(i)}
if additional_key:
md[additional_key] = str(i)
manager = factory.content_manager()
manager.add_content_unit(typedef.id, unit_id, md)
manager = factory.repo_unit_association_manager()
manager.associate_unit_by_id(self.REPO_ID, typedef.id, unit_id)
self.UNIT_ID += 1
def POST(self, dest_repo_id):
# Params
params = self.params()
source_repo_id = params.get('source_repo_id', None)
overrides = params.get('override_config', None)
if source_repo_id is None:
raise exceptions.MissingValue(['source_repo_id'])
# A 404 only applies to things in the URL, so the destination repo
# check allows the MissingResource to bubble up, but if the source
# repo doesn't exist, it's considered bad data.
repo_query_manager = manager_factory.repo_query_manager()
repo_query_manager.get_repository(dest_repo_id)
try:
repo_query_manager.get_repository(source_repo_id)
except exceptions.MissingResource:
raise exceptions.InvalidValue(['source_repo_id'])
criteria = params.get('criteria', None)
if criteria is not None:
try:
criteria = UnitAssociationCriteria.from_client_input(criteria)
except:
_LOG.error('Error parsing association criteria [%s]' %
criteria)
raise exceptions.PulpDataException(), None, sys.exc_info()[2]
association_manager = manager_factory.repo_unit_association_manager()
tags = [
resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE,
dest_repo_id),
resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE,
source_repo_id),
action_tag('associate')
]
call_request = CallRequest(
association_manager.associate_from_repo,
[source_repo_id, dest_repo_id], {
'criteria': criteria,
'import_config_override': overrides
},
tags=tags,
archive=True,
kwarg_blacklist=['criteria', 'import_config_override'])
call_request.reads_resource(
dispatch_constants.RESOURCE_REPOSITORY_TYPE, source_repo_id)
call_request.updates_resource(
dispatch_constants.RESOURCE_REPOSITORY_TYPE, dest_repo_id)
return execution.execute_async(self, call_request)
def __init__(self, repo_id, importer_id, importer_object_id):
RepoScratchPadMixin.__init__(self, repo_id, ImporterConduitException)
ImporterScratchPadMixin.__init__(self, repo_id, importer_id)
AddUnitMixin.__init__(self, repo_id, importer_id)
SingleRepoUnitsMixin.__init__(self, repo_id, ImporterConduitException)
StatusMixin.__init__(self, importer_id, ImporterConduitException)
SearchUnitsMixin.__init__(self, ImporterConduitException)
self.importer_object_id = importer_object_id
self._association_manager = manager_factory.repo_unit_association_manager()
self._content_query_manager = manager_factory.content_query_manager()
self._removed_count = 0
def __init__(self, repo_id, importer_id, importer_object_id):
RepoScratchPadMixin.__init__(self, repo_id, ImporterConduitException)
ImporterScratchPadMixin.__init__(self, repo_id, importer_id)
AddUnitMixin.__init__(self, repo_id, importer_id)
SingleRepoUnitsMixin.__init__(self, repo_id, ImporterConduitException)
StatusMixin.__init__(self, importer_id, ImporterConduitException)
SearchUnitsMixin.__init__(self, ImporterConduitException)
self.importer_object_id = importer_object_id
self._association_manager = manager_factory.repo_unit_association_manager()
self._content_query_manager = manager_factory.content_query_manager()
self._removed_count = 0
def __init__(self, source_repo_id, dest_repo_id, source_importer_id, dest_importer_id):
ImporterScratchPadMixin.__init__(self, dest_repo_id, dest_importer_id)
RepoScratchPadMixin.__init__(self, dest_repo_id, ImporterConduitException)
self.source_repo_id = source_repo_id
self.dest_repo_id = dest_repo_id
self.source_importer_id = source_importer_id
self.dest_importer_id = dest_importer_id
self.__association_manager = manager_factory.repo_unit_association_manager()
self.__association_query_manager = manager_factory.repo_unit_association_query_manager()
self.__importer_manager = manager_factory.repo_importer_manager()
def populate_units(self, key, typedef):
for i in range(1,10):
unit_id = 'unit-%s' % self.UNIT_ID
md = {key:str(i)}
manager = factory.content_manager()
manager.add_content_unit(typedef.id, unit_id, md)
manager = factory.repo_unit_association_manager()
manager.associate_unit_by_id(
self.REPO_ID,
typedef.id,
unit_id,
OWNER_TYPE_IMPORTER,
'test-importer')
self.UNIT_ID += 1
def populate_units(self, key, typedef, additional_key=None):
for i in range(1, 10):
unit_id = 'unit-%s' % self.UNIT_ID
md = {key: str(i)}
if additional_key:
md[additional_key] = str(i)
manager = factory.content_manager()
manager.add_content_unit(typedef.id, unit_id, md)
manager = factory.repo_unit_association_manager()
manager.associate_unit_by_id(
self.REPO_ID,
typedef.id,
unit_id)
self.UNIT_ID += 1
def test_syntactic_sugar_methods(self):
"""
Tests the syntactic sugar methods for retrieving specific managers.
"""
# Test
self.assertTrue(isinstance(factory.repo_manager(), RepoManager))
self.assertTrue(isinstance(factory.repo_unit_association_manager(), RepoUnitAssociationManager))
self.assertTrue(isinstance(factory.repo_publish_manager(), RepoPublishManager))
self.assertTrue(isinstance(factory.repo_query_manager(), RepoQueryManager))
self.assertTrue(isinstance(factory.repo_sync_manager(), RepoSyncManager))
self.assertTrue(isinstance(factory.content_manager(), ContentManager))
self.assertTrue(isinstance(factory.content_query_manager(), ContentQueryManager))
self.assertTrue(isinstance(factory.content_upload_manager(), ContentUploadManager))
self.assertTrue(isinstance(factory.consumer_manager(), ConsumerManager))
def populate(self):
# make content/ dir.
os.makedirs(os.path.join(self.parentfs, 'content'))
pulp_conf.set('server', 'storage_dir', self.parentfs)
# create repo
manager = managers.repo_manager()
manager.create_repo(self.REPO_ID)
# add units
units = []
for n in range(0, self.NUM_UNITS):
unit_id = self.UNIT_ID % n
unit = dict(self.UNIT_METADATA)
unit['N'] = n
# add unit file
storage_dir = pulp_conf.get('server', 'storage_dir')
storage_path = \
os.path.join(storage_dir, 'content',
'.'.join((unit_id, self.UNIT_TYPE_ID)))
unit['_storage_path'] = storage_path
fp = open(storage_path, 'w+')
fp.write(unit_id)
fp.close()
# add unit
manager = managers.content_manager()
manager.add_content_unit(
self.UNIT_TYPE_ID,
unit_id,
unit)
manager = managers.repo_unit_association_manager()
# associate unit
manager.associate_unit_by_id(
self.REPO_ID,
self.UNIT_TYPE_ID,
unit_id,
RepoContentUnit.OWNER_TYPE_IMPORTER,
constants.HTTP_IMPORTER)
units.append(unit)
# CA
self.units = units
path = os.path.join(self.parentfs, 'ca.crt')
fp = open(path, 'w+')
fp.write(self.CA_CERT)
fp.close()
# client cert
path = os.path.join(self.parentfs, 'local.crt')
fp = open(path, 'w+')
fp.write(self.CLIENT_CERT)
fp.close()
def save_unit(self, unit):
"""
Performs two distinct steps on the Pulp server:
- Creates or updates Pulp's knowledge of the content unit.
- Associates the unit to the repository being synchronized.
This call is idempotent. If the unit already exists or the association
already exists, this call will have no effect.
A reference to the provided unit is returned from this call. This call
will populate the unit's id field with the UUID for the unit.
@param unit: unit object returned from the init_unit call
@type unit: L{Unit}
@return: object reference to the provided unit, its state updated from the call
@rtype: L{Unit}
"""
try:
content_query_manager = manager_factory.content_query_manager()
content_manager = manager_factory.content_manager()
association_manager = manager_factory.repo_unit_association_manager(
)
# Save or update the unit
pulp_unit = common_utils.to_pulp_unit(unit)
try:
existing_unit = content_query_manager.get_content_unit_by_keys_dict(
unit.type_id, unit.unit_key)
unit.id = existing_unit['_id']
content_manager.update_content_unit(unit.type_id, unit.id,
pulp_unit)
self._updated_count += 1
except MissingResource:
unit.id = content_manager.add_content_unit(
unit.type_id, None, pulp_unit)
self._added_count += 1
# Associate it with the repo
association_manager.associate_unit_by_id(
self.repo_id, unit.type_id, unit.id,
self.association_owner_type, self.association_owner_id)
return unit
except Exception, e:
_LOG.exception(
_('Content unit association failed [%s]' % str(unit)))
raise ImporterConduitException(e), None, sys.exc_info()[2]
def _purge_unlinked_blobs(repo, manifest):
"""
Purge blobs associated with the given Manifests when removing it would leave them no longer
referenced by any remaining Manifests.
:param repo: The affected repository.
:type repo: pulp.server.db.model.Repository
:param units: List of removed units.
:type units: list of: pulp.plugins.model.AssociatedUnit
"""
# Find blob digests referenced by removed manifests (orphaned)
orphaned = set()
map((lambda layer: orphaned.add(layer.blob_sum)), manifest.fs_layers)
# in manifest schema version 2 there is an additional blob layer called config_layer
if manifest.config_layer:
orphaned.add(manifest.config_layer)
if not orphaned:
# nothing orphaned
return
# Find blob digests still referenced by other manifests (adopted)
adopted = set()
criteria = UnitAssociationCriteria(
type_ids=[constants.MANIFEST_TYPE_ID],
unit_filters={'digest': {
'$ne': manifest.digest
}})
for manifest in unit_association.RepoUnitAssociationManager._units_from_criteria(
repo, criteria):
map((lambda layer: adopted.add(layer.blob_sum)),
manifest.fs_layers)
if manifest.config_layer:
adopted.add(manifest.config_layer)
# Remove unreferenced blobs
orphaned = orphaned.difference(adopted)
if not orphaned:
# all adopted
return
unit_filter = {'digest': {'$in': sorted(orphaned)}}
criteria = UnitAssociationCriteria(type_ids=[constants.BLOB_TYPE_ID],
unit_filters=unit_filter)
manager = manager_factory.repo_unit_association_manager()
manager.unassociate_by_criteria(repo_id=repo.repo_id,
criteria=criteria,
notify_plugins=False)
def add_units(num_units=10):
units = []
n = 0
for type_id in ALL_TYPES:
for x in range(0, num_units):
unit_id = create_unit_id(type_id, n)
unit = dict(UNIT_METADATA)
unit['N'] = n
unit['_storage_path'] = create_storage_path(unit_id)
manager = managers.content_manager()
manager.add_content_unit(type_id, unit_id, unit)
manager = managers.repo_unit_association_manager()
# associate unit
manager.associate_unit_by_id(REPO_ID, type_id, unit_id)
units.append(unit)
n += 1
return units
def add_units(num_units=10):
units = []
n = 0
for type_id in ALL_TYPES:
for x in range(0, num_units):
unit_id = create_unit_id(type_id, n)
unit = dict(UNIT_METADATA)
unit['N'] = n
unit['_storage_path'] = create_storage_path(unit_id)
manager = managers.content_manager()
manager.add_content_unit(type_id, unit_id, unit)
manager = managers.repo_unit_association_manager()
# associate unit
with mock.patch('pulp.server.managers.repo.unit_association.repo_controller'):
manager.associate_unit_by_id(REPO_ID, type_id, unit_id)
units.append(unit)
n += 1
return units
def _purge_unlinked_blobs(repo, manifest):
"""
Purge blobs associated with the given Manifests when removing it would leave them no longer
referenced by any remaining Manifests.
:param repo: The affected repository.
:type repo: pulp.server.db.model.Repository
:param units: List of removed units.
:type units: list of: pulp.plugins.model.AssociatedUnit
"""
# Find blob digests referenced by removed manifests (orphaned)
orphaned = set()
map((lambda layer: orphaned.add(layer.blob_sum)), manifest.fs_layers)
if not orphaned:
# nothing orphaned
return
# Find blob digests still referenced by other manifests (adopted)
adopted = set()
criteria = UnitAssociationCriteria(type_ids=[constants.MANIFEST_TYPE_ID],
unit_filters={'digest__ne': manifest.digest})
for manifest in unit_association.RepoUnitAssociationManager._units_from_criteria(
repo, criteria):
map((lambda layer: adopted.add(layer.blob_sum)), manifest.fs_layers)
# Remove unreferenced blobs
orphaned = orphaned.difference(adopted)
if not orphaned:
# all adopted
return
unit_filter = {
'digest': {
'$in': sorted(orphaned)
}
}
criteria = UnitAssociationCriteria(
type_ids=[constants.BLOB_TYPE_ID],
unit_filters=unit_filter)
manager = manager_factory.repo_unit_association_manager()
manager.unassociate_by_criteria(
repo_id=repo.repo_id,
criteria=criteria,
notify_plugins=False)
def POST(self, dest_repo_id):
# Params
params = self.params()
source_repo_id = params.get('source_repo_id', None)
overrides = params.get('override_config', None)
if source_repo_id is None:
raise exceptions.MissingValue(['source_repo_id'])
# A 404 only applies to things in the URL, so the destination repo
# check allows the MissingResource to bubble up, but if the source
# repo doesn't exist, it's considered bad data.
repo_query_manager = manager_factory.repo_query_manager()
repo_query_manager.get_repository(dest_repo_id)
try:
repo_query_manager.get_repository(source_repo_id)
except exceptions.MissingResource:
raise exceptions.InvalidValue(['source_repo_id'])
criteria = params.get('criteria', None)
if criteria is not None:
try:
criteria = UnitAssociationCriteria.from_client_input(criteria)
except:
_LOG.error('Error parsing association criteria [%s]' % criteria)
raise exceptions.PulpDataException(), None, sys.exc_info()[2]
association_manager = manager_factory.repo_unit_association_manager()
resources = {dispatch_constants.RESOURCE_REPOSITORY_TYPE: {source_repo_id: dispatch_constants.RESOURCE_READ_OPERATION,
dest_repo_id: dispatch_constants.RESOURCE_UPDATE_OPERATION}}
tags = [resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE, dest_repo_id),
resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE, source_repo_id),
action_tag('associate')]
call_request = CallRequest(association_manager.associate_from_repo,
[source_repo_id, dest_repo_id],
{'criteria': criteria, 'import_config_override': overrides},
resources=resources,
tags=tags,
archive=True)
return execution.execute_async(self, call_request)
def save_unit(self, unit):
"""
Performs two distinct steps on the Pulp server:
- Creates or updates Pulp's knowledge of the content unit.
- Associates the unit to the repository being synchronized.
If a unit with the provided unit key already exists, it is updated with
the attributes on the passed-in unit.
A reference to the provided unit is returned from this call. This call
will populate the unit's id field with the UUID for the unit.
@param unit: unit object returned from the init_unit call
@type unit: L{Unit}
@return: object reference to the provided unit, its state updated from the call
@rtype: L{Unit}
"""
try:
content_query_manager = manager_factory.content_query_manager()
content_manager = manager_factory.content_manager()
association_manager = manager_factory.repo_unit_association_manager()
# Save or update the unit
pulp_unit = common_utils.to_pulp_unit(unit)
try:
existing_unit = content_query_manager.get_content_unit_by_keys_dict(unit.type_id, unit.unit_key)
unit.id = existing_unit['_id']
content_manager.update_content_unit(unit.type_id, unit.id, pulp_unit)
self._updated_count += 1
except MissingResource:
unit.id = content_manager.add_content_unit(unit.type_id, None, pulp_unit)
self._added_count += 1
# Associate it with the repo
association_manager.associate_unit_by_id(self.repo_id, unit.type_id, unit.id, self.association_owner_type, self.association_owner_id)
return unit
except Exception, e:
_LOG.exception(_('Content unit association failed [%s]' % str(unit)))
raise ImporterConduitException(e), None, sys.exc_info()[2]
def __init__(self, source_repo_id, dest_repo_id, source_importer_id,
dest_importer_id, association_owner_type,
association_owner_id):
"""
:param source_repo_id: ID of the repository from which units are being copied
:type source_repo_id: str
:param dest_repo_id: ID of the repository into which units are being copied
:type dest_repo_id: str
:param source_importer_id: ID of the importer on the source repository
:type source_importer_id: str
:param dest_importer_id: ID of the importer on the destination repository
:type dest_importer_id: str
:param association_owner_type: distinguishes the owner when creating an
association through this conduit
:type association_owner_type: str
:param association_owner_id: specific ID of the owner when creating an
association through this conduit
:type association_owner_id: str
"""
ImporterScratchPadMixin.__init__(self, dest_repo_id, dest_importer_id)
RepoScratchPadMixin.__init__(self, dest_repo_id,
ImporterConduitException)
SearchUnitsMixin.__init__(self, ImporterConduitException)
AddUnitMixin.__init__(self, dest_repo_id, dest_importer_id,
association_owner_type, association_owner_id)
self.source_repo_id = source_repo_id
self.dest_repo_id = dest_repo_id
self.source_importer_id = source_importer_id
self.dest_importer_id = dest_importer_id
self.association_owner_type = association_owner_type
self.association_owner_id = association_owner_id
self.__association_manager = manager_factory.repo_unit_association_manager(
)
self.__association_query_manager = manager_factory.repo_unit_association_query_manager(
)
self.__importer_manager = manager_factory.repo_importer_manager()
def add_units(num_units=10):
units = []
n = 0
for type_id in ALL_TYPES:
for x in range(0, num_units):
unit_id = create_unit_id(type_id, n)
unit = dict(UNIT_METADATA)
unit['N'] = n
unit['_storage_path'] = create_storage_path(unit_id)
manager = managers.content_manager()
manager.add_content_unit(type_id, unit_id, unit)
manager = managers.repo_unit_association_manager()
# associate unit
manager.associate_unit_by_id(
REPO_ID,
type_id,
unit_id,
RepoContentUnit.OWNER_TYPE_IMPORTER,
constants.HTTP_IMPORTER)
units.append(unit)
n += 1
return units
def add_units(self, begin, end):
units = []
storage_dir = os.path.join(pulp_conf.get('server', 'storage_dir'), 'content')
if not os.path.exists(storage_dir):
os.makedirs(storage_dir)
for n in range(begin, end):
unit_id = self.UNIT_ID % n
unit = dict(self.UNIT_KEY)
unit.update(self.UNIT_METADATA)
unit['N'] = n
# add unit file
storage_path = os.path.join(storage_dir, '.'.join((unit_id, self.UNIT_TYPE_ID)))
if n % 2 == 0: # even numbered has file associated
unit['_storage_path'] = storage_path
if n == 0: # 1st one is a directory of files
os.makedirs(storage_path)
dist_path = os.path.join(os.path.dirname(__file__), 'data/distribution.tar')
tb = tarfile.open(dist_path)
tb.extractall(path=storage_path)
tb.close()
else:
with open(storage_path, 'w+') as fp:
fp.write(unit_id)
# add unit
manager = managers.content_manager()
manager.add_content_unit(
self.UNIT_TYPE_ID,
unit_id,
unit)
manager = managers.repo_unit_association_manager()
# associate unit
manager.associate_unit_by_id(
self.REPO_ID,
self.UNIT_TYPE_ID,
unit_id,
RepoContentUnit.OWNER_TYPE_IMPORTER,
constants.HTTP_IMPORTER)
units.append(unit)
return units
def _purge_unlinked_tags(repo, manifest):
"""
Purge Tags associated with the given Manifest (image or list) in the repository.
We don't want to leave Tags that reference Manifests (image or lists) that no longer exist.
:param repo: The affected repository.
:type repo: pulp.server.db.model.Repository
:param manifest: The Manifest(image or list) that is being removed
:type manifest: pulp_docker.plugins.models.Manifest/ManifestList
"""
# Find Tag objects that reference the removed Manifest. We can remove any such Tags from
# the repository, and from Pulp as well (since Tag objects are repository specific).
unit_filter = {'manifest_digest': manifest.digest}
criteria = UnitAssociationCriteria(type_ids=[constants.TAG_TYPE_ID],
unit_filters=unit_filter)
manager = manager_factory.repo_unit_association_manager()
manager.unassociate_by_criteria(repo_id=repo.repo_id,
criteria=criteria,
notify_plugins=False)
# Finally, we can remove the Tag objects from Pulp entirely, since Tags are repository
# specific.
models.Tag.objects.filter(repo_id=repo.repo_id,
manifest_digest=manifest.digest).delete()
def POST(self, repo_id):
params = self.params()
criteria = params.get('criteria', None)
if criteria is not None:
try:
criteria = UnitAssociationCriteria.from_client_input(criteria)
except:
_LOG.exception('Error parsing unassociation criteria [%s]' % criteria)
raise exceptions.PulpDataException(), None, sys.exc_info()[2]
association_manager = manager_factory.repo_unit_association_manager()
tags = [resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE, repo_id),
action_tag('unassociate')]
call_request = CallRequest(association_manager.unassociate_by_criteria,
[repo_id, criteria, RepoContentUnit.OWNER_TYPE_USER, manager_factory.principal_manager().get_principal()['login']],
tags=tags,
archive=True)
call_request.updates_resource(dispatch_constants.RESOURCE_REPOSITORY_TYPE, repo_id)
return execution.execute_async(self, call_request)
def _purge_unlinked_tags(repo, manifest):
"""
Purge Tags associated with the given Manifest in the repository. We don't want to leave Tags
that reference Manifests that no longer exist.
:param repo: The affected repository.
:type repo: pulp.server.db.model.Repository
:param manifest: The Manifest that is being removed
:type manifest: pulp_docker.plugins.models.Manifest
"""
# Find Tag objects that reference the removed Manifest. We can remove any such Tags from
# the repository, and from Pulp as well (since Tag objects are repository specific).
unit_filter = {'manifest_digest': manifest.digest}
criteria = UnitAssociationCriteria(
type_ids=[constants.TAG_TYPE_ID],
unit_filters=unit_filter)
manager = manager_factory.repo_unit_association_manager()
manager.unassociate_by_criteria(
repo_id=repo.repo_id,
criteria=criteria,
notify_plugins=False)
# Finally, we can remove the Tag objects from Pulp entirely, since Tags are repository
# specific.
models.Tag.objects.filter(repo_id=repo.repo_id, manifest_digest=manifest.digest).delete()
def associate_units(self):
manager = managers.repo_unit_association_manager()
# RPMs
collection = database.type_units_collection(self.TYPE_ID)
for unit in collection.find():
manager.associate_unit_by_id(
self.REPO_ID,
self.TYPE_ID,
unit['_id'],
RepoContentUnit.OWNER_TYPE_IMPORTER,
'stuffed',
False
)
# ERRATA
collection = database.type_units_collection(self.ERRATA_TYPE_ID)
for unit in collection.find():
manager.associate_unit_by_id(
self.REPO_ID,
self.ERRATA_TYPE_ID,
unit['_id'],
RepoContentUnit.OWNER_TYPE_IMPORTER,
'stuffed',
False
)
def associate_unit(mongo_id, to_repo_id, type_id, mock_update_last):
manager = factory.repo_unit_association_manager()
manager.associate_unit_by_id(to_repo_id,
type_id,
mongo_id,
update_repo_metadata=True)
def associate_unit(mongo_id, to_repo_id, type_id, mock_update_last):
manager = factory.repo_unit_association_manager()
manager.associate_unit_by_id(to_repo_id, type_id, mongo_id, update_repo_metadata=True)
import copy
import logging
import pymongo.errors
from pulp.server.managers import factory
from pulp.server.managers.repo.cud import RepoContentUnit
from pulp.server.managers.repo.unit_association_query import UnitAssociationCriteria
from pulp_rpm.common import ids
_log = logging.getLogger('pulp')
# Initialize plugin loader api and other managers
factory.initialize()
ass_query_mgr = factory.repo_unit_association_query_manager()
ass_mgr = factory.repo_unit_association_manager()
content_mgr = factory.content_manager()
repo_mgr = factory.repo_manager()
def _get_repos():
"""
Lookups all the yum based repos in pulp.
@return a list of repoids
"""
repos = factory.repo_query_manager().find_with_importer_type(
"yum_importer")
if not repos:
_log.debug("No repos found to perform db migrate")
return []
repo_ids = [repo['id'] for repo in repos]
def associate_unit(mongo_id, to_repo_id, type_id):
manager = factory.repo_unit_association_manager()
manager.associate_unit_by_id(to_repo_id, type_id, mongo_id, 'importer',
'yum_importer', update_repo_metadata=True)
import logging
from pulp.plugins.types import database as types_db
from pulp.server.managers import factory
from pulp.server.managers.repo.cud import RepoContentUnit
from pulp.server.managers.repo.unit_association_query import UnitAssociationCriteria
import pymongo.errors
from pulp_rpm.common import ids
_log = logging.getLogger('pulp')
# Initialize plugin loader api and other managers
factory.initialize()
ass_query_mgr = factory.repo_unit_association_query_manager()
ass_mgr = factory.repo_unit_association_manager()
content_mgr = factory.content_manager()
repo_mgr = factory.repo_manager()
def _get_repos():
"""
Lookups all the yum based repos in pulp.
@return a list of repoids
"""
repos = factory.repo_query_manager().find_with_importer_type("yum_importer")
if not repos:
_log.debug("No repos found to perform db migrate")
return []
repo_ids = [repo['id'] for repo in repos]
return repo_ids