def get_units_by_type(self, repo_id, type_id, criteria=None, as_generator=False):
"""
Retrieves data describing units of the given type associated with the
given repository. Information on the associations themselves is also
provided.
The sort fields may be from either the association data OR the
unit fields. A mix of both is not supported. Multiple sort fields
are supported as long as they come from the same area.
If a sort is not provided, the units will be sorted ascending by each
value in the unit key for the given type.
:param repo_id: identifies the repository
:type repo_id: str
:param type_id: limits returned units to the given type
:type type_id: str
:param criteria: if specified will drive the query
:type criteria: UnitAssociationCriteria
:param as_generator: if true, return a generator; if false, a list
:type as_generator: bool
"""
# Get_units now defaults to batch behavior, so use a list of length 1 to
# specify the unit types and pass it through.
criteria = criteria or UnitAssociationCriteria()
# Just overwrite the type_ids if the user was dumb enough to provide
# them in this call.
criteria.type_ids = [type_id]
return self.get_units(repo_id, criteria, as_generator)
def post(self, request, dest_repo_id):
"""
Associate units matching the criteria into the given repository
:param request: WSGI request object
:type request: django.core.handlers.wsgi.WSGIRequest
:param dest_repo_id: id of the repository content will be copied to
:type dest_repo_id: str
:raises exceptions.MissingValue: if required param source_repo_id is not passed
:raises exceptions.InvalidValue: if source_repo_id is not found or if criteria
params cannot be parsed
:raises exceptions.OperationPostponed: dispatch a publish repo task
"""
model.Repository.objects.get_repo_or_missing_resource(dest_repo_id)
criteria_body = request.body_as_json.get('criteria', {})
overrides = request.body_as_json.get('override_config', None)
source_repo_id = request.body_as_json.get('source_repo_id', None)
if source_repo_id is None:
raise exceptions.MissingValue(['source_repo_id'])
# Catch MissingResource because this is body data, raise 400 rather than 404
try:
model.Repository.objects.get_repo_or_missing_resource(source_repo_id)
except exceptions.MissingResource:
raise exceptions.InvalidValue(['source_repo_id'])
try:
criteria = UnitAssociationCriteria.from_client_input(criteria_body)
except exceptions.InvalidValue, e:
invalid_criteria = exceptions.InvalidValue('criteria')
invalid_criteria.add_child_exception(e)
raise invalid_criteria
def unassociate_by_criteria(cls, repo_id, criteria, notify_plugins=True):
"""
Unassociate units that are matched by the given criteria.
:param repo_id: identifies the repo
:type repo_id: str
:param criteria:
:param notify_plugins: if true, relevant plugins will be informed of the removal
:type notify_plugins: bool
"""
criteria = UnitAssociationCriteria.from_dict(criteria)
repo = model.Repository.objects.get_repo_or_missing_resource(repo_id)
unassociate_units = load_associated_units(repo_id, criteria)
if len(unassociate_units) == 0:
return {}
# Convert the units into transfer units. This happens regardless of whether or not
# the plugin will be notified as it's used to generate the return result.
# If all source types have been converted to mongo, search via new style.
repo_unit_types = set(repo.content_unit_counts.keys())
if repo_unit_types.issubset(set(plugin_api.list_unit_models())):
transfer_units = list(cls._units_from_criteria(repo, criteria))
else:
transfer_units = None
if unassociate_units is not None:
transfer_units = list(create_transfer_units(unassociate_units))
if notify_plugins:
remove_from_importer(repo_id, transfer_units)
unit_map = {} # maps unit_type_id to a list of unit_ids
for unit in unassociate_units:
id_list = unit_map.setdefault(unit["unit_type_id"], [])
id_list.append(unit["unit_id"])
collection = RepoContentUnit.get_collection()
for unit_type_id, unit_ids in unit_map.items():
spec = {"repo_id": repo_id, "unit_type_id": unit_type_id, "unit_id": {"$in": unit_ids}}
collection.remove(spec)
unique_count = sum(
1
for unit_id in unit_ids
if not RepoUnitAssociationManager.association_exists(repo_id, unit_id, unit_type_id)
)
if not unique_count:
continue
repo_controller.update_unit_count(repo_id, unit_type_id, -unique_count)
repo_controller.update_last_unit_removed(repo_id)
# Match the return type/format as copy
serializable_units = [u.to_id_dict() for u in transfer_units]
return {"units_successful": serializable_units}
def _generate_response(cls, query, options, *args, **kwargs):
"""
Perform the database query using the given search data, and return the resuls as a JSON
serialized HttpReponse object.
This overrides the base class so we can validate repo existance and to choose the search
method depending on how many unit types we are dealing with.
:param query: The criteria that should be used to search for objects
:type query: dict
:param options: additional options for including extra data
:type options: dict
:return: The serialized search results in an HttpReponse
:rtype: django.http.HttpResponse
"""
repo_id = kwargs.get('repo_id')
model.Repository.objects.get_repo_or_missing_resource(repo_id)
criteria = UnitAssociationCriteria.from_client_input(query)
manager = manager_factory.repo_unit_association_query_manager()
if criteria.type_ids is not None and len(criteria.type_ids) == 1:
type_id = criteria.type_ids[0]
units = manager.get_units_by_type(repo_id, type_id, criteria=criteria)
else:
units = manager.get_units(repo_id, criteria=criteria)
return generate_json_response_with_pulp_encoder(units)
def unassociate_all_by_ids(self, repo_id, unit_type_id, unit_id_list, owner_type, owner_id,
notify_plugins=True):
"""
Removes the association between a repo and a number of units. Only the
association made by the given owner will be removed. It is possible the
repo will still have a manually created association will for the unit.
@param repo_id: identifies the repo
@type repo_id: str
@param unit_type_id: identifies the type of units being removed
@type unit_type_id: str
@param unit_id_list: list of unique identifiers for units within the given type
@type unit_id_list: list of str
@param owner_type: category of the caller who created the association;
must be one of the OWNER_* variables in this module
@type owner_type: str
@param owner_id: identifies the caller who created the association, either
the importer ID or user login
@type owner_id: str
@param notify_plugins: if true, relevant plugins will be informed of the
removal
@type notify_plugins: bool
"""
association_filters = {'unit_id' : {'$in' : unit_id_list}}
criteria = UnitAssociationCriteria(type_ids=[unit_type_id], association_filters=association_filters)
return self.unassociate_by_criteria(repo_id, criteria, owner_type, owner_id,
notify_plugins=notify_plugins)
def POST(self, repo_id):
# Params
params = self.params()
query = params.get('criteria', {})
options = params.get('options', {})
timeout = params.get('timeout', 60)
try:
criteria = UnitAssociationCriteria.from_client_input(query)
except:
_LOG.exception('Error parsing association criteria [%s]' % query)
raise exceptions.PulpDataException(), None, sys.exc_info()[2]
try:
timeout = int(timeout)
except ValueError:
raise exceptions.InvalidValue(['timeout']), None, sys.exc_info()[2]
# Coordinator configuration
resources = {dispatch_constants.RESOURCE_REPOSITORY_TYPE: {repo_id: dispatch_constants.RESOURCE_READ_OPERATION}}
tags = [resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE, repo_id),
action_tag('resolve_dependencies')]
dependency_manager = manager_factory.dependency_manager()
call_request = CallRequest(dependency_manager.resolve_dependencies_by_criteria,
[repo_id, criteria, options],
resources=resources, tags=tags, archive=True)
return execution.execute_sync_ok(self, call_request, timeout=timedelta(seconds=timeout))
def test_4(self):
"""
Scenario: Standard repository size but all associations are duplicated
(unlikely in the real world). Criteria set to remove duplicates.
Test Properties:
- 3000 units, single unit type
- 2 associations per unit
- Criteria removes duplicates
"""
if not GetUnitsStressTest.ENABLED:
return
# Setup
repo_id = 'repo-3'
for i in range(0, 3000):
unit_id = 'unit_%d' % i
metadata = {'key_1': unit_id}
for j in range(0, 10):
metadata['md_%d' % j] = 'value_%d' % i
self.content_manager.add_content_unit('alpha', unit_id, metadata)
self.association_manager.associate_unit_by_id(
repo_id, 'alpha', unit_id,
association_manager.OWNER_TYPE_IMPORTER, 'stress-importer')
self.association_manager.associate_unit_by_id(
repo_id, 'alpha', unit_id, association_manager.OWNER_TYPE_USER,
'admin')
# Test
criteria = UnitAssociationCriteria(remove_duplicates=True)
units = self.manager.get_units_across_types(repo_id, criteria)
self.assertEqual(3000, len(units))
def test_get_units_by_type_not_query(self):
"""
Mongo really doesn't like $not queries when regular expressions are
involved. This test is to make sure that across mongo and pymongo
versions a not expression against a regular expression continues to
work.
There is an important step in the parsing of the criteria from the
REST call into the Criteria object. This call will use that method
to more closely test the end to end experience.
"""
# Setup
# I got bit by the fact that incoming REST requests are in unicode;
# the criteria parsing didn't account for this. This example specifically
# replicates that by having the not and its value in unicode.
query_string = {'filters': {'unit': {'key_1': {u'$not': u'.*aa.*'}}}}
criteria = UnitAssociationCriteria.from_client_input(query_string)
# Test
units = self.manager.get_units_by_type('repo-1', 'alpha', criteria)
# Verify
self.assertEqual(len(self.units['alpha']) - 1, len(units))
for u in units:
self.assertTrue(u['metadata']['key_1'] != 'aardvark')
def test_get_units_limit(self):
# Test
low_criteria = UnitAssociationCriteria(limit=2)
low_units = self.manager.get_units_across_types('repo-1', low_criteria)
high_criteria = UnitAssociationCriteria(limit=10000)
high_units = self.manager.get_units_across_types(
'repo-1', high_criteria)
# Verify
self.assertEqual(2, len(low_units))
self.assertEqual(self.repo_1_count, len(high_units))
# Make sure the limit was applied to the front of the results
self.assertEqual(low_units[0], high_units[0])
self.assertEqual(low_units[1], high_units[1])
def test_unassociate_via_criteria(self, mock_call):
self.manager.associate_unit_by_id(self.repo_id, self.unit_type_id,
self.unit_id, OWNER_TYPE_USER,
'admin')
self.manager.associate_unit_by_id(self.repo_id, self.unit_type_id,
self.unit_id_2, OWNER_TYPE_USER,
'admin')
criteria_doc = {
'filters': {
'association': {
'unit_id': {
'$in': [self.unit_id, 'unit-X']
}
}
}
}
criteria = UnitAssociationCriteria.from_client_input(criteria_doc)
self.manager.unassociate_by_criteria(self.repo_id, criteria,
OWNER_TYPE_USER, 'admin')
self.assertFalse(
self.manager.association_exists(self.repo_id, self.unit_id,
self.unit_type_id))
self.assertTrue(
self.manager.association_exists(self.repo_id, self.unit_id_2,
self.unit_type_id))
mock_call.assert_called_once_with(self.repo_id)
def POST(self, dest_repo_id):
# Params
params = self.params()
source_repo_id = params.get('source_repo_id', None)
overrides = params.get('override_config', None)
if source_repo_id is None:
raise exceptions.MissingValue(['source_repo_id'])
criteria = params.get('criteria', None)
if criteria is not None:
try:
criteria = UnitAssociationCriteria.from_client_input(criteria)
except:
_LOG.exception('Error parsing association criteria [%s]' % criteria)
raise exceptions.PulpDataException(), None, sys.exc_info()[2]
association_manager = manager_factory.repo_unit_association_manager()
resources = {dispatch_constants.RESOURCE_REPOSITORY_TYPE: {source_repo_id: dispatch_constants.RESOURCE_READ_OPERATION,
dest_repo_id: dispatch_constants.RESOURCE_UPDATE_OPERATION}}
tags = [resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE, dest_repo_id),
resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE, source_repo_id),
action_tag('associate')]
call_request = CallRequest(association_manager.associate_from_repo,
[source_repo_id, dest_repo_id],
{'criteria': criteria, 'import_config_override': overrides},
resources=resources,
tags=tags,
archive=True)
return execution.execute_async(self, call_request)
def POST(self, repo_id):
# Params
params = self.params()
query = params.get('criteria', None)
repo_query_manager = manager_factory.repo_query_manager()
repo = repo_query_manager.find_by_id(repo_id)
if repo is None:
raise exceptions.MissingResource(repo_id=repo_id)
if query is None:
raise exceptions.MissingValue(['criteria'])
try:
criteria = UnitAssociationCriteria.from_client_input(query)
except:
_logger.error('Error parsing association criteria [%s]' % query)
raise exceptions.PulpDataException(), None, sys.exc_info()[2]
# Data lookup
manager = manager_factory.repo_unit_association_query_manager()
if criteria.type_ids is not None and len(criteria.type_ids) == 1:
type_id = criteria.type_ids[0]
units = manager.get_units_by_type(repo_id,
type_id,
criteria=criteria)
else:
units = manager.get_units_across_types(repo_id, criteria=criteria)
return self.ok(units)
def POST(self, repo_id):
params = self.params()
criteria = params.get('criteria', None)
if criteria is not None:
try:
criteria = UnitAssociationCriteria.from_client_input(criteria)
except:
_logger.error('Error parsing unassociation criteria [%s]' %
criteria)
raise exceptions.PulpDataException(), None, sys.exc_info()[2]
task_tags = [
tags.resource_tag(tags.RESOURCE_REPOSITORY_TYPE, repo_id),
tags.action_tag('unassociate')
]
async_result = unassociate_by_criteria.apply_async_with_reservation(
tags.RESOURCE_REPOSITORY_TYPE,
repo_id, [
repo_id, criteria, RepoContentUnit.OWNER_TYPE_USER,
manager_factory.principal_manager().get_principal()['login']
],
tags=task_tags)
raise exceptions.OperationPostponed(async_result)
def POST(self, repo_id):
params = self.params()
criteria = params.get('criteria', None)
if criteria is not None:
try:
criteria = UnitAssociationCriteria.from_client_input(criteria)
except:
_LOG.error('Error parsing unassociation criteria [%s]' %
criteria)
raise exceptions.PulpDataException(), None, sys.exc_info()[2]
association_manager = manager_factory.repo_unit_association_manager()
tags = [
resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE, repo_id),
action_tag('unassociate')
]
call_request = CallRequest(
association_manager.unassociate_by_criteria, [
repo_id, criteria, RepoContentUnit.OWNER_TYPE_USER,
manager_factory.principal_manager().get_principal()['login']
],
tags=tags,
archive=True)
call_request.updates_resource(
dispatch_constants.RESOURCE_REPOSITORY_TYPE, repo_id)
return execution.execute_async(self, call_request)
def POST(self, repo_id):
params = self.params()
criteria = params.get("criteria", None)
if criteria is not None:
try:
criteria = UnitAssociationCriteria.from_client_input(criteria)
except:
logger.error("Error parsing unassociation criteria [%s]" % criteria)
raise exceptions.PulpDataException(), None, sys.exc_info()[2]
tags = [resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE, repo_id), action_tag("unassociate")]
async_result = unassociate_by_criteria.apply_async_with_reservation(
dispatch_constants.RESOURCE_REPOSITORY_TYPE,
repo_id,
[
repo_id,
criteria,
RepoContentUnit.OWNER_TYPE_USER,
manager_factory.principal_manager().get_principal()["login"],
],
tags=tags,
)
raise exceptions.OperationPostponed(async_result)
def _generate_response(cls, query, options, *args, **kwargs):
"""
Perform the database query using the given search data, and return the resuls as a JSON
serialized HttpReponse object.
This overrides the base class so we can validate repo existance and to choose the search
method depending on how many unit types we are dealing with.
:param query: The criteria that should be used to search for objects
:type query: dict
:param options: additional options for including extra data
:type options: dict
:return: The serialized search results in an HttpReponse
:rtype: django.http.HttpResponse
"""
repo_id = kwargs.get('repo_id')
model.Repository.objects.get_repo_or_missing_resource(repo_id)
criteria = UnitAssociationCriteria.from_client_input(query)
manager = manager_factory.repo_unit_association_query_manager()
if criteria.type_ids is not None and len(criteria.type_ids) == 1:
type_id = criteria.type_ids[0]
units = manager.get_units_by_type(repo_id, type_id, criteria=criteria)
else:
units = manager.get_units(repo_id, criteria=criteria)
for unit in units:
content.remap_fields_with_serializer(unit['metadata'])
return generate_json_response_with_pulp_encoder(units)
def POST(self, repo_id):
# Params
params = self.params()
query = params.get('criteria', {})
options = params.get('options', {})
timeout = params.get('timeout', 60)
try:
criteria = UnitAssociationCriteria.from_client_input(query)
except:
_LOG.error('Error parsing association criteria [%s]' % query)
raise exceptions.PulpDataException(), None, sys.exc_info()[2]
try:
timeout = int(timeout)
except ValueError:
raise exceptions.InvalidValue(['timeout']), None, sys.exc_info()[2]
# Coordinator configuration
tags = [
resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE, repo_id),
action_tag('resolve_dependencies')
]
dependency_manager = manager_factory.dependency_manager()
call_request = CallRequest(
dependency_manager.resolve_dependencies_by_criteria,
[repo_id, criteria, options],
tags=tags,
archive=True)
call_request.reads_resource(
dispatch_constants.RESOURCE_REPOSITORY_TYPE, repo_id)
return execution.execute_sync_ok(self,
call_request,
timeout=timedelta(seconds=timeout))
def POST(self, repo_id):
# Params
params = self.params()
query = params.get('criteria', None)
repo_query_manager = manager_factory.repo_query_manager()
repo = repo_query_manager.find_by_id(repo_id)
if repo is None:
raise exceptions.MissingResource(repo_id=repo_id)
if query is None:
raise exceptions.MissingValue(['criteria'])
try:
criteria = UnitAssociationCriteria.from_client_input(query)
except:
_LOG.exception('Error parsing association criteria [%s]' % query)
raise exceptions.PulpDataException(), None, sys.exc_info()[2]
# Data lookup
manager = manager_factory.repo_unit_association_query_manager()
if criteria.type_ids is not None and len(criteria.type_ids) == 1:
type_id = criteria.type_ids[0]
units = manager.get_units_by_type(repo_id, type_id, criteria=criteria)
else:
units = manager.get_units_across_types(repo_id, criteria=criteria)
return self.ok(units)
def test_unassociate_via_criteria(self, mock_ctrl, mock_repo):
self.manager.associate_unit_by_id(self.repo_id, self.unit_type_id,
self.unit_id)
self.manager.associate_unit_by_id(self.repo_id, self.unit_type_id,
self.unit_id_2)
criteria_doc = {
'filters': {
'association': {
'unit_id': {
'$in': [self.unit_id, 'unit-X']
}
}
}
}
criteria = UnitAssociationCriteria.from_client_input(criteria_doc)
self.manager.unassociate_by_criteria(self.repo_id, criteria)
self.assertFalse(
self.manager.association_exists(self.repo_id, self.unit_id,
self.unit_type_id))
self.assertTrue(
self.manager.association_exists(self.repo_id, self.unit_id_2,
self.unit_type_id))
mock_repo.objects.get_repo_or_missing_resource.assert_called_once_with(
self.repo_id)
def unassociate_by_criteria(repo_id, criteria, notify_plugins=True):
"""
Unassociate units that are matched by the given criteria.
:param repo_id: identifies the repo
:type repo_id: str
:param criteria:
:param notify_plugins: if true, relevant plugins will be informed of the removal
:type notify_plugins: bool
"""
criteria = UnitAssociationCriteria.from_dict(criteria)
association_query_manager = manager_factory.repo_unit_association_query_manager(
)
unassociate_units = association_query_manager.get_units(
repo_id, criteria=criteria)
if len(unassociate_units) == 0:
return {}
unit_map = {} # maps unit_type_id to a list of unit_ids
for unit in unassociate_units:
id_list = unit_map.setdefault(unit['unit_type_id'], [])
id_list.append(unit['unit_id'])
collection = RepoContentUnit.get_collection()
for unit_type_id, unit_ids in unit_map.items():
spec = {
'repo_id': repo_id,
'unit_type_id': unit_type_id,
'unit_id': {
'$in': unit_ids
}
}
collection.remove(spec)
unique_count = sum(
1 for unit_id in unit_ids
if not RepoUnitAssociationManager.association_exists(
repo_id, unit_id, unit_type_id))
if not unique_count:
continue
repo_controller.update_unit_count(repo_id, unit_type_id,
-unique_count)
repo_controller.update_last_unit_removed(repo_id)
# Convert the units into transfer units. This happens regardless of whether or not
# the plugin will be notified as it's used to generate the return result,
transfer_units = create_transfer_units(unassociate_units)
if notify_plugins:
remove_from_importer(repo_id, transfer_units)
# Match the return type/format as copy
serializable_units = [u.to_id_dict() for u in transfer_units]
return {'units_successful': serializable_units}
def post(self, request, dest_repo_id):
"""
Associate units matching the criteria into the given repository
:param request: WSGI request object
:type request: django.core.handlers.wsgi.WSGIRequest
:param dest_repo_id: id of the repository content will be copied to
:type dest_repo_id: str
:raises exceptions.MissingValue: if required param source_repo_id is not passed
:raises exceptions.InvalidValue: if source_repo_id is not found or if criteria
params cannot be parsed
:raises exceptions.OperationPostponed: dispatch a publish repo task
"""
model.Repository.objects.get_repo_or_missing_resource(dest_repo_id)
criteria_body = request.body_as_json.get('criteria', {})
overrides = request.body_as_json.get('override_config', None)
source_repo_id = request.body_as_json.get('source_repo_id', None)
if source_repo_id is None:
raise exceptions.MissingValue(['source_repo_id'])
# Catch MissingResource because this is body data, raise 400 rather than 404
try:
model.Repository.objects.get_repo_or_missing_resource(source_repo_id)
except exceptions.MissingResource:
raise exceptions.InvalidValue(['source_repo_id'])
try:
criteria = UnitAssociationCriteria.from_client_input(criteria_body)
except exceptions.InvalidValue, e:
invalid_criteria = exceptions.InvalidValue('criteria')
invalid_criteria.add_child_exception(e)
raise invalid_criteria
def test_get_units_filter_created(self):
# Test
after_criteria = UnitAssociationCriteria(association_filters={'created' : {'$gt' : self.timestamps[0]}})
after_units = self.manager.get_units_across_types('repo-1', after_criteria)
before_criteria = UnitAssociationCriteria(association_filters={'created' : {'$lt' : self.timestamps[1]}})
before_units = self.manager.get_units_across_types('repo-1', before_criteria)
# Verify
# The first association in each type/owner combination will be timestamps[0],
# the second timestamps[1]. There are 4 such type/owner combinations,
# however the user associations in gamma have timestamp offsets of i+1.
self.assertEqual(self.repo_1_count - 3, len(after_units))
self.assertEqual(3, len(before_units))
def test_resolve_dependencies_by_criteria(self):
# Setup
report = 'dep report'
mock_plugins.MOCK_IMPORTER.resolve_dependencies.return_value = report
unit_id_1 = manager_factory.content_manager().add_content_unit(
'type-1', None, {'key-1': 'unit-id-1'})
unit_id_2 = manager_factory.content_manager().add_content_unit(
'type-1', None, {'key-1': 'dep-1'})
association_manager = manager_factory.repo_unit_association_manager()
association_manager.associate_unit_by_id(self.repo_id, 'type-1',
unit_id_1)
association_manager.associate_unit_by_id(self.repo_id, 'type-1',
unit_id_2)
criteria = UnitAssociationCriteria(type_ids=['type-1'],
unit_filters={'key-1': 'unit-id-1'})
# Test
result = self.manager.resolve_dependencies_by_criteria(
self.repo_id, criteria, {})
# Verify
self.assertEqual(report, result)
self.assertEqual(
1, mock_plugins.MOCK_IMPORTER.resolve_dependencies.call_count)
args = mock_plugins.MOCK_IMPORTER.resolve_dependencies.call_args[0]
self.assertEqual(1, len(args[1]))
def _purge_unlinked_blobs(repo, manifest):
"""
Purge blobs associated with the given Manifests when removing it would leave them no longer
referenced by any remaining Manifests.
:param repo: The affected repository.
:type repo: pulp.server.db.model.Repository
:param units: List of removed units.
:type units: list of: pulp.plugins.model.AssociatedUnit
"""
# Find blob digests referenced by removed manifests (orphaned)
orphaned = set()
map((lambda layer: orphaned.add(layer.blob_sum)), manifest.fs_layers)
# in manifest schema version 2 there is an additional blob layer called config_layer
if manifest.config_layer:
orphaned.add(manifest.config_layer)
if not orphaned:
# nothing orphaned
return
# Find blob digests still referenced by other manifests (adopted)
adopted = set()
criteria = UnitAssociationCriteria(
type_ids=[constants.MANIFEST_TYPE_ID],
unit_filters={'digest': {
'$ne': manifest.digest
}})
for manifest in unit_association.RepoUnitAssociationManager._units_from_criteria(
repo, criteria):
map((lambda layer: adopted.add(layer.blob_sum)),
manifest.fs_layers)
if manifest.config_layer:
adopted.add(manifest.config_layer)
# Remove unreferenced blobs
orphaned = orphaned.difference(adopted)
if not orphaned:
# all adopted
return
unit_filter = {'digest': {'$in': sorted(orphaned)}}
criteria = UnitAssociationCriteria(type_ids=[constants.BLOB_TYPE_ID],
unit_filters=unit_filter)
manager = manager_factory.repo_unit_association_manager()
manager.unassociate_by_criteria(repo_id=repo.repo_id,
criteria=criteria,
notify_plugins=False)
def get_repo_units(self,
repo_id,
content_type_id,
additional_unit_fields=None):
"""
Searches for units in the given repository with given content type
and returns a plugin unit containing unit id, unit key and any additional
fields requested.
:param repo_id: repo id
:type repo_id: str
:param content_type_id: content type id of the units
:type content_type_id: str
:param additional_unit_fields: additional fields from the unit metadata to be added
in the result
:type additional_unit_fields: list of str
:return: list of unit instances
:rtype: list of pulp.plugins.model.Unit
"""
additional_unit_fields = additional_unit_fields or []
try:
unit_key_fields = units_controller.get_unit_key_fields_for_type(
content_type_id)
# Query repo association manager to get all units of given type
# associated with given repo. Limit data by requesting only the fields
# that are needed.
query_manager = managers.repo_unit_association_query_manager()
unit_fields = list(unit_key_fields) + list(additional_unit_fields)
criteria = UnitAssociationCriteria(association_fields=['unit_id'],
unit_fields=unit_fields)
units = query_manager.get_units_by_type(repo_id, content_type_id,
criteria)
# Convert units to plugin units with unit_key and required metadata values for each unit
all_units = []
for unit in units:
unit_key = {}
metadata = {}
for k in unit_key_fields:
unit_key[k] = unit['metadata'].pop(k)
# Add unit_id and any additional unit fields requested by plugins
metadata['unit_id'] = unit.pop('unit_id')
for field in additional_unit_fields:
metadata[field] = unit['metadata'].pop(field, None)
u = Unit(content_type_id, unit_key, metadata, None)
all_units.append(u)
return all_units
except Exception, e:
_logger.exception(
_('Exception from server getting units from repo [%s]' %
repo_id))
raise self.exception_class(e), None, sys.exc_info()[2]
def import_units(self,
source_repo,
dest_repo,
import_conduit,
config,
units=None):
"""
Import content units into the given repository. This method will be
called in a number of different situations:
* A user is attempting to copy a content unit from one repository
into the repository that uses this importer
* A user is attempting to add an orphaned unit into a repository.
This call has two options for handling the requested units:
* Associate the given units with the destination repository. This will
link the repository with the existing unit directly; changes to the
unit will be reflected in all repositories that reference it.
* Create a new unit and save it to the repository. This would act as
a deep copy of sorts, creating a unique unit in the database. Keep
in mind that the unit key must change in order for the unit to
be considered different than the supplied one.
The APIs for both approaches are similar to those in the sync conduit.
In the case of a simple association, the init_unit step can be skipped
and save_unit simply called on each specified unit.
The units argument is optional. If None, all units in the source
repository should be imported. The conduit is used to query for those
units. If specified, only the units indicated should be imported (this
is the case where the caller passed a filter to Pulp).
:param source_repo: metadata describing the repository containing the
units to import
:type source_repo: pulp.plugins.model.Repository
:param dest_repo: metadata describing the repository to import units
into
:type dest_repo: pulp.plugins.model.Repository
:param import_conduit: provides access to relevant Pulp functionality
:type import_conduit: pulp.plugins.conduits.unit_import.ImportUnitConduit
:param config: plugin configuration
:type config: pulp.plugins.config.PluginCallConfiguration
:param units: optional list of pre-filtered units to import
:type units: list of pulp.plugins.model.Unit
:return: list of Unit instances that were saved to the destination repository
:rtype: list
"""
if units is None:
criteria = UnitAssociationCriteria(type_ids=[ids.TYPE_ID_ISO])
units = import_conduit.get_source_units(criteria=criteria)
for u in units:
import_conduit.associate_unit(u)
return units
def unassociate_by_criteria(repo_id, criteria, notify_plugins=True):
"""
Unassociate units that are matched by the given criteria.
:param repo_id: identifies the repo
:type repo_id: str
:param criteria:
:param notify_plugins: if true, relevant plugins will be informed of the removal
:type notify_plugins: bool
"""
criteria = UnitAssociationCriteria.from_dict(criteria)
repo = model.Repository.objects.get_repo_or_missing_resource(repo_id)
unassociate_units = load_associated_units(repo_id, criteria)
if len(unassociate_units) == 0:
return {}
# Convert the units into transfer units. This happens regardless of whether or not
# the plugin will be notified as it's used to generate the return result.
# If all source types have been converted to mongo, search via new style.
repo_unit_types = set(repo.content_unit_counts.keys())
if repo_unit_types.issubset(set(plugin_api.list_unit_models())):
transfer_units = list(
RepoUnitAssociationManager._units_from_criteria(
repo, criteria))
else:
transfer_units = None
if unassociate_units is not None:
transfer_units = list(create_transfer_units(unassociate_units))
if notify_plugins:
remove_from_importer(repo_id, transfer_units)
unit_map = {} # maps unit_type_id to a list of unit_ids
for unit in unassociate_units:
id_list = unit_map.setdefault(unit['unit_type_id'], [])
id_list.append(unit['unit_id'])
collection = RepoContentUnit.get_collection()
for unit_type_id, unit_ids in unit_map.items():
spec = {
'repo_id': repo_id,
'unit_type_id': unit_type_id,
'unit_id': {
'$in': unit_ids
}
}
collection.remove(spec)
repo_controller.update_last_unit_removed(repo_id)
repo_controller.rebuild_content_unit_counts(repo)
# Match the return type/format as copy
serializable_units = [u.to_id_dict() for u in transfer_units]
return {'units_successful': serializable_units}
def test_get_units_by_type_unit_metadata_sort_skip(self):
# Test
criteria = UnitAssociationCriteria(unit_sort=[('md_2', association_manager.SORT_DESCENDING)], skip=1)
units = self.manager.get_units_by_type('repo-1', 'alpha', criteria)
# Verify
expected_count = len(self.units['alpha']) - 1 # skip the first
self.assertEqual(expected_count, len(units))
def test_get_units_by_type_filter_wildcard(self):
# Test
criteria = UnitAssociationCriteria(unit_filters={'key_1' : {'$regex' : 'aa.*'}})
units = self.manager.get_units_by_type('repo-1', 'alpha', criteria)
# Verify
self.assertEqual(1, len(units))
self.assertEqual('aardvark', units[0]['metadata']['key_1'])
def test_get_destination_units(self, mock_get):
# Test
criteria = UnitAssociationCriteria()
self.conduit.get_destination_units(criteria=criteria)
# Verify the correct propagation to the mixin method
mock_get.assert_called_once_with(self.dest_repo_id, criteria,
ImporterConduitException)
def test_get_units_by_type_remove_duplicates(self):
# Test
criteria = UnitAssociationCriteria(remove_duplicates=True)
units = self.manager.get_units_by_type('repo-1', 'gamma', criteria)
# only one association per gamma unit
self.assertEqual(len(self.units['gamma']), len(units))
self.assertEqual(units[0]['unit_id'], 'garden')
def test_get_units_filter_owner_type(self):
# Test
criteria = UnitAssociationCriteria(association_filters={'owner_type' : OWNER_TYPE_IMPORTER})
units = self.manager.get_units_across_types('repo-1', criteria)
# Verify
expected_count = reduce(lambda x, y: x + len(self.units[y]), ['alpha', 'gamma'], 0)
self.assertEqual(expected_count, len(units))
def test_criteria_str(self):
# Setup
c1 = UnitAssociationCriteria()
c2 = UnitAssociationCriteria(type_ids=['a'],
association_filters={'a': 'a'},
unit_filters={'b': 'b'},
association_sort=['c'],
unit_sort=['d'],
limit=1,
skip=2,
association_fields=['e'],
unit_fields=['f'],
remove_duplicates=True)
# Test no exceptions are raised
str(c1)
str(c2)
def _do_import_modules(self, metadata):
"""
Actual logic of the import. This method will do a best effort per module;
if an individual module fails it will be recorded and the import will
continue. This method will only raise an exception in an extreme case
where it cannot react and continue.
"""
def unit_key_str(unit_key_dict):
"""
Converts the unit key dict form into a single string that can be
used as the key in a dict lookup.
"""
template = '%s-%s-%s'
return template % (encode_unicode(unit_key_dict['name']),
encode_unicode(unit_key_dict['version']),
encode_unicode(unit_key_dict['author']))
downloader = self._create_downloader()
self.downloader = downloader
# Ease lookup of modules
modules_by_key = dict([(unit_key_str(m.unit_key()), m)
for m in metadata.modules])
# Collect information about the repository's modules before changing it
module_criteria = UnitAssociationCriteria(
type_ids=[constants.TYPE_PUPPET_MODULE])
existing_units = self.sync_conduit.get_units(criteria=module_criteria)
existing_modules = [Module.from_unit(x) for x in existing_units]
existing_module_keys = [
unit_key_str(m.unit_key()) for m in existing_modules
]
new_unit_keys = self._resolve_new_units(existing_module_keys,
modules_by_key.keys())
remove_unit_keys = self._resolve_remove_units(existing_module_keys,
modules_by_key.keys())
# Once we know how many things need to be processed, we can update the
# progress report
self.progress_report.modules_total_count = len(new_unit_keys)
self.progress_report.modules_finished_count = 0
self.progress_report.modules_error_count = 0
self.progress_report.update_progress()
# Add new units
for key in new_unit_keys:
if self._canceled:
break
module = modules_by_key[key]
try:
self._add_new_module(downloader, module)
self.progress_report.modules_finished_count += 1
except Exception, e:
self.progress_report.add_failed_module(module, e,
sys.exc_info()[2])
self.progress_report.update_progress()
def test_get_units_by_type_remove_duplicates(self):
# Test
criteria = UnitAssociationCriteria(remove_duplicates=True)
units = self.manager.get_units_by_type('repo-1', 'gamma', criteria)
# Verify
self.assertEqual(len(self.units['gamma']), len(units)) # only one association per gamma unit
for u in units:
self.assertEqual(u['owner_type'], association_manager.OWNER_TYPE_USER) # all user associations have earlier created date
def test_get_units_by_type_association_filter(self):
# Test
criteria = UnitAssociationCriteria(
association_filters={'updated': self.timestamps[1]})
units = self.manager.get_units_by_type('repo-1', 'gamma', criteria)
# Verify
self.assertEqual(1, len(units))
def test_get_units_with_fields(self):
# Test
criteria = UnitAssociationCriteria(association_fields=['created'])
units = self.manager.get_units_across_types('repo-1', criteria)
# Verify
for u in units:
self.assertTrue('created' in u)
self.assertFalse('updated' in u)
def unassociate_by_criteria(repo_id, criteria, notify_plugins=True):
"""
Unassociate units that are matched by the given criteria.
:param repo_id: identifies the repo
:type repo_id: str
:param criteria:
:param notify_plugins: if true, relevant plugins will be informed of the removal
:type notify_plugins: bool
"""
criteria = UnitAssociationCriteria.from_dict(criteria)
association_query_manager = manager_factory.repo_unit_association_query_manager()
unassociate_units = association_query_manager.get_units(repo_id, criteria=criteria)
if len(unassociate_units) == 0:
return {}
unit_map = {} # maps unit_type_id to a list of unit_ids
for unit in unassociate_units:
id_list = unit_map.setdefault(unit['unit_type_id'], [])
id_list.append(unit['unit_id'])
collection = RepoContentUnit.get_collection()
for unit_type_id, unit_ids in unit_map.items():
spec = {'repo_id': repo_id,
'unit_type_id': unit_type_id,
'unit_id': {'$in': unit_ids}
}
collection.remove(spec)
unique_count = sum(
1 for unit_id in unit_ids if not RepoUnitAssociationManager.association_exists(
repo_id, unit_id, unit_type_id))
if not unique_count:
continue
repo_controller.update_unit_count(repo_id, unit_type_id, -unique_count)
repo_controller.update_last_unit_removed(repo_id)
# Convert the units into transfer units. This happens regardless of whether or not
# the plugin will be notified as it's used to generate the return result,
transfer_units = create_transfer_units(unassociate_units)
if notify_plugins:
remove_from_importer(repo_id, transfer_units)
# Match the return type/format as copy
serializable_units = [u.to_id_dict() for u in transfer_units]
return {'units_successful': serializable_units}
def test_unassociate_via_criteria(self):
self.manager.associate_unit_by_id(self.repo_id, self.unit_type_id, self.unit_id, OWNER_TYPE_USER, 'admin')
self.manager.associate_unit_by_id(self.repo_id, self.unit_type_id, self.unit_id_2, OWNER_TYPE_USER, 'admin')
criteria_doc = {'filters': {'association': {'unit_id': {'$in': [self.unit_id, 'unit-X']}}}}
criteria = UnitAssociationCriteria.from_client_input(criteria_doc)
self.manager.unassociate_by_criteria(self.repo_id, criteria, OWNER_TYPE_USER, 'admin')
self.assertFalse(self.manager.association_exists(self.repo_id, self.unit_id, self.unit_type_id))
self.assertTrue(self.manager.association_exists(self.repo_id, self.unit_id_2, self.unit_type_id))
def test_unassociate_via_criteria_no_matches(self):
self.manager.associate_unit_by_id(self.repo_id, "type-1", "unit-1", OWNER_TYPE_USER, "admin")
self.manager.associate_unit_by_id(self.repo_id, "type-1", "unit-2", OWNER_TYPE_USER, "admin")
criteria_doc = {"type_ids": ["type-2"]}
criteria = UnitAssociationCriteria.from_client_input(criteria_doc)
self.manager.unassociate_by_criteria(self.repo_id, criteria, OWNER_TYPE_USER, "admin")
self.assertTrue(self.manager.association_exists(self.repo_id, "unit-1", "type-1"))
self.assertTrue(self.manager.association_exists(self.repo_id, "unit-2", "type-1"))
def test_unassociate_via_criteria(self):
self.manager.associate_unit_by_id(self.repo_id, self.unit_type_id, self.unit_id, OWNER_TYPE_USER, "admin")
self.manager.associate_unit_by_id(self.repo_id, self.unit_type_id, self.unit_id_2, OWNER_TYPE_USER, "admin")
criteria_doc = {"filters": {"association": {"unit_id": {"$in": [self.unit_id, "unit-X"]}}}}
criteria = UnitAssociationCriteria.from_client_input(criteria_doc)
self.manager.unassociate_by_criteria(self.repo_id, criteria, OWNER_TYPE_USER, "admin")
self.assertFalse(self.manager.association_exists(self.repo_id, self.unit_id, self.unit_type_id))
self.assertTrue(self.manager.association_exists(self.repo_id, self.unit_id_2, self.unit_type_id))
def test_unassociate_via_criteria_no_matches(self):
self.manager.associate_unit_by_id('repo-1', 'type-1', 'unit-1', OWNER_TYPE_USER, 'admin')
self.manager.associate_unit_by_id('repo-1', 'type-1', 'unit-2', OWNER_TYPE_USER, 'admin')
criteria_doc = {'type_ids': ['type-2']}
criteria = UnitAssociationCriteria.from_client_input(criteria_doc)
self.manager.unassociate_by_criteria('repo-1', criteria, OWNER_TYPE_USER, 'admin')
self.assertTrue(self.manager.association_exists('repo-1', 'unit-1', 'type-1'))
self.assertTrue(self.manager.association_exists('repo-1', 'unit-2', 'type-1'))
def test_unassociate_via_criteria_no_matches(self, mock_ctrl):
self.manager.associate_unit_by_id(self.repo_id, 'type-1', 'unit-1')
self.manager.associate_unit_by_id(self.repo_id, 'type-1', 'unit-2')
criteria_doc = {'type_ids': ['type-2']}
criteria = UnitAssociationCriteria.from_client_input(criteria_doc)
result = self.manager.unassociate_by_criteria(self.repo_id, criteria)
self.assertEquals(result, {})
self.assertTrue(self.manager.association_exists(self.repo_id, 'unit-1', 'type-1'))
self.assertTrue(self.manager.association_exists(self.repo_id, 'unit-2', 'type-1'))
def test_unassociate_via_criteria(self, mock_repo_qs, mock_ctrl):
self.manager.associate_unit_by_id(self.repo_id, self.unit_type_id, self.unit_id)
self.manager.associate_unit_by_id(self.repo_id, self.unit_type_id, self.unit_id_2)
criteria_doc = {'filters': {'association': {'unit_id': {'$in': [self.unit_id, 'unit-X']}}}}
criteria = UnitAssociationCriteria.from_client_input(criteria_doc)
self.manager.unassociate_by_criteria(self.repo_id, criteria)
self.assertFalse(self.manager.association_exists(self.repo_id, self.unit_id,
self.unit_type_id))
self.assertTrue(self.manager.association_exists(self.repo_id, self.unit_id_2,
self.unit_type_id))
mock_repo_qs.get_repo_or_missing_resource.assert_called_once_with(self.repo_id)
def test_unassociate_via_criteria(self):
self.manager.associate_unit_by_id('repo-1', 'type-1', 'unit-1', OWNER_TYPE_USER, 'admin')
self.manager.associate_unit_by_id('repo-1', 'type-1', 'unit-2', OWNER_TYPE_USER, 'admin')
self.manager.associate_unit_by_id('repo-1', 'type-1', 'unit-3', OWNER_TYPE_USER, 'admin')
self.manager.associate_unit_by_id('repo-1', 'type-2', 'unit-1', OWNER_TYPE_IMPORTER, 'yum')
self.manager.associate_unit_by_id('repo-1', 'type-2', 'unit-2', OWNER_TYPE_IMPORTER, 'yum')
criteria_doc = {'filters': {'association': {'unit_id': {'$in': ['unit-1', 'unit-3']}}}}
criteria = UnitAssociationCriteria.from_client_input(criteria_doc)
self.manager.unassociate_by_criteria('repo-1', criteria, OWNER_TYPE_USER, 'admin')
self.assertFalse(self.manager.association_exists('repo-1', 'unit-1', 'type-1'))
self.assertTrue(self.manager.association_exists('repo-1', 'unit-2', 'type-1'))
self.assertFalse(self.manager.association_exists('repo-1', 'unit-3', 'type-1'))
self.assertTrue(self.manager.association_exists('repo-1', 'unit-1', 'type-2'))
self.assertTrue(self.manager.association_exists('repo-1', 'unit-2', 'type-2'))
def POST(self, repo_id):
params = self.params()
criteria = params.get('criteria', None)
if criteria is not None:
try:
criteria = UnitAssociationCriteria.from_client_input(criteria)
except:
_logger.error('Error parsing unassociation criteria [%s]' % criteria)
raise exceptions.PulpDataException(), None, sys.exc_info()[2]
task_tags = [tags.resource_tag(tags.RESOURCE_REPOSITORY_TYPE, repo_id),
tags.action_tag('unassociate')]
async_result = unassociate_by_criteria.apply_async_with_reservation(
tags.RESOURCE_REPOSITORY_TYPE, repo_id,
[repo_id, criteria, RepoContentUnit.OWNER_TYPE_USER,
manager_factory.principal_manager().get_principal()['login']], tags=task_tags)
raise exceptions.OperationPostponed(async_result)
def POST(self, dest_repo_id):
# Params
params = self.params()
source_repo_id = params.get("source_repo_id", None)
overrides = params.get("override_config", None)
if source_repo_id is None:
raise exceptions.MissingValue(["source_repo_id"])
# A 404 only applies to things in the URL, so the destination repo
# check allows the MissingResource to bubble up, but if the source
# repo doesn't exist, it's considered bad data.
repo_query_manager = manager_factory.repo_query_manager()
repo_query_manager.get_repository(dest_repo_id)
try:
repo_query_manager.get_repository(source_repo_id)
except exceptions.MissingResource:
raise exceptions.InvalidValue(["source_repo_id"])
criteria = params.get("criteria", None)
if criteria is not None:
try:
criteria = UnitAssociationCriteria.from_client_input(criteria)
except:
logger.error("Error parsing association criteria [%s]" % criteria)
raise exceptions.PulpDataException(), None, sys.exc_info()[2]
tags = [
resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE, dest_repo_id),
resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE, source_repo_id),
action_tag("associate"),
]
async_result = associate_from_repo.apply_async_with_reservation(
dispatch_constants.RESOURCE_REPOSITORY_TYPE,
dest_repo_id,
[source_repo_id, dest_repo_id],
{"criteria": criteria, "import_config_override": overrides},
tags=tags,
)
raise exceptions.OperationPostponed(async_result)
def test_parse_criteria(self):
# Setup
query = {
'type_ids': ['rpm'],
'filters': {
'unit': {'$and': [
{'$regex': '^p.*'},
{'$not': 'ython$'},
]},
'association': {'created': {'$gt': 'now'}},
},
'limit': 100,
'skip': 200,
'fields': {
'unit': ['name', 'version'],
'association': ['created'],
},
'remove_duplicates': True,
}
# Test
criteria = UnitAssociationCriteria.from_client_input(query)
# Verify
self.assertEqual(criteria.type_ids, ['rpm'])
self.assertEqual(criteria.association_filters, {'created': {'$gt': 'now'}})
self.assertEqual(criteria.limit, 100)
self.assertEqual(criteria.skip, 200)
self.assertEqual(criteria.unit_fields, ['name', 'version'])
self.assertEqual(criteria.association_fields, ['created', 'unit_id', 'unit_type_id'])
self.assertEqual(criteria.remove_duplicates, True)
# Check the special $not handling in the unit filter
self.assertTrue('$and' in criteria.unit_filters)
and_list = criteria.unit_filters['$and']
self.assertTrue('$regex' in and_list[0])
self.assertEqual(and_list[0]['$regex'], '^p.*')
self.assertTrue('$not' in and_list[1])
self.assertEqual(and_list[1]['$not'], re.compile('ython$'))
def POST(self, dest_repo_id):
# Params
params = self.params()
source_repo_id = params.get('source_repo_id', None)
overrides = params.get('override_config', None)
if source_repo_id is None:
raise exceptions.MissingValue(['source_repo_id'])
# A 404 only applies to things in the URL, so the destination repo
# check allows the MissingResource to bubble up, but if the source
# repo doesn't exist, it's considered bad data.
repo_query_manager = manager_factory.repo_query_manager()
repo_query_manager.get_repository(dest_repo_id)
try:
repo_query_manager.get_repository(source_repo_id)
except exceptions.MissingResource:
raise exceptions.InvalidValue(['source_repo_id'])
criteria = params.get('criteria', None)
if criteria is not None:
try:
criteria = UnitAssociationCriteria.from_client_input(criteria)
except:
_LOG.error('Error parsing association criteria [%s]' % criteria)
raise exceptions.PulpDataException(), None, sys.exc_info()[2]
association_manager = manager_factory.repo_unit_association_manager()
resources = {dispatch_constants.RESOURCE_REPOSITORY_TYPE: {source_repo_id: dispatch_constants.RESOURCE_READ_OPERATION,
dest_repo_id: dispatch_constants.RESOURCE_UPDATE_OPERATION}}
tags = [resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE, dest_repo_id),
resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE, source_repo_id),
action_tag('associate')]
call_request = CallRequest(association_manager.associate_from_repo,
[source_repo_id, dest_repo_id],
{'criteria': criteria, 'import_config_override': overrides},
resources=resources,
tags=tags,
archive=True)
return execution.execute_async(self, call_request)
def post(self, request, repo_id):
"""
Unassociate units that match the criteria from the given repository.
:param request: WSGI request object
:type request: django.core.handlers.wsgi.WSGIRequest
:param repo_id: id of the repository to unassociate content from
:type repo_id: str
:raises exceptions.InvalidValue: if criteria params cannot be parsed
:raises exceptions.OperationPostponed: dispatch a unassociate_by_criteria task
"""
criteria_body = request.body_as_json.get('criteria', {})
try:
criteria = UnitAssociationCriteria.from_client_input(criteria_body)
except exceptions.InvalidValue, e:
invalid_criteria = exceptions.InvalidValue('criteria')
invalid_criteria.add_child_exception(e)
raise invalid_criteria
def POST(self, repo_id):
# Params
params = self.params()
query = params.get('criteria', {})
options = params.get('options', {})
timeout = params.get('timeout', 60)
try:
criteria = UnitAssociationCriteria.from_client_input(query)
except:
logger.error('Error parsing association criteria [%s]' % query)
raise exceptions.PulpDataException(), None, sys.exc_info()[2]
try:
timeout = int(timeout)
except ValueError:
raise exceptions.InvalidValue(['timeout']), None, sys.exc_info()[2]
dependency_manager = manager_factory.dependency_manager()
result = dependency_manager.resolve_dependencies_by_criteria(repo_id, criteria, options)
return self.ok(result)
def POST(self, repo_id):
params = self.params()
criteria = params.get('criteria', None)
if criteria is not None:
try:
criteria = UnitAssociationCriteria.from_client_input(criteria)
except:
_LOG.exception('Error parsing unassociation criteria [%s]' % criteria)
raise exceptions.PulpDataException(), None, sys.exc_info()[2]
association_manager = manager_factory.repo_unit_association_manager()
tags = [resource_tag(dispatch_constants.RESOURCE_REPOSITORY_TYPE, repo_id),
action_tag('unassociate')]
call_request = CallRequest(association_manager.unassociate_by_criteria,
[repo_id, criteria, RepoContentUnit.OWNER_TYPE_USER, manager_factory.principal_manager().get_principal()['login']],
tags=tags,
archive=True)
call_request.updates_resource(dispatch_constants.RESOURCE_REPOSITORY_TYPE, repo_id)
return execution.execute_async(self, call_request)
def test_get_units_by_type_not_query(self):
"""
Mongo really doesn't like $not queries when regular expressions are
involved. This test is to make sure that across mongo and pymongo
versions a not expression against a regular expression continues to
work.
There is an important step in the parsing of the criteria from the
REST call into the Criteria object. This call will use that method
to more closely test the end to end experience.
"""
# Setup
# I got bit by the fact that incoming REST requests are in unicode;
# the criteria parsing didn't account for this. This example specifically
# replicates that by having the not and its value in unicode.
query_string = {
'filters' : {
'unit' : {
'key_1' : {
u'$not' : u'.*aa.*'
}
}
}
}
#criteria = unit_association_criteria(query_string)
criteria = UnitAssociationCriteria.from_client_input(query_string)
# Test
units = self.manager.get_units_by_type('repo-1', 'alpha', criteria)
# Verify
self.assertEqual(len(self.units['alpha']) - 1, len(units))
for u in units:
self.assertTrue(u['metadata']['key_1'] != 'aardvark')
def associate_from_repo(cls, source_repo_id, dest_repo_id, criteria,
import_config_override=None):
"""
Creates associations in a repository based on the contents of a source
repository. Units from the source repository can be filtered by
specifying a criteria object.
The destination repository must have an importer that can support
the types of units being associated. This is done by analyzing the
unit list and the importer metadata and takes place before the
destination repository is called.
Pulp does not actually perform the associations as part of this call.
The unit list is determined and passed to the destination repository's
importer. It is the job of the importer to make the associate calls
back into Pulp where applicable.
If criteria is None, the effect of this call is to copy the source
repository's associations into the destination repository.
:param source_repo_id: identifies the source repository
:type source_repo_id: str
:param dest_repo_id: identifies the destination repository
:type dest_repo_id: str
:param criteria: optional; if specified, will filter the units retrieved from
the source repository
:type criteria: pulp.server.db.model.criteria.UnitAssociationCriteria
:param import_config_override: optional config containing values to use for this import only
:type import_config_override: dict
:return: dict with key 'units_successful' whose
value is a list of unit keys that were copied.
units that were associated by this operation
:rtype: dict
:raise MissingResource: if either of the specified repositories don't exist
"""
criteria = UnitAssociationCriteria.from_dict(criteria)
source_repo = model.Repository.objects.get_repo_or_missing_resource(source_repo_id)
dest_repo = model.Repository.objects.get_repo_or_missing_resource(dest_repo_id)
dest_repo_importer = model.Importer.objects.get_or_404(repo_id=dest_repo_id)
source_repo_importer = model.Importer.objects.get_or_404(repo_id=source_repo_id)
# The docs are incorrect on the list_importer_types call; it actually
# returns a dict with the types under key "types" for some reason.
supported_type_ids = set(plugin_api.list_importer_types(
dest_repo_importer.importer_type_id)['types'])
# Get the unit types from the repo source repo
source_repo_unit_types = set(source_repo.content_unit_counts.keys())
# Now we can make sure the destination repository's importer is capable
# of importing either the selected units or all of the units
if not source_repo_unit_types.issubset(supported_type_ids):
raise exceptions.PulpCodedException(error_code=error_codes.PLP0044)
transfer_units = None
# if all source types have been converted to mongo - search via new style
if source_repo_unit_types.issubset(set(plugin_api.list_unit_models())):
transfer_units = cls._units_from_criteria(source_repo, criteria)
else:
# else, search via old style
associate_us = load_associated_units(source_repo_id, criteria)
# If units were supposed to be filtered but none matched, we're done
if len(associate_us) == 0:
# Return an empty list to indicate nothing was copied
return {'units_successful': []}
# Convert all of the units into the plugin standard representation if
# a filter was specified
transfer_units = None
if associate_us is not None:
transfer_units = create_transfer_units(associate_us)
# Convert the two repos into the plugin API model
transfer_dest_repo = dest_repo.to_transfer_repo()
transfer_source_repo = source_repo.to_transfer_repo()
# Invoke the importer
importer_instance, plugin_config = plugin_api.get_importer_by_id(
dest_repo_importer.importer_type_id)
call_config = PluginCallConfiguration(plugin_config, dest_repo_importer.config,
import_config_override)
conduit = ImportUnitConduit(
source_repo_id, dest_repo_id, source_repo_importer.importer_type_id,
dest_repo_importer.importer_type_id)
try:
copied_units = importer_instance.import_units(
transfer_source_repo, transfer_dest_repo, conduit, call_config,
units=transfer_units)
if isinstance(copied_units, tuple):
suc_units_ids = [u.to_id_dict() for u in copied_units[0] if u is not None]
unsuc_units_ids = [u.to_id_dict() for u in copied_units[1]]
repo_controller.rebuild_content_unit_counts(dest_repo)
return {'units_successful': suc_units_ids,
'units_failed_signature_filter': unsuc_units_ids}
unit_ids = [u.to_id_dict() for u in copied_units if u is not None]
repo_controller.rebuild_content_unit_counts(dest_repo)
return {'units_successful': unit_ids}
except Exception:
msg = _('Exception from importer [%(i)s] while importing units into repository [%(r)s]')
msg_dict = {'i': dest_repo_importer.importer_type_id, 'r': dest_repo_id}
logger.exception(msg % msg_dict)
raise exceptions.PulpExecutionException(), None, sys.exc_info()[2]
