def POST(self, repo_id):
# Params
params = self.params()
query = params.get('criteria', None)
repo_query_manager = manager_factory.repo_query_manager()
repo = repo_query_manager.find_by_id(repo_id)
if repo is None:
raise exceptions.MissingResource(repo_id=repo_id)
if query is None:
raise exceptions.MissingValue(['criteria'])
try:
criteria = UnitAssociationCriteria.from_client_input(query)
except:
_LOG.exception('Error parsing association criteria [%s]' % query)
raise exceptions.PulpDataException(), None, sys.exc_info()[2]
# Data lookup
manager = manager_factory.repo_unit_association_query_manager()
if criteria.type_ids is not None and len(criteria.type_ids) == 1:
type_id = criteria.type_ids[0]
units = manager.get_units_by_type(repo_id, type_id, criteria=criteria)
else:
units = manager.get_units_across_types(repo_id, criteria=criteria)
return self.ok(units)
def test_migrate_category(self):
# Setup
orig_cat_id = add_unit('c1', self.source_repo_id, ids.TYPE_ID_PKG_CATEGORY)
associate_unit(orig_cat_id, self.source_repo_id, ids.TYPE_ID_PKG_CATEGORY)
associate_unit(orig_cat_id, self.dest_repo_id, ids.TYPE_ID_PKG_CATEGORY)
# Test
self.migration.migrate()
group_coll = types_db.type_units_collection(ids.TYPE_ID_PKG_CATEGORY)
all_cats = group_coll.find({}).sort('repo_id', 1)
self.assertEqual(2, all_cats.count())
dest_cat = all_cats[0] # ordered by ID, this will be first
self.assertEqual(dest_cat['id'], 'c1')
self.assertEqual(dest_cat['repo_id'], self.dest_repo_id)
source_cat = all_cats[1]
self.assertEqual(source_cat['id'], 'c1')
self.assertEqual(source_cat['repo_id'], self.source_repo_id)
# Verify the associations
query_manager = factory.repo_unit_association_query_manager()
source_units = query_manager.get_units(self.source_repo_id)
self.assertEqual(1, len(source_units))
self.assertEqual(source_units[0]['unit_type_id'], ids.TYPE_ID_PKG_CATEGORY)
self.assertEqual(source_units[0]['unit_id'], source_cat['_id'])
dest_units = query_manager.get_units(self.dest_repo_id)
self.assertEqual(1, len(dest_units))
self.assertEqual(dest_units[0]['unit_type_id'], ids.TYPE_ID_PKG_CATEGORY)
self.assertEqual(dest_units[0]['unit_id'], dest_cat['_id'])
def __init__(self, source_repo_id, dest_repo_id, source_importer_id,
dest_importer_id):
"""
:param source_repo_id: ID of the repository from which units are being copied
:type source_repo_id: str
:param dest_repo_id: ID of the repository into which units are being copied
:type dest_repo_id: str
:param source_importer_id: ID of the importer on the source repository
:type source_importer_id: str
:param dest_importer_id: ID of the importer on the destination repository
:type dest_importer_id: str
"""
ImporterScratchPadMixin.__init__(self, dest_repo_id, dest_importer_id)
RepoScratchPadMixin.__init__(self, dest_repo_id,
ImporterConduitException)
SearchUnitsMixin.__init__(self, ImporterConduitException)
AddUnitMixin.__init__(self, dest_repo_id, dest_importer_id)
self.source_repo_id = source_repo_id
self.dest_repo_id = dest_repo_id
self.source_importer_id = source_importer_id
self.dest_importer_id = dest_importer_id
self.__association_manager = manager_factory.repo_unit_association_manager(
)
self.__association_query_manager = manager_factory.repo_unit_association_query_manager(
)
self.__importer_manager = manager_factory.repo_importer_manager()
def _add_repo_memberships(units, type_id):
"""
For a list of units, find what repos each is a member of and add a list
of repo_ids to each unit.
:param units: list of unit documents
:type units: list of dicts
:param type_id: content type id
:type type_id: str
:return: same list of units that was passed in, only for convenience.
units are modified in-place
"""
# quick return if there is nothing to do
if not units:
return units
unit_ids = [unit["_id"] for unit in units]
criteria = Criteria(
filters={"unit_id": {"$in": unit_ids}, "unit_type_id": type_id}, fields=("repo_id", "unit_id")
)
associations = factory.repo_unit_association_query_manager().find_by_criteria(criteria)
unit_ids = None
criteria = None
association_map = {}
for association in associations:
association_map.setdefault(association["unit_id"], set()).add(association["repo_id"])
for unit in units:
unit["repository_memberships"] = list(association_map.get(unit["_id"], []))
return units
def do_get_repo_units(repo_id, criteria, exception_class, as_generator=False):
"""
Performs a repo unit association query. This is split apart so we can have
custom mixins with different signatures.
"""
try:
association_query_manager = manager_factory.repo_unit_association_query_manager(
)
# Use a get_units as_generator here and cast to a list later, if necessary.
units = association_query_manager.get_units(repo_id,
criteria=criteria,
as_generator=True)
# Load all type definitions so we don't hammer the database.
type_defs = dict((t['id'], t) for t in types_db.all_type_definitions())
# Transfer object generator.
def _transfer_object_generator():
for u in units:
yield common_utils.to_plugin_associated_unit(
u, type_defs[u['unit_type_id']])
if as_generator:
return _transfer_object_generator()
# Maintain legacy behavior by default.
return list(_transfer_object_generator())
except Exception, e:
logger.exception(
'Exception from server requesting all content units for repository [%s]'
% repo_id)
raise exception_class(e), None, sys.exc_info()[2]
def calculate_associated_type_ids(source_repo_id, associated_units):
if associated_units is not None:
associated_unit_type_ids = set([u['unit_type_id'] for u in associated_units])
else:
association_query_manager = manager_factory.repo_unit_association_query_manager()
associated_unit_type_ids = association_query_manager.unit_type_ids_for_repo(source_repo_id)
return associated_unit_type_ids
def _add_repo_memberships(units, type_id):
"""
For a list of units, find what repos each is a member of, and add a list
of repo_ids to each unit.
:param units: list of unit documents
:type units: list of dicts
:param type_id: content type id
:type type_id: str
:return: same list of units that was passed in, only for convenience.
units are modified in-place
"""
# quick return if there is nothing to do
if not units:
return units
unit_ids = [unit['_id'] for unit in units]
criteria = Criteria(
filters={'unit_id': {'$in': unit_ids}, 'unit_type_id': type_id},
fields=('repo_id', 'unit_id')
)
associations = factory.repo_unit_association_query_manager().find_by_criteria(criteria)
unit_ids = None
criteria = None
association_map = {}
for association in associations:
association_map.setdefault(association['unit_id'], set()).add(
association['repo_id'])
for unit in units:
unit['repository_memberships'] = list(association_map.get(unit['_id'], []))
return units
def __init__(self, source_repo_id, dest_repo_id, source_importer_id, dest_importer_id,
association_owner_type, association_owner_id):
"""
:param source_repo_id: ID of the repository from which units are being copied
:type source_repo_id: str
:param dest_repo_id: ID of the repository into which units are being copied
:type dest_repo_id: str
:param source_importer_id: ID of the importer on the source repository
:type source_importer_id: str
:param dest_importer_id: ID of the importer on the destination repository
:type dest_importer_id: str
:param association_owner_type: distinguishes the owner when creating an
association through this conduit
:type association_owner_type: str
:param association_owner_id: specific ID of the owner when creating an
association through this conduit
:type association_owner_id: str
"""
ImporterScratchPadMixin.__init__(self, dest_repo_id, dest_importer_id)
RepoScratchPadMixin.__init__(self, dest_repo_id, ImporterConduitException)
SearchUnitsMixin.__init__(self, ImporterConduitException)
AddUnitMixin.__init__(self, dest_repo_id, dest_importer_id, association_owner_type, association_owner_id)
self.source_repo_id = source_repo_id
self.dest_repo_id = dest_repo_id
self.source_importer_id = source_importer_id
self.dest_importer_id = dest_importer_id
self.association_owner_type = association_owner_type
self.association_owner_id = association_owner_id
self.__association_manager = manager_factory.repo_unit_association_manager()
self.__association_query_manager = manager_factory.repo_unit_association_query_manager()
self.__importer_manager = manager_factory.repo_importer_manager()
def _generate_response(cls, query, options, *args, **kwargs):
"""
Perform the database query using the given search data, and return the resuls as a JSON
serialized HttpReponse object.
This overrides the base class so we can validate repo existance and to choose the search
method depending on how many unit types we are dealing with.
:param query: The criteria that should be used to search for objects
:type query: dict
:param options: additional options for including extra data
:type options: dict
:return: The serialized search results in an HttpReponse
:rtype: django.http.HttpResponse
"""
repo_id = kwargs.get('repo_id')
model.Repository.objects.get_repo_or_missing_resource(repo_id)
criteria = UnitAssociationCriteria.from_client_input(query)
manager = manager_factory.repo_unit_association_query_manager()
if criteria.type_ids is not None and len(criteria.type_ids) == 1:
type_id = criteria.type_ids[0]
units = manager.get_units_by_type(repo_id, type_id, criteria=criteria)
else:
units = manager.get_units(repo_id, criteria=criteria)
for unit in units:
content.remap_fields_with_serializer(unit['metadata'])
return generate_json_response_with_pulp_encoder(units)
def _generate_response(cls, query, options, *args, **kwargs):
"""
Perform the database query using the given search data, and return the resuls as a JSON
serialized HttpReponse object.
This overrides the base class so we can validate repo existance and to choose the search
method depending on how many unit types we are dealing with.
:param query: The criteria that should be used to search for objects
:type query: dict
:param options: additional options for including extra data
:type options: dict
:return: The serialized search results in an HttpReponse
:rtype: django.http.HttpResponse
"""
repo_id = kwargs.get('repo_id')
model.Repository.objects.get_repo_or_missing_resource(repo_id)
criteria = UnitAssociationCriteria.from_client_input(query)
manager = manager_factory.repo_unit_association_query_manager()
if criteria.type_ids is not None and len(criteria.type_ids) == 1:
type_id = criteria.type_ids[0]
units = manager.get_units_by_type(repo_id, type_id, criteria=criteria)
else:
units = manager.get_units(repo_id, criteria=criteria)
return generate_json_response_with_pulp_encoder(units)
def __init__(self, source_repo_id, dest_repo_id, source_importer_id, dest_importer_id):
"""
:param source_repo_id: ID of the repository from which units are being copied
:type source_repo_id: str
:param dest_repo_id: ID of the repository into which units are being copied
:type dest_repo_id: str
:param source_importer_id: ID of the importer on the source repository
:type source_importer_id: str
:param dest_importer_id: ID of the importer on the destination repository
:type dest_importer_id: str
"""
ImporterScratchPadMixin.__init__(self, dest_repo_id, dest_importer_id)
RepoScratchPadMixin.__init__(self, dest_repo_id, ImporterConduitException)
SearchUnitsMixin.__init__(self, ImporterConduitException)
AddUnitMixin.__init__(self, dest_repo_id, dest_importer_id)
self.source_repo_id = source_repo_id
self.dest_repo_id = dest_repo_id
self.source_importer_id = source_importer_id
self.dest_importer_id = dest_importer_id
self.__association_manager = manager_factory.repo_unit_association_manager()
self.__association_query_manager = manager_factory.repo_unit_association_query_manager()
self.__importer_manager = manager_factory.repo_importer_manager()
def test_migrate_duplicates_doesnt_delete_from_source_repo(self):
"""
This tests the correct behavior when we try to change the repo_id on an object,
and end up causing
a duplicate error due to our uniqueness constraint. It also makes sure the units are not
deleted from
the source repository if they are in the source repository.
"""
# Let's put two units here with the same IDs with two different repo_ids, and the run the
# migration.
source_repo_group_id = add_unit('group', self.source_repo_id,
ids.TYPE_ID_PKG_GROUP)
dest_repo_group_id = add_unit('group', self.dest_repo_id,
ids.TYPE_ID_PKG_GROUP)
# Associate the source_repo_group_id with both source and destination repos
associate_unit(source_repo_group_id, self.source_repo_id,
ids.TYPE_ID_PKG_GROUP)
associate_unit(source_repo_group_id, self.dest_repo_id,
ids.TYPE_ID_PKG_GROUP)
associate_unit(dest_repo_group_id, self.dest_repo_id,
ids.TYPE_ID_PKG_GROUP)
# Migrate should not cause a DuplicateKeyError
self.migration.migrate()
# Verify that both groups remain, because the migration should not have removed either
group_collection = types_db.type_units_collection(
ids.TYPE_ID_PKG_GROUP)
all_groups = list(group_collection.find())
self.assertEqual(len(all_groups), 2)
self.assertEqual(
group_collection.find({
'id': 'group',
'repo_id': self.dest_repo_id
}).count(), 1)
self.assertEqual(
group_collection.find({
'id': 'group',
'repo_id': self.source_repo_id
}).count(), 1)
# Let's make sure that there are two associations, and that they are correct.
query_manager = factory.repo_unit_association_query_manager()
dest_units = query_manager.get_units(self.dest_repo_id)
self.assertEqual(len(dest_units), 1)
dest_unit = dest_units[0]
self.assertEqual(dest_unit['unit_type_id'], ids.TYPE_ID_PKG_GROUP)
self.assertEqual(dest_unit['unit_id'], dest_repo_group_id)
source_units = query_manager.get_units(self.source_repo_id)
self.assertEqual(len(source_units), 1)
source_unit = source_units[0]
self.assertEqual(source_unit['unit_type_id'], ids.TYPE_ID_PKG_GROUP)
self.assertEqual(source_unit['unit_id'], source_repo_group_id)
# Verify the repo counts
source = model.Repository.objects.get(repo_id='source-repo')
self.assertEqual(source.content_unit_counts, {'package_group': 1})
dest = model.Repository.objects.get(repo_id='dest-repo')
self.assertEqual(dest.content_unit_counts, {'package_group': 1})
def do_get_repo_units(repo_id, criteria, exception_class, as_generator=False):
"""
Performs a repo unit association query. This is split apart so we can have
custom mixins with different signatures.
"""
try:
association_query_manager = manager_factory.repo_unit_association_query_manager()
# Use a get_units as_generator here and cast to a list later, if necessary.
units = association_query_manager.get_units(repo_id, criteria=criteria, as_generator=True)
# Transfer object generator.
def _transfer_object_generator():
unit_key_fields_cache = {}
for u in units:
type_id = u['unit_type_id']
if type_id not in unit_key_fields_cache:
fields = units_controller.get_unit_key_fields_for_type(type_id)
unit_key_fields_cache[type_id] = fields
yield common_utils.to_plugin_associated_unit(u, type_id,
unit_key_fields_cache[type_id])
if as_generator:
return _transfer_object_generator()
# Maintain legacy behavior by default.
return list(_transfer_object_generator())
except Exception, e:
_logger.exception(
'Exception from server requesting all content units for repository [%s]' % repo_id)
raise exception_class(e), None, sys.exc_info()[2]
def do_get_repo_units(repo_id, criteria, exception_class):
"""
Performs a repo unit association query. This is split apart so we can have
custom mixins with different signatures.
"""
try:
association_query_manager = manager_factory.repo_unit_association_query_manager(
)
units = association_query_manager.get_units(repo_id, criteria=criteria)
all_units = []
# Load all type definitions in use so we don't hammer the database
unique_type_defs = set([u['unit_type_id'] for u in units])
type_defs = {}
for def_id in unique_type_defs:
type_def = types_db.type_definition(def_id)
type_defs[def_id] = type_def
# Convert to transfer object
for unit in units:
type_id = unit['unit_type_id']
u = common_utils.to_plugin_associated_unit(unit,
type_defs[type_id])
all_units.append(u)
return all_units
except Exception, e:
_LOG.exception(
'Exception from server requesting all content units for repository [%s]'
% repo_id)
raise exception_class(e), None, sys.exc_info()[2]
def do_get_repo_units(repo_id, criteria, exception_class, as_generator=False):
"""
Performs a repo unit association query. This is split apart so we can have
custom mixins with different signatures.
"""
try:
association_query_manager = manager_factory.repo_unit_association_query_manager()
# Use a get_units as_generator here and cast to a list later, if necessary.
units = association_query_manager.get_units(repo_id, criteria=criteria, as_generator=True)
# Load all type definitions so we don't hammer the database.
type_defs = dict((t['id'], t) for t in types_db.all_type_definitions())
# Transfer object generator.
def _transfer_object_generator():
for u in units:
yield common_utils.to_plugin_associated_unit(u, type_defs[u['unit_type_id']])
if as_generator:
return _transfer_object_generator()
# Maintain legacy behavior by default.
return list(_transfer_object_generator())
except Exception, e:
_LOG.exception('Exception from server requesting all content units for repository [%s]' % repo_id)
raise exception_class(e), None, sys.exc_info()[2]
def _get_total(self, id_list=None, ignore_filter=False):
"""
Return the total number of units that are processed by this step.
This is used generally for progress reporting. The value returned should not change
during the processing of the step.
:param id_list: List of type ids to get the total count of
:type id_list: list of str
:param ignore_filter: Ignore the association filter and get all units of the given types
:type ignore_filter: bool
"""
if id_list is None:
id_list = self.unit_type
total = 0
types_to_query = set(id_list).difference(self.skip_list)
if not ignore_filter and self.association_filters:
# We are copying using a filter so we have to get everything
new_filter = copy.deepcopy(self.association_filters)
new_filter['unit_type_id'] = {'$in': list(types_to_query)}
criteria = Criteria(filters=new_filter)
association_query_manager = manager_factory.repo_unit_association_query_manager()
units_cursor = association_query_manager.find_by_criteria(criteria)
total = units_cursor.count()
else:
for type_id in types_to_query:
total += self.parent.repo.content_unit_counts.get(type_id, 0)
return total
def verify(self, num_units=PluginTestBase.NUM_UNITS):
# repository
manager = managers.repo_query_manager()
manager.get_repository(self.REPO_ID)
# importer
manager = managers.repo_importer_manager()
importer = manager.get_importer(self.REPO_ID)
manifest_url = importer['config'][constants.MANIFEST_URL_KEYWORD]
self.assertTrue(manifest_url.endswith('%s/manifest.json.gz' % self.REPO_ID))
# distributor
manager = managers.repo_distributor_manager()
manager.get_distributor(self.REPO_ID, FAKE_DISTRIBUTOR)
self.assertRaises(MissingResource, manager.get_distributor, self.REPO_ID, constants.HTTP_DISTRIBUTOR)
# check units
manager = managers.repo_unit_association_query_manager()
units = manager.get_units(self.REPO_ID)
units = dict([(u['metadata']['N'], u) for u in units])
self.assertEqual(len(units), num_units)
for n in range(0, num_units):
unit = units[n]
unit_id = self.UNIT_ID % n
metadata = unit['metadata']
storage_path = metadata['_storage_path'].replace('//', '/')
self.assertEqual(unit['unit_type_id'], self.UNIT_TYPE_ID)
self.assertEqual(unit['repo_id'], self.REPO_ID)
self.assertEqual(unit['owner_id'], constants.HTTP_IMPORTER)
file_path = '.'.join((unit_id, self.UNIT_TYPE_ID))
self.assertEqual(storage_path, os.path.join(self.childfs, 'content', file_path))
self.assertTrue(os.path.exists(storage_path))
fp = open(storage_path)
content = fp.read()
fp.close()
self.assertEqual(content, unit_id)
def POST(self, repo_id):
# Params
params = self.params()
query = params.get('criteria', None)
repo_query_manager = manager_factory.repo_query_manager()
repo = repo_query_manager.find_by_id(repo_id)
if repo is None:
raise exceptions.MissingResource(repo_id=repo_id)
if query is None:
raise exceptions.MissingValue(['criteria'])
try:
criteria = UnitAssociationCriteria.from_client_input(query)
except:
_logger.error('Error parsing association criteria [%s]' % query)
raise exceptions.PulpDataException(), None, sys.exc_info()[2]
# Data lookup
manager = manager_factory.repo_unit_association_query_manager()
if criteria.type_ids is not None and len(criteria.type_ids) == 1:
type_id = criteria.type_ids[0]
units = manager.get_units_by_type(repo_id,
type_id,
criteria=criteria)
else:
units = manager.get_units_across_types(repo_id, criteria=criteria)
return self.ok(units)
def resolve_dependencies_by_criteria(self, repo_id, criteria, options):
"""
Calculates dependencies for units in the given repositories. The
repository's importer is used to perform the calculation. The units
to resolve dependencies for are calculated by applying the given
criteria against the repository.
@param repo_id: identifies the repository
@type repo_id: str
@param criteria: used to determine which units to resolve dependencies for
@type criteria: UnitAssociationCriteria
@param options: dict of options to pass the importer to drive the resolution
@type options: dict
@return: report from the plugin
@rtype: object
"""
association_query_manager = manager_factory.repo_unit_association_query_manager()
units = association_query_manager.get_units(repo_id, criteria=criteria)
# The bulk of the validation will be done in the chained call below
return self.resolve_dependencies_by_units(repo_id, units, options)
def _get_total(self, id_list=None, ignore_filter=False):
"""
Return the total number of units that are processed by this step.
This is used generally for progress reporting. The value returned should not change
during the processing of the step.
:param id_list: List of type ids to get the total count of
:type id_list: list of str
:param ignore_filter: Ignore the association filter and get all units of the given types
:type ignore_filter: bool
"""
if id_list is None:
id_list = self.unit_type
total = 0
types_to_query = set(id_list).difference(self.skip_list)
if not ignore_filter and self.association_filters:
# We are copying using a filter so we have to get everything
new_filter = copy.deepcopy(self.association_filters)
new_filter['unit_type_id'] = {'$in': list(types_to_query)}
criteria = Criteria(filters=new_filter)
association_query_manager = manager_factory.repo_unit_association_query_manager(
)
units_cursor = association_query_manager.find_by_criteria(criteria)
total = units_cursor.count()
else:
for type_id in types_to_query:
total += self.parent.repo.content_unit_counts.get(type_id, 0)
return total
def unassociate_by_criteria(repo_id, criteria, notify_plugins=True):
"""
Unassociate units that are matched by the given criteria.
:param repo_id: identifies the repo
:type repo_id: str
:param criteria:
:param notify_plugins: if true, relevant plugins will be informed of the removal
:type notify_plugins: bool
"""
criteria = UnitAssociationCriteria.from_dict(criteria)
association_query_manager = manager_factory.repo_unit_association_query_manager(
)
unassociate_units = association_query_manager.get_units(
repo_id, criteria=criteria)
if len(unassociate_units) == 0:
return {}
unit_map = {} # maps unit_type_id to a list of unit_ids
for unit in unassociate_units:
id_list = unit_map.setdefault(unit['unit_type_id'], [])
id_list.append(unit['unit_id'])
collection = RepoContentUnit.get_collection()
for unit_type_id, unit_ids in unit_map.items():
spec = {
'repo_id': repo_id,
'unit_type_id': unit_type_id,
'unit_id': {
'$in': unit_ids
}
}
collection.remove(spec)
unique_count = sum(
1 for unit_id in unit_ids
if not RepoUnitAssociationManager.association_exists(
repo_id, unit_id, unit_type_id))
if not unique_count:
continue
repo_controller.update_unit_count(repo_id, unit_type_id,
-unique_count)
repo_controller.update_last_unit_removed(repo_id)
# Convert the units into transfer units. This happens regardless of whether or not
# the plugin will be notified as it's used to generate the return result,
transfer_units = create_transfer_units(unassociate_units)
if notify_plugins:
remove_from_importer(repo_id, transfer_units)
# Match the return type/format as copy
serializable_units = [u.to_id_dict() for u in transfer_units]
return {'units_successful': serializable_units}
def calculate_associated_type_ids(source_repo_id, associated_units):
if associated_units is not None:
associated_unit_type_ids = set([u['unit_type_id'] for u in associated_units])
else:
association_query_manager = manager_factory.repo_unit_association_query_manager()
associated_unit_type_ids = association_query_manager.unit_type_ids_for_repo(source_repo_id)
return associated_unit_type_ids
def resolve_dependencies_by_criteria(self, repo_id, criteria, options):
"""
Calculates dependencies for units in the given repositories. The
repository's importer is used to perform the calculation. The units
to resolve dependencies for are calculated by applying the given
criteria against the repository.
@param repo_id: identifies the repository
@type repo_id: str
@param criteria: used to determine which units to resolve dependencies for
@type criteria: UnitAssociationCriteria
@param options: dict of options to pass the importer to drive the resolution
@type options: dict
@return: report from the plugin
@rtype: object
"""
association_query_manager = manager_factory.repo_unit_association_query_manager(
)
units = association_query_manager.get_units(repo_id, criteria=criteria)
# The bulk of the validation will be done in the chained call below
return self.resolve_dependencies_by_units(repo_id, units, options)
def do_get_repo_units(repo_id, criteria, exception_class):
"""
Performs a repo unit association query. This is split apart so we can have
custom mixins with different signatures.
"""
try:
association_query_manager = manager_factory.repo_unit_association_query_manager()
units = association_query_manager.get_units(repo_id, criteria=criteria)
all_units = []
# Load all type definitions in use so we don't hammer the database
unique_type_defs = set([u['unit_type_id'] for u in units])
type_defs = {}
for def_id in unique_type_defs:
type_def = types_db.type_definition(def_id)
type_defs[def_id] = type_def
# Convert to transfer object
for unit in units:
type_id = unit['unit_type_id']
u = common_utils.to_plugin_unit(unit, type_defs[type_id])
all_units.append(u)
return all_units
except Exception, e:
_LOG.exception('Exception from server requesting all content units for repository [%s]' % repo_id)
raise exception_class(e), None, sys.exc_info()[2]
def get_repo_units(self,
repo_id,
content_type_id,
additional_unit_fields=None):
"""
Searches for units in the given repository with given content type
and returns a plugin unit containing unit id, unit key and any additional
fields requested.
:param repo_id: repo id
:type repo_id: str
:param content_type_id: content type id of the units
:type content_type_id: str
:param additional_unit_fields: additional fields from the unit metadata to be added
in the result
:type additional_unit_fields: list of str
:return: list of unit instances
:rtype: list of pulp.plugins.model.Unit
"""
additional_unit_fields = additional_unit_fields or []
try:
unit_key_fields = units_controller.get_unit_key_fields_for_type(
content_type_id)
# Query repo association manager to get all units of given type
# associated with given repo. Limit data by requesting only the fields
# that are needed.
query_manager = managers.repo_unit_association_query_manager()
unit_fields = list(unit_key_fields) + list(additional_unit_fields)
criteria = UnitAssociationCriteria(association_fields=['unit_id'],
unit_fields=unit_fields)
units = query_manager.get_units_by_type(repo_id, content_type_id,
criteria)
# Convert units to plugin units with unit_key and required metadata values for each unit
all_units = []
for unit in units:
unit_key = {}
metadata = {}
for k in unit_key_fields:
unit_key[k] = unit['metadata'].pop(k)
# Add unit_id and any additional unit fields requested by plugins
metadata['unit_id'] = unit.pop('unit_id')
for field in additional_unit_fields:
metadata[field] = unit['metadata'].pop(field, None)
u = Unit(content_type_id, unit_key, metadata, None)
all_units.append(u)
return all_units
except Exception, e:
_logger.exception(
_('Exception from server getting units from repo [%s]' %
repo_id))
raise self.exception_class(e), None, sys.exc_info()[2]
def load_associated_units(source_repo_id, criteria):
criteria.association_fields = None
# Retrieve the units to be associated
association_query_manager = manager_factory.repo_unit_association_query_manager()
associate_us = association_query_manager.get_units(source_repo_id, criteria=criteria)
return associate_us
def load_associated_units(source_repo_id, criteria):
criteria.association_fields = None
# Retrieve the units to be associated
association_query_manager = manager_factory.repo_unit_association_query_manager()
associate_us = association_query_manager.get_units(source_repo_id, criteria=criteria)
return associate_us
def unassociate_by_criteria(repo_id, criteria, owner_type, owner_id, notify_plugins=True):
"""
Unassociate units that are matched by the given criteria.
:param repo_id: identifies the repo
:type repo_id: str
:param criteria:
:param owner_type: category of the caller who created the association
:type owner_type: str
:param owner_id: identifies the call who created the association
:type owner_id: str
:param notify_plugins: if true, relevant plugins will be informed of the removal
:type notify_plugins: bool
"""
association_query_manager = manager_factory.repo_unit_association_query_manager()
unassociate_units = association_query_manager.get_units(repo_id, criteria=criteria)
if len(unassociate_units) == 0:
return {}
unit_map = {} # maps unit_type_id to a list of unit_ids
for unit in unassociate_units:
id_list = unit_map.setdefault(unit['unit_type_id'], [])
id_list.append(unit['unit_id'])
collection = RepoContentUnit.get_collection()
repo_manager = manager_factory.repo_manager()
for unit_type_id, unit_ids in unit_map.items():
spec = {'repo_id': repo_id,
'unit_type_id': unit_type_id,
'unit_id': {'$in': unit_ids}
}
collection.remove(spec, safe=True)
unique_count = sum(
1 for unit_id in unit_ids if not RepoUnitAssociationManager.association_exists(
repo_id, unit_id, unit_type_id))
if not unique_count:
continue
repo_manager.update_unit_count(repo_id, unit_type_id, -unique_count)
repo_manager.update_last_unit_removed(repo_id)
# Convert the units into transfer units. This happens regardless of whether or not
# the plugin will be notified as it's used to generate the return result,
unit_type_ids = calculate_associated_type_ids(repo_id, unassociate_units)
transfer_units = create_transfer_units(unassociate_units, unit_type_ids)
if notify_plugins:
remove_from_importer(repo_id, transfer_units)
# Match the return type/format as copy
serializable_units = [u.to_id_dict() for u in transfer_units]
return {'units_successful': serializable_units}
def __unit_ids_to_plugin_unit_keys(self, unit_ids_by_type, repo_ids):
"""
Parse a dictionary of unit ids keyed by content type id and return a dictionary of
corresponding plugin unit keys keyed by content type id.
:param unit_ids_by_type: dictionary of <content type id> : <list of unit ids>
:type unit_ids_by_type: dict
:return: if units are specified, return the corresponding plugin unit_keys. If unit_ids_by_type dict
is empty, return plugin unit keys corresponging to all units in given repo ids.
If unit ids list for a particular unit type is empty, return all plugin unit_keys
in given repo ids with that unit type.
:rtype: dict
"""
repo_unit_association_query_manager = managers.repo_unit_association_query_manager()
content_query_manager = managers.content_query_manager()
result_unit_keys = {}
if unit_ids_by_type is not None:
for unit_type_id, unit_ids in unit_ids_by_type.items():
# Get unit type specific collection
collection = content_query_manager.get_content_unit_collection(type_id=unit_type_id)
type_def = content_types_db.type_definition(unit_type_id)
if not unit_ids:
# If unit_list is empty for a unit_type, consider all units of specific type
criteria = UnitAssociationCriteria(unit_fields = ['unit_id'])
for repo_id in repo_ids:
repo_units = repo_unit_association_query_manager.get_units_by_type(repo_id, unit_type_id, criteria)
# Get metadata for each unit from type specific collection
pulp_units = [collection.find_one({'_id': u['unit_id']}) for u in repo_units]
# Convert pulp units to plugin unit keys
plugin_unit_keys = [common_utils.to_plugin_unit(u, type_def).unit_key for u in pulp_units]
result_unit_keys.setdefault(unit_type_id, []).extend(plugin_unit_keys)
else:
# Get metadata for each unit from type specific collection
pulp_units = [collection.find_one({'_id': unit_id}) for unit_id in unit_ids]
# Convert pulp units to plugin unit keys
plugin_unit_keys = [common_utils.to_plugin_unit(u, type_def).unit_key for u in pulp_units]
result_unit_keys.setdefault(unit_type_id, []).extend(plugin_unit_keys)
else:
# If units are not specified, consider all units in given repos.
for repo_id in repo_ids:
all_unit_type_ids = content_types_db.all_type_ids()
for unit_type_id in all_unit_type_ids:
criteria = UnitAssociationCriteria(type_ids=[unit_type_id], unit_fields = ['unit_id', 'unit_type_id'])
repo_units = repo_unit_association_query_manager.get_units(repo_id, criteria)
# Get unit metadata for each unit from type specific collection
collection = content_query_manager.get_content_unit_collection(type_id=unit_type_id)
pulp_units = [collection.find_one({'_id': u['unit_id']}) for u in repo_units]
# Convert pulp units to plugin unit keys
type_def = content_types_db.type_definition(unit_type_id)
plugin_unit_keys = [common_utils.to_plugin_unit(u, type_def).unit_key for u in pulp_units]
result_unit_keys.setdefault(unit_type_id, []).extend(plugin_unit_keys)
return result_unit_keys
def __parse_units(self, user_units, repo_ids):
"""
Parse units specified by user and return a dictionary of all plugin unit_keys
to be considered for applicability keyed by unit_type_id.
:param user_units: dictionary of unit metadata filters keyed by unit-type-id specified by user
:type user_units: dict
:return: if specific units are specified, return the corresponding plugin unit_keys. If units dict is empty,
return all plugin unit_keys corresponging to units in given repo ids keyed by unit_type_id.
If units list for a particular unit type in units is empty, return all plugin unit_keys
in given repo ids with that unit type keyed by unit_type_id.
:rtype: dict
"""
repo_unit_association_query_manager = managers.repo_unit_association_query_manager()
content_query_manager = managers.content_query_manager()
result_unit_keys = {}
if user_units is not None:
for unit_type_id, unit_list in user_units.items():
# Get unit type specific collection
collection = content_query_manager.get_content_unit_collection(type_id=unit_type_id)
type_def = content_types_db.type_definition(unit_type_id)
if not unit_list:
# If unit_list is empty for a unit_type, consider all units of specific type
criteria = UnitAssociationCriteria(unit_fields = ['unit_id'])
for repo_id in repo_ids:
repo_units = repo_unit_association_query_manager.get_units_by_type(repo_id, unit_type_id, criteria)
# Get unit metadata for each unit from type specific collection
pulp_units = [collection.find_one({'_id': u['unit_id']}) for u in repo_units]
plugin_units = [common_utils.to_plugin_unit(u, type_def) for u in pulp_units]
plugin_unit_keys = [u.unit_key for u in plugin_units]
result_unit_keys.setdefault(unit_type_id, []).extend(plugin_unit_keys)
else:
for unit in unit_list:
criteria = UnitAssociationCriteria(unit_filters=unit, unit_fields = ['unit_id'])
for repo_id in repo_ids:
repo_units = repo_unit_association_query_manager.get_units_by_type(repo_id, unit_type_id, criteria)
# Get unit metadata for each unit from type specific collection
pulp_units = [collection.find_one({'_id': u['unit_id']}) for u in repo_units]
plugin_units = [common_utils.to_plugin_unit(u, type_def) for u in pulp_units]
plugin_unit_keys = [u.unit_key for u in plugin_units]
result_unit_keys.setdefault(unit_type_id, []).extend(plugin_unit_keys)
else:
# If units are not specified, consider all units in repo_ids list.
for repo_id in repo_ids:
criteria = UnitAssociationCriteria(unit_fields = ['unit_id','unit_type_id'])
repo_units = repo_unit_association_query_manager.get_units(repo_id, criteria)
# Get unit metadata for each unit from type specific collection
for repo_unit in repo_units:
collection = content_query_manager.get_content_unit_collection(type_id=repo_unit['unit_type_id'])
type_def = content_types_db.type_definition(repo_unit['unit_type_id'])
pulp_unit = collection.find_one({'_id': repo_unit['unit_id']})
plugin_unit = common_utils.to_plugin_unit(pulp_unit, type_def)
result_unit_keys.setdefault(repo_unit['unit_type_id'], []).append(plugin_unit.unit_key)
return result_unit_keys
def _purge_orphaned_blobs(repo, units):
"""
Purge blobs associated with removed manifests when no longer
referenced by any remaining manifests.
:param repo: The affected repository.
:type repo: pulp.plugins.model.Repository
:param units: List of removed units.
:type units: list of: pulp.plugins.model.AssociatedUnit
"""
# Find blob digests referenced by removed manifests (orphaned)
orphaned = set()
for unit in units:
if unit.type_id != constants.MANIFEST_TYPE_ID:
continue
manifest = unit
for layer in manifest.metadata['fs_layers']:
digest = layer['blobSum']
orphaned.add(digest)
# Find blob digests still referenced by other manifests (adopted)
if not orphaned:
# nothing orphaned
return
adopted = set()
manager = manager_factory.repo_unit_association_query_manager()
for manifest in manager.get_units_by_type(repo.id, constants.MANIFEST_TYPE_ID):
for layer in manifest.metadata['fs_layers']:
digest = layer['blobSum']
adopted.add(digest)
# Remove unreferenced blobs
orphaned = orphaned.difference(adopted)
if not orphaned:
# all adopted
return
unit_filter = {
'digest': {
'$in': sorted(orphaned)
}
}
criteria = UnitAssociationCriteria(
type_ids=[constants.BLOB_TYPE_ID],
unit_filters=unit_filter)
manager = manager_factory.repo_unit_association_manager()
manager.unassociate_by_criteria(
repo_id=repo.id,
criteria=criteria,
owner_type='', # unused
owner_id='', # unused
notify_plugins=False)
def get_repo_units(self, repo_id, content_type_id, additional_unit_fields=None):
"""
Searches for units in the given repository with given content type
and returns a plugin unit containing unit id, unit key and any additional
fields requested.
:param repo_id: repo id
:type repo_id: str
:param content_type_id: content type id of the units
:type content_type_id: str
:param additional_unit_fields: additional fields from the unit metadata to be added
in the result
:type additional_unit_fields: list of str
:return: list of unit instances
:rtype: list of pulp.plugins.model.Unit
"""
additional_unit_fields = additional_unit_fields or []
try:
unit_key_fields = units_controller.get_unit_key_fields_for_type(content_type_id)
serializer = units_controller.get_model_serializer_for_type(content_type_id)
# Query repo association manager to get all units of given type
# associated with given repo. Limit data by requesting only the fields
# that are needed.
query_manager = managers.repo_unit_association_query_manager()
unit_fields = list(unit_key_fields) + list(additional_unit_fields)
criteria = UnitAssociationCriteria(association_fields=['unit_id'],
unit_fields=unit_fields)
units = query_manager.get_units_by_type(repo_id, content_type_id, criteria)
# Convert units to plugin units with unit_key and required metadata values for each unit
all_units = []
for unit in units:
if serializer:
serializer.serialize(unit['metadata'])
unit_key = {}
metadata = {}
for k in unit_key_fields:
unit_key[k] = unit['metadata'].pop(k)
# Add unit_id and any additional unit fields requested by plugins
metadata['unit_id'] = unit.pop('unit_id')
for field in additional_unit_fields:
metadata[field] = unit['metadata'].pop(field, None)
u = Unit(content_type_id, unit_key, metadata, None)
all_units.append(u)
return all_units
except Exception, e:
_logger.exception(_('Exception from server getting units from repo [%s]' % repo_id))
raise self.exception_class(e), None, sys.exc_info()[2]
def unassociate_by_criteria(self, repo_id, criteria, owner_type, owner_id, notify_plugins=True):
"""
Unassociate units that are matched by the given criteria.
@param repo_id: identifies the repo
@type repo_id: str
@param criteria:
@param owner_type: category of the caller who created the association
@type owner_type: str
@param owner_id: identifies the call who created the association
@type owner_id: str
@param notify_plugins: if true, relevant plugins will be informed of the
removal
@type notify_plugins: bool
"""
association_query_manager = manager_factory.repo_unit_association_query_manager()
unassociate_units = association_query_manager.get_units(repo_id, criteria=criteria)
if len(unassociate_units) is 0:
return []
unit_map = {} # maps unit_type_id to a list of unit_ids
for unit in unassociate_units:
id_list = unit_map.setdefault(unit['unit_type_id'], [])
id_list.append(unit['unit_id'])
collection = RepoContentUnit.get_collection()
repo_manager = manager_factory.repo_manager()
for unit_type_id, unit_ids in unit_map.items():
spec = {'repo_id': repo_id,
'unit_type_id': unit_type_id,
'unit_id': {'$in': unit_ids},
'owner_type': owner_type,
'owner_id': owner_id}
collection.remove(spec, safe=True)
unique_count = sum(1 for unit_id in unit_ids
if not self.association_exists(repo_id, unit_id, unit_type_id))
if not unique_count:
continue
repo_manager.update_unit_count(repo_id, unit_type_id, -unique_count)
# Convert the units into transfer units. This happens regardless of whether or not
# the plugin will be notified as it's used to generate the return result,
unit_type_ids = calculate_associated_type_ids(repo_id, unassociate_units)
transfer_units = create_transfer_units(unassociate_units, unit_type_ids)
if notify_plugins:
remove_from_importer(repo_id, transfer_units)
# Match the return type/format as copy
serializable_units = [u.to_id_dict() for u in transfer_units]
return serializable_units
def __populate_units(self, unit_ids_by_type, repo_ids):
"""
Parse a dictionary of unit ids keyed by content type id and populate units for each type
if the criteria is empty.
:param unit_ids_by_type: dictionary of <content type id> : <list of unit ids>
:type unit_ids_by_type: dict
:return: if units are specified, return the corresponding units. If unit_ids_by_type dict
is empty, return unit ids corresponging to all units in given repo ids.
If unit ids list for a particular unit type is empty, return all unit ids
in given repo ids with that unit type.
:rtype: dict
"""
repo_unit_association_query_manager = managers.repo_unit_association_query_manager(
)
result_units = {}
if unit_ids_by_type is not None:
for unit_type_id, unit_ids in unit_ids_by_type.items():
# Get unit type specific collection
if not unit_ids:
# If unit_list is empty for a unit_type, consider all units of specific type
criteria = Criteria(filters={
"repo_id": {
"$in": repo_ids
},
"unit_type_id": unit_type_id
},
fields=['unit_id'])
repo_units = repo_unit_association_query_manager.find_by_criteria(
criteria)
pulp_unit_ids = [u['unit_id'] for u in repo_units]
result_units.setdefault(unit_type_id, []).extend(
list(set(pulp_unit_ids)))
else:
result_units.setdefault(unit_type_id, []).extend(unit_ids)
else:
# If units are not specified, consider all units in given repos.
all_unit_type_ids = content_types_db.all_type_ids()
for unit_type_id in all_unit_type_ids:
criteria = Criteria(filters={
"repo_id": {
"$in": repo_ids
},
"unit_type_id": unit_type_id
},
fields=['unit_id'])
repo_units = repo_unit_association_query_manager.find_by_criteria(
criteria)
pulp_unit_ids = [u['unit_id'] for u in repo_units]
result_units.setdefault(unit_type_id,
[]).extend(list(set(pulp_unit_ids)))
return result_units
def test_migrate_duplicates(self):
"""
This tests the correct behavior when we try to change the repo_id on an object,
and end up causing
a duplicate error due to our uniqueness constraint.
"""
# Let's put two units here with the same IDs with two different repo_ids, and the run the
# migration.
source_repo_group_id = add_unit('group', self.source_repo_id,
ids.TYPE_ID_PKG_GROUP)
dest_repo_group_id = add_unit('group', self.dest_repo_id,
ids.TYPE_ID_PKG_GROUP)
associate_unit(source_repo_group_id, self.dest_repo_id,
ids.TYPE_ID_PKG_GROUP)
associate_unit(dest_repo_group_id, self.dest_repo_id,
ids.TYPE_ID_PKG_GROUP)
# Migrate should not cause a DuplicateKeyError
self.migration.migrate()
# Verify that both groups remain.
group_collection = types_db.type_units_collection(
ids.TYPE_ID_PKG_GROUP)
all_groups = list(group_collection.find())
self.assertEqual(len(all_groups), 2)
self.assertEqual(
group_collection.find({
'id': 'group',
'repo_id': self.dest_repo_id
}).count(), 1)
self.assertEqual(
group_collection.find({
'id': 'group',
'repo_id': self.source_repo_id
}).count(), 1)
# Let's make sure that the dest group is associated, but not the source one
query_manager = factory.repo_unit_association_query_manager()
dest_units = query_manager.get_units(self.dest_repo_id)
self.assertEqual(len(dest_units), 1)
dest_unit = dest_units[0]
self.assertEqual(dest_unit['unit_type_id'], ids.TYPE_ID_PKG_GROUP)
self.assertEqual(dest_unit['unit_id'], dest_repo_group_id)
self.assertEqual(query_manager.get_units(self.source_repo_id), [])
# Verify the repo counts
self.assertEqual(
Repo.get_collection().find({'id': 'source-repo'
})[0]['content_unit_counts'], {})
self.assertEqual(
Repo.get_collection().find({'id': 'dest-repo'
})[0]['content_unit_counts'],
{'package_group': 1})
def get_repo_units(self, repo_id, content_type_id, additional_unit_fields=[]):
"""
Searches for units in the given repository with given content type
and returns a plugin unit containing unit id, unit key and any additional
fields requested.
:param repo_id: repo id
:type repo_id: str
:param content_type_id: content type id of the units
:type content_type_id: str
:param additional_unit_fields: additional fields from the unit metadata to be added
in the result
:type additional_unit_fields: list of str
:return: list of unit instances
:rtype: list of pulp.plugins.model.Unit
"""
try:
# Get type definition and unit_key for given content type
type_def = types_db.type_definition(content_type_id)
unit_key_fields = type_def["unit_key"]
# Query repo association manager to get all units of given type
# associated with given repo. Limit data by requesting only the fields
# that are needed.
query_manager = managers.repo_unit_association_query_manager()
unit_fields = list(set(unit_key_fields + additional_unit_fields))
criteria = UnitAssociationCriteria(association_fields=["unit_id"], unit_fields=unit_fields)
units = query_manager.get_units_by_type(repo_id, content_type_id, criteria)
# Convert units to plugin units with unit_key and required metadata values for each unit
all_units = []
for unit in units:
unit_key = {}
metadata = {}
for k in unit_key_fields:
unit_key[k] = unit["metadata"].pop(k)
# Add unit_id and any additional unit fields requested by plugins
metadata["unit_id"] = unit.pop("unit_id")
for field in additional_unit_fields:
metadata[field] = unit["metadata"].pop(field, None)
u = Unit(content_type_id, unit_key, metadata, None)
all_units.append(u)
return all_units
except Exception, e:
_logger.exception(_("Exception from server getting units from repo [%s]" % repo_id))
raise self.exception_class(e), None, sys.exc_info()[2]
def __init__(self, source_repo_id, dest_repo_id, source_importer_id, dest_importer_id):
ImporterScratchPadMixin.__init__(self, dest_repo_id, dest_importer_id)
RepoScratchPadMixin.__init__(self, dest_repo_id, ImporterConduitException)
self.source_repo_id = source_repo_id
self.dest_repo_id = dest_repo_id
self.source_importer_id = source_importer_id
self.dest_importer_id = dest_importer_id
self.__association_manager = manager_factory.repo_unit_association_manager()
self.__association_query_manager = manager_factory.repo_unit_association_query_manager()
self.__importer_manager = manager_factory.repo_importer_manager()
def calculate_associated_type_ids(source_repo_id, associated_units):
if associated_units is not None:
associated_unit_type_ids = set([u['unit_type_id'] for u in associated_units])
else:
association_query_manager = manager_factory.repo_unit_association_query_manager()
# We may want to make a call here that only retrieves the unique
# type IDs instead of all of the units, but for now it doesn't exist
# and I'm not entirely sure this will be a huge problem.
all_units = association_query_manager.get_units(source_repo_id)
associated_unit_type_ids = set(u['unit_type_id'] for u in all_units)
return associated_unit_type_ids
def calculate_associated_type_ids(source_repo_id, associated_units):
if associated_units is not None:
associated_unit_type_ids = set([u['unit_type_id'] for u in associated_units])
else:
association_query_manager = manager_factory.repo_unit_association_query_manager()
# We may want to make a call here that only retrieves the unique
# type IDs instead of all of the units, but for now it doesn't exist
# and I'm not entirely sure this will be a huge problem.
all_units = association_query_manager.get_units(source_repo_id)
associated_unit_type_ids = set(u['unit_type_id'] for u in all_units)
return associated_unit_type_ids
def unassociate_by_criteria(self, repo_id, criteria, owner_type, owner_id):
"""
Unassociate units that are matched by the given criteria.
@param repo_id: identifies the repo
@type repo_id: str
@param criteria:
@param owner_type: category of the caller who created the association
@type owner_type: str
@param owner_id: identifies the call who created the association
@type owner_id: str
"""
association_query_manager = manager_factory.repo_unit_association_query_manager()
unassociate_units = association_query_manager.get_units(repo_id, criteria=criteria)
if len(unassociate_units) is 0:
return
unit_map = {} # maps unit_type_id to a list of unit_ids
for unit in unassociate_units:
id_list = unit_map.setdefault(unit['unit_type_id'], [])
id_list.append(unit['unit_id'])
collection = RepoContentUnit.get_collection()
repo_manager = manager_factory.repo_manager()
for unit_type_id, unit_ids in unit_map.items():
spec = {'repo_id': repo_id,
'unit_type_id': unit_type_id,
'unit_id': {'$in': unit_ids},
'owner_type': owner_type,
'owner_id': owner_id}
collection.remove(spec, safe=True)
unique_count = sum(1 for unit_id in unit_ids
if not self.association_exists(repo_id, unit_id, unit_type_id))
if not unique_count:
continue
repo_manager.update_unit_count(repo_id, -unique_count)
try:
repo_query_manager = manager_factory.repo_query_manager()
repo = repo_query_manager.get_repository(repo_id)
importer_manager = manager_factory.repo_importer_manager()
importer = importer_manager.get_importer(repo_id)
importer.remove_units(repo, unassociate_units)
except:
_LOG.exception('Exception informing importer for [%s] of unassociation' % repo_id)
def test_migrate_duplicates_doesnt_delete_from_source_repo(self):
"""
This tests the correct behavior when we try to change the repo_id on an object,
and end up causing
a duplicate error due to our uniqueness constraint. It also makes sure the units are not
deleted from
the source repository if they are in the source repository.
"""
# Let's put two units here with the same IDs with two different repo_ids, and the run the
# migration.
source_repo_group_id = add_unit('group', self.source_repo_id, ids.TYPE_ID_PKG_GROUP)
dest_repo_group_id = add_unit('group', self.dest_repo_id, ids.TYPE_ID_PKG_GROUP)
# Associate the source_repo_group_id with both source and destination repos
associate_unit(source_repo_group_id, self.source_repo_id, ids.TYPE_ID_PKG_GROUP)
associate_unit(source_repo_group_id, self.dest_repo_id, ids.TYPE_ID_PKG_GROUP)
associate_unit(dest_repo_group_id, self.dest_repo_id, ids.TYPE_ID_PKG_GROUP)
# Migrate should not cause a DuplicateKeyError
self.migration.migrate()
# Verify that both groups remain, because the migration should not have removed either
group_collection = types_db.type_units_collection(ids.TYPE_ID_PKG_GROUP)
all_groups = list(group_collection.find())
self.assertEqual(len(all_groups), 2)
self.assertEqual(
group_collection.find({'id': 'group', 'repo_id': self.dest_repo_id}).count(), 1)
self.assertEqual(
group_collection.find({'id': 'group', 'repo_id': self.source_repo_id}).count(), 1)
# Let's make sure that there are two associations, and that they are correct.
query_manager = factory.repo_unit_association_query_manager()
dest_units = query_manager.get_units(self.dest_repo_id)
self.assertEqual(len(dest_units), 1)
dest_unit = dest_units[0]
self.assertEqual(dest_unit['unit_type_id'], ids.TYPE_ID_PKG_GROUP)
self.assertEqual(dest_unit['unit_id'], dest_repo_group_id)
source_units = query_manager.get_units(self.source_repo_id)
self.assertEqual(len(source_units), 1)
source_unit = source_units[0]
self.assertEqual(source_unit['unit_type_id'], ids.TYPE_ID_PKG_GROUP)
self.assertEqual(source_unit['unit_id'], source_repo_group_id)
# Verify the repo counts
self.assertEqual(
Repo.get_collection().find({'id': 'source-repo'})[0]['content_unit_counts'],
{'package_group': 1})
self.assertEqual(Repo.get_collection().find({'id': 'dest-repo'})[0]['content_unit_counts'],
{'package_group': 1})
def get_repo_units(self, repo_id, content_type_id, additional_unit_fields=[]):
"""
Searches for units in the given repository with given content type
and returns a plugin unit with containing unit key and any additional
fields requested.
:param repo_id: repo id
:type repo_id: str
:param content_type_id: content type id of the units
:type content_type_id: str
:param additional_unit_fields: additional fields from the unit metadata to be added
in the result
:type additional_unit_fields: list of str
:return: list of unit instances
:rtype: list of pulp.plugins.model.Unit
"""
try:
query_manager = managers.repo_unit_association_query_manager()
criteria = UnitAssociationCriteria(type_ids=[content_type_id])
units = query_manager.get_units(repo_id, criteria)
# Get type definition and unit_key for given content type
type_def = types_db.type_definition(content_type_id)
key_list = type_def['unit_key']
# Return plugin units with unit_key and required metadata values for each unit
all_units = []
for unit in units:
unit_key = {}
metadata = {}
for k in key_list:
unit_key[k] = unit['metadata'].pop(k)
# Add unit_id and any additional unit fields requested by plugins
metadata['unit_id'] = unit.pop('unit_id', None)
for field in additional_unit_fields:
metadata[field] = unit['metadata'].pop(field, None)
u = Unit(content_type_id, unit_key, metadata, None)
all_units.append(u)
return all_units
except Exception, e:
_LOG.exception(_('Exception from server getting units from repo [%s]' % repo_id))
raise self.exception_class(e), None, sys.exc_info()[2]
def unassociate_by_criteria(self, repo_id, criteria, owner_type, owner_id, notify_plugins=True):
"""
Unassociate units that are matched by the given criteria.
@param repo_id: identifies the repo
@type repo_id: str
@param criteria:
@param owner_type: category of the caller who created the association
@type owner_type: str
@param owner_id: identifies the call who created the association
@type owner_id: str
@param notify_plugins: if true, relevant plugins will be informed of the
removal
@type notify_plugins: bool
"""
association_query_manager = manager_factory.repo_unit_association_query_manager()
unassociate_units = association_query_manager.get_units(repo_id, criteria=criteria)
if len(unassociate_units) is 0:
return
unit_map = {} # maps unit_type_id to a list of unit_ids
for unit in unassociate_units:
id_list = unit_map.setdefault(unit['unit_type_id'], [])
id_list.append(unit['unit_id'])
collection = RepoContentUnit.get_collection()
repo_manager = manager_factory.repo_manager()
for unit_type_id, unit_ids in unit_map.items():
spec = {'repo_id': repo_id,
'unit_type_id': unit_type_id,
'unit_id': {'$in': unit_ids},
'owner_type': owner_type,
'owner_id': owner_id}
collection.remove(spec, safe=True)
unique_count = sum(1 for unit_id in unit_ids
if not self.association_exists(repo_id, unit_id, unit_type_id))
if not unique_count:
continue
repo_manager.update_unit_count(repo_id, unit_type_id, -unique_count)
if notify_plugins:
remove_from_importer(repo_id, unassociate_units)
def test_migrate_groups(self):
# Setup
orig_group_id = add_unit('g1', self.source_repo_id,
ids.TYPE_ID_PKG_GROUP)
associate_unit(orig_group_id, self.source_repo_id,
ids.TYPE_ID_PKG_GROUP)
associate_unit(orig_group_id, self.dest_repo_id, ids.TYPE_ID_PKG_GROUP)
# Test
self.migration.migrate()
# Verify
# Verify a new group was created with the correct metadata
group_coll = types_db.type_units_collection(ids.TYPE_ID_PKG_GROUP)
all_groups = group_coll.find({}).sort('repo_id', 1)
self.assertEqual(2, all_groups.count())
dest_group = all_groups[0] # ordered by ID, this will be first
self.assertEqual(dest_group['id'], 'g1')
self.assertEqual(dest_group['repo_id'], self.dest_repo_id)
source_group = all_groups[1]
self.assertEqual(source_group['id'], 'g1')
self.assertEqual(source_group['repo_id'], self.source_repo_id)
# Verify the associations
query_manager = factory.repo_unit_association_query_manager()
source_units = query_manager.get_units(self.source_repo_id)
self.assertEqual(1, len(source_units))
self.assertEqual(source_units[0]['unit_type_id'],
ids.TYPE_ID_PKG_GROUP)
self.assertEqual(source_units[0]['unit_id'], source_group['_id'])
dest_units = query_manager.get_units(self.dest_repo_id)
self.assertEqual(1, len(dest_units))
self.assertEqual(dest_units[0]['unit_type_id'], ids.TYPE_ID_PKG_GROUP)
self.assertEqual(dest_units[0]['unit_id'], dest_group['_id'])
def test_migrate_duplicates(self):
"""
This tests the correct behavior when we try to change the repo_id on an object,
and end up causing
a duplicate error due to our uniqueness constraint.
"""
# Let's put two units here with the same IDs with two different repo_ids, and the run the
# migration.
source_repo_group_id = add_unit('group', self.source_repo_id, ids.TYPE_ID_PKG_GROUP)
dest_repo_group_id = add_unit('group', self.dest_repo_id, ids.TYPE_ID_PKG_GROUP)
associate_unit(source_repo_group_id, self.dest_repo_id, ids.TYPE_ID_PKG_GROUP)
associate_unit(dest_repo_group_id, self.dest_repo_id, ids.TYPE_ID_PKG_GROUP)
# Migrate should not cause a DuplicateKeyError
self.migration.migrate()
# Verify that both groups remain.
group_collection = types_db.type_units_collection(ids.TYPE_ID_PKG_GROUP)
all_groups = list(group_collection.find())
self.assertEqual(len(all_groups), 2)
self.assertEqual(
group_collection.find({'id': 'group', 'repo_id': self.dest_repo_id}).count(), 1)
self.assertEqual(
group_collection.find({'id': 'group', 'repo_id': self.source_repo_id}).count(), 1)
# Let's make sure that the dest group is associated, but not the source one
query_manager = factory.repo_unit_association_query_manager()
dest_units = query_manager.get_units(self.dest_repo_id)
self.assertEqual(len(dest_units), 1)
dest_unit = dest_units[0]
self.assertEqual(dest_unit['unit_type_id'], ids.TYPE_ID_PKG_GROUP)
self.assertEqual(dest_unit['unit_id'], dest_repo_group_id)
self.assertEqual(query_manager.get_units(self.source_repo_id), [])
# Verify the repo counts
source = model.Repository.objects.get(repo_id='source-repo')
self.assertDictEqual(source.content_unit_counts, {})
dest = model.Repository.objects.get(repo_id='dest-repo')
self.assertDictEqual(dest.content_unit_counts, {'package_group': 1})
def __init__(self, source_repo_id, dest_repo_id, source_importer_id,
dest_importer_id, association_owner_type,
association_owner_id):
"""
:param source_repo_id: ID of the repository from which units are being copied
:type source_repo_id: str
:param dest_repo_id: ID of the repository into which units are being copied
:type dest_repo_id: str
:param source_importer_id: ID of the importer on the source repository
:type source_importer_id: str
:param dest_importer_id: ID of the importer on the destination repository
:type dest_importer_id: str
:param association_owner_type: distinguishes the owner when creating an
association through this conduit
:type association_owner_type: str
:param association_owner_id: specific ID of the owner when creating an
association through this conduit
:type association_owner_id: str
"""
ImporterScratchPadMixin.__init__(self, dest_repo_id, dest_importer_id)
RepoScratchPadMixin.__init__(self, dest_repo_id,
ImporterConduitException)
SearchUnitsMixin.__init__(self, ImporterConduitException)
AddUnitMixin.__init__(self, dest_repo_id, dest_importer_id,
association_owner_type, association_owner_id)
self.source_repo_id = source_repo_id
self.dest_repo_id = dest_repo_id
self.source_importer_id = source_importer_id
self.dest_importer_id = dest_importer_id
self.association_owner_type = association_owner_type
self.association_owner_id = association_owner_id
self.__association_manager = manager_factory.repo_unit_association_manager(
)
self.__association_query_manager = manager_factory.repo_unit_association_query_manager(
)
self.__importer_manager = manager_factory.repo_importer_manager()
def __populate_units(self, unit_ids_by_type, repo_ids):
"""
Parse a dictionary of unit ids keyed by content type id and populate units for each type
if the criteria is empty.
:param unit_ids_by_type: dictionary of <content type id> : <list of unit ids>
:type unit_ids_by_type: dict
:return: if units are specified, return the corresponding units. If unit_ids_by_type dict
is empty, return unit ids corresponging to all units in given repo ids.
If unit ids list for a particular unit type is empty, return all unit ids
in given repo ids with that unit type.
:rtype: dict
"""
repo_unit_association_query_manager = managers.repo_unit_association_query_manager()
result_units = {}
if unit_ids_by_type is not None:
for unit_type_id, unit_ids in unit_ids_by_type.items():
# Get unit type specific collection
if not unit_ids:
# If unit_list is empty for a unit_type, consider all units of specific type
criteria = Criteria(filters={"repo_id": {"$in": repo_ids}, "unit_type_id":unit_type_id},
fields=['unit_id'])
repo_units = repo_unit_association_query_manager.find_by_criteria(criteria)
pulp_unit_ids = [u['unit_id'] for u in repo_units]
result_units.setdefault(unit_type_id, []).extend(list(set(pulp_unit_ids)))
else:
result_units.setdefault(unit_type_id, []).extend(unit_ids)
else:
# If units are not specified, consider all units in given repos.
all_unit_type_ids = content_types_db.all_type_ids()
for unit_type_id in all_unit_type_ids:
criteria = Criteria(filters={"repo_id": {"$in": repo_ids}, "unit_type_id":unit_type_id},
fields=['unit_id'])
repo_units = repo_unit_association_query_manager.find_by_criteria(criteria)
pulp_unit_ids = [u['unit_id'] for u in repo_units]
result_units.setdefault(unit_type_id, []).extend(list(set(pulp_unit_ids)))
return result_units
def test_migrate_category(self):
# Setup
orig_cat_id = add_unit('c1', self.source_repo_id,
ids.TYPE_ID_PKG_CATEGORY)
associate_unit(orig_cat_id, self.source_repo_id,
ids.TYPE_ID_PKG_CATEGORY)
associate_unit(orig_cat_id, self.dest_repo_id,
ids.TYPE_ID_PKG_CATEGORY)
# Test
self.migration.migrate()
group_coll = types_db.type_units_collection(ids.TYPE_ID_PKG_CATEGORY)
all_cats = group_coll.find({}).sort('repo_id', 1)
self.assertEqual(2, all_cats.count())
dest_cat = all_cats[0] # ordered by ID, this will be first
self.assertEqual(dest_cat['id'], 'c1')
self.assertEqual(dest_cat['repo_id'], self.dest_repo_id)
source_cat = all_cats[1]
self.assertEqual(source_cat['id'], 'c1')
self.assertEqual(source_cat['repo_id'], self.source_repo_id)
# Verify the associations
query_manager = factory.repo_unit_association_query_manager()
source_units = query_manager.get_units(self.source_repo_id)
self.assertEqual(1, len(source_units))
self.assertEqual(source_units[0]['unit_type_id'],
ids.TYPE_ID_PKG_CATEGORY)
self.assertEqual(source_units[0]['unit_id'], source_cat['_id'])
dest_units = query_manager.get_units(self.dest_repo_id)
self.assertEqual(1, len(dest_units))
self.assertEqual(dest_units[0]['unit_type_id'],
ids.TYPE_ID_PKG_CATEGORY)
self.assertEqual(dest_units[0]['unit_id'], dest_cat['_id'])
def verify(self, num_units=PluginTestBase.NUM_UNITS):
# repository
manager = managers.repo_query_manager()
manager.get_repository(self.REPO_ID)
# importer
manager = managers.repo_importer_manager()
importer = manager.get_importer(self.REPO_ID)
manifest_url = importer['config']['manifest_url']
self.assertTrue(manifest_url.endswith('%s/units.json.gz' % self.REPO_ID))
# distributor
manager = managers.repo_distributor_manager()
distributor = manager.get_distributor(self.REPO_ID, CITRUS_DISTRUBUTOR)
protocol = distributor['config']['protocol']
self.assertEqual(protocol, 'file')
alias = distributor['config'][protocol]['alias']
self.assertEqual(alias[0], self.upfs)
self.assertEqual(alias[1], self.upfs)
# check units
manager = managers.repo_unit_association_query_manager()
units = manager.get_units(self.REPO_ID)
units = dict([(u['metadata']['N'], u) for u in units])
self.assertEqual(len(units), num_units)
for n in range(0, num_units):
unit = units[n]
unit_id = self.UNIT_ID % n
metadata = unit['metadata']
storage_path = metadata['_storage_path'].replace('//', '/')
self.assertEqual(unit['unit_type_id'], self.UNIT_TYPE_ID)
self.assertEqual(unit['repo_id'], self.REPO_ID)
self.assertEqual(unit['owner_id'], CITRUS_IMPORTER)
file = '.'.join((unit_id, self.UNIT_TYPE_ID))
self.assertEqual(storage_path, os.path.join(self.downfs, 'content', file))
self.assertTrue(os.path.exists(storage_path))
fp = open(storage_path)
content = fp.read()
fp.close()
self.assertEqual(content, unit_id)
def test_migrate_groups(self):
# Setup
orig_group_id = add_unit('g1', self.source_repo_id, ids.TYPE_ID_PKG_GROUP)
associate_unit(orig_group_id, self.source_repo_id, ids.TYPE_ID_PKG_GROUP)
associate_unit(orig_group_id, self.dest_repo_id, ids.TYPE_ID_PKG_GROUP)
# Test
self.migration.migrate()
# Verify
# Verify a new group was created with the correct metadata
group_coll = types_db.type_units_collection(ids.TYPE_ID_PKG_GROUP)
all_groups = group_coll.find({}).sort('repo_id', 1)
self.assertEqual(2, all_groups.count())
dest_group = all_groups[0] # ordered by ID, this will be first
self.assertEqual(dest_group['id'], 'g1')
self.assertEqual(dest_group['repo_id'], self.dest_repo_id)
source_group = all_groups[1]
self.assertEqual(source_group['id'], 'g1')
self.assertEqual(source_group['repo_id'], self.source_repo_id)
# Verify the associations
query_manager = factory.repo_unit_association_query_manager()
source_units = query_manager.get_units(self.source_repo_id)
self.assertEqual(1, len(source_units))
self.assertEqual(source_units[0]['unit_type_id'], ids.TYPE_ID_PKG_GROUP)
self.assertEqual(source_units[0]['unit_id'], source_group['_id'])
dest_units = query_manager.get_units(self.dest_repo_id)
self.assertEqual(1, len(dest_units))
self.assertEqual(dest_units[0]['unit_type_id'], ids.TYPE_ID_PKG_GROUP)
self.assertEqual(dest_units[0]['unit_id'], dest_group['_id'])
import copy
import logging
import pymongo.errors
from pulp.server.managers import factory
from pulp.server.managers.repo.cud import RepoContentUnit
from pulp.server.managers.repo.unit_association_query import UnitAssociationCriteria
from pulp_rpm.common import ids
_log = logging.getLogger('pulp')
# Initialize plugin loader api and other managers
factory.initialize()
ass_query_mgr = factory.repo_unit_association_query_manager()
ass_mgr = factory.repo_unit_association_manager()
content_mgr = factory.content_manager()
repo_mgr = factory.repo_manager()
def _get_repos():
"""
Lookups all the yum based repos in pulp.
@return a list of repoids
"""
repos = factory.repo_query_manager().find_with_importer_type(
"yum_importer")
if not repos:
_log.debug("No repos found to perform db migrate")
return []
