def _fix_pkg_group_category_repoid(repoid, typeid):
"""
Looks up units with in a repo and validate if the repoid in the unit metadata matches the repo
the unit is associated with. If they dont match,
* take a deep copy of the pkg group or category unit
* create(save) new unit with fixed repoid
* re-associate new unit with the repo
"""
units = ass_query_mgr.get_units(repo_id=repoid,
criteria=UnitAssociationCriteria(type_ids=typeid))
for unit in units:
if unit['metadata']['repo_id'] != repoid:
_log.debug("Found unit %s to migrate" % unit['id'])
# take a copy of the unit and fix the repoid
new_unit_metadata = _safe_copy_unit(unit['metadata'])
new_unit_metadata['repo_id'] = repoid
try:
new_unit_id = content_mgr.add_content_unit(content_type=typeid, unit_id=None,
unit_metadata=new_unit_metadata)
# Grab the association doc itself from the DB directly
association = RepoContentUnit.get_collection().find_one({'_id': unit['_id']})
# Update to point to the new unit
association['unit_id'] = new_unit_id
# Save it back to the DB
RepoContentUnit.get_collection().save(association)
except pymongo.errors.DuplicateKeyError:
# If migrating this Unit to have the correct repo_id causes a duplicate,
# then there already
# is a Unit that has the correct metadata in place in this repository. Because of
# this, we
# should remove the association of the unit with the repository
RepoContentUnit.get_collection().remove({'_id': unit['_id']})
# Since we removed a Unit from the repo, we should decrement the repo unit count
repo_controller.update_unit_count(repoid, typeid, -1)
def unassociate_by_criteria(repo_id, criteria, notify_plugins=True):
"""
Unassociate units that are matched by the given criteria.
:param repo_id: identifies the repo
:type repo_id: str
:param criteria:
:param notify_plugins: if true, relevant plugins will be informed of the removal
:type notify_plugins: bool
"""
criteria = UnitAssociationCriteria.from_dict(criteria)
association_query_manager = manager_factory.repo_unit_association_query_manager(
)
unassociate_units = association_query_manager.get_units(
repo_id, criteria=criteria)
if len(unassociate_units) == 0:
return {}
unit_map = {} # maps unit_type_id to a list of unit_ids
for unit in unassociate_units:
id_list = unit_map.setdefault(unit['unit_type_id'], [])
id_list.append(unit['unit_id'])
collection = RepoContentUnit.get_collection()
for unit_type_id, unit_ids in unit_map.items():
spec = {
'repo_id': repo_id,
'unit_type_id': unit_type_id,
'unit_id': {
'$in': unit_ids
}
}
collection.remove(spec)
unique_count = sum(
1 for unit_id in unit_ids
if not RepoUnitAssociationManager.association_exists(
repo_id, unit_id, unit_type_id))
if not unique_count:
continue
repo_controller.update_unit_count(repo_id, unit_type_id,
-unique_count)
repo_controller.update_last_unit_removed(repo_id)
# Convert the units into transfer units. This happens regardless of whether or not
# the plugin will be notified as it's used to generate the return result,
transfer_units = create_transfer_units(unassociate_units)
if notify_plugins:
remove_from_importer(repo_id, transfer_units)
# Match the return type/format as copy
serializable_units = [u.to_id_dict() for u in transfer_units]
return {'units_successful': serializable_units}
def unassociate_by_criteria(cls, repo_id, criteria, notify_plugins=True):
"""
Unassociate units that are matched by the given criteria.
:param repo_id: identifies the repo
:type repo_id: str
:param criteria:
:param notify_plugins: if true, relevant plugins will be informed of the removal
:type notify_plugins: bool
"""
criteria = UnitAssociationCriteria.from_dict(criteria)
repo = model.Repository.objects.get_repo_or_missing_resource(repo_id)
unassociate_units = load_associated_units(repo_id, criteria)
if len(unassociate_units) == 0:
return {}
# Convert the units into transfer units. This happens regardless of whether or not
# the plugin will be notified as it's used to generate the return result.
# If all source types have been converted to mongo, search via new style.
repo_unit_types = set(repo.content_unit_counts.keys())
if repo_unit_types.issubset(set(plugin_api.list_unit_models())):
transfer_units = list(cls._units_from_criteria(repo, criteria))
else:
transfer_units = None
if unassociate_units is not None:
transfer_units = list(create_transfer_units(unassociate_units))
if notify_plugins:
remove_from_importer(repo_id, transfer_units)
unit_map = {} # maps unit_type_id to a list of unit_ids
for unit in unassociate_units:
id_list = unit_map.setdefault(unit["unit_type_id"], [])
id_list.append(unit["unit_id"])
collection = RepoContentUnit.get_collection()
for unit_type_id, unit_ids in unit_map.items():
spec = {"repo_id": repo_id, "unit_type_id": unit_type_id, "unit_id": {"$in": unit_ids}}
collection.remove(spec)
unique_count = sum(
1
for unit_id in unit_ids
if not RepoUnitAssociationManager.association_exists(repo_id, unit_id, unit_type_id)
)
if not unique_count:
continue
repo_controller.update_unit_count(repo_id, unit_type_id, -unique_count)
repo_controller.update_last_unit_removed(repo_id)
# Match the return type/format as copy
serializable_units = [u.to_id_dict() for u in transfer_units]
return {"units_successful": serializable_units}
def associate_unit_by_id(self,
repo_id,
unit_type_id,
unit_id,
update_repo_metadata=True):
"""
Creates an association between the given repository and content unit.
If there is already an association between the given repo and content
unit where all other metadata matches the input to this method,
this call has no effect.
Both repo and unit must exist in the database prior to this call,
however this call will not verify that for performance reasons. Care
should be taken by the caller to preserve the data integrity.
@param repo_id: identifies the repo
@type repo_id: str
@param unit_type_id: identifies the type of unit being added
@type unit_type_id: str
@param unit_id: uniquely identifies the unit within the given type
@type unit_id: str
@param update_repo_metadata: if True, updates the unit association count
after the new association is made. The last
unit added field will also be updated. Set this
to False when doing bulk associations, and
make one call to update the count at the end.
defaults to True
@type update_repo_metadata: bool
@raise InvalidType: if the given owner type is not of the valid enumeration
"""
# If the association already exists, no need to do anything else
spec = {
'repo_id': repo_id,
'unit_id': unit_id,
'unit_type_id': unit_type_id
}
existing_association = RepoContentUnit.get_collection().find_one(spec)
if existing_association is not None:
return
similar_exists = False
if update_repo_metadata:
similar_exists = RepoUnitAssociationManager.association_exists(
repo_id, unit_id, unit_type_id)
# Create the database entry
association = RepoContentUnit(repo_id, unit_id, unit_type_id)
RepoContentUnit.get_collection().save(association, safe=True)
# update the count and times of associated units on the repo object
if update_repo_metadata and not similar_exists:
repo_controller.update_unit_count(repo_id, unit_type_id, 1)
repo_controller.update_last_unit_added(repo_id)
def associate_unit_by_id(self, repo_id, unit_type_id, unit_id, update_repo_metadata=True):
"""
Creates an association between the given repository and content unit.
If there is already an association between the given repo and content
unit where all other metadata matches the input to this method,
this call has no effect.
Both repo and unit must exist in the database prior to this call,
however this call will not verify that for performance reasons. Care
should be taken by the caller to preserve the data integrity.
@param repo_id: identifies the repo
@type repo_id: str
@param unit_type_id: identifies the type of unit being added
@type unit_type_id: str
@param unit_id: uniquely identifies the unit within the given type
@type unit_id: str
@param update_repo_metadata: if True, updates the unit association count
after the new association is made. The last
unit added field will also be updated. Set this
to False when doing bulk associations, and
make one call to update the count at the end.
defaults to True
@type update_repo_metadata: bool
@raise InvalidType: if the given owner type is not of the valid enumeration
"""
# If the association already exists, no need to do anything else
spec = {'repo_id': repo_id,
'unit_id': unit_id,
'unit_type_id': unit_type_id}
existing_association = RepoContentUnit.get_collection().find_one(spec)
if existing_association is not None:
return
similar_exists = False
if update_repo_metadata:
similar_exists = RepoUnitAssociationManager.association_exists(repo_id, unit_id,
unit_type_id)
# Create the database entry
association = RepoContentUnit(repo_id, unit_id, unit_type_id)
RepoContentUnit.get_collection().save(association, safe=True)
# update the count and times of associated units on the repo object
if update_repo_metadata and not similar_exists:
repo_controller.update_unit_count(repo_id, unit_type_id, 1)
repo_controller.update_last_unit_added(repo_id)
def unassociate_by_criteria(repo_id, criteria, notify_plugins=True):
"""
Unassociate units that are matched by the given criteria.
:param repo_id: identifies the repo
:type repo_id: str
:param criteria:
:param notify_plugins: if true, relevant plugins will be informed of the removal
:type notify_plugins: bool
"""
association_query_manager = manager_factory.repo_unit_association_query_manager()
unassociate_units = association_query_manager.get_units(repo_id, criteria=criteria)
if len(unassociate_units) == 0:
return {}
unit_map = {} # maps unit_type_id to a list of unit_ids
for unit in unassociate_units:
id_list = unit_map.setdefault(unit['unit_type_id'], [])
id_list.append(unit['unit_id'])
collection = RepoContentUnit.get_collection()
for unit_type_id, unit_ids in unit_map.items():
spec = {'repo_id': repo_id,
'unit_type_id': unit_type_id,
'unit_id': {'$in': unit_ids}
}
collection.remove(spec, safe=True)
unique_count = sum(
1 for unit_id in unit_ids if not RepoUnitAssociationManager.association_exists(
repo_id, unit_id, unit_type_id))
if not unique_count:
continue
repo_controller.update_unit_count(repo_id, unit_type_id, -unique_count)
repo_controller.update_last_unit_removed(repo_id)
# Convert the units into transfer units. This happens regardless of whether or not
# the plugin will be notified as it's used to generate the return result,
unit_type_ids = calculate_associated_type_ids(repo_id, unassociate_units)
transfer_units = create_transfer_units(unassociate_units, unit_type_ids)
if notify_plugins:
remove_from_importer(repo_id, transfer_units)
# Match the return type/format as copy
serializable_units = [u.to_id_dict() for u in transfer_units]
return {'units_successful': serializable_units}
def associate_all_by_ids(self, repo_id, unit_type_id, unit_id_list):
"""
Creates multiple associations between the given repo and content units.
See associate_unit_by_id for semantics.
@param repo_id: identifies the repo
@type repo_id: str
@param unit_type_id: identifies the type of unit being added
@type unit_type_id: str
@param unit_id_list: list or generator of unique identifiers for units within the given type
@type unit_id_list: list or generator of str
:return: number of new units added to the repo
:rtype: int
@raise InvalidType: if the given owner type is not of the valid enumeration
"""
# There may be a way to batch this in mongo which would be ideal for a
# bulk operation like this. But for deadline purposes, this call will
# simply loop and call the single method.
unique_count = 0
for unit_id in unit_id_list:
if not RepoUnitAssociationManager.association_exists(
repo_id, unit_id, unit_type_id):
unique_count += 1
self.associate_unit_by_id(repo_id, unit_type_id, unit_id, False)
# update the count of associated units on the repo object
if unique_count:
repo_controller.update_unit_count(repo_id, unit_type_id,
unique_count)
repo_controller.update_last_unit_added(repo_id)
return unique_count
def _fix_pkg_group_category_repoid(repoid, typeid):
"""
Looks up units with in a repo and validate if the repoid in the unit metadata matches the repo
the unit is associated with. If they dont match,
* take a deep copy of the pkg group or category unit
* create(save) new unit with fixed repoid
* re-associate new unit with the repo
"""
units = ass_query_mgr.get_units(
repo_id=repoid, criteria=UnitAssociationCriteria(type_ids=typeid))
for unit in units:
if unit['metadata']['repo_id'] != repoid:
_log.debug("Found unit %s to migrate" % unit['id'])
# take a copy of the unit and fix the repoid
new_unit_metadata = _safe_copy_unit(unit['metadata'])
new_unit_metadata['repo_id'] = repoid
try:
new_unit_id = content_mgr.add_content_unit(
content_type=typeid,
unit_id=None,
unit_metadata=new_unit_metadata)
# Grab the association doc itself from the DB directly
association = RepoContentUnit.get_collection().find_one(
{'_id': unit['_id']})
# Update to point to the new unit
association['unit_id'] = new_unit_id
# Save it back to the DB
RepoContentUnit.get_collection().save(association)
except pymongo.errors.DuplicateKeyError:
# If migrating this Unit to have the correct repo_id causes a duplicate,
# then there already
# is a Unit that has the correct metadata in place in this repository. Because of
# this, we
# should remove the association of the unit with the repository
RepoContentUnit.get_collection().remove({'_id': unit['_id']})
# Since we removed a Unit from the repo, we should decrement the repo unit count
repo_controller.update_unit_count(repoid, typeid, -1)
def associate_all_by_ids(self, repo_id, unit_type_id, unit_id_list):
"""
Creates multiple associations between the given repo and content units.
See associate_unit_by_id for semantics.
@param repo_id: identifies the repo
@type repo_id: str
@param unit_type_id: identifies the type of unit being added
@type unit_type_id: str
@param unit_id_list: list or generator of unique identifiers for units within the given type
@type unit_id_list: list or generator of str
:return: number of new units added to the repo
:rtype: int
@raise InvalidType: if the given owner type is not of the valid enumeration
"""
# There may be a way to batch this in mongo which would be ideal for a
# bulk operation like this. But for deadline purposes, this call will
# simply loop and call the single method.
unique_count = 0
for unit_id in unit_id_list:
if not RepoUnitAssociationManager.association_exists(repo_id, unit_id, unit_type_id):
unique_count += 1
self.associate_unit_by_id(repo_id, unit_type_id, unit_id, False)
# update the count of associated units on the repo object
if unique_count:
repo_controller.update_unit_count(repo_id, unit_type_id, unique_count)
repo_controller.update_last_unit_added(repo_id)
return unique_count
def unassociate_by_criteria(repo_id, criteria, notify_plugins=True):
"""
Unassociate units that are matched by the given criteria.
:param repo_id: identifies the repo
:type repo_id: str
:param criteria:
:param notify_plugins: if true, relevant plugins will be informed of the removal
:type notify_plugins: bool
"""
criteria = UnitAssociationCriteria.from_dict(criteria)
repo = model.Repository.objects.get_repo_or_missing_resource(repo_id)
unassociate_units = load_associated_units(repo_id, criteria)
if len(unassociate_units) == 0:
return {}
# Convert the units into transfer units. This happens regardless of whether or not
# the plugin will be notified as it's used to generate the return result.
# If all source types have been converted to mongo, search via new style.
repo_unit_types = set(repo.content_unit_counts.keys())
if repo_unit_types.issubset(set(plugin_api.list_unit_models())):
transfer_units = list(RepoUnitAssociationManager._units_from_criteria(repo, criteria))
else:
transfer_units = None
if unassociate_units is not None:
transfer_units = list(create_transfer_units(unassociate_units))
if notify_plugins:
remove_from_importer(repo_id, transfer_units)
unit_map = {} # maps unit_type_id to a list of unit_ids
for unit in unassociate_units:
id_list = unit_map.setdefault(unit['unit_type_id'], [])
id_list.append(unit['unit_id'])
collection = RepoContentUnit.get_collection()
for unit_type_id, unit_ids in unit_map.items():
spec = {
'repo_id': repo_id,
'unit_type_id': unit_type_id,
'unit_id': {'$in': unit_ids}
}
collection.remove(spec)
unique_count = sum(
1 for unit_id in unit_ids if not RepoUnitAssociationManager.association_exists(
repo_id, unit_id, unit_type_id))
if not unique_count:
continue
repo_controller.update_unit_count(repo_id, unit_type_id, -unique_count)
repo_controller.update_last_unit_removed(repo_id)
# Match the return type/format as copy
serializable_units = [u.to_id_dict() for u in transfer_units]
return {'units_successful': serializable_units}
