def associated_unit(association, unit):
"""
Create a dictionary that is a composite of a unit association and the unit.
:param association: A unit association DB record.
:type association: dict
:param unit: A DB unit record.
:type unit: dict
:return: A composite of the unit association and the unit.
:rtype: dict
"""
unit_key = {}
unit_id = unit.pop('_id')
type_id = association['unit_type_id']
for key in get_unit_key_fields_for_type(type_id):
unit_key[key] = unit.pop(key, None)
storage_dir = pulp_conf.get('server', 'storage_dir')
storage_path = unit.pop('_storage_path', None)
last_updated = unit.pop('_last_updated', 0.0)
if storage_path:
relative_path = storage_path[len(storage_dir):].lstrip('/')
else:
relative_path = None
return dict(unit_id=unit_id,
type_id=type_id,
unit_key=unit_key,
storage_path=storage_path,
relative_path=relative_path,
last_updated=last_updated,
metadata=unit)
def find_unit_by_unit_key(self, type_id, unit_key):
"""
Finds a unit based on its unit key. If more than one unit comes back,
an exception will be raised.
@param type_id: indicates the type of units being retrieved
@type type_id: str
@param unit_key: the unit key for the unit
@type unit_key: dict
@return: a single unit
@rtype: L{Unit}
"""
content_query_manager = manager_factory.content_query_manager()
try:
# this call returns a unit or raises MissingResource
existing_unit = content_query_manager.get_content_unit_by_keys_dict(
type_id, unit_key)
unit_key_fields = units_controller.get_unit_key_fields_for_type(
type_id)
plugin_unit = common_utils.to_plugin_unit(existing_unit, type_id,
unit_key_fields)
return plugin_unit
except pulp_exceptions.MissingResource:
return None
def search_all_units(self, type_id, criteria):
"""
Searches for units of a given type in the server, regardless of their
associations to any repositories.
@param type_id: indicates the type of units being retrieved
@type type_id: str
@param criteria: used to query which units are returned
@type criteria: pulp.server.db.model.criteria.Criteria
@return: list of unit instances
@rtype: list of L{Unit}
"""
try:
query_manager = manager_factory.content_query_manager()
units = query_manager.find_by_criteria(type_id, criteria)
unit_key_fields = units_controller.get_unit_key_fields_for_type(type_id)
transfer_units = []
for pulp_unit in units:
u = common_utils.to_plugin_unit(pulp_unit, type_id, unit_key_fields)
transfer_units.append(u)
return transfer_units
except Exception, e:
_logger.exception('Exception from server requesting all units of type [%s]' % type_id)
raise self.exception_class(e), None, sys.exc_info()[2]
def get_content_unit_keys(self, content_type, unit_ids):
"""
Return the keys and values that will uniquely identify the content units
that match the given unique ids.
@param content_type: unique id of content collection
@type content_type: str
@param unit_ids: list of unique content unit ids
@type unit_ids: list of str's
@return: two tuples of the same length, one of ids the second of key dicts
the same index in each tuple corresponds to a single content unit
@rtype: tuple of (possibly empty) tuples
"""
try:
key_fields = units_controller.get_unit_key_fields_for_type(
content_type)
except ValueError:
raise InvalidValue(['content_type'])
all_fields = ['_id']
_flatten_keys(all_fields, key_fields)
collection = content_types_db.type_units_collection(content_type)
cursor = collection.find({'_id': {
'$in': unit_ids
}},
projection=all_fields)
dicts = tuple(dict(d) for d in cursor)
ids = tuple(d.pop('_id') for d in dicts)
return (ids, dicts)
def associated_unit(association, unit):
"""
Create a dictionary that is a composite of a unit association and the unit.
:param association: A unit association DB record.
:type association: dict
:param unit: A DB unit record.
:type unit: dict
:return: A composite of the unit association and the unit.
:rtype: dict
"""
unit_key = {}
unit_id = unit.pop('_id')
type_id = association['unit_type_id']
for key in get_unit_key_fields_for_type(type_id):
unit_key[key] = unit.pop(key, None)
storage_dir = pulp_conf.get('server', 'storage_dir')
storage_path = unit.pop('_storage_path', None)
last_updated = unit.pop('_last_updated', 0.0)
if storage_path:
relative_path = storage_path[len(storage_dir):].lstrip('/')
else:
relative_path = None
return dict(
unit_id=unit_id,
type_id=type_id,
unit_key=unit_key,
storage_path=storage_path,
relative_path=relative_path,
last_updated=last_updated,
metadata=unit)
def get_repo_units(self,
repo_id,
content_type_id,
additional_unit_fields=None):
"""
Searches for units in the given repository with given content type
and returns a plugin unit containing unit id, unit key and any additional
fields requested.
:param repo_id: repo id
:type repo_id: str
:param content_type_id: content type id of the units
:type content_type_id: str
:param additional_unit_fields: additional fields from the unit metadata to be added
in the result
:type additional_unit_fields: list of str
:return: list of unit instances
:rtype: list of pulp.plugins.model.Unit
"""
additional_unit_fields = additional_unit_fields or []
try:
unit_key_fields = units_controller.get_unit_key_fields_for_type(
content_type_id)
# Query repo association manager to get all units of given type
# associated with given repo. Limit data by requesting only the fields
# that are needed.
query_manager = managers.repo_unit_association_query_manager()
unit_fields = list(unit_key_fields) + list(additional_unit_fields)
criteria = UnitAssociationCriteria(association_fields=['unit_id'],
unit_fields=unit_fields)
units = query_manager.get_units_by_type(repo_id, content_type_id,
criteria)
# Convert units to plugin units with unit_key and required metadata values for each unit
all_units = []
for unit in units:
unit_key = {}
metadata = {}
for k in unit_key_fields:
unit_key[k] = unit['metadata'].pop(k)
# Add unit_id and any additional unit fields requested by plugins
metadata['unit_id'] = unit.pop('unit_id')
for field in additional_unit_fields:
metadata[field] = unit['metadata'].pop(field, None)
u = Unit(content_type_id, unit_key, metadata, None)
all_units.append(u)
return all_units
except Exception, e:
_logger.exception(
_('Exception from server getting units from repo [%s]' %
repo_id))
raise self.exception_class(e), None, sys.exc_info()[2]
def test_returns_from_typedb(self, mock_type_def, mock_get_model):
"""
test when the requested type is defined the old way
"""
mock_get_model.return_value = None
mock_type_def.return_value = {'unit_key': ['id']}
ret = units_controller.get_unit_key_fields_for_type('faketype')
self.assertEqual(ret, ('id',))
def test_returns_from_model(self, mock_type_def, mock_get_model):
"""
test when the requested type is a mongoengine model
"""
mock_get_model.return_value = DemoModel
mock_type_def.return_value = None
ret = units_controller.get_unit_key_fields_for_type(DemoModel.type_id)
self.assertEqual(ret, DemoModel.unit_key_fields)
def test_returns_from_typedb(self, mock_type_def, mock_get_model):
"""
test when the requested type is defined the old way
"""
mock_get_model.return_value = None
mock_type_def.return_value = {'unit_key': ['id']}
ret = units_controller.get_unit_key_fields_for_type('faketype')
self.assertEqual(ret, ('id',))
def _transfer_object_generator():
unit_key_fields_cache = {}
for u in units:
type_id = u['unit_type_id']
if type_id not in unit_key_fields_cache:
fields = units_controller.get_unit_key_fields_for_type(type_id)
unit_key_fields_cache[type_id] = fields
yield common_utils.to_plugin_associated_unit(u, type_id,
unit_key_fields_cache[type_id])
def test_returns_from_model(self, mock_type_def, mock_get_model):
"""
test when the requested type is a mongoengine model
"""
mock_get_model.return_value = DemoModel
mock_type_def.return_value = None
ret = units_controller.get_unit_key_fields_for_type(DemoModel.type_id)
self.assertEqual(ret, DemoModel.unit_key_fields)
def get_repo_units(self, repo_id, content_type_id, additional_unit_fields=None):
"""
Searches for units in the given repository with given content type
and returns a plugin unit containing unit id, unit key and any additional
fields requested.
:param repo_id: repo id
:type repo_id: str
:param content_type_id: content type id of the units
:type content_type_id: str
:param additional_unit_fields: additional fields from the unit metadata to be added
in the result
:type additional_unit_fields: list of str
:return: list of unit instances
:rtype: list of pulp.plugins.model.Unit
"""
additional_unit_fields = additional_unit_fields or []
try:
unit_key_fields = units_controller.get_unit_key_fields_for_type(content_type_id)
serializer = units_controller.get_model_serializer_for_type(content_type_id)
# Query repo association manager to get all units of given type
# associated with given repo. Limit data by requesting only the fields
# that are needed.
query_manager = managers.repo_unit_association_query_manager()
unit_fields = list(unit_key_fields) + list(additional_unit_fields)
criteria = UnitAssociationCriteria(association_fields=['unit_id'],
unit_fields=unit_fields)
units = query_manager.get_units_by_type(repo_id, content_type_id, criteria)
# Convert units to plugin units with unit_key and required metadata values for each unit
all_units = []
for unit in units:
if serializer:
serializer.serialize(unit['metadata'])
unit_key = {}
metadata = {}
for k in unit_key_fields:
unit_key[k] = unit['metadata'].pop(k)
# Add unit_id and any additional unit fields requested by plugins
metadata['unit_id'] = unit.pop('unit_id')
for field in additional_unit_fields:
metadata[field] = unit['metadata'].pop(field, None)
u = Unit(content_type_id, unit_key, metadata, None)
all_units.append(u)
return all_units
except Exception, e:
_logger.exception(_('Exception from server getting units from repo [%s]' % repo_id))
raise self.exception_class(e), None, sys.exc_info()[2]
def create_transfer_units(associate_units):
unit_key_fields = {}
transfer_units = []
for unit in associate_units:
type_id = unit['unit_type_id']
if type_id not in unit_key_fields:
unit_key_fields[type_id] = units_controller.get_unit_key_fields_for_type(type_id)
u = conduit_common_utils.to_plugin_associated_unit(unit, type_id, unit_key_fields[type_id])
transfer_units.append(u)
return transfer_units
def create_transfer_units(associate_units):
unit_key_fields = {}
transfer_units = []
for unit in associate_units:
type_id = unit['unit_type_id']
if type_id not in unit_key_fields:
unit_key_fields[type_id] = units_controller.get_unit_key_fields_for_type(type_id)
u = conduit_common_utils.to_plugin_associated_unit(unit, type_id, unit_key_fields[type_id])
transfer_units.append(u)
return transfer_units
def _build_multi_keys_spec(content_type, unit_keys_dicts):
"""
Build a mongo db spec document for a query on the given content_type
collection out of multiple content unit key dictionaries.
:param content_type: unique id of the content type collection
:type content_type: str
:param unit_keys_dicts: list of key dictionaries whose key, value pairs can be
used as unique identifiers for a single content unit
:type unit_keys_dicts: list of dict
:return: mongo db spec document for locating documents in a collection
:rtype: dict
:raises ValueError: if any of the key dictionaries do not match the unique
fields of the collection
"""
# keys dicts validation constants
try:
unit_key_fields = units_controller.get_unit_key_fields_for_type(
content_type)
except ValueError:
raise InvalidValue(['content_type'])
key_fields = []
_flatten_keys(key_fields, unit_key_fields)
key_fields_set = set(key_fields)
extra_keys_msg = _(
'keys dictionary found with superfluous keys %(a)s, valid keys are %(b)s'
)
missing_keys_msg = _(
'keys dictionary missing keys %(a)s, required keys are %(b)s')
keys_errors = []
# Validate all of the keys in the unit_keys_dict
for keys_dict in unit_keys_dicts:
# keys dict validation
keys_dict_set = set(keys_dict)
extra_keys = keys_dict_set.difference(key_fields_set)
if extra_keys:
keys_errors.append(extra_keys_msg % {
'a': ','.join(extra_keys),
'b': ','.join(key_fields)
})
missing_keys = key_fields_set.difference(keys_dict_set)
if missing_keys:
keys_errors.append(missing_keys_msg % {
'a': ','.join(missing_keys),
'b': ','.join(key_fields)
})
if keys_errors:
value_error_msg = '\n'.join(keys_errors)
raise ValueError(value_error_msg)
# Build the spec
spec = {'$or': unit_keys_dicts}
return spec
def delete(self, request, content_type):
"""
Dispatch a delete_orphans_by_type task.
:param request: WSGI request object
:type request: django.core.handlers.wsgi.WSGIRequest
:param content_type: restrict the list of orphans to be deleted to this content type
:type content_type: str
:raises: OperationPostponed when an async operation is performed
:raises: MissingResource when the content type does not exist
"""
try:
# this tests if the type exists
units.get_unit_key_fields_for_type(content_type)
except ValueError:
raise MissingResource(content_type_id=content_type)
task_tags = [tags.resource_tag(tags.RESOURCE_CONTENT_UNIT_TYPE, 'orphans')]
async_task = content_orphan.delete_orphans_by_type.apply_async(
(content_type,), tags=task_tags
)
raise OperationPostponed(async_task)
def delete_orphans_by_type(content_type_id, content_unit_ids=None):
"""
Delete the orphaned content units for the given content type.
If the content_unit_ids parameter is not None, is acts as a filter of
the specific orphaned content units that may be deleted.
NOTE: this method deletes the content unit's bits from disk, if applicable.
:param content_type_id: id of the content type
:type content_type_id: basestring
:param content_unit_ids: list of content unit ids to delete; None means delete them all
:type content_unit_ids: iterable or None
:return: count of units deleted
:rtype: int
"""
content_units_collection = content_types_db.type_units_collection(
content_type_id)
content_model = plugin_api.get_unit_model_by_id(content_type_id)
try:
unit_key_fields = units_controller.get_unit_key_fields_for_type(
content_type_id)
except ValueError:
raise MissingResource(content_type_id=content_type_id)
fields = ('_id', '_storage_path') + unit_key_fields
count = 0
for content_unit in OrphanManager.generate_orphans_by_type(
content_type_id, fields=fields):
if content_unit_ids is not None and content_unit[
'_id'] not in content_unit_ids:
continue
model.LazyCatalogEntry.objects(
unit_id=content_unit['_id'],
unit_type_id=content_type_id).delete()
content_units_collection.remove(content_unit['_id'])
if hasattr(content_model, 'do_post_delete_actions'):
content_model.do_post_delete_actions(content_unit)
storage_path = content_unit.get('_storage_path', None)
if storage_path is not None:
OrphanManager.delete_orphaned_file(storage_path)
count += 1
return count
def delete(self, request, content_type):
"""
Dispatch a delete_orphans_by_type task.
:param request: WSGI request object
:type request: django.core.handlers.wsgi.WSGIRequest
:param content_type: restrict the list of orphans to be deleted to this content type
:type content_type: str
:raises: OperationPostponed when an async operation is performed
:raises: MissingResource when the content type does not exist
"""
try:
# this tests if the type exists
units.get_unit_key_fields_for_type(content_type)
except ValueError:
raise MissingResource(content_type_id=content_type)
task_tags = [
tags.resource_tag(tags.RESOURCE_CONTENT_UNIT_TYPE, 'orphans')
]
async_task = content_orphan.delete_orphans_by_type.apply_async(
(content_type, ), tags=task_tags)
raise OperationPostponed(async_task)
def _build_multi_keys_spec(content_type, unit_keys_dicts):
"""
Build a mongo db spec document for a query on the given content_type
collection out of multiple content unit key dictionaries.
:param content_type: unique id of the content type collection
:type content_type: str
:param unit_keys_dicts: list of key dictionaries whose key, value pairs can be
used as unique identifiers for a single content unit
:type unit_keys_dicts: list of dict
:return: mongo db spec document for locating documents in a collection
:rtype: dict
:raises ValueError: if any of the key dictionaries do not match the unique
fields of the collection
"""
# keys dicts validation constants
try:
unit_key_fields = units_controller.get_unit_key_fields_for_type(content_type)
except ValueError:
raise InvalidValue(['content_type'])
key_fields = []
_flatten_keys(key_fields, unit_key_fields)
key_fields_set = set(key_fields)
extra_keys_msg = _('keys dictionary found with superfluous keys %(a)s, valid keys are %(b)s')
missing_keys_msg = _('keys dictionary missing keys %(a)s, required keys are %(b)s')
keys_errors = []
# Validate all of the keys in the unit_keys_dict
for keys_dict in unit_keys_dicts:
# keys dict validation
keys_dict_set = set(keys_dict)
extra_keys = keys_dict_set.difference(key_fields_set)
if extra_keys:
keys_errors.append(extra_keys_msg % {'a': ','.join(extra_keys),
'b': ','.join(key_fields)})
missing_keys = key_fields_set.difference(keys_dict_set)
if missing_keys:
keys_errors.append(missing_keys_msg % {'a': ','.join(missing_keys),
'b': ','.join(key_fields)})
if keys_errors:
value_error_msg = '\n'.join(keys_errors)
raise ValueError(value_error_msg)
# Build the spec
spec = {'$or': unit_keys_dicts}
return spec
def _associated_units_by_type_cursor(unit_type_id, criteria,
associated_unit_ids):
"""
Retrieve a pymongo cursor for units associated with a repository of a
give unit type that meet to the provided criteria.
:type unit_type_id: str
:type criteria: UnitAssociationCriteria
:type associated_unit_ids: list
:rtype: pymongo.cursor.Cursor
"""
collection = types_db.type_units_collection(unit_type_id)
spec = criteria.unit_filters.copy()
spec['_id'] = {'$in': associated_unit_ids}
fields = criteria.unit_fields
# The _content_type_id is required for looking up the association.
if fields is not None and '_content_type_id' not in fields:
fields = list(fields)
fields.append('_content_type_id')
cursor = collection.find(spec, fields=fields)
sort = criteria.unit_sort
if sort is None:
try:
unit_key = units_controller.get_unit_key_fields_for_type(
unit_type_id)
except ValueError:
unit_key = None
if unit_key is not None:
sort = [(u, SORT_ASCENDING) for u in unit_key]
if sort is not None:
cursor.sort(sort)
return cursor
def generate_orphans_by_type_with_unit_keys(content_type_id):
"""
Return an generator of all orphaned content units of the given content type.
Each content unit will contain the fields specified in the content type
definition's search indexes.
:param content_type_id: id of the content type
:type content_type_id: basestring
:return: generator of orphaned content units for the given content type
:rtype: generator
"""
try:
unit_key_fields = units_controller.get_unit_key_fields_for_type(content_type_id)
except ValueError:
raise MissingResource(content_type_id=content_type_id)
fields = ['_id', '_content_type_id']
fields.extend(unit_key_fields)
for content_unit in OrphanManager.generate_orphans_by_type(content_type_id, fields):
yield content_unit
def generate_orphans_by_type_with_unit_keys(content_type_id):
"""
Return an generator of all orphaned content units of the given content type.
Each content unit will contain the fields specified in the content type
definition's search indexes.
:param content_type_id: id of the content type
:type content_type_id: basestring
:return: generator of orphaned content units for the given content type
:rtype: generator
"""
try:
unit_key_fields = units_controller.get_unit_key_fields_for_type(content_type_id)
except ValueError:
raise MissingResource(content_type_id=content_type_id)
fields = ['_id', '_content_type_id']
fields.extend(unit_key_fields)
for content_unit in OrphanManager.generate_orphans_by_type(content_type_id, fields):
yield content_unit
def _associated_units_by_type_cursor(unit_type_id, criteria, associated_unit_ids):
"""
Retrieve a pymongo cursor for units associated with a repository of a
give unit type that meet to the provided criteria.
:type unit_type_id: str
:type criteria: UnitAssociationCriteria
:type associated_unit_ids: list
:rtype: pymongo.cursor.Cursor
"""
collection = types_db.type_units_collection(unit_type_id)
spec = criteria.unit_filters.copy()
spec['_id'] = {'$in': associated_unit_ids}
fields = criteria.unit_fields
# The _content_type_id is required for looking up the association.
if fields is not None and '_content_type_id' not in fields:
fields = list(fields)
fields.append('_content_type_id')
cursor = collection.find(spec, fields=fields)
sort = criteria.unit_sort
if sort is None:
try:
unit_key = units_controller.get_unit_key_fields_for_type(unit_type_id)
except ValueError:
unit_key = None
if unit_key is not None:
sort = [(u, SORT_ASCENDING) for u in unit_key]
if sort is not None:
cursor.sort(sort)
return cursor
def find_unit_by_unit_key(self, type_id, unit_key):
"""
Finds a unit based on its unit key. If more than one unit comes back,
an exception will be raised.
@param type_id: indicates the type of units being retrieved
@type type_id: str
@param unit_key: the unit key for the unit
@type unit_key: dict
@return: a single unit
@rtype: L{Unit}
"""
content_query_manager = manager_factory.content_query_manager()
try:
# this call returns a unit or raises MissingResource
existing_unit = content_query_manager.get_content_unit_by_keys_dict(type_id, unit_key)
unit_key_fields = units_controller.get_unit_key_fields_for_type(type_id)
plugin_unit = common_utils.to_plugin_unit(existing_unit, type_id, unit_key_fields)
return plugin_unit
except pulp_exceptions.MissingResource:
return None
def get_content_unit_keys(self, content_type, unit_ids):
"""
Return the keys and values that will uniquely identify the content units
that match the given unique ids.
@param content_type: unique id of content collection
@type content_type: str
@param unit_ids: list of unique content unit ids
@type unit_ids: list of str's
@return: two tuples of the same length, one of ids the second of key dicts
the same index in each tuple corresponds to a single content unit
@rtype: tuple of (possibly empty) tuples
"""
try:
key_fields = units_controller.get_unit_key_fields_for_type(content_type)
except ValueError:
raise InvalidValue(['content_type'])
all_fields = ['_id']
_flatten_keys(all_fields, key_fields)
collection = content_types_db.type_units_collection(content_type)
cursor = collection.find({'_id': {'$in': unit_ids}}, projection=all_fields)
dicts = tuple(dict(d) for d in cursor)
ids = tuple(d.pop('_id') for d in dicts)
return (ids, dicts)
def delete_orphan_content_units_by_type(type_id, content_unit_ids=None):
"""
Delete the orphaned content units for the given content type.
This method only applies to new style content units that are loaded via entry points
NOTE: this method deletes the content unit's bits from disk, if applicable.
:param type_id: id of the content type
:type type_id: basestring
:param content_unit_ids: list of content unit ids to delete; None means delete them all
:type content_unit_ids: iterable or None
:return: count of units deleted
:rtype: int
"""
# get the model matching the type
content_model = plugin_api.get_unit_model_by_id(type_id)
try:
unit_key_fields = units_controller.get_unit_key_fields_for_type(
type_id)
except ValueError:
raise MissingResource(content_type_id=type_id)
fields = ('id', '_storage_path') + unit_key_fields
if content_unit_ids:
query_sets = []
for page in plugin_misc.paginate(content_unit_ids):
qs = content_model.objects(id__in=page).only(*fields)
query_sets.append(qs)
content_units = itertools.chain(*query_sets)
else:
content_units = content_model.objects.only(*fields)
count = 0
# Paginate the content units
for units_group in plugin_misc.paginate(content_units):
# Build the list of ids to search for an easier way to access units in the group by id
unit_dict = dict()
for unit in units_group:
unit_dict[unit.id] = unit
id_list = list(unit_dict.iterkeys())
# Clear the units that are currently associated from unit_dict
non_orphan = model.RepositoryContentUnit.objects(unit_id__in=id_list)\
.distinct('unit_id')
for non_orphan_id in non_orphan:
unit_dict.pop(non_orphan_id)
# Remove the unit, lazy catalog entries, and any content in storage.
for unit_to_delete in unit_dict.itervalues():
model.LazyCatalogEntry.objects(
unit_id=str(unit_to_delete.id),
unit_type_id=str(type_id)).delete()
unit_to_delete.delete()
if hasattr(content_model, 'do_post_delete_actions'):
content_model.do_post_delete_actions(unit_to_delete)
if unit_to_delete._storage_path:
OrphanManager.delete_orphaned_file(
unit_to_delete._storage_path)
count += 1
return count
