def _fetch_objects(self, doc_type=None): """Fetch all references and convert to their document objects """ object_map = {} for col, dbrefs in self.reference_map.iteritems(): keys = object_map.keys() refs = list(set([dbref for dbref in dbrefs if str(dbref) not in keys])) if hasattr(col, "objects"): # We have a document class for the refs references = col.objects.in_bulk(refs) for key, doc in references.iteritems(): object_map[key] = doc else: # Generic reference: use the refs data to convert to document if doc_type and not isinstance(doc_type, (ListField, DictField, MapField)): references = doc_type._get_db()[col].find({"_id": {"$in": refs}}) for ref in references: doc = doc_type._from_son(ref) object_map[doc.id] = doc else: references = get_db()[col].find({"_id": {"$in": refs}}) for ref in references: if "_cls" in ref: doc = get_document(ref["_cls"])._from_son(ref) elif doc_type is None: doc = get_document("".join(x.capitalize() for x in col.split("_")))._from_son(ref) else: doc = doc_type._from_son(ref) object_map[doc.id] = doc return object_map
def _fetch_objects(self, doc_type=None): """Fetch all references and convert to their document objects """ object_map = {} for collection, dbrefs in self.reference_map.iteritems(): keys = object_map.keys() refs = list(set([dbref for dbref in dbrefs if unicode(dbref).encode('utf-8') not in keys])) if hasattr(collection, 'objects'): # We have a document class for the refs references = collection.objects.in_bulk(refs) for key, doc in references.iteritems(): object_map[key] = doc else: # Generic reference: use the refs data to convert to document if isinstance(doc_type, (ListField, DictField, MapField,)): continue if doc_type: references = doc_type._get_db()[collection].find({'_id': {'$in': refs}}) for ref in references: doc = doc_type._from_son(ref) object_map[doc.id] = doc else: references = get_db()[collection].find({'_id': {'$in': refs}}) for ref in references: if '_cls' in ref: doc = get_document(ref["_cls"])._from_son(ref) elif doc_type is None: doc = get_document( ''.join(x.capitalize() for x in collection.split('_')))._from_son(ref) else: doc = doc_type._from_son(ref) object_map[doc.id] = doc return object_map
def _find_references(self, items, depth=0): """ Recursively finds all db references to be dereferenced :param items: The iterable (dict, list, queryset) :param depth: The current depth of recursion """ reference_map = {} if not items or depth >= self.max_depth: return reference_map # Determine the iterator to use if not hasattr(items, 'items'): iterator = enumerate(items) else: iterator = items.iteritems() # Recursively find dbreferences depth += 1 for k, item in iterator: if isinstance(item, Document): for field_name, field in item._fields.iteritems(): v = getattr(item, field_name) if isinstance(v, (DBRef)): reference_map.setdefault(field.document_type, []).append(v.id) elif isinstance(v, Document) and getattr( v, '_lazy', False): reference_map.setdefault(field.document_type, []).append(v.pk) elif isinstance(v, (dict, SON)) and '_ref' in v: reference_map.setdefault(get_document(v['_cls']), []).append(v['_ref'].id) elif isinstance( v, (dict, list, tuple)) and depth <= self.max_depth: field_cls = getattr(getattr(field, 'field', None), 'document_type', None) references = self._find_references(v, depth) for key, refs in references.iteritems(): if isinstance( field_cls, (Document, TopLevelDocumentMetaclass)): key = field_cls reference_map.setdefault(key, []).extend(refs) elif isinstance(item, (DBRef)): reference_map.setdefault(item.collection, []).append(item.id) elif isinstance(item, (dict, SON)) and '_ref' in item: reference_map.setdefault(get_document(item['_cls']), []).append(item['_ref'].id) elif isinstance( item, (dict, list, tuple)) and depth - 1 <= self.max_depth: references = self._find_references(item, depth - 1) for key, refs in references.iteritems(): reference_map.setdefault(key, []).extend(refs) return reference_map
def _fetch_objects(self, doc_type=None): """Fetch all references and convert to their document objects """ object_map = {} for col, dbrefs in self.reference_map.iteritems(): keys = object_map.keys() refs = list( set([dbref for dbref in dbrefs if str(dbref) not in keys])) if hasattr(col, 'objects'): # We have a document class for the refs references = col.objects.in_bulk(refs) for key, doc in references.iteritems(): object_map[key] = doc else: # Generic reference: use the refs data to convert to document if doc_type and not isinstance(doc_type, ( ListField, DictField, MapField, )): references = doc_type._get_db()[col].find( {'_id': { '$in': refs }}) for ref in references: doc = doc_type._from_son(ref) object_map[doc.id] = doc else: references = get_db()[col].find({'_id': {'$in': refs}}) for ref in references: if '_cls' in ref: doc = get_document(ref["_cls"])._from_son(ref) else: doc = doc_type._from_son(ref) object_map[doc.id] = doc return object_map
def document_type(self): if isinstance(self.document_type_obj, basestring): if self.document_type_obj == RECURSIVE_REFERENCE_CONSTANT: self.document_type_obj = self.owner_document else: self.document_type_obj = get_document(self.document_type_obj) return self.document_type_obj
def dereference(self, value): doc_cls = get_document(value['_cls']) reference = value['_ref'] doc = doc_cls._get_db().dereference(reference) if doc is not None: doc = doc_cls._from_son(doc) return doc
def dereference(self, value): doc_cls = get_document(value['_cls']) reference = value['_ref'] doc = _get_db(doc_cls._meta['db_name']).dereference(reference) if doc is not None: doc = doc_cls._from_son(doc) doc._default_load_status = FieldStatus.LOADED return doc
def _find_references(self, items, depth=0, doc_type=None): """ Recursively finds all db references to be dereferenced :param items: The iterable (dict, list, queryset) :param depth: The current depth of recursion """ reference_map = {} if not items or depth >= self.max_depth: return reference_map # Determine the iterator to use if not hasattr(items, 'items'): iterator = enumerate(items) else: iterator = items.iteritems() # Recursively find dbreferences depth += 1 for k, item in iterator: if hasattr(item, '_fields'): for field_name, field in item._fields.iteritems(): v = item._data.get(field_name, None) if isinstance(v, (DBRef)): reference_map.setdefault(field.document_type, []).append(v.id) elif isinstance(v, (dict, SON)) and '_ref' in v: reference_map.setdefault(get_document(v['_cls']), []).append(v['_ref'].id) elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth: field_cls = getattr(getattr(field, 'field', None), 'document_type', None) references = self._find_references(v, depth) for key, refs in references.iteritems(): if isinstance(field_cls, (Document, TopLevelDocumentMetaclass)): key = field_cls reference_map.setdefault(key, []).extend(refs) elif isinstance(item, (DBRef)): reference_map.setdefault(item.collection, []).append(item.id) elif isinstance(item, ObjectId) and doc_type and not isinstance(doc_type, ObjectIdField): reference_map.setdefault(doc_type._get_collection_name(), []).append(item) elif isinstance(item, (dict, SON)) and '_ref' in item: reference_map.setdefault(get_document(item['_cls']), []).append(item['_ref'].id) elif isinstance(item, (dict, list, tuple)) and depth - 1 <= self.max_depth: references = self._find_references(item, depth - 1) for key, refs in references.iteritems(): reference_map.setdefault(key, []).extend(refs) return reference_map
def _find_references(self, items, depth=0): """ Recursively finds all db references to be dereferenced :param items: The iterable (dict, list, queryset) :param depth: The current depth of recursion """ reference_map = {} if not items or depth >= self.max_depth: return reference_map # Determine the iterator to use if not hasattr(items, "items"): iterator = enumerate(items) else: iterator = items.iteritems() # Recursively find dbreferences depth += 1 for k, item in iterator: if isinstance(item, (Document, EmbeddedDocument)): for field_name, field in item._fields.iteritems(): v = item._data.get(field_name, None) if isinstance(v, (DBRef)): reference_map.setdefault(field.document_type, []).append(v.id) elif isinstance(v, (dict, SON)) and "_ref" in v: reference_map.setdefault(get_document(v["_cls"]), []).append(v["_ref"].id) elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth: field_cls = getattr(getattr(field, "field", None), "document_type", None) references = self._find_references(v, depth) for key, refs in references.iteritems(): if isinstance(field_cls, (Document, TopLevelDocumentMetaclass)): key = field_cls reference_map.setdefault(key, []).extend(refs) elif isinstance(item, (DBRef)): reference_map.setdefault(item.collection, []).append(item.id) elif isinstance(item, (dict, SON)) and "_ref" in item: reference_map.setdefault(get_document(item["_cls"]), []).append(item["_ref"].id) elif isinstance(item, (dict, list, tuple)) and depth - 1 <= self.max_depth: references = self._find_references(item, depth - 1) for key, refs in references.iteritems(): reference_map.setdefault(key, []).extend(refs) return reference_map
def _fetch_objects(self): """Fetch all references and convert to their document objects """ object_map = {} for col, dbrefs in self.reference_map.iteritems(): keys = object_map.keys() refs = list(set([dbref for dbref in dbrefs if str(dbref) not in keys])) if hasattr(col, 'objects'): # We have a document class for the refs references = col.objects.in_bulk(refs) for key, doc in references.iteritems(): object_map[key] = doc else: # Generic reference: use the refs data to convert to document references = _get_db()[col].find({'_id': {'$in': refs}}) for ref in references: doc = get_document(ref['_cls'])._from_son(ref) object_map[doc.id] = doc return object_map
def to_python(self, value): if isinstance(value, dict): doc_cls = get_document(value['_cls']) value = doc_cls._from_son(value) return value
def _attach_objects(self, items, depth=0, instance=None, name=None): """ Recursively finds all db references to be dereferenced :param items: The iterable (dict, list, queryset) :param depth: The current depth of recursion :param instance: The owning instance used for tracking changes by :class:`~mongoengine.base.ComplexBaseField` :param name: The name of the field, used for tracking changes by :class:`~mongoengine.base.ComplexBaseField` """ if not items: if isinstance(items, (BaseDict, BaseList)): return items if instance: if isinstance(items, dict): return BaseDict(items, instance, name) else: return BaseList(items, instance, name) if isinstance(items, (dict, SON)): if '_ref' in items: return self.object_map.get(items['_ref'].id, items) elif '_cls' in items: doc = get_document(items['_cls'])._from_son(items) doc._data = self._attach_objects(doc._data, depth, doc, name) return doc if not hasattr(items, 'items'): is_list = True iterator = enumerate(items) data = [] else: is_list = False iterator = items.iteritems() data = {} depth += 1 for k, v in iterator: if is_list: data.append(v) else: data[k] = v if k in self.object_map: data[k] = self.object_map[k] elif hasattr(v, '_fields'): for field_name, field in v._fields.iteritems(): v = data[k]._data.get(field_name, None) if isinstance(v, (DBRef)): data[k]._data[field_name] = self.object_map.get(v.id, v) elif isinstance(v, (dict, SON)) and '_ref' in v: data[k]._data[field_name] = self.object_map.get(v['_ref'].id, v) elif isinstance(v, dict) and depth <= self.max_depth: data[k]._data[field_name] = self._attach_objects(v, depth, instance=instance, name=name) elif isinstance(v, (list, tuple)) and depth <= self.max_depth: data[k]._data[field_name] = self._attach_objects(v, depth, instance=instance, name=name) elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth: data[k] = self._attach_objects(v, depth - 1, instance=instance, name=name) elif hasattr(v, 'id'): data[k] = self.object_map.get(v.id, v) if instance and name: if is_list: return BaseList(data, instance, name) return BaseDict(data, instance, name) depth += 1 return data
def _attach_objects(self, items, depth=0, instance=None, name=None): """ Recursively finds all db references to be dereferenced :param items: The iterable (dict, list, queryset) :param depth: The current depth of recursion :param instance: The owning instance used for tracking changes by :class:`~mongoengine.base.ComplexBaseField` :param name: The name of the field, used for tracking changes by :class:`~mongoengine.base.ComplexBaseField` """ if not items: if isinstance(items, (BaseDict, BaseList)): return items if instance: if isinstance(items, dict): return BaseDict(items, instance, name) else: return BaseList(items, instance, name) if isinstance(items, (dict, SON)): if '_ref' in items: return self.object_map.get( (items['_ref'].collection, items['_ref'].id), items) elif '_cls' in items: doc = get_document(items['_cls'])._from_son(items) _cls = doc._data.pop('_cls', None) del items['_cls'] doc._data = self._attach_objects(doc._data, depth, doc, None) if _cls is not None: doc._data['_cls'] = _cls return doc if not hasattr(items, 'items'): is_list = True list_type = BaseList if isinstance(items, EmbeddedDocumentList): list_type = EmbeddedDocumentList as_tuple = isinstance(items, tuple) iterator = enumerate(items) data = [] else: is_list = False iterator = items.iteritems() data = {} depth += 1 for k, v in iterator: if is_list: data.append(v) else: data[k] = v if k in self.object_map and not is_list: data[k] = self.object_map[k] elif isinstance(v, (Document, EmbeddedDocument)): for field_name, field in v._fields.iteritems(): v = data[k]._data.get(field_name, None) if isinstance(v, DBRef): data[k]._data[field_name] = self.object_map.get( (v.collection, v.id), v) elif isinstance(v, (dict, SON)) and '_ref' in v: data[k]._data[field_name] = self.object_map.get( (v['_ref'].collection, v['_ref'].id), v) elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth: item_name = "{0}.{1}.{2}".format(name, k, field_name) data[k]._data[field_name] = self._attach_objects(v, depth, instance=instance, name=item_name) elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth: item_name = '%s.%s' % (name, k) if name else name data[k] = self._attach_objects(v, depth - 1, instance=instance, name=item_name) elif hasattr(v, 'id'): data[k] = self.object_map.get((v.collection, v.id), v) if instance and name: if is_list: return tuple(data) if as_tuple else list_type(data, instance, name) return BaseDict(data, instance, name) depth += 1 return data
def _attach_objects(self, items, depth=0, instance=None, name=None): """ Recursively finds all db references to be dereferenced :param items: The iterable (dict, list, queryset) :param depth: The current depth of recursion :param instance: The owning instance used for tracking changes by :class:`~mongoengine.base.ComplexBaseField` :param name: The name of the field, used for tracking changes by :class:`~mongoengine.base.ComplexBaseField` """ if not items: if isinstance(items, (BaseDict, BaseList)): return items if instance: if isinstance(items, dict): return BaseDict(items, instance, name) else: return BaseList(items, instance, name) if isinstance(items, (dict, SON)): if '_ref' in items: return self.object_map.get(items['_ref'].id, items) elif '_cls' in items: doc = get_document(items['_cls'])._from_son(items) _cls = doc._data.pop('_cls', None) del items['_cls'] doc._data = self._attach_objects(doc._data, depth, doc, None) if _cls is not None: doc._data['_cls'] = _cls return doc if not hasattr(items, 'items'): is_list = True list_type = BaseList if isinstance(items, EmbeddedDocumentList): list_type = EmbeddedDocumentList as_tuple = isinstance(items, tuple) iterator = enumerate(items) data = [] else: is_list = False iterator = items.iteritems() data = {} depth += 1 for k, v in iterator: if is_list: data.append(v) else: data[k] = v if k in self.object_map and not is_list: data[k] = self.object_map[k] elif isinstance(v, (Document, EmbeddedDocument)): for field_name, field in v._fields.iteritems(): v = data[k]._data.get(field_name, None) if isinstance(v, (DBRef)): data[k]._data[field_name] = self.object_map.get(v.id, v) elif isinstance(v, (dict, SON)) and '_ref' in v: data[k]._data[field_name] = self.object_map.get(v['_ref'].id, v) elif isinstance(v, dict) and depth <= self.max_depth: data[k]._data[field_name] = self._attach_objects(v, depth, instance=instance, name=name) elif isinstance(v, (list, tuple)) and depth <= self.max_depth: data[k]._data[field_name] = self._attach_objects(v, depth, instance=instance, name=name) elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth: item_name = '%s.%s' % (name, k) if name else name data[k] = self._attach_objects(v, depth - 1, instance=instance, name=item_name) elif hasattr(v, 'id'): data[k] = self.object_map.get(v.id, v) if instance and name: if is_list: return tuple(data) if as_tuple else list_type(data, instance, name) return BaseDict(data, instance, name) depth += 1 return data
def _attach_objects(self, items, depth=0, instance=None, name=None, get=False): """ Recursively finds all db references to be dereferenced :param items: The iterable (dict, list, queryset) :param depth: The current depth of recursion :param instance: The owning instance used for tracking changes by :class:`~mongoengine.base.ComplexBaseField` :param name: The name of the field, used for tracking changes by :class:`~mongoengine.base.ComplexBaseField` :param get: A boolean determining if being called by __get__ """ if not items: if isinstance(items, (BaseDict, BaseList)): return items if instance: if isinstance(items, dict): return BaseDict(items, instance=instance, name=name) else: return BaseList(items, instance=instance, name=name) if isinstance(items, (dict, pymongo.son.SON)): if '_ref' in items: return self.object_map.get(items['_ref'].id, items) elif '_types' in items and '_cls' in items: doc = get_document(items['_cls'])._from_son(items) if not get: doc._data = self._attach_objects(doc._data, depth, doc, name, get) return doc if not hasattr(items, 'items'): is_list = True iterator = enumerate(items) data = [] else: is_list = False iterator = items.iteritems() data = {} for k, v in iterator: if is_list: data.append(v) else: data[k] = v if k in self.object_map: data[k] = self.object_map[k] elif hasattr(v, '_fields'): for field_name, field in v._fields.iteritems(): v = data[k]._data.get(field_name, None) if isinstance(v, (pymongo.dbref.DBRef)): data[k]._data[field_name] = self.object_map.get( v.id, v) elif isinstance(v, (dict, pymongo.son.SON)) and '_ref' in v: data[k]._data[field_name] = self.object_map.get( v['_ref'].id, v) elif isinstance(v, dict) and depth < self.max_depth: data[k]._data[field_name] = self._attach_objects( v, depth, instance=instance, name=name, get=get) elif isinstance(v, (list, tuple)): data[k]._data[field_name] = self._attach_objects( v, depth, instance=instance, name=name, get=get) elif isinstance(v, (dict, list, tuple)) and depth < self.max_depth: data[k] = self._attach_objects(v, depth, instance=instance, name=name, get=get) elif hasattr(v, 'id'): data[k] = self.object_map.get(v.id, v) if instance and name: if is_list: return BaseList(data, instance=instance, name=name) return BaseDict(data, instance=instance, name=name) depth += 1 return data
def _transform_key(key, context, prefix='', is_find=False): from fields import BaseField, DictField, ListField, \ EmbeddedDocumentField, ArbitraryField parts = key.split('.', 1) first_part = parts[0] if len(parts) > 1: rest = parts[1] else: rest = None # a key as a digit means a list index... set context as the list's value if first_part.isdigit() or first_part == '$': if isinstance(context, DictField): context = ArbitraryField() elif isinstance(context.field, basestring): context = get_document(context.field) elif isinstance(context.field, BaseField): context = context.field if first_part == '_id': context = context.pk_field() # atomic ops, digits (list indexes), or _ids get no processing if first_part[0] == '$' or first_part.isdigit() or first_part == '_id': if prefix: new_prefix = "%s.%s" % (prefix, first_part) else: new_prefix = first_part if rest: return Document._transform_key(rest, context, prefix=new_prefix, is_find=is_find) else: return new_prefix, context def is_subclass_or_instance(obj, parent): try: if issubclass(obj, parent): return True except TypeError: if isinstance(obj, parent): return True return False field = None if is_subclass_or_instance(context, BaseDocument): field = context._fields.get(first_part, None) elif is_subclass_or_instance(context, EmbeddedDocumentField): field = context.document_type._fields.get(first_part, None) elif is_subclass_or_instance(context, ListField): if is_subclass_or_instance(context.field, basestring): field = get_document(context.field) elif is_subclass_or_instance(context.field, BaseField): field = context.field else: raise ValueError("Can't parse field %s" % first_part) field._in_list = True # if we hit a DictField, values can be anything, so use the sentinal # ArbitraryField value (I prefer this over None, since None can be # introduced in other ways that would be considered errors & should not # be silently ignored) elif is_subclass_or_instance(context, DictField): field = ArbitraryField() elif is_subclass_or_instance(context, ArbitraryField): field = context if not field: raise ValueError("Can't find field %s" % first_part) # another unfortunate hack... in find queries "list.field_name" means # field_name inside of the list's field... but in updates, # list.0.field_name means that... need to differentiate here list_field_name = None if is_subclass_or_instance(field, ListField) and is_find: list_field_name = field.db_field if is_subclass_or_instance(field.field, basestring): field = get_document(field.field) elif is_subclass_or_instance(field.field, BaseField): field = field.field else: raise ValueError("Can't parse field %s" % first_part) field._in_list = True if is_subclass_or_instance(field, ArbitraryField): db_field = first_part elif list_field_name: db_field = list_field_name else: db_field = field.db_field if prefix: result = "%s.%s" % (prefix, db_field) else: result = db_field if rest: return Document._transform_key(rest, field, prefix=result, is_find=is_find) else: return result, field