def obj_from_oplog(data, _filter, projection=None, pop_fields=None):
_id = None
obj = {}
_filter = _filter or {}
op = _.get(data, 'op')
ns = '.'.join(_.get(data, 'ns').split('.')[1:])
if op == 'i':
i_obj = copy.deepcopy(_.get(data, 'o') or {})
_id = _.get(i_obj, '_id', None)
if i_obj and dict_filter(i_obj, _filter, ns):
i_obj.pop('_id', None)
obj = i_obj
elif op == 'u':
_id = _.get(data, 'o2._id')
u_obj = copy.deepcopy(_.get(data, 'o') or {})
if u_obj:
if not _.get(u_obj, '$set'):
if u_obj and dict_filter(u_obj, _filter, ns):
obj = u_obj
obj.pop('_id', None)
else:
U_filter = dict(_filter, **{'_id': _id})
collection = models.get_collection(ns)
if collection.count(U_filter) > 0:
obj = _.get(u_obj, '$set')
obj.pop('_id', None)
elif op == 'd':
_id = _.get(data, 'o._id')
if _id and pop_fields:
collection = models.get_collection(ns)
collection.__class__ = BaseModel
obj = collection.populate_one(filter={'_id': _id},
projection=projection,
pop_fields=pop_fields)
obj.pop('_id', None)
return str(_id) if _id else None, obj or {}
def __populate(self, local_data_list, pop_fields, field_value_filter=lambda v: v):
# 获取外键collection，并判断是否存在
foreign_collection = models.get_collection(pop_fields.get('from'))
# 主键set，用于存储主键（不重复）
local_field_set = set()
local_field = pop_fields.get('local_field')
_as = pop_fields.get('as') or local_field
# 收集主键
for local_data in local_data_list:
local_field_set.add(field_value_filter(_.get(local_data, local_field)))
# 无主键，返回主collection数据
if len(local_field_set) <= 0:
return local_data_list
foreign_field = pop_fields.get('foreign_field')
foreign_data_dict = dict()
f_filter = {pop_fields.get('foreign_field'): {'$in': list(local_field_set)}}
# 查询外键collection，并形成dict
foreign_data_list = list(foreign_collection.find(filter=f_filter, projection=pop_fields.get('projection')))
# 递归查询populate数据
recursion_pop_fields = pop_fields.get('pop_fields')
if recursion_pop_fields:
for key, value in recursion_pop_fields.items():
foreign_data_list = self.__populate(foreign_data_list, value, field_value_filter)
for foreign_data in foreign_data_list:
if isinstance(_.get(foreign_data, foreign_field), list):
for arr_id in _.get(foreign_data, foreign_field):
foreign_data_dict[field_value_filter(arr_id)] = foreign_data
else:
foreign_data_dict[field_value_filter(_.get(foreign_data, foreign_field))] = foreign_data
# 匹配外键，并将数据添加到主collection
for local_data in local_data_list:
local_data[_as] = foreign_data_dict.get(field_value_filter(_.get(local_data, local_field))) or {}
return local_data_list
def dict_filter(data, _filter, ns):
"""
判断数据是否符合筛选器, 支持mongo的filter语法, 支持两种模式
1. 简单模式: {key:value}, 直接判断处理
2. 复杂模式: filter中存在复杂语义, 直接通过数据库进行查询
"""
if data and _filter:
for key, value in _filter.items():
if key.find('$') > -1 or key.find('{') > -1 or str(value).find(
'$') > -1 or str(value).find('{') > -1:
# 复杂语义
return models.get_collection(ns).count(_filter) > 0
# 简单语义
v = _.get(data, key)
if isinstance(v, list):
if not _.includes(v, value):
return False
elif isinstance(v, type(value)):
if v != value:
return False
return True