def get_flattened_tree(self, queryset: QuerySet):
project_tree = OrderedDict()
for project in queryset.select_related('parent'):
if project.parent_id:
project_tree.setdefault(project.parent_id, []).append(project)
else:
project_tree.setdefault(project.pk, []).append(project)
flattened_tree = itertools.chain(*project_tree.itervalues())
return flattened_tree
def admin_keyword_search(model, fields, keywords):
"""
"""
if not keywords:
return []
keywords = keywords.split(" ")
qs = QuerySet(model)
for keyword in keywords:
or_queries = [ Q(**{'%s__icontains' % field: keyword}) for field in fields ]
other_qs = QuerySet(model)
if qs.select_related:
other_qs = other_qs.select_related()
other_qs = other_qs.filter(reduce(operator.or_, or_queries))
qs = qs & other_qs
return qs
def get_query_set(self):
qs = self.manager.get_query_set()
lookup_params = self.params.copy() # a dictionary of the query string
for i in (ALL_VAR, ORDER_VAR, ORDER_TYPE_VAR, SEARCH_VAR, IS_POPUP_VAR):
if i in lookup_params:
del lookup_params[i]
for key, value in lookup_params.items():
if not isinstance(key, str):
# 'key' will be used as a keyword argument later, so Python
# requires it to be a string.
del lookup_params[key]
lookup_params[smart_str(key)] = value
# Apply lookup parameters from the query string.
qs = qs.filter(**lookup_params)
# Use select_related() if one of the list_display options is a field
# with a relationship.
if self.lookup_opts.admin.list_select_related:
qs = qs.select_related()
else:
for field_name in self.lookup_opts.admin.list_display:
try:
f = self.lookup_opts.get_field(field_name)
except models.FieldDoesNotExist:
pass
else:
if isinstance(f.rel, models.ManyToOneRel):
qs = qs.select_related()
break
# Calculate lookup_order_field.
# If the order-by field is a field with a relationship, order by the
# value in the related table.
lookup_order_field = self.order_field
try:
f = self.lookup_opts.get_field(self.order_field, many_to_many=False)
except models.FieldDoesNotExist:
pass
else:
if isinstance(f.rel, models.OneToOneRel):
# For OneToOneFields, don't try to order by the related object's ordering criteria.
pass
elif isinstance(f.rel, models.ManyToOneRel):
rel_ordering = f.rel.to._meta.ordering and f.rel.to._meta.ordering[0] or f.rel.to._meta.pk.column
lookup_order_field = '%s.%s' % (f.rel.to._meta.db_table, rel_ordering)
# Set ordering.
qs = qs.order_by((self.order_type == 'desc' and '-' or '') + lookup_order_field)
# Apply keyword searches.
def construct_search(field_name):
if field_name.startswith('^'):
return "%s__istartswith" % field_name[1:]
elif field_name.startswith('='):
return "%s__iexact" % field_name[1:]
elif field_name.startswith('@'):
return "%s__search" % field_name[1:]
else:
return "%s__icontains" % field_name
if self.lookup_opts.admin.search_fields and self.query:
for bit in self.query.split():
or_queries = [models.Q(**{construct_search(field_name): bit}) for field_name in self.lookup_opts.admin.search_fields]
other_qs = QuerySet(self.model)
if qs._select_related:
other_qs = other_qs.select_related()
other_qs = other_qs.filter(reduce(operator.or_, or_queries))
qs = qs & other_qs
if self.opts.one_to_one_field:
qs = qs.complex_filter(self.opts.one_to_one_field.rel.limit_choices_to)
return qs
def get_query_set(self):
qs = self.manager.get_query_set()
lookup_params = self.params.copy() # a dictionary of the query string
for i in (ALL_VAR, ORDER_VAR, ORDER_TYPE_VAR, SEARCH_VAR, IS_POPUP_VAR):
if lookup_params.has_key(i):
del lookup_params[i]
# Apply lookup parameters from the query string.
qs = qs.filter(**lookup_params)
# Use select_related() if one of the list_display options is a field
# with a relationship.
if self.lookup_opts.admin.list_select_related:
qs = qs.select_related()
else:
for field_name in self.lookup_opts.admin.list_display:
try:
f = self.lookup_opts.get_field(field_name)
except models.FieldDoesNotExist:
pass
else:
if isinstance(f.rel, models.ManyToOneRel):
qs = qs.select_related()
break
# Calculate lookup_order_field.
# If the order-by field is a field with a relationship, order by the
# value in the related table.
lookup_order_field = self.order_field
try:
f = self.lookup_opts.get_field(self.order_field, many_to_many=False)
except models.FieldDoesNotExist:
pass
else:
if isinstance(f.rel, models.OneToOneRel):
# For OneToOneFields, don't try to order by the related object's ordering criteria.
pass
elif isinstance(f.rel, models.ManyToOneRel):
rel_ordering = f.rel.to._meta.ordering and f.rel.to._meta.ordering[0] or f.rel.to._meta.pk.column
lookup_order_field = '%s.%s' % (f.rel.to._meta.db_table, rel_ordering)
# Set ordering.
qs = qs.order_by((self.order_type == 'desc' and '-' or '') + lookup_order_field)
# Apply keyword searches.
def construct_search(field_name):
if field_name.startswith('^'):
return "%s__istartswith" % field_name[1:]
elif field_name.startswith('='):
return "%s__iexact" % field_name[1:]
elif field_name.startswith('@'):
return "%s__search" % field_name[1:]
else:
return "%s__icontains" % field_name
if self.lookup_opts.admin.search_fields and self.query:
for bit in self.query.split():
or_queries = [models.Q(**{construct_search(field_name): bit}) for field_name in self.lookup_opts.admin.search_fields]
other_qs = QuerySet(self.model)
if qs._select_related:
other_qs = other_qs.select_related()
other_qs = other_qs.filter(reduce(operator.or_, or_queries))
qs = qs & other_qs
if self.opts.one_to_one_field:
qs = qs.complex_filter(self.opts.one_to_one_field.rel.limit_choices_to)
return qs
def get_query_set(self):
if (not self.opts.admin.show_all_rows) and self.opts.row_level_permissions and (not self.user.has_perm(self.opts.app_label + "."+self.opts.get_change_permission())):
from django.contrib.auth.models import RowLevelPermission
model_id_name = self.opts.pk.attname
param_name = model_id_name.replace("_", "__")+"__in"
param = {param_name:RowLevelPermission.objects.get_model_list(self.user,
self.model,
self.opts.get_change_permission())}
qs = self.manager.filter(**param)
else:
qs = self.manager.get_query_set()
lookup_params = self.params.copy() # a dictionary of the query string
for i in (ALL_VAR, ORDER_VAR, ORDER_TYPE_VAR, SEARCH_VAR, IS_POPUP_VAR):
if i in lookup_params:
del lookup_params[i]
# Apply lookup parameters from the query string.
qs = qs.filter(**lookup_params)
# Use select_related() if one of the list_display options is a field
# with a relationship.
if self.lookup_opts.admin.list_select_related:
qs = qs.select_related()
else:
for field_name in self.lookup_opts.admin.list_display:
try:
f = self.lookup_opts.get_field(field_name)
except models.FieldDoesNotExist:
pass
else:
if isinstance(f.rel, models.ManyToOneRel):
qs = qs.select_related()
break
# Calculate lookup_order_field.
# If the order-by field is a field with a relationship, order by the
# value in the related table.
lookup_order_field = self.order_field
try:
f = self.lookup_opts.get_field(self.order_field, many_to_many=False)
except models.FieldDoesNotExist:
pass
else:
if isinstance(f.rel, models.OneToOneRel):
# For OneToOneFields, don't try to order by the related object's ordering criteria.
pass
elif isinstance(f.rel, models.ManyToOneRel):
rel_ordering = f.rel.to._meta.ordering and f.rel.to._meta.ordering[0] or f.rel.to._meta.pk.column
lookup_order_field = '%s.%s' % (f.rel.to._meta.db_table, rel_ordering)
# Set ordering.
qs = qs.order_by((self.order_type == 'desc' and '-' or '') + lookup_order_field)
# Apply keyword searches.
def construct_search(field_name):
if field_name.startswith('^'):
return "%s__istartswith" % field_name[1:]
elif field_name.startswith('='):
return "%s__iexact" % field_name[1:]
elif field_name.startswith('@'):
return "%s__search" % field_name[1:]
else:
return "%s__icontains" % field_name
if self.lookup_opts.admin.search_fields and self.query:
for bit in self.query.split():
or_queries = [models.Q(**{construct_search(field_name): bit}) for field_name in self.lookup_opts.admin.search_fields]
other_qs = QuerySet(self.model)
if qs._select_related:
other_qs = other_qs.select_related()
other_qs = other_qs.filter(reduce(operator.or_, or_queries))
qs = qs & other_qs
if self.opts.one_to_one_field:
qs = qs.complex_filter(self.opts.one_to_one_field.rel.limit_choices_to)
return qs
def qs_to_result(result: QuerySet,
stride: int = 1,
group: bool = False,
shuffle: bool = False,
deterministic_order: bool = False,
custom_order_by_id: List[int] = None,
frame_major: bool = True,
show_count: bool = False,
limit: int = 100,
color: str = "red") -> Dict:
count = result.count() if show_count else 0
if shuffle:
result = result.order_by('?')
materialized_result = []
cls = result.model
bases = cls.__bases__
if bases[0] is base_models.Frame:
if custom_order_by_id is not None:
result = sorted(result, key=lambda x: custom_order_by_id.index(x.id))
elif not shuffle and deterministic_order:
result = result.order_by('video', 'number')
for frame in result[:limit * stride:stride]:
materialized_result.append({
'video': frame.video.id,
'min_frame': frame.number,
'objects': []
})
elif (cls is Face or cls is FaceGender or cls is FaceIdentity
or cls is Object or cls is Pose or cls is FaceLandmarks):
if cls is FaceGender or cls is FaceIdentity or cls is FaceLandmarks:
frame_path = 'face__frame'
if cls is FaceGender:
result = result.select_related('face', 'gender')
elif cls is FaceIdentity:
result = result.select_related('face', 'identity')
else:
result = result.select_related('face')
else:
frame_path = 'frame'
result = result.select_related(frame_path)
if not shuffle and deterministic_order:
result = result.order_by(frame_path + '__video', frame_path + '__number')
if cls is Face:
fn = bbox_to_dict
elif cls is Object:
fn = object_to_dict
elif cls is FaceGender:
fn = gender_to_dict
elif cls is FaceIdentity:
fn = identity_to_dict
elif cls is Pose:
fn = pose_to_dict
elif cls is FaceLandmarks:
fn = face_landmarks_to_dict
if frame_major:
frame_ids = set()
def get_all_results():
all_results = collect(
list(result.filter(**{frame_path + '__in': list(frame_ids)})),
lambda t: access(t, frame_path + '__id'))
return all_results
if custom_order_by_id is None:
frames = set()
for inst in list(
result.values(
frame_path + '__video', frame_path + '__number',
frame_path + '__id').annotate(m=F('id') % stride).filter(m=0)[:limit]):
frames.add((inst[frame_path + '__video'], inst[frame_path + '__number'],
inst[frame_path + '__id']))
frame_ids.add(inst[frame_path + '__id'])
all_results = get_all_results()
frames = list(frames)
frames.sort(key=itemgetter(0, 1))
else:
frames = {}
id_to_position = defaultdict(lambda: float('inf'))
for i, id_ in enumerate(custom_order_by_id):
id_to_position[id_] = i
for inst in list(
result.values(
'id', frame_path + '__video', frame_path + '__number',
frame_path + '__id').annotate(m=F('id') % stride).filter(m=0)):
frame_key = (inst[frame_path + '__video'], inst[frame_path + '__number'],
inst[frame_path + '__id'])
frames[frame_key] = min(id_to_position[inst['id']], frames[frame_key]
if frame_key in frames else float('inf'))
frame_ids.add(inst[frame_path + '__id'])
all_results = get_all_results()
frames = sorted([x for x in frames.items()], key=lambda x: x[1])
frames = [x[0] for x in frames[:limit]]
for (video, frame_num, frame_id) in frames:
materialized_result.append({
'video': video,
'min_frame': frame_num,
'objects': [fn(inst) for inst in all_results[frame_id]]
})
else:
for inst in result[:limit * stride:stride]:
r = {
'video': inst.frame.video.id,
'min_frame': inst.frame.number,
'objects': [fn(inst)]
}
materialized_result.append(r)
elif bases[0] is base_models.Track:
if not shuffle and deterministic_order:
result = result.order_by('video', 'min_frame')
for t in result.annotate(duration=Track.duration_expr()).filter(duration__gt=0)[:limit]:
result = {
'video': t.video.id,
'track': t.id,
'min_frame': t.min_frame,
'max_frame': t.max_frame,
}
materialized_result.append(result)
if custom_order_by_id is not None:
materialized_result.sort(key=lambda x: custom_order_by_id.index(x['track']))
else:
materialized_result.sort(key=itemgetter('video', 'min_frame'))
elif bases[0] is base_models.Video:
if custom_order_by_id is not None:
raise NotImplementedError()
if not shuffle and deterministic_order:
result = result.order_by('id')
for v in result[:limit]:
materialized_result.append({
'video': v.id,
'min_frame': 0,
'max_frame': v.num_frames})
else:
raise Exception("Unsupported class")
ty_name = cls.__name__
if group:
by_video = collect(materialized_result, itemgetter('video'))
videos = collect(Video.objects.filter(id__in=by_video.keys()).all(),
attrgetter('id'))
groups = [{
'type': 'contiguous',
'label': video,
'num_frames': videos[video][0].num_frames,
'elements': [{
'video': video,
'segments': sorted(by_video[video], key=itemgetter('min_frame')),
'color': color
}]
} for video in sorted(by_video.keys())]
else:
groups = [{'type': 'flat', 'label': '', 'elements': [r]} for r in materialized_result]
return {'result': groups, 'count': count, 'type': ty_name}
class SphinxQuerySet(object):
__index_match = re.compile(r'[^a-z0-9_-]*', re.I)
def __init__(self, model=None, using=None, **kwargs):
self.model = model
self.using = using
self.queryset = QuerySet()
self.realtime = None
self._doc_ids = None
self._iter = None
self._query = None
self._query_args = None
self._only_meta = False
self._field_names = {}
self._fields = '*'
self._aliases = {}
self._group_by = ''
self._order_by = ''
self._group_order_by = ''
self._filters = {}
self._excludes = {}
self._excluded_fields = []
_q_opts = kwargs.pop('query_options', SPHINX_QUERY_OPTS)
if 'ranker' not in _q_opts:
_q_opts['ranker'] = 'bm25'
self._query_opts = self._format_options(**_q_opts)
self._result_cache = None
self._doc_fields_cache = {}
self._index_fields_cache = None
self._metadata = None
self._maxmatches = min(kwargs.pop('maxmatches', SPHINX_MAX_MATCHES), SPHINX_MAX_MATCHES)
self._limit = min(kwargs.pop('limit', SPHINX_QUERY_LIMIT), self._maxmatches)
self._offset = None
self._snippets = kwargs.pop('snippets', SPHINX_SNIPPETS)
self._snippets_opts = kwargs.pop('snippets_options', SPHINX_SNIPPETS_OPTS)
self._snippets_string = None
self._snippet_dict = {}
if model:
#self._indexes = self._parse_indexes(kwargs.pop('index', model._meta.db_table))
self._indexes = [model._meta.db_table]
model_options = model.__sphinx_options__
if model_options.get('realtime', False):
self.realtime = '%s_rt' % model._meta.db_table
self._indexes.append(self.realtime)
else:
self._indexes = self._parse_indexes(kwargs.pop('index', None))
def __len__(self):
return self.count()
def __iter__(self):
if self._result_cache is None:
try:
self._get_data()
except MySQLdb.ProgrammingError as e:
raise SearchError(e.args + (self.query_string % self._query_args,))
return iter(self._result_cache)
def __repr__(self):
return repr(self.__iter__())
def __getitem__(self, k):
"""
Retrieves an item or slice from the set of results.
"""
if not isinstance(k, (slice,) + six.integer_types):
raise TypeError
assert ((not isinstance(k, slice) and (k >= 0))
or (isinstance(k, slice) and (k.start is None or k.start >= 0)
and (k.stop is None or k.stop >= 0))),\
"Negative indexing is not supported."
if isinstance(k, slice):
qs = self._clone()
start = int(k.start) if k.start is not None else 0
stop = int(k.stop) if k.stop is not None else None
qs._set_limits(start, stop)
qs._get_data()
return k.step and list(qs)[::k.step] or qs
try:
qs = self._clone()
qs._set_limits(k, k + 1)
qs._get_data()
return list(qs)[0]
except Exception as e:
raise IndexError(e.args)
# Indexes
def add_index(self, index):
if self.model is not None:
raise SearchError('You can not add an index to the model')
_indexes = self._indexes[:]
for x in self._parse_indexes(index):
if x not in _indexes:
_indexes.append(x)
return self._clone(_indexes=_indexes)
def remove_index(self, index):
if self.model is not None:
raise SearchError('You can not remove an index from model')
_indexes = self._indexes[:]
for x in self._parse_indexes(index):
if x in _indexes:
_indexes.pop(_indexes.index(x))
return self._clone(_indexes=_indexes)
# For queryset
def select_related(self, *fields, **kwargs):
self.queryset = self.queryset.select_related(*fields, **kwargs)
return self
def prefetch_related(self, *lookups):
self.queryset = self.queryset.prefetch_related(*lookups)
return self
def only(self, *fields):
self.queryset = self.queryset.only(*fields)
return self
def defer(self, *fields):
c = self._clone()
c.queryset = self.queryset.defer(*fields)
c._excluded_fields = self._excluded_fields + list(fields)
return c
# Querying
def query(self, query):
return self._clone(_query=force_unicode(query))
def filter(self, **kwargs):
filters = self._filters.copy()
return self._clone(_filters=self._process_filters(filters, False, **kwargs))
def exclude(self, **kwargs):
filters = self._excludes.copy()
return self._clone(_excludes=self._process_filters(filters, True, **kwargs))
def fields(self, *args, **kwargs):
fields = ''
aliases = {}
if args:
fields = '`%s`' % '`, `'.join(args)
if kwargs:
for k, v in kwargs.iteritems():
aliases[k] = '%s AS `%s`' % (v, k)
if fields or aliases:
return self._clone(_fields=fields, _aliases=aliases)
return self
def options(self, **kwargs):
if not kwargs:
return self
return self._clone(_query_opts=self._format_options(**kwargs))
def snippets(self, snippets=True, **kwargs):
if snippets == self._snippets and not kwargs:
return self
for k, v in kwargs.iteritems():
if isinstance(v, bool):
v = int(v)
return self._clone(_snippets_opts=kwargs, _snippets=snippets, _snippets_opts_string=None)
def snippet(self, attr, to_filed=None):
to_filed = to_filed or attr
snippet_dict = self._snippet_dict.copy()
snippet_dict[to_filed] = attr
return self._clone(_snippet_dict=snippet_dict)
# Currently only supports grouping by a single column.
# The column however can be a computed expression
def group_by(self, field):
return self._clone(_group_by='GROUP BY `%s`' % field)
def order_by(self, *args):
sort_by = []
for arg in args:
order = 'ASC'
if arg[0] == '-':
order = 'DESC'
arg = arg[1:]
if arg == 'pk':
arg = 'id'
sort_by.append('`%s` %s' % (arg, order))
if sort_by:
return self._clone(_order_by='ORDER BY %s' % ', '.join(sort_by))
return self
def group_order_by(self, *args):
sort_by = []
for arg in args:
order = 'ASC'
if arg[0] == '-':
order = 'DESC'
arg = arg[1:]
if arg == 'pk':
arg = 'id'
if arg.startswith('@'):
sort_by.append('%s %s' % (arg, order))
else:
sort_by.append('`%s` %s' % (arg, order))
if sort_by:
return self._clone(_group_order_by='WITHIN GROUP ORDER BY %s' % ', '.join(sort_by))
return self
def count(self):
store_qs = self.queryset._clone()
self.only('id')
self._only_meta = True
result = min(int(self.meta.get('total_found', 0)), self._maxmatches)
self._only_meta = False
self.queryset = store_qs
return result
# Возвращяет все объекты из индекса. Размер списка ограничен только
# значением maxmatches
def all(self):
return self._clone(_limit=self._maxmatches, _offset=None)
def none(self):
qs = EmptySphinxQuerySet()
qs.__dict__.update(self.__dict__.copy())
return qs
def reset(self):
return self.__class__(self.model, self.using, index=self._get_index())
def _get_values_for_update(self, obj):
fields = self._get_index_fields()
values = []
for field in fields[:]:
if field == 'id':
f = getattr(obj, 'pk')
f = self._encode_document_id(f)
else:
f = getattr(obj, field)
if hasattr(f, 'through'): # ManyToMany
# пропускаем пока что...
f = [force_unicode(x.pk) for x in f.all()]
elif isinstance(f, six.string_types):
pass
elif isinstance(f, six.integer_types) or isinstance(f, (bool, date, datetime, float, decimal.Decimal)):
f = to_sphinx(f)
else:
model_filed = obj._meta.get_field(field)
if isinstance(model_filed, RelatedField):
f = to_sphinx(getattr(obj, model_filed.column))
else:
raise SearchError('Unknown field `%s`' % type(f))
values.append(f)
return values
def create(self, *args, **kwargs):
values = ()
if self.model:
assert len(args) == 1, \
'Model RT-index can be updated by object instance or queryset'
obj = args[0]
if isinstance(obj, self.model):
# один объект, один документ
values = (self._get_values_for_update(obj),)
elif isinstance(obj, QuerySet):
# несколько объектов, несколько документов
values = map(self._get_values_for_update, obj)
else:
raise SearchError('Can`t `%s` not an instance/queryset of `%s`' % (obj, self.model))
else:
raise NotImplementedError('Non-model RT-index update not supported yet')
if not values:
raise SearchError('Empty QuerySet? o_O')
query = ['REPLACE' if kwargs.pop('force_update', False) else 'INSERT']
query.append('INTO %s' % self.realtime)
query.append('(%s)' % ','.join(self._get_index_fields()))
query.append('VALUES')
query_args = []
q = []
for v in values:
f_list = []
for f in v:
if isinstance(f, six.string_types):
query_args.append(f)
f_list.append('%s')
elif isinstance(f, (list, tuple)):
f_list.append('(%s)' % ','.join(f))
else:
f_list.append(force_unicode(f))
q.append('(%s)' % ','.join(f_list))
query.append(', '.join(q))
cursor = conn_handler.cursor()
count = cursor.execute(' '.join(query), query_args)
return count
def update(self, **kwargs):
raise NotImplementedError('Update not implemented yet')
def delete(self):
"""
Удаляет из индекса документы, удовлетворяющие условиям filter
"""
assert self._can_modify(),\
"Cannot use 'limit' or 'offset' with delete."
q = ['DELETE FROM %s WHERE' % self.realtime]
if len(self._doc_ids) == 1:
where = 'id = %i' % self._doc_ids[0]
else:
where = 'id IN (%s)' % ','.join(str(id) for id in self._doc_ids)
q.append(where)
query = ' '.join(q)
cursor = conn_handler.cursor()
cursor.execute(query, self._query_args)
# misc
def keywords(self, text, index=None, hits=None):
"""\
Возвращает генератор со списком ключевых слов
для переданного текста\
"""
if index is None:
# пока только для одного индекса
index = self._indexes[0]
query = 'CALL KEYWORDS (%s)'
q = ['%s', '%s']
if hits is not None and hits:
q.append('1')
query = query % ', '.join(q)
cursor = conn_handler.cursor()
count = cursor.execute(query, [text, index])
for x in range(0, count):
yield cursor.fetchone()
def get_query_set(self, model):
if self.using is not None:
self.queryset.using(self.using)
self.queryset.model = model
self.queryset.query.model = model
return self.queryset
# Properties
def _meta(self):
if self._metadata is None:
self._get_data()
return self._metadata
meta = property(_meta)
def _get_snippets_string(self):
if self._snippets_string is None:
opts_list = []
for k, v in self._snippets_opts.iteritems():
opt = ('\'%s\' AS %s' if isinstance(v, six.string_types) else '%s AS %s') % (v, k)
opts_list.append(opt)
if opts_list:
self._snippets_string = ', %s' % ', '.join(opts_list)
return self._snippets_string or ''
#internal
def _set_limits(self, start, stop=None):
if start is not None:
self._offset = int(start)
else:
start = 0
if stop is not None:
self._limit = stop - start
def _can_modify(self):
if self.realtime is None:
raise SearchError('Documents can`t be modified on the non-realtime index')
assert self._doc_ids is not None \
and not self._excludes and self._query is None\
and len(self._filters) == 1 and 'id' in self._filters, \
'Only {id = value | id IN (val1 [, val2 [, ...]])} filters allowed here'
return self._offset is None
def _get_data(self):
if not self._indexes:
#warnings.warn('Index list is not set. Using all known indices.')
self._indexes = self._parse_indexes(all_indexes())
self._iter = SphinxQuery(self.query_string, self._query_args)
self._result_cache = []
self._metadata = self._iter.meta
if not self._only_meta:
self._fill_cache()
## Options
def _parse_indexes(self, index):
if index is None:
return list()
return [x.lower() for x in re.split(self.__index_match, index) if x]
def _get_index(self):
return ' '.join(self._indexes)
def _format_options_dict(self, d):
return '(%s)' % ', '.join(['%s=%s' % (x, d[x]) for x in d])
def _format_options(self, **kwargs):
if not kwargs:
return ''
opts = []
for k, v in kwargs.iteritems():
if isinstance(v, bool):
v = int(v)
elif isinstance(v, dict):
v = self._format_options_dict(v)
opts.append('%s=%s' % (k, v))
return 'OPTION %s' % ','.join(opts)
## Cache
def _fill_cache(self, num=None):
fields = self.meta['fields'].copy()
id_pos = fields.pop('id')
ct = None
results = {}
docs = OrderedDict()
if self._iter:
try:
while True:
doc = self._iter.next()
doc_id = doc[id_pos]
obj_id, ct = self._decode_document_id(int(doc_id))
results.setdefault(ct, {})[obj_id] = {}
docs.setdefault(doc_id, {})['results'] = results[ct][obj_id]
docs[doc_id]['data'] = {}
for field in fields:
docs[doc_id]['data'].setdefault('fields', {})[field] = doc[fields[field]]
except StopIteration:
self._iter = None
if not docs:
self._result_cache = []
return
if self.model is None and len(self._indexes) == 1 and ct is not None:
self.model = ContentType.objects.get(pk=ct).model_class()
if self.model:
qs = self.get_query_set(self.model)
qs = qs.filter(pk__in=results[ct].keys())
for obj in qs:
results[ct][obj.pk]['obj'] = obj
else:
for ct in results:
model_class = ContentType.objects.get(pk=ct).model_class()
qs = self.get_query_set(model_class).filter(pk__in=results[ct].keys())
for obj in qs:
results[ct][obj.pk]['obj'] = obj
#clear missing items
for pk in [pk for pk, doc in docs.items() if not 'obj' in doc['results']]:
del docs[pk]
if self._snippets:
for doc in docs.values():
doc['data']['snippets'] = self._get_snippets(doc['results']['obj'])
self._result_cache.append(SphinxProxy(doc['results']['obj'], doc['data']))
else:
for doc in docs.values():
self._result_cache.append(SphinxProxy(doc['results']['obj'], doc['data']))
## Snippets
def _get_snippets(self, instance):
(fields, docs) = zip(*[(f, getattr(instance, f)) for f in self._get_doc_fields(instance) if getattr(instance, f)])
opts = self._get_snippets_string()
doc_format = ', '.join('%s' for x in range(0, len(fields)))
query = 'CALL SNIPPETS (({0:>s}), \'{1:>s}\', %s {2:>s})'.format(doc_format,
instance.__sphinx_indexes__[0],
opts)
docs += (self._query or '',)
c = conn_handler.cursor()
c.execute(query, docs)
snippets = {}
for field in fields:
snippets[field] = c.fetchone()[0].decode('utf-8')
return snippets
def _get_doc_fields(self, instance):
cache = self._doc_fields_cache.get(type(instance), None)
if cache is None:
def _get_field(name):
return instance._meta.get_field(name)
opts = instance.__sphinx_options__
included = opts.get('included_fields', [])
excluded = opts.get('excluded_fields', []) + self._excluded_fields
stored_attrs = opts.get('stored_attributes', [])
stored_fields = opts.get('stored_fields', [])
if included:
included = [f for f in included if
f not in excluded
and
get_sphinx_attr_type_for_field(_get_field(f)) == 'string']
for f in stored_fields:
if get_sphinx_attr_type_for_field(_get_field(f)) == 'string':
included.append(f)
else:
included = [f.name for f in instance._meta.fields
if
f.name not in excluded
and
(f.name not in stored_attrs
or
f.name in stored_fields)
and
get_sphinx_attr_type_for_field(f) == 'string']
cache = self._doc_fields_cache[type(instance)] = included
return cache
def _get_index_fields(self):
if self._index_fields_cache is None:
opts = self.model.__sphinx_options__
excluded = opts.get('excluded_fields', [])
fields = []
for f in ['included_fields', 'stored_attributes',
'stored_fields', 'related_fields', 'mva_fields']:
fields.extend(opts.get(f, []))
for f in excluded:
if f in fields:
fields.pop(fields.index(f))
fields.insert(0, 'id')
self._index_fields_cache = fields
return self._index_fields_cache
## Documents
def _decode_document_id(self, doc_id):
"""\
Декодирует ID документа, полученного от Sphinx
:param doc_id: ID документа
:type doc_id: long
:returns: tuple(ContentTypeID, ObjectID)
:rtype: tuple\
"""
assert isinstance(doc_id, six.integer_types)
ct = (doc_id & CONTENT_TYPE_MASK) >> DOCUMENT_ID_SHIFT
return (doc_id & OBJECT_ID_MASK, ct)
def _encode_document_id(self, id):
if self.model:
ct = ContentType.objects.get_for_model(self.model)
id = int(ct.id) << DOCUMENT_ID_SHIFT | id
return id
## Filters
def _process_single_obj_operation(self, obj):
if isinstance(obj, models.Model):
if self.model is None:
raise ValueError('For non model or multiple model indexes comparsion with objects not supported')
value = obj.pk
elif not isinstance(obj, (list, tuple, QuerySet)):
value = obj
else:
raise TypeError('Comparison operations require a single object, not a `%s`' % type(obj))
return to_sphinx(value)
def _process_obj_list_operation(self, obj_list):
if isinstance(obj_list, (models.Model, QuerySet)):
if self.model is None:
raise ValueError('For non model or multiple model indexes comparsion with objects not supported')
if isinstance(obj_list, models.Model):
values = [obj_list.pk]
else:
values = [obj.pk for obj in obj_list]
elif hasattr(obj_list, '__iter__') or isinstance(obj_list, (list, tuple)):
values = list(obj_list)
elif isinstance(obj_list, (int, float, date, datetime)):
values = [obj_list]
else:
raise ValueError('`%s` is not a list of objects and not single object' % type(obj_list))
return map(to_sphinx, values)
def _process_filters(self, filters, exclude=False, **kwargs):
for k, v in kwargs.iteritems():
if len(k.split('__')) > 3:
raise NotImplementedError('Related model fields lookup not supported')
parts = k.rsplit('__', 1)
parts_len = len(parts)
field = parts[0]
lookup = parts[-1]
if field == 'pk': # приводим pk к id
field = 'id'
if parts_len == 1: # один
if field == 'id':
v = self._encode_document_id(self._process_single_obj_operation(v))
self._doc_ids = [v]
else:
v = self._process_single_obj_operation(v)
filters[field] = '%s %s %s' % (field,
'!=' if exclude else '=',
v)
elif parts_len == 2: # один exact или список, или сравнение
if lookup == 'in':
if field == 'id':
v = map(self._encode_document_id, self._process_obj_list_operation(v))
self._doc_ids = v
else:
v = self._process_obj_list_operation(v)
filters[field] = '%s %sIN (%s)' % (field,
'NOT ' if exclude else '',
','.join(str(x) for x in v))
elif lookup == 'range':
v = self._process_obj_list_operation(v)
if len(v) != 2:
raise ValueError('Range may consist of two values')
if exclude:
# not supported by sphinx. raises error!
warnings.warn('Exclude range not supported by SphinxQL now!')
filters[field] = 'NOT %s BETWEEN %i AND %i' % (field, v[0], v[1])
else:
filters[field] = '%s BETWEEN %i AND %i' % (field, v[0], v[1])
elif lookup in FILTER_CMP_OPERATIONS:
filters[field] = '%s %s %s' % (field,
FILTER_CMP_INVERSE[lookup]\
if exclude\
else FILTER_CMP_OPERATIONS[lookup],
self._process_single_obj_operation(v))
else: # stored related field
filters[k] = '%s %s %s' % (k,
'!=' if exclude else '=',
self._process_single_obj_operation(v))
return filters
## Query
def _build_query(self):
self._query_args = []
q = ['SELECT']
q.extend(self._build_fields())
q.extend(['FROM', ', '.join(self._indexes)])
q.extend(self._build_where())
q.append(self._build_group_by())
q.append(self._build_order_by())
q.append(self._build_group_order_by())
q.extend(self._build_limits())
if self._query_opts is not None:
q.append(self._query_opts)
return ' '.join(q)
query_string = property(_build_query)
def _build_fields(self):
q = []
if self._fields:
q.append(self._fields)
if self._aliases:
q.append(', ')
q.append(', '.join(self._aliases.values()))
if self._snippet_dict:
q.append(', ')
q.append(', '.join(["SNIPPET(%s, %%s) as `%s`" % (attr, to_field) for to_field, attr in self._snippet_dict.items()]))
self._query_args += [self._query] * len(self._snippet_dict)
return q
def _build_where(self):
q = []
if self._query or self._filters or self._excludes:
q.append('WHERE')
if self._query:
q.append('MATCH(%s)')
self._query_args.append(self._query)
if self._filters or self._excludes:
q.append('AND')
if self._filters:
q.append(' AND '.join(self._filters.values()))
if self._excludes:
q.append('AND')
if self._excludes:
q.append(' AND '.join(self._excludes.values()))
return q
def _build_group_by(self):
return self._group_by
def _build_order_by(self):
return self._order_by
def _build_group_order_by(self):
return self._group_order_by
def _build_limits(self):
if not self._limit is None and self._offset is None:
return ''
q = ['LIMIT']
if self._offset is not None:
q.append('%i,' % self._offset)
q.append('%i' % (self._limit if self._limit is not None else self._maxmatches))
return q
## Clone
def _clone(self, **kwargs):
"""\
Clones the queryset passing any changed args\
"""
c = self.__class__()
c.__dict__.update(self.__dict__.copy())
c._result_cache = None
c._metadata = None
c._iter = None
for k, v in kwargs.iteritems():
setattr(c, k, v)
return c
