def search(query, model):
""" Performs a search query and returns the object ids """
query = '%s*' % query.strip()
LOGGER.debug(query)
sqs = SearchQuerySet()
sqs = sqs.filter(content=query).models(model)
return [o.pk for o in sqs]
def _filter(request, object_, tags=None, models=(Image, Video), rng=None, more=False):
"""Filters Piece objects from self based on filters, search, and range
:param tags: List of tag IDs to filter
:type tags: list
:param models: List of model classes to filter on
:type models: list
:param rng: Range of objects to return. i.e. 0:100
:type rng: str
:param more -- bool, Returns more of the same filtered set of images based on session range
return list, Objects filtered
"""
NOW = time.clock()
res = Result()
idDict = {}
objDict = {}
lastIDs = {}
data = {}
LOGGER.debug('init: %f' % (time.clock() - NOW))
if request.user.is_anonymous():
gRange = 300
else:
Prefs = json.loads(UserPref.objects.get(user=request.user).data)
gRange = Prefs['batchSize']
request.session.setdefault('frog_range', '0:%i' % gRange)
if rng:
s, e = [int(x) for x in rng.split(':')]
else:
if more:
s = int(request.session.get('frog_range', '0:%i' % gRange).split(':')[1])
e = s + gRange
s, e = 0, gRange
else:
s, e = 0, gRange
## -- Gat all IDs for each model
for m in models:
indexes = list(m.model_class().objects.all().values_list('id', flat=True))
if not indexes:
continue
lastIndex = indexes[0]
if more:
## -- This is a request for more results
idx = request.session.get('last_%s_id' % m.model, lastIndex + 1)
lastIDs.setdefault('last_%s_id' % m.model, idx)
else:
lastIDs['last_%s_id' % m.model] = lastIndex + 1
## -- Start with objects within range
idDict[m.model] = m.model_class().objects.filter(gallery=object_, id__lt=lastIDs['last_%s_id' % m.model])
LOGGER.debug(m.model + '_initial_query: %f' % (time.clock() - NOW))
if tags:
for bucket in tags:
searchQuery = ""
o = None
for item in bucket:
## -- filter by tag
if isinstance(item, int) or isinstance(item, long):
if not o:
o = Q()
o |= Q(tags__id=item)
## -- add to search string
else:
searchQuery += item + ' '
if not HAYSTACK:
if not o:
o = Q()
## -- use a basic search
LOGGER.debug('search From LIKE')
o |= Q(title__icontains=item)
if HAYSTACK and searchQuery != "":
## -- once all tags have been filtered, filter by search
searchIDs = search(searchQuery, m.model_class())
if searchIDs:
if not o:
o = Q()
LOGGER.debug('searchFrom haystack:' + str(searchIDs))
o |= Q(id__in=searchIDs)
if o:
## -- apply the filters
idDict[m.model] = idDict[m.model].filter(o)
else:
idDict[m.model] = idDict[m.model].none()
LOGGER.debug(m.model + '_added_buckets(%i): %f' % (len(tags), time.clock() - NOW))
## -- Get all ids of filtered objects, this will be a very fast query
idDict[m.model] = list(idDict[m.model].values_list('id', flat=True))
LOGGER.debug(m.model + '_queried_ids: %f' % (time.clock() - NOW))
res.message = str(s) + ':' + str(e)
## -- perform the main query to retrieve the objects we want
objDict[m.model] = m.model_class().objects.filter(id__in=idDict[m.model]).select_related('author').prefetch_related('tags')
if not rng:
objDict[m.model] = objDict[m.model][:gRange]
objDict[m.model] = list(objDict[m.model])
LOGGER.debug(m.model + '_queried_obj: %f' % (time.clock() - NOW))
## -- combine and sort all objects by date
objects = _sortObjects(**objDict) if len(models) > 1 else objDict.values()[0]
objects = objects[s:e]
LOGGER.debug('sorted: %f' % (time.clock() - NOW))
## -- serialize objects
for i in objects:
for m in models:
if isinstance(i, m.model_class()):
## -- set the last ID per model for future lookups
lastIDs['last_%s_id' % m.model] = i.id
data['last_%s_id' % m.model] = i.id
res.append(i.json())
LOGGER.debug('serialized: %f' % (time.clock() - NOW))
request.session['frog_range'] = ':'.join((str(s),str(e)))
LOGGER.debug('total: %f' % (time.clock() - NOW))
request.session['last_image_id'] = lastIDs.get('last_image_id', 0)
request.session['last_video_id'] = lastIDs.get('last_video_id', 0)
data['count'] = len(objects)
data['queries'] = connection.queries
res.value = data
res.isSuccess = True
return JsonResponse(res)
