def set_cursor(queryset, start=None, end=None): if start is not None: start = Cursor.from_websafe_string(start) queryset.query._gae_start_cursor = start if end is not None: end = Cursor.from_websafe_string(end) queryset.query._gae_end_cursor = end # Evaluate QuerySet len(queryset)
def set_cursor(queryset, start=None, end=None): if start is not None: start = Cursor.from_websafe_string(start) queryset.query._gae_start_cursor = start if end is not None: end = Cursor.from_websafe_string(end) queryset.query._gae_end_cursor = end # Evaluate QuerySet len(queryset)
def set_cursor(queryset, start=None, end=None): queryset = _add_mixin(queryset) if start is not None: start = Cursor.from_websafe_string(start) setattr(queryset.query, '_gae_start_cursor', start) if end is not None: end = Cursor.from_websafe_string(end) setattr(queryset.query, '_gae_end_cursor', end) return queryset
def set_cursor(queryset, start=None, end=None): queryset = _add_mixin(queryset) if start is not None: start = Cursor.from_websafe_string(start) setattr(queryset.query, '_gae_start_cursor', start) if end is not None: end = Cursor.from_websafe_string(end) setattr(queryset.query, '_gae_end_cursor', end) return queryset
def set_cursor(queryset, start=None, end=None): queryset = queryset.all() class CursorQuery(CursorQueryMixin, queryset.query.__class__): pass queryset.query = queryset.query.clone(klass=CursorQuery) if start is not None: start = Cursor.from_websafe_string(start) queryset.query._gae_start_cursor = start if end is not None: end = Cursor.from_websafe_string(end) queryset.query._gae_end_cursor = end return queryset
def set_cursor(queryset, start=None, end=None): queryset = queryset.all() class CursorQuery(CursorQueryMixin, queryset.query.__class__): pass queryset.query = queryset.query.clone(klass=CursorQuery) if start is not None: start = Cursor.from_websafe_string(start) queryset.query._gae_start_cursor = start if end is not None: end = Cursor.from_websafe_string(end) queryset.query._gae_end_cursor = end # Evaluate QuerySet return queryset
def put(self): body = self.readJson() if body.get("cursor"): logging.info("Continuing consolidation of functions") (q, cursor, more) = counters.FunctionCounter.get_dirties().fetch_page( page_size=CHUNK_SIZE_FUNC, start_cursor=Cursor.from_websafe_string(body.get("cursor"))) else: logging.info("Starting consolidation of functions") (q, cursor, more) = counters.FunctionCounter.get_dirties().fetch_page( page_size=CHUNK_SIZE_FUNC) logging.info("{} Function Records taken".format(len(q))) if more: task = taskqueue.add(url='/function', target='consolidator', method='PUT', payload="{\"cursor\": \"%s\"}" % cursor.to_websafe_string()) logging.info( "Queueing taks in order to continue consolidating functions") for f in q: counters.FunctionCounter.consolidate(f) logging.info("{} Function Records proccessed".format(len(q))) if more: self.SendJsonOKMessage('Task {} enqueued, ETA {}.'.format( task.name, task.eta)) else: logging.info("Consolidation of functions done") self.SendJsonOKMessage('Consolidation done')
def retrieve_dbs(query, order=None, limit=None, cursor=None, **filters): """ Retrieves entities from datastore, by applying cursor pagination and equality filters. Returns dbs and more cursor value """ limit = limit or config.DEFAULT_DB_LIMIT cursor = Cursor.from_websafe_string(cursor) if cursor else None model_class = ndb.Model._kind_map[query.kind] if order: for o in order.split(","): if o.startswith("-"): query = query.order(-model_class._properties[o[1:]]) else: query = query.order(model_class._properties[o]) for prop in filters: if filters.get(prop, None) is None: continue if isinstance(filters[prop], list): for value in filters[prop]: query = query.filter(model_class._properties[prop] == value) else: query = query.filter(model_class._properties[prop] == filters[prop]) model_dbs, more_cursor, more = query.fetch_page(limit, start_cursor=cursor) more_cursor = more_cursor.to_websafe_string() if more else None return list(model_dbs), more_cursor
def retrieve_dbs(model_class, query, order=None, limit=None, cursor=None, **filters): ''' Retrieves entities from datastore, by applying cursor pagindation and equality filters. Returns dbs and more cursor value ''' limit = limit or config.DEFAULT_DB_LIMIT if cursor: cursor = Cursor.from_websafe_string(cursor) #apply order if any if order: for o in order.split(','): if o.startswith('-'): query = query.order(-model_class._properties[o[1:]]) else: query = query.order(model_class._properties[o]) for prop in filters: if filters.get(prop, None) is None: continue if type(filters[prop]) == list: for value in filters[prop]: query = query.filter(model_class._properties[prop] == value) else: query = query.filter(model_class._properties[prop] == filters[prop]) model_dbs, more_cursor, more = query.fetch_page(limit, start_cursor=cursor) if not more: more_cursor = None else: more_cursor = more_cursor.to_websafe_string() return list(model_dbs), more_cursor
def get_dbs( query, order=None, limit=None, cursor=None, keys_only=None, **filters ): limit = limit or config.DEFAULT_DB_LIMIT cursor = Cursor.from_websafe_string(cursor) if cursor else None model_class = ndb.Model._kind_map[query.kind] if order: for o in order.split(','): if o.startswith('-'): query = query.order(-model_class._properties[o[1:]]) else: query = query.order(model_class._properties[o]) for prop in filters: if filters.get(prop, None) is None: continue if isinstance(filters[prop], list): for value in filters[prop]: query = query.filter(model_class._properties[prop] == value) else: query = query.filter(model_class._properties[prop] == filters[prop]) model_dbs, next_cursor, more = query.fetch_page( limit, start_cursor=cursor, keys_only=keys_only, ) next_cursor = next_cursor.to_websafe_string() if more else None return list(model_dbs), next_cursor
def retrieve_dbs( query, order=None, limit=None, cursor=None, keys_only=None, **filters ): limit = limit or config.DEFAULT_DB_LIMIT cursor = Cursor.from_websafe_string(cursor) if cursor else None model_class = ndb.Model._kind_map[query.kind] if order: for o in order.split(','): if o.startswith('-'): query = query.order(-model_class._properties[o[1:]]) else: query = query.order(model_class._properties[o]) for prop in filters: if filters.get(prop, None) is None: continue if isinstance(filters[prop], list): for value in filters[prop]: query = query.filter(model_class._properties[prop] == value) else: query = query.filter(model_class._properties[prop] == filters[prop]) model_dbs, more_cursor, more = query.fetch_page( limit, start_cursor=cursor, keys_only=keys_only, ) more_cursor = more_cursor.to_websafe_string() if more else None return list(model_dbs), more_cursor
def retrieve_dbs(query, order=None, limit=None, cursor=None, **filters): ''' Retrieves entities from datastore, by applying cursor pagination and equality filters. Returns dbs and more cursor value ''' limit = limit or config.DEFAULT_DB_LIMIT cursor = Cursor.from_websafe_string(cursor) if cursor else None model_class = ndb.Model._kind_map[query.kind] if order: for o in order.split(','): if o.startswith('-'): query = query.order(-model_class._properties[o[1:]]) else: query = query.order(model_class._properties[o]) for prop in filters: if filters.get(prop, None) is None: continue if isinstance(filters[prop], list): for value in filters[prop]: query = query.filter(model_class._properties[prop] == value) else: query = query.filter( model_class._properties[prop] == filters[prop]) model_dbs, more_cursor, more = query.fetch_page(limit, start_cursor=cursor) more_cursor = more_cursor.to_websafe_string() if more else None return list(model_dbs), more_cursor
def all(cls, cursor): page_size = int(os.environ['PAGE_SIZE']) query = cls.query().order(-cls.creation_time) if cursor: cursor_obj = Cursor.from_websafe_string(cursor) return query.fetch_page(page_size, start_cursor=cursor_obj) return query.fetch_page(page_size)
def all(cls, cursor): page_size = int(os.environ['PAGE_SIZE']) q = cls.query(cls.status.IN([StatusType.ACTIVE, StatusType.BANNED]))\ .order(cls.key) if cursor: cursor_obj = Cursor.from_websafe_string(cursor) return q.fetch_page(page_size, start_cursor=cursor_obj) return q.fetch_page(page_size)
def by_creator(cls, user_key, cursor): page_size = int(os.environ['PAGE_SIZE']) query = cls.query( cls.status == StatusType.ACTIVE, cls.creator_key == user_key).order(cls.status, cls.key) if cursor: cursor_obj = Cursor.from_websafe_string(cursor) return query.fetch_page(page_size, start_cursor=cursor_obj) return query.fetch_page(page_size)
def set_cursor(queryset, start=None, end=None): from google.appengine.datastore.datastore_query import Cursor queryset = queryset.all() if CursorQueryMixin not in queryset.query.__class__.__bases__: class CursorQuery(CursorQueryMixin, queryset.query.__class__): pass queryset.query = queryset.query.clone(klass=CursorQuery) if start is not None: start = Cursor.from_websafe_string(start) queryset.query._gae_start_cursor = start if end is not None: end = Cursor.from_websafe_string(end) queryset.query._gae_end_cursor = end return queryset
def by_research(cls, research_key, cursor): page_size = int(os.environ['PAGE_SIZE']) query = cls.query( cls.status == StatusType.ACTIVE, cls.research_key == research_key)\ .order(-cls.creation_time, cls.status, cls.key) if cursor: cursor_obj = Cursor.from_websafe_string(cursor) return query.fetch_page(page_size, start_cursor=cursor_obj) return query.fetch_page(page_size)
def get_comments_more(self,key): cursor=self.param('next') entry=Entry.get(key) from google.appengine.datastore.datastore_query import Cursor cur=Cursor.from_websafe_string(cursor) comments,cursor,more=Comment.query().filter(Comment.entry ==entry.key).order(-Comment.date).fetch_page(10,start_cursor=cur) vals= dict(entry=entry, comments=comments,cursor=more and cursor.to_websafe_string() or '',more=more) html=self.get_render('comments_more',vals) self.write(html)
def by_user(cls, user_key, cursor): page_size = int(os.environ['PAGE_SIZE']) q = cls.query(ndb.OR( cls.supervisor_key == user_key, cls.researchers_keys == user_key), cls.status.IN([StatusType.ACTIVE, StatusType.BANNED]))\ .order(cls.key) if cursor: cursor_obj = Cursor.from_websafe_string(cursor) return q.fetch_page(page_size, start_cursor=cursor_obj) return q.fetch_page(page_size)
def get_dbs(query, order=None, limit=None, cursor=None, prev_cursor=False, keys_only=None, **filters): model_class = ndb.Model._kind_map[query.kind] query_prev = query if order: for o in order.split(','): if o.startswith('-'): query = query.order(-model_class._properties[o[1:]]) if prev_cursor: query_prev = query_prev.order( model_class._properties[o[1:]]) else: query = query.order(model_class._properties[o]) if prev_cursor: query_prev = query_prev.order(-model_class._properties[o]) for prop, value in filters.iteritems(): if value is None: continue for val in value if isinstance(value, list) else [value]: query = query.filter(model_class._properties[prop] == val) if prev_cursor: query_prev = query_prev.filter( model_class._properties[prop] == val) limit = limit or config.DEFAULT_DB_LIMIT if limit is -1: return list(query.fetch(keys_only=keys_only)), { 'next': None, 'prev': None } cursor = Cursor.from_websafe_string(cursor) if cursor else None model_dbs, next_cursor, more = query.fetch_page( limit, start_cursor=cursor, keys_only=keys_only, ) next_cursor = next_cursor.to_websafe_string() if more else None if not prev_cursor: return list(model_dbs), {'next': next_cursor, 'prev': None} model_dbs_prev, prev_cursor, prev_more = query_prev.fetch_page( limit, start_cursor=cursor.reversed() if cursor else None, keys_only=True) prev_cursor = prev_cursor.reversed().to_websafe_string() \ if prev_cursor and cursor else None return list(model_dbs), {'next': next_cursor, 'prev': prev_cursor}
def retrieve_list(cls, limit=None, order=None, cursor=None, keys_only=None, **filters): '''Retrieves entities from datastore, by applying cursor pagination and equality filters. Returns dbs or keys and more cursor value model_class = ndb.Model._kind_map[query.kind] ''' query = cls.query() limit = limit or config.DEFAULT_DB_LIMIT orderBy = order or "-created" # Cursor(urlsafe=cursor) cursor = Cursor.from_websafe_string(cursor) if cursor else None model_class = cls for prop in filters: if filters.get(prop, None) is None: continue fp = filters[prop] mp = model_class._properties[prop] if isinstance(fp, list): nodes = [ndb.OR(mp == fi) for fi in fp] query = query.filter(ndb.OR(*nodes)) # required key order: http://stackoverflow.com/questions/12449197/badargumenterror-multiquery-with-cursors-requires-key-order-in-ndb query = query.order(model_class._key) elif isinstance(fp, tuple): p_len = len(fp) if p_len == 1: query = query.filter(mp != fp[0]) elif p_len == 2: query = query.filter(ndb.AND(mp >= fp[0], mp <= fp[1])) # unequal filters require first sort property to be the sames orderBy = prop + "," + orderBy else: query = query.filter(mp == fp) if orderBy: for o in orderBy.split(','): if o.startswith('-'): query = query.order(-model_class._properties[o[1:]]) else: query = query.order(model_class._properties[o]) model_dbs, more_cursor, more = query.fetch_page( limit, start_cursor=cursor, keys_only=keys_only, ) # more_cursor = more_cursor.urlsafe() more_cursor = more_cursor.to_websafe_string() if more else None return list(model_dbs), more_cursor
def get_dbs( query, order=None, limit=None, cursor=None, keys_only=None, **filters ): limit = limit or config.DEFAULT_DB_LIMIT cursor = Cursor.from_websafe_string(cursor) if cursor else None model_class = ndb.Model._kind_map[query.kind] for prop in filters: if filters.get(prop, None) is None: continue if isinstance(filters[prop], list): for value in filters[prop]: query = query.filter(model_class._properties[prop] == value) # new custom wodor app ------------- elif isinstance(filters[prop], dict): if filters[prop]['test'] == '>': query = query.filter(model_class._properties[prop] > filters[prop]['value']) elif filters[prop]['test'] == '>=': query = query.filter(model_class._properties[prop] >= filters[prop]['value']) elif filters[prop]['test'] == '<': query = query.filter(model_class._properties[prop] < filters[prop]['value']) elif filters[prop]['test'] == '<=': query = query.filter(model_class._properties[prop] < filters[prop]['value']) elif filters[prop]['test'] == '==': query = query.filter(model_class._properties[prop] == filters[prop]['value']) elif filters[prop]['test'] == '!=': query = query.filter(model_class._properties[prop] != filters[prop]['value']) elif filters[prop]['test'] == 'IN': values = filters[prop]['value'] if isinstance(values, list): values = filters[prop]['value'] else: values = values.split(',') query = query.filter(model_class._properties[prop].IN(values)) query = query.order(model_class._key) query = query.order(model_class._properties[prop]) # TODO does it work? else: query = query.filter(model_class._properties[prop] == filters[prop]) # ---------------------------------- if order: for o in order.split(','): if o.startswith('-'): query = query.order(-model_class._properties[o[1:]]) else: query = query.order(model_class._properties[o]) model_dbs, next_cursor, more = query.fetch_page( limit, start_cursor=cursor, keys_only=keys_only, ) next_cursor = next_cursor.to_websafe_string() if more else None return list(model_dbs), next_cursor
def fetch_cursor(self,next_cursor='',prev_cursor='',order=None): from google.appengine.datastore.datastore_query import Cursor ## cur=Cursor.from_websafe_string(p) ## results,cur2,more= self.query.fetch_page(self.items_per_page,start_cursor=cur) ## # setup query if prev_cursor: query = self.query.order(order.reversed()) cur=Cursor.from_websafe_string(prev_cursor) elif next_cursor: query = self.query.order(order) cur=Cursor.from_websafe_string(next_cursor) else: query = self.query.order(order) cur=None # fetch results results,cur2,more = query.fetch_page(self.items_per_page,start_cursor=cur) if prev_cursor: results.reverse() sPrev='' sNext='' # pagination if prev_cursor: sNext=cur.reversed().to_websafe_string() sPrev=more and cur2.to_websafe_string() or '' else: if next_cursor: sPrev=cur.reversed().to_websafe_string() sNext=more and cur2.to_websafe_string() or '' links = { 'prev': sPrev, 'next': sNext, 'more': more } ## ## links = {'prev': cur2.reversed().to_websafe_string(), 'next':more and cur2.to_websafe_string() or '','more':more } ## return (results, links)
def artists(self, request): '''API endpoint to query for artists''' next_page = request.next_page or 'first_artists_page' cache = memcache.get(next_page) if cache: return ArtistsResponse(artists=cache[0], next_page=cache[1]) query = Artist.query() if next_page is 'first_artists_page': artists, cursor, more = query.fetch_page(300) else: artists, cursor, more = query.fetch_page(300, start_cursor=Cursor.from_websafe_string(next_page)) artists = [artist.to_message() for artist in artists] memcache.add(next_page, (artists, cursor.to_websafe_string())) return ArtistsResponse(artists=artists, next_page=cursor.to_websafe_string())
def find_all(cls, cursor, keyword): page_size = int(os.environ['PAGE_SIZE']) q = cls.query(cls.status.IN([StatusType.ACTIVE, StatusType.BANNED])) \ if keyword: q = q.filter(cls.email >= keyword) q = q.filter(cls.email < keyword + u'\ufffd') q = q.order(cls.email, cls.key) if cursor: cursor_obj = Cursor.from_websafe_string(cursor) return q.fetch_page(page_size, start_cursor=cursor_obj) return q.fetch_page(page_size)
def validate_request(cls, request): args = cls.get_dwc_args(request) if len(args) == 0 and request.get('q', None) is None: return None keywords = [x.lower() for x in request.get('q', '').split(',') if x] limit = request.get_range('limit', min_value=1, max_value=100, default=10) offset = request.get('offset', None) cursor = None if offset: cursor = Cursor.from_websafe_string(offset) return dict( args=args, keywords=keywords, limit=limit, offset=offset, cursor=cursor)
def get_dbs( query, order=None, limit=None, cursor=None, keys_only=None, **filters ): limit = limit or flask.current_app.config.get('DEFAULT_DB_LIMIT') cursor = Cursor.from_websafe_string(cursor) if cursor else None model_class = ndb.Model._kind_map[query.kind] query_reverse = query query_forward = query if order: for o in order.split(','): if o.startswith('-'): query_reverse = query_reverse.order(model_class._properties[o[1:]]) query_forward = query_forward.order(-model_class._properties[o[1:]]) else: query_reverse = query_reverse.order(-model_class._properties[o]) query_forward = query_forward.order(model_class._properties[o]) for prop in filters: if filters.get(prop, None) is None: continue if funcy.is_seqcoll(filters[prop]): for value in filters[prop]: query_reverse = query_reverse.filter( model_class._properties[prop] == value ) query_forward = query_forward.filter( model_class._properties[prop] == value ) else: query_reverse = query_reverse.filter( model_class._properties[prop] == filters[prop] ) query_forward = query_forward.filter( model_class._properties[prop] == filters[prop] ) model_dbs, next_cursor, more = query_forward.fetch_page( limit, start_cursor=cursor, keys_only=keys_only, ) model_dbs_prev, prev_cursor, rev_more = query_reverse.fetch_page( limit, start_cursor=cursor.reversed() if cursor else None, keys_only=True ) next_cursor = next_cursor.urlsafe() if more else None prev_cursor = prev_cursor.reversed().urlsafe()\ if prev_cursor and cursor else None return list(model_dbs), next_cursor, prev_cursor
def retrieve_list(cls, limit=None, order=None, cursor=None, keys_only=None, **filters): '''Retrieves entities from datastore, by applying cursor pagination and equality filters. Returns dbs or keys and more cursor value model_class = ndb.Model._kind_map[query.kind] ''' query = cls.query() limit = limit or config.DEFAULT_DB_LIMIT orderBy = order or "-created" # Cursor(urlsafe=cursor) cursor = Cursor.from_websafe_string(cursor) if cursor else None model_class = cls for prop in filters: if filters.get(prop, None) is None: continue fp = filters[prop] mp = model_class._properties[prop] if isinstance(fp, list): nodes = [ndb.OR(mp == fi) for fi in fp] query = query.filter(ndb.OR(*nodes)) # required key order: http://stackoverflow.com/questions/12449197/badargumenterror-multiquery-with-cursors-requires-key-order-in-ndb query = query.order(model_class._key) elif isinstance(fp, tuple): p_len = len(fp) if p_len == 1: query = query.filter(mp != fp[0]) elif p_len == 2: query = query.filter(ndb.AND(mp >= fp[0], mp <= fp[1])) # unequal filters require first sort property to be the sames orderBy = prop + "," + orderBy else: query = query.filter(mp == fp) if orderBy: for o in orderBy.split(','): if o.startswith('-'): query = query.order(-model_class._properties[o[1:]]) else: query = query.order(model_class._properties[o]) model_dbs, more_cursor, more = query.fetch_page( limit, start_cursor=cursor, keys_only=keys_only, ) # more_cursor = more_cursor.urlsafe() more_cursor = more_cursor.to_websafe_string() if more else None return list(model_dbs), more_cursor
def get(self): nextc = self.request.get("next") time = int(self.request.get("time")) if nextc: (q, cursor, more) = report.FunctionReportEntry.getEntriesLastHours( time).fetch_page(page_size=FUNCTIONS_PAGE_SIZE, start_cursor=Cursor.from_websafe_string(nextc)) else: (q, cursor, more) = report.FunctionReportEntry.getEntriesLastHours( time).fetch_page(page_size=FUNCTIONS_PAGE_SIZE) q = [{'name': e.name, 'total': e.total, 'date': e.date.isoformat()} for e in q] response = {} response['result'] = q response['more'] = more if more: response['next'] = cursor.to_websafe_string() self.SendJson(response)
def get(self): nextc = self.request.get("next") date = datetime.today() if nextc: (q, cursor, more) = report.ApplicationReportEntry.getEntriesByDate( date).fetch_page(page_size=APPLICATIONS_PAGE_SIZE, start_cursor=Cursor.from_websafe_string(nextc)) else: (q, cursor, more) = report.ApplicationReportEntry.getEntriesByDate( date).fetch_page(page_size=APPLICATIONS_PAGE_SIZE) q = [{'name': e.name, 'count': e.count, 'date': e.date.isoformat()} for e in q] response = {} response['result'] = q response['more'] = more if more: response['next'] = cursor.to_websafe_string() self.SendJson(response)
def get_dbs( query, order=None, limit=None, cursor=None, prev_cursor=False, keys_only=None, **filters ): model_class = ndb.Model._kind_map[query.kind] query_prev = query if order: for o in order.split(','): if o.startswith('-'): query = query.order(-model_class._properties[o[1:]]) if prev_cursor: query_prev = query_prev.order( model_class._properties[o[1:]]) else: query = query.order(model_class._properties[o]) if prev_cursor: query_prev = query_prev.order(-model_class._properties[o]) for prop, value in filters.iteritems(): if value is None: continue for val in value if isinstance(value, list) else [value]: query = query.filter(model_class._properties[prop] == val) if prev_cursor: query_prev = query_prev.filter( model_class._properties[prop] == val) limit = limit or config.DEFAULT_DB_LIMIT if limit is -1: return list(query.fetch(keys_only=keys_only)), {'next': None, 'prev': None} cursor = Cursor.from_websafe_string(cursor) if cursor else None model_dbs, next_cursor, more = query.fetch_page( limit, start_cursor=cursor, keys_only=keys_only, ) next_cursor = next_cursor.to_websafe_string() if more else None if not prev_cursor: return list(model_dbs), {'next': next_cursor, 'prev': None} model_dbs_prev, prev_cursor, prev_more = query_prev.fetch_page( limit, start_cursor=cursor.reversed() if cursor else None, keys_only=True ) prev_cursor = prev_cursor.reversed().to_websafe_string()\ if prev_cursor and cursor else None return list(model_dbs), {'next': next_cursor, 'prev': prev_cursor}
def retrieve_dbs_pager( query, prev=False, order=None, limit=None, cursor=None, keys_only=None, **filters ): '''Retrieves entities from datastore, by applying cursor pagination and equality filters. Returns dbs or keys and more cursor value and return prev_page and next_page cursor ''' limit = limit or config.DEFAULT_DB_LIMIT cursor = Cursor.from_websafe_string(cursor) if cursor else None if cursor and prev: cursor = cursor.reversed() model_class = ndb.Model._kind_map[query.kind] if order: for o in order.split(','): if o.startswith('-'): qry = query.order(-model_class._properties[o[1:]]) qry_r = query.order(model_class._properties[o[1:]]) else: qry = query.order(model_class._properties[o]) qry_r = query.order(-model_class._properties[o]) for prop in filters: if filters.get(prop, None) is None: continue if isinstance(filters[prop], list): for value in filters[prop]: qry = qry.filter(model_class._properties[prop] == value) qry_r = qry_r.filter(model_class._properties[prop] == value) else: qry = qry.filter(model_class._properties[prop] == filters[prop]) qry_r = qry_r.filter(model_class._properties[prop] == filters[prop]) model_dbs, more_cursor, more = qry.fetch_page( limit, start_cursor=cursor, keys_only=keys_only, ) more_cursor = more_cursor.to_websafe_string() if more else None prev_more_cursor = None if cursor: prev_model_dbs, prev_more_cursor, prev_more = qry_r.fetch_page( limit, start_cursor=cursor.reversed(),keys_only=True, ) prev_more_cursor = prev_more_cursor.to_websafe_string() if prev_more_cursor else None # thought prev_more is False, but we did't render that page, so there's a page left (prev) return list(model_dbs), more_cursor, prev_more_cursor
def get_closest_cursor(self, start, cursors): keys = filter(lambda x: x< start , cursors.keys()) if keys: lower_cursor = cursors.get(max(*keys)) else: lower_cursor = None diff_list = [start] if lower_cursor: lower_diff = start-lower_cursor.start diff_list.append(lower_diff) diff = sorted(diff_list)[0] if lower_cursor: if diff == lower_diff: cursor=Cursor.from_websafe_string(lower_cursor.cursor) if diff == start: cursor = None logging.debug('the closest cursor found, diff: %s' %(str(diff))) return cursor, diff
def get_dbs( query, order=None, limit=None, cursor=None, keys_only=None, **filters ): limit = limit or config.DEFAULT_DB_LIMIT cursor = Cursor.from_websafe_string(cursor) if cursor else None model_class = ndb.Model._kind_map[query.kind] if order: for o in order.split(','): if o.startswith('-'): query = query.order(-model_class._properties[o[1:]]) else: query = query.order(model_class._properties[o]) for prop, value in filters.iteritems(): if value is None: continue for val in value if isinstance(value, list) else [value]: query = query.filter(model_class._properties[prop] == val) model_dbs, next_cursor, more = query.fetch_page( limit, start_cursor=cursor, keys_only=keys_only, ) next_cursor = next_cursor.to_websafe_string() if more else None return list(model_dbs), next_cursor
def cursor_for(websafe_string): if websafe_string is None: return None return Cursor.from_websafe_string(websafe_string)
def __call__(self): # logging.debug('Paginator started with initial vals: currpage %s, q %s, rows %s ' %(str(currpage), str(q), str(rows))) FETCH_LIMIT = 1000 start = self.rows*(self.currpage - 1) query_kind = self.q.kind query_key = self.get_query_id(self.q, self.rows) query_map_changed = False query_map = memcache.get(query_kind) if query_map is None: query_map ={query_key: {}} #print 'memcache cursor:' + str(cursor) total_query_objects = query_map.setdefault(query_key, {}).get('total') totalpages = query_map.setdefault(query_key, {}).get('pages') if total_query_objects is None: total_query_objects = self.q.count() query_map[query_key]['total'] = total_query_objects totalpages = int(math.ceil(total_query_objects/float(self.rows))) query_map[query_key]['pages'] = totalpages query_map_changed = True objects = None more = False try: logging.debug('trying to get objects directly from memcache') objects = query_map[query_key]['cursors'][start]['objects'] more = query_map[query_key]['cursors'][start]['more'] logging.debug('got some objects!') except KeyError: query_map[query_key]['cursors'] = {} query_map_changed = True if objects is None: try: cursor = query_map[query_key]['cursors'][start]['cursor'] except KeyError: query_map[query_key]['cursors'] = {} query_map_changed = True cursor = None if cursor is not None: logging.debug('cursor key found in memcache') cursor = Cursor.from_websafe_string(cursor) else: logging.debug('cursor key not found in memcache. Start looking for the closest cursor') cursor, diff = self.get_closest_cursor(start, query_map[query_key]['cursors']) #firstly, lets set the cursor on start of page while diff > FETCH_LIMIT: logging.debug('diff is %s moving cursor...' %(diff,)) objects, cursor, more = self.q.fetch_page(FETCH_LIMIT, start_cursor=cursor, keys_only=True) query_map[query_key]['cursors'][start+diff] = dict(cursor=cursor.to_websafe_string()) query_map_changed = True diff -= FETCH_LIMIT #ok, lets set cursor more precisely if diff: objects, cursor, more = self.q.fetch_page(diff, start_cursor=cursor, keys_only=True) query_map[query_key]['cursors'][start] = dict(cursor=cursor.to_websafe_string()) query_map_changed = True objects, cursor, more = self.q.fetch_page(self.rows, start_cursor=cursor) if objects: query_map[query_key]['cursors'][start+len(objects)] = dict(cursor = cursor.to_websafe_string()) query_map[query_key]['cursors'].setdefault(start, {'objects': objects})['objects'] = objects query_map[query_key]['cursors'][start]['more'] = more if query_map_changed: memcache.set(query_kind, query_map) return dict(objects=objects, totalpages=totalpages, totalrecords=total_query_objects, more=more)
def cursor_for(websafe_string): if websafe_string is None: return None return Cursor.from_websafe_string(websafe_string)