def set_cursor(queryset, start=None, end=None):
if start is not None:
start = Cursor.from_websafe_string(start)
queryset.query._gae_start_cursor = start
if end is not None:
end = Cursor.from_websafe_string(end)
queryset.query._gae_end_cursor = end
# Evaluate QuerySet
len(queryset)
def set_cursor(queryset, start=None, end=None):
if start is not None:
start = Cursor.from_websafe_string(start)
queryset.query._gae_start_cursor = start
if end is not None:
end = Cursor.from_websafe_string(end)
queryset.query._gae_end_cursor = end
# Evaluate QuerySet
len(queryset)
def set_cursor(queryset, start=None, end=None):
queryset = _add_mixin(queryset)
if start is not None:
start = Cursor.from_websafe_string(start)
setattr(queryset.query, '_gae_start_cursor', start)
if end is not None:
end = Cursor.from_websafe_string(end)
setattr(queryset.query, '_gae_end_cursor', end)
return queryset
def set_cursor(queryset, start=None, end=None):
queryset = _add_mixin(queryset)
if start is not None:
start = Cursor.from_websafe_string(start)
setattr(queryset.query, '_gae_start_cursor', start)
if end is not None:
end = Cursor.from_websafe_string(end)
setattr(queryset.query, '_gae_end_cursor', end)
return queryset
def set_cursor(queryset, start=None, end=None):
queryset = queryset.all()
class CursorQuery(CursorQueryMixin, queryset.query.__class__):
pass
queryset.query = queryset.query.clone(klass=CursorQuery)
if start is not None:
start = Cursor.from_websafe_string(start)
queryset.query._gae_start_cursor = start
if end is not None:
end = Cursor.from_websafe_string(end)
queryset.query._gae_end_cursor = end
return queryset
def set_cursor(queryset, start=None, end=None):
queryset = queryset.all()
class CursorQuery(CursorQueryMixin, queryset.query.__class__):
pass
queryset.query = queryset.query.clone(klass=CursorQuery)
if start is not None:
start = Cursor.from_websafe_string(start)
queryset.query._gae_start_cursor = start
if end is not None:
end = Cursor.from_websafe_string(end)
queryset.query._gae_end_cursor = end
# Evaluate QuerySet
return queryset
def put(self):
body = self.readJson()
if body.get("cursor"):
logging.info("Continuing consolidation of functions")
(q, cursor,
more) = counters.FunctionCounter.get_dirties().fetch_page(
page_size=CHUNK_SIZE_FUNC,
start_cursor=Cursor.from_websafe_string(body.get("cursor")))
else:
logging.info("Starting consolidation of functions")
(q, cursor,
more) = counters.FunctionCounter.get_dirties().fetch_page(
page_size=CHUNK_SIZE_FUNC)
logging.info("{} Function Records taken".format(len(q)))
if more:
task = taskqueue.add(url='/function',
target='consolidator',
method='PUT',
payload="{\"cursor\": \"%s\"}" %
cursor.to_websafe_string())
logging.info(
"Queueing taks in order to continue consolidating functions")
for f in q:
counters.FunctionCounter.consolidate(f)
logging.info("{} Function Records proccessed".format(len(q)))
if more:
self.SendJsonOKMessage('Task {} enqueued, ETA {}.'.format(
task.name, task.eta))
else:
logging.info("Consolidation of functions done")
self.SendJsonOKMessage('Consolidation done')
def retrieve_dbs(query, order=None, limit=None, cursor=None, **filters):
""" Retrieves entities from datastore, by applying cursor pagination
and equality filters. Returns dbs and more cursor value
"""
limit = limit or config.DEFAULT_DB_LIMIT
cursor = Cursor.from_websafe_string(cursor) if cursor else None
model_class = ndb.Model._kind_map[query.kind]
if order:
for o in order.split(","):
if o.startswith("-"):
query = query.order(-model_class._properties[o[1:]])
else:
query = query.order(model_class._properties[o])
for prop in filters:
if filters.get(prop, None) is None:
continue
if isinstance(filters[prop], list):
for value in filters[prop]:
query = query.filter(model_class._properties[prop] == value)
else:
query = query.filter(model_class._properties[prop] == filters[prop])
model_dbs, more_cursor, more = query.fetch_page(limit, start_cursor=cursor)
more_cursor = more_cursor.to_websafe_string() if more else None
return list(model_dbs), more_cursor
def retrieve_dbs(model_class, query, order=None, limit=None, cursor=None,
**filters):
''' Retrieves entities from datastore, by applying cursor pagindation
and equality filters. Returns dbs and more cursor value
'''
limit = limit or config.DEFAULT_DB_LIMIT
if cursor:
cursor = Cursor.from_websafe_string(cursor)
#apply order if any
if order:
for o in order.split(','):
if o.startswith('-'):
query = query.order(-model_class._properties[o[1:]])
else:
query = query.order(model_class._properties[o])
for prop in filters:
if filters.get(prop, None) is None:
continue
if type(filters[prop]) == list:
for value in filters[prop]:
query = query.filter(model_class._properties[prop] == value)
else:
query = query.filter(model_class._properties[prop] == filters[prop])
model_dbs, more_cursor, more = query.fetch_page(limit, start_cursor=cursor)
if not more:
more_cursor = None
else:
more_cursor = more_cursor.to_websafe_string()
return list(model_dbs), more_cursor
def get_dbs(
query, order=None, limit=None, cursor=None, keys_only=None, **filters
):
limit = limit or config.DEFAULT_DB_LIMIT
cursor = Cursor.from_websafe_string(cursor) if cursor else None
model_class = ndb.Model._kind_map[query.kind]
if order:
for o in order.split(','):
if o.startswith('-'):
query = query.order(-model_class._properties[o[1:]])
else:
query = query.order(model_class._properties[o])
for prop in filters:
if filters.get(prop, None) is None:
continue
if isinstance(filters[prop], list):
for value in filters[prop]:
query = query.filter(model_class._properties[prop] == value)
else:
query = query.filter(model_class._properties[prop] == filters[prop])
model_dbs, next_cursor, more = query.fetch_page(
limit, start_cursor=cursor, keys_only=keys_only,
)
next_cursor = next_cursor.to_websafe_string() if more else None
return list(model_dbs), next_cursor
def retrieve_dbs(
query, order=None, limit=None, cursor=None, keys_only=None, **filters
):
limit = limit or config.DEFAULT_DB_LIMIT
cursor = Cursor.from_websafe_string(cursor) if cursor else None
model_class = ndb.Model._kind_map[query.kind]
if order:
for o in order.split(','):
if o.startswith('-'):
query = query.order(-model_class._properties[o[1:]])
else:
query = query.order(model_class._properties[o])
for prop in filters:
if filters.get(prop, None) is None:
continue
if isinstance(filters[prop], list):
for value in filters[prop]:
query = query.filter(model_class._properties[prop] == value)
else:
query = query.filter(model_class._properties[prop] == filters[prop])
model_dbs, more_cursor, more = query.fetch_page(
limit, start_cursor=cursor, keys_only=keys_only,
)
more_cursor = more_cursor.to_websafe_string() if more else None
return list(model_dbs), more_cursor
def retrieve_dbs(query, order=None, limit=None, cursor=None, **filters):
''' Retrieves entities from datastore, by applying cursor pagination
and equality filters. Returns dbs and more cursor value
'''
limit = limit or config.DEFAULT_DB_LIMIT
cursor = Cursor.from_websafe_string(cursor) if cursor else None
model_class = ndb.Model._kind_map[query.kind]
if order:
for o in order.split(','):
if o.startswith('-'):
query = query.order(-model_class._properties[o[1:]])
else:
query = query.order(model_class._properties[o])
for prop in filters:
if filters.get(prop, None) is None:
continue
if isinstance(filters[prop], list):
for value in filters[prop]:
query = query.filter(model_class._properties[prop] == value)
else:
query = query.filter(
model_class._properties[prop] == filters[prop])
model_dbs, more_cursor, more = query.fetch_page(limit, start_cursor=cursor)
more_cursor = more_cursor.to_websafe_string() if more else None
return list(model_dbs), more_cursor
def all(cls, cursor):
page_size = int(os.environ['PAGE_SIZE'])
query = cls.query().order(-cls.creation_time)
if cursor:
cursor_obj = Cursor.from_websafe_string(cursor)
return query.fetch_page(page_size, start_cursor=cursor_obj)
return query.fetch_page(page_size)
def all(cls, cursor):
page_size = int(os.environ['PAGE_SIZE'])
q = cls.query(cls.status.IN([StatusType.ACTIVE, StatusType.BANNED]))\
.order(cls.key)
if cursor:
cursor_obj = Cursor.from_websafe_string(cursor)
return q.fetch_page(page_size, start_cursor=cursor_obj)
return q.fetch_page(page_size)
def by_creator(cls, user_key, cursor):
page_size = int(os.environ['PAGE_SIZE'])
query = cls.query(
cls.status == StatusType.ACTIVE,
cls.creator_key == user_key).order(cls.status, cls.key)
if cursor:
cursor_obj = Cursor.from_websafe_string(cursor)
return query.fetch_page(page_size, start_cursor=cursor_obj)
return query.fetch_page(page_size)
def set_cursor(queryset, start=None, end=None):
from google.appengine.datastore.datastore_query import Cursor
queryset = queryset.all()
if CursorQueryMixin not in queryset.query.__class__.__bases__:
class CursorQuery(CursorQueryMixin, queryset.query.__class__):
pass
queryset.query = queryset.query.clone(klass=CursorQuery)
if start is not None:
start = Cursor.from_websafe_string(start)
queryset.query._gae_start_cursor = start
if end is not None:
end = Cursor.from_websafe_string(end)
queryset.query._gae_end_cursor = end
return queryset
def by_research(cls, research_key, cursor):
page_size = int(os.environ['PAGE_SIZE'])
query = cls.query(
cls.status == StatusType.ACTIVE,
cls.research_key == research_key)\
.order(-cls.creation_time, cls.status, cls.key)
if cursor:
cursor_obj = Cursor.from_websafe_string(cursor)
return query.fetch_page(page_size, start_cursor=cursor_obj)
return query.fetch_page(page_size)
def get_comments_more(self,key):
cursor=self.param('next')
entry=Entry.get(key)
from google.appengine.datastore.datastore_query import Cursor
cur=Cursor.from_websafe_string(cursor)
comments,cursor,more=Comment.query().filter(Comment.entry ==entry.key).order(-Comment.date).fetch_page(10,start_cursor=cur)
vals= dict(entry=entry, comments=comments,cursor=more and cursor.to_websafe_string() or '',more=more)
html=self.get_render('comments_more',vals)
self.write(html)
def by_user(cls, user_key, cursor):
page_size = int(os.environ['PAGE_SIZE'])
q = cls.query(ndb.OR(
cls.supervisor_key == user_key,
cls.researchers_keys == user_key),
cls.status.IN([StatusType.ACTIVE, StatusType.BANNED]))\
.order(cls.key)
if cursor:
cursor_obj = Cursor.from_websafe_string(cursor)
return q.fetch_page(page_size, start_cursor=cursor_obj)
return q.fetch_page(page_size)
def get_dbs(query,
order=None,
limit=None,
cursor=None,
prev_cursor=False,
keys_only=None,
**filters):
model_class = ndb.Model._kind_map[query.kind]
query_prev = query
if order:
for o in order.split(','):
if o.startswith('-'):
query = query.order(-model_class._properties[o[1:]])
if prev_cursor:
query_prev = query_prev.order(
model_class._properties[o[1:]])
else:
query = query.order(model_class._properties[o])
if prev_cursor:
query_prev = query_prev.order(-model_class._properties[o])
for prop, value in filters.iteritems():
if value is None:
continue
for val in value if isinstance(value, list) else [value]:
query = query.filter(model_class._properties[prop] == val)
if prev_cursor:
query_prev = query_prev.filter(
model_class._properties[prop] == val)
limit = limit or config.DEFAULT_DB_LIMIT
if limit is -1:
return list(query.fetch(keys_only=keys_only)), {
'next': None,
'prev': None
}
cursor = Cursor.from_websafe_string(cursor) if cursor else None
model_dbs, next_cursor, more = query.fetch_page(
limit,
start_cursor=cursor,
keys_only=keys_only,
)
next_cursor = next_cursor.to_websafe_string() if more else None
if not prev_cursor:
return list(model_dbs), {'next': next_cursor, 'prev': None}
model_dbs_prev, prev_cursor, prev_more = query_prev.fetch_page(
limit,
start_cursor=cursor.reversed() if cursor else None,
keys_only=True)
prev_cursor = prev_cursor.reversed().to_websafe_string() \
if prev_cursor and cursor else None
return list(model_dbs), {'next': next_cursor, 'prev': prev_cursor}
def retrieve_list(cls,
limit=None,
order=None,
cursor=None,
keys_only=None,
**filters):
'''Retrieves entities from datastore, by applying cursor pagination
and equality filters. Returns dbs or keys and more cursor value
model_class = ndb.Model._kind_map[query.kind]
'''
query = cls.query()
limit = limit or config.DEFAULT_DB_LIMIT
orderBy = order or "-created"
# Cursor(urlsafe=cursor)
cursor = Cursor.from_websafe_string(cursor) if cursor else None
model_class = cls
for prop in filters:
if filters.get(prop, None) is None:
continue
fp = filters[prop]
mp = model_class._properties[prop]
if isinstance(fp, list):
nodes = [ndb.OR(mp == fi) for fi in fp]
query = query.filter(ndb.OR(*nodes))
# required key order: http://stackoverflow.com/questions/12449197/badargumenterror-multiquery-with-cursors-requires-key-order-in-ndb
query = query.order(model_class._key)
elif isinstance(fp, tuple):
p_len = len(fp)
if p_len == 1:
query = query.filter(mp != fp[0])
elif p_len == 2:
query = query.filter(ndb.AND(mp >= fp[0], mp <= fp[1]))
# unequal filters require first sort property to be the sames
orderBy = prop + "," + orderBy
else:
query = query.filter(mp == fp)
if orderBy:
for o in orderBy.split(','):
if o.startswith('-'):
query = query.order(-model_class._properties[o[1:]])
else:
query = query.order(model_class._properties[o])
model_dbs, more_cursor, more = query.fetch_page(
limit,
start_cursor=cursor,
keys_only=keys_only,
)
# more_cursor = more_cursor.urlsafe()
more_cursor = more_cursor.to_websafe_string() if more else None
return list(model_dbs), more_cursor
def get_dbs(
query, order=None, limit=None, cursor=None, keys_only=None, **filters
):
limit = limit or config.DEFAULT_DB_LIMIT
cursor = Cursor.from_websafe_string(cursor) if cursor else None
model_class = ndb.Model._kind_map[query.kind]
for prop in filters:
if filters.get(prop, None) is None:
continue
if isinstance(filters[prop], list):
for value in filters[prop]:
query = query.filter(model_class._properties[prop] == value)
# new custom wodor app -------------
elif isinstance(filters[prop], dict):
if filters[prop]['test'] == '>':
query = query.filter(model_class._properties[prop] > filters[prop]['value'])
elif filters[prop]['test'] == '>=':
query = query.filter(model_class._properties[prop] >= filters[prop]['value'])
elif filters[prop]['test'] == '<':
query = query.filter(model_class._properties[prop] < filters[prop]['value'])
elif filters[prop]['test'] == '<=':
query = query.filter(model_class._properties[prop] < filters[prop]['value'])
elif filters[prop]['test'] == '==':
query = query.filter(model_class._properties[prop] == filters[prop]['value'])
elif filters[prop]['test'] == '!=':
query = query.filter(model_class._properties[prop] != filters[prop]['value'])
elif filters[prop]['test'] == 'IN':
values = filters[prop]['value']
if isinstance(values, list):
values = filters[prop]['value']
else:
values = values.split(',')
query = query.filter(model_class._properties[prop].IN(values))
query = query.order(model_class._key)
query = query.order(model_class._properties[prop]) # TODO does it work?
else:
query = query.filter(model_class._properties[prop] == filters[prop])
# ----------------------------------
if order:
for o in order.split(','):
if o.startswith('-'):
query = query.order(-model_class._properties[o[1:]])
else:
query = query.order(model_class._properties[o])
model_dbs, next_cursor, more = query.fetch_page(
limit, start_cursor=cursor, keys_only=keys_only,
)
next_cursor = next_cursor.to_websafe_string() if more else None
return list(model_dbs), next_cursor
def fetch_cursor(self,next_cursor='',prev_cursor='',order=None):
from google.appengine.datastore.datastore_query import Cursor
## cur=Cursor.from_websafe_string(p)
## results,cur2,more= self.query.fetch_page(self.items_per_page,start_cursor=cur)
## # setup query
if prev_cursor:
query = self.query.order(order.reversed())
cur=Cursor.from_websafe_string(prev_cursor)
elif next_cursor:
query = self.query.order(order)
cur=Cursor.from_websafe_string(next_cursor)
else:
query = self.query.order(order)
cur=None
# fetch results
results,cur2,more = query.fetch_page(self.items_per_page,start_cursor=cur)
if prev_cursor:
results.reverse()
sPrev=''
sNext=''
# pagination
if prev_cursor:
sNext=cur.reversed().to_websafe_string()
sPrev=more and cur2.to_websafe_string() or ''
else:
if next_cursor:
sPrev=cur.reversed().to_websafe_string()
sNext=more and cur2.to_websafe_string() or ''
links = { 'prev': sPrev, 'next': sNext, 'more': more }
##
## links = {'prev': cur2.reversed().to_websafe_string(), 'next':more and cur2.to_websafe_string() or '','more':more }
##
return (results, links)
def artists(self, request):
'''API endpoint to query for artists'''
next_page = request.next_page or 'first_artists_page'
cache = memcache.get(next_page)
if cache:
return ArtistsResponse(artists=cache[0], next_page=cache[1])
query = Artist.query()
if next_page is 'first_artists_page':
artists, cursor, more = query.fetch_page(300)
else:
artists, cursor, more = query.fetch_page(300,
start_cursor=Cursor.from_websafe_string(next_page))
artists = [artist.to_message() for artist in artists]
memcache.add(next_page, (artists, cursor.to_websafe_string()))
return ArtistsResponse(artists=artists, next_page=cursor.to_websafe_string())
def find_all(cls, cursor, keyword):
page_size = int(os.environ['PAGE_SIZE'])
q = cls.query(cls.status.IN([StatusType.ACTIVE, StatusType.BANNED])) \
if keyword:
q = q.filter(cls.email >= keyword)
q = q.filter(cls.email < keyword + u'\ufffd')
q = q.order(cls.email, cls.key)
if cursor:
cursor_obj = Cursor.from_websafe_string(cursor)
return q.fetch_page(page_size, start_cursor=cursor_obj)
return q.fetch_page(page_size)
def validate_request(cls, request):
args = cls.get_dwc_args(request)
if len(args) == 0 and request.get('q', None) is None:
return None
keywords = [x.lower() for x in request.get('q', '').split(',') if x]
limit = request.get_range('limit', min_value=1, max_value=100, default=10)
offset = request.get('offset', None)
cursor = None
if offset:
cursor = Cursor.from_websafe_string(offset)
return dict(
args=args,
keywords=keywords,
limit=limit,
offset=offset,
cursor=cursor)
def get_dbs(
query, order=None, limit=None, cursor=None, keys_only=None, **filters
):
limit = limit or flask.current_app.config.get('DEFAULT_DB_LIMIT')
cursor = Cursor.from_websafe_string(cursor) if cursor else None
model_class = ndb.Model._kind_map[query.kind]
query_reverse = query
query_forward = query
if order:
for o in order.split(','):
if o.startswith('-'):
query_reverse = query_reverse.order(model_class._properties[o[1:]])
query_forward = query_forward.order(-model_class._properties[o[1:]])
else:
query_reverse = query_reverse.order(-model_class._properties[o])
query_forward = query_forward.order(model_class._properties[o])
for prop in filters:
if filters.get(prop, None) is None:
continue
if funcy.is_seqcoll(filters[prop]):
for value in filters[prop]:
query_reverse = query_reverse.filter(
model_class._properties[prop] == value
)
query_forward = query_forward.filter(
model_class._properties[prop] == value
)
else:
query_reverse = query_reverse.filter(
model_class._properties[prop] == filters[prop]
)
query_forward = query_forward.filter(
model_class._properties[prop] == filters[prop]
)
model_dbs, next_cursor, more = query_forward.fetch_page(
limit, start_cursor=cursor, keys_only=keys_only,
)
model_dbs_prev, prev_cursor, rev_more = query_reverse.fetch_page(
limit, start_cursor=cursor.reversed() if cursor else None, keys_only=True
)
next_cursor = next_cursor.urlsafe() if more else None
prev_cursor = prev_cursor.reversed().urlsafe()\
if prev_cursor and cursor else None
return list(model_dbs), next_cursor, prev_cursor
def retrieve_list(cls, limit=None, order=None, cursor=None, keys_only=None, **filters):
'''Retrieves entities from datastore, by applying cursor pagination
and equality filters. Returns dbs or keys and more cursor value
model_class = ndb.Model._kind_map[query.kind]
'''
query = cls.query()
limit = limit or config.DEFAULT_DB_LIMIT
orderBy = order or "-created"
# Cursor(urlsafe=cursor)
cursor = Cursor.from_websafe_string(cursor) if cursor else None
model_class = cls
for prop in filters:
if filters.get(prop, None) is None:
continue
fp = filters[prop]
mp = model_class._properties[prop]
if isinstance(fp, list):
nodes = [ndb.OR(mp == fi) for fi in fp]
query = query.filter(ndb.OR(*nodes))
# required key order: http://stackoverflow.com/questions/12449197/badargumenterror-multiquery-with-cursors-requires-key-order-in-ndb
query = query.order(model_class._key)
elif isinstance(fp, tuple):
p_len = len(fp)
if p_len == 1:
query = query.filter(mp != fp[0])
elif p_len == 2:
query = query.filter(ndb.AND(mp >= fp[0], mp <= fp[1]))
# unequal filters require first sort property to be the sames
orderBy = prop + "," + orderBy
else:
query = query.filter(mp == fp)
if orderBy:
for o in orderBy.split(','):
if o.startswith('-'):
query = query.order(-model_class._properties[o[1:]])
else:
query = query.order(model_class._properties[o])
model_dbs, more_cursor, more = query.fetch_page(
limit, start_cursor=cursor, keys_only=keys_only,
)
# more_cursor = more_cursor.urlsafe()
more_cursor = more_cursor.to_websafe_string() if more else None
return list(model_dbs), more_cursor
def get(self):
nextc = self.request.get("next")
time = int(self.request.get("time"))
if nextc:
(q, cursor, more) = report.FunctionReportEntry.getEntriesLastHours(
time).fetch_page(page_size=FUNCTIONS_PAGE_SIZE,
start_cursor=Cursor.from_websafe_string(nextc))
else:
(q, cursor, more) = report.FunctionReportEntry.getEntriesLastHours(
time).fetch_page(page_size=FUNCTIONS_PAGE_SIZE)
q = [{'name': e.name, 'total': e.total, 'date': e.date.isoformat()} for
e in q]
response = {}
response['result'] = q
response['more'] = more
if more:
response['next'] = cursor.to_websafe_string()
self.SendJson(response)
def get(self):
nextc = self.request.get("next")
date = datetime.today()
if nextc:
(q, cursor, more) = report.ApplicationReportEntry.getEntriesByDate(
date).fetch_page(page_size=APPLICATIONS_PAGE_SIZE,
start_cursor=Cursor.from_websafe_string(nextc))
else:
(q, cursor, more) = report.ApplicationReportEntry.getEntriesByDate(
date).fetch_page(page_size=APPLICATIONS_PAGE_SIZE)
q = [{'name': e.name, 'count': e.count, 'date': e.date.isoformat()} for
e in q]
response = {}
response['result'] = q
response['more'] = more
if more:
response['next'] = cursor.to_websafe_string()
self.SendJson(response)
def get_dbs(
query, order=None, limit=None, cursor=None, prev_cursor=False,
keys_only=None, **filters
):
model_class = ndb.Model._kind_map[query.kind]
query_prev = query
if order:
for o in order.split(','):
if o.startswith('-'):
query = query.order(-model_class._properties[o[1:]])
if prev_cursor:
query_prev = query_prev.order(
model_class._properties[o[1:]])
else:
query = query.order(model_class._properties[o])
if prev_cursor:
query_prev = query_prev.order(-model_class._properties[o])
for prop, value in filters.iteritems():
if value is None:
continue
for val in value if isinstance(value, list) else [value]:
query = query.filter(model_class._properties[prop] == val)
if prev_cursor:
query_prev = query_prev.filter(
model_class._properties[prop] == val)
limit = limit or config.DEFAULT_DB_LIMIT
if limit is -1:
return list(query.fetch(keys_only=keys_only)), {'next': None, 'prev': None}
cursor = Cursor.from_websafe_string(cursor) if cursor else None
model_dbs, next_cursor, more = query.fetch_page(
limit, start_cursor=cursor, keys_only=keys_only,
)
next_cursor = next_cursor.to_websafe_string() if more else None
if not prev_cursor:
return list(model_dbs), {'next': next_cursor, 'prev': None}
model_dbs_prev, prev_cursor, prev_more = query_prev.fetch_page(
limit, start_cursor=cursor.reversed() if cursor else None, keys_only=True
)
prev_cursor = prev_cursor.reversed().to_websafe_string()\
if prev_cursor and cursor else None
return list(model_dbs), {'next': next_cursor, 'prev': prev_cursor}
def retrieve_dbs_pager(
query, prev=False, order=None, limit=None, cursor=None, keys_only=None, **filters
):
'''Retrieves entities from datastore, by applying cursor pagination
and equality filters. Returns dbs or keys and more cursor value
and return prev_page and next_page cursor
'''
limit = limit or config.DEFAULT_DB_LIMIT
cursor = Cursor.from_websafe_string(cursor) if cursor else None
if cursor and prev:
cursor = cursor.reversed()
model_class = ndb.Model._kind_map[query.kind]
if order:
for o in order.split(','):
if o.startswith('-'):
qry = query.order(-model_class._properties[o[1:]])
qry_r = query.order(model_class._properties[o[1:]])
else:
qry = query.order(model_class._properties[o])
qry_r = query.order(-model_class._properties[o])
for prop in filters:
if filters.get(prop, None) is None:
continue
if isinstance(filters[prop], list):
for value in filters[prop]:
qry = qry.filter(model_class._properties[prop] == value)
qry_r = qry_r.filter(model_class._properties[prop] == value)
else:
qry = qry.filter(model_class._properties[prop] == filters[prop])
qry_r = qry_r.filter(model_class._properties[prop] == filters[prop])
model_dbs, more_cursor, more = qry.fetch_page(
limit, start_cursor=cursor, keys_only=keys_only,
)
more_cursor = more_cursor.to_websafe_string() if more else None
prev_more_cursor = None
if cursor:
prev_model_dbs, prev_more_cursor, prev_more = qry_r.fetch_page(
limit, start_cursor=cursor.reversed(),keys_only=True,
)
prev_more_cursor = prev_more_cursor.to_websafe_string() if prev_more_cursor else None
# thought prev_more is False, but we did't render that page, so there's a page left (prev)
return list(model_dbs), more_cursor, prev_more_cursor
def get_closest_cursor(self, start, cursors):
keys = filter(lambda x: x< start , cursors.keys())
if keys:
lower_cursor = cursors.get(max(*keys))
else:
lower_cursor = None
diff_list = [start]
if lower_cursor:
lower_diff = start-lower_cursor.start
diff_list.append(lower_diff)
diff = sorted(diff_list)[0]
if lower_cursor:
if diff == lower_diff:
cursor=Cursor.from_websafe_string(lower_cursor.cursor)
if diff == start:
cursor = None
logging.debug('the closest cursor found, diff: %s' %(str(diff)))
return cursor, diff
def get_dbs(
query, order=None, limit=None, cursor=None, keys_only=None, **filters
):
limit = limit or config.DEFAULT_DB_LIMIT
cursor = Cursor.from_websafe_string(cursor) if cursor else None
model_class = ndb.Model._kind_map[query.kind]
if order:
for o in order.split(','):
if o.startswith('-'):
query = query.order(-model_class._properties[o[1:]])
else:
query = query.order(model_class._properties[o])
for prop, value in filters.iteritems():
if value is None:
continue
for val in value if isinstance(value, list) else [value]:
query = query.filter(model_class._properties[prop] == val)
model_dbs, next_cursor, more = query.fetch_page(
limit, start_cursor=cursor, keys_only=keys_only,
)
next_cursor = next_cursor.to_websafe_string() if more else None
return list(model_dbs), next_cursor
def cursor_for(websafe_string):
if websafe_string is None:
return None
return Cursor.from_websafe_string(websafe_string)
def __call__(self):
# logging.debug('Paginator started with initial vals: currpage %s, q %s, rows %s ' %(str(currpage), str(q), str(rows)))
FETCH_LIMIT = 1000
start = self.rows*(self.currpage - 1)
query_kind = self.q.kind
query_key = self.get_query_id(self.q, self.rows)
query_map_changed = False
query_map = memcache.get(query_kind)
if query_map is None:
query_map ={query_key: {}}
#print 'memcache cursor:' + str(cursor)
total_query_objects = query_map.setdefault(query_key, {}).get('total')
totalpages = query_map.setdefault(query_key, {}).get('pages')
if total_query_objects is None:
total_query_objects = self.q.count()
query_map[query_key]['total'] = total_query_objects
totalpages = int(math.ceil(total_query_objects/float(self.rows)))
query_map[query_key]['pages'] = totalpages
query_map_changed = True
objects = None
more = False
try:
logging.debug('trying to get objects directly from memcache')
objects = query_map[query_key]['cursors'][start]['objects']
more = query_map[query_key]['cursors'][start]['more']
logging.debug('got some objects!')
except KeyError:
query_map[query_key]['cursors'] = {}
query_map_changed = True
if objects is None:
try:
cursor = query_map[query_key]['cursors'][start]['cursor']
except KeyError:
query_map[query_key]['cursors'] = {}
query_map_changed = True
cursor = None
if cursor is not None:
logging.debug('cursor key found in memcache')
cursor = Cursor.from_websafe_string(cursor)
else:
logging.debug('cursor key not found in memcache. Start looking for the closest cursor')
cursor, diff = self.get_closest_cursor(start, query_map[query_key]['cursors'])
#firstly, lets set the cursor on start of page
while diff > FETCH_LIMIT:
logging.debug('diff is %s moving cursor...' %(diff,))
objects, cursor, more = self.q.fetch_page(FETCH_LIMIT, start_cursor=cursor, keys_only=True)
query_map[query_key]['cursors'][start+diff] = dict(cursor=cursor.to_websafe_string())
query_map_changed = True
diff -= FETCH_LIMIT
#ok, lets set cursor more precisely
if diff:
objects, cursor, more = self.q.fetch_page(diff, start_cursor=cursor, keys_only=True)
query_map[query_key]['cursors'][start] = dict(cursor=cursor.to_websafe_string())
query_map_changed = True
objects, cursor, more = self.q.fetch_page(self.rows, start_cursor=cursor)
if objects:
query_map[query_key]['cursors'][start+len(objects)] = dict(cursor = cursor.to_websafe_string())
query_map[query_key]['cursors'].setdefault(start, {'objects': objects})['objects'] = objects
query_map[query_key]['cursors'][start]['more'] = more
if query_map_changed:
memcache.set(query_kind, query_map)
return dict(objects=objects, totalpages=totalpages, totalrecords=total_query_objects, more=more)
def cursor_for(websafe_string):
if websafe_string is None:
return None
return Cursor.from_websafe_string(websafe_string)
