def get_stats(cls,
cursor_key=None,
limit=None,
year=None,
topic=None,
sort_by=None,
**kw):
if topic:
return RoshReviewUserTopicStats.get_stats(topic,
cursor_key=cursor_key,
limit=limit,
year=year,
**kw)
limit = limit if limit else 20
sort_by = sort_by if sort_by else 'performance'
cursor = Cursor(urlsafe=cursor_key) if cursor_key else None
q = cls.query()
if year:
q = q.filter(cls.year == year)
q = q.order(-ndb.GenericProperty(sort_by))
stats, cursor, _ = q.fetch_page(limit, start_cursor=cursor, **kw)
return stats, (cursor.urlsafe() if cursor else None),
def cursor_pagination(cls, ancestor_key, prev_cursor_str, next_cursor_str, page_count):
if not prev_cursor_str and not next_cursor_str:
objects, next_cursor, more = cls.query(ancestor=ancestor_key).order(-cls.createDate).fetch_page(page_count)
prev_cursor_str = ''
if next_cursor:
next_cursor_str = next_cursor.urlsafe()
else:
next_cursor_str = ''
next_ = True if more else False
prev = False
elif next_cursor_str:
cursor = Cursor(urlsafe=next_cursor_str)
objects, next_cursor, more = cls.query(ancestor=ancestor_key).order(-cls.createDate).fetch_page(page_count, start_cursor=cursor)
prev_cursor_str = next_cursor_str
next_cursor_str = next_cursor.urlsafe()
prev = True
next_ = True if more else False
elif prev_cursor_str:
cursor = Cursor(urlsafe=prev_cursor_str)
objects, next_cursor, more = cls.query(ancestor=ancestor_key).order(cls.createDate).fetch_page(page_count, start_cursor=cursor)
objects.reverse()
next_cursor_str = prev_cursor_str
prev_cursor_str = next_cursor.urlsafe()
prev = True if more else False
next_ = True
return {'objects': objects, 'next_cursor': next_cursor_str, 'prev_cursor': prev_cursor_str, 'prev': prev,
'next': next_}
def get_template_params(self, vessel_key):
self.vessel_key = vessel_key
vessel = vessel_key.get()
wpt_qry = Waypoint.query(ancestor=vessel.key).order(-Waypoint.report_date, -Waypoint.received_date)
curs = Cursor(urlsafe=self.request.get('cursor'))
params = {'loginurl': users.create_login_url('/'),
'vessel': vessel,
'map' : GoogleMapTrack(vessel)
}
if self.request.get('cursor'):
params['start_url'] = self.get_base_url()
else:
params['start_url'] = ''
params['waypoints'], next_curs, params['older'] = wpt_qry.fetch_page(self.NUM_WAYPOINTS, start_cursor=curs)
params['this_page_url'] = self.get_base_url() + "?cursor=" + curs.urlsafe()
if params['older'] and next_curs:
params['next_page_url'] = self.get_base_url() + "?cursor=" + next_curs.urlsafe()
else:
params['older'] = False
# #Formulate reverse pointer if there is more recent waypoints
# rev_wpt_qry = Waypoint.query(ancestor=vessel.key).order(Waypoint.report_date, Waypoint.received_date)
# rev_curs = curs.reversed()
# _, prev_curs, params['newer'] = wpt_qry.fetch_page(self.NUM_WAYPOINTS, start_cursor=rev_curs)
# if params['newer'] and prev_curs:
# params['prev_page_url'] = self.get_base_url() + "?cursor=" + prev_curs.reversed().urlsafe()
# else:
# params['newer'] = False
return params
def get(self, post_key):
from google.appengine.datastore.datastore_query import Cursor
args = {}
next_cursor = self.request.get('next_cursor')
ckey = 'PostHandler.%s.%s' % (post_key, next_cursor)
cdata = memcache.get(ckey)
if cdata is not None:
args = cdata
else:
args['tags'] = self.tags()
post = Feed.query(Feed.key == ndb.Key(urlsafe=post_key)).get()
next_cursor = Cursor(urlsafe=next_cursor)
entryRef, next_cursor, more = Comment.query(
Comment.parent == post.key).order(
Comment.created_time).fetch_page(100,
start_cursor=next_cursor)
entries = []
for _entry in entryRef:
entry = _entry.to_dict()
entry['member'] = _entry.member.get().to_dict()
entries.append(entry)
post_key = post.key.urlsafe()
args['post'] = post.to_dict()
args['post']['message'] = message(args['post']['message'])
args['post']['member'] = post.member.get().to_dict()
args['comments'] = {}
args['comments']['entries'] = entries
args['comments']['next_cursor'] = next_cursor.urlsafe(
) if next_cursor else None
args['comments']['more'] = more
if not memcache.add(ckey, args, CACEH_POST_IN_PERMLINK):
logging.error('Memcache set failed.')
template = JINJA_ENVIRONMENT.get_template('/view/post.html')
self.response.write(template.render(args))
def projects(bookmark_cursor=None):
cursor = None
if bookmark_cursor:
cursor = Cursor(urlsafe=bookmark_cursor)
query = Project.query()
next_query = query.order(-Project.updated_at)
previous_query = query.order(Project.updated_at)
entities, next_cursor, has_next = next_query.fetch_page(
PER_PAGE, start_cursor=cursor)
if next_cursor:
next_cursor = next_cursor.urlsafe()
has_previous = False
previous_cursor = None
if cursor:
previous_cursor = cursor.reversed()
previous_entities, previous_cursor, has_previous = previous_query.fetch_page(
PER_PAGE, start_cursor=previous_cursor)
if previous_cursor:
has_previous = True
previous_cursor = previous_cursor.urlsafe()
return {
'entities': entities,
'nextCursor': next_cursor,
'hasNext': has_next,
'previousCursor': previous_cursor,
'hasPrevious': has_previous
}
def get_stats(
cls,
cursor_key=None,
limit=None,
year=None,
topic=None,
sort_by=None,
**kw
):
if topic:
return RoshReviewUserTopicStats.get_stats(
topic,
cursor_key=cursor_key,
limit=limit,
year=year,
**kw
)
limit = limit if limit else 20
sort_by = sort_by if sort_by else 'performance'
cursor = Cursor(urlsafe=cursor_key) if cursor_key else None
q = cls.query()
if year:
q = q.filter(cls.year == year)
q = q.order(-ndb.GenericProperty(sort_by))
stats, cursor, _ = q.fetch_page(limit, start_cursor=cursor, **kw)
return stats, (cursor.urlsafe() if cursor else None),
def xdb_fetch_page(query, limit, offset=None, start_cursor=None):
"""Pagination-ready fetching a some entities."""
if isinstance(query, ndb.Query):
if start_cursor:
if isinstance(start_cursor, basestring):
start_cursor = Cursor(urlsafe=start_cursor)
objects, cursor, more_objects = query.fetch_page(limit, start_cursor=start_cursor)
else:
objects, cursor, more_objects = query.fetch_page(limit, offset=offset)
elif isinstance(query, db.Query) or isinstance(query, db.GqlQuery):
if start_cursor:
if isinstance(start_cursor, Cursor):
start_cursor = start_cursor.urlsafe()
query.with_cursor(start_cursor)
objects = query.fetch(limit)
cursor = Cursor(urlsafe=query.cursor())
more_objects = len(objects) == limit
else:
objects = query.fetch(limit, offset=offset)
_cursor = query.cursor()
more_objects = query.with_cursor(_cursor).count(1) > 0
cursor = Cursor(urlsafe=_cursor)
else:
raise RuntimeError('unknown query class: %s' % type(query))
return objects, cursor, more_objects
def list(limit=10, cursor=None):
if cursor:
cursor = Cursor(urlsafe=cursor)
query = Book.query().order(Book.description)
entities, cursor, more = query.fetch_page(limit, start_cursor=cursor)
entities = builtin_list(map(from_datastore, entities))
return entities, cursor.urlsafe() if len(entities) == limit else None
def tag(tag):
cursor_str = util.param('cursor', str)
cursor = None
try:
cursor = Cursor(urlsafe=cursor_str)
except TypeError:
key = None
story_dbs, next_cursor, more = model.Story.query(
model.Story.tags == tag).filter(model.Story.story_item_count > 0).fetch_page(24,
start_cursor=cursor)
if len(story_dbs) == 0:
not_found = exceptions.NotFound()
raise not_found
params = {
'next_cursor': next_cursor.urlsafe(),
'tag': tag,
'current_cursor': cursor.urlsafe()
}
resp_model = {}
resp_model['html_class'] = 'tag'
resp_model['canonical_path'] = flask.url_for('tag', tag=tag)
decorate_page_response_model(resp_model)
decorate_stories_page_model(resp_model, story_dbs, params)
return flask.render_template('public/story/story_list.html', model=resp_model)
def cursor_pagination(prev_cursor_str, next_cursor_str):
PAGE_SIZE = 20
if not prev_cursor_str and not next_cursor_str:
p, next_cursor, more = models.get_pages_init(
self.type_name, PAGE_SIZE)
prev_cursor_str = ''
if next_cursor:
next_cursor_str = next_cursor.urlsafe()
else:
next_cursor_str = ''
next_ = True if more else False
prev_ = False
elif next_cursor_str:
cursor = Cursor(urlsafe=next_cursor_str)
p, next_cursor, more = models.get_pages_next(
self.type_name, cursor, PAGE_SIZE)
prev_cursor_str = next_cursor_str
next_cursor_str = next_cursor.urlsafe()
prev_ = True
next_ = True if more else False
elif prev_cursor_str:
cursor = Cursor(urlsafe=prev_cursor_str)
p, next_cursor, more = models.get_pages_prev(
self.type_name, cursor, PAGE_SIZE)
p.reverse()
next_cursor_str = prev_cursor_str
prev_cursor_str = next_cursor.urlsafe()
prev_ = True if more else False
next_ = True
return p, \
prev_cursor_str, \
next_cursor_str, \
prev_, \
next_
def getPhotos(prev_cursor_str, next_cursor_str, photosPerPage):
if not prev_cursor_str and not next_cursor_str:
photos, next_cursor, more = Models.Photo.query().order(
-Models.Photo.mod_time).fetch_page(photosPerPage)
prev_cursor_str = ''
if next_cursor:
next_cursor_str = next_cursor.urlsafe()
else:
next_cursor_str = ''
next_ = True if more else False
prev = False
elif next_cursor_str:
cursor = Cursor(urlsafe=next_cursor_str)
photos, next_cursor, more = Models.Photo.query().order(
-Models.Photo.mod_time).fetch_page(photosPerPage,
start_cursor=cursor)
prev_cursor_str = next_cursor_str
next_cursor_str = next_cursor.urlsafe()
prev = True
next_ = True if more else False
elif prev_cursor_str:
cursor = Cursor(urlsafe=prev_cursor_str)
photos, next_cursor, more = Models.Photo.query().order(
Models.Photo.mod_time).fetch_page(photosPerPage,
start_cursor=cursor)
photos.reverse()
next_cursor_str = prev_cursor_str
prev_cursor_str = next_cursor.urlsafe()
prev = True if more else False
next_ = True
return photos, next_cursor_str, prev_cursor_str, prev, next_
def get(self):
f_cursor_param = self.request.get('f_cursor')
b_cursor_param = self.request.get('b_cursor')
template_values = {}
if b_cursor_param:
words_query = Word.query(ancestor=create_dictionary_key()).order(Word.date)
b_cursor = Cursor(urlsafe=self.request.get('b_cursor'))
template_values['f_cursor'] = b_cursor.urlsafe()
rev_curs = b_cursor.reversed()
words, next_cursor, more = words_query.fetch_page(WORDS_PER_PAGE, start_cursor=rev_curs)
words.reverse()
if more and next_cursor:
template_values['b_cursor'] = next_cursor.reversed().urlsafe()
else:
words_query = Word.query(ancestor=create_dictionary_key()).order(-Word.date)
f_cursor = Cursor(urlsafe=self.request.get('f_cursor'))
template_values['b_cursor'] = f_cursor.urlsafe()
words, next_cursor, more = words_query.fetch_page(WORDS_PER_PAGE, start_cursor=f_cursor)
if more and next_cursor:
template_values['f_cursor'] = next_cursor.urlsafe()
wordDTOs = []
for word in words:
wordDTOs.append(WordDTO(word.word, word.description, word.example))
template_values['words'] = wordDTOs
self.response.out.write(template.render(get_template_path('view_words.html'), template_values))
def show_books():
fwd_query = model.Book.query().order(model.Book.title, model.Book.key)
rev_query = model.Book.query().order(-model.Book.title)
cursor = Cursor(urlsafe=request.values.get('cursor'))
direction = request.values.get('dir') or 'next'
fetch_args = {}
query = fwd_query
fetch_args['start_cursor'] = cursor
if direction == 'prev':
fetch_args['start_cursor'] = cursor.reversed()
query = rev_query
all_books, next_cursor, more = query.fetch_page(10, **fetch_args)
all_books.reverse()
return render_template(
'list_books.html',
list_heading="All Books",
books=all_books,
n_cursor=cursor.reversed().urlsafe(),
p_cursor=next_cursor.urlsafe(),
)
else:
all_books, next_cursor, more = query.fetch_page(10, **fetch_args)
return render_template(
'list_books.html',
list_heading="All Books",
books=all_books,
p_cursor=cursor.reversed().urlsafe(),
n_cursor=next_cursor.urlsafe(),
)
def get(self, post_key):
from google.appengine.datastore.datastore_query import Cursor
import json
next_cursor = Cursor(urlsafe=self.request.get('next_cursor'))
entries = []
ckey = 'CommentDataHandler.%s.%s' % (post_key,
self.request.get('next_cursor'))
cdata = memcache.get(ckey)
if cdata is not None:
cache = cdata
else:
post = Feed.query(Feed.key == ndb.Key(urlsafe=post_key)).get()
#syncComment(post);
entryRef, next_cursor, more = Comment.query(
Comment.parent == ndb.Key(urlsafe=post_key)).order(
Comment.created_time).fetch_page(COMMEMT_PAGE_SCALE,
start_cursor=next_cursor)
for _entry in entryRef:
entry = _entry.to_dict()
entry['member'] = _entry.member.get().to_dict()
entries.append(entry)
cache = {
'entries': entries,
'next_cursor': next_cursor.urlsafe() if next_cursor else None,
'more': more
}
if not memcache.add(ckey, cache, CACHE_COMMENT_IN_POST_TIME):
logging.error('Memcache set failed.')
self.response.write(json.dumps(cache))
def process (self, cursor=None):
if cursor:
cursor = Cursor(urlsafe=cursor)
total_clicks = 0
total_opens = 0
self.temp_clicks = {}
self.temp_opens = {}
if cursor == None: #skip all cursor continuations, these values are already init'd
self.tags = {}
self.urls = {}
self.clients = {}
self.clicks = []
self.opens = []
self.total_sends = 0
self.total_clicks = 0
self.total_opens = 0
from bulkmail.api.models import Campaign
c = Campaign.query(Campaign.campaign_id == self.campaign_id, Campaign.list_id == self.list_id).get()
for key in c.send_data:
sd = key.get()
self.total_sends += len(sd.data)
tracks, cursor, more = Track.query(
Track.list_id == self.list_id,
Track.campaign_id == self.campaign_id,
ndb.OR(Track.ttype == 'click', Track.ttype == 'open')
).order(Track._key).fetch_page(100, start_cursor=cursor)
for t in tracks:
self.process_track(t)
if t.ttype == 'click':
total_clicks += 1
elif t.ttype == 'open':
total_opens += 1
#set total_clicks/total_opens
self.total_clicks = self.total_clicks + total_clicks
self.total_opens = self.total_opens + total_opens
#set clicks/opens
self.sort_data('clicks')
self.sort_data('opens')
self.put()
if more and cursor:
taskqueue.add(
url='/api/compile-stats',
params={
'list_id': self.list_id,
'campaign_id': self.campaign_id,
'key': self.key.urlsafe(),
'cursor': cursor.urlsafe()
},
queue_name='stats'
)
def get(self, post_key):
from google.appengine.datastore.datastore_query import Cursor
args = {}
next_cursor = self.request.get('next_cursor');
ckey = 'PostHandler.%s.%s' % (post_key,next_cursor)
cdata = memcache.get(ckey)
if cdata is not None:
args = cdata
else:
args['tags'] = self.tags()
post = Feed.query(Feed.key == ndb.Key(urlsafe = post_key)).get()
next_cursor = Cursor(urlsafe=next_cursor)
entryRef, next_cursor, more = Comment.query(Comment.parent == post.key).order(Comment.created_time).fetch_page(100, start_cursor = next_cursor)
entries = []
for _entry in entryRef:
entry = _entry.to_dict()
entry['member'] = _entry.member.get().to_dict()
entries.append(entry)
post_key = post.key.urlsafe()
args['post'] = post.to_dict();
args['post']['message'] = message(args['post']['message'])
args['post']['member'] = post.member.get().to_dict()
args['comments'] = {}
args['comments']['entries'] = entries;
args['comments']['next_cursor'] = next_cursor.urlsafe() if next_cursor else None
args['comments']['more'] = more
if not memcache.add(ckey, args, CACEH_POST_IN_PERMLINK):
logging.error('Memcache set failed.')
template = JINJA_ENVIRONMENT.get_template('/view/post.html')
self.response.write(template.render(args))
def get_staff(cls, cursor_key, limit=20, **kw):
cursor = Cursor(urlsafe=cursor_key) if cursor_key else None
is_staff = True
q = cls.query().filter(cls.is_staff == is_staff)
q = q.order(cls.display_name)
staff, cursor, _ = q.fetch_page(limit, start_cursor=cursor, **kw)
return staff, (cursor.urlsafe() if cursor else None)
def get_staff(cls, cursor_key, limit=20, **kw):
cursor = Cursor(urlsafe=cursor_key) if cursor_key else None
is_staff = True
q = cls.query().filter(cls.is_staff == is_staff)
q = q.order(cls.display_name)
staff, cursor, _ = q.fetch_page(limit, start_cursor=cursor, **kw)
return staff, (cursor.urlsafe() if cursor else None),
def get_students(cls, cursor_key=None, limit=None, name=None, years=[], **kw):
limit = 20 if limit is None else limit
cursor = Cursor(urlsafe=cursor_key) if cursor_key else None
q = cls.query()
if name:
q = q.filter(cls.names >= name)
q = q.filter(cls.names < "%s{" % name)
if years:
q = q.filter(cls.year.IN(years))
else:
q = q.filter(cls.is_active == True)
if name:
q = q.order(cls.names, cls.year)
elif years:
q = q.order(cls.year, cls.display_name, cls.key)
else:
q = q.order(cls.year, cls.display_name)
if name or limit == 0:
limit = limit if limit else None
students, cursor = q.fetch(limit, **kw), None
else:
students, cursor, _ = q.fetch_page(limit, start_cursor=cursor, **kw)
return ([s.details() for s in students], (cursor.urlsafe() if cursor else None))
def livros(_write_tmpl, **params):
index = params["pag"] if params.has_key("pag") else ""
_type = params["type"] if params.has_key("type") else "next"
cursor = Cursor(urlsafe=index)
query = Livro.query()
has_previous = False
if _type == "previous":
q_backward = query.order(-Livro.titulo)
livros, previous, has_previous = q_backward.fetch_page(5, start_cursor = cursor.reversed())
has_next = True
next = cursor.reversed()
for l in livros:
l.id = l.key.id()
livros = reversed(livros)
else:
q_forward = query.order(Livro.titulo)
livros, next, has_next = q_forward.fetch_page(5, start_cursor = cursor)
if params.has_key("pag") and params["pag"] != '':
has_previous = True
previous = cursor
for l in livros:
l.id = l.key.id()
var = {"livros":livros,
"has_next":has_next,
"has_previous":has_previous,
"previous_cursor":previous,
"next_cursor":next}
_write_tmpl('templates/panel/livros.html', var)
def cursor_pagination(cls, prev_cursor_str, next_cursor_str):
if not prev_cursor_str and not next_cursor_str:
objects, next_cursor, more = cls.query().order(cls.number).fetch_page(ITEMS)
prev_cursor_str = ''
if next_cursor:
next_cursor_str = next_cursor.urlsafe()
else:
next_cursor_str = ''
next_ = True if more else False
prev = False
elif next_cursor_str:
cursor = Cursor(urlsafe=next_cursor_str)
objects, next_cursor, more = cls.query().order(cls.number).fetch_page(ITEMS, start_cursor=cursor)
prev_cursor_str = next_cursor_str
next_cursor_str = next_cursor.urlsafe()
prev = True
next_ = True if more else False
elif prev_cursor_str:
cursor = Cursor(urlsafe=prev_cursor_str)
objects, next_cursor, more = cls.query().order(-cls.number).fetch_page(ITEMS, start_cursor=cursor)
objects.reverse()
next_cursor_str = prev_cursor_str
prev_cursor_str = next_cursor.urlsafe()
prev = True if more else False
next_ = True
return {'objects': objects, 'next_cursor': next_cursor_str, 'prev_cursor': prev_cursor_str, 'prev': prev,
'next': next_}
def list(limit=10, cursor=None):
if cursor:
cursor = Cursor(urlsafe=cursor)
query = Book.query().order(Book.title)
entities, cursor, more = query.fetch_page(limit, start_cursor=cursor)
entities = builtin_list(map(from_datastore, entities))
return entities, cursor.urlsafe() if len(entities) == limit else None
def get(self):
p = self.request.get('p')
q = self.request.get('q')
c = self.request.get('c')
forward = True if p not in ['prev'] else False
cursor = Cursor(urlsafe=c)
if q:
qry = models.BlogPost.query(
ndb.OR(models.BlogPost.title == q.lower(),
models.BlogPost.author == q.lower(),
models.BlogPost.category.IN(q.lower().split(','))))
count = qry.count()
blogs = qry
else:
qry = models.BlogPost.query()
count = qry.count()
PAGE_SIZE = 50
if forward:
blogs, next_cursor, more = qry.order(
-models.BlogPost.updated).fetch_page(PAGE_SIZE,
start_cursor=cursor)
if next_cursor and more:
self.view.next_cursor = next_cursor
if c:
self.view.prev_cursor = cursor.reversed()
else:
blogs, next_cursor, more = qry.order(
models.BlogPost.updated).fetch_page(PAGE_SIZE,
start_cursor=cursor)
blogs = list(reversed(blogs))
if next_cursor and more:
self.view.prev_cursor = next_cursor
self.view.next_cursor = cursor.reversed()
def pager_url(p, cursor):
params = OrderedDict()
if q:
params['q'] = q
if p in ['prev']:
params['p'] = p
if cursor:
params['c'] = cursor.urlsafe()
return self.uri_for('admin-blog', **params)
self.view.pager_url = pager_url
self.view.q = q
params = {
"list_columns": [('title', 'Title'), ('author', 'Author'),
('created', 'Created'), ('updated', 'Updated'),
('category', 'Categories')],
"blogs":
blogs,
"count":
count
}
params['nickname'] = g_users.get_current_user().email().lower()
return self.render_template('blog/admin_blog.html', **params)
def _run_query_with_cursor(self,
query,
params,
fetch_limit,
order_property,
extra_ordering=[]):
qm = QueryMetadata()
results = []
forward_query = query.order(order_property,
*extra_ordering) # Sort ascending
reverse_query = query.order(-order_property,
*extra_ordering) # Sort descending
# By default, set new cursors to old cursors, to begin
qm.new_top_cursor_str = params.curr_top_cursor_str
qm.new_bottom_cursor_str = params.curr_bottom_cursor_str
if ((params.get_newer and not params.curr_top_cursor_str) or
(not params.get_newer and not params.curr_bottom_cursor_str)):
# If we do not get a top/bottom cursor from the client, we assume
# that they want the first fetch_limit results.
results, bottom_cursor, more = reverse_query.fetch_page(
fetch_limit)
if bottom_cursor:
_, top_cursor, _ = forward_query.fetch_page(
fetch_limit, start_cursor=bottom_cursor)
qm.new_top_cursor_str = top_cursor.urlsafe()
qm.new_bottom_cursor_str = bottom_cursor.urlsafe()
qm.has_more_older_data = len(results) >= fetch_limit
elif params.get_newer:
cursor = Cursor(urlsafe=params.curr_top_cursor_str)
# With a forward_query, the bottom_cursor eventually becomes the
# new top. So we get the new posts so we can figure out what
# the new top cursor should be, ignoring the results.
_, bottom_cursor, _ = forward_query.fetch_page(fetch_limit,
start_cursor=cursor)
if (bottom_cursor
and not self._cursors_are_eq(cursor, bottom_cursor)):
qm.new_top_cursor_str = bottom_cursor.urlsafe()
# Then we fetch again, this time from the new top cursor, to the
# original bottom cursor so we can grab all the posts that we
# have served the client thus far. This is so that if there are
# any updates to any posts, the client can get them.
results, _, _ = reverse_query.fetch_page(
page_size=sys.getsizeof(int()),
start_cursor=Cursor(urlsafe=qm.new_top_cursor_str),
end_cursor=Cursor(urlsafe=params.curr_bottom_cursor_str))
else:
# By default, assume we have no older data to serve.
qm.has_more_older_data = False
cursor = Cursor(urlsafe=params.curr_bottom_cursor_str)
results, new_bottom_cursor, more = reverse_query.fetch_page(
fetch_limit, start_cursor=cursor)
# If there are actual results
if new_bottom_cursor and not self._cursors_are_eq(
new_bottom_cursor, cursor):
qm.new_bottom_cursor_str = new_bottom_cursor.urlsafe()
qm.has_more_older_data = more
return results, qm
def cursor_pagination(cls, prev_cursor_str, next_cursor_str, category_str):
ctg = True if category_str else False
if ctg:
cat_search = categoriesDict[category_str]
if not prev_cursor_str and not next_cursor_str:
# q.filter('title =', 'Imagine')
# objects, next_cursor, more = cls.query().filter(cls.category == 1).order(-cls.view_date).fetch_page(ITEMS)
if ctg:
objects, next_cursor, more = cls.query().filter(
cls.category == cat_search).order(
-cls.view_date).fetch_page(ITEMS)
else:
objects, next_cursor, more = cls.query().order(
-cls.view_date).fetch_page(ITEMS)
prev_cursor_str = ''
if next_cursor:
next_cursor_str = next_cursor.urlsafe()
else:
next_cursor_str = ''
next_ = True if more else False
prev = False
elif next_cursor_str:
cursor = Cursor(urlsafe=next_cursor_str)
if ctg:
objects, next_cursor, more = cls.query().filter(
cls.category == cat_search).order(
-cls.view_date).fetch_page(ITEMS, start_cursor=cursor)
else:
objects, next_cursor, more = cls.query().order(
-cls.view_date).fetch_page(ITEMS, start_cursor=cursor)
prev_cursor_str = next_cursor_str
next_cursor_str = next_cursor.urlsafe()
prev = True
next_ = True if more else False
elif prev_cursor_str:
cursor = Cursor(urlsafe=prev_cursor_str)
if ctg:
objects, next_cursor, more = cls.query().filter(
cls.category == cat_search).order(
cls.view_date).fetch_page(ITEMS, start_cursor=cursor)
else:
objects, next_cursor, more = cls.query().order(
cls.view_date).fetch_page(ITEMS, start_cursor=cursor)
objects.reverse()
next_cursor_str = prev_cursor_str
prev_cursor_str = next_cursor.urlsafe()
prev = True if more else False
next_ = True
return {
'objects': objects,
'next_cursor': next_cursor_str,
'prev_cursor': prev_cursor_str,
'prev': prev,
'next': next_,
'category': category_str,
'ctg': ctg
}
def set_cursor(queryset, start=None, end=None):
if start is not None:
start = Cursor.from_websafe_string(start)
queryset.query._gae_start_cursor = start
if end is not None:
end = Cursor.from_websafe_string(end)
queryset.query._gae_end_cursor = end
# Evaluate QuerySet
len(queryset)
def get(self):
p = self.request.get('p')
q = self.request.get('q')
c = self.request.get('c')
forward = True if p not in ['prev'] else False
cursor = Cursor(urlsafe=c)
if q:
qry = LogEmail.query(
ndb.OR(LogEmail.to == q.lower(), LogEmail.sender == q.lower(),
LogEmail.subject == q.lower()))
else:
qry = LogEmail.query()
PAGE_SIZE = 50
if forward:
emails, next_cursor, more = qry.order(LogEmail.key).fetch_page(
PAGE_SIZE, start_cursor=cursor)
if next_cursor and more:
self.view.next_cursor = next_cursor
if c:
self.view.prev_cursor = cursor.reversed()
else:
emails, next_cursor, more = qry.order(-LogEmail.key).fetch_page(
PAGE_SIZE, start_cursor=cursor)
emails = list(reversed(emails))
if next_cursor and more:
self.view.prev_cursor = next_cursor
self.view.next_cursor = cursor.reversed()
def pager_url(p, cursor):
params = OrderedDict()
if q:
params['q'] = q
if p in ['prev']:
params['p'] = p
if cursor:
params['c'] = cursor.urlsafe()
return self.uri_for('admin-logs-emails', **params)
self.view.pager_url = pager_url
self.view.q = q
params = {
"list_columns": [
('when', 'When'),
('to', 'To'),
('subject', 'Subject'),
('sender', 'Sender'),
# ('body', 'Body')
],
"emails":
emails,
"count":
qry.count()
}
return self.render_template('admin_logs_emails.html', **params)
def set_cursor(queryset, start=None, end=None):
if start is not None:
start = Cursor.from_websafe_string(start)
queryset.query._gae_start_cursor = start
if end is not None:
end = Cursor.from_websafe_string(end)
queryset.query._gae_end_cursor = end
# Evaluate QuerySet
len(queryset)
def GetPagedResults(query,
order_properties,
cursor=None,
direction=NEXT,
page_size=PAGE_SIZE):
"""Paging the query results with page_size.
Args:
query(ndb.Query): The ndb query to query entities.
order_properties: A list of tuples containing class attribute of entity
class to order the entities and ordering of the field.
cursor (Cursor): The cursor provides a cursor in the current query
results, allowing you to retrieve the next set based on the offset.
direction (str): Either previous or next.
page_size (int): Number of entities to show per page.
Returns:
A tuple of (entities, top_cursor, next_cursor).
entities (list): List of entities to be displayed at the current page.
top_cursor (str): The urlsafe encoding of the cursor, which is at the
top position of entities of the current page.
bottom_cursor (str): The urlsafe encoding of the cursor, which is at the
bottom position of entities of the current page.
"""
cursor = Cursor(urlsafe=cursor) if cursor else None
if direction.lower() == PREVIOUS:
for order_property, forward_order in order_properties:
if forward_order == DESC:
# Forward order is desc meaning backward order should be asc.
query = query.order(order_property)
if forward_order == ASC:
# Forward order is asc meaning backward order should be desc.
query = query.order(-order_property)
entities, next_cursor, more = query.fetch_page(
page_size, start_cursor=cursor.reversed())
entities.reverse()
else:
for order_property, forward_order in order_properties:
if forward_order == DESC:
query = query.order(-order_property)
if forward_order == ASC:
query = query.order(order_property)
entities, next_cursor, more = query.fetch_page(page_size,
start_cursor=cursor)
next_cursor = next_cursor.urlsafe() if next_cursor else ''
used_cursor = cursor.urlsafe() if cursor else ''
if direction.lower() == PREVIOUS:
top_cursor = next_cursor if more else ''
bottom_cursor = used_cursor
else:
top_cursor = used_cursor
bottom_cursor = next_cursor if more else ''
return entities, top_cursor, bottom_cursor
def __init__(self, model_cls, limit, order, cursor_str="", query=None):
cursor = Cursor(urlsafe=cursor_str) if cursor_str else Cursor()
rcursor = cursor.reversed()
cls_order = getattr(model_cls, order)
if query == None: query = model_cls.query()
q_forward = query.order(cls_order)
q_reverse = query.order(-cls_order)
self._items, self._next_cursor, self._more = q_forward.fetch_page(limit, start_cursor=cursor)
unused_itemss, self._prev_cursor, unused_prev_more = q_reverse.fetch_page(limit, start_cursor=rcursor)
self._cursor = cursor
def list(limit=10, cursor=None):
""" Returns the moviereview entities from the DataStore converted to a list (limit 10 per list), with each entry containing
the movie id, title, year, genre, rating, review and reviewer. """
if cursor:
cursor = Cursor(urlsafe=cursor)
query = MovieReview.query().order(MovieReview.movie)
entities, cursor, more = query.fetch_page(limit, start_cursor=cursor)
entities = builtin_list(map(from_datastore, entities))
return entities, cursor.urlsafe() if len(entities) == limit else None
def get(self):
p = self.request.get('p')
q = self.request.get('q')
c = self.request.get('c')
forward = True if p not in ['prev'] else False
cursor = Cursor(urlsafe=c)
if q:
user_key = ndb.Key(User, long(q.lower()))
qry = LogVisit.query(
ndb.OR(LogVisit.user == user_key,
LogVisit.timestamp == q.lower(),
LogVisit.uastring == q.lower(),
LogVisit.ip == q.lower()))
else:
qry = LogVisit.query()
PAGE_SIZE = 50
if forward:
visits, next_cursor, more = qry.order(LogVisit.key).fetch_page(
PAGE_SIZE, start_cursor=cursor)
if next_cursor and more:
self.view.next_cursor = next_cursor
if c:
self.view.prev_cursor = cursor.reversed()
else:
visits, next_cursor, more = qry.order(-LogVisit.key).fetch_page(
PAGE_SIZE, start_cursor=cursor)
visits = list(reversed(visits))
if next_cursor and more:
self.view.prev_cursor = next_cursor
self.view.next_cursor = cursor.reversed()
def pager_url(p, cursor):
params = OrderedDict()
if q:
params['q'] = q
if p in ['prev']:
params['p'] = p
if cursor:
params['c'] = cursor.urlsafe()
return self.uri_for('admin-visitlogs-list', **params)
self.view.pager_url = pager_url
self.view.q = q
params = {
"list_columns": [('timestamp', 'Timestamp'), ('ip', 'IP'),
('uastring', 'uastring')],
"visits":
visits,
"count":
qry.count()
}
return self.render_template('admin_visitlogs_list.html', **params)
def get(self, *args, **kwargs):
self.model_class = kwargs["_class"]
self.form_class = kwargs["_form_class"]
p = self.request.get("p")
q = self.request.get("q")
c = self.request.get("c")
forward = True if p not in ["prev"] else False
cursor = Cursor(urlsafe=c)
from google.appengine.ext.ndb.query import Node
if q:
nodes = []
for search in self.form_class.search_list:
nodes.append(FilterNode(search, "=", q))
qry = self.model_class.query(ndb.OR(*nodes))
else:
qry = self.model_class.query()
PAGE_SIZE = 5
if forward:
users, next_cursor, more = qry.order(self.model_class.key).fetch_page(PAGE_SIZE, start_cursor=cursor)
if next_cursor and more:
self.view.next_cursor = next_cursor
if c:
self.view.prev_cursor = cursor.reversed()
else:
users, next_cursor, more = qry.order(-self.model_class.key).fetch_page(PAGE_SIZE, start_cursor=cursor)
users = list(reversed(users))
if next_cursor and more:
self.view.prev_cursor = next_cursor
self.view.next_cursor = cursor.reversed()
def pager_url(p, cursor):
params = OrderedDict()
if q:
params["q"] = q
if p in ["prev"]:
params["p"] = p
if cursor:
params["c"] = cursor.urlsafe()
return self.uri_for("%s-list" % self.model_class.__name__.lower(), **params)
self.view.pager_url = pager_url
self.view.q = q
params = {
"class_name": self.model_class.__name__,
"list_columns": self.form_class.list_columns,
"users": users,
"count": qry.count(),
"add_uri": self.uri_for("%s-add" % self.model_class.__name__.lower()),
}
logger.warn(params)
return self.render_template("admin/list.html", **params)
def get(self):
p = self.request.get('p')
q = self.request.get('q')
c = self.request.get('c')
forward = True if p not in ['prev'] else False
cursor = Cursor(urlsafe=c)
if q:
qry = models.BlogPost.query(ndb.OR(models.BlogPost.title == q.lower(),
models.BlogPost.author == q.lower(),
models.BlogPost.category.IN(q.lower().split(','))))
count = qry.count()
blogs = qry
else:
qry = models.BlogPost.query()
count = qry.count()
PAGE_SIZE = 50
if forward:
blogs, next_cursor, more = qry.order(-models.BlogPost.updated).fetch_page(PAGE_SIZE, start_cursor=cursor)
if next_cursor and more:
self.view.next_cursor = next_cursor
if c:
self.view.prev_cursor = cursor.reversed()
else:
blogs, next_cursor, more = qry.order(models.BlogPost.updated).fetch_page(PAGE_SIZE, start_cursor=cursor)
blogs = list(reversed(blogs))
if next_cursor and more:
self.view.prev_cursor = next_cursor
self.view.next_cursor = cursor.reversed()
def pager_url(p, cursor):
params = OrderedDict()
if q:
params['q'] = q
if p in ['prev']:
params['p'] = p
if cursor:
params['c'] = cursor.urlsafe()
return self.uri_for('admin-blog', **params)
self.view.pager_url = pager_url
self.view.q = q
if self.app.config.get('app_lang') == 'es':
list_columns = [('title', u'Título'), ('author', 'Autor'), ('created', u'Fecha de creación'), ('updated', u'Fecha de actualización'), ('category', u'Categorías')]
else:
list_columns = [('title', u'Title'), ('author', 'Author'), ('created', u'Created'), ('updated', u'Updated'), ('category', u'Categories')]
params = {
"list_columns": list_columns,
"blogs": blogs,
"count": count
}
params['nickname'] = g_users.get_current_user().email().lower()
return self.render_template('%s/blog/admin_blog.html' % self.app.config.get('app_lang'), **params)
def get(self):
p = self.request.get('p')
q = self.request.get('q')
c = self.request.get('c')
forward = True if p not in ['prev'] else False
cursor = Cursor(urlsafe=c)
if q:
qry = self.user_model.query(
ndb.OR(self.user_model.last_name == q.lower(),
self.user_model.email == q.lower(),
self.user_model.username == q.lower()))
else:
qry = self.user_model.query()
PAGE_SIZE = 50
if forward:
users, next_cursor, more = qry.order(
self.user_model.key).fetch_page(PAGE_SIZE, start_cursor=cursor)
if next_cursor and more:
self.view.next_cursor = next_cursor
if c:
self.view.prev_cursor = cursor.reversed()
else:
users, next_cursor, more = qry.order(
-self.user_model.key).fetch_page(PAGE_SIZE,
start_cursor=cursor)
users = list(reversed(users))
if next_cursor and more:
self.view.prev_cursor = next_cursor
self.view.next_cursor = cursor.reversed()
def pager_url(p, cursor):
params = OrderedDict()
if q:
params['q'] = q
if p in ['prev']:
params['p'] = p
if cursor:
params['c'] = cursor.urlsafe()
return self.uri_for('admin-users-list', **params)
self.view.pager_url = pager_url
self.view.q = q
params = {
"list_columns": [('username', 'Username'), ('name', 'Name'),
('last_name', 'Last Name'), ('email', 'Email'),
('country', 'Country'), ('tz', 'TimeZone')],
"users":
users,
"count":
qry.count()
}
return self.render_template('admin_users_list.html', **params)
def get_stats(cls, topic, cursor_key=None, limit=None, year=None, **kw):
limit = limit if limit else 20
cursor = Cursor(urlsafe=cursor_key) if cursor_key else None
q = cls.query().filter(cls.topic == topic)
if year:
q = q.filter(cls.year == year)
q = q.order(-cls.performance)
stats, cursor, _ = q.fetch_page(limit, start_cursor=cursor, **kw)
return stats, (cursor.urlsafe() if cursor else None),
def get(self):
p = self.request.get('p')
q = self.request.get('q')
c = self.request.get('c')
forward = True if p not in ['prev'] else False
cursor = Cursor(urlsafe=c)
if q:
qry = Bom.query(ndb.OR(Bom.public_id == q,
Bom.name == q,
Bom.tag_name == q))
else:
qry = Bom.query()
PAGE_SIZE = 5
if forward:
boms, next_cursor, more = qry.order(Bom.key).fetch_page(PAGE_SIZE, start_cursor=cursor)
if next_cursor and more:
self.view.next_cursor = next_cursor
if c:
self.view.prev_cursor = cursor.reversed()
else:
boms, next_cursor, more = qry.order(-Bom.key).fetch_page(PAGE_SIZE, start_cursor=cursor)
boms = list(reversed(boms))
if next_cursor and more:
self.view.prev_cursor = next_cursor
self.view.next_cursor = cursor.reversed()
def pager_url(p, cursor):
params = OrderedDict()
if q:
params['q'] = q
if p in ['prev']:
params['p'] = p
if cursor:
params['c'] = cursor.urlsafe()
return self.uri_for('boms-admin', **params)
self.view.pager_url = pager_url
self.view.q = q
params = {
"list_columns": [('public_id', 'Public ID'),
('name', 'BOM Name'),
('tag_name', 'Tag Name'),
('public', 'public')],
"boms" : boms,
"count" : qry.count(),
}
# FIXME: admin_user should probably go into BaseHandler
params['admin_user'] = googleusers.is_current_user_admin()
return self.render_template('admin/boms.html', **params)
def set_cursor(queryset, start=None, end=None):
queryset = _add_mixin(queryset)
if start is not None:
start = Cursor.from_websafe_string(start)
setattr(queryset.query, '_gae_start_cursor', start)
if end is not None:
end = Cursor.from_websafe_string(end)
setattr(queryset.query, '_gae_end_cursor', end)
return queryset
def set_cursor(queryset, start=None, end=None):
queryset = _add_mixin(queryset)
if start is not None:
start = Cursor.from_websafe_string(start)
setattr(queryset.query, '_gae_start_cursor', start)
if end is not None:
end = Cursor.from_websafe_string(end)
setattr(queryset.query, '_gae_end_cursor', end)
return queryset
def get(self):
p = self.request.get('p')
q = self.request.get('q')
c = self.request.get('c')
forward = True if p not in ['prev'] else False
cursor = Cursor(urlsafe=c)
if q:
qry = models.User.query(ndb.OR(models.User.last_name == q,
models.User.email == q,
models.User.username == q))
else:
qry = models.User.query()
PAGE_SIZE = 5
if forward:
users, next_cursor, more = qry.order(models.User.key).fetch_page(PAGE_SIZE, start_cursor=cursor)
if next_cursor and more:
self.view.next_cursor = next_cursor
if c:
self.view.prev_cursor = cursor.reversed()
else:
users, next_cursor, more = qry.order(-models.User.key).fetch_page(PAGE_SIZE, start_cursor=cursor)
users = list(reversed(users))
if next_cursor and more:
self.view.prev_cursor = next_cursor
self.view.next_cursor = cursor.reversed()
def pager_url(p, cursor):
params = OrderedDict()
if q:
params['q'] = q
if p in ['prev']:
params['p'] = p
if cursor:
params['c'] = cursor.urlsafe()
return self.uri_for('user-list', **params)
self.view.pager_url = pager_url
self.view.q = q
params = {
"list_columns": [('username', 'Username'),
('last_name', 'Last Name'),
('email', 'E-Mail'),
('country', 'Country')],
"users" : users,
"count" : qry.count()
}
# FIXME: admin_user should probably go into BaseHandler
params['admin_user'] = googleusers.is_current_user_admin()
return self.render_template('admin/users.html', **params)
def get(self):
p = self.request.get('p')
q = self.request.get('q')
c = self.request.get('c')
forward = True if p not in ['prev'] else False
cursor = Cursor(urlsafe=c)
user_key = ndb.Key('User', long(self.user_id))
if q:
qry = Bom.query(
Bom.owners == user_key,
ndb.OR(Bom.public_id == q, Bom.name == q, Bom.tag_name == q))
else:
qry = Bom.query(Bom.owners == user_key)
PAGE_SIZE = 5
if forward:
boms, next_cursor, more = qry.order(Bom.key).fetch_page(
PAGE_SIZE, start_cursor=cursor)
if next_cursor and more:
self.view.next_cursor = next_cursor
if c:
self.view.prev_cursor = cursor.reversed()
else:
boms, next_cursor, more = qry.order(-Bom.key).fetch_page(
PAGE_SIZE, start_cursor=cursor)
boms = list(reversed(boms))
if next_cursor and more:
self.view.prev_cursor = next_cursor
self.view.next_cursor = cursor.reversed()
def pager_url(p, cursor):
params = OrderedDict()
if q:
params['q'] = q
if p in ['prev']:
params['p'] = p
if cursor:
params['c'] = cursor.urlsafe()
return self.uri_for('boms', **params)
self.view.pager_url = pager_url
self.view.q = q
params = {
"list_columns": [('name', 'BOM Name'), ('public_id', 'Public ID'),
('public', 'public'),
('change_time', 'Last Updated')],
"boms":
boms,
"count":
qry.count()
}
return self.render_template('boms.html', **params)
def get(self):
p = self.request.get('p')
q = self.request.get('q')
c = self.request.get('c')
forward = True if p not in ['prev'] else False
cursor = Cursor(urlsafe=c)
if q:
qry = models.Seller.query(ndb.OR(models.Seller.n == q,
models.Seller.cn == q))
else:
qry = models.Seller.query()
PAGE_SIZE = 5
if forward:
sellers, next_cursor, more = qry.order(models.User.key).fetch_page(PAGE_SIZE, start_cursor=cursor)
if next_cursor and more:
self.view.next_cursor = next_cursor
if c:
self.view.prev_cursor = cursor.reversed()
else:
sellers, next_cursor, more = qry.order(-models.Seller.key).fetch_page(PAGE_SIZE, start_cursor=cursor)
sellers = list(reversed(users))
if next_cursor and more:
self.view.prev_cursor = next_cursor
self.view.next_cursor = cursor.reversed()
def pager_url(p, cursor):
params = OrderedDict()
if q:
params['q'] = q
if p in ['prev']:
params['p'] = p
if cursor:
params['c'] = cursor.urlsafe()
return self.uri_for('user-list', **params)
self.view.pager_url = pager_url
self.view.q = q
params = {
"list_columns": [('n', 'Seller'),
('dom', 'Domain'),
('cn', 'Contact Name'),
('pn', 'Phone Number'),
('cat', 'Category'),
('active', 'Active'),
('auth', 'Authenticated')],
"sellers" : sellers,
'pageTitle': 'Sellers List',
"count" : qry.count()
}
return self.render_template('admin/list.html', **params)
def get_news_list_from_db(token):
curs = Cursor(urlsafe=token)
newss, curs, _ = News.query().order(-News.date).fetch_page(10, start_cursor=curs)
if newss:
newss_list = {}
if curs:
newss_list['pagetoken'] = curs.urlsafe()
newss_list['news'] = []
for news in newss:
newss_list['news'].append(news.maximize())
return newss_list
return None
def get_issues_list_from_db(token):
curs = Cursor(urlsafe=token)
issues, curs, _ = Issue.query().order(-Issue.id).fetch_page(10, start_cursor=curs)
if issues:
issues_list = {}
if curs:
issues_list['pagetoken'] = curs.urlsafe()
issues_list['issues'] = []
for issue in issues:
issues_list['issues'].append(issue.minimize())
return issues_list
return None
def remove_old_dashboard_data(self):
""" Removes old statistics from the AppScale dashboard application. """
last_cursor = None
last_model = None
# If we have state information beyond what function to use,
# load the last seen model and cursor if available.
if (len(self.groomer_state) > 1 and
self.groomer_state[0] == self.CLEAN_DASHBOARD_TASK):
last_model = self.DASHBOARD_DATA_MODELS[int(self.groomer_state[1])]
if len(self.groomer_state) > 2:
last_cursor = Cursor(self.groomer_state[2])
self.register_db_accessor(constants.DASHBOARD_APP_ID)
timeout = datetime.datetime.utcnow() - \
datetime.timedelta(seconds=self.DASHBOARD_DATA_TIMEOUT)
for model_number in range(len(self.DASHBOARD_DATA_MODELS)):
model_type = self.DASHBOARD_DATA_MODELS[model_number]
if last_model and model_type != last_model:
continue
counter = 0
while True:
query = model_type.query().filter(model_type.timestamp < timeout)
entities, next_cursor, more = query.fetch_page(self.BATCH_SIZE,
start_cursor=last_cursor)
for entity in entities:
entity.key.delete()
counter += 1
if time.time() > self.last_logged + self.LOG_PROGRESS_FREQUENCY:
logging.info('Removed {} {} entities.'
.format(counter, model_type.__class__.__name__))
self.last_logged = time.time()
if more:
last_cursor = next_cursor
self.update_groomer_state([self.CLEAN_DASHBOARD_TASK,
str(model_number), last_cursor.urlsafe()])
else:
break
if model_number != len(self.DASHBOARD_DATA_MODELS) - 1:
self.update_groomer_state([self.CLEAN_DASHBOARD_TASK,
str(model_number + 1)])
last_model = None
last_cursor = None
if counter > 0:
logging.info("Removed {0} {1} dashboard entities".format(counter,
model_type))
# Do a scan of all entities and remove any that
# do not have timestamps for AppScale versions 2.3 and before.
# This may take some time on the initial run, but subsequent runs should
# be quick given a low dashboard data timeout.
self.remove_deprecated_dashboard_data(model_type)
return
def get(self):
p = self.request.get('p')
q = self.request.get('q')
c = self.request.get('c')
forward = True if p not in ['prev'] else False
cursor = Cursor(urlsafe=c)
if q:
qry = LogEmail.query(ndb.OR(LogEmail.to == q.lower(),
LogEmail.sender == q.lower(),
LogEmail.subject == q.lower()))
else:
qry = LogEmail.query()
PAGE_SIZE = 50
if forward:
emails, next_cursor, more = qry.order(LogEmail.key).fetch_page(PAGE_SIZE, start_cursor=cursor)
if next_cursor and more:
self.view.next_cursor = next_cursor
if c:
self.view.prev_cursor = cursor.reversed()
else:
emails, next_cursor, more = qry.order(-LogEmail.key).fetch_page(PAGE_SIZE, start_cursor=cursor)
emails = list(reversed(emails))
if next_cursor and more:
self.view.prev_cursor = next_cursor
self.view.next_cursor = cursor.reversed()
def pager_url(p, cursor):
params = OrderedDict()
if q:
params['q'] = q
if p in ['prev']:
params['p'] = p
if cursor:
params['c'] = cursor.urlsafe()
return self.uri_for('admin-logs-emails', **params)
self.view.pager_url = pager_url
self.view.q = q
if self.app.config.get('app_lang') == 'es':
list_columns = [('when', 'Fecha'),('to', 'Destinatario'),('subject', 'Asunto'),('sender', 'Remitente')]
else:
list_columns = [('when', 'Date'),('to', 'Recipient'),('subject', 'Subject'),('sender', 'Sender')]
params = {
"list_columns": list_columns,
"emails": emails,
"count": qry.count()
}
params['nickname'] = g_users.get_current_user().email().lower()
return self.render_template('%s/emails/admin_logs_emails.html' % self.app.config.get('app_lang'), **params)
def post(self):
user = self._pre_post()
if not user:
return
# pull together all of the necessary parameters
parameters = self._post_parameters()
search, search_column, orders = self._post_aggregate()
search, search_valid = self._post_search(search, search_column)
# build the query
query = models.Hacker.query()
for order in orders:
query = query.order(order)
if search_valid and search:
search_column = ReviewDataTableHandler.COLUMNS[search_column]
query = query.filter(search_column == search)
offset = parameters['offset']
cursor = None
data, more = [], False
if search_valid:
cursor = Cursor(urlsafe=parameters['cursor'])
data, cursor, more = query.fetch_page(parameters['limit'], offset=0, start_cursor=cursor)
# the data that we send back is slightly different
# than what the serializer would yield; we can't include
# extra fields either
serialized_data = [{
"firstName": registrant.first_name,
"lastName": registrant.last_name,
"gender": registrant.registration.gender,
"school": registrant.registration.school,
"graduation": registrant.registration.graduation_year,
"major": registrant.registration.major,
"initiatives": registrant.registration.initiatives,
"registered": registrant.registration.created,
"status": registrant.status,
"wave": registrant.notification_wave,
"id": registrant.key.parent().id()
} for registrant in data if registrant.registration]
records_total = self._compute_records_total(parameters, more)
self.write_json(json.dumps({
"draw": parameters['draw'],
"data": serialized_data,
"recordsTotal": records_total,
"recordsFiltered": records_total,
"cursor": cursor.urlsafe() if cursor else None
}))
def remove_old_dashboard_data(self):
""" Removes old statistics from the AppScale dashboard application. """
last_cursor = None
last_model = None
# If we have state information beyond what function to use,
# load the last seen model and cursor if available.
if (len(self.groomer_state) > 1 and
self.groomer_state[0] == self.CLEAN_DASHBOARD_TASK):
last_model = self.DASHBOARD_DATA_MODELS[int(self.groomer_state[1])]
if len(self.groomer_state) > 2:
last_cursor = Cursor(self.groomer_state[2])
self.register_db_accessor(constants.DASHBOARD_APP_ID)
timeout = datetime.datetime.utcnow() - \
datetime.timedelta(seconds=self.DASHBOARD_DATA_TIMEOUT)
for model_number in range(len(self.DASHBOARD_DATA_MODELS)):
model_type = self.DASHBOARD_DATA_MODELS[model_number]
if last_model and model_type != last_model:
continue
counter = 0
while True:
query = model_type.query().filter(model_type.timestamp < timeout)
entities, next_cursor, more = query.fetch_page(self.BATCH_SIZE,
start_cursor=last_cursor)
for entity in entities:
entity.key.delete()
counter += 1
if time.time() > self.last_logged + self.LOG_PROGRESS_FREQUENCY:
logging.info('Removed {} {} entities.'
.format(counter, model_type.__class__.__name__))
self.last_logged = time.time()
if more:
last_cursor = next_cursor
self.update_groomer_state([self.CLEAN_DASHBOARD_TASK,
str(model_number), last_cursor.urlsafe()])
else:
break
if model_number != len(self.DASHBOARD_DATA_MODELS) - 1:
self.update_groomer_state([self.CLEAN_DASHBOARD_TASK,
str(model_number + 1)])
last_model = None
last_cursor = None
if counter > 0:
logging.info("Removed {0} {1} dashboard entities".format(counter,
model_type))
# Do a scan of all entities and remove any that
# do not have timestamps for AppScale versions 2.3 and before.
# This may take some time on the initial run, but subsequent runs should
# be quick given a low dashboard data timeout.
self.remove_deprecated_dashboard_data(model_type)
return
def set_cursor(queryset, start=None, end=None):
queryset = queryset.all()
class CursorQuery(CursorQueryMixin, queryset.query.__class__):
pass
queryset.query = queryset.query.clone(klass=CursorQuery)
if start is not None:
start = Cursor.from_websafe_string(start)
queryset.query._gae_start_cursor = start
if end is not None:
end = Cursor.from_websafe_string(end)
queryset.query._gae_end_cursor = end
return queryset
def get(self):
p = self.request.get('p')
q = self.request.get('q')
c = self.request.get('c')
forward = True if p not in ['prev'] else False
cursor = Cursor(urlsafe=c)
if q:
qry = models.BlogPost.query(ndb.OR(models.BlogPost.title == q.lower(),
models.BlogPost.author == q.lower(),
models.BlogPost.category.IN(q.lower().split(','))))
count = qry.count()
blogs = qry
else:
qry = models.BlogPost.query()
count = qry.count()
PAGE_SIZE = 50
if forward:
blogs, next_cursor, more = qry.order(-models.BlogPost.updated).fetch_page(PAGE_SIZE, start_cursor=cursor)
if next_cursor and more:
self.view.next_cursor = next_cursor
if c:
self.view.prev_cursor = cursor.reversed()
else:
blogs, next_cursor, more = qry.order(models.BlogPost.updated).fetch_page(PAGE_SIZE, start_cursor=cursor)
blogs = list(reversed(blogs))
if next_cursor and more:
self.view.prev_cursor = next_cursor
self.view.next_cursor = cursor.reversed()
def pager_url(p, cursor):
params = OrderedDict()
if q:
params['q'] = q
if p in ['prev']:
params['p'] = p
if cursor:
params['c'] = cursor.urlsafe()
return self.uri_for('admin-blog', **params)
self.view.pager_url = pager_url
self.view.q = q
params = {
"list_columns": [('title', 'Title'),
('author', 'Author'),
('created', 'Created'),
('updated', 'Updated'),
('category', 'Categories')],
"blogs": blogs,
"count": count
}
return self.render_template('admin_blog.html', **params)
def get(self):
p = self.request.get('p')
q = self.request.get('q')
c = self.request.get('c')
forward = True if p not in ['prev'] else False
cursor = Cursor(urlsafe=c)
if q:
qry = LogEmail.query(ndb.OR(LogEmail.to == q.lower(),
LogEmail.sender == q.lower(),
LogEmail.subject == q.lower()))
else:
qry = LogEmail.query()
PAGE_SIZE = 50
if forward:
emails, next_cursor, more = qry.order(LogEmail.key).fetch_page(PAGE_SIZE, start_cursor=cursor)
if next_cursor and more:
self.view.next_cursor = next_cursor
if c:
self.view.prev_cursor = cursor.reversed()
else:
emails, next_cursor, more = qry.order(-LogEmail.key).fetch_page(PAGE_SIZE, start_cursor=cursor)
emails = list(reversed(emails))
if next_cursor and more:
self.view.prev_cursor = next_cursor
self.view.next_cursor = cursor.reversed()
def pager_url(p, cursor):
params = OrderedDict()
if q:
params['q'] = q
if p in ['prev']:
params['p'] = p
if cursor:
params['c'] = cursor.urlsafe()
return self.uri_for('admin-logs-emails', **params)
self.view.pager_url = pager_url
self.view.q = q
params = {
"list_columns": [('when', 'When'),
('to', 'To'),
('subject', 'Subject'),
('sender', 'Sender'),
# ('body', 'Body')
],
"emails": emails,
"count": qry.count()
}
return self.render_template('admin_logs_emails.html', **params)
def get(self):
p = self.request.get('p')
q = self.request.get('q')
c = self.request.get('c')
forward = True if p not in ['prev'] else False
cursor = Cursor(urlsafe=c)
user_key = ndb.Key('User', long(self.user_id))
if q:
qry = Bom.query(Bom.owners == user_key,
ndb.OR(Bom.public_id == q,
Bom.name == q,
Bom.tag_name == q))
else:
qry = Bom.query(Bom.owners == user_key)
PAGE_SIZE = 5
if forward:
boms, next_cursor, more = qry.order(Bom.key).fetch_page(PAGE_SIZE, start_cursor=cursor)
if next_cursor and more:
self.view.next_cursor = next_cursor
if c:
self.view.prev_cursor = cursor.reversed()
else:
boms, next_cursor, more = qry.order(-Bom.key).fetch_page(PAGE_SIZE, start_cursor=cursor)
boms = list(reversed(boms))
if next_cursor and more:
self.view.prev_cursor = next_cursor
self.view.next_cursor = cursor.reversed()
def pager_url(p, cursor):
params = OrderedDict()
if q:
params['q'] = q
if p in ['prev']:
params['p'] = p
if cursor:
params['c'] = cursor.urlsafe()
return self.uri_for('boms', **params)
self.view.pager_url = pager_url
self.view.q = q
params = {
"list_columns": [('name', 'BOM Name'),
('public_id', 'Public ID'),
('public', 'public'),
('change_time', 'Last Updated')],
"boms" : boms,
"count" : qry.count()
}
return self.render_template('boms.html', **params)
def get(self):
p = self.request.get('p')
q = self.request.get('q')
c = self.request.get('c')
forward = True if p not in ['prev'] else False
cursor = Cursor(urlsafe=c)
if q:
user_key = ndb.Key(User,long(q.lower()))
qry = LogVisit.query(ndb.OR( LogVisit.user == user_key,
LogVisit.timestamp == q.lower(),
LogVisit.uastring == q.lower(),
LogVisit.ip == q.lower()))
else:
qry = LogVisit.query()
PAGE_SIZE = 50
if forward:
visits, next_cursor, more = qry.order(LogVisit.key).fetch_page(PAGE_SIZE, start_cursor=cursor)
if next_cursor and more:
self.view.next_cursor = next_cursor
if c:
self.view.prev_cursor = cursor.reversed()
else:
visits, next_cursor, more = qry.order(-LogVisit.key).fetch_page(PAGE_SIZE, start_cursor=cursor)
visits = list(reversed(visits))
if next_cursor and more:
self.view.prev_cursor = next_cursor
self.view.next_cursor = cursor.reversed()
def pager_url(p, cursor):
params = OrderedDict()
if q:
params['q'] = q
if p in ['prev']:
params['p'] = p
if cursor:
params['c'] = cursor.urlsafe()
return self.uri_for('admin-logs-visits', **params)
self.view.pager_url = pager_url
self.view.q = q
params = {
"list_columns": [('timestamp', 'Timestamp'),
('ip', 'IP'),
('uastring', 'uastring')
],
"visits": visits,
"count": qry.count()
}
return self.render_template('admin_logs_visits.html', **params)
def get(self):
p = self.request.get('p')
q = self.request.get('q')
c = self.request.get('c')
forward = True if p not in ['prev'] else False
cursor = Cursor(urlsafe=c)
if q:
qry = self.user_model.query(ndb.OR(self.user_model.last_name == q.lower(),
self.user_model.email == q.lower(),
self.user_model.username == q.lower()))
else:
qry = self.user_model.query()
PAGE_SIZE = 50
if forward:
users, next_cursor, more = qry.order(self.user_model.key).fetch_page(PAGE_SIZE, start_cursor=cursor)
if next_cursor and more:
self.view.next_cursor = next_cursor
if c:
self.view.prev_cursor = cursor.reversed()
else:
users, next_cursor, more = qry.order(-self.user_model.key).fetch_page(PAGE_SIZE, start_cursor=cursor)
users = list(reversed(users))
if next_cursor and more:
self.view.prev_cursor = next_cursor
self.view.next_cursor = cursor.reversed()
def pager_url(p, cursor):
params = OrderedDict()
if q:
params['q'] = q
if p in ['prev']:
params['p'] = p
if cursor:
params['c'] = cursor.urlsafe()
return self.uri_for('admin-users-list', **params)
self.view.pager_url = pager_url
self.view.q = q
params = {
"list_columns": [('username', 'Username'),
('name', 'Name'),
('last_name', 'Last Name'),
('email', 'Email'),
('country', 'Country'),
('tz', 'TimeZone')],
"users": users,
"count": qry.count()
}
return self.render_template('admin_users_list.html', **params)
def paginate_ndb(self, query, cursor, limit):
if cursor:
cursor = Cursor(urlsafe=cursor)
data, next_cursor, more = query.fetch_page(limit, start_cursor=cursor)
self.handler.set('paging', {
'cursor': cursor.urlsafe() if cursor else '',
'next_cursor': next_cursor.urlsafe() if more else '',
'limit': limit
})
return data
def paginate_ndb(self, query, cursor, limit):
if cursor:
cursor = Cursor(urlsafe=cursor)
data, next_cursor, more = query.fetch_page(limit, start_cursor=cursor)
self.handler.set('paging', {
'cursor': cursor.urlsafe() if cursor else '',
'next_cursor': next_cursor.urlsafe() if more else '',
'limit': limit
})
return data
def get(self):
p = self.request.get('p')
q = self.request.get('q')
c = self.request.get('c')
forward = True if p not in ['prev'] else False
cursor = Cursor(urlsafe=c)
if q:
qry = Group.query()
else:
qry = Group.query()
PAGE_SIZE = 50
if forward:
groups, next_cursor, more = qry.order(self.user_model.key).fetch_page(PAGE_SIZE, start_cursor=cursor)
if next_cursor and more:
self.view.next_cursor = next_cursor
if c:
self.view.prev_cursor = cursor.reversed()
else:
groups, next_cursor, more = qry.order(-self.user_model.key).fetch_page(PAGE_SIZE, start_cursor=cursor)
groups = list(reversed(groups))
if next_cursor and more:
self.view.prev_cursor = next_cursor
self.view.next_cursor = cursor.reversed()
def pager_url(p, cursor):
params = OrderedDict()
if q:
params['q'] = q
if p in ['prev']:
params['p'] = p
if cursor:
params['c'] = cursor.urlsafe()
return self.uri_for('admin-groups-list', **params)
self.view.pager_url = pager_url
self.view.q = q
params = {
"list_columns": [('group', 'Group'),
('can_view', 'Can View'),
('can_edit', 'Can Edit'),
('can_administer', 'Can Adminster'),
('can_upload', 'Can Upload'),
],
"groups": groups,
"count": qry.count()
}
return self.render_template('admin_groups_list.html', **params)
def post(self):
in_obj = self.request.POST
out_obj = {}
out_obj['draw'] = int(in_obj['draw'])
self.response.content_type = 'application/json'
count = 99999999 # FIXME if real number of records exceeds this, increment this!!
out_obj['recordsTotal'] = count
out_obj['recordsFiltered'] = count # OK?
start = in_obj['start'] # starts at 0
pagesize = int(in_obj['length']) # number of rows to return
cursor = None
if len(in_obj['next_cursor']):
cursor = Cursor(urlsafe=in_obj['next_cursor'])
print("cursor is")
if cursor:
print(cursor.urlsafe())
else:
print("(none)")
query = AMILaunch.query(ancestor=get_parent()).order(-AMILaunch.date)
rows, next_cursor, more = query.fetch_page(pagesize,
start_cursor=cursor)
if next_cursor:
print("urlsafe is")
print(next_cursor.urlsafe())
out_obj['next_cursor'] = next_cursor.urlsafe()
else:
out_obj['next_cursor'] = None
print("urlsafe is None")
data = []
for row in rows:
data.append([
row.ami_id, row.bioc_version, row.ami_name, row.instance_type,
row.region, row.availability_zone, row.is_bioc_account,
str(row.date), row.account_hash
])
out_obj['data'] = data
self.response.write(json.encode(out_obj))
