def run_to_queue(self, queue, conn, options=None):
"""Run this query, putting entities into the given queue."""
multiquery = self._maybe_multi_query()
if multiquery is not None:
multiquery.run_to_queue(queue, conn,
options=options) # No return value.
return
dsqry, post_filters = self._get_query(conn)
orig_options = options
if (post_filters and options is not None
and (options.offset or options.limit is not None)):
options = datastore_query.QueryOptions(offset=None,
limit=None,
config=orig_options)
assert options.limit is None and options.limit is None
rpc = dsqry.run_async(conn, options)
skipped = 0
count = 0
while rpc is not None:
batch = yield rpc
rpc = batch.next_batch_async(options)
for ent in batch.results:
if post_filters:
if not post_filters.apply(ent):
continue
if orig_options is not options:
if orig_options.offset and skipped < orig_options.offset:
skipped += 1
continue
if orig_options.limit is not None and count >= orig_options.limit:
rpc = None # Quietly throw away the next batch.
break
count += 1
queue.putq(ent)
queue.complete()
def _iter_key_range(self, k_range):
raw_entity_kind = util.get_short_name(self._entity_kind)
query = k_range.make_ascending_datastore_query(raw_entity_kind,
keys_only=True)
for key in query.Run(config=datastore_query.QueryOptions(
batch_size=self._batch_size)):
yield key, key
def __iter__(self):
self._query = self._key_range.make_ascending_datastore_query(
self._query_spec.entity_kind, filters=self._query_spec.filters)
for entity in self._query.Run(config=datastore_query.QueryOptions(
batch_size=self._query_spec.batch_size,
keys_only=self._KEYS_ONLY,
start_cursor=self._cursor)):
yield entity
def __iter__(self):
query = self._key_range.make_ascending_datastore_query(
self._query_spec.entity_kind, filters=self._query_spec.filters)
connection = datastore_rpc.Connection()
query_options = datastore_query.QueryOptions(
batch_size=self._query_spec.batch_size,
start_cursor=self._cursor,
produce_cursors=True)
self._query = datastore_query.ResultsIterator(query.GetQuery().run(
connection, query_options))
for entity_proto in self._query:
yield entity_proto
def config_iterable(plain_config, batch_size=50, limit=1000):
config = plain_config
try:
# This specific use of the QueryOptions private API was suggested to us by the App Engine team.
# Wrapping in try/except in case it ever goes away.
from google.appengine.datastore import datastore_query
config = datastore_query.QueryOptions(config=plain_config,
limit=limit,
offset=0,
prefetch_size=batch_size,
batch_size=batch_size)
except Exception, e:
logging.exception("Failed to create QueryOptions config object: %s", e)
def cursor(self, cursor, endCursor=None):
"""
Sets the start cursor for this query.
The result set will only include results behind that cursor.
The cursor is generated by an earlier query with exactly the same configuration.
Its safe to use client-supplied cursors, a cursor can't be abused to access entities
which don't match the current filters.
:param cursor: The cursor key to set to the Query.
:type cursor: str | datastore_query.Cursor
:returns: Returns the query itself for chaining.
:rtype: server.db.Query
"""
if isinstance(cursor, basestring):
cursor = datastore_query.Cursor(urlsafe=cursor)
elif isinstance(cursor, datastore_query.Cursor) or cursor == None:
pass
else:
raise ValueError(
"Cursor must be String, datastore_query.Cursor or None")
if endCursor is not None:
if isinstance(endCursor, basestring):
endCursor = datastore_query.Cursor(urlsafe=endCursor)
elif isinstance(cursor,
datastore_query.Cursor) or endCursor == None:
pass
else:
raise ValueError(
"endCursor must be String, datastore_query.Cursor or None")
qo = self.datastoreQuery.__query_options
self.datastoreQuery.__query_options = datastore_query.QueryOptions(
keys_only=qo.keys_only,
produce_cursors=qo.produce_cursors,
start_cursor=cursor,
end_cursor=endCursor or qo.end_cursor,
projection=qo.projection)
self._origCursor = cursor
return (self)
def _make_query(self):
qry = Message.query().order(-Message.when)
options = datastore_query.QueryOptions(batch_size=13, limit=43)
return qry, options
def get_activity_timeline(self, query_id=None):
from models_acc import UserTimelineSystem, UserTimeline, UserTimelineFollowersIndex, UserTimelineSuggest
from geovote.models import Comment, Vote
from geoalert.models import Event
from geolist.models import List
# definir las consultas
query_chrono = UserTimelineFollowersIndex.all().filter(
'followers =', self.key()).order('-created')
query_activity = UserTimelineSystem.all().filter(
'user ='******'visible =', True).order('-modified')
# recuperar cursores
if query_id is not None and len(query_id) >= 2:
cursor_chronology = query_id[0]
cursor_activity = query_id[1]
query_chrono = query_chrono.with_cursor(
start_cursor=cursor_chronology)
query_activity = query_activity.with_cursor(
start_cursor=cursor_activity)
else:
cursor_activity = None
cursor_chronology = None
# let's go!
timeline = []
timeline_chrono = []
activity_async = query_activity.run(
config=datastore_query.QueryOptions(limit=TIMELINE_PAGE_SIZE))
chrono_async = query_chrono.run(config=datastore_query.QueryOptions(
limit=TIMELINE_PAGE_SIZE))
_go_chrono = True
chrono = None
for activity_timeline in activity_async:
while _go_chrono:
if len(timeline) + len(timeline_chrono) >= TIMELINE_PAGE_SIZE:
_go_chrono = False
break
if chrono is None:
try:
chrono = chrono_async.next()
except:
_go_chrono = False
break
if chrono is not None and chrono.created > activity_timeline.modified:
timeline_chrono.append(chrono)
chrono = None
else:
break
timeline.append(activity_timeline)
if len(timeline) + len(timeline_chrono) >= TIMELINE_PAGE_SIZE:
break
# generar timeline
timeline_chrono = fetch_parents(timeline_chrono)
timeline = prefetch_refprops(timeline, UserTimeline.user)
timeline_chrono = prefetch_refprops(timeline_chrono,
UserTimeline.instance,
UserTimeline.user)
timeline.extend(timeline_chrono)
#seguimos cargando por lotes todas las referencias
from helpers_acc import _load_ref_instances
instances = _load_ref_instances(timeline)
timeline = [{
'id':
int(activity_timeline.id),
'created':
activity_timeline.created,
'modified':
activity_timeline.modified,
'msg':
activity_timeline.msg,
'username':
activity_timeline.user.username,
'msg_id':
activity_timeline.msg_id,
'instance':
instances.get(
UserTimeline.instance.get_value_for_datastore(
activity_timeline), activity_timeline.instance),
'has_voted':
Vote.objects.user_has_voted(self, activity_timeline.instance.key())
if activity_timeline.instance is not None else None,
'vote_counter':
Vote.objects.get_vote_counter(activity_timeline.instance.key())
if activity_timeline.instance is not None else None,
'comments':
Comment.objects.get_by_instance(activity_timeline.instance,
querier=self),
'list':
instances.get(
UserTimelineSuggest.list_id.get_value_for_datastore(
activity_timeline), activity_timeline.list_id)
if isinstance(activity_timeline, UserTimelineSuggest) else None,
'status':
activity_timeline.status
if hasattr(activity_timeline, 'status') else None,
'is_private':
True,
'user_follower':
instances.get(
UserTimeline.instance.get_value_for_datastore(
activity_timeline),
activity_timeline.instance).has_follower(self) if hasattr(
instances.get(
UserTimeline.instance.get_value_for_datastore(
activity_timeline), activity_timeline.instance),
'has_follower') else None,
} for activity_timeline in timeline]
from operator import itemgetter
timeline_sorted = sorted(timeline,
key=itemgetter('modified'),
reverse=True)
chronology = [[query_chrono.cursor(),
query_activity.cursor()], timeline_sorted]
return chronology
class CommentHelper(object):
""" Helper de la clase comentario """
def get_by_id(self, id):
try:
id = long(id)
except:
return None
comment = Comment.get_by_id(id)
if comment is not None:
if comment.deleted:
return None
if comment._is_public():
return comment
return None
def get_by_key(self, key):
"""
Obtiene el evento con ese key
"""
return Comment.get(key)
def get_by_user(self, user, query_id=None, page=1, querier=None):
"""
Obtiene una lista con todos los comentarios hechos por un usuario
:param user: usuario a buscar
:type user: :class:`geouser.models.User`
:param query_id: identificador de la busqueda paginada
:type query_id: :class:`long`
:param page: pagina a buscar
:type page: :class:`integer`
:returns: [query_id, [:class:`geovote.models.Comment`]]
"""
if querier is not None and not isinstance(querier, User):
raise TypeError
from georemindme.paging import PagedQuery
from google.appengine.api import datastore
q = datastore.Query('Comment', {
'user ='******'deleted =': False
})
q.Order(('created', datastore.Query.DESCENDING))
p = PagedQuery(q, id=query_id, page_size=7)
comments = p.fetch_page(page)
from georemindme.funcs import prefetch_refpropsEntity
prefetch = prefetch_refpropsEntity(comments, 'user', 'instance')
return [
p.id,
[{
'id':
comment.key().id(),
'username':
prefetch[comment['user']].username,
'has_voted':
Vote.objects.user_has_voted(querier, comment.key())
if querier is not None else None,
'vote_counter':
comment['votes'],
'instance':
prefetch[comment['instance']],
'msg':
comment['msg'],
'created':
comment['created'],
} for comment in comments]
]
def get_by_instance(self,
instance,
query_id=None,
page=1,
querier=None,
async=False):
"""
Obtiene una lista con todos los comentarios hechos en una instancia
:param instance: objeto al que buscar los comentarios
:type instance: :class:`db.Model`
:param query_id: identificador de la busqueda paginada
:type query_id: :class:`long`
:param page: pagina a buscar
:type page: :class:`integer`
:returns: [query_id, [:class:`geovote.models.Comment`]]
"""
if querier is not None and not querier.is_authenticated():
querier = None
if querier is not None and not isinstance(querier, User):
raise TypeError
if instance is None:
return None
from georemindme.paging import PagedQuery
from google.appengine.api import datastore
q = datastore.Query(kind='Comment',
filters={
'instance =': instance.key(),
'deleted =': False
})
q.Order(('created', datastore.Query.DESCENDING))
p = PagedQuery(q, id=query_id, page_size=7)
if async:
from google.appengine.datastore import datastore_query
q = Comment.all().filter('instance =',
instance).filter('deleted =',
False).order('-created')
return p.id, q.run(config=datastore_query.QueryOptions(limit=7))
comments = p.fetch_page(page)
from georemindme.funcs import prefetch_refpropsEntity
prefetch = prefetch_refpropsEntity(comments, 'user', 'instance')
return [
p.id,
[{
'id':
comment.key().id(),
'username':
prefetch[comment['user']].username,
'has_voted':
Vote.objects.user_has_voted(querier, comment.key())
if querier is not None else None,
'vote_counter':
comment['votes'],
'instance':
prefetch[comment['instance']],
'msg':
comment['msg'],
'created':
comment['created'],
} for comment in comments]
]
