def run_to_queue(self, queue, conn, options=None): """Run this query, putting entities into the given queue.""" multiquery = self._maybe_multi_query() if multiquery is not None: multiquery.run_to_queue(queue, conn, options=options) # No return value. return dsqry, post_filters = self._get_query(conn) orig_options = options if (post_filters and options is not None and (options.offset or options.limit is not None)): options = datastore_query.QueryOptions(offset=None, limit=None, config=orig_options) assert options.limit is None and options.limit is None rpc = dsqry.run_async(conn, options) skipped = 0 count = 0 while rpc is not None: batch = yield rpc rpc = batch.next_batch_async(options) for ent in batch.results: if post_filters: if not post_filters.apply(ent): continue if orig_options is not options: if orig_options.offset and skipped < orig_options.offset: skipped += 1 continue if orig_options.limit is not None and count >= orig_options.limit: rpc = None # Quietly throw away the next batch. break count += 1 queue.putq(ent) queue.complete()
def _iter_key_range(self, k_range): raw_entity_kind = util.get_short_name(self._entity_kind) query = k_range.make_ascending_datastore_query(raw_entity_kind, keys_only=True) for key in query.Run(config=datastore_query.QueryOptions( batch_size=self._batch_size)): yield key, key
def __iter__(self): self._query = self._key_range.make_ascending_datastore_query( self._query_spec.entity_kind, filters=self._query_spec.filters) for entity in self._query.Run(config=datastore_query.QueryOptions( batch_size=self._query_spec.batch_size, keys_only=self._KEYS_ONLY, start_cursor=self._cursor)): yield entity
def __iter__(self): query = self._key_range.make_ascending_datastore_query( self._query_spec.entity_kind, filters=self._query_spec.filters) connection = datastore_rpc.Connection() query_options = datastore_query.QueryOptions( batch_size=self._query_spec.batch_size, start_cursor=self._cursor, produce_cursors=True) self._query = datastore_query.ResultsIterator(query.GetQuery().run( connection, query_options)) for entity_proto in self._query: yield entity_proto
def config_iterable(plain_config, batch_size=50, limit=1000): config = plain_config try: # This specific use of the QueryOptions private API was suggested to us by the App Engine team. # Wrapping in try/except in case it ever goes away. from google.appengine.datastore import datastore_query config = datastore_query.QueryOptions(config=plain_config, limit=limit, offset=0, prefetch_size=batch_size, batch_size=batch_size) except Exception, e: logging.exception("Failed to create QueryOptions config object: %s", e)
def cursor(self, cursor, endCursor=None): """ Sets the start cursor for this query. The result set will only include results behind that cursor. The cursor is generated by an earlier query with exactly the same configuration. Its safe to use client-supplied cursors, a cursor can't be abused to access entities which don't match the current filters. :param cursor: The cursor key to set to the Query. :type cursor: str | datastore_query.Cursor :returns: Returns the query itself for chaining. :rtype: server.db.Query """ if isinstance(cursor, basestring): cursor = datastore_query.Cursor(urlsafe=cursor) elif isinstance(cursor, datastore_query.Cursor) or cursor == None: pass else: raise ValueError( "Cursor must be String, datastore_query.Cursor or None") if endCursor is not None: if isinstance(endCursor, basestring): endCursor = datastore_query.Cursor(urlsafe=endCursor) elif isinstance(cursor, datastore_query.Cursor) or endCursor == None: pass else: raise ValueError( "endCursor must be String, datastore_query.Cursor or None") qo = self.datastoreQuery.__query_options self.datastoreQuery.__query_options = datastore_query.QueryOptions( keys_only=qo.keys_only, produce_cursors=qo.produce_cursors, start_cursor=cursor, end_cursor=endCursor or qo.end_cursor, projection=qo.projection) self._origCursor = cursor return (self)
def _make_query(self): qry = Message.query().order(-Message.when) options = datastore_query.QueryOptions(batch_size=13, limit=43) return qry, options
def get_activity_timeline(self, query_id=None): from models_acc import UserTimelineSystem, UserTimeline, UserTimelineFollowersIndex, UserTimelineSuggest from geovote.models import Comment, Vote from geoalert.models import Event from geolist.models import List # definir las consultas query_chrono = UserTimelineFollowersIndex.all().filter( 'followers =', self.key()).order('-created') query_activity = UserTimelineSystem.all().filter( 'user ='******'visible =', True).order('-modified') # recuperar cursores if query_id is not None and len(query_id) >= 2: cursor_chronology = query_id[0] cursor_activity = query_id[1] query_chrono = query_chrono.with_cursor( start_cursor=cursor_chronology) query_activity = query_activity.with_cursor( start_cursor=cursor_activity) else: cursor_activity = None cursor_chronology = None # let's go! timeline = [] timeline_chrono = [] activity_async = query_activity.run( config=datastore_query.QueryOptions(limit=TIMELINE_PAGE_SIZE)) chrono_async = query_chrono.run(config=datastore_query.QueryOptions( limit=TIMELINE_PAGE_SIZE)) _go_chrono = True chrono = None for activity_timeline in activity_async: while _go_chrono: if len(timeline) + len(timeline_chrono) >= TIMELINE_PAGE_SIZE: _go_chrono = False break if chrono is None: try: chrono = chrono_async.next() except: _go_chrono = False break if chrono is not None and chrono.created > activity_timeline.modified: timeline_chrono.append(chrono) chrono = None else: break timeline.append(activity_timeline) if len(timeline) + len(timeline_chrono) >= TIMELINE_PAGE_SIZE: break # generar timeline timeline_chrono = fetch_parents(timeline_chrono) timeline = prefetch_refprops(timeline, UserTimeline.user) timeline_chrono = prefetch_refprops(timeline_chrono, UserTimeline.instance, UserTimeline.user) timeline.extend(timeline_chrono) #seguimos cargando por lotes todas las referencias from helpers_acc import _load_ref_instances instances = _load_ref_instances(timeline) timeline = [{ 'id': int(activity_timeline.id), 'created': activity_timeline.created, 'modified': activity_timeline.modified, 'msg': activity_timeline.msg, 'username': activity_timeline.user.username, 'msg_id': activity_timeline.msg_id, 'instance': instances.get( UserTimeline.instance.get_value_for_datastore( activity_timeline), activity_timeline.instance), 'has_voted': Vote.objects.user_has_voted(self, activity_timeline.instance.key()) if activity_timeline.instance is not None else None, 'vote_counter': Vote.objects.get_vote_counter(activity_timeline.instance.key()) if activity_timeline.instance is not None else None, 'comments': Comment.objects.get_by_instance(activity_timeline.instance, querier=self), 'list': instances.get( UserTimelineSuggest.list_id.get_value_for_datastore( activity_timeline), activity_timeline.list_id) if isinstance(activity_timeline, UserTimelineSuggest) else None, 'status': activity_timeline.status if hasattr(activity_timeline, 'status') else None, 'is_private': True, 'user_follower': instances.get( UserTimeline.instance.get_value_for_datastore( activity_timeline), activity_timeline.instance).has_follower(self) if hasattr( instances.get( UserTimeline.instance.get_value_for_datastore( activity_timeline), activity_timeline.instance), 'has_follower') else None, } for activity_timeline in timeline] from operator import itemgetter timeline_sorted = sorted(timeline, key=itemgetter('modified'), reverse=True) chronology = [[query_chrono.cursor(), query_activity.cursor()], timeline_sorted] return chronology
class CommentHelper(object): """ Helper de la clase comentario """ def get_by_id(self, id): try: id = long(id) except: return None comment = Comment.get_by_id(id) if comment is not None: if comment.deleted: return None if comment._is_public(): return comment return None def get_by_key(self, key): """ Obtiene el evento con ese key """ return Comment.get(key) def get_by_user(self, user, query_id=None, page=1, querier=None): """ Obtiene una lista con todos los comentarios hechos por un usuario :param user: usuario a buscar :type user: :class:`geouser.models.User` :param query_id: identificador de la busqueda paginada :type query_id: :class:`long` :param page: pagina a buscar :type page: :class:`integer` :returns: [query_id, [:class:`geovote.models.Comment`]] """ if querier is not None and not isinstance(querier, User): raise TypeError from georemindme.paging import PagedQuery from google.appengine.api import datastore q = datastore.Query('Comment', { 'user ='******'deleted =': False }) q.Order(('created', datastore.Query.DESCENDING)) p = PagedQuery(q, id=query_id, page_size=7) comments = p.fetch_page(page) from georemindme.funcs import prefetch_refpropsEntity prefetch = prefetch_refpropsEntity(comments, 'user', 'instance') return [ p.id, [{ 'id': comment.key().id(), 'username': prefetch[comment['user']].username, 'has_voted': Vote.objects.user_has_voted(querier, comment.key()) if querier is not None else None, 'vote_counter': comment['votes'], 'instance': prefetch[comment['instance']], 'msg': comment['msg'], 'created': comment['created'], } for comment in comments] ] def get_by_instance(self, instance, query_id=None, page=1, querier=None, async=False): """ Obtiene una lista con todos los comentarios hechos en una instancia :param instance: objeto al que buscar los comentarios :type instance: :class:`db.Model` :param query_id: identificador de la busqueda paginada :type query_id: :class:`long` :param page: pagina a buscar :type page: :class:`integer` :returns: [query_id, [:class:`geovote.models.Comment`]] """ if querier is not None and not querier.is_authenticated(): querier = None if querier is not None and not isinstance(querier, User): raise TypeError if instance is None: return None from georemindme.paging import PagedQuery from google.appengine.api import datastore q = datastore.Query(kind='Comment', filters={ 'instance =': instance.key(), 'deleted =': False }) q.Order(('created', datastore.Query.DESCENDING)) p = PagedQuery(q, id=query_id, page_size=7) if async: from google.appengine.datastore import datastore_query q = Comment.all().filter('instance =', instance).filter('deleted =', False).order('-created') return p.id, q.run(config=datastore_query.QueryOptions(limit=7)) comments = p.fetch_page(page) from georemindme.funcs import prefetch_refpropsEntity prefetch = prefetch_refpropsEntity(comments, 'user', 'instance') return [ p.id, [{ 'id': comment.key().id(), 'username': prefetch[comment['user']].username, 'has_voted': Vote.objects.user_has_voted(querier, comment.key()) if querier is not None else None, 'vote_counter': comment['votes'], 'instance': prefetch[comment['instance']], 'msg': comment['msg'], 'created': comment['created'], } for comment in comments] ]