def helper(): inq = tasklets.SerialQueueFuture() query.run_to_queue(inq, self._conn, options) is_ancestor_query = query.ancestor is not None while True: try: ent = yield inq.getq() except EOFError: break if isinstance(ent, model.Key): pass # It was a keys-only query and ent is really a Key. else: key = ent.key if key in self._cache: # Assume the cache is more up to date. if self._cache[key] is None: # This is a weird case. Apparently this entity was # deleted concurrently with the query. Let's just # pretend the delete happened first. logging.info('Conflict: entity %s was deleted', key) continue # Replace the entity the callback will see with the one # from the cache. if ent != self._cache[key]: logging.info('Conflict: entity %s was modified', key) ent = self._cache[key] else: if is_ancestor_query and self.should_cache(key): self._cache[key] = ent if callback is None: val = ent else: val = callback(ent) # TODO: If this raises, log and ignore mfut.putq(val) mfut.complete()
def testSerialQueueFuture_PutQ_1(self): sqf = tasklets.SerialQueueFuture() f1 = Future() sqf.putq(f1) sqf.complete() f1.set_result(1) self.assertEqual(sqf.getq().get_result(), 1)
def testSerialQueueFuture(self): q = tasklets.SerialQueueFuture() @tasklets.tasklet def produce_one(i): yield tasklets.sleep(random.randrange(10) * 0.01) raise tasklets.Return(i) @tasklets.tasklet def producer(): for i in range(10): q.add_dependent(produce_one(i)) q.complete() @tasklets.tasklet def consumer(): for i in range(10): val = yield q.getq() self.assertEqual(val, i) yield q self.assertRaises(EOFError, q.getq().get_result) yield q @tasklets.synctasklet def foo(): yield producer(), consumer() foo()
def testSerialQueueFuture_ItemException(self): sqf = tasklets.SerialQueueFuture() g1 = sqf.getq() f1 = Future() sqf.putq(f1) sqf.complete() f1.set_exception(ZeroDivisionError()) self.assertRaises(ZeroDivisionError, g1.get_result)
def helper(): try: inq = tasklets.SerialQueueFuture() query.run_to_queue(inq, self._conn, options) is_ancestor_query = query.ancestor is not None while True: try: batch, i, ent = yield inq.getq() except EOFError: break if isinstance(ent, model.Key): pass # It was a keys-only query and ent is really a Key. else: key = ent._key if key in self._cache: hit = self._cache[key] if hit is not None and hit.key != key: # The cached entry has been mutated to have a different key. # That's a false hit. Get rid of it. See issue #13. del self._cache[key] if key in self._cache: # Assume the cache is more up to date. if self._cache[key] is None: # This is a weird case. Apparently this entity was # deleted concurrently with the query. Let's just # pretend the delete happened first. logging.info('Conflict: entity %s was deleted', key) continue # Replace the entity the callback will see with the one # from the cache. if ent != self._cache[key]: logging.info( 'Conflict: entity %s was modified', key) ent = self._cache[key] else: # Cache the entity only if this is an ancestor query; # non-ancestor queries may return stale results, since in # the HRD these queries are "eventually consistent". # TODO: Shouldn't we check this before considering cache hits? if is_ancestor_query and self.should_cache(key): self._cache[key] = ent if callback is None: val = ent else: # TODO: If the callback raises, log and ignore. if options is not None and options.produce_cursors: val = callback(batch, i, ent) else: val = callback(ent) mfut.putq(val) except Exception, err: _, _, tb = sys.exc_info() mfut.set_exception(err, tb) raise
def run_to_queue(self, queue, conn, options=None): """Run this query, putting entities into the given queue.""" # Create a list of (first-entity, subquery-iterator) tuples. # TODO: Use the specified sort order. assert options is None # Don't know what to do with these yet. state = [] orderings = orders_to_orderings(self.__orders) for subq in self.__subqueries: subit = tasklets.SerialQueueFuture('MultiQuery.run_to_queue') subq.run_to_queue(subit, conn) try: ent = yield subit.getq() except EOFError: continue else: state.append(_SubQueryIteratorState(ent, subit, orderings)) # Now turn it into a sorted heap. The heapq module claims that # calling heapify() is more efficient than calling heappush() for # each item. heapq.heapify(state) # Repeatedly yield the lowest entity from the state vector, # filtering duplicates. This is essentially a multi-way merge # sort. One would think it should be possible to filter # duplicates simply by dropping other entities already in the # state vector that are equal to the lowest entity, but because of # the weird sorting of repeated properties, we have to explicitly # keep a set of all keys, so we can remove later occurrences. # Yes, this means that the output may not be sorted correctly. # Too bad. (I suppose you can do this in constant memory bounded # by the maximum number of entries in relevant repeated # properties, but I'm too lazy for now. And yes, all this means # MultiQuery is a bit of a toy. But where it works, it beats # expecting the user to do this themselves.) keys_seen = set() while state: item = heapq.heappop(state) ent = item.entity if ent._key not in keys_seen: keys_seen.add(ent._key) queue.putq((None, None, ent)) subit = item.iterator try: batch, i, ent = yield subit.getq() except EOFError: pass else: item.entity = ent heapq.heappush(state, item) queue.complete()
def iter_query(self, query, options=None): return self.map_query(query, callback=None, options=options, merge_future=tasklets.SerialQueueFuture())
def testSerialQueueFuture_GetQ(self): sqf = tasklets.SerialQueueFuture() sqf.set_exception(KeyError()) self.assertRaises(KeyError, sqf.getq().get_result)
def testSerialQueueFuture_PutQ_3(self): sqf = tasklets.SerialQueueFuture() g1 = sqf.getq() sqf.putq(1) sqf.complete() self.assertEqual(g1.get_result(), 1)
def testSerialQueueFuture_Complete(self): sqf = tasklets.SerialQueueFuture() g1 = sqf.getq() sqf.complete() self.assertRaises(EOFError, g1.get_result)