async def test_take(assert_run, event_loop): with event_loop.assert_cleanup(): xs = stream.count() | add_resource.pipe(1) | pipe.take(3) await assert_run(xs, [0, 1, 2]) with event_loop.assert_cleanup(): xs = stream.count() | add_resource.pipe(1) | pipe.take(0) await assert_run(xs, [])
async def test_merge(assert_run, event_loop): with event_loop.assert_cleanup(): xs = stream.range(1, 5, 2, interval=2) | pipe.delay(1) ys = stream.range(0, 5, 2, interval=2) | pipe.merge(xs) await assert_run(ys, [0, 1, 2, 3, 4]) assert event_loop.steps == [1, 1, 1, 1] with event_loop.assert_cleanup(): xs = stream.range(1, 5, 2, interval=2) | pipe.delay(1) ys = stream.range(0, 5, 2, interval=2) | pipe.merge(xs) await assert_run(ys[:3], [0, 1, 2]) assert event_loop.steps == [1, 1] with event_loop.assert_cleanup(): xs = stream.just(1) + stream.never() ys = xs | pipe.merge(xs) | pipe.timeout(1) await assert_run(ys, [1, 1], asyncio.TimeoutError()) assert event_loop.steps == [1] # Reproduce issue #65 with event_loop.assert_cleanup(): xs = stream.iterate([1, 2]) ys = stream.iterate([3, 4]) zs = stream.merge(xs, ys) | pipe.take(3) await assert_run(zs, [1, 2, 3]) with event_loop.assert_cleanup(): xs = stream.iterate([1, 2, 3]) ys = stream.throw(ZeroDivisionError) zs = stream.merge(xs, ys) | pipe.delay(1) | pipe.take(3) await assert_run(zs, [1, 2, 3]) # Silencing of a CancelledError async def agen1(): if False: yield try: await asyncio.sleep(2) except asyncio.CancelledError: return async def agen2(): yield 1 with event_loop.assert_cleanup(): xs = stream.merge(agen1(), agen2()) | pipe.delay(1) | pipe.take(1) await assert_run(xs, [1])
async def main(): xs = ( random() # Stream random numbers | square.pipe() # Square the values | pipe.take(5) # Take the first five | pipe.accumulate()) # Sum the values print(await xs)
async def download_files(urls): async with aiohttp.ClientSession() as session: await (stream.repeat(session) | pipe.zip(stream.iterate(fields.itertuples())) | pipe.flatmap(generate_urls) | pipe.filter(ignore_existing_file) | pipe.take(1000) | pipe.map(retrieve_file, ordered=False, task_limit=10))
async def main(): # This stream computes 11² + 13² in 1.5 second xs = ( stream.count(interval=0.1) # Count from zero every 0.1 s | pipe.skip(10) # Skip the first 10 numbers | pipe.take(5) # Take the following 5 | pipe.filter(lambda x: x % 2) # Keep odd numbers | pipe.map(lambda x: x**2) # Square the results | pipe.accumulate() # Add the numbers together ) print('11² + 13² = ', await xs)
async def main(): # This stream computes 11² + 13² in 1.5 second xs = ( stream.count(interval=0.1) # Count from zero every 0.1 s | pipe.skip(10) # Skip the first 10 numbers | pipe.take(5) # Take the following 5 | pipe.filter(lambda x: x % 2) # Keep odd numbers | pipe.map(lambda x: x ** 2) # Square the results | pipe.accumulate() # Add the numbers together ) print('11² + 13² = ', await xs)
async def handle(): async with KrakenClient() as client: LOGGER.info("Subscribing") await subscribe(client, SubscriptionKind.BOOK, PAIRS, SUBSCRIPTION_DEPTH) LOGGER.info("Waiting for snapshot messages") snapshot = await (client.stream() | pipe.filter(is_snapshot) | pipe.take(len(PAIRS)) | pipe.list()) LOGGER.info("Unsubscribing") await unsubscribe(client, SubscriptionKind.BOOK, PAIRS, SUBSCRIPTION_DEPTH) return snapshot
async def run( subscription_kinds: List[SubscriptionKind], topic: str, begin_timestamp=None, interval_sec=None, num_msgs=10, on_message: Optional[Callable[[Any], Any]] = None, ): def log_message(msg): if not isinstance(msg, list): #LOGGER.info("Status: %s", json.dumps(msg)) pass else: LOGGER.debug("Received message: %s", json.dumps(msg)) def maybe_callback(m): if on_message: on_message(m) # the continuous version takes messages up until a time point if interval_sec: async with KrakenClient() as client: for kind in subscription_kinds: await subscribe(client, kind, PAIRS, SUBSCRIPTION_DEPTH) await (client.stream(reconnect_count=WEBSOCKET_RECONNECT_COUNT) | pipe.action(log_message) | pipe.filter(lambda msg: isinstance(msg, list)) | pipe.map(json.dumps) #| pipe.print() | pipe.takewhile( lambda x: time.time() - begin_timestamp < interval_sec) | pipe.action(maybe_callback)) client.disconnect() # the discrete version takes 100 distinct messages else: async with KrakenClient() as client: for kind in subscription_kinds: await subscribe(client, kind, PAIRS, SUBSCRIPTION_DEPTH) await (client.stream(reconnect_count=WEBSOCKET_RECONNECT_COUNT) | pipe.action(log_message) | pipe.filter(lambda msg: isinstance(msg, list)) | pipe.map(json.dumps) #| pipe.print() | pipe.take(num_msgs) | pipe.action(maybe_callback)) print("disconnecting") client.disconnect()
async def test_concatmap(assert_run, event_loop): # Concurrent run with event_loop.assert_cleanup(): xs = stream.range(0, 6, 2, interval=1) ys = xs | pipe.concatmap(lambda x: stream.range(x, x + 2, interval=5)) await assert_run(ys, [0, 1, 2, 3, 4, 5]) assert event_loop.steps == [1, 1, 3, 5, 5] # Sequential run with event_loop.assert_cleanup(): xs = stream.range(0, 6, 2, interval=1) ys = xs | pipe.concatmap( lambda x: stream.range(x, x + 2, interval=5), task_limit=1 ) await assert_run(ys, [0, 1, 2, 3, 4, 5]) assert event_loop.steps == [5, 1, 5, 1, 5] # Limited run with event_loop.assert_cleanup(): xs = stream.range(0, 6, 2, interval=1) ys = xs | pipe.concatmap( lambda x: stream.range(x, x + 2, interval=5), task_limit=2 ) await assert_run(ys, [0, 1, 2, 3, 4, 5]) assert event_loop.steps == [1, 4, 1, 4, 5] # Make sure item arrive as soon as possible with event_loop.assert_cleanup(): xs = stream.just(2) ys = xs | pipe.concatmap(lambda x: stream.range(x, x + 4, interval=1)) zs = ys | pipe.timeout(2) # Sould NOT raise await assert_run(zs, [2, 3, 4, 5]) assert event_loop.steps == [1, 1, 1] # An exception might get discarded if the result can be produced before the # processing of the exception is required with event_loop.assert_cleanup(): xs = stream.iterate([True, False]) ys = xs | pipe.concatmap( lambda x: stream.range(0, 3, interval=1) if x else stream.throw(ZeroDivisionError) ) zs = ys | pipe.take(3) await assert_run(zs, [0, 1, 2]) assert event_loop.steps == [1, 1]
async def run(self): self.session = aiohttp.ClientSession(auth=self.auth) self.limiter = FrequencyLock(self.interval) asyncio.create_task(self.limiter.run()) pl = stream.iterate(self.find_images()) if self.limit: pl = pl | pipe.take(self.limit) pl = pl | pipe.map(self.tag, task_limit=self.tasks) | pipe.map( self.store) try: await pl except aiohttp.ClientResponseError as err: raise click.ClickException( "Imagga API failure ({.status}): probably exceeded " "concurrency limits".format(err)) finally: await self.limiter.stop() await self.session.close()