async def test_take(assert_run, event_loop):
with event_loop.assert_cleanup():
xs = stream.count() | add_resource.pipe(1) | pipe.take(3)
await assert_run(xs, [0, 1, 2])
with event_loop.assert_cleanup():
xs = stream.count() | add_resource.pipe(1) | pipe.take(0)
await assert_run(xs, [])
async def test_take(assert_run, event_loop):
with event_loop.assert_cleanup():
xs = stream.count() | add_resource.pipe(1) | pipe.take(3)
await assert_run(xs, [0, 1, 2])
with event_loop.assert_cleanup():
xs = stream.count() | add_resource.pipe(1) | pipe.take(0)
await assert_run(xs, [])
async def test_merge(assert_run, event_loop):
with event_loop.assert_cleanup():
xs = stream.range(1, 5, 2, interval=2) | pipe.delay(1)
ys = stream.range(0, 5, 2, interval=2) | pipe.merge(xs)
await assert_run(ys, [0, 1, 2, 3, 4])
assert event_loop.steps == [1, 1, 1, 1]
with event_loop.assert_cleanup():
xs = stream.range(1, 5, 2, interval=2) | pipe.delay(1)
ys = stream.range(0, 5, 2, interval=2) | pipe.merge(xs)
await assert_run(ys[:3], [0, 1, 2])
assert event_loop.steps == [1, 1]
with event_loop.assert_cleanup():
xs = stream.just(1) + stream.never()
ys = xs | pipe.merge(xs) | pipe.timeout(1)
await assert_run(ys, [1, 1], asyncio.TimeoutError())
assert event_loop.steps == [1]
# Reproduce issue #65
with event_loop.assert_cleanup():
xs = stream.iterate([1, 2])
ys = stream.iterate([3, 4])
zs = stream.merge(xs, ys) | pipe.take(3)
await assert_run(zs, [1, 2, 3])
with event_loop.assert_cleanup():
xs = stream.iterate([1, 2, 3])
ys = stream.throw(ZeroDivisionError)
zs = stream.merge(xs, ys) | pipe.delay(1) | pipe.take(3)
await assert_run(zs, [1, 2, 3])
# Silencing of a CancelledError
async def agen1():
if False:
yield
try:
await asyncio.sleep(2)
except asyncio.CancelledError:
return
async def agen2():
yield 1
with event_loop.assert_cleanup():
xs = stream.merge(agen1(), agen2()) | pipe.delay(1) | pipe.take(1)
await assert_run(xs, [1])
async def main():
xs = (
random() # Stream random numbers
| square.pipe() # Square the values
| pipe.take(5) # Take the first five
| pipe.accumulate()) # Sum the values
print(await xs)
async def main():
xs = (
random() # Stream random numbers
| square.pipe() # Square the values
| pipe.take(5) # Take the first five
| pipe.accumulate()) # Sum the values
print(await xs)
async def download_files(urls):
async with aiohttp.ClientSession() as session:
await (stream.repeat(session)
| pipe.zip(stream.iterate(fields.itertuples()))
| pipe.flatmap(generate_urls)
| pipe.filter(ignore_existing_file)
| pipe.take(1000)
| pipe.map(retrieve_file, ordered=False, task_limit=10))
async def main():
# This stream computes 11² + 13² in 1.5 second
xs = (
stream.count(interval=0.1) # Count from zero every 0.1 s
| pipe.skip(10) # Skip the first 10 numbers
| pipe.take(5) # Take the following 5
| pipe.filter(lambda x: x % 2) # Keep odd numbers
| pipe.map(lambda x: x**2) # Square the results
| pipe.accumulate() # Add the numbers together
)
print('11² + 13² = ', await xs)
async def main():
# This stream computes 11² + 13² in 1.5 second
xs = (
stream.count(interval=0.1) # Count from zero every 0.1 s
| pipe.skip(10) # Skip the first 10 numbers
| pipe.take(5) # Take the following 5
| pipe.filter(lambda x: x % 2) # Keep odd numbers
| pipe.map(lambda x: x ** 2) # Square the results
| pipe.accumulate() # Add the numbers together
)
print('11² + 13² = ', await xs)
async def handle():
async with KrakenClient() as client:
LOGGER.info("Subscribing")
await subscribe(client, SubscriptionKind.BOOK, PAIRS,
SUBSCRIPTION_DEPTH)
LOGGER.info("Waiting for snapshot messages")
snapshot = await (client.stream() | pipe.filter(is_snapshot)
| pipe.take(len(PAIRS)) | pipe.list())
LOGGER.info("Unsubscribing")
await unsubscribe(client, SubscriptionKind.BOOK, PAIRS,
SUBSCRIPTION_DEPTH)
return snapshot
async def run(
subscription_kinds: List[SubscriptionKind],
topic: str,
begin_timestamp=None,
interval_sec=None,
num_msgs=10,
on_message: Optional[Callable[[Any], Any]] = None,
):
def log_message(msg):
if not isinstance(msg, list):
#LOGGER.info("Status: %s", json.dumps(msg))
pass
else:
LOGGER.debug("Received message: %s", json.dumps(msg))
def maybe_callback(m):
if on_message:
on_message(m)
# the continuous version takes messages up until a time point
if interval_sec:
async with KrakenClient() as client:
for kind in subscription_kinds:
await subscribe(client, kind, PAIRS, SUBSCRIPTION_DEPTH)
await (client.stream(reconnect_count=WEBSOCKET_RECONNECT_COUNT)
| pipe.action(log_message)
| pipe.filter(lambda msg: isinstance(msg, list))
| pipe.map(json.dumps)
#| pipe.print()
| pipe.takewhile(
lambda x: time.time() - begin_timestamp < interval_sec)
| pipe.action(maybe_callback))
client.disconnect()
# the discrete version takes 100 distinct messages
else:
async with KrakenClient() as client:
for kind in subscription_kinds:
await subscribe(client, kind, PAIRS, SUBSCRIPTION_DEPTH)
await (client.stream(reconnect_count=WEBSOCKET_RECONNECT_COUNT)
| pipe.action(log_message)
| pipe.filter(lambda msg: isinstance(msg, list))
| pipe.map(json.dumps)
#| pipe.print()
| pipe.take(num_msgs)
| pipe.action(maybe_callback))
print("disconnecting")
client.disconnect()
async def test_concatmap(assert_run, event_loop):
# Concurrent run
with event_loop.assert_cleanup():
xs = stream.range(0, 6, 2, interval=1)
ys = xs | pipe.concatmap(lambda x: stream.range(x, x + 2, interval=5))
await assert_run(ys, [0, 1, 2, 3, 4, 5])
assert event_loop.steps == [1, 1, 3, 5, 5]
# Sequential run
with event_loop.assert_cleanup():
xs = stream.range(0, 6, 2, interval=1)
ys = xs | pipe.concatmap(
lambda x: stream.range(x, x + 2, interval=5), task_limit=1
)
await assert_run(ys, [0, 1, 2, 3, 4, 5])
assert event_loop.steps == [5, 1, 5, 1, 5]
# Limited run
with event_loop.assert_cleanup():
xs = stream.range(0, 6, 2, interval=1)
ys = xs | pipe.concatmap(
lambda x: stream.range(x, x + 2, interval=5), task_limit=2
)
await assert_run(ys, [0, 1, 2, 3, 4, 5])
assert event_loop.steps == [1, 4, 1, 4, 5]
# Make sure item arrive as soon as possible
with event_loop.assert_cleanup():
xs = stream.just(2)
ys = xs | pipe.concatmap(lambda x: stream.range(x, x + 4, interval=1))
zs = ys | pipe.timeout(2) # Sould NOT raise
await assert_run(zs, [2, 3, 4, 5])
assert event_loop.steps == [1, 1, 1]
# An exception might get discarded if the result can be produced before the
# processing of the exception is required
with event_loop.assert_cleanup():
xs = stream.iterate([True, False])
ys = xs | pipe.concatmap(
lambda x: stream.range(0, 3, interval=1)
if x
else stream.throw(ZeroDivisionError)
)
zs = ys | pipe.take(3)
await assert_run(zs, [0, 1, 2])
assert event_loop.steps == [1, 1]
async def run(self):
self.session = aiohttp.ClientSession(auth=self.auth)
self.limiter = FrequencyLock(self.interval)
asyncio.create_task(self.limiter.run())
pl = stream.iterate(self.find_images())
if self.limit:
pl = pl | pipe.take(self.limit)
pl = pl | pipe.map(self.tag, task_limit=self.tasks) | pipe.map(
self.store)
try:
await pl
except aiohttp.ClientResponseError as err:
raise click.ClickException(
"Imagga API failure ({.status}): probably exceeded "
"concurrency limits".format(err))
finally:
await self.limiter.stop()
await self.session.close()
