async def test_insert_value(autojump_clock):
async with Pipeline.create(
produce_alphabet(1, max=3, delay=1),
InsertValue('n')) as pipeline, pipeline.tap() as aiter:
start_time = trio.current_time()
results = [(v, trio.current_time() - start_time) async for v in aiter]
assert results == [('n', 0), ('a', 1), ('b', 2), ('c', 3)]
async def test_simple_process_section():
value = 'hello, world!'
async with Pipeline.create(
SimpleProcessSection(value)) as pipeline, pipeline.tap() as aiter:
results = [i async for i in aiter]
assert results[0] == value
async def test_welding_two_generator_functions_not_allowed(autojump_clock):
with pytest.raises(ValueError):
async with Pipeline.create(
produce_increasing_integers(1),
produce_increasing_integers(1),
) as pipeline, pipeline.tap() as aiter:
result = [i async for i in aiter]
async def test_thread_section_early_break(autojump_clock):
async with Pipeline.create(produce_increasing_integers(
1, max=5), SyncSquares()) as pipeline, pipeline.tap() as aiter:
async for i in aiter:
if i == 4:
break
assert i == 4
async def test_changes(autojump_clock):
async with Pipeline.create(
Merge(produce_increasing_integers(1, max=5),
produce_increasing_integers(1, max=5)),
Changes()) as pipeline, pipeline.tap() as aiter:
result = [i async for i in aiter]
assert result == [0, 1, 2, 3, 4]
async def test_spam_requests():
with pytest.raises(BitMEXWebsocketApiError):
async with open_bitmex_websocket('testnet') as ws:
async with Pipeline.create(ws.listen(
'instrument',
'PAROTCOIN')) as pipeline, pipeline.tap() as aiter:
async for bundle in aiter:
break
async def test_ratelimit_callable_subject(autojump_clock):
async with Pipeline.create(
RateLimit(1,
produce_mappings(0.5),
subject=lambda item: item['vehicle'][2])
) as pipeline, pipeline.tap() as aiter:
result = [item['number'] async for item in aiter]
assert result == [0, 1, 3, 4, 6, 7, 8]
async def test_thread_section_exception(autojump_clock):
with pytest.raises(RuntimeError):
async with Pipeline.create(produce_increasing_integers(
1, max=5), SyncSquares(
raise_after=4)) as pipeline, pipeline.tap() as aiter:
async for i in aiter:
pass
assert i == 9
async def test_delay(autojump_clock):
async def timestamp():
yield trio.current_time()
async with Pipeline.create(
Delay(1, timestamp())
) as pipeline, pipeline.tap() as aiter:
async for item in aiter:
assert trio.current_time() - item == 1
async def test_repeat_kwargs(autojump_clock):
results = []
async with Pipeline.create(Repeat(
1, default='a')) as pipeline, pipeline.tap() as aiter:
start_time = trio.current_time()
async for item in aiter:
results.append((item, trio.current_time() - start_time))
if len(results) == 5:
break
assert results == [('a', 0), ('a', 1), ('a', 2), ('a', 3), ('a', 4)]
async def test_repeat_input(autojump_clock):
results = []
async with Pipeline.create(produce_alphabet(1.5, max=3, delay=1),
Repeat(1)) as pipeline, pipeline.tap() as aiter:
start_time = trio.current_time()
async for item in aiter:
results.append((item, trio.current_time() - start_time))
if len(results) == 5:
break
assert results == [('a', 1), ('a', 2), ('b', 2.5), ('b', 3.5), ('c', 4)]
async def test_early_tap_closure_aiter(autojump_clock):
async def spammer(stop):
for i in range(stop):
yield i
for _ in range(100):
async with Pipeline.create(
spammer(10), ) as pipeline, pipeline.tap() as aiter:
async for i in aiter:
assert isinstance(i, int)
break
async def test_metronome():
async with Pipeline.create(produce_alphabet(
5, max=3), Metronome(5)) as pipeline, pipeline.tap() as aiter:
results = []
start_time = trio.current_time()
async for item in aiter:
results.append((item, trio.current_time() - start_time))
if len(results) == 2:
break
assert [x[0] for x in results] == ['a', 'b']
assert 5 - results[1][1] + results[0][1] < 0.1
async def test_json(echo_server, hello_world):
send_channel, receive_channel = trio.open_memory_channel(1)
async with Pipeline.create(
receive_channel,
Websocket.create(
'127.0.0.1', 49000, '/', use_ssl=False,
parse_json=True)) as pipeline, pipeline.tap() as tap:
await send_channel.send(hello_world)
response = await tap.receive()
assert 'message' in response
assert response['message'] == 'hello, world'
async def test_funding():
async with open_bitmex_websocket('testnet') as ws:
async with Pipeline.create(Group(
2, ws.listen('funding'))) as pipeline, pipeline.tap() as aiter:
async for bundle in aiter:
for funding in bundle:
funding['timestamp'] = pendulum.parse(funding['timestamp'])
funding['fundingInterval'] = pendulum.parse(
funding['fundingInterval'])
assert isinstance(bundle, tuple)
assert len(bundle) > 1
return
assert False, 'This should not happen.'
async def test_early_tap_closure_section(autojump_clock):
class Spammer(TrioSection):
def __init__(self, stop) -> None:
self.stop = stop
async def refine(self, input, output):
for i in range(self.stop):
await output(i)
for _ in range(100):
async with Pipeline.create(
Spammer(10), ) as pipeline, pipeline.tap() as aiter:
async for i in aiter:
assert isinstance(i, int)
break
async def test_disconnect(disconnecting_server, autojump_clock):
async def listener(tap):
async for item in tap:
assert item in {'bark', 'cart', 'suffer', 'impress'}
try:
async with Pipeline.create(
Websocket.create('127.0.0.1',
49000,
'/',
use_ssl=False,
parse_json=False)
) as pipeline, trio.open_nursery() as nursery:
nursery.start_soon(listener, pipeline.tap())
nursery.start_soon(listener, pipeline.tap())
except ConnectionClosed as cls:
assert cls.reason.code == 1000
assert cls.reason.reason == 'Plug pulled.'
async def test_window(autojump_clock):
async with Pipeline.create(
Window(3, produce_increasing_integers(1, max=5))
) as pipeline, pipeline.tap() as aiter:
result = [item async for item in aiter]
assert result == [(0,), (0, 1), (0, 1, 2), (1, 2, 3), (2, 3, 4)]
async def test_group_timeout(autojump_clock):
async with Pipeline.create(
Group(2.5, spam_wait_spam_integers(5))
) as pipeline, pipeline.tap() as aiter:
result = [item async for item in aiter]
assert result == [(0, 1, 2, 3, 4), (0, 1, 2, 3, 4)]
async def test_thread_section_section_input(autojump_clock):
async with Pipeline.create(
produce_increasing_integers(1), Map(lambda i: i),
SyncSquares()) as pipeline, pipeline.tap() as aiter:
result = [i async for i in aiter]
assert result == [0, 1, 4]
async def test_thread_section(autojump_clock):
async with Pipeline.create(produce_increasing_integers(
1, max=5), SyncSquares()) as pipeline, pipeline.tap() as aiter:
result = [i async for i in aiter]
assert result == [0, 1, 4, 9, 16]
async def test_ratelimit(autojump_clock):
async with Pipeline.create(
RateLimit(1, produce_mappings(0.5))
) as pipeline, pipeline.tap() as aiter:
result = [item['number'] async for item in aiter]
assert result == [0, 3, 6]
async def test_pipeline_create(autojump_clock):
async with Pipeline.create(None):
await trio.sleep(1)
async def test_fibonacci_section():
async with Pipeline.create(
FibonacciSection(20)) as pipeline, pipeline.tap() as aiter:
results = [i async for i in aiter]
assert len(results) == 20
assert results[-1] == 4181
async def test_skip(autojump_clock):
async with Pipeline.create(
Skip(5, produce_increasing_integers(1, max=10))
) as pipeline, pipeline.tap() as aiter:
result = [i async for i in aiter]
assert result == [5, 6, 7, 8, 9]
async def test_repeat_valid_args():
with pytest.raises(RuntimeError):
async with Pipeline.create(
Repeat(1)) as pipeline, pipeline.tap() as aiter:
async for item in aiter:
assert False, 'No items should be emitted due to invalid arguments provided.'
async def _connect(self, network, api_key, api_secret, dead_mans_switch):
"""Open a BitMEX websocket connection."""
try:
if network == 'mainnet':
url = 'wss://ws.bitmex.com/realtime'
else:
url = 'wss://ws.testnet.bitmex.com/realtime'
log.debug('Generating authentication headers.')
# To auth to the WS using an API key, we generate a signature of a nonce and
# the WS API endpoint.
headers = None
if api_key and api_secret:
nonce = generate_expires()
headers = [('api-expires', str(nonce)),
('api-signature',
generate_signature(api_secret, 'GET',
'/realtime', nonce, '')),
('api-key', api_key)]
send_channel, receive_channel = trio.open_memory_channel(math.inf)
self._send_channel = send_channel
if dead_mans_switch:
sections = [
Merge(
receive_channel,
Repeat(15,
default={
'op': 'cancelAllAfter',
'args': 60000
}))
]
else:
sections = [receive_channel]
self._websocket = Websocket(url, extra_headers=headers)
parser = Parser()
sections.append(self._websocket)
sections.append(parser)
sections.append(self.storage)
async with Pipeline.create(*sections) as pipeline:
self._pipeline = pipeline
# Force the websocket to connect
pipeline._enabled.set()
await parser._connected.wait()
log.info('BitMEXWebsocket open.')
yield self
log.debug(
'BitMEXWebsocket context exit. Cancelling running tasks.')
pipeline.nursery.cancel_scope.cancel()
except ConnectionClosed as cls:
log.warning('BitMEXWebsocket closed (%d) %s.', cls.reason.code,
cls.reason.name)
raise
except OSError as ose:
log.error('Connection attempt failed: %s', type(ose).__name__)
log.info('BitMEXWebsocket closed.')
async def test_pipeline_passthrough(autojump_clock):
async with Pipeline.create(produce_increasing_integers(1)) as pipeline:
async with pipeline.tap() as aiter:
result = [i async for i in aiter]
assert result == [0, 1, 2]
async def test_welding(autojump_clock):
async with Pipeline.create(produce_increasing_integers(1),
(Map(lambda i: i + 1), )) as pipeline:
async with pipeline.tap() as aiter:
result = [i async for i in aiter]
assert result == [1, 2, 3]
async def test_group_max_size(autojump_clock):
async with Pipeline.create(
Group(2.5, produce_increasing_integers(1, max=5), max_size=3)
) as pipeline, pipeline.tap() as aiter:
result = [item async for item in aiter]
assert result == [(0, 1, 2), (3, 4)]
