async def wait(self, shown_event: trio.Event = trio.Event()) -> str:
"""See :meth:`qtrio.dialogs.DialogProtocol.wait`."""
with _manage(dialog=self) as finished_event:
if self.dialog is None: # pragma: no cover
raise qtrio.InternalError(
"Dialog not assigned while it is being managed."
)
shown_event.set()
await finished_event.wait()
dialog_result = self.dialog.result()
if dialog_result == QtWidgets.QDialog.Rejected:
raise qtrio.UserCancelledError()
# TODO: `: str` is a workaround for
# https://github.com/spyder-ide/qtpy/pull/217
text_result: str = self.dialog.textValue()
self.result = text_result
return text_result
class Parser(Section):
def __init__(self) -> None:
self._connected = Event()
async def pump(self, input, output):
async for message in input:
if 'info' in message:
self._connected.set()
log.debug('Connected to BitMEX realtime api.')
elif 'subscribe' in message:
if message['success']:
log.debug('Subscribed to %s.', message["subscribe"])
else:
log.error(
'Unable to subscribe to %s. Error: "%s" Please check and restart.',
message["request"]["args"][0], message["error"])
elif 'action' in message:
await output(message)
elif 'request' in message and 'op' in message[
'request'] and message['request']['op'] == 'cancelAllAfter':
log.debug(
'Dead mans switch reset. All open orders will be cancelled at %s.',
message['cancelTime'])
elif 'error' in message:
log.error('%s - Request: %s', message['error'],
message['request'])
raise BitMEXWebsocketApiError(message['status'],
message['error'])
else:
log.warning('Received unknown message type: %s', message)
async def sleep_and_err(
ev: trio.Event,
task_status: TaskStatus = trio.TASK_STATUS_IGNORED,
):
await trio.sleep(0.5)
doggy() # noqa
ev.set()
task_status.started()
async def wrapper(prev_done: trio.Event, self_done: trio.Event, item: T) -> None:
async with limiter:
result = await function(item)
await prev_done.wait()
await send_result.send(result)
self_done.set()
async def wrapper(prev_done: trio.Event, self_done: trio.Event,
item: T) -> None:
# pylint: disable=not-async-context-manager
async with limiter:
result = await function(item)
await prev_done.wait()
await send_result.send(result)
self_done.set()
async def _ag_awaitable_wrapper(outcome: dict, end_signal: trio.Event,
ag_awaitable):
try:
outcome['return_value'] = await ag_awaitable
except GeneratorExit:
outcome['cancelled'] = True
raise
except Exception as e:
outcome['exception'] = e
finally:
end_signal.set()
async def wrapper(prev_done: trio.Event, self_done: trio.Event, item: T) -> None:
nonlocal collected_result
input_item = await wait_for(item)
if collected_result is SENTINEL:
# We are working on the first item, and initializer was not set.
collected_result = input_item
else:
async with limiter:
collected_result = await function(collected_result, input_item)
await prev_done.wait()
self_done.set()
async def spawn_child_nursery(spawn, shutdown_timeout=math.inf):
send_channel, receive_channel = trio.open_memory_channel(0)
async with receive_channel:
shutdown_trigger = Event()
spawn(_run_nursery_until_event, send_channel, shutdown_trigger,
shutdown_timeout)
return await receive_channel.receive(), shutdown_trigger
async def wait(self, shown_event: trio.Event = trio.Event()) -> None:
"""See :meth:`qtrio.dialogs.DialogProtocol.wait`."""
with _manage(dialog=self) as finished_event:
if self.dialog is None: # pragma: no cover
raise qtrio.InternalError(
"Dialog not assigned while it is being managed."
)
shown_event.set()
await finished_event.wait()
result = self.dialog.result()
if result == QtWidgets.QDialog.Rejected:
raise qtrio.UserCancelledError()
async def aiterassets(
dandiset: RemoteDandiset, done_flag: trio.Event
) -> AsyncIterator[Optional[RemoteAsset]]:
last_ts: Optional[datetime] = None
if dandiset.version_id == "draft":
vs = [v for v in dandiset.get_versions() if v.identifier != "draft"]
vs.sort(key=attrgetter("created"))
versions = deque(vs)
else:
versions = deque()
async with httpx.AsyncClient() as client:
url: Optional[
str
] = f"{dandiset.client.api_url}{dandiset.version_api_path}assets/?order=created"
while url is not None:
r = await arequest(client, "GET", url)
data = r.json()
for item in data["results"]:
r = await arequest(
client,
"GET",
f"{dandiset.client.api_url}/assets/{item['asset_id']}/",
)
metadata = r.json()
asset = RemoteAsset.from_data(dandiset, item, metadata)
assert last_ts is None or last_ts <= asset.created, (
f"Asset {asset.path} created at {asset.created} but"
f" returned after an asset created at {last_ts}!"
)
if (
versions
and (last_ts is None or last_ts < versions[0].created)
and asset.created >= versions[0].created
):
log.info(
"All assets up to creation of version %s found;"
" will commit soon",
versions[0].identifier,
)
versions.popleft()
yield None
last_ts = asset.created
yield asset
url = data.get("next")
log.info("Finished getting assets from API")
done_flag.set()
async def _enter_and_wait(
mngr: AsyncContextManager[T],
unwrapped: dict[int, T],
all_entered: trio.Event,
parent_exit: trio.Event,
) -> None:
'''
Open the async context manager deliver it's value
to this task's spawner and sleep until cancelled.
'''
async with mngr as value:
unwrapped[id(mngr)] = value
if all(unwrapped.values()):
all_entered.set()
await parent_exit.wait()
async def _enter_and_sleep(
mngr: AsyncContextManager[T],
to_yield: dict[int, T],
all_entered: trio.Event,
# task_status: TaskStatus[T] = trio.TASK_STATUS_IGNORED,
) -> T:
'''Open the async context manager deliver it's value
to this task's spawner and sleep until cancelled.
'''
async with mngr as value:
to_yield[id(mngr)] = value
if all(to_yield.values()):
all_entered.set()
# sleep until cancelled
await trio.sleep_forever()
async def _monitor_signals(
self, ready: trio.Event,
signal_aiter: AsyncIterator["signal.Signals"]) -> None:
import signal # noqa: F811
import logging
ready.set()
async for sig in signal_aiter:
logger = logging.getLogger()
if sig == signal.SIGTERM:
logger.info("Stopping: SIGTERM")
elif sig == signal.SIGINT:
logger.info("Stopping: CTRL+C")
else:
logger.error("Stopping: unexpected signal: %s", sig.name)
self.manager.cancel()
async def wait(self, shown_event: trio.Event = trio.Event()) -> trio.Path:
"""See :meth:`qtrio.dialogs.DialogProtocol.wait`."""
with _manage(dialog=self) as finished_event:
if self.dialog is None: # pragma: no cover
raise qtrio.InternalError(
"Dialog not assigned while it is being managed."
)
shown_event.set()
await finished_event.wait()
if self.dialog.result() != QtWidgets.QDialog.Accepted:
raise qtrio.UserCancelledError()
[path_string] = self.dialog.selectedFiles()
self.result = trio.Path(path_string)
return self.result
async def broadcast_until_stopped(self,
generate_message: callable(None),
stop_signal: trio.Event,
update_interval=0.25):
print("movement starting")
while not stop_signal.is_set():
msg = await generate_message()
await self.broadcast(msg)
await trio.sleep(update_interval)
print("movement stopped")
async def wait_on_coro_final_result(
to_trio: trio.MemorySendChannel,
coro: Awaitable,
aio_task_complete: trio.Event,
) -> None:
'''
Await ``coro`` and relay result back to ``trio``.
'''
nonlocal aio_err
nonlocal chan
orig = result = id(coro)
try:
result = await coro
except GeneratorExit:
# no need to relay error
raise
except BaseException as aio_err:
chan._aio_err = aio_err
raise
else:
if (result != orig and aio_err is None and
# in the ``open_channel_from()`` case we don't
# relay through the "return value".
not provide_channels):
to_trio.send_nowait(result)
finally:
# if the task was spawned using ``open_channel_from()``
# then we close the channels on exit.
if provide_channels:
# only close the sender side which will relay
# a ``trio.EndOfChannel`` to the trio (consumer) side.
to_trio.close()
aio_task_complete.set()
async def monitor(self, check_rate, max_time, stop_signal: trio.Event):
self._mon_stop = stop_signal
while not stop_signal.is_set():
if self.resources_available():
async with self.resource_lock:
if self.resources_available():
res = await self.recv_chan.receive()
last_used = time.time() - res.last_use
print(f"{res.name}:mon: res last used {last_used}")
if last_used > max_time:
print(f"closing res {res.name}")
self.current_resources -= 1
await res.aclose()
else:
await self.send_chan.send(res)
else:
print("no resources")
await trio.sleep(check_rate)
self._mon_stop = None
print("closing monitor...")
async def _heartbeat_task(self, kill_switch:trio.Event):
while not kill_switch.is_set():
await trio.sleep(1)
def __init__(self) -> None:
self._connected = Event()
async def stream_quotes(
send_chan: trio.abc.SendChannel,
symbols: list[str],
feed_is_live: trio.Event,
loglevel: str = None,
# backend specific
sub_type: str = 'ohlc',
# startup sync
task_status: TaskStatus[tuple[dict, dict]] = trio.TASK_STATUS_IGNORED,
) -> None:
'''
Subscribe for ohlc stream of quotes for ``pairs``.
``pairs`` must be formatted <crypto_symbol>/<fiat_symbol>.
'''
# XXX: required to propagate ``tractor`` loglevel to piker logging
get_console_log(loglevel or tractor.current_actor().loglevel)
ws_pairs = {}
sym_infos = {}
async with open_cached_client('kraken') as client, send_chan as send_chan:
# keep client cached for real-time section
for sym in symbols:
# transform to upper since piker style is always lower
sym = sym.upper()
si = Pair(**await client.symbol_info(sym)) # validation
syminfo = si.dict()
syminfo['price_tick_size'] = 1 / 10**si.pair_decimals
syminfo['lot_tick_size'] = 1 / 10**si.lot_decimals
syminfo['asset_type'] = 'crypto'
sym_infos[sym] = syminfo
ws_pairs[sym] = si.wsname
symbol = symbols[0].lower()
init_msgs = {
# pass back token, and bool, signalling if we're the writer
# and that history has been written
symbol: {
'symbol_info': sym_infos[sym],
'shm_write_opts': {'sum_tick_vml': False},
'fqsn': sym,
},
}
@acm
async def subscribe(ws: wsproto.WSConnection):
# XXX: setup subs
# https://docs.kraken.com/websockets/#message-subscribe
# specific logic for this in kraken's shitty sync client:
# https://github.com/krakenfx/kraken-wsclient-py/blob/master/kraken_wsclient_py/kraken_wsclient_py.py#L188
ohlc_sub = make_sub(
list(ws_pairs.values()),
{'name': 'ohlc', 'interval': 1}
)
# TODO: we want to eventually allow unsubs which should
# be completely fine to request from a separate task
# since internally the ws methods appear to be FIFO
# locked.
await ws.send_msg(ohlc_sub)
# trade data (aka L1)
l1_sub = make_sub(
list(ws_pairs.values()),
{'name': 'spread'} # 'depth': 10}
)
# pull a first quote and deliver
await ws.send_msg(l1_sub)
yield
# unsub from all pairs on teardown
await ws.send_msg({
'pair': list(ws_pairs.values()),
'event': 'unsubscribe',
'subscription': ['ohlc', 'spread'],
})
# XXX: do we need to ack the unsub?
# await ws.recv_msg()
# see the tips on reconnection logic:
# https://support.kraken.com/hc/en-us/articles/360044504011-WebSocket-API-unexpected-disconnections-from-market-data-feeds
ws: NoBsWs
async with open_autorecon_ws(
'wss://ws.kraken.com/',
fixture=subscribe,
) as ws:
# pull a first quote and deliver
msg_gen = process_data_feed_msgs(ws)
# TODO: use ``anext()`` when it lands in 3.10!
typ, ohlc_last = await msg_gen.__anext__()
topic, quote = normalize(ohlc_last)
task_status.started((init_msgs, quote))
# lol, only "closes" when they're margin squeezing clients ;P
feed_is_live.set()
# keep start of last interval for volume tracking
last_interval_start = ohlc_last.etime
# start streaming
async for typ, ohlc in msg_gen:
if typ == 'ohlc':
# TODO: can get rid of all this by using
# ``trades`` subscription...
# generate tick values to match time & sales pane:
# https://trade.kraken.com/charts/KRAKEN:BTC-USD?period=1m
volume = ohlc.volume
# new OHLC sample interval
if ohlc.etime > last_interval_start:
last_interval_start = ohlc.etime
tick_volume = volume
else:
# this is the tick volume *within the interval*
tick_volume = volume - ohlc_last.volume
ohlc_last = ohlc
last = ohlc.close
if tick_volume:
ohlc.ticks.append({
'type': 'trade',
'price': last,
'size': tick_volume,
})
topic, quote = normalize(ohlc)
elif typ == 'l1':
quote = ohlc
topic = quote['symbol'].lower()
await send_chan.send({topic: quote})
async def stream_quotes(
send_chan: trio.abc.SendChannel,
symbols: list[str],
feed_is_live: trio.Event,
loglevel: str = None,
# startup sync
task_status: TaskStatus[tuple[dict, dict]] = trio.TASK_STATUS_IGNORED,
) -> None:
# XXX: required to propagate ``tractor`` loglevel to piker logging
get_console_log(loglevel or tractor.current_actor().loglevel)
sym_infos = {}
uid = 0
async with (
open_cached_client('binance') as client,
send_chan as send_chan,
):
# keep client cached for real-time section
cache = await client.cache_symbols()
for sym in symbols:
d = cache[sym.upper()]
syminfo = Pair(**d) # validation
si = sym_infos[sym] = syminfo.dict()
# XXX: after manually inspecting the response format we
# just directly pick out the info we need
si['price_tick_size'] = float(syminfo.filters[0]['tickSize'])
si['lot_tick_size'] = float(syminfo.filters[2]['stepSize'])
si['asset_type'] = 'crypto'
symbol = symbols[0]
init_msgs = {
# pass back token, and bool, signalling if we're the writer
# and that history has been written
symbol: {
'symbol_info': sym_infos[sym],
'shm_write_opts': {'sum_tick_vml': False},
'fqsn': sym,
},
}
@acm
async def subscribe(ws: wsproto.WSConnection):
# setup subs
# trade data (aka L1)
# https://binance-docs.github.io/apidocs/spot/en/#symbol-order-book-ticker
l1_sub = make_sub(symbols, 'bookTicker', uid)
await ws.send_msg(l1_sub)
# aggregate (each order clear by taker **not** by maker)
# trades data:
# https://binance-docs.github.io/apidocs/spot/en/#aggregate-trade-streams
agg_trades_sub = make_sub(symbols, 'aggTrade', uid)
await ws.send_msg(agg_trades_sub)
# ack from ws server
res = await ws.recv_msg()
assert res['id'] == uid
yield
subs = []
for sym in symbols:
subs.append("{sym}@aggTrade")
subs.append("{sym}@bookTicker")
# unsub from all pairs on teardown
await ws.send_msg({
"method": "UNSUBSCRIBE",
"params": subs,
"id": uid,
})
# XXX: do we need to ack the unsub?
# await ws.recv_msg()
async with open_autorecon_ws(
'wss://stream.binance.com/ws',
fixture=subscribe,
) as ws:
# pull a first quote and deliver
msg_gen = stream_messages(ws)
typ, quote = await msg_gen.__anext__()
while typ != 'trade':
# TODO: use ``anext()`` when it lands in 3.10!
typ, quote = await msg_gen.__anext__()
task_status.started((init_msgs, quote))
# signal to caller feed is ready for consumption
feed_is_live.set()
# import time
# last = time.time()
# start streaming
async for typ, msg in msg_gen:
# period = time.time() - last
# hz = 1/period if period else float('inf')
# if hz > 60:
# log.info(f'Binance quotez : {hz}')
topic = msg['symbol'].lower()
await send_chan.send({topic: msg})
async def open_chain(
self,
portal: tractor.Portal,
complete: trio.Event,
started: trio.Event,
fqsn: str,
dst_shm: ShmArray,
conf: dict,
target: Fsp,
loglevel: str,
) -> None:
'''
Task which opens a remote FSP endpoint in the managed
cluster and sleeps until signalled to exit.
'''
ns_path = str(target.ns_path)
async with (
portal.open_context(
# chaining entrypoint
cascade,
# data feed key
fqsn=fqsn,
# mems
src_shm_token=self.src_shm.token,
dst_shm_token=dst_shm.token,
# target
ns_path=ns_path,
loglevel=loglevel,
zero_on_step=conf.get('zero_on_step', False),
shm_registry=[
(token.as_msg(), fsp_name, dst_token.as_msg())
for (token, fsp_name), dst_token
in self._flow_registry.items()
],
) as (ctx, last_index),
ctx.open_stream() as stream,
):
# register output data
self._registry[
(fqsn, ns_path)
] = (
stream,
dst_shm,
complete
)
started.set()
# wait for graceful shutdown signal
async with stream.subscribe() as stream:
async for msg in stream:
if msg == 'update':
# if the chart isn't hidden try to update
# the data on screen.
if not self.linked.isHidden():
log.info(f'Re-syncing graphics for fsp: {ns_path}')
self.linked.graphics_cycle(trigger_all=True)
else:
log.info(f'recved unexpected fsp engine msg: {msg}')
await complete.wait()
async def open_order_mode(
feed: Feed,
chart: 'ChartPlotWidget', # noqa
fqsn: str,
started: trio.Event,
) -> None:
'''Activate chart-trader order mode loop:
- connect to emsd
- load existing positions
- begin EMS response handling loop which updates local
state, mostly graphics / UI.
'''
multistatus = chart.window().status_bar
done = multistatus.open_status('starting order mode..')
book: OrderBook
trades_stream: tractor.MsgStream
# The keys in this dict **must** be in set our set of "normalized"
# symbol names (i.e. the same names you'd get back in search
# results) in order for position msgs to correctly trigger the
# display of a position indicator on screen.
position_msgs: dict[str, list[BrokerdPosition]]
# spawn EMS actor-service
async with (
open_ems(fqsn) as (
book,
trades_stream,
position_msgs,
brokerd_accounts,
),
trio.open_nursery() as tn,
):
log.info(f'Opening order mode for {fqsn}')
view = chart.view
# annotations editors
lines = LineEditor(chart=chart)
arrows = ArrowEditor(chart, {})
# symbol id
symbol = chart.linked.symbol
symkey = symbol.front_fqsn()
# map of per-provider account keys to position tracker instances
trackers: dict[str, PositionTracker] = {}
# load account names from ``brokers.toml``
accounts_def = config.load_accounts(
providers=symbol.brokers
)
# XXX: ``brokerd`` delivers a set of account names that it allows
# use of but the user also can define the accounts they'd like
# to use, in order, in their `brokers.toml` file.
accounts = {}
for name in brokerd_accounts:
# ensure name is in ``brokers.toml``
accounts[name] = accounts_def[name]
# first account listed is the one we select at startup
# (aka order based selection).
pp_account = next(
# choose first account based on line order from `brokers.toml`.
iter(accounts.keys())
) if accounts else 'paper'
# NOTE: requires the backend exactly specifies
# the expected symbol key in its positions msg.
pp_msgs = position_msgs.get(symkey, ())
pps_by_account = {msg['account']: msg for msg in pp_msgs}
# update pp trackers with data relayed from ``brokerd``.
for account_name in accounts:
# net-zero pp
startup_pp = Position(
symbol=symbol,
size=0,
avg_price=0,
)
msg = pps_by_account.get(account_name)
if msg:
log.info(f'Loading pp for {symkey}:\n{pformat(msg)}')
startup_pp.update_from_msg(msg)
# allocator config
alloc = mk_allocator(
symbol=symbol,
account=account_name,
# if this startup size is greater the allocator limit,
# the limit is increased internally in this factory.
startup_pp=startup_pp,
)
pp_tracker = PositionTracker(
chart,
alloc,
startup_pp
)
pp_tracker.hide()
trackers[account_name] = pp_tracker
assert pp_tracker.startup_pp.size == pp_tracker.live_pp.size
# TODO: do we even really need the "startup pp" or can we
# just take the max and pass that into the some state / the
# alloc?
pp_tracker.update_from_pp()
# on existing position, show pp tracking graphics
if pp_tracker.startup_pp.size != 0:
pp_tracker.show()
pp_tracker.hide_info()
# setup order mode sidepane widgets
form = chart.sidepane
vbox = form.vbox
from textwrap import dedent
from PyQt5.QtCore import Qt
from ._style import _font, _font_small
from ..calc import humanize
feed_label = FormatLabel(
fmt_str=dedent("""
actor: **{actor_name}**\n
|_ @**{host}:{port}**\n
|_ throttle_hz: **{throttle_rate}**\n
|_ streams: **{symbols}**\n
|_ shm: **{shm}**\n
"""),
font=_font.font,
font_size=_font_small.px_size,
font_color='default_lightest',
)
form.feed_label = feed_label
# add feed info label to top
vbox.insertWidget(
0,
feed_label,
alignment=Qt.AlignBottom,
)
# vbox.setAlignment(feed_label, Qt.AlignBottom)
# vbox.setAlignment(Qt.AlignBottom)
_ = chart.height() - (
form.height() +
form.fill_bar.height()
# feed_label.height()
)
vbox.setSpacing(
int((1 + 5/8)*_font.px_size)
)
# fill in brokerd feed info
host, port = feed.portal.channel.raddr
if host == '127.0.0.1':
host = 'localhost'
mpshm = feed.shm._shm
shmstr = f'{humanize(mpshm.size)}'
form.feed_label.format(
actor_name=feed.portal.channel.uid[0],
host=host,
port=port,
symbols=len(feed.symbols),
shm=shmstr,
throttle_rate=feed.throttle_rate,
)
order_pane = SettingsPane(
form=form,
# XXX: ugh, so hideous...
fill_bar=form.fill_bar,
pnl_label=form.left_label,
step_label=form.bottom_label,
limit_label=form.top_label,
)
order_pane.set_accounts(list(trackers.keys()))
# update pp icons
for name, tracker in trackers.items():
order_pane.update_account_icons({name: tracker.live_pp})
# top level abstraction which wraps all this crazyness into
# a namespace..
mode = OrderMode(
chart,
tn,
feed,
book,
lines,
arrows,
multistatus,
pane=order_pane,
trackers=trackers,
)
# XXX: MUST be set
order_pane.order_mode = mode
# select a pp to track
tracker = trackers[pp_account]
mode.current_pp = tracker
tracker.show()
tracker.hide_info()
# XXX: would love to not have to do this separate from edit
# fields (which are done in an async loop - see below)
# connect selection signals (from drop down widgets)
# to order sync pane handler
for key in ('account', 'size_unit',):
w = form.fields[key]
w.currentTextChanged.connect(
partial(
order_pane.on_selection_change,
key=key,
)
)
# make fill bar and positioning snapshot
order_pane.on_ui_settings_change('limit', tracker.alloc.limit())
order_pane.update_status_ui(pp=tracker)
# TODO: create a mode "manager" of sorts?
# -> probably just call it "UxModes" err sumthin?
# so that view handlers can access it
view.order_mode = mode
order_pane.on_ui_settings_change('account', pp_account)
mode.pane.display_pnl(mode.current_pp)
# Begin order-response streaming
done()
# start async input handling for chart's view
async with (
# ``ChartView`` input async handler startup
chart.view.open_async_input_handler(),
# pp pane kb inputs
open_form_input_handling(
form,
focus_next=chart.linked.godwidget,
on_value_change=order_pane.on_ui_settings_change,
),
):
# signal to top level symbol loading task we're ready
# to handle input since the ems connection is ready
started.set()
tn.start_soon(
process_trades_and_update_ui,
tn,
feed,
mode,
trades_stream,
book,
)
yield mode
async def stop_task(self, kill_switch:trio.Event):
kill_switch.set()
async def manage_history(
mod: ModuleType,
bus: _FeedsBus,
fqsn: str,
some_data_ready: trio.Event,
feed_is_live: trio.Event,
task_status: TaskStatus = trio.TASK_STATUS_IGNORED,
) -> None:
'''
Load and manage historical data including the loading of any
available series from `marketstore` as well as conducting real-time
update of both that existing db and the allocated shared memory
buffer.
'''
# (maybe) allocate shm array for this broker/symbol which will
# be used for fast near-term history capture and processing.
shm, opened = maybe_open_shm_array(
key=fqsn,
# use any broker defined ohlc dtype:
dtype=getattr(mod, '_ohlc_dtype', base_iohlc_dtype),
# we expect the sub-actor to write
readonly=False,
)
# TODO: history validation
if not opened:
raise RuntimeError(
"Persistent shm for sym was already open?!"
)
log.info('Scanning for existing `marketstored`')
is_up = await check_for_service('marketstored')
# for now only do backfilling if no tsdb can be found
do_legacy_backfill = not is_up and opened
bfqsn = fqsn.replace('.' + mod.name, '')
open_history_client = getattr(mod, 'open_history_client', None)
if is_up and opened and open_history_client:
log.info('Found existing `marketstored`')
from . import marketstore
async with marketstore.open_storage_client(
fqsn,
) as storage:
# TODO: this should be used verbatim for the pure
# shm backfiller approach below.
# start history anal and load missing new data via backend.
series, _, last_tsdb_dt = await storage.load(fqsn)
broker, symbol, expiry = unpack_fqsn(fqsn)
(
shm,
latest_start_dt,
latest_end_dt,
bf_done,
) = await bus.nursery.start(
partial(
start_backfill,
mod,
bfqsn,
shm,
last_tsdb_dt=last_tsdb_dt,
storage=storage,
)
)
# if len(shm.array) < 2:
# TODO: there's an edge case here to solve where if the last
# frame before market close (at least on ib) was pushed and
# there was only "1 new" row pushed from the first backfill
# query-iteration, then the sample step sizing calcs will
# break upstream from here since you can't diff on at least
# 2 steps... probably should also add logic to compute from
# the tsdb series and stash that somewhere as meta data on
# the shm buffer?.. no se.
task_status.started(shm)
some_data_ready.set()
await bf_done.wait()
# do diff against last start frame of history and only fill
# in from the tsdb an allotment that allows for most recent
# to be loaded into mem *before* tsdb data.
if last_tsdb_dt:
dt_diff_s = (
latest_start_dt - last_tsdb_dt
).seconds
else:
dt_diff_s = 0
# await trio.sleep_forever()
# TODO: see if there's faster multi-field reads:
# https://numpy.org/doc/stable/user/basics.rec.html#accessing-multiple-fields
# re-index with a `time` and index field
prepend_start = shm._first.value
# sanity check on most-recent-data loading
assert prepend_start > dt_diff_s
history = list(series.values())
if history:
fastest = history[0]
to_push = fastest[:prepend_start]
shm.push(
to_push,
# insert the history pre a "days worth" of samples
# to leave some real-time buffer space at the end.
prepend=True,
# update_first=False,
# start=prepend_start,
field_map=marketstore.ohlc_key_map,
)
# load as much from storage into shm as space will
# allow according to user's shm size settings.
count = 0
end = fastest['Epoch'][0]
while shm._first.value > 0:
count += 1
series = await storage.read_ohlcv(
fqsn,
end=end,
)
history = list(series.values())
fastest = history[0]
end = fastest['Epoch'][0]
prepend_start -= len(to_push)
to_push = fastest[:prepend_start]
shm.push(
to_push,
# insert the history pre a "days worth" of samples
# to leave some real-time buffer space at the end.
prepend=True,
# update_first=False,
# start=prepend_start,
field_map=marketstore.ohlc_key_map,
)
# manually trigger step update to update charts/fsps
# which need an incremental update.
for delay_s in sampler.subscribers:
await broadcast(delay_s)
if count > 6:
break
log.info(f'Loaded {to_push.shape} datums from storage')
# TODO: write new data to tsdb to be ready to for next read.
if do_legacy_backfill:
# do a legacy incremental backfill from the provider.
log.info('No existing `marketstored` found..')
# start history backfill task ``backfill_bars()`` is
# a required backend func this must block until shm is
# filled with first set of ohlc bars
await bus.nursery.start(
partial(
start_backfill,
mod,
bfqsn,
shm,
)
)
# yield back after client connect with filled shm
task_status.started(shm)
# indicate to caller that feed can be delivered to
# remote requesting client since we've loaded history
# data that can be used.
some_data_ready.set()
# history retreival loop depending on user interaction and thus
# a small RPC-prot for remotely controllinlg what data is loaded
# for viewing.
await trio.sleep_forever()