async def update_minute_price(symbol, time, price, volume):
# datetime(2020, 5, 18, 16, 55, 33) -> datetime(2020, 5, 18, 16, 56, 00)
rounded_time = time.replace(second=0, microsecond=0)
if rounded_time < time: # pretty always
rounded_time += timedelta(seconds=60)
while True:
try:
return await MINUTE_COLLECTION.update_one(
{
"symbol": symbol,
"time": rounded_time,
},
{
"$set": {"updated": time, "close": price, "volume": volume},
"$min": {"low": price},
"$max": {"high": price},
"$setOnInsert": {"open": price},
"$inc": {"updates": 1},
},
upsert=True
)
except PyMongoError as e:
logger.error(f"Update stat exc {type(e)}: {e}", extra={"MESSAGE_ID": "MONGODB_EXC"})
await asyncio.sleep(MONGODB_ERROR_INTERVAL)
async def get_minute_stats(symbol, limit=3000):
collection = get_mongodb_collection(collection_name=MONGODB_MINUTE_COLLECTION)
result = []
try:
cursor = collection.find(
{"symbol": symbol},
)
async for obj in cursor.limit(limit):
obj["id"] = str(obj.pop("_id"))
del obj["symbol"]
result.append(obj)
except PyMongoError as e:
logger.error(f"Mongodb error {type(e)}: {e}", extra={"MESSAGE_ID": "MONGODB_EXC"})
raise web.HTTPInternalServerError()
else:
return {"symbol": symbol, "data": result}
async def get_symbols(offset=0, limit=3000):
collection = get_mongodb_collection(collection_name=MONGODB_COLLECTION)
result = []
query = {}
try:
cursor = collection.find(
query,
{"name": 1}
)
async for obj in cursor.skip(offset).limit(limit):
result.append(dict(id=obj["_id"], name=obj["name"]))
except PyMongoError as e:
logger.error(f"Mongodb error {type(e)}: {e}", extra={"MESSAGE_ID": "MONGODB_EXC"})
raise web.HTTPInternalServerError()
else:
return result
async def main(symbols):
if not symbols:
symbols = [s["id"] for s in await get_symbols()]
conn = aiohttp.TCPConnector(ttl_dns_cache=300)
async with aiohttp.ClientSession(connector=conn,
headers=WS_SESSION_HEADER) as session:
ws = await session.ws_connect(WS_ENDPOINT)
msg_data = {"subscribe": symbols}
await ws.send_str(json.dumps(msg_data))
logger.info("Sent sub", extra={"MESSAGE": msg_data})
while RUN:
msg = await ws.receive()
if msg.type == aiohttp.WSMsgType.TEXT:
await process_message(msg.data)
else:
logger.error(f"Unexpected msg.type {msg}")
break
async def incoming_listener(ws, queue, functions):
subscribed_symbols = set()
try:
while not ws.closed:
incoming = await ws.receive()
logger.info("Got incoming msg", extra={"MESSAGE": incoming})
if incoming.type is WSMsgType.TEXT:
try:
data = json.loads(incoming.data)
except ValueError as e:
logger.error(e)
else:
new_functions = data.get("functions")
if new_functions:
# the same obj, but new contents
functions[:] = new_functions
subscribe = data.get("subscribe")
if subscribe:
subscribe = set(subscribe)
delete_symbols = subscribed_symbols - subscribe
for symbol in delete_symbols:
symbol_changes.unsubscribe(symbol, ws)
add_symbols = subscribe - subscribed_symbols
for symbol in add_symbols:
symbol_changes.subscribe(symbol, ws, queue)
logger.info(
f"Subscribed to {add_symbols} and unsubscribed from {delete_symbols}"
)
for symbol in add_symbols:
start_data = await get_minute_stats(symbol)
queue.put_nowait(start_data)
except ConnectionResetError as e:
logger.warning(f"ConnectionResetError while getting data {e}")
finally:
logger.info(f"Unsubscribe socket from {subscribed_symbols}")
for symbol in subscribed_symbols:
symbol_changes.unsubscribe(symbol, ws)
async def watch_changes_minute_stats():
collection = get_mongodb_collection(collection_name=MONGODB_MINUTE_COLLECTION)
resume_after = None
while True:
logger.info(f"Start watching mongodb changes after {resume_after}")
changes = collection.watch(full_document="updateLookup", resume_after=resume_after)
while True:
try:
change = await changes.next()
except PyMongoError as e:
logger.error(f"Got feed error {type(e)}: {e}", extra={"MESSAGE_ID": "MONGODB_EXC"})
await asyncio.sleep(MONGODB_ERROR_INTERVAL)
except StopAsyncIteration as e:
logger.exception(e)
resume_after = None
break
except Exception as e:
logger.exception(e)
await asyncio.sleep(MONGODB_ERROR_INTERVAL)
else:
resume_after = change["_id"]
if "fullDocument" in change: # present in insert/update changes
yield change["fullDocument"]