def test_ttl(self):
cache = TTLCache(maxsize=2, ttl=2, timer=Timer())
self.assertEqual(0, cache.timer())
self.assertEqual(2, cache.ttl)
cache[1] = 1
self.assertEqual(1, cache[1])
self.assertEqual(1, len(cache))
self.assertEqual({1}, set(cache))
cache.timer.tick()
self.assertEqual(1, cache[1])
self.assertEqual(1, len(cache))
self.assertEqual({1}, set(cache))
cache[2] = 2
self.assertEqual(1, cache[1])
self.assertEqual(2, cache[2])
self.assertEqual(2, len(cache))
self.assertEqual({1, 2}, set(cache))
cache.timer.tick()
self.assertNotIn(1, cache)
self.assertEqual(2, cache[2])
self.assertEqual(1, len(cache))
self.assertEqual({2}, set(cache))
cache[3] = 3
self.assertNotIn(1, cache)
self.assertEqual(2, cache[2])
self.assertEqual(3, cache[3])
self.assertEqual(2, len(cache))
self.assertEqual({2, 3}, set(cache))
cache.timer.tick()
self.assertNotIn(1, cache)
self.assertNotIn(2, cache)
self.assertEqual(3, cache[3])
self.assertEqual(1, len(cache))
self.assertEqual({3}, set(cache))
cache.timer.tick()
self.assertNotIn(1, cache)
self.assertNotIn(2, cache)
self.assertNotIn(3, cache)
with self.assertRaises(KeyError):
del cache[1]
with self.assertRaises(KeyError):
cache.pop(2)
with self.assertRaises(KeyError):
del cache[3]
self.assertEqual(0, len(cache))
self.assertEqual(set(), set(cache))
def test_ttl(self):
cache = TTLCache(maxsize=2, ttl=1, timer=Timer())
self.assertEqual(0, cache.timer())
self.assertEqual(1, cache.ttl)
cache[1] = 1
self.assertEqual({1}, set(cache))
self.assertEqual(1, len(cache))
self.assertEqual(1, cache[1])
cache.timer.tick()
self.assertEqual({1}, set(cache))
self.assertEqual(1, len(cache))
self.assertEqual(1, cache[1])
cache[2] = 2
self.assertEqual({1, 2}, set(cache))
self.assertEqual(2, len(cache))
self.assertEqual(1, cache[1])
self.assertEqual(2, cache[2])
cache.timer.tick()
self.assertEqual({2}, set(cache))
self.assertEqual(1, len(cache))
self.assertNotIn(1, cache)
self.assertEqual(2, cache[2])
cache[3] = 3
self.assertEqual({2, 3}, set(cache))
self.assertEqual(2, len(cache))
self.assertNotIn(1, cache)
self.assertEqual(2, cache[2])
self.assertEqual(3, cache[3])
cache.timer.tick()
self.assertEqual({3}, set(cache))
self.assertEqual(1, len(cache))
self.assertNotIn(1, cache)
self.assertNotIn(2, cache)
self.assertEqual(3, cache[3])
cache.timer.tick()
self.assertEqual(set(), set(cache))
self.assertEqual(0, len(cache))
self.assertNotIn(1, cache)
self.assertNotIn(2, cache)
self.assertNotIn(3, cache)
with self.assertRaises(KeyError):
del cache[1]
with self.assertRaises(KeyError):
cache.pop(2)
with self.assertRaises(KeyError):
del cache[3]
class MemoryCache:
#
# コンストラクタ
#
def __init__(self):
self._cache = TTLCache(maxsize=1024, ttl=3600)
#
# set
#
def set(self, key, value):
self._cache[key] = value
#
# get
#
def get(self, key):
if key in self._cache: return self._cache[key]
return None
#
# delete
#
def delete(self, key):
return self._cache.pop(key)
def test_atomic(self):
cache = TTLCache(maxsize=1, ttl=1, timer=Timer(auto=True))
cache[1] = 1
self.assertEqual(1, cache[1])
cache[1] = 1
self.assertEqual(1, cache.get(1))
cache[1] = 1
self.assertEqual(1, cache.pop(1))
cache[1] = 1
self.assertEqual(1, cache.setdefault(1))
def test_atomic(self):
cache = TTLCache(maxsize=1, ttl=1, timer=Timer(auto=True))
cache[1] = 1
self.assertEqual(1, cache[1])
cache[1] = 1
self.assertEqual(1, cache.get(1))
cache[1] = 1
self.assertEqual(1, cache.pop(1))
cache[1] = 1
self.assertEqual(1, cache.setdefault(1))
class BaseBackend:
TIMEOUT = 30 * 1000
def __init__(self, ttl):
self.ttl = ttl
self.values = TTLCache(maxsize=1024, ttl=ttl / 1000)
def load_data(self, key: str, data):
try:
self.values[key] = json.loads(data)
except (TypeError, ValueError):
self.values[key] = data
def fetch_value(self, key: str):
pass
def get(self, key: str) -> Union[bytes, dict]:
if key not in self.values:
self.fetch_value(key)
try:
# still maybe empty
return self.values[key]
except KeyError:
return MISSING
def clear(self, key: str):
self.values.pop(key, None)
def save(self, key: str, value: Any, ttl: int = None):
self.values[key] = value
def wait(self, key: str, timeout: int):
# same as get in BaseBackend
return self.get(key)
def request_renew(self, key: str, value: str, timeout: int):
return True
def finish_renew(self, key: str, value: str):
pass
class FuturesTrader:
def __init__(self):
self.client: AsyncClient = None
self.state: dict = None
self.prices: dict = {}
self.symbols: dict = {}
self.price_streamer = None
self.clocks = NamedLock()
self.olock = asyncio.Lock() # lock to place only one order at a time
self.slock = asyncio.Lock() # lock for stream subscriptions
self.order_queue = asyncio.Queue()
# cache to disallow orders with same symbol, entry and first TP for 10 mins
self.sig_cache = TTLCache(maxsize=100, ttl=600)
self.balance = 0
self.results_handler = None
async def init(self, api_key, api_secret, state={}, test=False, loop=None):
self.state = state
self.client = await AsyncClient.create(api_key=api_key,
api_secret=api_secret,
testnet=test,
loop=loop)
self.manager = BinanceSocketManager(self.client, loop=loop)
self.user_stream = UserStream(api_key, api_secret, test=test)
if not self.state.get("streams"):
self.state["streams"] = []
if not self.state.get("orders"):
self.state["orders"] = {}
await self._gather_orders()
await self._watch_orders()
await self._subscribe_futures_user()
resp = await self.client.futures_exchange_info()
for info in resp["symbols"]:
self.symbols[info["symbol"]] = info
resp = await self.client.futures_account_balance()
for item in resp:
if item["asset"] == "USDT":
self.balance = float(item["balance"])
logging.info(f"Account balance: {self.balance} USDT", on="blue")
async def queue_signal(self, signal: Signal):
await self.order_queue.put(signal)
async def close_trades(self, tag, coin=None):
if coin is None:
logging.info(
f"Attempting to close all trades associated with channel {tag}",
color="yellow")
else:
logging.info(
f"Attempting to close {coin} trades associated with channel {tag}",
color="yellow")
async with self.olock:
removed = []
for order_id, order in self.state["orders"].items():
if order.get("tag") != tag:
continue
if coin is not None and order["sym"] != f"{coin}USDT":
continue
children = [] + order["t_ord"]
if order.get("s_ord"):
children.append(order["s_ord"])
removed += children
for oid in children:
await self._cancel_order(oid, order["sym"])
quantity = 0
for tid, q in zip(order["t_ord"], order["t_q"]):
if not self.state["orders"].get(tid, {}).get("filled"):
quantity += q
try:
if quantity > 0:
resp = await self.client.futures_create_order(
symbol=order["sym"],
positionSide="LONG"
if order["side"] == "BUY" else "SHORT",
side="SELL" if order["side"] == "BUY" else "BUY",
type=OrderType.MARKET,
quantity=self._round_qty(order["sym"], quantity),
)
else:
resp = await self.client.futures_cancel_order(
symbol=order["sym"],
origClientOrderId=order_id,
)
logging.info(
f"Closed position for order {order}, resp: {resp}",
color="yellow")
except Exception as err:
logging.error(
f"Failed to close position for order {order}, err: {err}"
)
for oid in removed:
self.state["orders"].pop(oid, None)
if not removed:
logging.info(
f"Didn't find any matching positions for {tag} to close",
color="yellow")
async def _gather_orders(self):
async def _gatherer():
logging.info("Waiting for orders to be queued...")
while True:
signal = await self.order_queue.get()
if self.symbols.get(f"{signal.coin}USDT") is None:
logging.info(f"Unknown symbol {signal.coin} in signal",
color="yellow")
continue
async def _process(signal):
# Process one order at a time for each symbol
async with self.clocks.lock(signal.coin):
registered = await self._register_order_for_signal(
signal)
if not registered:
logging.info(
f"Ignoring signal from {signal.tag} because order exists "
f"for {signal.coin}",
color="yellow")
return
for i in range(ORDER_MAX_RETRIES):
try:
await self._place_order(signal)
return
except PriceUnavailableException:
logging.info(
f"Price unavailable for {signal.coin}",
color="red")
except EntryCrossedException as err:
logging.info(
f"Price went too fast ({err.price}) for signal {signal}",
color="yellow")
except InsufficientQuantityException as err:
logging.info(
f"Allocated ${round(err.alloc_funds, 2)} for {err.alloc_q} {signal.coin} "
f"but requires ${round(err.est_funds, 2)} for {err.est_q} {signal.coin}",
color="red")
except Exception as err:
logging.error(
f"Failed to place order: {traceback.format_exc()} {err}"
)
break # unknown error - don't block future signals
if i < ORDER_MAX_RETRIES - 1:
await asyncio.sleep(ORDER_RETRY_SLEEP)
await self._unregister_order(signal)
asyncio.ensure_future(_process(signal))
asyncio.ensure_future(_gatherer())
async def _place_order(self, signal: Signal):
await self._subscribe_futures(signal.coin)
for _ in range(10):
if self.prices.get(signal.coin) is not None:
break
logging.info(f"Waiting for {signal.coin} price to be available")
await asyncio.sleep(1)
if self.prices.get(signal.coin) is None:
raise PriceUnavailableException()
symbol = f"{signal.coin}USDT"
logging.info(f"Modifying leverage to {signal.leverage}x for {symbol}",
color="green")
await self.client.futures_change_leverage(symbol=symbol,
leverage=signal.leverage)
price = self.prices[signal.coin]
signal.correct(price)
alloc_funds = self.balance * signal.fraction
quantity = alloc_funds / (price / signal.leverage)
logging.info(f"Corrected signal: {signal}", color="cyan")
qty = self._round_qty(symbol, quantity)
est_funds = qty * signal.entry / signal.leverage
if (est_funds / alloc_funds) > PRICE_SLIPPAGE:
raise InsufficientQuantityException(quantity, alloc_funds, qty,
est_funds)
order_id = OrderID.wait() if signal.wait_entry else OrderID.market()
params = {
"symbol": symbol,
"positionSide": "LONG" if signal.is_long else "SHORT",
"side": "BUY" if signal.is_long else "SELL",
"type": OrderType.MARKET,
"quantity": qty,
"newClientOrderId": order_id,
}
if (signal.is_long and price > signal.max_entry) or (
signal.is_short and price < signal.max_entry):
raise EntryCrossedException(price)
if signal.wait_entry:
logging.info(
f"Placing stop limit order for {signal.coin} (price @ {price}, entry @ {signal.entry})"
)
params["type"] = OrderType.STOP
params["stopPrice"] = self._round_price(symbol, signal.entry)
params["price"] = self._round_price(symbol, signal.max_entry)
else:
logging.info(
f"Placing market order for {signal.coin} (price @ {price}, entry @ {signal.entry}"
)
async with self.olock: # Lock only for interacting with orders
try:
resp = await self.client.futures_create_order(**params)
self.state["orders"][order_id] = {
"id": resp["orderId"],
"qty": float(resp["origQty"]),
"sym": symbol,
"side": params["side"],
"ent": signal.entry if signal.wait_entry else price,
"sl": signal.sl,
"tgt": signal.targets,
"fnd": alloc_funds,
"lev": signal.leverage,
"tag": signal.tag,
"crt": int(time.time()),
"t_ord": [],
"t_q": [],
}
logging.info(f"Created order {order_id} for signal: {signal}, "
f"params: {json.dumps(params)}, resp: {resp}")
except Exception as err:
logging.error(
f"Failed to create order for signal {signal}: {err}, "
f"params: {json.dumps(params)}")
if isinstance(err, BinanceAPIException) and err.code == -2021:
raise EntryCrossedException(price)
async def _place_collection_orders(self, order_id):
await self._place_sl_order(order_id)
async with self.olock:
odata = self.state["orders"][order_id]
await self.results_handler(
Trade.entry(odata["tag"], odata["sym"], odata["ent"],
odata["qty"], odata["lev"], odata["side"]))
if odata.get("t_ord"):
logging.warning(
f"TP order(s) already exist for parent {order_id}")
return
targets = odata["tgt"][:MAX_TARGETS]
remaining = quantity = odata["qty"]
for i, tgt in enumerate(targets):
quantity *= 0.5
# NOTE: Don't close position (as it'll affect other orders)
if i == len(targets) - 1:
quantity = self._round_qty(odata["sym"], remaining)
else:
quantity = self._round_qty(odata["sym"], quantity)
tgt_order_id = await self._create_target_order(
order_id, odata["sym"], odata["side"], tgt, quantity)
if tgt_order_id is None:
continue
odata["t_ord"].append(tgt_order_id)
odata["t_q"].append(quantity)
self.state["orders"][tgt_order_id] = {
"parent": order_id,
"filled": False,
}
remaining -= quantity
async def _create_target_order(self, order_id, symbol, side, tgt_price,
rounded_qty):
tgt_order_id = OrderID.target()
params = {
"symbol": symbol,
"type": OrderType.LIMIT,
"timeInForce": "GTC",
"positionSide": "LONG" if side == "BUY" else "SHORT",
"side": "SELL" if side == "BUY" else "BUY",
"newClientOrderId": tgt_order_id,
"price": self._round_price(symbol, tgt_price),
"quantity": rounded_qty,
}
try:
resp = await self.client.futures_create_order(**params)
logging.info(
f"Created limit order {tgt_order_id} for parent {order_id}, "
f"resp: {resp}, params: {json.dumps(params)}")
return tgt_order_id
except Exception as err:
logging.error(
f"Failed to create target order for parent {order_id}: {err}, "
f"params: {json.dumps(params)}")
async def _subscribe_futures_user(self):
async def _handler():
while True:
async with self.user_stream.message() as msg:
try:
data = msg
event = msg['e']
if event == UserEventType.AccountUpdate:
data = msg["a"]
elif event == UserEventType.OrderTradeUpdate:
data = msg["o"]
elif event == UserEventType.AccountConfigUpdate:
data = msg.get("ac", msg.get("ai"))
logging.info(f"{event}: {data}")
await self._handle_event(msg)
except Exception as err:
logging.exception(
f"Failed to handle event {msg}: {err}")
asyncio.ensure_future(_handler())
async def _handle_event(self, msg: dict):
if msg["e"] == UserEventType.AccountUpdate:
for info in msg["a"]["B"]:
if info["a"] == "USDT":
self.balance = float(info["cw"])
logging.info(f"Account balance: {self.balance} USDT",
on="blue")
elif msg["e"] == UserEventType.OrderTradeUpdate:
info = msg["o"]
order_id = info["c"]
async with self.olock:
o = self.state["orders"].get(order_id)
if o is None:
logging.warning(
f"Received order {order_id} but missing in state")
return
if info["X"] == "FILLED":
if OrderID.is_wait(order_id) or OrderID.is_market(order_id):
entry = float(info["ap"])
logging.info(
f"Placing TP/SL orders for fulfilled order {order_id} (entry: {entry})",
color="green")
async with self.olock:
self.state["orders"][order_id]["ent"] = entry
await self._place_collection_orders(order_id)
elif OrderID.is_stop_loss(order_id):
async with self.olock:
logging.info(
f"Order {order_id} hit stop loss. Removing TP orders...",
color="red")
sl = self.state["orders"].pop(order_id)
parent = self.state["orders"].pop(sl["parent"])
for oid in parent["t_ord"]:
self.state["orders"].pop(
oid, None) # It might not exist
await self._cancel_order(oid, parent["sym"])
await self.results_handler(
Trade.target(parent["tag"],
parent["sym"],
parent["ent"],
parent["qty"],
parent["lev"],
float(info["ap"]),
float(info["q"]),
is_long=parent["side"] == "BUY",
is_sl=True))
elif OrderID.is_target(order_id):
logging.info(f"TP order {order_id} hit.", color="green")
await self._move_stop_loss(order_id)
async def _move_stop_loss(self, tp_id: str):
async with self.olock:
tp = self.state["orders"][tp_id]
tp["filled"] = True
parent = self.state["orders"][tp["parent"]]
targets = parent["t_ord"]
if tp_id not in targets:
if parent.get("s_ord") is None:
logging.warning(f"SL doesn't exist for order {parent}")
return
logging.warning(
f"Couldn't find TP order {tp_id} in parent {parent}, closing trade",
color="red")
await self.close_trades(parent["tag"],
parent["sym"].replace("USDT", ""))
return
idx = targets.index(tp_id)
await self.results_handler(
Trade.target(parent["tag"],
parent["sym"],
parent["ent"],
parent["qty"],
parent["lev"],
parent["tgt"][idx],
parent["t_q"][idx],
is_long=parent["side"] == "BUY"))
if tp_id == targets[-1]:
logging.info(f"All TP orders hit. Removing parent {parent}")
parent = self.state["orders"].pop(tp["parent"])
for oid in parent["t_ord"]:
self.state["orders"].pop(oid, None) # It might not exist
self.state["orders"].pop(parent["s_ord"])
await self._cancel_order(parent["s_ord"], parent["sym"])
return
else:
new_price, quantity = parent["ent"], sum(parent["t_q"][(idx +
1):])
await self._place_sl_order(tp["parent"], new_price, quantity)
async def _place_sl_order(self,
parent_id: str,
new_price=None,
quantity=None):
async with self.olock:
odata = self.state["orders"][parent_id]
symbol = odata["sym"]
sl_order_id = OrderID.stop_loss()
if odata.get("s_ord") is not None:
logging.info(
f"Moving SL order for {parent_id} to new price {new_price}"
)
await self._cancel_order(odata["s_ord"], symbol)
params = {
"symbol":
symbol,
"positionSide":
"LONG" if odata["side"] == "BUY" else "SHORT",
"side":
"SELL" if odata["side"] == "BUY" else "BUY",
"type":
OrderType.STOP_MARKET,
"newClientOrderId":
sl_order_id,
"stopPrice":
self._round_price(
symbol,
new_price if new_price is not None else odata["sl"]),
"quantity":
self._round_qty(
symbol,
(quantity if quantity is not None else odata["qty"])),
}
for _ in range(2):
try:
resp = await self.client.futures_create_order(**params)
odata["s_ord"] = sl_order_id
self.state["orders"][sl_order_id] = {
"parent": parent_id,
"filled": False,
}
logging.info(
f"Created SL order {sl_order_id} for parent {parent_id}, "
f"resp: {resp}, params: {json.dumps(params)}")
break
except Exception as err:
logging.error(
f"Failed to create SL order for parent {parent_id}: {err}, "
f"params: {json.dumps(params)}")
if isinstance(
err, BinanceAPIException
) and err.code == -2021: # price is around SL now
logging.info(
f"Placing market order for parent {parent_id} "
"after attempt to create SL order",
color="yellow")
params.pop("stopPrice")
params["type"] = OrderType.MARKET
async def _watch_orders(self):
async def _watcher():
while True:
try:
open_symbols = await self._expire_outdated_orders_and_get_open_symbols(
)
except Exception as err:
logging.exception(
f"Failed to expire outdated orders: {err}")
async with self.slock:
redundant = set(
self.state["streams"]).difference(open_symbols)
if redundant:
logging.warning(
f"Resetting price streams to {open_symbols}",
color="yellow")
self.state["streams"] = open_symbols
await self._subscribe_futures(resub=redundant)
for sym in redundant:
self.prices.pop(f"{sym}USDT", None)
now = time.time()
async with self.olock:
removed = []
for order_id, order in self.state["orders"].items():
if not OrderID.is_wait(order_id):
continue
if order.get("t_ord"):
continue
if now < (order["crt"] + WAIT_ORDER_EXPIRY):
continue
logging.warning(
f"Wait order {order_id} has expired. Removing...",
color="yellow")
removed.append(order_id)
await self._cancel_order(order_id, order["sym"])
for order_id in removed:
self.state["orders"].pop(order_id)
await asyncio.sleep(ORDER_WATCH_INTERVAL)
asyncio.ensure_future(_watcher())
async def _expire_outdated_orders_and_get_open_symbols(self):
open_symbols = []
open_orders, positions = {}, {}
sl_orders = []
async with self.olock:
resp = await self.client.futures_account()
for pos in resp["positions"]:
amount, side = float(pos["positionAmt"]), pos["positionSide"]
if side == "BOTH" or amount == 0:
continue
positions[pos["symbol"] + side] = amount
resp = await self.client.futures_get_open_orders()
for order in resp:
open_orders[order["clientOrderId"]] = order
logging.info(
f"Checking {len(open_orders)} orders for {len(positions)} positions: {positions}"
)
for oid, order in open_orders.items():
odata = self.state["orders"].get(oid)
if OrderID.is_market(oid) or OrderID.is_wait(oid):
open_symbols.append(order["symbol"][:-4])
elif odata and (OrderID.is_target(oid)
or OrderID.is_stop_loss(oid)):
odata["filled"] = False
removed = []
for oid, odata in list(self.state["orders"].items()):
if (OrderID.is_target(oid) or OrderID.is_stop_loss(oid)):
if self.state["orders"].get(odata["parent"]) is None:
logging.warning(
f"Order {oid} is now an orphan. Flagging for removal"
)
removed.append(oid)
if not (OrderID.is_wait(oid) or OrderID.is_market(oid)):
continue
if open_orders.get(oid) is not None: # only filled orders
continue
if (time.time() -
odata["crt"]) < NEW_ORDER_TIMEOUT: # must be old
continue
if odata.get("s_ord") is None: # must have stop loss order
continue
side = "LONG" if odata["side"] == "BUY" else "SHORT"
if not positions.get(odata["sym"] + side):
logging.warning(
f"Order {oid} missing in open positions. Flagging for removal"
)
removed.append(oid)
continue
children = [odata["s_ord"]] + odata["t_ord"]
for cid in children:
if open_orders.get(cid) is not None: # ignore open orders
continue
is_filled = False
try:
cdata = await self.client.futures_get_order(
symbol=odata["sym"], origClientOrderId=cid)
is_filled = cdata["status"] == "FILLED"
except Exception as err:
logging.warning(
f"Error fetching order {cid} for parent {odata}: {err}"
)
if is_filled:
self.state["orders"][cid] = {
"parent": oid,
"filled": True,
}
else:
logging.info(
f"Missing order {cid} detected for parent {oid}",
color="yellow")
self.state["orders"].pop(cid, None)
order_hit = []
for i in range(len(children)):
cdata = self.state["orders"].get(children[i])
order_hit.append(cdata["filled"] if cdata else False)
if cdata is not None:
continue
if i == 0:
sl_orders.append({"id": oid})
continue
tgt_id = await self._create_target_order(
oid, odata["sym"], odata["side"], odata["tgt"][i - 1],
odata["t_q"][i - 1])
if tgt_id is not None:
odata["t_ord"][i - 1] = tgt_id
self.state["orders"][tgt_id] = {
"parent": oid,
"filled": False,
}
if order_hit[0] or all(order_hit[1:]): # All TPs or SL hit
removed.append(oid)
for oid in removed:
logging.warning(f"Removing outdated order {oid}",
color="yellow")
parent = self.state["orders"].pop(oid, None)
if not (OrderID.is_wait(oid) or OrderID.is_market(oid)):
continue
if parent is None:
continue
sym = parent["sym"]
for cid in [parent["s_ord"]] + parent["t_ord"]:
logging.warning(f"Removing outdated child {cid}",
color="yellow")
c = self.state["orders"].pop(cid, None)
if c and not c["filled"]:
await self._cancel_order(cid, sym)
for o in sl_orders:
oid = o["id"]
for i, tid in enumerate(self.state["orders"][oid]["t_ord"]):
if self.state["orders"][tid]["filled"]:
o["tp"] = tid
break
for o in sl_orders:
if o.get("tp"):
await self._move_stop_loss(o["tp"])
else:
await self._place_sl_order(o["id"])
return open_symbols
async def _subscribe_futures(self, coin: str = None, resub=False):
async with self.slock:
num_streams = len(set(self.state["streams"]))
resub = resub or (self.price_streamer is None)
if coin is not None:
coin = coin.upper()
# We should have duplicates because it should be possible to long/short
# on top of an existing long/short.
self.state["streams"].append(coin)
if num_streams != len(set(self.state["streams"])):
resub = True
if resub and self.price_streamer is not None:
logging.info("Cancelling ongoing ws stream for resubscribing")
self.price_streamer.cancel()
symbols = set(self.state["streams"])
if not resub or not symbols:
return
async def _streamer():
subs = list(map(lambda s: s.lower() + "usdt@aggTrade", symbols))
logging.info(
f"Spawning listener for {len(symbols)} symbol(s): {symbols}",
color="magenta")
async with self.manager.futures_multiplex_socket(subs) as stream:
while True:
msg = await stream.recv()
if msg is None:
logging.warning("Received 'null' in price stream",
color="red")
continue
try:
symbol = msg["stream"].split("@")[0][:-4].upper()
self.prices[symbol.upper()] = float(msg["data"]["p"])
except Exception as err:
logging.error(
f"Failed to get price for {msg['stream']}: {err}")
self.price_streamer = asyncio.ensure_future(_streamer())
async def _cancel_order(self, oid: str, symbol: str):
try:
resp = await self.client.futures_cancel_order(
symbol=symbol, origClientOrderId=oid)
logging.info(f"Cancelled order {oid}: {resp}")
except Exception as err:
logging.error(f"Failed to cancel order {oid}: {err}")
async def _register_order_for_signal(self, signal: Signal):
async with self.olock:
self.sig_cache.expire()
# Same provider can't give signal for same symbol within 20 seconds
key = self._cache_key(signal)
if self.sig_cache.get(key) is not None:
return False
self.sig_cache[key] = ()
return True
async def _unregister_order(self, signal: Signal):
async with self.olock:
self.sig_cache.pop(self._cache_key(signal), None)
def _cache_key(self, signal: Signal):
return f"{signal.coin}_{signal.targets[0]}" # coin and first target for filter
# MARK: Rouding for min quantity and min price for symbols
def _round_price(self, symbol: str, price: float):
info = self.symbols[symbol]
for f in info["filters"]:
if f["filterType"] == "PRICE_FILTER":
return round(
price, int(round(math.log(1 / float(f["tickSize"]), 10),
0)))
return price
def _round_qty(self, symbol: str, qty: float):
info = self.symbols[symbol]
for f in info["filters"]:
if f["filterType"] == "LOT_SIZE":
return round(
qty, int(round(math.log(1 / float(f["minQty"]), 10), 0)))
return qty
class CustomThreatService(BaseController):
"""
Implements a custom threat service for Resilient.
The root path (/xxx/ below) is configurable.
The service provides the following URLs:
OPTIONS /<root_path>/<any_sub_path>
POST /<root_path>/<any_sub_path>
GET /<root_path>/<any_sub_path>/<id>
"""
# Arbitrary constant
namespace = UUID('18222d9c-adf0-409c-aa19-beb27130ba12')
def __init__(self, opts):
super(CustomThreatService, self).__init__(**_make_args(opts))
# Configurable options
self.options = opts.get(CONFIG_SECTION, {})
# Do we support "file-content" artifacts? Default is no.
# TODO add implementation support to parse the file content
self.support_upload_file = strtobool(
self.options.get(CONFIG_UPLOAD_FILE.key,
CONFIG_UPLOAD_FILE.default))
# Default time that this service will tell Resilient to retry
self.first_retry_secs = int(
self.options.get(CONFIG_FIRST_RETRY_SECS.key,
CONFIG_FIRST_RETRY_SECS.default)) or 5
self.later_retry_secs = int(
self.options.get(CONFIG_LATER_RETRY_SECS.key,
CONFIG_LATER_RETRY_SECS.default)) or 60
# Size of the request cache
self.cache_size = int(
self.options.get(CONFIG_CACHE_SIZE.key, CONFIG_CACHE_SIZE.default))
# TTL of the request cache (millis before we give up on a request lookup)
self.cache_ttl = int(
self.options.get(CONFIG_CACHE_TTL.key, CONFIG_CACHE_TTL.default))
# Limit to the number of queries we'll answer for unfinished searchers (count before giving up on them)
self.max_retries = int(
self.options.get(CONFIG_MAX_RETRIES.key,
CONFIG_MAX_RETRIES.default))
# IDs and their results are maintained in a cache so that we can set
# an upper bound on the number of in-progress and recent lookups.
self.cache = TTLCache(maxsize=self.cache_size, ttl=self.cache_ttl)
# Helper component does event dispatch work
self.async_helper = CustomThreatServiceHelper(self)
(self.helper_thread, self.bridge) = self.async_helper.start()
urls = ["{0}/{1}".format(self.channel, e) for e in self.events()]
LOG.info("Web handler for %s", ", ".join(urls))
self.auth_user = self.options.get(CONFIG_AUTH_USER.key,
CONFIG_AUTH_USER.default)
self.auth_password = self.options.get(CONFIG_AUTH_PASSWORD.key,
CONFIG_AUTH_USER.default)
# Web endpoints
@exposeWeb("OPTIONS")
def _options_request(self, event, *args, **kwargs):
"""
Options indicate to Resilient whether file upload is supported.
"""
LOG.info(event.args[0])
options = {"upload_file": bool(self.support_upload_file)}
return options
@exposeWeb("POST")
def _post_request(self, event, *args, **kwargs):
LOG.info(event.args[0])
if not self.check_authentication(event.args[0]):
LOG.error("Custom Threat Service Authentication Error")
response = event.args[1]
response.status = 500
return {"id": None, "hits": []}
result = self._handle_post_request(event, *args, **kwargs)
LOG.info("%s: %s", event.args[1].status, json.dumps(result))
return result
def _handle_post_request(self, event, *args, **kwargs):
"""
Responds to POST /cts/<anything>
The URL below /cts/ is specific to this threat service. For example,
/cts/one and /cts/two can be registered as two separate threat sources.
The string 'one' or 'two' becomes the channel that searcher events are dispatched on.
Request is a ThreatServiceArtifactDTO containing the artifact to be scanned
Response is a ResponseDTO containing the response, or 'please retry' (HTTP status 303).
"""
request = event.args[0]
response = event.args[1]
# The channels that searchers are listening for events
cts_channel = searcher_channel(*args)
value = request.body.getvalue()
if not value:
err = "Empty request"
LOG.warn(err)
return {"id": str(uuid4()), "hits": []}
# Resilient sends artifacts in two formats: multi-part MIME, or plain JSON.
# server may send either, even for cases where there is no file content,
# so check content-type and decode appropriately.
try:
if request.headers and "form-data" in request.headers.get(
"Content-Type", ""):
multipart_data = decoder.MultipartDecoder(
value, request.headers["Content-Type"])
body = json.loads(multipart_data.parts[0].text)
LOG.debug(body)
else:
body = json.loads(value.decode("utf-8"))
LOG.debug(body)
except (ValueError, NonMultipartContentTypeException) as e:
err = "Can't handle request: {}".format(e)
LOG.warn(err)
LOG.debug(value)
return {"id": str(uuid4()), "hits": []}
if not isinstance(body, dict):
# Valid JSON but not a valid request.
err = "Invalid request: {}".format(json.dumps(body))
LOG.warn(err)
return {"id": str(uuid4()), "hits": []}
# Generate a request ID, derived from the artifact being requested.
request_id = str(uuid5(self.namespace, json.dumps(body)))
artifact_type = body.get("type", "unknown")
artifact_value = body.get("value")
response_object = {"id": request_id, "hits": []}
cache_key = (cts_channel, request_id)
if artifact_type == "net.name" and artifact_value == "localhost":
# Hard-coded response to 'net.name' of 'localhost'
# because this is used in 'resutil threatservicetest'
# and we want to return an immediate (not async) response
return response_object
# If we already have a completed query for this key, return it immmediately
request_data = self.cache.get(cache_key)
if request_data and request_data.get("complete"):
response_object["hits"] = request_data.get("hits", [])
return response_object
response.status = 303
response_object["retry_secs"] = self.first_retry_secs
# Add the request to the cache, then notify searchers that there's a new request
self.cache.setdefault(cache_key, {
"id": request_id,
"artifact": body,
"hits": [],
"complete": False
})
evt = ThreatServiceLookupEvent(request_id=request_id,
name=artifact_type,
artifact=body,
channel=cts_channel)
self.async_helper.fire(evt, HELPER_CHANNEL)
return response_object
@exposeWeb("GET")
def _get_request(self, event, *args, **kwargs):
LOG.info(event.args[0])
if not self.check_authentication(event.args[0]):
LOG.error("Custom Threat Service Authentication Error")
response = event.args[1]
response.status = 500
return {"id": None, "hits": []}
result = self._handle_get_request(event, *args, **kwargs)
LOG.info("%s: %s", event.args[1].status, json.dumps(result))
return result
def _handle_get_request(self, event, *args, **kwargs):
"""
Responds to GET /cts/<anything>/<request-id>
The URL below /cts/ is specific to this threat service. For example,
/cts/one and /cts/two are considered two separate threat sources.
Response is a ResponseDTO containing the response, or 'please retry'
"""
LOG.info(event.args[0])
response = event.args[1]
request_id = None
if not args:
return {"id": request_id, "hits": []}
# The ID of the lookup request
request_id = args[-1]
# The channel that searchers are listening for events
cts_channel = searcher_channel(*args[:-1])
response_object = {"id": request_id, "hits": []}
cache_key = (cts_channel, request_id)
request_data = self.cache.get(cache_key)
if not request_data:
# There's no record of this request in our cache, return empty hits
response.status = 200
return response_object
response_object["hits"] = request_data["hits"]
if not request_data["complete"]:
# The searchers haven't finished yet, return partial hits if available
response.status = 303
response_object["retry_secs"] = self.later_retry_secs
# Update the counter, so we can detect "stale" failures
request_data["count"] = request_data.get("count", 0) + 1
if request_data["count"] > self.max_retries:
LOG.info("Exceeded max retries for {}".format(cache_key))
try:
self.cache.pop(cache_key)
except KeyError:
pass
response.status = 200
return response_object
return response_object
# Remove the result from cache
# self.cache.pop(cache_key)
return response_object
@handler(channel=LOOKUP_COMPLETE_CHANNEL)
def _lookup_complete(self, event, *args, **kwargs):
"""
A lookup event was completed
"""
if not isinstance(event.parent, ThreatServiceLookupEvent):
return
results = event.parent.value.getValue()
artifact = event.parent.artifact
cts_channel = event.parent.cts_channel
request_id = event.parent.request_id
LOG.info("Lookup complete: %s, %s", event.parent, results)
# Depending on how many components handled this lookup event,
# the results can be a single value (dict), or an array, or None,
# or an exception, or a tuple (type, exception, traceback)
hits = []
complete = True
if isinstance(results, list):
for result in results:
if result:
if isinstance(result,
(tuple, ThreatLookupIncompleteException)):
LOG.info("Retry later!")
complete = False
elif isinstance(result, (tuple, Exception)):
LOG.error("No hits due to exception")
else:
hits.append(result)
elif results:
if isinstance(results, (tuple, ThreatLookupIncompleteException)):
LOG.info("Retry later!")
complete = False
elif isinstance(results, (tuple, Exception)):
LOG.error("No hits due to exception")
else:
hits.append(results)
# Store the result and mark as complete (or not)
cache_key = (cts_channel, request_id)
self.cache[cache_key] = {
"id": request_id,
"artifact": artifact,
"hits": hits,
"complete": complete
}
def _get_authentication_headers(self, request):
"""[extract user/password info in http header: Authentication Basic into a list]"""
if request.headers and "Basic" in request.headers.get(
"Authorization", ""):
auth = request.headers.get("Authorization", "").split(' ')
user_password = base64.b64decode(auth[1])
return b_to_s(user_password).split(":")
return [None, None]
def _is_authenticated(self, user_password_list):
"""[check if a user/password pair matches values set in app.config]"""
return self.auth_user == user_password_list[
0] and self.auth_password == user_password_list[1]
def check_authentication(self, request):
"""[check if the headers contain user/password information and they match the settings in app.config]"""
return self._is_authenticated(
self._get_authentication_headers(request))
async def cut_stop(user_id: str, uid: str, offset: float) -> Optional[str]:
if res := check_sub(user_id, uid):
return res
key = task_key(user_id, uid)
if key not in cut_tasks:
return "不存在未结束的切片任务"
room_id = get_sub_info_by_uid(uid)["room_id"]
path = await check_task(room_id)
if isinstance(path, str):
return path
if str(path.absolute()) != str(cut_tasks[key].file_path.absolute()):
cut_tasks.pop(key)
return "录播文件路径发生变动，切片失败..."
time = await get_keyframe(room_id, offset)
if isinstance(time, str):
return time
task = cut_tasks[key]
if time <= task.start_time:
return "切片结束时间必须大于开始时间！"
if time - task.start_time <= 10:
return "切片时长必须大于10s！"
task.stop_time = time
task.start()
cut_tasks.pop(key)
class AdminView:
"""
i can use adminView to control how to access the database easily
"""
cache_ttl: int = 43200
client = ndb.Client(namespace="main", project=Config().PROJECT)
mem_cache = TTLCache(maxsize=2048, ttl=cache_ttl)
def __init__(self, config):
super(AdminView, self).__init__()
self.cache_ttl = config.CACHE_TTL
self.mem_cache = TTLCache(maxsize=config.CACHE_SIZE,
ttl=config.CACHE_TTL)
def get_cache(self, key: str) -> any:
"""
:param key:
:return:
"""
try:
return self.mem_cache.pop(key)
except KeyError as e:
pass
with self.client.context():
cache_values: list = StoreCache().query(
StoreCache.cache_key == key).fetch()
if len(cache_values) > 0:
cache = cache_values[0]
if timestamp() - cache.last_accessed > self.cache_ttl:
return None
return cache.response
return None
def set_cache(self, key: str, response: dict) -> bool:
self.mem_cache[key] = response
with self.client.context():
cache_values: list = StoreCache().query(
StoreCache.cache_key == key).fetch()
if len(cache_values) > 0:
cache = cache_values[0]
else:
cache = StoreCache()
cache.cache_key = key
cache.response = response
cache.last_accessed = timestamp()
cache.put()
return True
def is_shutdown(self) -> bool:
with self.client.context():
api_settings_list: list = SettingsAPI.query().fetch()
if len(api_settings_list) > 0:
api_settings = api_settings_list[0]
else:
api_settings = SettingsAPI()
return api_settings.user_shutdown
def update_property_types(self, property_selections: list) -> tuple:
selected_properties: list = []
for _, value in enumerate(property_selections):
selected_properties.append(
value) if property_selections[value] else ""
with self.client.context():
defaults_list: list = DefaultAPIQueries.query().fetch()
if len(defaults_list) > 0:
default_api = defaults_list[0]
else:
default_api = DefaultAPIQueries()
default_api.set_property_types(property_types=selected_properties)
default_api.put()
return jsonify({
'status': 'success',
'message': "property types successfully updated"
}), 200
def update_dates_selected(self, dates_selected: list) -> tuple:
selected_properties: list = []
for _, value in enumerate(dates_selected):
selected_properties.append(value) if dates_selected[value] else ""
with self.client.context():
defaults_list: list = DefaultAPIQueries.query().fetch()
if len(defaults_list) > 0:
default_api = defaults_list[0]
else:
default_api = DefaultAPIQueries()
default_api.set_construction_dates(
construction_dates=selected_properties)
default_api.put()
return jsonify({
'status': 'success',
'message': "construction dates successfully updated"
}), 200
def update_finish_quality(self, finish_quality: list) -> tuple:
with self.client.context():
defaults_list: list = DefaultAPIQueries.query().fetch()
if len(defaults_list) > 0:
default_api = defaults_list[0]
else:
default_api = DefaultAPIQueries()
default_api.set_finish_quality(finish_quality=[
finish_quality[value] for _, value in enumerate(finish_quality)
])
default_api.put()
return jsonify({
'status': 'success',
'message': "finish quality successfully updated"
}), 200
def fetch_all_admin_defaults(self) -> dict:
with self.client.context():
defaults_list: list = DefaultAPIQueries.query().fetch()
if len(defaults_list) > 0:
default_api = defaults_list[0]
else:
default_api = DefaultAPIQueries()
return {
'status': 'success',
'payload': default_api.to_dict(),
'message': 'successfully fetched default api values'
}
def fetch_property_types(self) -> dict:
with self.client.context():
defaults_list: list = DefaultAPIQueries.query().fetch()
if len(defaults_list) > 0:
default_api = defaults_list[0]
else:
default_api = DefaultAPIQueries()
return jsonify({
'status': 'success',
'payload': default_api.get_property_types(),
'message': 'successfully fetched property types'
})
def fetch_finishing_quality(self) -> dict:
with self.client.context():
defaults_list: list = DefaultAPIQueries.query().fetch()
if len(defaults_list) > 0:
default_api = defaults_list[0]
else:
default_api = DefaultAPIQueries()
return jsonify({
'status': 'success',
'payload': default_api.get_finish_quality(),
'message': 'successfully fetched property types'
})
def get_construction_dates(self) -> dict:
with self.client.context():
defaults_list: list = DefaultAPIQueries.query().fetch()
if len(defaults_list) > 0:
default_api = defaults_list[0]
else:
default_api = DefaultAPIQueries()
return jsonify({
'status': 'success',
'payload': default_api.get_construction_dates(),
'message': 'successfully fetched construction dates'
})
def set_shutdown_status(self, status: bool) -> dict:
with self.client.context():
settings_list: list = SettingsAPI.query().fetch()
if len(settings_list) > 0:
default_settings: SettingsAPI = settings_list[0]
else:
default_settings: SettingsAPI = SettingsAPI()
default_settings.set_shutdown_status(status=status)
default_settings.put()
if status:
message: str = 'API is Shutting down ...'
else:
message: str = 'API is Restarting ....'
return jsonify({
'status': 'success',
'payload': default_settings.to_dict(),
'message': message
})
def get_settings(self) -> dict:
with self.client.context():
settings_list: list = SettingsAPI.query().fetch()
if len(settings_list) > 0:
default_settings: SettingsAPI = settings_list[0]
else:
default_settings: SettingsAPI = SettingsAPI()
return jsonify({
'status': 'success',
'payload': default_settings.to_dict(),
'message': 'Successfully fetched api settings'
})
def add_successful_request(self) -> bool:
with self.client.context():
settings_list: list = SettingsAPI.query().fetch()
if len(settings_list) > 0:
default_settings: SettingsAPI = settings_list[0]
else:
default_settings: SettingsAPI = SettingsAPI()
default_settings.add_successful_request()
default_settings.put()
return True
def add_failed_request(self) -> bool:
with self.client.context():
settings_list: list = SettingsAPI.query().fetch()
if len(settings_list) > 0:
default_settings: SettingsAPI = settings_list[0]
else:
default_settings: SettingsAPI = SettingsAPI()
default_settings.add_error_request()
default_settings.put()
return True
def add_cached_request(self) -> bool:
with self.client.context():
settings_list: list = SettingsAPI.query().fetch()
if len(settings_list) > 0:
default_settings: SettingsAPI = settings_list[0]
else:
default_settings: SettingsAPI = SettingsAPI()
default_settings.add_cached_request()
default_settings.put()
return True
def save_notification_settings(self, search: str, postcode: str,
email: str) -> tuple:
with self.client.context():
notifications_list: typing.List[
NotificationsSettings] = NotificationsSettings.query(
NotificationsSettings.email == email).fetch()
if len(notifications_list) == 0:
try:
key = NotificationsSettings(email=email,
search=search,
postcode=postcode).put()
except TypeError:
return jsonify({
'status': 'failure',
'message': 'Database Error inform admin'
}), 500
except BadValueError:
return jsonify({
'status': 'failure',
'message': 'Database Error inform admin'
}), 500
if key is not None:
message: str = '''Successfully updated notifications, you will periodically receive emails with listed
properties suitable to your search criteria'''
return jsonify({
'status': 'success',
'message': message
}), 200
else:
message: str = '''Error saving notification settings please inform admin through chat'''
return jsonify({
'status': 'failure',
'message': message
}), 500
else:
message: str = '''Notifications Setting already saved'''
return jsonify({'status': 'failure', 'message': message}), 500
def add_notifications_user(self, email: str, name: str, uid: str) -> tuple:
with self.client.context():
email_list_instance: typing.List[
EmailListUsers] = EmailListUsers.query(
EmailListUsers.email == email).fetch()
if len(email_list_instance) > 0:
message: str = ''' Notification user already exist'''
return jsonify({'status': 'failure', 'message': message}), 500
else:
try:
key = EmailListUsers(email=email, name=name, uid=uid).put()
return jsonify({
'status':
'failure',
'message':
'Successfully created new mailing list user'
}), 200
except TypeError:
return jsonify({
'status': 'failure',
'message': 'Database Error inform admin'
}), 500
except BadValueError:
return jsonify({
'status': 'failure',
'message': 'Database Error inform admin'
}), 500
class Client:
def __init__(self, config):
self.auth_token = config["authtoken"]
self.zoho_org_id = config["zohoOrgId"]
try:
self.cache_enabled = config["cache_enabled"]
except KeyError:
self.cache_enabled = DEFAULT_CACHE_MODE
try:
self.cache_ttl = config["cache_ttl"]
except KeyError:
self.cache_ttl = DEFAULT_CACHE_TTL
self.requests = requests.Session()
self.cache = TTLCache(ttl=self.cache_ttl,
maxsize=DEFAULT_CACHE_MAXSIZE)
def add_to_cache(self, key, value):
if (self.cache_enabled is None) or (self.cache_enabled is False):
pass
else:
self.cache[key] = value
def get_from_cache(self, key):
if (self.cache_enabled is None) or (self.cache_enabled is False):
return None
else:
try:
return self.cache[key]
except KeyError:
return None
def delete_from_cache(self, key):
if (self.cache_enabled is None) or (self.cache_enabled is False):
return False
else:
try:
self.cache.pop(key=key)
return True
except KeyError:
return False
# my_key = ast.literal_eval(key)
# return self.cache.pop(key=key)
def get_request_headers(self, headers):
default_headers = {
ZOHO_AUTH_HEADER: ZOHO_AUTH_TOKEN_HEADER_PREFIX + self.auth_token,
ZOHO_ORG_ID_HEADER: self.zoho_org_id,
'Content-Type': "application/json"
}
if (headers is not None) and len(headers) > 0:
default_headers.update(headers)
return default_headers
def send_request(self, method, uri, data=None, headers=None):
try:
response = requests.request(
method,
ZOHO_SUBSCRIPTION_API_URL + uri,
data=json.dumps(data),
headers=self.get_request_headers(headers))
response.raise_for_status()
except HTTPError as http_err:
return http_err
except Exception as err:
return None
if response.headers[
'Content-Type'] == 'application/json;charset=UTF-8':
return json.loads(response.text)
else:
return response.content
class AutoTTLCache(MutableMapping):
def __init__(self,
items=None,
*,
maxsize,
ttl,
timer=time.monotonic,
getsizeof=None):
self._cache_lock = threading.Lock()
self._cache = TTLCache(maxsize, ttl, timer=timer, getsizeof=getsizeof)
if items is not None:
self._cache.update(items)
self._monitor = CacheMonitor(self)
@property
def ttl(self):
with self._cache_lock:
return self._cache.ttl
@property
def maxsize(self):
with self._cache_lock:
return self._cache.maxsize
@property
def timer(self):
with self._cache_lock:
return self._cache.timer
def expire(self):
with self._cache_lock:
self._cache.expire()
def __contains__(self, key):
with self._cache_lock:
return key in self._cache
def __setitem__(self, k, v):
with self._cache_lock:
self._cache[k] = v
def __delitem__(self, k):
with self._cache_lock:
del self._cache[k]
def __getitem__(self, k):
with self._cache_lock:
return self._cache[k]
def __len__(self) -> int:
with self._cache_lock:
return len(self._cache)
def __iter__(self):
with self._cache_lock:
keys = list(self._cache)
yield from keys
# TODO: __reduce__ and __setstate__
def __repr__(self):
return f"{type(self).__name__}(max_size={self.maxsize}, ttl={self.ttl})"
def clear(self):
with self._cache_lock:
self._cache.clear()
def get(self, *args, **kwargs):
with self._cache_lock:
self._cache.get(*args, **kwargs)
def pop(self, *args, **kwargs):
with self._cache_lock:
self._cache.pop(*args, **kwargs)
def setdefault(self, *args, **kwargs):
with self._cache_lock:
self._cache.setdefault(*args, **kwargs)
def popitem(self):
with self._cache_lock:
self._cache.popitem()
