def __lightning_fetch_balance(self):
balance = Dotdict()
if self.lightning_enabled:
res = self.lightning.inventories()
for k, v in res.items():
balance[k] = Dotdict(v)
return balance
def __restapi_fetch_balance(self):
balance = Dotdict()
if self.private_api_enabled:
res = self.exchange.private_get_getbalance()
for v in res:
balance[v['currency_code']] = Dotdict(v)
return balance
def create_rich_ohlcv(self, ohlcv):
if self.settings.disable_rich_ohlcv:
rich_ohlcv = Dotdict()
for k in ohlcv[0].keys():
rich_ohlcv[k] = [v[k] for v in ohlcv]
else:
rich_ohlcv = pd.DataFrame.from_records(ohlcv, index="created_at")
return rich_ohlcv
def __restapi_fetch_collateral(self):
collateral = Dotdict()
collateral.collateral = 0
collateral.open_position_pnl = 0
collateral.require_collateral = 0
collateral.keep_rate = 0
if self.private_api_enabled:
collateral = Dotdict(self.exchange.private_get_getcollateral())
# self.logger.info("COLLATERAL: {collateral} open {open_position_pnl} require {require_collateral:.2f} rate {keep_rate}".format(**collateral))
return collateral
def __init__(self, yourlogic):
# トレーディングロジック設定
self.yourlogic = yourlogic
# ポジションサイズ
self.position_size = 0.0
# 取引所情報
self.settings = Dotdict()
self.settings.exchange = self.cfg.get_env("EXCHANGE", default='bitflyer')
self.settings.symbol = self.cfg.get_env("SYMBOL", default='FX_BTC_JPY')
self.settings.topics = ['ticker', 'executions']
self.settings.apiKey = self.cfg.get_env("BITFLYER_KEY")
self.settings.secret = self.cfg.get_env("BITFLYER_SECRET_KEY")
# 取引所
self.exchange = None
# LightningAPI設定
self.settings.use_lightning = self.cfg.get_env("BITFLER_LIGHTNING_API", default=False, type=bool)
self.settings.lightning_userid = self.cfg.get_env("BITFLYER_LIGHTNING_USERID")
self.settings.lightning_password = self.cfg.get_env("BITFLYER_LIGHTNING_PASSWORD")
# 動作タイミング
self.settings.interval = int(self.cfg.get_env("INTERVAL", default=60))
self.settings.timeframe = self.cfg.get_env("TIMEFRAME", default=60)
# OHLCV生成オプション
self.settings.max_ohlcv_size = 1000
self.settings.use_lazy_ohlcv = False
self.settings.disable_create_ohlcv = False
self.settings.disable_rich_ohlcv = False
# その他
self.hft = False
self.settings.show_last_n_orders = 0
self.settings.safe_order = True
# リスク設定
self.risk = Dotdict()
self.risk.max_position_size = 1.0
self.risk.max_num_of_orders = 1
def get_open_order(self, myid):
my_open_orders = self.get_open_orders()
my_order = [v for v in my_open_orders.values() if v['myid'] == myid]
if len(my_order) == 1:
return Dotdict(my_order[0])
elif len(my_order) == 0:
raise OrderNotFoundException(
"MyOrderID:%s is not open. MyOpenOrders:%s" %
(myid, my_open_orders))
else:
raise OrderDuplicateFoundException("MyOrderID:%s is duplicated." %
myid)
def __init__(self, yourlogic):
# ログ設定
self.logger, self.test_handler = SystemLogger(yourlogic.__name__).get_logger()
self.logger.info("Initializing Strategy...")
# トレーディングロジック設定
self.yourlogic = yourlogic
# 取引所情報
self.settings = Dotdict()
self.settings.exchange = self.cfg.get_env("EXCHANGE", default='bitmex')
self.settings.symbol = self.cfg.get_env("SYMBOL", default='BTC/USD')
self.settings.use_websocket = self.cfg.get_env("USE_WEB_SOCKET", type=bool, default=True)
self.logger.info("USE_WEB_SOCKET: %s" % self.settings.use_websocket)
if self.cfg.env.is_real():
self.settings.apiKey = self.cfg.get_env("BITMEX_KEY")
self.settings.secret = self.cfg.get_env("BITMEX_SECRET_KEY")
else:
self.settings.apiKey = self.cfg.get_env("TEST_BITMEX_KEY")
self.settings.secret = self.cfg.get_env("TEST_BITMEX_SECRET_KEY")
self.settings.close_position_at_start_stop = False
# 動作タイミング
self.settings.interval = int(self.cfg.get_env("INTERVAL", default=60))
# ohlcv設定
self.settings.timeframe = self.cfg.get_env("TIMEFRAME", default='1m')
self.settings.partial = False
self.hoge = None
# リスク設定
self.risk = Dotdict()
self.risk.max_position_size = 1000
self.risk.max_drawdown = 5000
# ポジション情報
self.position = Dotdict()
self.position.currentQty = 0
# 資金情報
self.balance = None
# 注文情報
self.orders = Dotdict()
# ティッカー情報
self.ticker = Dotdict()
# ohlcv情報
self.ohlcv = None
self.ohlcv_updated = False
# 約定情報
self.executions = Dotdict()
# 取引所接続
self.exchange = Exchange(self.settings, apiKey=self.settings.apiKey, secret=self.settings.secret)
self.logger.info("Completed to initialize Strategy.")
def __restapi_create_order(self, myid, side, qty, limit, stop,
time_in_force, minute_to_expire, symbol):
# raise ccxt.ExchangeNotAvailable('sendchildorder {"status":-208,"error_message":"Order is not accepted"}')
qty = round(qty, 8) # 有効桁数8桁
order_type = 'market'
params = {}
if limit is not None:
order_type = 'limit'
limit = float(limit)
if time_in_force is not None:
params['time_in_force'] = time_in_force
if minute_to_expire is not None:
params['minute_to_expire'] = minute_to_expire
order = Dotdict(
self.exchange.create_order(symbol, order_type, side, qty, limit,
params))
order.myid = myid
order.accepted_at = datetime.utcnow()
order = self.om.add_order(order)
self.logger.info(
"NEW: {myid} {status} {side} {price} {filled}/{amount} {id}".
format(**order))
def __restapi_fetch_position(self, symbol):
#raise ccxt.ExchangeError("ConnectionResetError(104, 'Connection reset by peer')")
position = Dotdict()
position.currentQty = 0
position.avgCostPrice = 0
position.unrealisedPnl = 0
position.all = []
if self.private_api_enabled:
res = self.exchange.private_get_getpositions(
params={'product_code': self.exchange.market_id(symbol)})
position.all = res
for r in res:
size = r['size'] if r['side'] == 'BUY' else r['size'] * -1
cost = (position.avgCostPrice * abs(position.currentQty) +
r['price'] * abs(size))
position.currentQty = round(position.currentQty + size, 8)
position.avgCostPrice = int(cost / abs(position.currentQty))
position.unrealisedPnl = position.unrealisedPnl + r['pnl']
self.logger.info('{side} {price} {size} ({pnl})'.format(**r))
self.logger.info(
"POSITION: qty {currentQty} cost {avgCostPrice:.0f} pnl {unrealisedPnl}"
.format(**position))
return position
def edit_order(self, myid, side, qty, limit=None, stop=None, trailing_offset=None, symbol=None):
type = 'market'
params = {}
if stop is not None and limit is not None:
type = 'stopLimit'
params['stopPx'] = stop
# params['price'] = limit
elif stop is not None:
type = 'stop'
params['stopPx'] = stop
elif limit is not None:
type = 'limit'
# params['price'] = limit
if trailing_offset is not None:
params['pegOffsetValue'] = trailing_offset
symbol = symbol or self.settings.symbol
res = self.exchange.edit_order(myid, symbol, type, side, qty, limit, params)
self.logger.info("EDIT: {id} {order_type} {amount} {rate}({stop_loss_rate})".format(**res['info']))
return Dotdict(res)
def create_order(self, myid, side, qty, limit, stop, trailing_offset, symbol):
type = 'market'
params = {}
if stop is not None and limit is not None:
type = 'stopLimit'
params['stopPx'] = stop
params['execInst'] = 'LastPrice'
# params['price'] = limit
elif stop is not None:
type = 'stop'
params['stopPx'] = stop
params['execInst'] = 'LastPrice'
elif limit is not None:
type = 'limit'
# params['price'] = limit
if trailing_offset is not None:
params['pegPriceType'] = 'TrailingStopPeg'
params['pegOffsetValue'] = trailing_offset
symbol = symbol or self.settings.symbol
res = self.exchange.create_order(myid, symbol, type, side, qty, limit, params)
self.logger.info("ORDER: {id} {order_type} {amount} {rate}({stop_loss_rate})".format(**res['info']))
return Dotdict(res)
def test_dotdict(self):
d = Dotdict({"a": 1, 'b': 4})
self.assertEqual({'a': 1, 'b': 4}, d)
self.assertEqual(1, d.a)
self.assertEqual(4, d.b)
def __lightning_fetch_position_and_collateral(self, symbol):
position = Dotdict()
position.currentQty = 0
position.avgCostPrice = 0
position.unrealisedPnl = 0
collateral = Dotdict()
collateral.collateral = 0
collateral.open_position_pnl = 0
collateral.require_collateral = 0
collateral.keep_rate = 0
if self.lightning_enabled:
res = self.lightning.getmyCollateral(
product_code=self.exchange.market_id(symbol))
collateral.collateral = res['collateral']
collateral.open_position_pnl = res['open_position_pnl']
collateral.require_collateral = res['require_collateral']
collateral.keep_rate = res['keep_rate']
position.all = res['positions']
for r in position.all:
size = r['size'] if r['side'] == 'BUY' else r['size'] * -1
cost = (position.avgCostPrice * abs(position.currentQty) +
r['price'] * abs(size))
position.currentQty = round(position.currentQty + size, 8)
position.avgCostPrice = cost / abs(position.currentQty)
position.unrealisedPnl = position.unrealisedPnl + r['pnl']
self.logger.info('{side} {price} {size} ({pnl})'.format(**r))
self.logger.info(
"POSITION: qty {currentQty} cost {avgCostPrice:.0f} pnl {unrealisedPnl}"
.format(**position))
self.logger.info(
"COLLATERAL: {collateral} open {open_position_pnl} require {require_collateral:.2f} rate {keep_rate}"
.format(**collateral))
return position, collateral
def __lightning_create_order(self, myid, side, qty, limit, stop,
time_in_force, minute_to_expire, symbol):
# raise LightningError({'status':-208})
qty = round(qty, 8) # 有効桁数8桁
ord_type = 'MARKET'
if limit is not None:
ord_type = 'LIMIT'
limit = int(limit)
res = self.lightning.sendorder(self.exchange.market_id(symbol),
ord_type, side.upper(), limit, qty,
minute_to_expire, time_in_force)
order = Dotdict()
order.myid = myid
order.accepted_at = datetime.utcnow()
order.id = res['order_ref_id']
order.status = 'accepted'
order.symbol = symbol
order.type = ord_type.lower()
order.side = side
order.price = limit if limit is not None else 0
order.average_price = 0
order.cost = 0
order.amount = qty
order.filled = 0
order.remaining = 0
order.fee = 0
order = self.om.add_order(order)
self.logger.info(
"NEW: {myid} {status} {side} {price} {filled}/{amount} {id}".
format(**order))
def __restapi_fetch_board_state(self, symbol):
res = Dotdict(
self.exchange.public_get_getboardstate(
params={'product_code': self.exchange.market_id(symbol)}))
self.logger.info("health {health} state {state}".format(**res))
return res
def loop(self, ohlcv, ticker, board_state, strategy, **other):
out = Dotdict()
out.health = board_state.health
out.state = board_state.state
out.datetime = datetime.utcnow()
out.open = ohlcv.open[-1]
out.high = ohlcv.high[-1]
out.low = ohlcv.low[-1]
out.close = ohlcv.close[-1]
out.volume = ohlcv.volume[-1]
out.average = ohlcv.average[-1]
out.stdev = ohlcv.stdev[-1]
out.trades = ohlcv.trades[-1]
out.imbalance = ohlcv.imbalance[-1]
out.volume_imbalance = ohlcv.volume_imbalance[-1]
# out.opened_at = ohlcv.opened_at[-1]
# out.closed_at = ohlcv.closed_at[-1]
out.delay_seconds = ohlcv.delay_seconds[-1]
if self.once:
logger.info(','.join(str(v) for v in out.keys()))
self.once = False
logger.info(','.join(str(v) for v in out.values()))
class OrderManager(metaclass=Singleton):
logger, test_handler = SystemLogger(__name__).get_logger()
INVALID_ORDER = Dotdict({
'No': 0,
'myid': '__INVALID_ORDER__',
'id': '__INVALID_ORDER__',
'accepted_at': '1989-10-28T0:00:00.000',
'status': 'closed',
'symbol': 'FX_BTC_JPY',
'type': 'market',
'side': 'none',
'price': 0,
'average_price': 0,
'amount': 0,
'filled': 0,
'remaining': 0,
'fee': 0,
})
lock = threading.Lock()
number_of_orders = 0
orders = OrderedDict()
positions = deque()
def clear_for_test(self):
self.number_of_orders = 0
self.orders.clear()
def add_order(self, new_order):
with self.lock:
self.number_of_orders += 1
new_order['No'] = self.number_of_orders
self.orders[new_order['myid']] = new_order
return new_order
def add_position(self, p):
# 建玉追加
self.positions.append(p)
while len(self.positions) >= 2:
r = self.positions.pop()
l = self.positions.popleft()
if r['side'] == l['side']:
# 売買方向が同じなら取り出したポジションを戻す
self.positions.append(r)
self.positions.appendleft(l)
break
else:
if l['size'] >= r['size']:
# 決済
l['size'] = round(l['size'] - r['size'], 8)
if l['size'] > 0:
# サイズが残っている場合、ポジションを戻す
self.positions.appendleft(l)
else:
# 決済
r['size'] = round(r['size'] - l['size'], 8)
if r['size'] > 0:
# サイズが残っている場合、ポジションを戻す
self.positions.append(r)
def execute(self, o, e):
updated = False
with self.lock:
if o['filled'] < o['amount']:
# ポジション追加
self.add_position({
'side': o['side'],
'size': e['size'],
'price': e['price']
})
# 注文情報更新
last = o['filled'] * o['average_price']
curr = e['size'] * e['price']
filled = round(o['filled'] + e['size'], 8)
average_price = (last + curr) / filled
o['filled'] = filled
o['average_price'] = average_price
o['remaining'] = round(o['amount'] - filled, 8)
o['status'] = o['status']
if o['remaining'] <= 0:
o['status'] = 'closed'
else:
if o['status'] == 'accepted':
o['status'] = 'open'
updated = True
return updated
def open_or_cancel(self, o, size):
updated = False
with self.lock:
if o['status'] == 'cancel':
# 板サイズが注文サイズ未満ならキャンセル完了
if size < o['amount']:
o['status'] = 'canceled'
updated = True
elif o['status'] == 'accepted':
# 板サイズが注文サイズ以上ならオープン（他のユーザの注文と被る可能性があるが許容する）
if size >= o['amount']:
o['status'] = 'open'
updated = True
return updated
def expire(self, o):
with self.lock:
if o['status'] in ['cancel', 'open']:
o['status'] = 'canceled'
def overwrite(self, o, latest):
with self.lock:
# ローカルで更新した状態は残しておく
if latest['status'] == 'open':
if o['status'] in ['cancel', 'canceled', 'closed']:
latest['status'] = o['status']
if latest['filled'] < o['filled']:
latest['filled'] = o['filled']
for k in [
'id', 'accepted_at', 'status', 'average_price', 'filled',
'remaining', 'fee'
]:
o[k] = latest[k]
def cancel_order(self, myid):
cancelable = ['open', 'accepted']
with self.lock:
my_orders = [
v for v in self.orders.values()
if (v['type'] != 'market') and (v['myid'] == myid) and (
v['status'] in cancelable)
]
for o in my_orders:
o['status'] = 'cancel'
return my_orders
def cancel_order_all(self):
cancelable = ['open', 'accepted']
with self.lock:
my_orders = [
v for v in self.orders.values()
if (v['type'] != 'market') and (v['status'] in cancelable)
]
for o in my_orders:
o['status'] = 'cancel'
return my_orders
def get_order(self, myid):
with self.lock:
my_orders = [v for v in self.orders.values() if v['myid'] == myid]
if len(my_orders):
return my_orders[-1]
raise OrderNotFoundException("MyOrderID:%s is not found. MyOrders:%s" %
(myid, my_orders))
def get_open_order(self, myid):
my_open_orders = self.get_open_orders()
my_order = [v for v in my_open_orders.values() if v['myid'] == myid]
if len(my_order) == 1:
return Dotdict(my_order[0])
elif len(my_order) == 0:
raise OrderNotFoundException(
"MyOrderID:%s is not open. MyOpenOrders:%s" %
(myid, my_open_orders))
else:
raise OrderDuplicateFoundException("MyOrderID:%s is duplicated." %
myid)
def get_open_orders(self):
return self.get_orders(status_filter=['open', 'accepted', 'cancel'])
def get_orders(self, status_filter=None):
with self.lock:
if status_filter is None:
my_orders = self.orders.copy()
else:
my_orders = {
k: v
for k, v in self.orders.items()
if (v['status'] in status_filter)
}
return my_orders
def is_active(self, myid):
try:
self.get_open_order(myid)
except OrderNotFoundException as e:
return False
except Exception as e:
return False
return True
def update_order(self, o):
self.get_order(o['myid']).update(o)
return o
def cleaning_if_needed(self, limit_orders=200, remaining_orders=20):
"""注文情報整理"""
open_status = ['open', 'accepted', 'cancel']
with self.lock:
if len(self.orders) > limit_orders:
all_orders = list(self.orders.items())
# open/accepted状態の注文は残す
orders = [(k, v) for k, v in all_orders[:-remaining_orders]
if v['status'] in open_status]
# 最新n件の注文は残す
orders.extend(all_orders[-remaining_orders:])
self.orders = OrderedDict(orders)
def printall(self):
print('\n' + '\t'.join(self.INVALID_ORDER.keys()))
for v in self.orders.values():
print('\t'.join([str(v) for v in v.values()]))
def start(self):
self.logger.info("Start Trading")
self.setup()
def async_inverval(func, interval, parallels):
next_exec_time = 0
@wraps(func)
def wrapper(*args, **kargs):
nonlocal next_exec_time
f_result = None
t = time()
if t > next_exec_time:
next_exec_time = ((t//interval)+1)*interval
f_result = func(*args,**kargs)
if parallels is not None:
parallels.append(f_result)
return f_result
return wrapper
def async_result(f_result, last):
if f_result is not None and f_result.done():
try:
return None, f_result.result()
except Exception as e:
self.logger.warning(type(e).__name__ + ": {0}".format(e))
f_result = None
return f_result, last
async_requests = []
fetch_position = async_inverval(self.exchange.fetch_position, 30, async_requests)
check_order_status = async_inverval(self.exchange.check_order_status, 5, async_requests)
errorWait = 0
f_position = position = f_check = None
once = True
while True:
self.interval = self.settings.interval
try:
# 注文処理の完了待ち
self.exchange.wait_for_completion()
# 待ち時間
self.monitoring_ep.suspend(False)
if self.interval:
if not self.hft:
self.logger.info("Waiting...")
wait_sec = (-time() % self.interval) or self.interval
sleep(wait_sec)
self.monitoring_ep.suspend(True)
# 例外発生時の待ち
no_needs_err_wait = (errorWait == 0) or (errorWait < time())
# ポジション等の情報取得
if no_needs_err_wait:
f_position = f_position or fetch_position(self.settings.symbol)
f_check = f_check or check_order_status(show_last_n_orders=self.settings.show_last_n_orders)
# リクエスト完了を待つ
if not self.hft or once:
for f in concurrent.futures.as_completed(async_requests):
pass
once = False
async_requests.clear()
# 建玉取得
if self.settings.use_lightning:
f_position, res = async_result(f_position, (position, None))
position, _ = res
else:
f_position, position = async_result(f_position, position)
# 内部管理のポジション数をAPIで取得した値に更新
if 'checked' not in position:
self.exchange.restore_position(position.all)
position['checked'] = True
# 内部管理のポジション数取得
self.position_size, self.position_avg_price, self.openprofit, self.positions = self.exchange.get_position()
# 注文情報取得
f_check, _ = async_result(f_check, None)
# REST API状態取得
self.api_state, self.api_avg_responce_time = self.exchange.api_state()
if self.api_state is not 'normal':
self.logger.info("REST API: {0} ({1:.1f}ms)".format(self.api_state, self.api_avg_responce_time*1000))
# 価格データ取得
ticker, executions, ohlcv = Dotdict(self.ep.get_ticker()), None, None
# インターバルが0の場合、約定履歴の到着を待つ
if self.settings.interval==0:
self.ep.wait_any(['executions'], timeout=0.5)
# OHLCVを作成しない場合、約定履歴を渡す
if self.settings.disable_create_ohlcv:
executions = self.ep.get_executions()
else:
if self.settings.use_lazy_ohlcv:
ohlcv = self.create_rich_ohlcv(self.ep.get_lazy_ohlcv())
else:
ohlcv = self.create_rich_ohlcv(self.ep.get_boundary_ohlcv())
# 資金情報取得
balance = self.fetch_balance()
# 売買ロジック呼び出し
args = {
'strategy': self,
'ticker': ticker,
'ohlcv': ohlcv,
'position': position,
'balance': balance,
'executions': executions
}
if no_needs_err_wait:
self.yourlogic.bizlogic(self.yourlogic, **args)
errorWait = 0
else:
self.logger.info("Waiting for Error...")
except ccxt.DDoSProtection as e:
self.logger.warning(type(e).__name__ + ": {0}".format(e))
errorWait = time() + 60
except ccxt.RequestTimeout as e:
self.logger.warning(type(e).__name__ + ": {0}".format(e))
errorWait = time() + 30
except ccxt.ExchangeNotAvailable as e:
self.logger.warning(type(e).__name__ + ": {0}".format(e))
errorWait = time() + 5
except ccxt.AuthenticationError as e:
self.logger.warning(type(e).__name__ + ": {0}".format(e))
self.private_api_enabled = False
errorWait = time() + 5
except ccxt.ExchangeError as e:
self.logger.warning(type(e).__name__ + ": {0}".format(e))
errorWait = time() + 5
except (KeyboardInterrupt, SystemExit):
self.logger.info('Shutdown!')
break
except Exception as e:
self.logger.exception(e)
errorWait = time() + 1
self.logger.info("Stop Trading")
# 停止
self.running = False
# ストリーミング停止
self.streaming.stop()
# 取引所停止
self.exchange.stop()
def fetch_order_book(self, symbol):
"""板情報取得"""
return Dotdict(
self.exchange.public_get_getboard(
params={'product_code': self.exchange.market_id(symbol)}))
def make_ohlcv(self, executions):
price = [e['price'] for e in executions]
buy = [e for e in executions if e['side'] == 'BUY']
sell = [e for e in executions if e['side'] == 'SELL']
ohlcv = Dotdict()
ohlcv.open = price[0]
ohlcv.high = max(price)
ohlcv.low = min(price)
ohlcv.close = price[-1]
ohlcv.volume = sum(e['size'] for e in executions)
ohlcv.buy_volume = sum(e['size'] for e in buy)
ohlcv.sell_volume = sum(e['size'] for e in sell)
ohlcv.volume_imbalance = ohlcv.buy_volume - ohlcv.sell_volume
ohlcv.buy_count = len(buy)
ohlcv.sell_count = len(sell)
ohlcv.trades = ohlcv.buy_count + ohlcv.sell_count
ohlcv.imbalance = ohlcv.buy_count - ohlcv.sell_count
ohlcv.average = sum(price) / len(price)
ohlcv.average_sq = sum(p**2 for p in price) / len(price)
ohlcv.variance = ohlcv.average_sq - (ohlcv.average * ohlcv.average)
ohlcv.stdev = math.sqrt(ohlcv.variance)
ohlcv.vwap = sum(e['price']*e['size'] for e in executions) / ohlcv.volume if ohlcv.volume > 0 else price[-1]
ohlcv.created_at = datetime.utcnow()
ohlcv.closed_at = self.parse_exec_date(executions[-1]['exec_date'])
e = executions[-1]
if e['side']=='SELL':
ohlcv.market_order_delay = (ohlcv.closed_at-self.parse_order_ref_id(e['sell_child_order_acceptance_id'])).total_seconds()
elif e['side']=='BUY':
ohlcv.market_order_delay = (ohlcv.closed_at-self.parse_order_ref_id(e['buy_child_order_acceptance_id'])).total_seconds()
else:
ohlcv.market_order_delay = 0
ohlcv.distribution_delay = (ohlcv.created_at - ohlcv.closed_at).total_seconds()
return ohlcv