def __init__(self, debug=False):
super().__init__()
self.debug = debug
if self.debug:
from xross_common.SystemLogger import SystemLogger
self.logger, test_handler = SystemLogger(
"SystemContext").get_logger()
def __init__(self, yourlogic):
# ログ設定
self.logger, self.test_handler = SystemLogger(yourlogic.__name__).get_logger()
self.logger.info("Initializing Strategy...")
# トレーディングロジック設定
self.yourlogic = yourlogic
# 取引所情報
self.settings = Dotdict()
self.settings.exchange = self.cfg.get_env("EXCHANGE", default='bitmex')
self.settings.symbol = self.cfg.get_env("SYMBOL", default='BTC/USD')
self.settings.use_websocket = self.cfg.get_env("USE_WEB_SOCKET", type=bool, default=True)
self.logger.info("USE_WEB_SOCKET: %s" % self.settings.use_websocket)
if self.cfg.env.is_real():
self.settings.apiKey = self.cfg.get_env("BITMEX_KEY")
self.settings.secret = self.cfg.get_env("BITMEX_SECRET_KEY")
else:
self.settings.apiKey = self.cfg.get_env("TEST_BITMEX_KEY")
self.settings.secret = self.cfg.get_env("TEST_BITMEX_SECRET_KEY")
self.settings.close_position_at_start_stop = False
# 動作タイミング
self.settings.interval = int(self.cfg.get_env("INTERVAL", default=60))
# ohlcv設定
self.settings.timeframe = self.cfg.get_env("TIMEFRAME", default='1m')
self.settings.partial = False
self.hoge = None
# リスク設定
self.risk = Dotdict()
self.risk.max_position_size = 1000
self.risk.max_drawdown = 5000
# ポジション情報
self.position = Dotdict()
self.position.currentQty = 0
# 資金情報
self.balance = None
# 注文情報
self.orders = Dotdict()
# ティッカー情報
self.ticker = Dotdict()
# ohlcv情報
self.ohlcv = None
self.ohlcv_updated = False
# 約定情報
self.executions = Dotdict()
# 取引所接続
self.exchange = Exchange(self.settings, apiKey=self.settings.apiKey, secret=self.settings.secret)
self.logger.info("Completed to initialize Strategy.")
class TestSystemThrottle(XrossTestBase):
logger, test_handler = SystemLogger("TestSystemThrottle").get_logger()
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.sys_throttle = SystemThrottle()
def test_check(self):
# setup
self.sys_throttle.set_throttle(THROTTLE_KEY, 10, 1)
# check
for i in range(9):
self.assertEqual(
Right("Success").value,
self.sys_throttle.check(THROTTLE_KEY).value)
result = self.sys_throttle.check(THROTTLE_KEY)
self.assertEqual("SystemThrottle is detected. 10 times per 1 secs.",
str(result.value))
def test_check_raise_exception(self):
# setup
self.sys_throttle.set_throttle(THROTTLE_KEY, 10, 1)
# check
try:
for i in range(10):
self.sys_throttle.check_raise_exception(THROTTLE_KEY)
self.fail()
except SystemThrottleError as e:
self.assertEqual(
"SystemThrottle is detected. 10 times per 1 secs.", str(e))
class ReloadableJsondict(Dotdict):
logger, test_handler = SystemLogger("ReloadableJsondict").get_logger()
def __init__(self, jsonfile, default_value={}):
super().__init__()
self.reloaded = False
self.mtime = 0
self.path = os.path.normpath(os.getcwd() + jsonfile)
self.update(default_value)
self.reload()
def reload(self):
try:
mtime = os.path.getmtime(self.path)
if mtime > self.mtime:
json_dict = json.load(open(self.path, 'r'),
object_hook=Dotdict)
self.update(json_dict)
self.mtime = mtime
self.reloaded = True
except Exception as e:
self.logger.warning(
type(e).__name__ + ": %s where %s" % (e, self.path))
raise e
return self
class SystemThrottle(metaclass=Singleton):
logger, test_handler = SystemLogger("SystemThrottle").get_logger()
timemgr = None
throttle = {}
counter = ActorCounter().start()
def __init__(self, timemgr=None):
super().__init__()
self.timemgr = timemgr
if not self.timemgr:
from datetime import datetime
def __get_now(self):
if not self.timemgr:
return datetime.utcnow().timestamp()
else:
return self.timemgr.get_systemtimestamp().to_unix_timestamp()
def check(self, key: str) -> "Either":
self.counter.ask({
'action': 'put',
'key': key,
'now': self.__get_now()
})
l = self.counter.ask({'action': 'list', 'key': key})
throttle = self.throttle[key]
if len(l) < throttle.limit:
return Right("Success")
elif l[-throttle.limit] + throttle.interval < self.__get_now():
return Right("Success")
else:
msg = "SystemThrottle is detected. %s times per %s secs." % (
throttle.limit, throttle.interval)
self.logger.warning(msg)
return Left(msg)
def check_raise_exception(self, key: str) -> None:
result = self.check(key)
if isinstance(result, Left):
raise SystemThrottleError(result.value)
def check_and_wait(self, key: str) -> None:
self.check(key)
time.sleep(l[-throttle.limit] + throttle.interval - self.__get_now())
def set_throttle(self, key: str, limit: int, interval: int) -> None:
self.throttle.update({key: Throttle(limit, interval)})
self.counter.ask({'action': 'initialize', 'key': key, 'limit': limit})
def get_throttle(self, key: str) -> int:
return self.throttle[key]
def clear_throttle_for_test(self, key=None) -> None:
if not key:
self.throttle.clear()
else:
self.throttle[key].clear()
class SystemContext(Dotdict):
logger, test_handler = None, None
debug = False
def __init__(self, debug=False):
super().__init__()
self.debug = debug
if self.debug:
from xross_common.SystemLogger import SystemLogger
self.logger, test_handler = SystemLogger(
"SystemContext").get_logger()
def set(self, dic: dict):
for k, v in dic.items():
setattr(self, str(k).upper(), v)
if self.debug:
self.logger.debug("set:%s" % self)
def __get(self, key: str, default: object = None):
if self.debug:
self.logger.debug("get:%s" % self)
item = getattr(self, conv(key))
return item if item else default
def has(self, key: str) -> bool:
return key.upper() in self.keys()
def get_int(self, key: str, default: int = 0) -> int:
if not str(self.__get(key, default=0)).isdecimal():
raise TypeError("Value:%s (Key:%s) is not decimal" %
(self.__get(key), key))
return int(self.__get(key, default=default))
def get_str(self, key: str, default: str = "") -> str:
return self.__get(key, default)
def increment(self, key: str, delta: int = 1) -> int:
new_val = int(self.get_int(key)) + delta
self.set({conv(key): new_val})
return new_val
def __repr__(self):
return "SystemContext%s" % super().__repr__()
class TestXrossTestBase(XrossTestBase):
logger, test_handler = SystemLogger("TestTestBase").get_logger()
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def test_assertRegexList_different_length(self):
try:
self.assertRegexList([], ["hoge"])
self.fail()
except Exception as e:
self.assertEqual("Length of lists doesn't match. 0!=1", str(e))
class TestSystemTimestamp(XrossTestBase):
logger, test_handler = SystemLogger("TestSystemTimestamp").get_logger()
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def test_default(self):
ts = SystemTimestamp()
print(datetime.now(JST).astimezone(tz=timezone.utc))
print(ts.to_datetime())
self.assertAlmostEqual(datetime.now(JST).astimezone(tz=timezone.utc), ts.to_datetime(), delta=timedelta(0, 1))
def test_from_datetime(self):
dt = datetime(2018, 2, 11, 8, 38, 0, 0, tzinfo=timezone.utc)
ts = SystemTimestamp(2018, 2, 11, 8, 38, 0, 0)
self.assertEqual(ts, SystemTimestamp.from_datetime(dt))
def test_to_datetime(self):
ts = SystemTimestamp(2018, 2, 11, 8, 38, 0, 0)
self.assertEqual(datetime(2018, 2, 11, 8, 38, 0, 0, tzinfo=timezone.utc), ts.to_datetime())
def test_to_string(self):
ts = SystemTimestamp(2018, 2, 11, 8, 38, 0, 0)
self.assertEqual("2018-02-11 08:38:00.000000", ts.to_string())
def test_to_string_iso(self):
ts = SystemTimestamp(2018, 2, 11, 8, 38, 0, 0)
self.assertEqual("2018-02-11T08:38:00.000000Z", ts.to_string_iso8601())
def test_from_string(self):
self.assertEqual(SystemTimestamp(2018, 2, 11, 8, 38, 0, 0),
SystemTimestamp.from_string("2018-02-11 08:38:00.000000"))
def test_from_string_iso(self):
self.assertEqual(SystemTimestamp(2018, 2, 11, 8, 38, 0, 0),
SystemTimestamp.from_string("2018-02-11T08:38:00.000000Z", format=FORMAT_ISO8601))
def test_to_unix_timestamp(self):
ts = SystemTimestamp(2018, 2, 11, 8, 38, 0, 0)
self.assertEqual(1518338280.0, ts.to_unix_timestamp())
def test_to_string_tz(self):
expiry_date = SystemTimestamp(2015, 1, 30, 15, 0, 0, 0).to_datetime()
self.assertEqual(datetime(2015, 1, 30, 15, 0, 0, 0, tzinfo=timezone.utc), expiry_date)
def test_custom_log_set_logger_level(self):
# set up
os.environ["LOGGER_LEVEL"] = "TRACE"
logger, test_handler = SystemLogger("TestSystemLogger").get_logger()
logger.trace("TRACE log is available.")
self.assertEqual(
['Loaded SystemLogger LOGGER_LEVEL:TRACE', 'TRACE log is available.'],
test_handler.formatted
)
logger.verbose("VERBOSE log is available.")
self.assertEqual(
['Loaded SystemLogger LOGGER_LEVEL:TRACE',
'TRACE log is available.',
'VERBOSE log is available.'],
test_handler.formatted
)
# teardown
os.environ["LOGGER_LEVEL"] = "DEBUG"
class Exchange:
env = SystemEnv.create()
logger, test_handler = SystemLogger(__name__).get_logger()
def __init__(self, settings, apiKey='', secret=''):
self.settings = settings
self.apiKey = apiKey
self.secret = secret
# 注文管理
self.om = None
# 取引所情報
self.exchange = None
# ステータス
self.running = False
# WebSocket
# self.settings.use_websocket = False
self.ws = None
# TODO: safe_api_caller
self.trades = None
self.__last_timestamp_recent_trades = datetime(year=1970,
month=1,
day=1)
def start(self):
self.running = True
# 取引所セットアップ
self.exchange = getattr(ccxt, self.settings.exchange)({
'apiKey':
self.settings.apiKey,
'secret':
self.settings.secret,
})
self.exchange.nonce = ccxt.Exchange.milliseconds
self.logger.info('Start Exchange')
self.exchange.load_markets()
# マーケット一覧表示
for k, v in self.exchange.markets.items():
self.logger.info('Markets: ' + v['symbol'])
# マーケット情報表示
market = self.exchange.market(self.settings.symbol)
# self.logger.debug(market)
self.logger.info('{symbol}: base:{base}'.format(**market))
self.logger.info('{symbol}: quote:{quote}'.format(**market))
# self.logger.info('{symbol}: active:{active}'.format(**market))
self.logger.info('{symbol}: taker:{taker}'.format(**market))
self.logger.info('{symbol}: maker:{maker}'.format(**market))
# self.logger.info('{symbol}: type:{type}'.format(**market))
self.om = OrderManager()
def __safe_api_recent_trades(self, symbol, **params):
if datetime.now() - self.__last_timestamp_recent_trades > timedelta(
seconds=self.settings.interval):
self.trades = self.exchange.fetch_trades(symbol, **params)
self.__last_timestamp_recent_trades = datetime.fromtimestamp(
int(self.trades[0]['id']))
return self.trades
def __safe_api_recent_trades_ws(self, **params):
if datetime.now() - self.__last_timestamp_recent_trades > timedelta(
seconds=self.settings.interval):
self.trades = self.ws.recent_trades(**params)
self.__last_timestamp_recent_trades = datetime.strptime(
self.trades[0]['timestamp'], DATETIME_FORMAT)
return self.trades
def fetch_my_executions(self, symbol, orders, **params):
trades = self.__safe_api_recent_trades(symbol, **params)
order_ids = [v['id'] for myid, v in orders.items()]
trade_ids = [trade['id'] for trade in trades]
return [order_id for order_id in order_ids if order_id in trade_ids]
def fetch_my_executions_ws(self, orders, **params):
trades = self.__safe_api_recent_trades_ws(**params)
order_ids = {{v['id']: myid} for myid, v in orders.items()}
trade_ids = {trade['id'] for trade in trades}
return [order_id for order_id in order_ids if order_id in trade_ids]
def fetch_ticker(self, symbol=None, timeframe=None):
symbol = symbol or self.settings.symbol
timeframe = timeframe or self.settings.timeframe
book = self.exchange.fetch_order_book(symbol, limit=10)
trade = self.__safe_api_recent_trades(symbol,
limit=1,
params={"reverse": True})
ticker = Dotdict()
ticker.bid, ticker.bidsize = book['bids'][0]
ticker.ask, ticker.asksize = book['asks'][0]
ticker.bids = book['bids']
ticker.asks = book['asks']
ticker.last = trade[0]['price']
ticker.datetime = pd.to_datetime(trade[0]['datetime'],
utc=True).tz_convert('Asia/Tokyo')
self.logger.info(
"TICK: bid {bid} ask {ask} last {last}".format(**ticker))
self.logger.info(
"TRD: price {rate} size {amount} side {order_type} ".format(
**(trade[0]['info'])))
return ticker, trade
def fetch_ticker_ws(self):
trade = self.__safe_api_recent_trades_ws()[-1]
ticker = Dotdict(self.ws.get_ticker())
ticker.datetime = pd.to_datetime(trade['timestamp'])
self.logger.info(
"TICK: bid {bid} ask {ask} last {last}".format(**ticker))
self.logger.info(
"TRD: price {price} size {size} side {side} tick {tickDirection} ".
format(**(trade['info'])))
return ticker, trade
def fetch_ohlcv(self, symbol=None, timeframe=None):
"""過去100件のOHLCVを取得"""
symbol = symbol or self.settings.symbol
timeframe = timeframe or self.settings.timeframe
partial = 'true' if self.settings.partial else 'false'
rsinf = RESAMPLE_INFO[timeframe]
market = self.exchange.market(symbol)
# req = {
# 'symbol': market['id'],
# 'binSize': rsinf['binSize'],
# 'count': rsinf['count'],
# 'partial': partial, # True == include yet-incomplete current bins
# 'reverse': 'false',
# 'startTime': datetime.utcnow() - (rsinf['delta'] * rsinf['count']),
# }
res = self.exchange.fetchOHLCV(symbol=symbol,
timeframe=timeframe,
since=None,
limit=rsinf['count'],
params={'reverse': True})
res2 = res.copy()
for i in range(len(res)):
res2[i][0] = datetime.fromtimestamp(int(res[i][0]) / 1000.0)
del res
keys = ['timestamp', 'open', 'high', 'low', 'close', 'volume']
dict_res = [dict(zip(keys, r)) for r in res2]
df = pd.DataFrame(dict_res).set_index('timestamp')
self.logger.info("OHLCV: {open} {high} {low} {close} {volume}".format(
**df.iloc[-1]))
return df
def fetch_position(self, symbol=None):
"""現在のポジションを取得"""
# symbol = symbol or self.settings.symbol
# res = self.exchange.fetchMyTrades(symbol=symbol)
# self.logger.info("POSITION: qty {currentQty} cost {avgCostPrice} pnl {unrealisedPnl}({unrealisedPnlPcnt100:.2f}%) {realisedPnl}".format(**pos))
# TODO: please implement someday
return None
def fetch_position_ws(self):
pos = Dotdict(self.ws.position())
pos.unrealisedPnlPcnt100 = pos.unrealisedPnlPcnt * 100
self.logger.info(
"POSITION: qty {currentQty} cost {avgCostPrice} pnl {unrealisedPnl}({unrealisedPnlPcnt100:.2f}%) {realisedPnl}"
.format(**pos))
return pos
def fetch_balance(self):
"""資産情報取得"""
balance = Dotdict(self.exchange.fetch_balance())
balance.BTC = Dotdict(balance.BTC)
self.logger.info(
"BALANCE: free {free:.3f} used {used:.3f} total {total:.3f}".
format(**balance.BTC))
return balance
def fetch_balance_ws(self):
balance = Dotdict(self.ws.funds())
balance.BTC = Dotdict()
balance.BTC.free = balance.availableMargin * 0.00000001
balance.BTC.total = balance.marginBalance * 0.00000001
balance.BTC.used = balance.BTC.total - balance.BTC.free
self.logger.info(
"BALANCE: free {free:.3f} used {used:.3f} total {total:.3f}".
format(**balance.BTC))
return balance
def fetch_order(self, order_id):
order = Dotdict({'status': 'closed', 'id': order_id})
try:
order = Dotdict(self.exchange.fetch_open_orders())
except ccxt.OrderNotFound as e:
self.logger.exception(type(e).__name__ + ": {0}".format(e))
return order
def fetch_order_ws(self, order_id):
orders = self.ws.all_orders()
for o in orders:
if o['id'] == order_id:
order = Dotdict(self.exchange.parse_order(o))
order.info = Dotdict(order.info)
return order
return Dotdict({'status': 'closed', 'id': order_id})
def create_order(self, myid, symbol, type, side, qty, price, params):
self.logger.info(
"Requesting to create a new order. symbol:%s, type:%s, side:%s, qty:%s, price:%s"
% (symbol, type, side, qty, price))
try:
result = Right(
self.exchange.create_order(symbol,
type,
side,
qty,
price=price,
params=params))
order = Dotdict()
order.myid = myid
order.accepted_at = datetime.utcnow().strftime(DATETIME_FORMAT)
order.id = result.value['info']['id']
order.status = 'accepted'
order.symbol = symbol
order.type = type.lower()
order.side = side
order.price = price if price is not None else 0
order.average_price = 0
order.cost = 0
order.amount = qty
order.filled = 0
order.remaining = 0
order.fee = 0
self.om.add_order(order)
except ccxt.BadRequest as e:
self.logger.warning("Returned BadRequest: %s" % str(e))
result = Left("Returned BadRequest: %s" % str(e))
except Exception as e:
self.logger.warning("Returned Exception: %s" % str(e))
result = Left(str(e))
return result.value
def edit_order(self, myid, symbol, type, side, qty, price, params):
self.logger.info(
"Requesting to amend the order. myid:%s, symbol:%s, type:%s, side:%s, qty:%s, price:%s"
% (myid, symbol, type, side, qty, price))
try:
active_order = self.om.get_open_order(myid)
if active_order is None:
self.logger.warning("No ActiveOrder exists.")
return
result = Right(
self.exchange.edit_order(active_order.id,
symbol,
type,
side,
amount=qty,
price=price,
params=params))
active_order.myid = myid
active_order.accepted_at = datetime.utcnow().strftime(
DATETIME_FORMAT)
active_order.id = result.value['info']['id']
active_order.status = 'accepted'
# order.symbol = symbol
active_order.type = type.lower()
# order.side = side
active_order.price = price if price is not None else 0
active_order.average_price = 0
active_order.cost = 0
active_order.amount = qty
active_order.filled = 0 # FIXME
active_order.remaining = 0 # FIXME
active_order.fee = 0
self.om.add_order(active_order)
except ccxt.BadRequest as e:
self.logger.warning("Returned BadRequest: %s" % str(e))
result = Left("Returned BadRequest: %s" % str(e))
self.om.cancel_order(myid)
except Exception as e:
self.logger.warning("Returned Exception: %s" % str(e))
result = Left(str(e))
return result.value
@excahge_error
def close_position(self, symbol=None):
"""現在のポジションを閉じる"""
symbol = symbol or self.settings.symbol
market = self.exchange.market(symbol)
req = {'symbol': market['id']}
res = self.exchange.privatePostOrderClosePosition(req)
self.logger.info(
"CLOSE: {id} {order_type} {amount} {rate}".format(**res))
def cancel_order(self, myid):
"""注文をキャンセル"""
open_orders = self.om.get_open_orders()
if myid in open_orders:
try:
details = open_orders[myid]
self.logger.info(
"Requesting to cancel the order. myid:%s, id:%s, symbol:%s, type:%s, side:%s, qty:%s, price:%s"
% (myid, details.id, details.symbol, details.type,
details.side, details.qty, details.price))
res = self.exchange.cancel_order(details.id)
self.logger.info(
"CANCEL: {id} {order_type} {amount} {rate}".format(
**res['info']))
except ccxt.OrderNotFound as e:
self.logger.warning(type(e).__name__ + ": {0}".format(e))
except Exception as e:
self.logger.warning(type(e).__name__ + ": {0}".format(e))
# del open_orders[myid]
self.om.cancel_order(myid)
@excahge_error
def cancel_order_all(self, symbol=None):
"""現在の注文をキャンセル"""
symbol = symbol or self.settings.symbol
res = self.exchange.fetch_open_orders(symbol=symbol)
for r in res:
self.logger.info(
"CANCEL: {id} {order_type} {amount} {rate}".format(**r))
def reconnect_websocket(self):
# 再接続が必要がチェック
need_reconnect = False
if self.ws is None:
need_reconnect = True
else:
if self.ws.connected == False:
self.ws.exit()
need_reconnect = True
# 再接続
if need_reconnect:
market = self.exchange.market(self.settings.symbol)
# ストリーミング設定
# if self.env.is_real():
# self.ws = BitMEXWebsocket(
# endpoint='wss://www.bitmex.com',
# symbol=market['id'],
# api_key=self.settings.apiKey,
# api_secret=self.settings.secret
# )
# else:
# self.ws = BitMEXWebsocket(
# endpoint='wss://testnet.bitmex.com/realtime',
# symbol=market['id'],
# api_key=self.settings.apiKey,
# api_secret=self.settings.secret
# )
# ネットワーク負荷の高いトピックの配信を停止
self.ws.unsubscribe(['orderBookL2'])
class TestSystemLogger(XrossTestBase):
logger, test_handler = SystemLogger("TestSystemLogger").get_logger()
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def tearDown(self):
self.test_handler.flush()
def test_log_once(self):
# action
self.logger.info("hoge")
# assert
self.assertEqual(["hoge"], self.test_handler.formatted)
def test_log_twice(self):
# action
self.logger.info("hoge1")
self.logger.info("hoge2")
# assert
self.assertEqual(["hoge1", "hoge2"], self.test_handler.formatted)
def test_log_multi_process(self):
# action
self.logger.info("AlgoProcess is starting.")
algo_proc = multiprocessing.Process(target=self.logger.info("hoge"), name="AlgoProcess")
algo_proc.start()
algo_proc.join()
self.logger.info("AlgoProcess is stopped.")
# assert
self.assertEqual(
['AlgoProcess is starting.',
'hoge',
'AlgoProcess is stopped.'],
self.test_handler.formatted)
def test_custom_log(self):
self.logger.trace("TRACE log is available.")
self.assertEqual(
['Loaded SystemLogger LOGGER_LEVEL:DEBUG'],
self.test_handler.formatted
)
self.logger.verbose("VERBOSE log is available.")
self.assertEqual(
['Loaded SystemLogger LOGGER_LEVEL:DEBUG', 'VERBOSE log is available.'],
self.test_handler.formatted
)
def test_custom_log_set_logger_level(self):
# set up
os.environ["LOGGER_LEVEL"] = "TRACE"
logger, test_handler = SystemLogger("TestSystemLogger").get_logger()
logger.trace("TRACE log is available.")
self.assertEqual(
['Loaded SystemLogger LOGGER_LEVEL:TRACE', 'TRACE log is available.'],
test_handler.formatted
)
logger.verbose("VERBOSE log is available.")
self.assertEqual(
['Loaded SystemLogger LOGGER_LEVEL:TRACE',
'TRACE log is available.',
'VERBOSE log is available.'],
test_handler.formatted
)
# teardown
os.environ["LOGGER_LEVEL"] = "DEBUG"
class Exchange:
logger, test_handler = SystemLogger(__name__).get_logger()
def __init__(self, apiKey='', secret=''):
self.apiKey = apiKey
self.secret = secret
self.responce_times = deque(maxlen=3)
self.lightning_enabled = False
self.lightning_collateral = None
self.order_is_not_accepted = None
self.ltp = 0
self.last_position_size = 0
self.running = False
self.exchange = None
self.om = None
def safe_api_call(self, func):
@wraps(func)
def wrapper(*args, **kargs):
retry = 3
while retry > 0:
retry = retry - 1
try:
start = time()
result = func(*args, **kargs)
responce_time = (time() - start)
self.responce_times.append(responce_time)
return result
except ccxt.ExchangeError as e:
if (retry == 0) or ('Connection reset by peer'
not in e.args[0]):
raise e
sleep(0.3)
return wrapper
def api_state(self):
res_times = list(self.responce_times)
mean_time = sum(res_times) / len(res_times)
health = 'super busy'
if mean_time < 0.2:
health = 'normal'
elif mean_time < 0.5:
health = 'busy'
elif mean_time < 1.0:
health = 'very busy'
return health, mean_time
def start(self):
self.logger.info('Start Exchange')
self.running = True
# 取引所セットアップ
self.exchange = ccxt.bitflyer({
'apiKey': self.apiKey,
'secret': self.secret
})
self.exchange.urls['api'] = 'https://api.bitflyer.com'
self.exchange.timeout = 60 * 1000
# 応答時間計測用にラッパーをかぶせる
# self.wait_for_completion = stop_watch(self.wait_for_completion)
self.safe_create_order = self.safe_api_call(
self.__restapi_create_order)
self.safe_cancel_order = self.safe_api_call(
self.__restapi_cancel_order)
self.safe_cancel_order_all = self.safe_api_call(
self.__restapi_cancel_order_all)
self.safe_fetch_collateral = self.safe_api_call(
self.__restapi_fetch_collateral)
self.safe_fetch_position = self.safe_api_call(
self.__restapi_fetch_position)
self.safe_fetch_balance = self.safe_api_call(
self.__restapi_fetch_balance)
self.safe_fetch_orders = self.safe_api_call(
self.__restapi_fetch_orders)
self.safe_fetch_board_state = self.safe_api_call(
self.__restapi_fetch_board_state)
self.inter_check_order_status = self.__restapi_check_order_status
# プライベートAPI有効判定
self.private_api_enabled = len(self.apiKey) > 0 and len(
self.secret) > 0
# マーケット一覧表示
self.exchange.load_markets()
for k, v in self.exchange.markets.items():
self.logger.info('Markets: ' + v['symbol'])
# スレッドプール作成
self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=8)
# 並列注文処理完了待ち用リスト
self.parallel_orders = []
# 注文管理
self.om = OrderManager()
# Lightningログイン
if self.lightning_enabled:
self.lightning.login()
# LightningAPIに置き換える
self.safe_create_order = self.safe_api_call(
self.__lightning_create_order)
# self.safe_cancel_order = self.safe_api_call(self.__lightning_cancel_order)
self.safe_cancel_order_all = self.safe_api_call(
self.__lightning_cancel_order_all)
self.safe_fetch_position = self.safe_api_call(
self.__lightning_fetch_position_and_collateral)
self.safe_fetch_balance = self.safe_api_call(
self.__lightning_fetch_balance)
# self.safe_fetch_orders = self.safe_api_call(self.__lightning_fetch_orders)
# self.safe_fetch_board_state = self.safe_api_call(self.__lightning_fetch_board_state)
# self.inter_check_order_status = self.__lightning_check_order_status
def stop(self):
if self.running:
self.logger.info('Stop Exchange')
self.running = False
# すべてのワーカスレッド停止
self.executor.shutdown()
# Lightningログオフ
if self.lightning_enabled:
self.lightning.logoff()
def get_order(self, myid):
return self.om.get_order(myid)
def get_open_orders(self):
orders = self.om.get_orders(status_filter=['open', 'accepted'])
orders_by_myid = OrderedDict()
for o in orders.values():
orders_by_myid[o['myid']] = o
return orders_by_myid
def create_order(self, myid, side, qty, limit, stop, time_in_force,
minute_to_expire, symbol):
"""新規注文"""
if self.private_api_enabled:
self.parallel_orders.append(
self.executor.submit(self.safe_create_order, myid, side, qty,
limit, stop, time_in_force,
minute_to_expire, symbol))
def cancel(self, myid):
"""注文をキャンセルする"""
if self.private_api_enabled:
cancel_orders = self.om.cancel_order(myid)
for o in cancel_orders:
self.logger.info(
"CANCEL: {myid} {status} {side} {price} {filled}/{amount} {id}"
.format(**o))
self.parallel_orders.append(
self.executor.submit(self.safe_cancel_order,
order_id=o['id'],
symbol=o['symbol']))
def cancel_order_all(self, symbol):
"""すべての注文をキャンセルする"""
if self.private_api_enabled:
cancel_orders = self.om.cancel_order_all()
for o in cancel_orders:
self.logger.info(
"CANCEL: {myid} {status} {side} {price} {filled}/{amount} {id}"
.format(**o))
self.parallel_orders.append(
self.executor.submit(self.safe_cancel_order,
order_id=o['id'],
symbol=o['symbol']))
def __restapi_cancel_order_all(self, symbol):
self.exchange.private_post_cancelallchildorders(
params={'product_code': self.exchange.market_id(symbol)})
def __restapi_cancel_order(self, order_id, symbol):
self.exchange.cancel_order(order_id, symbol)
def __restapi_create_order(self, myid, side, qty, limit, stop,
time_in_force, minute_to_expire, symbol):
# raise ccxt.ExchangeNotAvailable('sendchildorder {"status":-208,"error_message":"Order is not accepted"}')
qty = round(qty, 8) # 有効桁数8桁
order_type = 'market'
params = {}
if limit is not None:
order_type = 'limit'
limit = float(limit)
if time_in_force is not None:
params['time_in_force'] = time_in_force
if minute_to_expire is not None:
params['minute_to_expire'] = minute_to_expire
order = Dotdict(
self.exchange.create_order(symbol, order_type, side, qty, limit,
params))
order.myid = myid
order.accepted_at = datetime.utcnow()
order = self.om.add_order(order)
self.logger.info(
"NEW: {myid} {status} {side} {price} {filled}/{amount} {id}".
format(**order))
def __restapi_fetch_position(self, symbol):
#raise ccxt.ExchangeError("ConnectionResetError(104, 'Connection reset by peer')")
position = Dotdict()
position.currentQty = 0
position.avgCostPrice = 0
position.unrealisedPnl = 0
position.all = []
if self.private_api_enabled:
res = self.exchange.private_get_getpositions(
params={'product_code': self.exchange.market_id(symbol)})
position.all = res
for r in res:
size = r['size'] if r['side'] == 'BUY' else r['size'] * -1
cost = (position.avgCostPrice * abs(position.currentQty) +
r['price'] * abs(size))
position.currentQty = round(position.currentQty + size, 8)
position.avgCostPrice = int(cost / abs(position.currentQty))
position.unrealisedPnl = position.unrealisedPnl + r['pnl']
self.logger.info('{side} {price} {size} ({pnl})'.format(**r))
self.logger.info(
"POSITION: qty {currentQty} cost {avgCostPrice:.0f} pnl {unrealisedPnl}"
.format(**position))
return position
def fetch_position(self, symbol, _async=True):
"""建玉一覧取得"""
if _async:
return self.executor.submit(self.safe_fetch_position, symbol)
return self.safe_fetch_position(symbol)
def __restapi_fetch_collateral(self):
collateral = Dotdict()
collateral.collateral = 0
collateral.open_position_pnl = 0
collateral.require_collateral = 0
collateral.keep_rate = 0
if self.private_api_enabled:
collateral = Dotdict(self.exchange.private_get_getcollateral())
# self.logger.info("COLLATERAL: {collateral} open {open_position_pnl} require {require_collateral:.2f} rate {keep_rate}".format(**collateral))
return collateral
def fetch_collateral(self, _async=True):
"""証拠金情報を取得"""
if _async:
return self.executor.submit(self.safe_fetch_collateral)
return self.safe_fetch_collateral()
def __restapi_fetch_balance(self):
balance = Dotdict()
if self.private_api_enabled:
res = self.exchange.private_get_getbalance()
for v in res:
balance[v['currency_code']] = Dotdict(v)
return balance
def fetch_balance(self, _async=True):
"""資産情報取得"""
if _async:
return self.executor.submit(self.safe_fetch_balance)
return self.safe_fetch_balance()
def fetch_open_orders(self, symbol, limit=100):
orders = []
if self.private_api_enabled:
orders = self.exchange.fetch_open_orders(symbol=symbol,
limit=limit)
# for order in orders:
# self.logger.info("{side} {price} {amount} {status} {id}".format(**order))
return orders
def __restapi_fetch_orders(self, symbol, limit):
orders = []
if self.private_api_enabled:
orders = self.exchange.fetch_orders(symbol=symbol, limit=limit)
# for order in orders:
# self.logger.info("{side} {price} {amount} {status} {id}".format(**order))
return orders
def fetch_orders(self, symbol, limit=100, _async=False):
if _async:
return self.executor.submit(self.safe_fetch_orders, symbol, limit)
return self.safe_fetch_orders(symbol, limit)
def fetch_order_book(self, symbol):
"""板情報取得"""
return Dotdict(
self.exchange.public_get_getboard(
params={'product_code': self.exchange.market_id(symbol)}))
def wait_for_completion(self):
# 新規注文拒否解除
if self.order_is_not_accepted:
past = datetime.utcnow() - self.order_is_not_accepted
if past > timedelta(seconds=3):
self.order_is_not_accepted = None
# 注文完了確認
for f in concurrent.futures.as_completed(self.parallel_orders):
try:
res = {}
try:
f.result()
except ccxt.ExchangeNotAvailable as e:
self.logger.warning(type(e).__name__ + ": {0}".format(e))
msg = e.args[0]
if '{' in msg:
res = json.loads(msg[msg.find('{'):])
except ccxt.DDoSProtection as e:
self.logger.warning(type(e).__name__ + ": {0}".format(e))
self.order_is_not_accepted = datetime.utcnow() + timedelta(
seconds=12)
except LightningError as e:
self.logger.warning(type(e).__name__ + ": {0}".format(e))
res = e.args[0]
# 注文を受付けることができませんでした.
if 'status' in res and res['status'] == -208:
self.order_is_not_accepted = datetime.utcnow()
except Exception as e:
self.logger.warning(type(e).__name__ + ": {0}".format(e))
self.parallel_orders = []
def get_position(self):
size = 0
avg = 0
pnl = 0
with self.om.lock:
positions = list(self.om.positions)
if len(positions):
size = fsum(p['size'] for p in positions)
avg = fsum(p['price'] * p['size'] for p in positions) / size
size = size if positions[0]['side'] == 'buy' else size * -1
pnl = (self.ltp * size) - (avg * size)
if self.last_position_size != size:
# self.logger.info("POSITION: qty {0:.8f} cost {1:.0f} pnl {2:.8f}".format(size, avg, pnl))
self.last_position_size = size
return size, avg, pnl, positions
def restore_position(self, positions):
with self.om.lock:
self.om.positions = deque()
for p in positions:
self.om.positions.append({
'side': p['side'].lower(),
'size': p['size'],
'price': p['price']
})
def order_exec(self, o, e):
if o is not None:
if self.om.execute(o, e):
self.logger.info(
"EXEC: {myid} {status} {side} {price} {filled}/{amount} {average_price} {id}"
.format(**o))
def check_order_execution(self, executions):
if len(executions):
self.ltp = executions[-1]['price']
my_orders = self.om.get_orders(
status_filter=['open', 'accepted', 'cancel', 'canceled'])
if len(my_orders):
for e in executions:
o = my_orders.get(e['buy_child_order_acceptance_id'], None)
self.order_exec(o, e)
o = my_orders.get(e['sell_child_order_acceptance_id'],
None)
self.order_exec(o, e)
def check_order_open_and_cancel(self, boards):
if len(boards):
my_orders = self.om.get_open_orders()
if len(my_orders):
for board in boards:
bids = {b['price']: b['size'] for b in board['bids']}
asks = {b['price']: b['size'] for b in board['asks']}
for o in my_orders.values():
size = None
if o['side'] == 'buy':
size = bids.get(o['price'], None)
elif o['side'] == 'sell':
size = asks.get(o['price'], None)
if size is not None:
if self.om.open_or_cancel(o, size):
self.logger.info(
"UPDATE: {myid} {status} {side} {price} {filled}/{amount} {average_price} {id}"
.format(**o))
def start_monitoring(self, endpoint):
def monitoring_main(ep):
self.logger.info('Start Monitoring')
while self.running and not ep.closed:
try:
ep.wait_any()
executions = ep.get_executions()
self.check_order_execution(executions)
boards = ep.get_boards()
self.check_order_open_and_cancel(boards)
except Exception as e:
self.logger.warning(type(e).__name__ + ": {0}".format(e))
self.logger.info('Stop Monitoring')
self.executor.submit(monitoring_main, endpoint)
def __restapi_check_order_status(self, show_last_n_orders=0):
my_orders = self.om.get_open_orders()
if len(my_orders):
# マーケット毎の注文一覧を取得する
symbols = list(set(v['symbol'] for v in my_orders.values()))
latest_orders = []
for symbol in symbols:
latest_orders.extend(
self.safe_fetch_orders(symbol=symbol, limit=50))
# 注文情報更新
for latest in latest_orders:
o = my_orders.get(latest['id'], None)
if o is not None:
# 最新の情報で上書き
self.om.overwrite(o, latest)
# self.logger.info("STATUS: {myid} {status} {side} {price} {filled}/{amount} {id}".format(**o))
del my_orders[latest['id']]
# 注文一覧から消えた注文はcanceledとする
for o in my_orders.values():
self.om.expire(o)
# 直近n個の注文を表示
if show_last_n_orders:
my_orders = list(self.om.get_orders().values())
if len(my_orders) > show_last_n_orders:
my_orders = my_orders[-show_last_n_orders:]
self.logger.info(
'No myid status side price amount filled average_price'
)
for o in my_orders:
self.logger.info(
'{No:<5} {myid:<8} {status:<8} {side:<4} {price:>8.0f} {amount:6.2f} {filled:6.2f} {average_price:>13.0f} {type} {symbol} {id}'
.format(**o))
# 注文情報整理
remaining_orders = max(show_last_n_orders, 30)
self.om.cleaning_if_needed(limit_orders=remaining_orders * 2,
remaining_orders=remaining_orders)
def check_order_status(self, show_last_n_orders=0, _async=True):
"""注文の状態を確認"""
if _async:
return self.executor.submit(self.inter_check_order_status,
show_last_n_orders)
self.inter_check_order_status(show_last_n_orders)
def __restapi_fetch_board_state(self, symbol):
res = Dotdict(
self.exchange.public_get_getboardstate(
params={'product_code': self.exchange.market_id(symbol)}))
self.logger.info("health {health} state {state}".format(**res))
return res
def fetch_board_state(self, symbol, _async=True):
"""板状態取得"""
if _async:
return self.executor.submit(self.safe_fetch_board_state, symbol)
return self.safe_fetch_board_state(symbol)
def enable_lightning_api(self, userid, password):
"""LightningAPIを有効にする"""
self.lightning = LightningAPI(userid, password)
self.lightning_enabled = True
def __lightning_create_order(self, myid, side, qty, limit, stop,
time_in_force, minute_to_expire, symbol):
# raise LightningError({'status':-208})
qty = round(qty, 8) # 有効桁数8桁
ord_type = 'MARKET'
if limit is not None:
ord_type = 'LIMIT'
limit = int(limit)
res = self.lightning.sendorder(self.exchange.market_id(symbol),
ord_type, side.upper(), limit, qty,
minute_to_expire, time_in_force)
order = Dotdict()
order.myid = myid
order.accepted_at = datetime.utcnow()
order.id = res['order_ref_id']
order.status = 'accepted'
order.symbol = symbol
order.type = ord_type.lower()
order.side = side
order.price = limit if limit is not None else 0
order.average_price = 0
order.cost = 0
order.amount = qty
order.filled = 0
order.remaining = 0
order.fee = 0
order = self.om.add_order(order)
self.logger.info(
"NEW: {myid} {status} {side} {price} {filled}/{amount} {id}".
format(**order))
def __lightning_cancel_order(self, order_id, symbol):
self.lightning.cancelorder(
product_code=self.exchange.market_id(symbol), order_id=order_id)
def __lightning_cancel_order_all(self, symbol):
self.lightning.cancelallorder(
product_code=self.exchange.market_id(symbol))
def __lightning_fetch_position_and_collateral(self, symbol):
position = Dotdict()
position.currentQty = 0
position.avgCostPrice = 0
position.unrealisedPnl = 0
collateral = Dotdict()
collateral.collateral = 0
collateral.open_position_pnl = 0
collateral.require_collateral = 0
collateral.keep_rate = 0
if self.lightning_enabled:
res = self.lightning.getmyCollateral(
product_code=self.exchange.market_id(symbol))
collateral.collateral = res['collateral']
collateral.open_position_pnl = res['open_position_pnl']
collateral.require_collateral = res['require_collateral']
collateral.keep_rate = res['keep_rate']
position.all = res['positions']
for r in position.all:
size = r['size'] if r['side'] == 'BUY' else r['size'] * -1
cost = (position.avgCostPrice * abs(position.currentQty) +
r['price'] * abs(size))
position.currentQty = round(position.currentQty + size, 8)
position.avgCostPrice = cost / abs(position.currentQty)
position.unrealisedPnl = position.unrealisedPnl + r['pnl']
self.logger.info('{side} {price} {size} ({pnl})'.format(**r))
self.logger.info(
"POSITION: qty {currentQty} cost {avgCostPrice:.0f} pnl {unrealisedPnl}"
.format(**position))
self.logger.info(
"COLLATERAL: {collateral} open {open_position_pnl} require {require_collateral:.2f} rate {keep_rate}"
.format(**collateral))
return position, collateral
def __lightning_fetch_balance(self):
balance = Dotdict()
if self.lightning_enabled:
res = self.lightning.inventories()
for k, v in res.items():
balance[k] = Dotdict(v)
return balance
def __lightning_check_order_status(self, show_last_n_orders=0):
pass
class TestSystemUtil(XrossTestBase):
logger, test_handler = SystemLogger("TestSystemUtil").get_logger()
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.cfg = SystemUtil()
def test_skip_load_ini(self):
self.cfg = SystemUtil()
self.assertEqual("SystemContext{}", str(self.cfg.get_all_sysprop()))
def test_show_sysprop(self):
self.cfg.clear_gcfg_for_test()
self.assertEqual("SystemContext{}", str(self.cfg.get_all_sysprop()))
def test_set_sysprop(self):
# setup
self.sysprop = self.cfg.get_all_sysprop()
# action
self.cfg.set_sysprop(TEST_KEY_PARAM, TEST_VALUE_PARAM)
# assert
self.assertEqual(TEST_VALUE_PARAM, self.sysprop.get(TEST_KEY_PARAM))
def test_get_sysprop(self):
# setup
self.test_set_sysprop()
# action/assert
self.assertEqual(TEST_VALUE_PARAM,
self.cfg.get_sysprop(TEST_KEY_PARAM))
def test_set_get_sysprop_dict(self):
# setup
TEST_KEY_PARAM_DICT = "FRASHCRASHER_1.DELTA_PRICE"
TEST_VALUE_PARAM_DICT = "{'MCO/BTC':0.00002,'BNB/BTC':0.000001}"
sysprop = self.cfg.get_all_sysprop()
# action
self.cfg.set_sysprop(TEST_KEY_PARAM_DICT, TEST_VALUE_PARAM_DICT)
# assert
self.assertEqual(TEST_VALUE_PARAM_DICT,
sysprop.get(TEST_KEY_PARAM_DICT))
# action/assert
self.assertEqual({
'MCO/BTC': 0.00002,
'BNB/BTC': 0.000001
}, self.cfg.get_sysprop(TEST_KEY_PARAM_DICT, type=dict))
def test_set_get_sysprop_pack(self):
# setup
TEST_KEY_PARAM_PACK = "FRASHCRASHER_1.INTERVALS"
TEST_VALUE_PARAM_PACK = "5,1"
sysprop = self.cfg.get_all_sysprop()
# action
self.cfg.set_sysprop(TEST_KEY_PARAM_PACK, TEST_VALUE_PARAM_PACK)
# assert
self.assertEqual(TEST_VALUE_PARAM_PACK,
sysprop.get(TEST_KEY_PARAM_PACK))
# action/assert
self.assertEqual(['5', '1'],
self.cfg.get_sysprop(TEST_KEY_PARAM_PACK, type=dict))
def test_set_get_sysprop_list(self):
# setup
TEST_KEY_PARAM_LIST = "FRASHCRASHER_1.INTERVALS"
TEST_VALUE_PARAM_LIST = "[5,1]"
sysprop = self.cfg.get_all_sysprop()
# action
self.cfg.set_sysprop(TEST_KEY_PARAM_LIST, TEST_VALUE_PARAM_LIST)
# assert
self.assertEqual(TEST_VALUE_PARAM_LIST,
sysprop.get(TEST_KEY_PARAM_LIST))
# action/assert
self.assertEqual(['5', '1'],
self.cfg.get_sysprop(TEST_KEY_PARAM_LIST, type=list))
def test_remove_sysprop(self):
self.test_get_sysprop()
# action/teardown
self.cfg.remove_sysprop(TEST_KEY_PARAM)
# teardown
expected_none = self.sysprop.get(TEST_KEY_PARAM)
self.assertEqual(None, expected_none)
def test_set_env(self):
# action
self.cfg.set_env(TEST_KEY_PARAM, TEST_VALUE_PARAM)
# assert
self.assertEqual(TEST_VALUE_PARAM, os.environ.get(TEST_KEY_PARAM))
def test_get_env(self):
# setup
self.test_set_env()
# action/assert
self.assertEqual(TEST_VALUE_PARAM, self.cfg.get_env(TEST_KEY_PARAM))
def test_get_envs_prefixed(self):
# setup
self.test_set_env()
# action/assert
self.assertEqual({TEST_KEY_PARAM: TEST_VALUE_PARAM},
self.cfg.get_envs_prefixed("TEST"))
def test_set_get_env_dict(self):
# setup
TEST_KEY_PARAM_DICT = "FRASHCRASHER_1.DELTA_PRICE"
TEST_VALUE_PARAM_DICT = "{'MCO/BTC':0.00002,'BNB/BTC':0.000001}"
self.cfg.set_env(TEST_KEY_PARAM_DICT, TEST_VALUE_PARAM_DICT)
# action/assert
self.assertEqual({
'MCO/BTC': 0.00002,
'BNB/BTC': 0.000001
}, self.cfg.get_env(TEST_KEY_PARAM_DICT, type=dict))
def test_get_env_from_setenvsh(self):
# assert
self.assertFalse(None, self.cfg.get_env("DOCKER_DIST_DIR"))
self.assertFalse("", self.cfg.get_env("DOCKER_DIST_DIR"))
def test_remove_env(self):
# setup
self.test_get_env()
# action/teardown
self.cfg.remove_env(TEST_KEY_PARAM)
# assert
self.assertEqual(None, os.environ.get(TEST_KEY_PARAM))
def test_read_config(self):
# setup
self.cfg.set_sysprop('TEST_READ_CONFIG', "True")
self.cfg.set_sysprop('TEST_READ_CONFIG_NUM', "1.0")
# assert
self.assertEqual("True",
self.cfg.get_sysprop_or_env('TEST_READ_CONFIG'))
self.assertTrue(
self.cfg.get_sysprop_or_env('TEST_READ_CONFIG', type=bool))
self.assertEqual(
1.0, self.cfg.get_sysprop_or_env('TEST_READ_CONFIG_NUM',
type=float))
try:
self.assertEqual(
1, self.cfg.get_sysprop_or_env('TEST_READ_CONFIG_NUM',
type=int))
except ValueError as e:
self.assertEqual("invalid literal for int() with base 10: '1.0'",
str(e))
# MEMO: confirming extra picture with the bottom code enabled
def test_sys_args_and_env(self):
print("=======ENVIRONMENT VARIABLES========")
# print(self.cfg.get_all_env_for_test()) # not to show SECRET_KEY in log
print("=======SYSPROP========")
print(sys.argv[1:])
print("config.ini and sys.argv[1:] contains " +
str(len(self.cfg.get_all_sysprop())))
def test_system_env(self):
self.logger.info("SystemEnv is " + str(self.cfg.env))
self.logger.info("IS_LOCAL : " + str(self.cfg.env.is_local()))
self.logger.info("IS_DOCKER : " + str(self.cfg.env.is_docker()))
self.logger.info("IS_UNITTEST : " + str(self.cfg.env.is_unittest()))
self.assertFalse(self.cfg.env.is_real())
self.assertNotEqual(SystemEnv.UNKNOWN, self.cfg.env)
def test_get_env_written_nowhere(self):
self.assertIsNone(self.cfg.get_env("WRITTEN_NOWHERE"))
self.assertFalse(
self.cfg.get_env("WRITTEN_NOWHERE", default=False, type=bool))
self.assertFalse(
self.cfg.get_env("WRITTEN_NOWHERE", default="False", type=bool))
class Streaming:
logger, test_handler = SystemLogger("Streaming").get_logger()
def __init__(self):
self.ws = None
self.running = False
self.subscribed_channels = []
self.endpoints = []
self.connected = False
# self.on_message = stop_watch(self.on_message)
def on_message(self, message):
message = json.loads(message)
if message["method"] == "channelMessage":
channel = message["params"]["channel"]
message = message["params"]["message"]
for ep in self.endpoints:
ep.put(channel, message)
def on_error(self, error):
self.logger.info(error)
def on_close(self):
self.logger.info('disconnected')
self.connected = False
def on_open(self):
self.logger.info('connected')
self.connected = True
if len(self.subscribed_channels):
for channel in self.subscribed_channels:
self.ws.send(json.dumps({'method': 'subscribe', 'params': {'channel': channel}}))
def get_endpoint(self, product_id='FX_BTC_JPY', topics=['ticker', 'executions'], timeframe=60, max_ohlcv_size=100):
ep = Streaming.Endpoint(product_id, topics, self.logger, timeframe, max_ohlcv_size)
self.endpoints.append(ep)
for channel in ep.channels:
if channel not in self.subscribed_channels:
if self.connected:
self.ws.send(json.dumps({'method': 'subscribe', 'params': {'channel': channel}}))
self.subscribed_channels.append(channel)
return ep
def ws_run_loop(self):
while self.running:
try:
self.ws = websocket.WebSocketApp("wss://ws.lightstream.bitflyer.com/json-rpc",
on_message=self.on_message,
on_error=self.on_error,
on_close=self.on_close)
self.ws.on_open = self.on_open
self.ws.run_forever()
except Exception as e:
self.logger.exception(e)
if self.running:
sleep(5)
def start(self):
self.logger.info('Start Streaming')
self.running = True
self.thread = threading.Thread(target=self.ws_run_loop)
self.thread.start()
def stop(self):
if self.running:
self.logger.info('Stop Streaming')
self.running = False
self.ws.close()
self.thread.join()
for ep in self.endpoints:
ep.shutdown()
class Endpoint:
def __init__(self, product_id, topics, logger, timeframe, max_ohlcv_size):
self.logger = logger
self.product_id = product_id.replace('/','_')
self.topics = topics
self.cond = threading.Condition()
self.channels = ['lightning_' + t + '_' + self.product_id for t in topics]
self.data = {}
self.last = {}
self.closed = False
self.suspend_count = 0
for channel in self.channels:
self.data[channel] = deque(maxlen=10000)
self.last[channel] = None
# self.get_data = stop_watch(self.get_data)
# self.put = stop_watch(self.put)
# self.parse_exec_date = stop_watch(self.parse_exec_date)
# self.make_ohlcv = stop_watch(self.make_ohlcv)
# self.create_ohlcv = stop_watch(self.create_ohlcv)
# self.get_lazy_ohlcv = stop_watch(self.get_lazy_ohlcv)
# self.get_boundary_ohlcv = stop_watch(self.get_boundary_ohlcv)
self.timeframe = timeframe
self.ohlcv = deque(maxlen=max_ohlcv_size)
self.remain_executions = []
self.lst_timeframe = datetime.utcnow().timestamp() // timeframe
def put(self, channel, message):
if channel in self.data:
with self.cond:
self.data[channel].append(message)
self.last[channel] = message
self.cond.notify_all()
def suspend(self, flag):
with self.cond:
if flag:
self.suspend_count += 1
else:
self.suspend_count = max(self.suspend_count-1, 0)
self.cond.notify_all()
def wait_for(self, topics = None):
topics = topics or self.topics
for topic in topics:
channel = 'lightning_' + topic + '_' + self.product_id
while True:
data = self.data[channel]
if len(data) or self.closed:
break
else:
self.logger.info('Waiting for stream data...')
sleep(1)
def wait_any(self, topics = None, timeout = None):
topics = topics or self.topics
channels = ['lightning_' + t + '_' + self.product_id for t in topics]
result = True
with self.cond:
while True:
available = 0
if self.suspend_count == 0:
for channel in channels:
available = available + len(self.data[channel])
if available or self.closed:
break
else:
if self.cond.wait(timeout) == False:
result = False
break
return result
def shutdown(self):
with self.cond:
self.closed = True
self.cond.notify_all()
def get_data(self, topic, blocking, timeout):
channel = 'lightning_' + topic + '_' + self.product_id
if channel in self.data:
with self.cond:
if blocking:
while True:
if len(self.data[channel]) or self.closed:
break
else:
if self.cond.wait(timeout) == False:
break
data = list(self.data[channel])
last = self.last[channel]
self.data[channel].clear()
return data, last
return [], None
def get_last(self, topic):
channel = 'lightning_' + topic + '_' + self.product_id
return self.last[channel] if channel in self.data else None
def get_ticker(self, blocking = False, timeout = None):
data, last = self.get_data('ticker', blocking, timeout)
return last
def get_tickers(self, blocking = False, timeout = None):
data, last = self.get_data('ticker', blocking, timeout)
return data
def get_executions(self, blocking = False, timeout = None):
data, last = self.get_data('executions', blocking, timeout)
return list(chain.from_iterable(data))
def get_board_snapshot(self, blocking = False, timeout = None):
data, last = self.get_data('board_snapshot', blocking, timeout)
return last
def get_boards(self, blocking = False, timeout = None):
data, last = self.get_data('board', blocking, timeout)
return data
@staticmethod
def parse_exec_date(exec_date):
exec_date = exec_date.rstrip('Z')+'0000000'
return datetime(
int(exec_date[0:4]),
int(exec_date[5:7]),
int(exec_date[8:10]),
int(exec_date[11:13]),
int(exec_date[14:16]),
int(exec_date[17:19]),
int(exec_date[20:26]))
@staticmethod
def parse_order_ref_id(order_ref_id):
return datetime(
int(order_ref_id[3:7]),
int(order_ref_id[7:9]),
int(order_ref_id[9:11]),
int(order_ref_id[12:14]),
int(order_ref_id[14:16]),
int(order_ref_id[16:18]),
int(order_ref_id[19:]))
def get_lazy_ohlcv(self):
data, last = self.get_data('executions',False,None)
if len(self.remain_executions)>0:
self.ohlcv.pop()
if len(data)==0:
e = last[-1].copy()
e['size'] = 0
e['side'] = ''
data.append([e])
for dat in data:
closed_at = self.parse_exec_date(dat[-1]['exec_date'])
cur_timeframe = closed_at.timestamp() // self.timeframe
if cur_timeframe > self.lst_timeframe:
if len(self.remain_executions) > 0:
self.ohlcv.append(self.make_ohlcv(self.remain_executions))
self.remain_executions = []
self.lst_timeframe = cur_timeframe
self.remain_executions.extend(dat)
if len(self.remain_executions) > 0:
self.ohlcv.append(self.make_ohlcv(self.remain_executions))
return list(self.ohlcv)
def get_boundary_ohlcv(self):
data, last = self.get_data('executions',False,None)
executions = list(chain.from_iterable(data))
if len(executions)==0:
e = last[-1].copy()
e['size'] = 0
e['side'] = ''
executions.append(e)
self.ohlcv.append(self.make_ohlcv(executions))
return list(self.ohlcv)
def make_ohlcv(self, executions):
price = [e['price'] for e in executions]
buy = [e for e in executions if e['side'] == 'BUY']
sell = [e for e in executions if e['side'] == 'SELL']
ohlcv = Dotdict()
ohlcv.open = price[0]
ohlcv.high = max(price)
ohlcv.low = min(price)
ohlcv.close = price[-1]
ohlcv.volume = sum(e['size'] for e in executions)
ohlcv.buy_volume = sum(e['size'] for e in buy)
ohlcv.sell_volume = sum(e['size'] for e in sell)
ohlcv.volume_imbalance = ohlcv.buy_volume - ohlcv.sell_volume
ohlcv.buy_count = len(buy)
ohlcv.sell_count = len(sell)
ohlcv.trades = ohlcv.buy_count + ohlcv.sell_count
ohlcv.imbalance = ohlcv.buy_count - ohlcv.sell_count
ohlcv.average = sum(price) / len(price)
ohlcv.average_sq = sum(p**2 for p in price) / len(price)
ohlcv.variance = ohlcv.average_sq - (ohlcv.average * ohlcv.average)
ohlcv.stdev = math.sqrt(ohlcv.variance)
ohlcv.vwap = sum(e['price']*e['size'] for e in executions) / ohlcv.volume if ohlcv.volume > 0 else price[-1]
ohlcv.created_at = datetime.utcnow()
ohlcv.closed_at = self.parse_exec_date(executions[-1]['exec_date'])
e = executions[-1]
if e['side']=='SELL':
ohlcv.market_order_delay = (ohlcv.closed_at-self.parse_order_ref_id(e['sell_child_order_acceptance_id'])).total_seconds()
elif e['side']=='BUY':
ohlcv.market_order_delay = (ohlcv.closed_at-self.parse_order_ref_id(e['buy_child_order_acceptance_id'])).total_seconds()
else:
ohlcv.market_order_delay = 0
ohlcv.distribution_delay = (ohlcv.created_at - ohlcv.closed_at).total_seconds()
return ohlcv
class Strategy:
cfg = SystemUtil(skip=True)
logger, test_handler = SystemLogger(__name__).get_logger()
def __init__(self, yourlogic):
# トレーディングロジック設定
self.yourlogic = yourlogic
# ポジションサイズ
self.position_size = 0.0
# 取引所情報
self.settings = Dotdict()
self.settings.exchange = self.cfg.get_env("EXCHANGE", default='bitflyer')
self.settings.symbol = self.cfg.get_env("SYMBOL", default='FX_BTC_JPY')
self.settings.topics = ['ticker', 'executions']
self.settings.apiKey = self.cfg.get_env("BITFLYER_KEY")
self.settings.secret = self.cfg.get_env("BITFLYER_SECRET_KEY")
# 取引所
self.exchange = None
# LightningAPI設定
self.settings.use_lightning = self.cfg.get_env("BITFLER_LIGHTNING_API", default=False, type=bool)
self.settings.lightning_userid = self.cfg.get_env("BITFLYER_LIGHTNING_USERID")
self.settings.lightning_password = self.cfg.get_env("BITFLYER_LIGHTNING_PASSWORD")
# 動作タイミング
self.settings.interval = int(self.cfg.get_env("INTERVAL", default=60))
self.settings.timeframe = self.cfg.get_env("TIMEFRAME", default=60)
# OHLCV生成オプション
self.settings.max_ohlcv_size = 1000
self.settings.use_lazy_ohlcv = False
self.settings.disable_create_ohlcv = False
self.settings.disable_rich_ohlcv = False
# その他
self.hft = False
self.settings.show_last_n_orders = 0
self.settings.safe_order = True
# リスク設定
self.risk = Dotdict()
self.risk.max_position_size = 1.0
self.risk.max_num_of_orders = 1
# ログ設定
# self.logger, self.test_handler = SystemLogger(__name__).get_logger()
# self.logger = logging.getLogger(__name__)
# self.create_rich_ohlcv = stop_watch(self.create_rich_ohlcv)
def fetch_order_book(self, symbol = None):
"""板情報取得"""
return self.exchange.fetch_order_book(symbol or self.settings.symbol)
def fetch_balance(self):
"""資産情報取得"""
return self.exchange.fetch_balance()
def fetch_collateral(self):
"""証拠金情報取得"""
return self.exchange.fetch_collateral()
def cancel(self, myid):
"""注文をキャンセル"""
# 注文情報取得
order = self.exchange.get_order(myid)
# 注文一覧にのるまでキャンセルは受け付けない
if (order.status == 'accepted') and self.settings.safe_order:
delta = datetime.utcnow() - order.accepted_at
if delta < timedelta(seconds=30):
if not self.hft:
self.logger.info("REJECT: %s order creating..." % myid)
return
self.exchange.cancel(myid)
def cancel_order_all(self, symbol = None):
"""すべての注文をキャンセル"""
self.exchange.cancel_order_all(symbol or self.settings.symbol)
def close_position(self, myid, symbol = None):
"""ポジションクローズ"""
if self.exchange.order_is_not_accepted is not None:
if not self.hft:
self.logger.info("REJECT: %s order is not accepted..." % myid)
return
# 最小注文サイズ取得
symbol = symbol or self.settings.symbol
if symbol == 'FX_BTC_JPY':
min_qty = 0.01
else:
min_qty = 0.001
buysize = sellsize = 0
# 買いポジあり
if self.position_size > 0:
sellsize = self.position_size
if sellsize < min_qty:
buysize = min_qty
sellsize = fsum([sellsize,min_qty])
# 売りポジあり
elif self.position_size < 0:
buysize = -self.position_size
if buysize < min_qty:
buysize = fsum([buysize,min_qty])
sellsize = min_qty
# 注文作成
close_orders = []
if sellsize:
close_orders.append(('__Lc__', 'sell', sellsize))
if buysize:
close_orders.append(('__Sc__', 'buy', buysize))
for order in close_orders:
myid, side, size = order
# 約定するまで次の注文は受け付けない
o = self.exchange.get_order(myid)
if o.status == 'open' or o.status == 'accepted':
delta = datetime.utcnow() - o.accepted_at
if delta < timedelta(seconds=60):
continue
self.exchange.create_order(myid, side, size, None, None, None, None, symbol)
def order(self, myid, side, qty,
limit=None, stop=None, time_in_force = None, minute_to_expire = None, symbol = None, limit_mask = 0,
seconds_to_keep_order=None):
"""注文"""
if self.exchange.order_is_not_accepted is not None:
if not self.hft:
self.logger.info("REJECT: %s order is not accepted..." % myid)
return
qty_total = qty
qty_limit = self.risk.max_position_size
# 買いポジあり
if self.position_size > 0:
# 買い増し
if side == 'buy':
# 現在のポジ数を加算
qty_total = qty_total + self.position_size
else:
# 反対売買の場合、ドテンできるように上限を引き上げる
qty_limit = qty_limit + self.position_size
# 売りポジあり
if self.position_size < 0:
# 売りまし
if side == 'sell':
# 現在のポジ数を加算
qty_total = qty_total + -self.position_size
else:
# 反対売買の場合、ドテンできるように上限を引き上げる
qty_limit = qty_limit + -self.position_size
# 購入数をポジション最大サイズに抑える
if qty_total > qty_limit:
qty = qty - (qty_total - qty_limit)
# 注文情報取得
order = self.exchange.get_order(myid)
# 前の注文が成り行き
if order['type'] == 'market':
# 約定するまで次の注文は受け付けない
if order.status == 'open' or order.status == 'accepted':
delta = datetime.utcnow() - order.accepted_at
if delta < timedelta(seconds=60):
if not self.hft:
self.logger.info("REJECT: {0} order creating...".format(myid))
return
else:
if order.status == 'open' or order.status == 'accepted':
# 前の注文と価格とサイズが同じなら何もしない
if (abs(order.price - limit)<=limit_mask) and (order.amount == qty) and (order.side == side):
return
# 新しい注文を制限する（指値を市場に出している最小時間を保証）
if seconds_to_keep_order is not None:
past = datetime.utcnow() - order.accepted_at
if past < timedelta(seconds=seconds_to_keep_order):
return
# 安全な空の旅
if self.settings.safe_order:
# 前の注文が注文一覧にのるまで次の注文は受け付けない
if order.status == 'accepted':
delta = datetime.utcnow() - order.accepted_at
if delta < timedelta(seconds=60):
if not self.hft:
self.logger.info("REJECT: {0} order creating...".format(myid))
return
# 同じIDのオープン状態の注文が2つ以上ある場合、注文は受け付けない（2つ前の注文がキャンセル中）
orders = {k: v for k, v in self.exchange.get_open_orders().items() if v['myid'] == myid}
if len(orders) >= 2:
if not self.hft:
self.logger.info("REJECT: {0} too many orders...".format(myid))
return
# 前の注文がオープンならキャンセル
if (order.status == 'open') or (order.status == 'accepted'):
self.exchange.cancel(myid)
# 最小発注サイズ(FX 0.01/現物・先物は0.001)に切り上げる
symbol = symbol or self.settings.symbol
if symbol == 'FX_BTC_JPY':
min_qty = 0.01
else:
min_qty = 0.001
# 新規注文
if qty > 0:
qty = max(qty, min_qty)
self.exchange.create_order(myid, side, qty, limit, stop, time_in_force, minute_to_expire, symbol)
def get_order(self, myid):
return self.exchange.get_order(myid)
def get_open_orders(self):
return self.exchange.get_open_orders()
def entry(self, myid, side, qty,
limit=None, stop=None, time_in_force = None, minute_to_expire = None, symbol = None, limit_mask = 0,
seconds_to_keep_order = None):
"""注文"""
# 買いポジションがある場合、清算する
if side=='sell' and self.position_size > 0:
qty = qty + self.position_size
# 売りポジションがある場合、清算する
if side=='buy' and self.position_size < 0:
qty = qty - self.position_size
# 注文
self.order(myid, side, qty, limit, stop, time_in_force, minute_to_expire, symbol, limit_mask,
seconds_to_keep_order=seconds_to_keep_order)
def create_rich_ohlcv(self, ohlcv):
if self.settings.disable_rich_ohlcv:
rich_ohlcv = Dotdict()
for k in ohlcv[0].keys():
rich_ohlcv[k] = [v[k] for v in ohlcv]
else:
rich_ohlcv = pd.DataFrame.from_records(ohlcv, index="created_at")
return rich_ohlcv
def setup(self):
# 実行中フラグセット
self.running = True
# 高頻度取引？
self.hft = self.settings.interval < 3
# 取引所セットアップ
self.exchange = Exchange(apiKey=self.settings.apiKey, secret=self.settings.secret)
if self.settings.use_lightning:
self.exchange.enable_lightning_api(
self.settings.lightning_userid,
self.settings.lightning_password)
self.exchange.start()
# ストリーミング開始
self.streaming = Streaming()
self.streaming.start()
self.ep = self.streaming.get_endpoint(self.settings.symbol, ['ticker', 'executions'],
timeframe=self.settings.timeframe,
max_ohlcv_size=self.settings.max_ohlcv_size)
self.ep.wait_for(['ticker'])
# 約定履歴・板差分から注文状態監視
if self.hft:
ep = self.streaming.get_endpoint(self.settings.symbol, ['executions', 'board'])
else:
ep = self.streaming.get_endpoint(self.settings.symbol, ['executions'])
self.exchange.start_monitoring(ep)
self.monitoring_ep = ep
# 売買ロジックセットアップ
self.yourlogic()
self.logger.info("Finished Setup")
def start(self):
self.logger.info("Start Trading")
self.setup()
def async_inverval(func, interval, parallels):
next_exec_time = 0
@wraps(func)
def wrapper(*args, **kargs):
nonlocal next_exec_time
f_result = None
t = time()
if t > next_exec_time:
next_exec_time = ((t//interval)+1)*interval
f_result = func(*args,**kargs)
if parallels is not None:
parallels.append(f_result)
return f_result
return wrapper
def async_result(f_result, last):
if f_result is not None and f_result.done():
try:
return None, f_result.result()
except Exception as e:
self.logger.warning(type(e).__name__ + ": {0}".format(e))
f_result = None
return f_result, last
async_requests = []
fetch_position = async_inverval(self.exchange.fetch_position, 30, async_requests)
check_order_status = async_inverval(self.exchange.check_order_status, 5, async_requests)
errorWait = 0
f_position = position = f_check = None
once = True
while True:
self.interval = self.settings.interval
try:
# 注文処理の完了待ち
self.exchange.wait_for_completion()
# 待ち時間
self.monitoring_ep.suspend(False)
if self.interval:
if not self.hft:
self.logger.info("Waiting...")
wait_sec = (-time() % self.interval) or self.interval
sleep(wait_sec)
self.monitoring_ep.suspend(True)
# 例外発生時の待ち
no_needs_err_wait = (errorWait == 0) or (errorWait < time())
# ポジション等の情報取得
if no_needs_err_wait:
f_position = f_position or fetch_position(self.settings.symbol)
f_check = f_check or check_order_status(show_last_n_orders=self.settings.show_last_n_orders)
# リクエスト完了を待つ
if not self.hft or once:
for f in concurrent.futures.as_completed(async_requests):
pass
once = False
async_requests.clear()
# 建玉取得
if self.settings.use_lightning:
f_position, res = async_result(f_position, (position, None))
position, _ = res
else:
f_position, position = async_result(f_position, position)
# 内部管理のポジション数をAPIで取得した値に更新
if 'checked' not in position:
self.exchange.restore_position(position.all)
position['checked'] = True
# 内部管理のポジション数取得
self.position_size, self.position_avg_price, self.openprofit, self.positions = self.exchange.get_position()
# 注文情報取得
f_check, _ = async_result(f_check, None)
# REST API状態取得
self.api_state, self.api_avg_responce_time = self.exchange.api_state()
if self.api_state is not 'normal':
self.logger.info("REST API: {0} ({1:.1f}ms)".format(self.api_state, self.api_avg_responce_time*1000))
# 価格データ取得
ticker, executions, ohlcv = Dotdict(self.ep.get_ticker()), None, None
# インターバルが0の場合、約定履歴の到着を待つ
if self.settings.interval==0:
self.ep.wait_any(['executions'], timeout=0.5)
# OHLCVを作成しない場合、約定履歴を渡す
if self.settings.disable_create_ohlcv:
executions = self.ep.get_executions()
else:
if self.settings.use_lazy_ohlcv:
ohlcv = self.create_rich_ohlcv(self.ep.get_lazy_ohlcv())
else:
ohlcv = self.create_rich_ohlcv(self.ep.get_boundary_ohlcv())
# 資金情報取得
balance = self.fetch_balance()
# 売買ロジック呼び出し
args = {
'strategy': self,
'ticker': ticker,
'ohlcv': ohlcv,
'position': position,
'balance': balance,
'executions': executions
}
if no_needs_err_wait:
self.yourlogic.bizlogic(self.yourlogic, **args)
errorWait = 0
else:
self.logger.info("Waiting for Error...")
except ccxt.DDoSProtection as e:
self.logger.warning(type(e).__name__ + ": {0}".format(e))
errorWait = time() + 60
except ccxt.RequestTimeout as e:
self.logger.warning(type(e).__name__ + ": {0}".format(e))
errorWait = time() + 30
except ccxt.ExchangeNotAvailable as e:
self.logger.warning(type(e).__name__ + ": {0}".format(e))
errorWait = time() + 5
except ccxt.AuthenticationError as e:
self.logger.warning(type(e).__name__ + ": {0}".format(e))
self.private_api_enabled = False
errorWait = time() + 5
except ccxt.ExchangeError as e:
self.logger.warning(type(e).__name__ + ": {0}".format(e))
errorWait = time() + 5
except (KeyboardInterrupt, SystemExit):
self.logger.info('Shutdown!')
break
except Exception as e:
self.logger.exception(e)
errorWait = time() + 1
self.logger.info("Stop Trading")
# 停止
self.running = False
# ストリーミング停止
self.streaming.stop()
# 取引所停止
self.exchange.stop()
class BitMEXWebsocket:
logger, test_handler = SystemLogger(__name__).get_logger()
# Don't grow a table larger than this amount. Helps cap memory usage.
MAX_TABLE_LEN = 200
def __init__(self, endpoint, symbol, api_key=None, api_secret=None):
'''Connect to the websocket and initialize data stores.'''
self.logger.debug("Initializing WebSocket.")
self.endpoint = endpoint
self.symbol = symbol
if api_key is not None and api_secret is None:
raise ValueError('api_secret is required if api_key is provided')
if api_key is None and api_secret is not None:
raise ValueError('api_key is required if api_secret is provided')
self.api_key = api_key
self.api_secret = api_secret
self.data = {}
self.keys = {}
self.exited = False
# findItemByKeys高速化のため、インデックスを作成・格納するための変数を作っておく
self.itemIdxs = {}
# 高速化のため、各処理の処理時間を格納するtimearkを作成
"""
self.timemark = {}
self.timemark['partial'] = 0
self.timemark['insert'] = 0
self.timemark['update'] = 0
self.timemark['delete'] = 0
self.timemark['find'] = 0
"""
# We can subscribe right in the connection querystring, so let's build that.
# Subscribe to all pertinent endpoints
wsURL = self.__get_url()
self.logger.info("Connecting to %s" % wsURL)
self.__connect(wsURL, symbol)
self.logger.info('Connected to WS.')
# Connected. Wait for partials
self.__wait_for_symbol(symbol)
if api_key:
self.__wait_for_account()
self.logger.info('Got all market data. Starting.')
def exit(self):
'''Call this to exit - will close websocket.'''
self.exited = True
self.ws.close()
def get_instrument(self):
'''Get the raw instrument data for this symbol.'''
# Turn the 'tickSize' into 'tickLog' for use in rounding
instrument = self.data['instrument'][0]
instrument['tickLog'] = int(
math.fabs(math.log10(instrument['tickSize'])))
return instrument
def get_ticker(self):
'''Return a ticker object. Generated from quote and trade.'''
lastQuote = self.data['quote'][-1]
lastTrade = self.data['trade'][-1]
ticker = {
"last":
lastTrade['price'],
"buy":
lastQuote['bidPrice'],
"sell":
lastQuote['askPrice'],
"mid": (float(lastQuote['bidPrice'] or 0) +
float(lastQuote['askPrice'] or 0)) / 2
}
# The instrument has a tickSize. Use it to round values.
instrument = self.data['instrument'][0]
return {
k: round(float(v or 0), instrument['tickLog'])
for k, v in ticker.items()
}
def funds(self):
'''Get your margin details.'''
return self.data['margin'][0]
def market_depth(self):
'''Get market depth (orderbook). Returns all levels.'''
return self.data['orderBookL2']
def open_orders(self, clOrdIDPrefix):
'''Get all your open orders.'''
orders = self.data['order']
# Filter to only open orders (leavesQty > 0) and those that we actually placed
return [
o for o in orders if str(o['clOrdID']).startswith(clOrdIDPrefix)
and o['leavesQty'] > 0
]
def recent_trades(self):
'''Get recent trades.'''
return self.data['trade']
#
# End Public Methods
#
def __connect(self, wsURL, symbol):
'''Connect to the websocket in a thread.'''
self.logger.debug("Starting thread")
self.ws = websocket.WebSocketApp(wsURL,
on_message=self.__on_message,
on_close=self.__on_close,
on_open=self.__on_open,
on_error=self.__on_error,
header=self.__get_auth())
self.wst = threading.Thread(target=lambda: self.ws.run_forever())
self.wst.daemon = True
self.wst.start()
self.logger.debug("Started thread")
# Wait for connect before continuing
conn_timeout = 5
while not self.ws.sock or not self.ws.sock.connected and conn_timeout:
sleep(1)
conn_timeout -= 1
if not conn_timeout:
self.logger.error("Couldn't connect to WS! Exiting.")
self.exit()
raise websocket.WebSocketTimeoutException(
'Couldn\'t connect to WS! Exiting.')
def __get_auth(self):
'''Return auth headers. Will use API time.time() if present in settings.'''
if self.api_key:
self.logger.info("Authenticating with API Key.")
# To auth to the WS using an API key, we generate a signature of a nonce and
# the WS API endpoint.
nonce = generate_nonce()
return [
"api-nonce: " + str(nonce),
"api-signature: " + generate_signature(self.api_secret, 'GET',
'/realtime', nonce, ''),
"api-key:" + self.api_key
]
else:
self.logger.info("Not authenticating.")
return []
def __get_url(self):
'''
Generate a connection URL. We can define subscriptions right in the querystring.
Most subscription topics are scoped by the symbol we're listening to.
'''
# You can sub to orderBookL2 for all levels, or orderBook10 for top 10 levels & save bandwidth
symbolSubs = [
"execution", "instrument", "order", "orderBookL2", "position",
"quote", "trade"
]
genericSubs = ["margin"]
subscriptions = [sub + ':' + self.symbol for sub in symbolSubs]
subscriptions += genericSubs
urlParts = list(urllib.parse.urlparse(self.endpoint))
urlParts[0] = urlParts[0].replace('http', 'ws')
urlParts[2] = "/realtime?subscribe={}".format(','.join(subscriptions))
return urllib.parse.urlunparse(urlParts)
def __wait_for_account(self):
'''On subscribe, this data will come down. Wait for it.'''
# Wait for the time.time() to show up from the ws
while not {'margin', 'position', 'order', 'orderBookL2'} <= set(
self.data):
sleep(0.1)
def __wait_for_symbol(self, symbol):
'''On subscribe, this data will come down. Wait for it.'''
while not {'instrument', 'trade', 'quote'} <= set(self.data):
sleep(0.1)
def __send_command(self, command, args=None):
'''Send a raw command.'''
if args is None:
args = []
self.ws.send(json.dumps({"op": command, "args": args}))
def __on_message(self, message):
'''Handler for parsing WS messages.'''
message = json.loads(message)
self.logger.debug(json.dumps(message))
table = message['table'] if 'table' in message else None
action = message['action'] if 'action' in message else None
try:
if 'subscribe' in message:
self.logger.debug("Subscribed to %s." % message['subscribe'])
elif action:
if table not in self.data:
self.data[table] = []
# index格納用objにtable用のobjを追加
if table not in self.itemIdxs:
self.itemIdxs[table] = {}
# keysが含まれない情報があるので、追加
if table not in self.keys:
self.keys[table] = {}
# There are four possible actions from the WS:
# 'partial' - full table image
# 'insert' - new row
# 'update' - update row
# 'delete' - delete row
if action == 'partial':
#処理時間計測開始
#start = time.time()
self.logger.debug("%s: partial" % table)
self.data[table] += message['data']
# time.time() are communicated on partials to let you know how to uniquely identify
# an item. We use it for updates.
self.keys[table] = message['keys']
#indexを作成します
# self.itemIdxs[table][keyvalue(kye1val-key2val-key3val)] に
# 対象データのdata[table]上のインデックスが格納されます
for i in range(len(self.data[table])):
item = self.data[table][i]
keyvalues = "-".join([
str(v) for k, v in item.items()
if k in self.keys[table]
])
self.itemIdxs[table][keyvalues] = i
# 処理時間計測終了・登録
#end = time.time()
#self.timemark['partial'] += (end - start)
elif action == 'insert':
#処理時間計測開始
#start = time.time()
self.logger.debug('%s: inserting %s' %
(table, message['data']))
self.data[table] += message['data']
#最後尾アイテムのindexを追加します
item = self.data[table][-1]
keyvalues = "-".join([
str(v) for k, v in item.items()
if k in self.keys[table]
])
self.itemIdxs[table][keyvalues] = len(self.data[table]) - 1
# Limit the max length of the table to avoid excessive memory usage.
# Don't trim orders because we'll lose valuable state if we do.
if table not in ['order', 'orderBookL2'] and len(
self.data[table]) > BitMEXWebsocket.MAX_TABLE_LEN:
self.data[table] = self.data[table][
int(BitMEXWebsocket.MAX_TABLE_LEN / 2):]
# インデックスの再構築をします
for i in range(len(self.data[table])):
item = self.data[table][i]
keyvalues = "-".join([
str(v) for k, v in item.items()
if k in self.keys[table]
])
self.itemIdxs[table][keyvalues] = i
# 処理時間計測終了・登録
#end = time.time()
#self.timemark['insert'] += (end - start)
elif action == 'update':
#処理時間計測開始
#start = time.time()
self.logger.debug('%s: updating %s' %
(table, message['data']))
# Locate the item in the collection and update it.
for updateData in message['data']:
# 高速化のため、itemIdxsを追加で引数指定
item = self.findItemByKeys(self.keys[table],
self.data[table],
updateData,
self.itemIdxs[table])
if not item:
return # No item found to update. Could happen before push
item.update(updateData)
# Remove cancelled / filled orders
if table == 'order' and item['leavesQty'] <= 0:
self.data[table].remove(item)
# 処理時間計測終了・登録
#end = time.time()
#self.timemark['update'] += (end - start)
elif action == 'delete':
#処理時間計測開始
#start = time.time()
self.logger.debug('%s: deleting %s' %
(table, message['data']))
# Locate the item in the collection and remove it.
for deleteData in message['data']:
# 高速化のため、itemIdxsを追加で引数指定
item = self.findItemByKeys(self.keys[table],
self.data[table],
deleteData,
self.itemIdxs[table])
self.data[table].remove(item)
# インデックスの再構築をします
for i in range(len(self.data[table])):
item = self.data[table][i]
keyvalues = "-".join([
str(v) for k, v in item.items()
if k in self.keys[table]
])
self.itemIdxs[table][keyvalues] = i
# 処理時間計測終了・登録
#end = time.time()
#self.timemark['delete'] += (end - start)
else:
raise Exception("Unknown action: %s" % action)
except:
self.logger.error(traceback.format_exc())
def __on_error(self, error):
'''Called on fatal websocket errors. We exit on these.'''
if not self.exited:
self.logger.error("Error : %s" % error)
raise websocket.WebSocketException(error)
def __on_open(self):
'''Called when the WS opens.'''
self.logger.debug("Websocket Opened.")
def __on_close(self):
'''Called on websocket close.'''
self.logger.info('Websocket Closed')
# 処理時間計測処理を加えるため、クラスメソッドに変更
# Utility method for finding an item in the store.
# When an update comes through on the websocket, we need to figure out which item in the array it is
# in order to match that item.
#
# Helpfully, on a data push (or on an HTTP hit to /api/v1/schema), we have a "keys" array. These are the
# fields we can use to uniquely identify an item. Sometimes there is more than one, so we iterate through all
# provided keys.
def findItemByKeys(self, keys, table, matchData, itemIdxs):
#処理時間計測開始
#start = time.time()
md_keyvalue = "-".join(
[str(v) for k, v in matchData.items() if k in keys])
if md_keyvalue in itemIdxs.keys(
) and len(table) > itemIdxs[md_keyvalue]:
# 処理時間計測終了・登録
#end = time.time()
#self.timemark['find'] += (end - start)
return table[itemIdxs[md_keyvalue]]
#end = time.time()
#self.timemark['find'] += (end - start)
""" 旧ロジック
class TestSystemContext(XrossTestBase):
logger, test_handler = SystemLogger("TestSystemContext").get_logger()
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def setUp(self):
self.cxt = SystemContext(debug=True)
def tearDown(self):
self.cxt = None
def test_set(self):
# action
self.cxt.set({"hogemanager": HOGEMANAGER})
# assert
self.assertTrue(hasattr(self.cxt, "HOGEMANAGER"))
def test_get(self):
# setup
self.test_set()
# action
mgr = self.cxt.get_str("HOGEMANAGER")
# assert
self.assertEqual("HOGEMANAGER", str(mgr.__name__))
def test_has(self):
# setup
self.test_set()
# action
result = self.cxt.has("HOGEMANAGER")
result2 = self.cxt.has("HOGEMANAGER2")
# assert
self.assertTrue(result)
self.assertFalse(result2)
def test_pop(self):
# setup
self.test_set()
# action
result = self.cxt.pop("HOGEMANAGER")
# assert
self.assertEqual(result, HOGEMANAGER)
self.assertEqual("SystemContext{'debug': True}", str(self.cxt))
def test_clear(self):
# setup
self.test_set()
# action
self.cxt.clear()
# assert
self.assertEqual("SystemContext{}", str(self.cxt))
def test_get_mgr_fail(self):
# setup
self.test_set()
# action
try:
self.XXX = self.cxt.get_str("XXX")
except AttributeError as ex:
self.assertEqual("'SystemContext' object has no attribute 'xxx'",
str(ex))
def test_get_int_fail_unless_set(self):
self.assertEqual(0, self.cxt.get_int(PARAM_KEY))
def test_get_int_fail_not_decimal(self):
self.assertEqual(0, self.cxt.get_int(PARAM_KEY))
self.cxt.increment(PARAM_KEY)
self.assertEqual(1, self.cxt.get_int(PARAM_KEY))
self.cxt.set({PARAM_KEY: "a"})
try:
self.cxt.get_int(PARAM_KEY)
self.fail()
except Exception as e:
self.assertEqual("Value:a (Key:hoge) is not decimal", str(e))
def test_increment(self):
# setup
self.cxt.set({PARAM_KEY: '99'})
# action
self.cxt.increment(PARAM_KEY)
# assert
self.assertEqual(100, self.cxt.get_int(PARAM_KEY))
# action
self.cxt.increment(PARAM_KEY, 100)
# assert
self.assertEqual(200, self.cxt.get_int(PARAM_KEY))
class Strategy:
cfg = SystemUtil(skip=True)
def __init__(self, yourlogic):
# ログ設定
self.logger, self.test_handler = SystemLogger(yourlogic.__name__).get_logger()
self.logger.info("Initializing Strategy...")
# トレーディングロジック設定
self.yourlogic = yourlogic
# 取引所情報
self.settings = Dotdict()
self.settings.exchange = self.cfg.get_env("EXCHANGE", default='coincheck')
self.settings.symbol = self.cfg.get_env("SYMBOL", default='BTC/JPY')
self.settings.lot = int(self.cfg.get_env("LOT", default=1000))
self.settings.use_websocket = self.cfg.get_env("USE_WEB_SOCKET", type=bool, default=True)
self.logger.info("USE_WEB_SOCKET: %s" % self.settings.use_websocket)
self.settings.apiKey = self.cfg.get_env("COINCHECK_KEY")
self.settings.secret = self.cfg.get_env("COINCHECK_SECRET_KEY")
self.settings.close_position_at_start_stop = False
# 動作タイミング
self.settings.interval = int(self.cfg.get_env("INTERVAL", default=86400))
# ohlcv設定
self.settings.timeframe = self.cfg.get_env("TIMEFRAME", default='1m')
self.settings.partial = False
# リスク設定
self.risk = Dotdict()
self.risk.max_position_size = 20000
self.risk.max_drawdown = 200000
# ポジション情報
self.position = Dotdict()
self.position.currentQty = 0
# 資金情報
self.balance = None
# 注文情報
self.orders = Dotdict()
# ティッカー情報
self.ticker = Dotdict()
# ohlcv情報
self.ohlcv = None
self.ohlcv_updated = False
# 約定情報
self.executions = Dotdict()
# 取引所接続
self.exchange = Exchange(self.settings, apiKey=self.settings.apiKey, secret=self.settings.secret)
self.logger.info("Completed to initialize Strategy.")
def create_order(self, myid, side, qty, limit, stop, trailing_offset, symbol):
type = 'market'
params = {}
if stop is not None and limit is not None:
type = 'stopLimit'
params['stopPx'] = stop
params['execInst'] = 'LastPrice'
# params['price'] = limit
elif stop is not None:
type = 'stop'
params['stopPx'] = stop
params['execInst'] = 'LastPrice'
elif limit is not None:
type = 'limit'
# params['price'] = limit
if trailing_offset is not None:
params['pegPriceType'] = 'TrailingStopPeg'
params['pegOffsetValue'] = trailing_offset
symbol = symbol or self.settings.symbol
res = self.exchange.create_order(myid, symbol, type, side, qty, limit, params)
self.logger.info("ORDER: {id} {order_type} {amount} {rate}({stop_loss_rate})".format(**res['info']))
return Dotdict(res)
def edit_order(self, myid, side, qty, limit=None, stop=None, trailing_offset=None, symbol=None):
type = 'market'
params = {}
if stop is not None and limit is not None:
type = 'stopLimit'
params['stopPx'] = stop
# params['price'] = limit
elif stop is not None:
type = 'stop'
params['stopPx'] = stop
elif limit is not None:
type = 'limit'
# params['price'] = limit
if trailing_offset is not None:
params['pegOffsetValue'] = trailing_offset
symbol = symbol or self.settings.symbol
res = self.exchange.edit_order(myid, symbol, type, side, qty, limit, params)
self.logger.info("EDIT: {id} {order_type} {amount} {rate}({stop_loss_rate})".format(**res['info']))
return Dotdict(res)
def order(self, myid, side, qty, limit=None, stop=None, trailing_offset=None, symbol=None):
"""注文"""
qty_total = qty
qty_limit = self.risk.max_position_size
# # 買いポジあり
# if self.position.currentQty > 0:
# # 買い増し
# if side == 'buy':
# # 現在のポジ数を加算
# qty_total = qty_total + self.position.currentQty
# else:
# # 反対売買の場合、ドテンできるように上限を引き上げる
# qty_limit = qty_limit + self.position.currentQty
#
# # 売りポジあり
# if self.position.currentQty < 0:
# # 売りまし
# if side == 'sell':
# # 現在のポジ数を加算
# qty_total = qty_total + -self.position.currentQty
# else:
# # 反対売買の場合、ドテンできるように上限を引き上げる
# qty_limit = qty_limit + -self.position.currentQty
# 購入数をポジション最大サイズに抑える
if qty_total > qty_limit:
qty = qty - (qty_total - qty_limit)
if qty > 0:
symbol = symbol or self.settings.symbol
if myid in self.orders:
order = self.exchange.fetch_order(self.orders[myid].id)
# 未約定・部分約定の場合、注文を編集
if order.status == 'open':
# オーダータイプが異なる or STOP注文がトリガーされたら編集に失敗するのでキャンセルしてから新規注文する
order_type = 'stop' if stop is not None else ''
order_type = order_type + 'limit' if limit is not None else order_type
if (order_type != order.order_type) or (order.order_type == 'stoplimit'):
# 注文キャンセルに失敗した場合、ポジション取得からやり直す
self.exchange.cancel_order(myid)
order = self.create_order(myid, side, qty, limit, stop, trailing_offset, symbol)
else:
# 指値・ストップ価格・数量に変更がある場合のみ編集を行う
if ((order.info.price is not None and order.info.price != limit) or
(order.info.stopPx is not None and order.info.stopPx != stop) or
(order.info.orderQty is not None and order.info.orderQty != qty)):
order = self.edit_order(myid, side, qty, limit, stop, trailing_offset, symbol)
# 約定済みの場合、新規注文
else:
order = self.create_order(myid, side, qty, limit, stop, trailing_offset, symbol)
# 注文がない場合、新規注文
else:
order = self.create_order(myid, side, qty, limit, stop, trailing_offset, symbol)
self.orders[myid] = order
return order
def entry(self, myid, side, qty, limit=None, stop=None, trailing_offset=None, symbol=None):
"""注文"""
# # 買いポジションがある場合、清算する
# if side == 'sell' and self.position.currentQty > 0:
# qty = qty + self.position.currentQty
#
# # 売りポジションがある場合、清算する
# if side == 'buy' and self.position.currentQty < 0:
# qty = qty - self.position.currentQty
# qty validation
price = limit or self.ticker.ask if side == 'buy' else self.ticker.bid
if qty < price * 0.005:
self.logger.warning("Quantity validation. Order quantity %s JPY (%s BTC) is lower than 0.005 BTC" % (qty, qty/price))
return
# 注文
return self.order(myid, side, qty, limit=limit, stop=stop, trailing_offset=trailing_offset, symbol=symbol)
def cancel(self, myid):
return self.exchange.cancel_order(myid)
def update_ohlcv(self, ticker_time=None, force_update=False):
if self.settings.partial or force_update:
self.ohlcv = self.exchange.fetch_ohlcv()
self.ohlcv_updated = True
else:
# 次に足取得する時間
timestamp = self.ohlcv.index
if len(timestamp) > 2:
t0 = timestamp[-1]
t1 = timestamp[-2]
next_fetch_time = t0 + (t0 - t1)
# 足取得
if ticker_time > next_fetch_time.tz_localize('Asia/Tokyo'):
self.ohlcv = self.exchange.fetch_ohlcv()
# 更新確認
timestamp = self.ohlcv.index
if timestamp[-1] >= next_fetch_time:
self.ohlcv_updated = True
else:
self.ohlcv = self.exchange.fetch_ohlcv()
def setup(self):
validate(self, "self.settings.apiKey")
validate(self, "self.settings.secret")
self.exchange.start()
self.yourlogic()
args = {
'strategy': self
}
self.yourlogic.use(self.yourlogic, **args)
def add_arguments(self, parser):
parser.add_argument('--apikey', type=str, default=self.settings.apiKey)
parser.add_argument('--secret', type=str, default=self.settings.secret)
parser.add_argument('--symbol', type=str, default=self.settings.symbol)
parser.add_argument('--timeframe', type=str, default=self.settings.timeframe)
parser.add_argument('--interval', type=float, default=self.settings.interval)
return parser
def start(self):
self.logger.info("Setup Strategy")
self.setup()
# 全注文キャンセル
self.exchange.cancel_order_all()
# ポジションクローズ
if self.settings.close_position_at_start_stop:
self.exchange.close_position()
self.logger.info("Start Trading")
# 強制足取得
self.update_ohlcv(force_update=True)
errorWait = 0
while True:
try:
# 例外発生時の待ち
if errorWait:
sleep(errorWait)
errorWait = 0
if self.settings.use_websocket:
# WebSocketの接続が切れていたら再接続
self.exchange.reconnect_websocket()
# ティッカー取得
self.ticker, last_execution = self.exchange.fetch_ticker_ws()
# ポジション取得
self.position = self.exchange.fetch_position_ws()
# 資金情報取得
self.balance = self.exchange.fetch_balance_ws()
# 約定情報
self.executions = self.exchange.fetch_my_executions_ws(self.orders)
else:
# ティッカー取得
self.ticker, last_execution = self.exchange.fetch_ticker()
# ポジション取得
self.position = self.exchange.fetch_position()
# 資金情報取得
self.balance = self.exchange.fetch_balance()
# 約定情報
self.executions = self.exchange.fetch_my_executions(self.settings.symbol, self.orders)
# 足取得（足確定後取得）
self.update_ohlcv(ticker_time=self.ticker.datetime)
# メインロジックコール
arg = {
'strategy': self,
'ticker': self.ticker,
'ohlcv': self.ohlcv,
'position': self.position,
'balance': self.balance,
'execution': self.executions
}
self.yourlogic.bizlogic(self.yourlogic, **arg)
except ccxt.DDoSProtection as e:
self.logger.exception(type(e).__name__ + ": {0}".format(e))
errorWait = 30
except ccxt.RequestTimeout as e:
self.logger.exception(type(e).__name__ + ": {0}".format(e))
errorWait = 5
except ccxt.ExchangeNotAvailable as e:
self.logger.exception(type(e).__name__ + ": {0}".format(e))
errorWait = 20
except ccxt.AuthenticationError as e:
self.logger.exception(type(e).__name__ + ": {0}".format(e))
break
except ccxt.ExchangeError as e:
self.logger.exception(type(e).__name__ + ": {0}".format(e))
errorWait = 5
except (KeyboardInterrupt, SystemExit):
self.logger.info('Shutdown!')
break
except Exception as e:
self.logger.exception(e)
errorWait = 5
# 通常待ち
sleep(self.settings.interval)
self.logger.info("Stop Trading")
# 全注文キャンセル
self.exchange.cancel_order_all()
# ポジションクローズ
if self.settings.close_position_at_start_stop:
self.exchange.close_position()
class XrossTestBase(unittest.TestCase):
_logger, _test_handler = SystemLogger("XrossTestBase").get_logger()
_logger.setLevel(logging.DEBUG)
cxt = None
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._logger.info("XrossTestBase has been loaded.")
def setUp(self):
super().setUp()
self._logger.addHandler(self._test_handler)
self._logger.debug("XrossTestBase.setUp()")
def tearDown(self):
super().tearDown()
self._logger.debug("XrossTestBase.tearDown()")
self._logger.removeHandler(self._test_handler)
self._test_handler.flush()
# noinspection PyMethodParameters
def do_test():
unittest.main()
def assertSelectSetEqual(self, expected, actual):
sorted_expect_list = sorted(expected, key=lambda x: x['timestamp'])
sorted_actual_list = sorted(actual, key=lambda x: x['timestamp'])
for d1, d2 in zip(sorted_expect_list, sorted_actual_list):
try:
self.assertDictEqual(
OrderedDict(sorted(d1.items(), key=lambda x: x[0])),
OrderedDict(sorted(d2.items(), key=lambda x: x[0])))
except AssertionError as e:
self._logger.warning(
"expected_element: %s, but actual_element: %s" %
(str(d1), str(d2)))
self._logger.warning(
"expected: %s, but actual: %s" %
(str(sorted_expect_list), str(sorted_actual_list)))
raise e
def assertRegexList(self, expected_regex_list, actual_list):
if len(actual_list) != len(expected_regex_list):
self.fail("Length of lists doesn't match. %s!=%s" %
(len(expected_regex_list), len(actual_list)))
for text, regex in zip(actual_list, expected_regex_list):
self.assertRegex(text, regex)
def assertObject(self, expect, actual):
# print("assertObject is scanning in %s" % dir(expect))
for e, a in zip(dir(expect), dir(actual)):
if not e.startswith("_") and not a.startswith(
"_") and not callable(getattr(expect, e)):
try:
self.assertEqual(getattr(expect, e), getattr(actual, a))
except AssertionError as ex:
print("AssertionError has occurred comparing between %s" %
e)
raise ex
@staticmethod
def retry(exception_to_check, tries=4, delay=1, backoff=2, logger=None):
"""Retry calling the decorated function using an exponential backoff.
http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry
:param exception_to_check: the exception to check. may be a tuple of
exceptions to check
:type exception_to_check: Exception or tuple
:param tries: number of times to try (not retry) before giving up
:type tries: int
:param delay: initial delay between retries in seconds
:type delay: int
:param backoff: backoff multiplier e.g. value of 2 will double the delay
each retry
:type backoff: int
:param logger: logger to use. If None, print
:type logger: logging.Logger instance
"""
def deco_retry(f):
@wraps(f)
def f_retry(*args, **kwargs):
self, mtries, mdelay = args[0], tries, delay
while mtries > 0:
try:
self.setUp()
return f(*args, **kwargs)
except exception_to_check as e:
msg = "%s, Retrying in %d seconds..." % (str(e),
mdelay)
if logger:
logger.warning(msg)
else:
print(msg)
time.sleep(mdelay)
mtries -= 1
mdelay *= backoff
finally:
self.tearDown()
if mtries == 0:
self.fail()
return f(*args, **kwargs)
return f_retry # true decorator
return deco_retry
class OrderManager(metaclass=Singleton):
logger, test_handler = SystemLogger(__name__).get_logger()
INVALID_ORDER = Dotdict({
'No': 0,
'myid': '__INVALID_ORDER__',
'id': '__INVALID_ORDER__',
'accepted_at': '1989-10-28T0:00:00.000',
'status': 'closed',
'symbol': 'FX_BTC_JPY',
'type': 'market',
'side': 'none',
'price': 0,
'average_price': 0,
'amount': 0,
'filled': 0,
'remaining': 0,
'fee': 0,
})
lock = threading.Lock()
number_of_orders = 0
orders = OrderedDict()
positions = deque()
def clear_for_test(self):
self.number_of_orders = 0
self.orders.clear()
def add_order(self, new_order):
with self.lock:
self.number_of_orders += 1
new_order['No'] = self.number_of_orders
self.orders[new_order['myid']] = new_order
return new_order
def add_position(self, p):
# 建玉追加
self.positions.append(p)
while len(self.positions) >= 2:
r = self.positions.pop()
l = self.positions.popleft()
if r['side'] == l['side']:
# 売買方向が同じなら取り出したポジションを戻す
self.positions.append(r)
self.positions.appendleft(l)
break
else:
if l['size'] >= r['size']:
# 決済
l['size'] = round(l['size'] - r['size'], 8)
if l['size'] > 0:
# サイズが残っている場合、ポジションを戻す
self.positions.appendleft(l)
else:
# 決済
r['size'] = round(r['size'] - l['size'], 8)
if r['size'] > 0:
# サイズが残っている場合、ポジションを戻す
self.positions.append(r)
def execute(self, o, e):
updated = False
with self.lock:
if o['filled'] < o['amount']:
# ポジション追加
self.add_position({
'side': o['side'],
'size': e['size'],
'price': e['price']
})
# 注文情報更新
last = o['filled'] * o['average_price']
curr = e['size'] * e['price']
filled = round(o['filled'] + e['size'], 8)
average_price = (last + curr) / filled
o['filled'] = filled
o['average_price'] = average_price
o['remaining'] = round(o['amount'] - filled, 8)
o['status'] = o['status']
if o['remaining'] <= 0:
o['status'] = 'closed'
else:
if o['status'] == 'accepted':
o['status'] = 'open'
updated = True
return updated
def open_or_cancel(self, o, size):
updated = False
with self.lock:
if o['status'] == 'cancel':
# 板サイズが注文サイズ未満ならキャンセル完了
if size < o['amount']:
o['status'] = 'canceled'
updated = True
elif o['status'] == 'accepted':
# 板サイズが注文サイズ以上ならオープン（他のユーザの注文と被る可能性があるが許容する）
if size >= o['amount']:
o['status'] = 'open'
updated = True
return updated
def expire(self, o):
with self.lock:
if o['status'] in ['cancel', 'open']:
o['status'] = 'canceled'
def overwrite(self, o, latest):
with self.lock:
# ローカルで更新した状態は残しておく
if latest['status'] == 'open':
if o['status'] in ['cancel', 'canceled', 'closed']:
latest['status'] = o['status']
if latest['filled'] < o['filled']:
latest['filled'] = o['filled']
for k in [
'id', 'accepted_at', 'status', 'average_price', 'filled',
'remaining', 'fee'
]:
o[k] = latest[k]
def cancel_order(self, myid):
cancelable = ['open', 'accepted']
with self.lock:
my_orders = [
v for v in self.orders.values()
if (v['type'] != 'market') and (v['myid'] == myid) and (
v['status'] in cancelable)
]
for o in my_orders:
o['status'] = 'cancel'
return my_orders
def cancel_order_all(self):
cancelable = ['open', 'accepted']
with self.lock:
my_orders = [
v for v in self.orders.values()
if (v['type'] != 'market') and (v['status'] in cancelable)
]
for o in my_orders:
o['status'] = 'cancel'
return my_orders
def get_order(self, myid):
with self.lock:
my_orders = [v for v in self.orders.values() if v['myid'] == myid]
if len(my_orders):
return my_orders[-1]
raise OrderNotFoundException("MyOrderID:%s is not found. MyOrders:%s" %
(myid, my_orders))
def get_open_order(self, myid):
my_open_orders = self.get_open_orders()
my_order = [v for v in my_open_orders.values() if v['myid'] == myid]
if len(my_order) == 1:
return Dotdict(my_order[0])
elif len(my_order) == 0:
raise OrderNotFoundException(
"MyOrderID:%s is not open. MyOpenOrders:%s" %
(myid, my_open_orders))
else:
raise OrderDuplicateFoundException("MyOrderID:%s is duplicated." %
myid)
def get_open_orders(self):
return self.get_orders(status_filter=['open', 'accepted', 'cancel'])
def get_orders(self, status_filter=None):
with self.lock:
if status_filter is None:
my_orders = self.orders.copy()
else:
my_orders = {
k: v
for k, v in self.orders.items()
if (v['status'] in status_filter)
}
return my_orders
def is_active(self, myid):
try:
self.get_open_order(myid)
except OrderNotFoundException as e:
return False
except Exception as e:
return False
return True
def update_order(self, o):
self.get_order(o['myid']).update(o)
return o
def cleaning_if_needed(self, limit_orders=200, remaining_orders=20):
"""注文情報整理"""
open_status = ['open', 'accepted', 'cancel']
with self.lock:
if len(self.orders) > limit_orders:
all_orders = list(self.orders.items())
# open/accepted状態の注文は残す
orders = [(k, v) for k, v in all_orders[:-remaining_orders]
if v['status'] in open_status]
# 最新n件の注文は残す
orders.extend(all_orders[-remaining_orders:])
self.orders = OrderedDict(orders)
def printall(self):
print('\n' + '\t'.join(self.INVALID_ORDER.keys()))
for v in self.orders.values():
print('\t'.join([str(v) for v in v.values()]))
class LightningAPI:
logger, test_handler = SystemLogger(__name__).get_logger()
def __init__(self, id, password, timeout=60):
self.id = id
self.password = password
self.account_id = ''
self.timeout = timeout
self.api_url = 'https://lightning.bitflyer.com/api/trade'
self.session = requests.session()
self.logon = False
self.driver = None
validate(self, "self.id")
validate(self, "self.password")
# #ブラウザを起ち上げっぱなしにしたいのでthreading
# self.thread = threading.Thread(target=lambda: self.login())
# self.thread.daemon = True
# self.thread.start()
def login(self):
"""ログイン処理"""
try:
# ヘッドレスブラウザがらみの設定など
# WEB_DRIVER_PATH = './chromedriver.exe' #windows
# WEB_DRIVER_PATH = './chromedriver' #mac linux
cfg = SystemUtil(skip=True)
if not cfg.get_env("WEB_DRIVER_PATH"):
self.logger.warning("Failed to find WEB_DRIVER_PATH")
return
# ヘッドレスブラウザのオプションを設定
options = Options()
# options.binary_location = 'C:/*********/chrome.exe' #windowsのみPATH指定
options.add_argument(
'--headless') # ヘッドレスモードを有効、指定しなければ通常通りブラウザが立ち上がる
options.add_argument('--no-sandbox')
options.add_argument('--disable-gpu')
options.add_argument(
'--user-agent=Mozilla/5.0 (iPhone; U; CPU iPhone OS 5_1_1 like Mac OS X; en) AppleWebKit/534.46.0 (KHTML, like Gecko) CriOS/19.0.1084.60 Mobile/9B206 Safari/7534.48.3'
)
# ヘッドレスブラウザ(webdriver)インスタンスを作成
# driver = webdriver.Chrome(WEB_DRIVER_PATH, chrome_options=options)
self.logger.info('Start WebDriver...')
driver = webdriver.Chrome(options=options)
# bitFlyerへアクセス
self.logger.info('Access lightning...')
driver.get('https://lightning.bitflyer.jp/')
driver.save_screenshot("login.png")
# ログインフォームへID、PASSを入力
login_id = driver.find_element_by_id('LoginId')
login_id.send_keys(self.id)
login_password = driver.find_element_by_id('Password')
login_password.send_keys(self.password)
# ログインボタンをクリック(2段階認証が無い場合はログイン完了)
self.logger.info('Login lightning...')
driver.find_element_by_id('login_btn').click()
driver.save_screenshot("2factor.png")
# 通常2段階認証の処理が入るが種類が多いので割愛
print("Input 2 Factor Code >>")
driver.find_element_by_name("ConfirmationCode").send_keys(input())
# 確認ボタンを押す
driver.find_element_by_xpath(
"/html/body/main/div/section/form/button").click()
# driver.save_screenshot("trade.png")
# account_idを取得(実は必要ない、たぶん)
self.account_id = driver.find_element_by_tag_name(
'body').get_attribute('data-account')
# ヘッドレスブラウザで取得したcookieをrequestsにセット
for cookie in driver.get_cookies():
self.session.cookies.set(cookie['name'], cookie['value'])
self.logon = True
driver.get('https://lightning.bitflyer.jp/performance')
# driver.save_screenshot("performance.png")
self.logger.info('Lightning API Ready')
self.driver = driver
# ヘッドレスブラウザを起ち上げっぱなしにしたいので、とりあえずループさせる？
# cookieを定期的に更新させてもよいのかもしれない
# while True:
# pass
except Exception as e:
self.logger.exception(type(e).__name__ + ": {0}".format(e))
def logoff(self):
self.logger.info('Lightning Logoff')
self.driver.quit()
def sendorder(self,
product_code,
ord_type,
side,
price,
size,
minuteToExpire=43200,
time_in_force='GTC'):
"""注文送信"""
params = {
'account_id': self.account_id,
'is_check': 'false',
'lang': 'ja',
'minuteToExpire': minuteToExpire,
'ord_type': ord_type,
'price': price,
'product_code': product_code,
'side': side,
'size': size,
'time_in_force': time_in_force,
}
return self.do_request('/sendorder', params)
def getMyActiveParentOrders(self, product_code):
"""注文取得(アクティブ)"""
params = {
'account_id': self.account_id,
'lang': 'ja',
'product_code': product_code
}
return self.do_request('/getMyActiveParentOrders', params)
def getMyBoardOrders(self, product_code):
"""注文取得(全て/キャンセルが含まれるかも)"""
params = {
'account_id': self.account_id,
'lang': 'ja',
'product_code': product_code
}
return self.do_request('/getMyBoardOrders', params)
def cancelorder(self, product_code, order_id):
"""注文キャンセル"""
params = {
'account_id': self.account_id,
'lang': 'ja',
'order_id': order_id,
'parent_order_id': '',
'product_code': product_code
}
return self.do_request('/cancelorder', params)
def cancelallorder(self, product_code):
"""注文全キャンセル"""
params = {
'account_id': self.account_id,
'lang': 'ja',
'product_code': product_code
}
return self.do_request('/cancelallorder', params)
def getmyCollateral(self, product_code):
"""証拠金の状態やポジションを取得"""
params = {
'account_id': self.account_id,
'lang': 'ja',
'product_code': product_code
}
return self.do_request('/getmyCollateral', params)
def inventories(self):
"""資産情報取得"""
params = {
'account_id': self.account_id,
'lang': 'ja',
}
return self.do_request('/inventories', params)
def do_request(self, endpoint, params):
"""リクエスト送信"""
headers = {
'Content-Type': 'application/json; charset=utf-8',
'X-Requested-With': 'XMLHttpRequest'
}
response = self.session.post(self.api_url + endpoint,
data=json.dumps(params),
headers=headers,
timeout=self.timeout)
content = ''
if len(response.content) > 0:
content = json.loads(response.content.decode("utf-8"))
if isinstance(content, dict):
if 'status' in content:
if content['status'] < 0:
raise LightningError(content)
return content['data']
return content