def update_order_price(self, oid, new_price):
with self.__lock_order_data:
if oid in self.order_id:
self.__tmp_trade_log.append('#' + str(self.account_id) +
' - updated order price from ' +
str(self.order_price[oid]) +
' to ' + str(new_price))
print('#', self.account_id, ' - updated order price from',
self.order_price[oid], ' to ', new_price)
LineNotification.send_message('#' + str(self.account_id) +
' - updated order price from ' +
str(self.order_price[oid]) +
' to ' + str(new_price))
self.order_price[oid] = round(float(new_price), 1)
else:
self.__tmp_trade_log.append(
'#' + str(self.account_id) +
' - updated order price but order_id is not existed in BotAccount !'
)
print(
'#', self.account_id,
' - updated order price but order_id is not existed in BotAccount !'
)
LineNotification.send_message(
'#' + str(self.account_id) +
' - updated order price but order_id is not existed in BotAccount !'
)
def market_order_wait_till_execution(cls, side, size) -> dict:
id = cls.order(side, 0, size, 'market', 0)
i = 0
print('waiting order execution...')
flg_activated = False
time.sleep(1)
while True:
status = cls.get_order_status(id)
if len(status) > 0:
if status[0]['child_order_state'] == 'COMPLETED': # order executed
print('order has been executed')
return status[0]
elif status[0]['child_order_state'] == 'ACTIVE':
flg_activated = True
else:
if flg_activated:
print('order has been expired')
return None
i += 1
if i > 50:
print('market order wait till execution - ')
print(status[0])
LogMaster.add_log('market order wait till execution - ', 0, None)
LineNotification.send_error('market order wait till execution - ')
return status[0]
time.sleep(0.3)
def __confirm_order(self, order_id):
for i in range(10):
order_data = Trade.get_order_byid(order_id, 10)
if len(order_data) > 0:
if self.order_side[order_id] != order_data[-1]['side']:
print('order side is not matched !')
LineNotification.send_error('order side is not matched !')
self.order_side[order_id] = order_data[-1]['side']
if self.order_price[order_id] != order_data[-1]['price']:
print('order price is not matched !')
LineNotification.send_error('order price is not matched !')
self.order_price[order_id] = order_data[-1]['price']
if self.order_size[order_id] != order_data[-1]['orderQty']:
print('order size is not matched !')
LineNotification.send_error('order size is not matched !')
self.order_size[order_id] = order_data[-1]['orderQty']
if self.order_type[order_id].upper() != order_data[-1]['ordType'].upper():
print('order type is not matched !')
LineNotification.send_error('order type is not matched !')
self.order_type[order_id] = order_data[-1]['ordType']
self.order_status[order_id] = 'Onboarded'
return 0
else:
time.sleep(1)
print('__confirm_order: order id not found in order data!')
LineNotification.send_error('__confirm_order: order id not found in order data!')
return None
def update_order_price(cls, order_id, new_price):
for i in range(cls.error_trial):
cls.num_private_access += 1
order_info = ''
error_message = ''
order_data = cls.get_order_byid(order_id)
if 'user_id' in order_data:
try:
order_info = cls.bb.edit_order(
order_id, 'BTC/USD', order_data['order_type'],
order_data['side'], None, new_price,
{'time_in_force': 'GoodTillCancel'})
except Exception as e:
error_message = str(e)
print('Trade.update_order_price: Error', e)
print('order_data', order_data)
print('order_info', order_info)
LineNotification.send_message(
'Trade.update_order_price: Error\n' + str(e))
cls.initialize()
time.sleep(1)
finally:
if 'info' in order_info:
if order_info['info']['ret_msg'] == 'ok':
print(
'Trade.update_order_price:Order price successfully updated.'
)
else:
print('Trade.update_order_price:Order price failed.',
order_info)
return order_info
else:
print('Trade.update_order_price: Order id is not found!',
order_id)
def cancel_and_wait_completion(cls, oid) -> dict:
status = cls.get_order_status(oid)
if len(status) == 0:
n = 0
while len(status) == 0:
time.sleep(0.2)
n += 1
status = cls.get_order_status(oid)
if n > 25:
print('cancel_and_wait_completion - order status is not available!')
return []
cls.cancel_order(oid)
print('waiting cancel order ' + oid)
n = 0
while True: # loop for check cancel completion or execution
status = cls.get_order_status(oid)
if len(status) > 0:
if (status[0]['child_order_state'] == 'COMPLETED' or status[0]['child_order_state'] == 'CANCELED') and status[0]['executed_size'] > 0:
print('cancel failed order has been partially executed. exe size='+str(status[0]['executed_size']))
return status[0]
elif (status[0]['child_order_state'] == 'COMPLETED' or status[0]['child_order_state'] == 'CANCELED') and status[0]['executed_size'] == 0:
print('order has been successfully cancelled')
return []
else:
print('order has been successfully cancelled')
return []
n +=1
if n > 25:
print('5 sec passed but cancel order completion was not confirmed!')
LineNotification.send_error('5 sec passed but cancel order completion was not confirmed!')
return []
time.sleep(0.2)
def __display_thread(cls):
while SystemFlg.get_system_flg():
print('pl=', cls.total_pl, 'num_trade=', cls.num_trade,
'win_rate=', cls.win_rate, 'pl_per_min=', cls.pl_per_min,
'total_fee=', cls.total_fee)
print(cls.get_order_data())
print('holding_side=', cls.holding_side, 'holding_price',
cls.holding_price, 'holding_qty=', cls.holding_qty)
#LineNotification.send_message('holding_side:'+cls.holding_side+', holding_price:'+str(cls.holding_price)+', holding_qty:'+str(cls.holding_qty))
order_data = cls.get_order_data()
LineNotification.send_message('pl=' + str(cls.total_pl) +
', num_trade=' + str(cls.num_trade) +
', win_rate=' + str(cls.win_rate) +
'\n' + '********Holding********\n' +
cls.holding_side + ' @' +
str(cls.holding_price) + ' x ' +
str(cls.holding_qty) + '\n' +
'********Order********\n' +
str(order_data['side']) + ' @' +
str(order_data['price']) + ' x ' +
str(order_data['leaves_qty']))
LogMaster.add_account_log(datetime.datetime.now(),
Trade.get_bid_ask()[0], cls.total_pl,
cls.total_fee, cls.num_trade,
cls.win_rate)
#loop = asyncio.new_event_loop()
#loop.run_until_complete(cls.__generate_and_send_pl_image())
if cls.image_sending_flg >= 60:
cls.__generate_and_send_pl_image()
cls.image_sending_flg = 0
else:
cls.image_sending_flg += 1
time.sleep(60)
def order(cls, side, price, type, amount):
for i in range(cls.error_trial):
cls.num_private_access += 1
order_info = ''
error_message = ''
try:
if type == 'Limit':
order_info = cls.bm.create_order(
symbol='BTC/USD',
type=type,
side=side,
price=price,
amount=amount #×0.0001btc
)
elif type == 'Market':
order_info = cls.bm.create_order(
symbol='BTC/USD',
type=type,
side=side,
amount=amount # ×0.0001btc
)
except Exception as e:
error_message = str(e)
print('Trade-order error!, '+str(e))
print('side=',side, ', price=',price, ', type', type, ', amount', amount)
LineNotification.send_error('error in order! ' + '\r\n' + order_info + '\r\n' + str(e))
finally:
if 'error' not in error_message:
return order_info
else:
time.sleep(cls.rest_interval)
return None
def get_order_status(cls, id) -> []:
if cls.flg_api_limit == False:
res = []
try:
cls.num_private_access += 1
res = cls.bf.private_get_getchildorders(
params={
'product_code': 'FX_BTC_JPY',
'child_order_acceptance_id': id
})
except Exception as e:
if cls.check_exception(e) == 'ok':
pass
print('error in get_order_status ' + str(e))
LogMaster.add_log(
'api_error - Trade-get order status error! ' + str(e),
None)
LineNotification.send_error(
'api_error:Trade-get order status error!' + str(e))
finally:
return res
else:
print(
'get_order_status is temporary exhibited due to API access limitation!'
)
LogMaster.add_log(
'get_order_status is temporary exhibited due to API access limitation!',
None)
return None
def start_flyer_bot(self, num_term, window_term, pl, ls, upper_kijun,
lower_kijun):
self.__bot_initializer(num_term, window_term, pl, ls, upper_kijun,
lower_kijun)
self.start_time = time.time()
self.fixed_order_size = 0.05
while SystemFlg.get_system_flg():
self.__check_system_maintenance(num_term, window_term)
self.__update_ohlc()
if self.ac.holding_side == '' and self.ac.pt_side == '' and self.flg_loss_cut == False: #no position no pt order
if self.prediction == 1 or self.prediction == -1:
self.entry_price_tracing_order(self.pred_side,
self.fixed_order_size)
elif self.ac.holding_side != '' and self.ac.pt_side == '' and self.flg_loss_cut == False: #holding position and no order
self.entry_pt_order()
elif (self.ac.holding_side == 'buy' and self.prediction
== 2) or (self.ac.holding_side == 'sell' and self.prediction
== 1): # ポジションが判定と逆の時にexit, もしplがあればキャンセル。。
self.cancel_pt_order()
self.exit_order()
self.entry_price_tracing_order(self.pred_side,
self.fixed_order_size)
elif self.ac.holding_side == '' and self.ac.pt_side != '': #unexpected situation, no holding position but pt order exist
print('no position but pt order exist!')
LogMaster.add_log('no position but pt order exist!',
self.prediction, self.ac)
LineNotification.send_error('no position but pt order exist!')
self.cancel_pt_order()
self.check_and_do_lc() #check and do loss cut
if Trade.flg_api_limit:
time.sleep(60)
print('Bot sleeping for 60sec due to API access limitation')
else:
time.sleep(0.1)
def get_order_data(self, oid):
with self.__lock_order_data:
if oid in self.order_id:
return {
'id': oid,
'dt': self.order_dt[oid],
'side': self.order_side[oid],
'price': self.order_price[oid],
'qty': self.order_qty[oid],
'leaves_qty': self.order_leaves_qty[oid],
'type': self.order_type[oid],
'status': self.order_status[oid],
'fee': self.order_cum_exec_fee[oid]
}
else:
print('BotAccount.get_order_data: Unknown order id !', oid)
LineNotification.send_message(
'BotAccount.get_order_data: Unknown order id ! - ' + oid)
return {
'id': '',
'dt': '',
'side': '',
'price': 0,
'qty': 0,
'leaves_qty': 0,
'type': '',
'status': '',
'fee': 0
}
def update_order_amount(self, oid, new_amount):
with self.__lock_order_data:
if oid in self.order_id:
self.__tmp_trade_log.append('#' + str(self.account_id) +
' - updated order amount from ' +
str(self.order_qty[oid]) + ' to ' +
str(new_amount))
print('#', self.account_id, ' - updated order amount from',
self.order_qty[oid], ' to ', new_amount)
LineNotification.send_message('#' + str(self.account_id) +
' - updated order amount from ' +
str(self.order_qty[oid]) +
' to ' + str(new_amount))
self.order_leaves_qty[oid] += int(new_amount) - int(
self.order_qty[oid])
self.order_qty[oid] = int(new_amount)
else:
self.__tmp_trade_log.append(
'#' + str(self.account_id) +
' - updated order price but order_id is not existed in BotAccount !'
)
print(
'#', self.account_id,
' - updated order price but order_id is not existed in BotAccount !'
)
LineNotification.send_message(
'#' + str(self.account_id) +
' - updated order price but order_id is not existed in BotAccount !'
)
def start(self):
LineNotification.initialize()
OneMinMarketData.initialize_for_bot()
ws = RealtimeWSAPI()
bot = Bot('./Data/sim_log.csv', True)
while True:
time.sleep(1)
def cancel_order(self, oid):
self.__remove_order_data(oid)
self.__tmp_trade_log.append('#' + str(self.account_id) +
' - cancelled order.')
print('#', self.account_id, ' - cancelled order.')
LineNotification.send_message('#' + str(self.account_id) +
' - cancelled order.')
def restart_thread(cls):
LineNotification.send_error('restarted ws ohlc thread.')
with cls.lock_tmp_data:
cls.tmp_exec_data = []
cls.last_ohlc_min = int(datetime.now(cls.JST).minute)
cls.flg_ohlc_loop = True
th = threading.Thread(target=cls.__calc_ohlc_thread)
th.start()
def get_positions(cls): # None
cls.num_private_access += 1
try:
positions = cls.bb.private_get_position()
except Exception as e:
print('error in get_positions ' + e)
LineNotification.send_message('error in get_positions ' + e)
return positions
def market_order_wait_till_execution3(cls, side, total_size):
size = []
price = []
max_wait = 10
num = 0
bid = TickData.get_bid_price()
ask = TickData.get_ask_price()
try:
cls.num_private_access += 1
order_id = cls.bf.create_order(
symbol='BTC/JPY',
type='market',
side=side,
amount=total_size,
params={'product_code': 'FX_BTC_JPY'})
except Exception as e:
print('market order failed! ' + str(e))
LogMaster.add_log('market order failed! ' + str(e), None)
LineNotification.send_error('market order failed! ' + str(e))
cls.check_exception(e)
return -1, 0, 0, ''
order_id = order_id['info']['child_order_acceptance_id']
print('waiting for market order execution...')
start = time.time()
while sum(size) < total_size:
exe_data = TickData.get_exe_data()[-100:]
if time.time() - start > max_wait:
exe_data = cls.get_executions()
exe_data = TickData.get_exe_data()[-100:]
for exec in exe_data:
if exec[side + "_child_order_acceptance_id"] == order_id:
size.append(exec["size"])
price.append(exec["price"])
if time.time() - start > max_wait:
print(
'can not complete Trade - check_and_wait_till_all_execution!'
)
LogMaster.add_log(
'can not complete Trade - check_and_wait_till_all_execution!',
None)
LineNotification.send_error(
'can not complete Trade - check_and_wait_till_all_execution!'
)
return -1, sum(size), round(
sum(price[i] * size[i]
for i in range(len(price))) / sum(size)), order_id
num += 1
ave_p = round(
sum(price[i] * size[i] for i in range(len(price))) / sum(size))
print('market order has been successfully executed.' + 'side=' + side +
', ave price=' + str(ave_p) + ', size=' +
str(round(sum(size), 2)))
sp = ask - ave_p if side == 'buy' else ave_p - bid
print('market order spread loss/profit = ' + str(sp))
cls.spread_loss_profit += sp
cls.spread_loss_profit_log.append(sp)
return 0, round(sum(size), 2), ave_p, order_id
def save_file(cls, file_name):
try:
s3 = boto3.resource('s3')
s3.Object(cls.__bucket_name, file_name).upload_file(file_name)
return 0
except Exception as e:
print('s3 bot trade log save file error!={}'.format(e))
LineNotification.send_error('s3 bot trade log save file error!={}'.format(e))
return -1
def __calc_ohlc_thread(cls):
while cls.flg_ohlc_loop:
target_min = -1
next_min = -1
while SystemFlg.get_system_flg():
data = cls.get_tmp_exec_data()
if len(data) > 0:
target_min = int(data[-1]['timestamp'].split('T')[1].split(':')[1]) #計算開始の基準minuteを決定
target_min = target_min +1 if target_min +1 < 60 else 0
next_min = target_min +1 if target_min +1 < 60 else 0
break
time.sleep(0.1)
while SystemFlg.get_system_flg():
data = cls.get_tmp_exec_data()
if len(data) > 0 and int(data[-1]['timestamp'].split('T')[1].split(':')[1]) == next_min or (datetime.now().minute == next_min and datetime.now().second > 2): #次の分のデータが入り出したらohlc計算する
next_data = []
target_data = []
for d in data:
minut = int(d['timestamp'].split('T')[1].split(':')[1])
#if (next_min != 0 and minut >= next_min) or (next_min == 0 and minut >= 0):
if minut == next_min:
next_data.append(d) #次回の分のデータは次に回す
elif minut == target_min:
target_data.append(d)
if len(next_data) > 0:
cls.add_tmp_exec_data(next_data)
if len(target_data) == 0:
print('RealtimeWSAPI: target data len is 0 !')
LineNotification.send_error('RealtimeWSAPI: target data len is 0 !')
#ここに来たら、計算開始の基準minuteを決定まで戻ったほうがいい。
#reset target min
cls.flg_ohlc_loop = False
break
#if len(data) > 0:
# target_min = int(data[-1]['timestamp'].split('T')[1].split(':')[1])
# target_min = target_min + 1 if target_min + 1 < 60 else 0
# next_min = target_min + 1 if target_min + 1 < 60 else 0
#else:
# print('RealtimeWSAPI: data len is 0 !')
# LineNotification.send_error('RealtimeWSAPI: data len is 0 !')
else:
p = [d.get('price') for d in target_data]
size = [d.get('size') for d in target_data]
dt = dateutil.parser.parse(target_data[-1]['timestamp'])
OneMinMarketData.add_tmp_ohlc(dt.timestamp(), dt, p[0], max(p), min(p), p[-1], sum(size))
#print(datetime.now())
#print('RealtimeWSAPI: ', dt.timestamp(), p[0], max(p), min(p), p[-1], sum(size))
target_min = target_min + 1 if target_min + 1 < 60 else 0
next_min = target_min + 1 if target_min + 1 < 60 else 0
else:
cls.add_tmp_exec_data(data)
time.sleep(1)
if cls.flg_ohlc_loop == False:
cls.restart_thread()
def check_train_test_index_duplication(self, train_x, test_x):
train_list = list(train_x.index.values)
test_list = list(test_x.index.values)
dupli = set(train_list) & set(test_list)
if len(dupli) > 0:
print('Index duplication in train and test df was found!')
LineNotification.send_error(
'LgbModel:' +
'Index duplication in train and test df was found!')
print(dupli)
def get_current_asset(cls):
try:
cls.num_private_access += 1
res = cls.bf.fetch_balance()
except Exception as e:
print('error i get_current_asset ' + e)
LogMaster.add_log('action_message - Trade-get current asset error! ' + str(e),0,None)
LineNotification.send_error('action_message - Trade-get current asset error! ' + str(e))
if cls.check_exception(e) == 'ok':
pass
finally:
return res['total']['BTC'] * TickData.get_ltp() + res['total']['JPY']
def get_collateral(cls):
res=''
try:
cls.num_private_access += 1
res = cls.bf.fetch2(path='getcollateral', api='private', method='GET')
except Exception as e:
print('error i get_collateral ' + e)
LogMaster.add_log('api_error - Trade-get get_collateral error! ' + str(e),0,None)
LineNotification.send_error('api_error - Trade-get get_collateral error! ' + str(e))
if cls.check_exception(e) == 'ok':
pass
finally:
return res
def get_positions(cls): # None
try:
cls.num_private_access += 1
positions = cls.bf.private_get_getpositions(params={"product_code": "FX_BTC_JPY"})
except Exception as e:
print('error in get_positions ' + str(e))
LogMaster.add_log('api_error - Trade-get get_positions error! ' + str(e),0,None)
LineNotification.send_error('api_error - Trade-get get_positions error! ' + str(e))
if cls.check_exception(e) == 'ok':
pass
else:
return cls.get_positions()
return positions
def get_order(cls, order_id):
try:
cls.num_private_access += 1
order = cls.bf.fetch_open_orders(symbol='BTC/JPY', params={"product_code": "FX_BTC_JPY", 'child_order_acceptance_id': order_id})
except Exception as e:
print('error in get_order ' + str(e))
LogMaster.add_log('api_error - Trade-get get_order error! ' + str(e), None)
LineNotification.send_error('api_error - Trade-get get_order error! ' + str(e))
if cls.check_exception(e) == 'ok':
pass
else:
return cls.get_order(order_id)
return order
def calc_opt_pl(self):
if TickData.get_1m_std() > 10000:
newpl = self.pl_kijun * math.log(
(TickData.get_1m_std() / 100000)) + 5
print('changed opt pl kijun to ' + str(newpl))
LogMaster.add_log(
'action_message - changed opt pl kijun to ' + str(newpl),
self.prediction[0], self.ac)
LineNotification.send_error('changed opt pl kijun to ' +
str(newpl))
return newpl
else:
return self.pl_kijun
def get_orders(cls, count):
cls.num_private_access += 1
order_data = None
try:
order_data = cls.bb.fetch_orders(symbol="BTC/USD",
params={"count": count})
except Exception as e:
print('Error in Trade.get_orders:', str(e))
LineNotification.send_message('Error in Trade.get_orders:\n' +
str(e))
cls.initialize()
finally:
return order_data
def market_order_wait_till_execution2(cls, side, total_size):
size = []
price = []
wait_sec = 0.3
max_wait = 2.1
num = 0
try:
cls.num_private_access += 1
order_id = cls.bf.create_order(
symbol='BTC/JPY',
type='market',
side=side,
amount=total_size,
params={'product_code': 'FX_BTC_JPY'})
except Exception as e:
print('market order failed! ' + str(e))
LogMaster.add_log('market order failed! ' + str(e), None)
LineNotification.send_error('market order failed! ' + str(e))
cls.check_exception(e)
return -1, 0, 0, ''
order_id = order_id['info']['child_order_acceptance_id']
print('waiting for market order execution...')
while sum(size) < total_size:
executions = cls.get_executions()
for exec in executions:
if exec["child_order_acceptance_id"] == order_id:
size.append(exec["size"])
price.append(exec["price"])
time.sleep(wait_sec)
num += 1
if num * wait_sec > max_wait:
print(
'can not complete Trade - check_and_wait_till_all_execution!'
)
LogMaster.add_log(
'can not complete Trade - check_and_wait_till_all_execution!',
0, None)
LineNotification.send_error(
'can not complete Trade - check_and_wait_till_all_execution!'
)
return -1, sum(size), round(
sum(price[i] * size[i]
for i in range(len(price))) / sum(size)), order_id
ave_p = round(
sum(price[i] * size[i] for i in range(len(price))) / sum(size))
print('market order has been successfully executed.' + 'side=' + side +
', ave price=' + str(ave_p) + ', size=' +
str(round(sum(size), 2)))
return 0, round(sum(size), 2), ave_p, order_id
def calc_opt_size(self):
collateral = Trade.get_collateral()['collateral']
if TickData.get_1m_std() > 10000:
multiplier = 0.5
print('changed opt size multiplier to 0.5')
LogMaster.add_log(
'action_message - changed opt size multiplier to 0.5',
self.prediction[0], self.ac)
LineNotification.send_error('changed opt size multiplier to 0.5')
else:
multiplier = 1.5
size = round((multiplier * collateral * self.margin_rate) /
TickData.get_ltp() * 1.0 / self.leverage, 2)
return size
def get_order_byid(cls, order_id, count):
cls.num_private_access += 1
orders = None
try:
orders = cls.bm.fetch_orders(symbol='BTC/USD', since=None, limit=None, params={'count': count, 'reverse': True})
if len(order_id) > 0:
orders = map(lambda x: x['info'] if x['info']['orderID'] == order_id else {}, orders)
orders = [x for x in orders if x != {}]
if len(orders) == 0:
order_id = None
except Exception as e:
print('error in get_order_byid' + str(e))
LineNotification.send_error('error in get_order_byid! ' + '\r\n' + str(e))
return orders
def __check_system_maintenance(self, num_term, window_term):
if (datetime.now(tz=self.JST).hour == 3
and datetime.now(tz=self.JST).minute >= 59):
print('sleep waiting for system maintenance')
#TickData.stop(30,30)
LineNotification.send_error('sleep waiting for system maintenance')
if self.ac.order_side != '':
self.cancel_order()
time.sleep(720) # wait for daily system maintenace
TickData.initialize()
CryptowatchDataGetter.get_and_add_to_csv()
OneMinMarketData.initialize_for_bot(num_term, window_term)
LineNotification.send_error('resumed from maintenance time sleep')
print('resumed from maintenance time sleep')
def order_wait_till_boarding(cls, side, price, size, expire_m) -> dict:
oid = cls.order(side, price, size, 'limit', expire_m)
n = 0
while True:
status = cls.get_order_status(oid)
if len(status) > 0:
if status[0]['child_order_state'] == 'ACTIVE' or status[0]['child_order_state'] == 'COMPLETED':
print('confirmed the order has been boarded')
return status[0]
n += 1
if n > 30:
print('6 sec was passed but order boarding was not confirmed!')
LineNotification.send_error('6 sec was passed but order boarding was not confirmed!')
time.sleep(0.2)