def download_data(self, file, bin_size, start_time, end_time):
"""
データをサーバーから取得する。
"""
if not os.path.exists(os.path.dirname(file)):
os.makedirs(os.path.dirname(file))
data = pd.DataFrame()
left_time = None
source = None
is_last_fetch = False
while True:
if left_time is None:
left_time = start_time
right_time = left_time + delta(allowed_range[bin_size][0]) * 99
else:
left_time = source.iloc[-1].name + + delta(allowed_range[bin_size][0]) * allowed_range[bin_size][2]
right_time = left_time + delta(allowed_range[bin_size][0]) * 99
if right_time > end_time:
right_time = end_time
is_last_fetch = True
source = self.fetch_ohlcv(bin_size=bin_size, start_time=left_time, end_time=right_time)
data = pd.concat([data, source])
if is_last_fetch:
data.to_csv(file)
break
time.sleep(2)
def fetch_ohlcv(self, bin_size, start_time, end_time):
"""
fetch OHLCV data
:param start_time: start time
:param end_time: end time
:return:
"""
self.__init_client()
fetch_bin_size = allowed_range[bin_size][0]
left_time = start_time
right_time = end_time
data = to_data_frame([])
while True:
source = retry(lambda: self.public_client.Trade.Trade_getBucketed(
symbol=self.pair,
binSize=fetch_bin_size,
startTime=left_time,
endTime=right_time,
count=500,
partial=True).result())
if len(source) == 0:
break
source = to_data_frame(source)
data = pd.concat([data, source])
if right_time > source.iloc[-1].name + delta(fetch_bin_size):
left_time = source.iloc[-1].name + delta(fetch_bin_size)
time.sleep(2)
else:
break
return resample(data, bin_size)
def __update_ohlcv(self, action, new_data):
"""
데이터를 취득한 후, 전략을 실행
데이터가 없으면 서버와 접속해서 다운로드를 처음 함, 그 후는 어떻게 할까
"""
if self.data is None:
end_time = datetime.now(timezone.utc)
start_time = end_time - self.ohlcv_len * delta(self.bin_size)
d1 = self.fetch_ohlcv(self.bin_size, start_time, end_time)
if len(d1) > 0:
d2 = self.fetch_ohlcv(
allowed_range[self.bin_size][0],
d1.iloc[-1].name + delta(allowed_range[self.bin_size][0]),
end_time)
self.data = pd.concat([d1, d2], sort=True)
else:
self.data = d1
else:
self.data = pd.concat([self.data, new_data], sort=True)
# 마지막행은 불학정정보이기에 베제한다. (original)
# re_sample_data = resample(self.data, self.bin_size)[:-1]
# 마지막행도 반영한다. (by neo)
re_sample_data = resample(self.data, self.bin_size)[:]
if self.data.iloc[-1].name == re_sample_data.iloc[-1].name:
self.data = re_sample_data.iloc[-1 * self.ohlcv_len:, :]
if self.last_action_time is not None and \
self.last_action_time == re_sample_data.iloc[-1].name:
return
open = re_sample_data['open'].values
close = re_sample_data['close'].values
high = re_sample_data['high'].values
low = re_sample_data['low'].values
volume = re_sample_data['volume'].values
try:
if self.strategy is not None:
self.strategy(open, close, high, low, volume)
self.last_action_time = re_sample_data.iloc[-1].name
except FatalError as e:
# 致命的エラー
logger.error(f"Fatal error. {e}")
logger.error(traceback.format_exc())
notify(f"Fatal error occurred. Stopping Bot. {e}")
notify(traceback.format_exc())
self.stop()
except Exception as e:
logger.error(f"An error occurred. {e}")
logger.error(traceback.format_exc())
notify(f"An error occurred. {e}")
notify(traceback.format_exc())
def __update_ohlcv(self, action, new_data):
"""
get OHLCV data and execute the strategy
"""
if self.data is None:
end_time = datetime.now(timezone.utc)
start_time = end_time - self.ohlcv_len * delta(self.bin_size)
#logger.info(f"start time fetch ohlcv: {start_time}")
#logger.info(f"end time fetch ohlcv: {end_time}")
d1 = self.fetch_ohlcv(self.bin_size, start_time, end_time)
if len(d1) > 0:
d2 = self.fetch_ohlcv(
allowed_range[self.bin_size][0],
d1.iloc[-1].name + delta(allowed_range[self.bin_size][0]),
end_time)
self.data = pd.concat([d1, d2])
else:
self.data = d1
else:
self.data = pd.concat([self.data, new_data])
# exclude current candle data
re_sample_data = resample(self.data, self.bin_size)[:-1]
if self.data.iloc[-1].name == re_sample_data.iloc[-1].name:
self.data = re_sample_data.iloc[-1 * self.ohlcv_len:, :]
if self.last_action_time is not None and \
self.last_action_time == re_sample_data.iloc[-1].name:
return
open = re_sample_data['open'].values
close = re_sample_data['close'].values
high = re_sample_data['high'].values
low = re_sample_data['low'].values
volume = re_sample_data['volume'].values
try:
if self.strategy is not None:
self.strategy(open, close, high, low, volume)
self.last_action_time = re_sample_data.iloc[-1].name
except FatalError as e:
# Fatal error
logger.error(f"Fatal error. {e}")
logger.error(traceback.format_exc())
notify(f"Fatal error occurred. Stopping Bot. {e}")
notify(traceback.format_exc())
self.stop()
except Exception as e:
logger.error(f"An error occurred. {e}")
logger.error(traceback.format_exc())
notify(f"An error occurred. {e}")
notify(traceback.format_exc())
def fetch_ohlcv(self, bin_size, start_time, end_time):
"""
fetch OHLCV data
:param start_time: start time
:param end_time: end time
:return:
"""
self.__init_client()
fetch_bin_size = allowed_range[bin_size][0]
left_time = start_time
right_time = end_time
data = to_data_frame([])
while True:
if left_time > right_time:
break
logger.info(f"fetching OHLCV data")
left_time_to_timestamp = int(datetime.timestamp(left_time) * 1000)
right_time_to_timestamp = int(
datetime.timestamp(right_time) * 1000)
source = retry(lambda: self.client.futures_klines(
symbol=self.pair,
interval=fetch_bin_size,
startTime=left_time_to_timestamp,
endTime=right_time_to_timestamp,
limit=1500))
if len(source) == 0:
break
source_to_object_list = []
for s in source:
timestamp_to_datetime = datetime.fromtimestamp(
s[6] / 1000).astimezone(UTC)
source_to_object_list.append({
"timestamp": timestamp_to_datetime,
"high": float(s[2]),
"low": float(s[3]),
"open": float(s[1]),
"close": float(s[4]),
"volume": float(s[5])
})
source = to_data_frame(source_to_object_list)
data = pd.concat([data, source])
if right_time > source.iloc[-1].name + delta(fetch_bin_size):
left_time = source.iloc[-1].name + delta(fetch_bin_size)
time.sleep(2)
else:
break
return resample(data, bin_size)
def __update_ohlcv(self, action, new_data):
"""
データを取得して、戦略を実行する。
"""
if self.data is None:
end_time = datetime.now(timezone.utc)
start_time = end_time - self.ohlcv_len * delta(self.bin_size)
d1 = self.fetch_ohlcv(self.bin_size, start_time, end_time)
if len(d1) > 0:
d2 = self.fetch_ohlcv(allowed_range[self.bin_size][0],
d1.iloc[-1].name + delta(allowed_range[self.bin_size][0]), end_time)
self.data = pd.concat([d1, d2], sort=True)
else:
self.data = d1
else:
self.data = pd.concat([self.data, new_data], sort=True)
# 最後の行は不確定情報のため、排除する
re_sample_data = resample(self.data, self.bin_size)[:-1]
if self.data.iloc[-1].name == re_sample_data.iloc[-1].name:
self.data = re_sample_data.iloc[-1 * self.ohlcv_len:, :]
if self.last_action_time is not None and \
self.last_action_time == re_sample_data.iloc[-1].name:
return
open = re_sample_data['open'].values
close = re_sample_data['close'].values
high = re_sample_data['high'].values
low = re_sample_data['low'].values
volume = re_sample_data['volume'].values
try:
if self.strategy is not None:
self.strategy(open, close, high, low, volume)
self.last_action_time = re_sample_data.iloc[-1].name
except FatalError as e:
# 致命的エラー
logger.error(f"Fatal error. {e}")
logger.error(traceback.format_exc())
notify(f"Fatal error occurred. Stopping Bot. {e}")
notify(traceback.format_exc())
self.stop()
except Exception as e:
logger.error(f"An error occurred. {e}")
logger.error(traceback.format_exc())
notify(f"An error occurred. {e}")
notify(traceback.format_exc())
def autorelay_download_save(conn, table, bin_size, ohlcv_len, append):
cur = conn.cursor()
startd = time.time()
bitmex = BitMex(threading=True)
sql = str("select ROWID, * from '%s_%s' order by ROWID desc limit 1" % (table, bin_size))
cur.execute(sql)
item_list = cur.fetchall()
end_time = datetime.now(timezone.utc)
start_time = end_time - ohlcv_len * delta(bin_size)
# 데이터가 있으면 릴레이 다운로드 자동으로 기간설정
if item_list:
lasttime = item_list[0][1]
# print(lasttime)
# print(lasttime[0:16])
last_time = datetime.strptime(lasttime[0:16], '%Y-%m-%d %H:%M')
start_time = last_time + timedelta(minutes=1)
else:
# 데이터가 없으면,
print('No data and will start from ', start_time)
pass
# print('start_time:', start_time)
# print('end_time:', end_time)
# relay download 디폴트 기간 설정만큼 다운로드 한다.
df = bitmex.fetch_ohlcv(bin_size, start_time, end_time)
print('download_df_ohlcv time: ', time.time() - startd)
# insert to database
df.to_sql(table+'_'+bin_size, conn, if_exists=append)
show_table(conn, table, bin_size, 1)
def __update_ohlcv(self, action, new_data):
new_data = new_data.rename(index={new_data.iloc[0].name: new_data.iloc[0].name.ceil(freq="1T")})
"""
get OHLCV data and execute the strategy
"""
if self.data is None:
end_time = datetime.now(timezone.utc)
start_time = end_time - self.ohlcv_len * delta(self.bin_size)
# logger.info(f"start time fetch ohlcv: {start_time}")
# logger.info(f"end time fetch ohlcv: {end_time}")
self.data = self.fetch_ohlcv(self.bin_size, start_time, end_time)
if self.data.iloc[-1].name > end_time:
last_candle = self.data.iloc[-1].values
self.data = self.data[:-1]
self.data.loc[end_time.replace(microsecond=0)] = last_candle
logger.info(f"Initial Buffer Fill - Last Candle: {self.data.iloc[-1].name}")
else:
if self.data.iloc[-1].name == new_data.iloc[0].name:
self.data = pd.concat([self.data[:-1], new_data])
else:
self.data = pd.concat([self.data, new_data])
# exclude current candle data
re_sample_data = resample(self.data, self.bin_size)[:-1]
if self.last_action_time is not None and self.last_action_time == re_sample_data.iloc[-1].name:
return
self.data = pd.concat([re_sample_data.iloc[-1 * self.ohlcv_len :, :], self.data.iloc[[-1]]])
open = re_sample_data["open"].values
close = re_sample_data["close"].values
high = re_sample_data["high"].values
low = re_sample_data["low"].values
volume = re_sample_data["volume"].values
try:
if self.strategy is not None:
self.timestamp = re_sample_data.iloc[-1].name.isoformat()
self.strategy(open, close, high, low, volume)
self.eval_exit()
self.last_action_time = re_sample_data.iloc[-1].name
except FatalError as e:
# Fatal error
logger.error(f"Fatal error. {e}")
logger.error(traceback.format_exc())
notify(f"Fatal error occurred. Stopping Bot. {e}")
notify(traceback.format_exc())
self.stop()
except Exception as e:
logger.error(f"An error occurred. {e}")
logger.error(traceback.format_exc())
notify(f"An error occurred. {e}")
notify(traceback.format_exc())
def test_fetch_ohlcv_11m(self):
ohlcv_len = 100
bin_size = '11m'
bitmex = BitMex(threading=False)
end_time = datetime.now(timezone.utc)
start_time = end_time - ohlcv_len * delta(bin_size)
d1 = bitmex.fetch_ohlcv(bin_size, start_time, end_time)
print(f"{d1}")
def download_data(self, bin_size, start_time, end_time):
"""
download or get the data and set variables related to ohlcv data
"""
data = pd.DataFrame()
left_time = None
source = None
is_last_fetch = False
if self.minute_granularity == True:
#self.timeframe = bin_size.add('1m') # add 1m timeframe to the set (sets wont allow duplicates) in case we need minute granularity
bin_size = '1m'
minute_gran = True
else:
bin_size = bin_size[0]
while True:
if left_time is None:
left_time = start_time
right_time = left_time + delta(allowed_range[bin_size][0]) * 99
else:
left_time = source.iloc[
-1].name #+ delta(allowed_range[bin_size][0]) * allowed_range[bin_size][2]
right_time = left_time + delta(allowed_range[bin_size][0]) * 99
if right_time > end_time:
right_time = end_time
is_last_fetch = True
source = self.fetch_ohlcv(bin_size=bin_size,
start_time=left_time,
end_time=right_time)
# if(data.shape[0]):
# logger.info(f"Last: {data.iloc[-1].name} Left: {left_time} Start: {source.iloc[0].name} Right: {right_time} End: {source.iloc[-1].name}")
data = pd.concat([data, source])
if is_last_fetch:
return data
time.sleep(0.25)
def download_data(self, bin_size, start_time, end_time):
"""
download or get the data
"""
data = pd.DataFrame()
left_time = None
source = None
is_last_fetch = False
while True:
if left_time is None:
left_time = start_time
right_time = left_time + delta(allowed_range[bin_size][0]) * 99
else:
left_time = source.iloc[
-1].name # + + delta(allowed_range[bin_size][0]) * allowed_range[bin_size][2]
right_time = left_time + delta(allowed_range[bin_size][0]) * 99
if right_time > end_time:
right_time = end_time
is_last_fetch = True
source = self.fetch_ohlcv(bin_size=bin_size,
start_time=left_time,
end_time=right_time)
# if(data.shape[0]):
# logger.info(f"Last: {data.iloc[-1].name} Left: {left_time} Start: {source.iloc[0].name} Right: {right_time} End: {source.iloc[-1].name}")
data = pd.concat([data, source])
if is_last_fetch:
return data
time.sleep(0.25)
def download_df_ohlcv(bin_size, ohlcv_len, **kwargs):
'''
df => DataFrame Type
------------------------------------------------------------------
open high low close volume
2019-06-15 14:29:00+00:00 8669.5 8670.0 8667.0 8667.0 1454667
2019-06-15 14:30:00+00:00 8667.0 8667.5 8667.0 8667.5 424940
:return:
'''
print('download data from server')
start = time.time()
# bitmex = BitMex(threading=False)
bitmex = BitMex(threading=True)
end_time = datetime.now(timezone.utc)
start_time = end_time - ohlcv_len * delta(bin_size)
df = bitmex.fetch_ohlcv(bin_size, start_time, end_time)
print('download_df_ohlcv time:', time.time() - start)
return df
def __update_ohlcv(self, action, new_data):
"""
get and update OHLCV data and execute the strategy
"""
# Binance can output wierd timestamps - Eg. 2021-05-25 16:04:59.999000+00:00
# We need to round up to the nearest second for further processing
new_data = new_data.rename(index={new_data.iloc[0].name: new_data.iloc[0].name.ceil(freq='1T')})
if self.timeframe_data is None:
self.timeframe_data = {}
for t in self.bin_size:
bin_size = t
end_time = datetime.now(timezone.utc)
start_time = end_time - self.ohlcv_len * delta(bin_size)
self.timeframe_data[bin_size] = self.fetch_ohlcv(bin_size, start_time, end_time)
self.timeframe_info[bin_size] = {
"allowed_range": allowed_range_minute_granularity[t][0] if self.minute_granularity else allowed_range[t][0],
"ohlcv": self.timeframe_data[t][:-1], # Dataframe with closed candles
"last_action_time": None,#self.timeframe_data[bin_size].iloc[-1].name, # Last strategy execution time
"last_candle": self.timeframe_data[bin_size].iloc[-2].values, # Store last complete candle
"partial_candle": self.timeframe_data[bin_size].iloc[-1].values # Store incomplete candle
}
# The last candle is an incomplete candle with timestamp in future
if self.timeframe_data[bin_size].iloc[-1].name > end_time:
last_candle = self.timeframe_data[t].iloc[-1].values # Store last candle
self.timeframe_data[bin_size] = self.timeframe_data[t][:-1] # Exclude last candle
self.timeframe_data[bin_size].loc[end_time.replace(microsecond=0)] = last_candle #set last candle to end_time
logger.info(f"Initial Buffer Fill - Last Candle: {self.timeframe_data[bin_size].iloc[-1].name}")
#logger.info(f"{self.timeframe_data}")
timeframes_to_update = []
for t in self.timeframe_info:
if self.timeframe_info[t]["allowed_range"] == action:
# append minute count of a timeframe when sorting when sorting is need otherwise just add a string timeframe
timeframes_to_update.append(allowed_range_minute_granularity[t][3]) if self.timeframes_sorted != None else timeframes_to_update.append(t)
# Sorting timeframes that will be updated
if self.timeframes_sorted == True:
timeframes_to_update.sort(reverse=True)
if self.timeframes_sorted == False:
timeframes_to_update.sort(reverse=False)
#logger.info(f"timefeames to update: {timeframes_to_update}")
for t in timeframes_to_update:
# Find timeframe string based on its minute count value
if self.timeframes_sorted != None:
t = find_timeframe_string(t)
# replace latest candle if timestamp is same or append
if self.timeframe_data[t].iloc[-1].name == new_data.iloc[0].name:
self.timeframe_data[t] = pd.concat([self.timeframe_data[t][:-1], new_data])
else:
self.timeframe_data[t] = pd.concat([self.timeframe_data[t], new_data])
# exclude current candle data and store partial candle data
re_sample_data = resample(self.timeframe_data[t], t, minute_granularity=True if self.minute_granularity else False)
self.timeframe_info[t]['partial_candle'] = re_sample_data.iloc[-1].values # store partial candle data
re_sample_data =re_sample_data[:-1] # exclude current candle data
#logger.info(f"{self.timeframe_info[t]['last_action_time']} : {self.timeframe_data[t].iloc[-1].name} : {re_sample_data.iloc[-1].name}")
if self.timeframe_info[t]["last_action_time"] is None:
self.timeframe_info[t]["last_action_time"] = re_sample_data.iloc[-1].name
if self.timeframe_info[t]["last_action_time"] == re_sample_data.iloc[-1].name:
continue
# The last candle in the buffer needs to be preserved
# while resetting the buffer as it may be incomlete
# or contains latest data from WS
self.timeframe_data[t] = pd.concat([re_sample_data.iloc[-1 * self.ohlcv_len:, :], self.timeframe_data[t].iloc[[-1]]])
#store ohlcv dataframe to timeframe_info dictionary
self.timeframe_info[t]["ohlcv"] = re_sample_data
#logger.info(f"Buffer Right Edge: {self.data.iloc[-1]}")
open = re_sample_data['open'].values
close = re_sample_data['close'].values
high = re_sample_data['high'].values
low = re_sample_data['low'].values
volume = re_sample_data['volume'].values
try:
if self.strategy is not None:
self.timestamp = re_sample_data.iloc[-1].name.isoformat()
self.strategy(t, open, close, high, low, volume)
self.timeframe_info[t]['last_action_time'] = re_sample_data.iloc[-1].name
except FatalError as e:
# Fatal error
logger.error(f"Fatal error. {e}")
logger.error(traceback.format_exc())
notify(f"Fatal error occurred. Stopping Bot. {e}")
notify(traceback.format_exc())
self.stop()
except Exception as e:
logger.error(f"An error occurred. {e}")
logger.error(traceback.format_exc())
notify(f"An error occurred. {e}")
notify(traceback.format_exc())
def __update_ohlcv(self, action, new_data):
# Binance can output wierd timestamps - Eg. 2021-05-25 16:04:59.999000+00:00
# We need to round up to the nearest second for further processing
new_data = new_data.rename(index={new_data.iloc[0].name: new_data.iloc[0].name.ceil(freq='1T')})
"""
get OHLCV data and execute the strategy
"""
if self.data is None:
end_time = datetime.now(timezone.utc)
start_time = end_time - self.ohlcv_len * delta(self.bin_size)
#logger.info(f"start time fetch ohlcv: {start_time}")
#logger.info(f"end time fetch ohlcv: {end_time}")
self.data = self.fetch_ohlcv(self.bin_size, start_time, end_time)
# The last candle is an incomplete candle with timestamp
# in future
if(self.data.iloc[-1].name > end_time):
last_candle = self.data.iloc[-1].values # Store last candle
self.data = self.data[:-1] # exclude last candle
self.data.loc[end_time.replace(microsecond=0)] = last_candle #set last candle to end_time
logger.info(f"Initial Buffer Fill - Last Candle: {self.data.iloc[-1].name}")
else:
#replace latest candle if timestamp is same or append
if(self.data.iloc[-1].name == new_data.iloc[0].name):
self.data = pd.concat([self.data[:-1], new_data])
else:
self.data = pd.concat([self.data, new_data])
# exclude current candle data
re_sample_data = resample(self.data, self.bin_size)[:-1]
# logger.info(f"{self.last_action_time} : {self.data.iloc[-1].name} : {re_sample_data.iloc[-1].name}")
if self.last_action_time is not None and \
self.last_action_time == re_sample_data.iloc[-1].name:
return
# The last candle in the buffer needs to be preserved
# while resetting the buffer as it may be incomlete
# or contains latest data from WS
self.data = pd.concat([re_sample_data.iloc[-1 * self.ohlcv_len:, :], self.data.iloc[[-1]]])
#logger.info(f"Buffer Right Edge: {self.data.iloc[-1]}")
open = re_sample_data['open'].values
close = re_sample_data['close'].values
high = re_sample_data['high'].values
low = re_sample_data['low'].values
volume = re_sample_data['volume'].values
try:
if self.strategy is not None:
self.timestamp = re_sample_data.iloc[-1].name.isoformat()
self.strategy(open, close, high, low, volume)
self.last_action_time = re_sample_data.iloc[-1].name
except FatalError as e:
# Fatal error
logger.error(f"Fatal error. {e}")
logger.error(traceback.format_exc())
notify(f"Fatal error occurred. Stopping Bot. {e}")
notify(traceback.format_exc())
self.stop()
except Exception as e:
logger.error(f"An error occurred. {e}")
logger.error(traceback.format_exc())
notify(f"An error occurred. {e}")
notify(traceback.format_exc())
