def download_data(self, file, bin_size, start_time, end_time): """ データをサーバーから取得する。 """ if not os.path.exists(os.path.dirname(file)): os.makedirs(os.path.dirname(file)) data = pd.DataFrame() left_time = None source = None is_last_fetch = False while True: if left_time is None: left_time = start_time right_time = left_time + delta(allowed_range[bin_size][0]) * 99 else: left_time = source.iloc[-1].name + + delta(allowed_range[bin_size][0]) * allowed_range[bin_size][2] right_time = left_time + delta(allowed_range[bin_size][0]) * 99 if right_time > end_time: right_time = end_time is_last_fetch = True source = self.fetch_ohlcv(bin_size=bin_size, start_time=left_time, end_time=right_time) data = pd.concat([data, source]) if is_last_fetch: data.to_csv(file) break time.sleep(2)
def fetch_ohlcv(self, bin_size, start_time, end_time): """ fetch OHLCV data :param start_time: start time :param end_time: end time :return: """ self.__init_client() fetch_bin_size = allowed_range[bin_size][0] left_time = start_time right_time = end_time data = to_data_frame([]) while True: source = retry(lambda: self.public_client.Trade.Trade_getBucketed( symbol=self.pair, binSize=fetch_bin_size, startTime=left_time, endTime=right_time, count=500, partial=True).result()) if len(source) == 0: break source = to_data_frame(source) data = pd.concat([data, source]) if right_time > source.iloc[-1].name + delta(fetch_bin_size): left_time = source.iloc[-1].name + delta(fetch_bin_size) time.sleep(2) else: break return resample(data, bin_size)
def __update_ohlcv(self, action, new_data): """ 데이터를 취득한 후, 전략을 실행 데이터가 없으면 서버와 접속해서 다운로드를 처음 함, 그 후는 어떻게 할까 """ if self.data is None: end_time = datetime.now(timezone.utc) start_time = end_time - self.ohlcv_len * delta(self.bin_size) d1 = self.fetch_ohlcv(self.bin_size, start_time, end_time) if len(d1) > 0: d2 = self.fetch_ohlcv( allowed_range[self.bin_size][0], d1.iloc[-1].name + delta(allowed_range[self.bin_size][0]), end_time) self.data = pd.concat([d1, d2], sort=True) else: self.data = d1 else: self.data = pd.concat([self.data, new_data], sort=True) # 마지막행은 불학정정보이기에 베제한다. (original) # re_sample_data = resample(self.data, self.bin_size)[:-1] # 마지막행도 반영한다. (by neo) re_sample_data = resample(self.data, self.bin_size)[:] if self.data.iloc[-1].name == re_sample_data.iloc[-1].name: self.data = re_sample_data.iloc[-1 * self.ohlcv_len:, :] if self.last_action_time is not None and \ self.last_action_time == re_sample_data.iloc[-1].name: return open = re_sample_data['open'].values close = re_sample_data['close'].values high = re_sample_data['high'].values low = re_sample_data['low'].values volume = re_sample_data['volume'].values try: if self.strategy is not None: self.strategy(open, close, high, low, volume) self.last_action_time = re_sample_data.iloc[-1].name except FatalError as e: # 致命的エラー logger.error(f"Fatal error. {e}") logger.error(traceback.format_exc()) notify(f"Fatal error occurred. Stopping Bot. {e}") notify(traceback.format_exc()) self.stop() except Exception as e: logger.error(f"An error occurred. {e}") logger.error(traceback.format_exc()) notify(f"An error occurred. {e}") notify(traceback.format_exc())
def __update_ohlcv(self, action, new_data): """ get OHLCV data and execute the strategy """ if self.data is None: end_time = datetime.now(timezone.utc) start_time = end_time - self.ohlcv_len * delta(self.bin_size) #logger.info(f"start time fetch ohlcv: {start_time}") #logger.info(f"end time fetch ohlcv: {end_time}") d1 = self.fetch_ohlcv(self.bin_size, start_time, end_time) if len(d1) > 0: d2 = self.fetch_ohlcv( allowed_range[self.bin_size][0], d1.iloc[-1].name + delta(allowed_range[self.bin_size][0]), end_time) self.data = pd.concat([d1, d2]) else: self.data = d1 else: self.data = pd.concat([self.data, new_data]) # exclude current candle data re_sample_data = resample(self.data, self.bin_size)[:-1] if self.data.iloc[-1].name == re_sample_data.iloc[-1].name: self.data = re_sample_data.iloc[-1 * self.ohlcv_len:, :] if self.last_action_time is not None and \ self.last_action_time == re_sample_data.iloc[-1].name: return open = re_sample_data['open'].values close = re_sample_data['close'].values high = re_sample_data['high'].values low = re_sample_data['low'].values volume = re_sample_data['volume'].values try: if self.strategy is not None: self.strategy(open, close, high, low, volume) self.last_action_time = re_sample_data.iloc[-1].name except FatalError as e: # Fatal error logger.error(f"Fatal error. {e}") logger.error(traceback.format_exc()) notify(f"Fatal error occurred. Stopping Bot. {e}") notify(traceback.format_exc()) self.stop() except Exception as e: logger.error(f"An error occurred. {e}") logger.error(traceback.format_exc()) notify(f"An error occurred. {e}") notify(traceback.format_exc())
def fetch_ohlcv(self, bin_size, start_time, end_time): """ fetch OHLCV data :param start_time: start time :param end_time: end time :return: """ self.__init_client() fetch_bin_size = allowed_range[bin_size][0] left_time = start_time right_time = end_time data = to_data_frame([]) while True: if left_time > right_time: break logger.info(f"fetching OHLCV data") left_time_to_timestamp = int(datetime.timestamp(left_time) * 1000) right_time_to_timestamp = int( datetime.timestamp(right_time) * 1000) source = retry(lambda: self.client.futures_klines( symbol=self.pair, interval=fetch_bin_size, startTime=left_time_to_timestamp, endTime=right_time_to_timestamp, limit=1500)) if len(source) == 0: break source_to_object_list = [] for s in source: timestamp_to_datetime = datetime.fromtimestamp( s[6] / 1000).astimezone(UTC) source_to_object_list.append({ "timestamp": timestamp_to_datetime, "high": float(s[2]), "low": float(s[3]), "open": float(s[1]), "close": float(s[4]), "volume": float(s[5]) }) source = to_data_frame(source_to_object_list) data = pd.concat([data, source]) if right_time > source.iloc[-1].name + delta(fetch_bin_size): left_time = source.iloc[-1].name + delta(fetch_bin_size) time.sleep(2) else: break return resample(data, bin_size)
def __update_ohlcv(self, action, new_data): """ データを取得して、戦略を実行する。 """ if self.data is None: end_time = datetime.now(timezone.utc) start_time = end_time - self.ohlcv_len * delta(self.bin_size) d1 = self.fetch_ohlcv(self.bin_size, start_time, end_time) if len(d1) > 0: d2 = self.fetch_ohlcv(allowed_range[self.bin_size][0], d1.iloc[-1].name + delta(allowed_range[self.bin_size][0]), end_time) self.data = pd.concat([d1, d2], sort=True) else: self.data = d1 else: self.data = pd.concat([self.data, new_data], sort=True) # 最後の行は不確定情報のため、排除する re_sample_data = resample(self.data, self.bin_size)[:-1] if self.data.iloc[-1].name == re_sample_data.iloc[-1].name: self.data = re_sample_data.iloc[-1 * self.ohlcv_len:, :] if self.last_action_time is not None and \ self.last_action_time == re_sample_data.iloc[-1].name: return open = re_sample_data['open'].values close = re_sample_data['close'].values high = re_sample_data['high'].values low = re_sample_data['low'].values volume = re_sample_data['volume'].values try: if self.strategy is not None: self.strategy(open, close, high, low, volume) self.last_action_time = re_sample_data.iloc[-1].name except FatalError as e: # 致命的エラー logger.error(f"Fatal error. {e}") logger.error(traceback.format_exc()) notify(f"Fatal error occurred. Stopping Bot. {e}") notify(traceback.format_exc()) self.stop() except Exception as e: logger.error(f"An error occurred. {e}") logger.error(traceback.format_exc()) notify(f"An error occurred. {e}") notify(traceback.format_exc())
def autorelay_download_save(conn, table, bin_size, ohlcv_len, append): cur = conn.cursor() startd = time.time() bitmex = BitMex(threading=True) sql = str("select ROWID, * from '%s_%s' order by ROWID desc limit 1" % (table, bin_size)) cur.execute(sql) item_list = cur.fetchall() end_time = datetime.now(timezone.utc) start_time = end_time - ohlcv_len * delta(bin_size) # 데이터가 있으면 릴레이 다운로드 자동으로 기간설정 if item_list: lasttime = item_list[0][1] # print(lasttime) # print(lasttime[0:16]) last_time = datetime.strptime(lasttime[0:16], '%Y-%m-%d %H:%M') start_time = last_time + timedelta(minutes=1) else: # 데이터가 없으면, print('No data and will start from ', start_time) pass # print('start_time:', start_time) # print('end_time:', end_time) # relay download 디폴트 기간 설정만큼 다운로드 한다. df = bitmex.fetch_ohlcv(bin_size, start_time, end_time) print('download_df_ohlcv time: ', time.time() - startd) # insert to database df.to_sql(table+'_'+bin_size, conn, if_exists=append) show_table(conn, table, bin_size, 1)
def __update_ohlcv(self, action, new_data): new_data = new_data.rename(index={new_data.iloc[0].name: new_data.iloc[0].name.ceil(freq="1T")}) """ get OHLCV data and execute the strategy """ if self.data is None: end_time = datetime.now(timezone.utc) start_time = end_time - self.ohlcv_len * delta(self.bin_size) # logger.info(f"start time fetch ohlcv: {start_time}") # logger.info(f"end time fetch ohlcv: {end_time}") self.data = self.fetch_ohlcv(self.bin_size, start_time, end_time) if self.data.iloc[-1].name > end_time: last_candle = self.data.iloc[-1].values self.data = self.data[:-1] self.data.loc[end_time.replace(microsecond=0)] = last_candle logger.info(f"Initial Buffer Fill - Last Candle: {self.data.iloc[-1].name}") else: if self.data.iloc[-1].name == new_data.iloc[0].name: self.data = pd.concat([self.data[:-1], new_data]) else: self.data = pd.concat([self.data, new_data]) # exclude current candle data re_sample_data = resample(self.data, self.bin_size)[:-1] if self.last_action_time is not None and self.last_action_time == re_sample_data.iloc[-1].name: return self.data = pd.concat([re_sample_data.iloc[-1 * self.ohlcv_len :, :], self.data.iloc[[-1]]]) open = re_sample_data["open"].values close = re_sample_data["close"].values high = re_sample_data["high"].values low = re_sample_data["low"].values volume = re_sample_data["volume"].values try: if self.strategy is not None: self.timestamp = re_sample_data.iloc[-1].name.isoformat() self.strategy(open, close, high, low, volume) self.eval_exit() self.last_action_time = re_sample_data.iloc[-1].name except FatalError as e: # Fatal error logger.error(f"Fatal error. {e}") logger.error(traceback.format_exc()) notify(f"Fatal error occurred. Stopping Bot. {e}") notify(traceback.format_exc()) self.stop() except Exception as e: logger.error(f"An error occurred. {e}") logger.error(traceback.format_exc()) notify(f"An error occurred. {e}") notify(traceback.format_exc())
def test_fetch_ohlcv_11m(self): ohlcv_len = 100 bin_size = '11m' bitmex = BitMex(threading=False) end_time = datetime.now(timezone.utc) start_time = end_time - ohlcv_len * delta(bin_size) d1 = bitmex.fetch_ohlcv(bin_size, start_time, end_time) print(f"{d1}")
def download_data(self, bin_size, start_time, end_time): """ download or get the data and set variables related to ohlcv data """ data = pd.DataFrame() left_time = None source = None is_last_fetch = False if self.minute_granularity == True: #self.timeframe = bin_size.add('1m') # add 1m timeframe to the set (sets wont allow duplicates) in case we need minute granularity bin_size = '1m' minute_gran = True else: bin_size = bin_size[0] while True: if left_time is None: left_time = start_time right_time = left_time + delta(allowed_range[bin_size][0]) * 99 else: left_time = source.iloc[ -1].name #+ delta(allowed_range[bin_size][0]) * allowed_range[bin_size][2] right_time = left_time + delta(allowed_range[bin_size][0]) * 99 if right_time > end_time: right_time = end_time is_last_fetch = True source = self.fetch_ohlcv(bin_size=bin_size, start_time=left_time, end_time=right_time) # if(data.shape[0]): # logger.info(f"Last: {data.iloc[-1].name} Left: {left_time} Start: {source.iloc[0].name} Right: {right_time} End: {source.iloc[-1].name}") data = pd.concat([data, source]) if is_last_fetch: return data time.sleep(0.25)
def download_data(self, bin_size, start_time, end_time): """ download or get the data """ data = pd.DataFrame() left_time = None source = None is_last_fetch = False while True: if left_time is None: left_time = start_time right_time = left_time + delta(allowed_range[bin_size][0]) * 99 else: left_time = source.iloc[ -1].name # + + delta(allowed_range[bin_size][0]) * allowed_range[bin_size][2] right_time = left_time + delta(allowed_range[bin_size][0]) * 99 if right_time > end_time: right_time = end_time is_last_fetch = True source = self.fetch_ohlcv(bin_size=bin_size, start_time=left_time, end_time=right_time) # if(data.shape[0]): # logger.info(f"Last: {data.iloc[-1].name} Left: {left_time} Start: {source.iloc[0].name} Right: {right_time} End: {source.iloc[-1].name}") data = pd.concat([data, source]) if is_last_fetch: return data time.sleep(0.25)
def download_df_ohlcv(bin_size, ohlcv_len, **kwargs): ''' df => DataFrame Type ------------------------------------------------------------------ open high low close volume 2019-06-15 14:29:00+00:00 8669.5 8670.0 8667.0 8667.0 1454667 2019-06-15 14:30:00+00:00 8667.0 8667.5 8667.0 8667.5 424940 :return: ''' print('download data from server') start = time.time() # bitmex = BitMex(threading=False) bitmex = BitMex(threading=True) end_time = datetime.now(timezone.utc) start_time = end_time - ohlcv_len * delta(bin_size) df = bitmex.fetch_ohlcv(bin_size, start_time, end_time) print('download_df_ohlcv time:', time.time() - start) return df
def __update_ohlcv(self, action, new_data): """ get and update OHLCV data and execute the strategy """ # Binance can output wierd timestamps - Eg. 2021-05-25 16:04:59.999000+00:00 # We need to round up to the nearest second for further processing new_data = new_data.rename(index={new_data.iloc[0].name: new_data.iloc[0].name.ceil(freq='1T')}) if self.timeframe_data is None: self.timeframe_data = {} for t in self.bin_size: bin_size = t end_time = datetime.now(timezone.utc) start_time = end_time - self.ohlcv_len * delta(bin_size) self.timeframe_data[bin_size] = self.fetch_ohlcv(bin_size, start_time, end_time) self.timeframe_info[bin_size] = { "allowed_range": allowed_range_minute_granularity[t][0] if self.minute_granularity else allowed_range[t][0], "ohlcv": self.timeframe_data[t][:-1], # Dataframe with closed candles "last_action_time": None,#self.timeframe_data[bin_size].iloc[-1].name, # Last strategy execution time "last_candle": self.timeframe_data[bin_size].iloc[-2].values, # Store last complete candle "partial_candle": self.timeframe_data[bin_size].iloc[-1].values # Store incomplete candle } # The last candle is an incomplete candle with timestamp in future if self.timeframe_data[bin_size].iloc[-1].name > end_time: last_candle = self.timeframe_data[t].iloc[-1].values # Store last candle self.timeframe_data[bin_size] = self.timeframe_data[t][:-1] # Exclude last candle self.timeframe_data[bin_size].loc[end_time.replace(microsecond=0)] = last_candle #set last candle to end_time logger.info(f"Initial Buffer Fill - Last Candle: {self.timeframe_data[bin_size].iloc[-1].name}") #logger.info(f"{self.timeframe_data}") timeframes_to_update = [] for t in self.timeframe_info: if self.timeframe_info[t]["allowed_range"] == action: # append minute count of a timeframe when sorting when sorting is need otherwise just add a string timeframe timeframes_to_update.append(allowed_range_minute_granularity[t][3]) if self.timeframes_sorted != None else timeframes_to_update.append(t) # Sorting timeframes that will be updated if self.timeframes_sorted == True: timeframes_to_update.sort(reverse=True) if self.timeframes_sorted == False: timeframes_to_update.sort(reverse=False) #logger.info(f"timefeames to update: {timeframes_to_update}") for t in timeframes_to_update: # Find timeframe string based on its minute count value if self.timeframes_sorted != None: t = find_timeframe_string(t) # replace latest candle if timestamp is same or append if self.timeframe_data[t].iloc[-1].name == new_data.iloc[0].name: self.timeframe_data[t] = pd.concat([self.timeframe_data[t][:-1], new_data]) else: self.timeframe_data[t] = pd.concat([self.timeframe_data[t], new_data]) # exclude current candle data and store partial candle data re_sample_data = resample(self.timeframe_data[t], t, minute_granularity=True if self.minute_granularity else False) self.timeframe_info[t]['partial_candle'] = re_sample_data.iloc[-1].values # store partial candle data re_sample_data =re_sample_data[:-1] # exclude current candle data #logger.info(f"{self.timeframe_info[t]['last_action_time']} : {self.timeframe_data[t].iloc[-1].name} : {re_sample_data.iloc[-1].name}") if self.timeframe_info[t]["last_action_time"] is None: self.timeframe_info[t]["last_action_time"] = re_sample_data.iloc[-1].name if self.timeframe_info[t]["last_action_time"] == re_sample_data.iloc[-1].name: continue # The last candle in the buffer needs to be preserved # while resetting the buffer as it may be incomlete # or contains latest data from WS self.timeframe_data[t] = pd.concat([re_sample_data.iloc[-1 * self.ohlcv_len:, :], self.timeframe_data[t].iloc[[-1]]]) #store ohlcv dataframe to timeframe_info dictionary self.timeframe_info[t]["ohlcv"] = re_sample_data #logger.info(f"Buffer Right Edge: {self.data.iloc[-1]}") open = re_sample_data['open'].values close = re_sample_data['close'].values high = re_sample_data['high'].values low = re_sample_data['low'].values volume = re_sample_data['volume'].values try: if self.strategy is not None: self.timestamp = re_sample_data.iloc[-1].name.isoformat() self.strategy(t, open, close, high, low, volume) self.timeframe_info[t]['last_action_time'] = re_sample_data.iloc[-1].name except FatalError as e: # Fatal error logger.error(f"Fatal error. {e}") logger.error(traceback.format_exc()) notify(f"Fatal error occurred. Stopping Bot. {e}") notify(traceback.format_exc()) self.stop() except Exception as e: logger.error(f"An error occurred. {e}") logger.error(traceback.format_exc()) notify(f"An error occurred. {e}") notify(traceback.format_exc())
def __update_ohlcv(self, action, new_data): # Binance can output wierd timestamps - Eg. 2021-05-25 16:04:59.999000+00:00 # We need to round up to the nearest second for further processing new_data = new_data.rename(index={new_data.iloc[0].name: new_data.iloc[0].name.ceil(freq='1T')}) """ get OHLCV data and execute the strategy """ if self.data is None: end_time = datetime.now(timezone.utc) start_time = end_time - self.ohlcv_len * delta(self.bin_size) #logger.info(f"start time fetch ohlcv: {start_time}") #logger.info(f"end time fetch ohlcv: {end_time}") self.data = self.fetch_ohlcv(self.bin_size, start_time, end_time) # The last candle is an incomplete candle with timestamp # in future if(self.data.iloc[-1].name > end_time): last_candle = self.data.iloc[-1].values # Store last candle self.data = self.data[:-1] # exclude last candle self.data.loc[end_time.replace(microsecond=0)] = last_candle #set last candle to end_time logger.info(f"Initial Buffer Fill - Last Candle: {self.data.iloc[-1].name}") else: #replace latest candle if timestamp is same or append if(self.data.iloc[-1].name == new_data.iloc[0].name): self.data = pd.concat([self.data[:-1], new_data]) else: self.data = pd.concat([self.data, new_data]) # exclude current candle data re_sample_data = resample(self.data, self.bin_size)[:-1] # logger.info(f"{self.last_action_time} : {self.data.iloc[-1].name} : {re_sample_data.iloc[-1].name}") if self.last_action_time is not None and \ self.last_action_time == re_sample_data.iloc[-1].name: return # The last candle in the buffer needs to be preserved # while resetting the buffer as it may be incomlete # or contains latest data from WS self.data = pd.concat([re_sample_data.iloc[-1 * self.ohlcv_len:, :], self.data.iloc[[-1]]]) #logger.info(f"Buffer Right Edge: {self.data.iloc[-1]}") open = re_sample_data['open'].values close = re_sample_data['close'].values high = re_sample_data['high'].values low = re_sample_data['low'].values volume = re_sample_data['volume'].values try: if self.strategy is not None: self.timestamp = re_sample_data.iloc[-1].name.isoformat() self.strategy(open, close, high, low, volume) self.last_action_time = re_sample_data.iloc[-1].name except FatalError as e: # Fatal error logger.error(f"Fatal error. {e}") logger.error(traceback.format_exc()) notify(f"Fatal error occurred. Stopping Bot. {e}") notify(traceback.format_exc()) self.stop() except Exception as e: logger.error(f"An error occurred. {e}") logger.error(traceback.format_exc()) notify(f"An error occurred. {e}") notify(traceback.format_exc())