def __get_klines_cache(self, symbol, interval, s_time, e_time): """ 获取k线缓存 """ if (interval not in self.klines_cache or s_time != self.klines_cache[interval]["s_time"] or e_time != self.klines_cache[interval]["e_time"]): cache = {} cache["data"] = self.__get_klines( xq.get_kline_collection(symbol, interval), s_time, e_time) cache["s_time"] = s_time cache["e_time"] = e_time self.klines_cache[interval] = cache return self.klines_cache[interval]["data"]
from setup import * import pandas as pd from importer import add_common_arguments, split_time_range if __name__ == "__main__": parser = add_common_arguments('Binance Importer') args = parser.parse_args() # print(args) if not (args.s and args.k and args.m): parser.print_help() exit(1) symbol = args.s interval = timedelta(seconds=xq.get_interval_seconds(args.k)) collection = xq.get_kline_collection(symbol, args.k) #print("collection: ", collection) db = get_mongodb(args.m) db.ensure_index(collection, [("open_time",1)], unique=True) if args.r: start_time, end_time = split_time_range(args.r) else: # 续接db中最后一条记录,至今天之前 klines = db.find_sort(collection, {}, 'open_time', -1, 1) if len(klines) > 0: start_time = (datetime.fromtimestamp(klines[0]["open_time"]/1000) + interval) else: start_time = None end_time = datetime.now()
def download_from_exchange(exchange, db, symbol, kline_type, time_range): print('%12s %6s ' % (' ', kline_type), end='') collection = xq.get_kline_collection(symbol, kline_type) db.ensure_index(collection, [("open_time", 1)], unique=True) interval = timedelta(seconds=xq.get_interval_seconds(kline_type)) if time_range: start_time, end_time = split_time_range(time_range) else: # 续接db中最后一条记录,至今天之前 klines = db.find_sort(collection, {}, 'open_time', -1, 1) if len(klines) > 0: start_time = ( datetime.fromtimestamp(klines[0]["open_time"] / 1000) + interval) else: start_time = exchange.start_time end_time = datetime.now() #print(xq.get_open_time(kline_type, end_time)) """ if start_time.hour != exchange.start_time.hour: print("open time(%s) hour error! %s open time hour: %s" % (start_time, exchange.name, exchange.start_time.hour)) exit(1) if end_time.hour < exchange.start_time.hour: end_time -= timedelta(days=1) end_time = end_time.replace(hour=exchange.start_time.hour, minute=0, second=0, microsecond=0) """ end_time = end_time.replace(minute=0, second=0, microsecond=0) end_time = xq.get_open_time(kline_type, end_time) print("time range: %s ~ %s " % (start_time, end_time)) size = 1000 tmp_time = start_time while tmp_time < end_time: size_interval = size * interval if (tmp_time + size_interval) > end_time: batch = int((end_time - tmp_time) / interval) else: batch = size # print(batch) klines = exchange.get_klines(symbol, kline_type, size=batch, since=1000 * int(tmp_time.timestamp())) klines_df = pd.DataFrame(klines, columns=exchange.get_kline_column_names()) klen = len(klines) print(" %20s start time: %s %s" % (' ', tmp_time, klen)) for i in range(klen - 1, -1, -1): last_open_time = datetime.fromtimestamp( klines_df["open_time"].values[i] / 1000) if last_open_time + interval <= end_time: break klines_df = klines_df.drop([i]) # last_kline = klines[i] # print("%s %s" % (datetime.fromtimestamp(last_kline[0]/1000),last_kline)) if not db.insert_many(collection, klines_df.to_dict('records')): for item in klines_df.to_dict('records'): db.insert_one(collection, item) last_time = datetime.fromtimestamp( klines_df["open_time"].values[-1] / 1000) + interval if last_time > tmp_time + batch * interval: batch = int((last_time - tmp_time) / interval) tmp_time += batch * interval
def __get_klines_1min(self, symbol, s_time, e_time): """ 获取分钟k线 """ return self.__get_klines( xq.get_kline_collection(symbol, xq.KLINE_INTERVAL_1MINUTE), s_time, e_time)
parser.add_argument('-d', '--display', action='store_true', help='display info') args = parser.parse_args() # print(args) if not (args.s and args.r and args.k and args.m): parser.print_help() exit(1) start_time, end_time = split_time_range(args.r) print("range: [%s, %s)" % (start_time, end_time)) interval = args.k collection = xq.get_kline_collection(args.s, interval) td = xq.get_interval_timedelta(interval) period = xq.get_interval_seconds(interval) tick_time = xq.get_open_time(interval, start_time) if tick_time < start_time: tick_time = xq.get_open_time(interval, start_time + td) db = md.MongoDB(mongo_user, mongo_pwd, args.m, db_url) target_len = int( (int(end_time.timestamp()) - int(start_time.timestamp())) / period) print("Target length:", target_len) klines = db.find_sort( collection, { "open_time": { "$gte": int(start_time.timestamp()) * 1000,