def __get_klines_cache(self, symbol, interval, s_time, e_time):
""" 获取k线缓存 """
if (interval not in self.klines_cache
or s_time != self.klines_cache[interval]["s_time"]
or e_time != self.klines_cache[interval]["e_time"]):
cache = {}
cache["data"] = self.__get_klines(
xq.get_kline_collection(symbol, interval), s_time, e_time)
cache["s_time"] = s_time
cache["e_time"] = e_time
self.klines_cache[interval] = cache
return self.klines_cache[interval]["data"]
from setup import *
import pandas as pd
from importer import add_common_arguments, split_time_range
if __name__ == "__main__":
parser = add_common_arguments('Binance Importer')
args = parser.parse_args()
# print(args)
if not (args.s and args.k and args.m):
parser.print_help()
exit(1)
symbol = args.s
interval = timedelta(seconds=xq.get_interval_seconds(args.k))
collection = xq.get_kline_collection(symbol, args.k)
#print("collection: ", collection)
db = get_mongodb(args.m)
db.ensure_index(collection, [("open_time",1)], unique=True)
if args.r:
start_time, end_time = split_time_range(args.r)
else:
# 续接db中最后一条记录，至今天之前
klines = db.find_sort(collection, {}, 'open_time', -1, 1)
if len(klines) > 0:
start_time = (datetime.fromtimestamp(klines[0]["open_time"]/1000) + interval)
else:
start_time = None
end_time = datetime.now()
def download_from_exchange(exchange, db, symbol, kline_type, time_range):
print('%12s %6s ' % (' ', kline_type), end='')
collection = xq.get_kline_collection(symbol, kline_type)
db.ensure_index(collection, [("open_time", 1)], unique=True)
interval = timedelta(seconds=xq.get_interval_seconds(kline_type))
if time_range:
start_time, end_time = split_time_range(time_range)
else:
# 续接db中最后一条记录，至今天之前
klines = db.find_sort(collection, {}, 'open_time', -1, 1)
if len(klines) > 0:
start_time = (
datetime.fromtimestamp(klines[0]["open_time"] / 1000) +
interval)
else:
start_time = exchange.start_time
end_time = datetime.now()
#print(xq.get_open_time(kline_type, end_time))
"""
if start_time.hour != exchange.start_time.hour:
print("open time(%s) hour error! %s open time hour: %s" % (start_time, exchange.name, exchange.start_time.hour))
exit(1)
if end_time.hour < exchange.start_time.hour:
end_time -= timedelta(days=1)
end_time = end_time.replace(hour=exchange.start_time.hour, minute=0, second=0, microsecond=0)
"""
end_time = end_time.replace(minute=0, second=0, microsecond=0)
end_time = xq.get_open_time(kline_type, end_time)
print("time range: %s ~ %s " % (start_time, end_time))
size = 1000
tmp_time = start_time
while tmp_time < end_time:
size_interval = size * interval
if (tmp_time + size_interval) > end_time:
batch = int((end_time - tmp_time) / interval)
else:
batch = size
# print(batch)
klines = exchange.get_klines(symbol,
kline_type,
size=batch,
since=1000 * int(tmp_time.timestamp()))
klines_df = pd.DataFrame(klines,
columns=exchange.get_kline_column_names())
klen = len(klines)
print(" %20s start time: %s %s" % (' ', tmp_time, klen))
for i in range(klen - 1, -1, -1):
last_open_time = datetime.fromtimestamp(
klines_df["open_time"].values[i] / 1000)
if last_open_time + interval <= end_time:
break
klines_df = klines_df.drop([i])
# last_kline = klines[i]
# print("%s %s" % (datetime.fromtimestamp(last_kline[0]/1000),last_kline))
if not db.insert_many(collection, klines_df.to_dict('records')):
for item in klines_df.to_dict('records'):
db.insert_one(collection, item)
last_time = datetime.fromtimestamp(
klines_df["open_time"].values[-1] / 1000) + interval
if last_time > tmp_time + batch * interval:
batch = int((last_time - tmp_time) / interval)
tmp_time += batch * interval
def __get_klines_1min(self, symbol, s_time, e_time):
""" 获取分钟k线 """
return self.__get_klines(
xq.get_kline_collection(symbol, xq.KLINE_INTERVAL_1MINUTE), s_time,
e_time)
parser.add_argument('-d',
'--display',
action='store_true',
help='display info')
args = parser.parse_args()
# print(args)
if not (args.s and args.r and args.k and args.m):
parser.print_help()
exit(1)
start_time, end_time = split_time_range(args.r)
print("range: [%s, %s)" % (start_time, end_time))
interval = args.k
collection = xq.get_kline_collection(args.s, interval)
td = xq.get_interval_timedelta(interval)
period = xq.get_interval_seconds(interval)
tick_time = xq.get_open_time(interval, start_time)
if tick_time < start_time:
tick_time = xq.get_open_time(interval, start_time + td)
db = md.MongoDB(mongo_user, mongo_pwd, args.m, db_url)
target_len = int(
(int(end_time.timestamp()) - int(start_time.timestamp())) / period)
print("Target length:", target_len)
klines = db.find_sort(
collection, {
"open_time": {
"$gte": int(start_time.timestamp()) * 1000,
