def __join_klines(self, symbol, interval, size, since=None):
""" 拼接k线 """
# print("tick_time : ", self.tick_time)
tick_open_time = kl.get_open_time(interval, self.tick_time)
td = kl.get_timedelta(interval, size)
#print("tick_open_time: ", tick_open_time)
if since is None:
# 取出之前的k线
s_time = tick_open_time - td
e_time = tick_open_time
else:
s_time = kl.get_open_time(interval, since)
e_time = s_time + td
#print("%s~%s"%(s_time, e_time))
ks = self.__get_klines_cache(symbol, interval, s_time, e_time)
k = self.__create_kline_from_1min(symbol, interval, tick_open_time,
self.tick_time)
return ks + k
if __name__ == "__main__":
parser = add_common_arguments('fix')
args = parser.parse_args()
# print(args)
if not (args.s and args.r and args.k and args.m):
parser.print_help()
exit(1)
start_time, end_time = split_time_range(args.r)
interval = args.k
collection = kl.get_kline_collection(args.s, interval)
td = kl.get_interval_timedelta(interval)
period = kl.get_interval_seconds(interval)
tick_time = kl.get_open_time(interval, start_time)
if tick_time < start_time:
tick_time = kl.get_open_time(interval, start_time+td)
db = md.MongoDB(mongo_user, mongo_pwd, args.m, db_url)
klines = db.find_sort(collection, {"open_time": {
"$gte": int(start_time.timestamp())*1000,
"$lt": int(end_time.timestamp())*1000}}, 'open_time', 1)
i = 0
miss_count = 0
print(len(klines))
period_ms = period * 1000
for kline in klines:
def download_from_exchange(exchange, db, symbol, kline_type, time_range):
print('%12s %6s ' % (' ', kline_type), end='')
collection = kl.get_kline_collection(symbol, kline_type)
open_time_key = exchange.kline_key_open_time
db.ensure_index(collection, [(open_time_key, 1)], unique=True)
interval = kl.get_interval_timedelta(kline_type)
if time_range:
start_time, end_time = split_time_range(time_range)
else:
# 续接db中最后一条记录，至今天之前
klines = db.find_sort(collection, {}, open_time_key, -1, 1)
if len(klines) > 0:
start_time = (
exchange.get_time_from_data_ts(klines[0][open_time_key]) +
interval)
else:
start_time = exchange.start_time
end_time = datetime.now()
#print(kl.get_open_time(kline_type, end_time))
"""
if start_time.hour != exchange.start_time.hour:
print("open time(%s) hour error! %s open time hour: %s" % (start_time, exchange.name, exchange.start_time.hour))
exit(1)
if end_time.hour < exchange.start_time.hour:
end_time -= timedelta(days=1)
end_time = end_time.replace(hour=exchange.start_time.hour, minute=0, second=0, microsecond=0)
"""
end_time = end_time.replace(minute=0, second=0, microsecond=0)
end_time = kl.get_open_time(kline_type, end_time)
print("time range: %s ~ %s " % (start_time, end_time))
size = exchange.max_count_of_single_download_kl
tmp_time = start_time
while tmp_time < end_time:
size_interval = size * interval
if (tmp_time + size_interval) > end_time:
batch = int((end_time - tmp_time) / interval)
else:
batch = size
# print(batch)
if batch == 0:
break
klines = exchange.get_klines(
symbol,
kline_type,
size=batch,
since=exchange.get_data_ts_from_time(tmp_time))
klines_df = pd.DataFrame(klines, columns=exchange.kline_column_names)
klen = len(klines)
print(" %20s start time: %s %s" % (' ', tmp_time, klen))
for i in range(klen - 1, -1, -1):
last_open_time = exchange.get_time_from_data_ts(
klines_df[open_time_key].values[i])
if last_open_time + interval <= end_time:
break
klines_df = klines_df.drop([i])
db_datalines = klines_df.to_dict('records')
if len(db_datalines) == 0:
break
if not db.insert_many(collection, db_datalines):
for item in db_datalines:
db.insert_one(collection, item)
last_time = exchange.get_time_from_data_ts(
klines_df[open_time_key].values[-1]) + interval
if last_time > tmp_time + batch * interval:
batch = int((last_time - tmp_time) / interval)
tmp_time += batch * interval