def download_from_exchange(exchange, db, symbol, kline_type, time_range):
print('%12s %6s ' % (' ', kline_type), end='')
collection = xq.get_kline_collection(symbol, kline_type)
db.ensure_index(collection, [("open_time", 1)], unique=True)
interval = timedelta(seconds=xq.get_interval_seconds(kline_type))
if time_range:
start_time, end_time = split_time_range(time_range)
else:
# 续接db中最后一条记录，至今天之前
klines = db.find_sort(collection, {}, 'open_time', -1, 1)
if len(klines) > 0:
start_time = (
datetime.fromtimestamp(klines[0]["open_time"] / 1000) +
interval)
else:
start_time = exchange.start_time
end_time = datetime.now()
#print(xq.get_open_time(kline_type, end_time))
"""
if start_time.hour != exchange.start_time.hour:
print("open time(%s) hour error! %s open time hour: %s" % (start_time, exchange.name, exchange.start_time.hour))
exit(1)
if end_time.hour < exchange.start_time.hour:
end_time -= timedelta(days=1)
end_time = end_time.replace(hour=exchange.start_time.hour, minute=0, second=0, microsecond=0)
"""
end_time = end_time.replace(minute=0, second=0, microsecond=0)
end_time = xq.get_open_time(kline_type, end_time)
print("time range: %s ~ %s " % (start_time, end_time))
size = 1000
tmp_time = start_time
while tmp_time < end_time:
size_interval = size * interval
if (tmp_time + size_interval) > end_time:
batch = int((end_time - tmp_time) / interval)
else:
batch = size
# print(batch)
klines = exchange.get_klines(symbol,
kline_type,
size=batch,
since=1000 * int(tmp_time.timestamp()))
klines_df = pd.DataFrame(klines,
columns=exchange.get_kline_column_names())
klen = len(klines)
print(" %20s start time: %s %s" % (' ', tmp_time, klen))
for i in range(klen - 1, -1, -1):
last_open_time = datetime.fromtimestamp(
klines_df["open_time"].values[i] / 1000)
if last_open_time + interval <= end_time:
break
klines_df = klines_df.drop([i])
# last_kline = klines[i]
# print("%s %s" % (datetime.fromtimestamp(last_kline[0]/1000),last_kline))
if not db.insert_many(collection, klines_df.to_dict('records')):
for item in klines_df.to_dict('records'):
db.insert_one(collection, item)
last_time = datetime.fromtimestamp(
klines_df["open_time"].values[-1] / 1000) + interval
if last_time > tmp_time + batch * interval:
batch = int((last_time - tmp_time) / interval)
tmp_time += batch * interval
}
print(mid_kline)
database.insert_one(collection, mid_kline)
return mid_kline
if __name__ == "__main__":
parser = add_common_arguments('fix')
args = parser.parse_args()
# print(args)
if not (args.s and args.r and args.k and args.m):
parser.print_help()
exit(1)
start_time, end_time = split_time_range(args.r)
interval = args.k
collection = xq.get_kline_collection(args.s, interval)
td = xq.get_interval_timedelta(interval)
period = xq.get_interval_seconds(interval)
tick_time = xq.get_open_time(interval, start_time)
if tick_time < start_time:
tick_time = xq.get_open_time(interval, start_time + td)
db = md.MongoDB(mongo_user, mongo_pwd, args.m, db_url)
klines = db.find_sort(
collection, {
"open_time": {
"$gte": int(start_time.timestamp()) * 1000,
def download_from_exchange(exchange, db, symbol, kline_type, time_range):
print('%12s %6s ' % (' ', kline_type), end='')
collection = kl.get_kline_collection(symbol, kline_type)
open_time_key = exchange.kline_key_open_time
db.ensure_index(collection, [(open_time_key, 1)], unique=True)
interval = kl.get_interval_timedelta(kline_type)
if time_range:
start_time, end_time = split_time_range(time_range)
else:
# 续接db中最后一条记录，至今天之前
klines = db.find_sort(collection, {}, open_time_key, -1, 1)
if len(klines) > 0:
start_time = (
exchange.get_time_from_data_ts(klines[0][open_time_key]) +
interval)
else:
start_time = exchange.start_time
end_time = datetime.now()
#print(kl.get_open_time(kline_type, end_time))
"""
if start_time.hour != exchange.start_time.hour:
print("open time(%s) hour error! %s open time hour: %s" % (start_time, exchange.name, exchange.start_time.hour))
exit(1)
if end_time.hour < exchange.start_time.hour:
end_time -= timedelta(days=1)
end_time = end_time.replace(hour=exchange.start_time.hour, minute=0, second=0, microsecond=0)
"""
end_time = end_time.replace(minute=0, second=0, microsecond=0)
end_time = kl.get_open_time(kline_type, end_time)
print("time range: %s ~ %s " % (start_time, end_time))
size = exchange.max_count_of_single_download_kl
tmp_time = start_time
while tmp_time < end_time:
size_interval = size * interval
if (tmp_time + size_interval) > end_time:
batch = int((end_time - tmp_time) / interval)
else:
batch = size
# print(batch)
if batch == 0:
break
klines = exchange.get_klines(
symbol,
kline_type,
size=batch,
since=exchange.get_data_ts_from_time(tmp_time))
klines_df = pd.DataFrame(klines, columns=exchange.kline_column_names)
klen = len(klines)
print(" %20s start time: %s %s" % (' ', tmp_time, klen))
for i in range(klen - 1, -1, -1):
last_open_time = exchange.get_time_from_data_ts(
klines_df[open_time_key].values[i])
if last_open_time + interval <= end_time:
break
klines_df = klines_df.drop([i])
db_datalines = klines_df.to_dict('records')
if len(db_datalines) == 0:
break
if not db.insert_many(collection, db_datalines):
for item in db_datalines:
db.insert_one(collection, item)
last_time = exchange.get_time_from_data_ts(
klines_df[open_time_key].values[-1]) + interval
if last_time > tmp_time + batch * interval:
batch = int((last_time - tmp_time) / interval)
tmp_time += batch * interval