def get_oldest_time(self, symbol, kline_type):
collection = kl.get_kline_collection(symbol, kline_type)
klines = self.md_db.find_sort(collection, {}, self.kline_key_open_time,
1, 1)
if len(klines) <= 0:
return None
return self.get_kline_open_time(klines[0])
def get_oldest_time(self, symbol, kline_type):
collection = kl.get_kline_collection(symbol, kline_type)
klines = self.md_db.find_sort(collection, {}, self.kline_key_open_time,
1, 1)
if len(klines) <= 0:
return None
return (datetime.fromtimestamp(klines[0][self.kline_key_open_time] /
1000))
def get_latest_time(self, symbol, kline_type):
collection = kl.get_kline_collection(symbol, kline_type)
interval = timedelta(seconds=kl.get_interval_seconds(kline_type))
klines = self.md_db.find_sort(collection, {}, self.kline_key_open_time,
-1, 1)
if len(klines) <= 0:
return None
return (self.get_kline_open_time(klines[0]) + interval)
def get_latest_pirce(self, symbol):
collection = kl.get_kline_collection(symbol, kl.KLINE_INTERVAL_1MINUTE)
klines = self.md_db.find_sort(collection, {}, self.kline_key_open_time,
-1, 1)
if len(klines) <= 0:
return None, None
latest_kl = klines[0]
latest_price = float(latest_kl[self.kline_key_close])
latest_time = self.get_kline_close_time(latest_kl)
return latest_price, latest_time
def __get_klines_cache(self, symbol, interval, s_time, e_time):
""" 获取k线缓存 """
if (interval not in self.klines_cache
or s_time != self.klines_cache[interval]["s_time"]
or e_time != self.klines_cache[interval]["e_time"]):
cache = {}
cache["data"] = self.get_original_klines(
kl.get_kline_collection(symbol, interval), s_time, e_time)
cache["s_time"] = s_time
cache["e_time"] = e_time
self.klines_cache[interval] = cache
return self.klines_cache[interval]["data"]
def chart(title,
md,
symbol,
interval,
start_time,
end_time,
ordersets,
args,
signalsets=[]):
display_count = int((end_time - start_time).total_seconds() /
kl.get_interval_seconds(interval))
print("display_count: %s" % display_count)
#klines = md.get_klines(symbol, interval, 150+display_count)
klines = md.get_original_klines(kl.get_kline_collection(symbol, interval),
start_time, end_time)
#print("klines: %s"%klines)
if len(klines) > 0:
chart_mpf(title, args, symbol, ordersets, klines, md, display_count,
signalsets)
def perf_get_klines_adv(self):
total_interval_count = int((self.end_time - self.start_time) /
self.interval_td) + self.pre_count
print("total_interval_count: %s" % (total_interval_count))
interval_collection = kl.get_kline_collection(self.symbol,
self.interval)
interval_klines = self.md.get_original_klines(
interval_collection,
self.start_time - self.interval_td * self.pre_count, self.end_time)
kl = interval_klines[0]
print("open_time: %s" % (self.md.get_kline_open_time(kl)))
#print("json: %s" % (kl))
ti.EMA(interval_klines, "close", 13)
ti.EMA(interval_klines, "close", 21)
ti.EMA(interval_klines, "close", 55)
ti.BIAS_EMA(interval_klines, 13, 21)
ti.BIAS_EMA(interval_klines, 21, 55)
ti.RSI(interval_klines, "close")
#print(interval_klines[self.pre_count:self.pre_count+2])
#print(interval_klines[-2:])
#pprint(interval_klines[self.pre_count])
#pprint(interval_klines[-1])
for i in range(self.pre_count + 1):
if self.md.get_kline_open_time(
interval_klines[i]) >= self.start_time:
break
interval_idx = i
kl = interval_klines[interval_idx]
print("interval_idx: %s" % (interval_idx))
print("open time: %s" % (self.md.get_kline_open_time(kl)))
#print("json: %s" % (kl))
for i in range(interval_idx, len(interval_klines)):
start_i = i - self.pre_count
if start_i < 0:
start_i = 0
history_kls = interval_klines[start_i:i]
#print(len(history_kls))
interval_open_time = self.md.get_kline_open_time(
interval_klines[i])
#print(interval_open_time)
tick_klines = self.md.get_original_klines(
self.tick_collection, interval_open_time,
interval_open_time + self.interval_td)
new_interval_kl = tick_klines[0]
for tick_kl in tick_klines[1:]:
tick_open_time = self.md.get_kline_open_time(tick_kl)
#print(tick_open_time)
new_interval_kl["close"] = tick_kl["close"]
new_interval_kl["close_time"] = tick_kl["close_time"]
if new_interval_kl["high"] < tick_kl["high"]:
new_interval_kl["high"] = tick_kl["high"]
if new_interval_kl["low"] > tick_kl["low"]:
new_interval_kl["low"] = tick_kl["low"]
cur_kls = history_kls + [new_interval_kl]
ti.EMA(cur_kls, "close", 13)
ti.EMA(cur_kls, "close", 21)
ti.EMA(cur_kls, "close", 55)
ti.BIAS_EMA(cur_kls, 13, 21)
ti.BIAS_EMA(cur_kls, 21, 55)
ti.RSI(cur_kls, "close")
class TestDBMD(unittest.TestCase):
symbol = "eth_usdt"
digits = 4
interval = kl.KLINE_INTERVAL_1DAY
interval_td = kl.get_interval_timedelta(interval)
pre_count = 150
display_count = 10
tick_interval = kl.KLINE_INTERVAL_1MINUTE
tick_collection = kl.get_kline_collection(symbol, tick_interval)
tick_td = kl.get_interval_timedelta(tick_interval)
start_time = datetime(2017, 9, 1)
end_time = datetime(2020, 8, 1)
md = DBMD("binance")
def setUp(self):
self.md.tick_time = self.start_time
tick = (self.end_time - self.start_time) / self.tick_td
print("tick td=%s, tick=%d, time rang: %s ~ %s" %
(self.tick_td, tick, self.start_time, self.end_time))
def tearDown(self):
pass
def perf_get_json_klines(self):
while self.md.tick_time < self.end_time:
klines = self.md.get_json_klines(
self.symbol, self.interval,
self.pre_count + self.display_count)
self.md.tick_time += self.tick_td
def perf_get_klines(self):
while self.md.tick_time < self.end_time:
klines = self.md.get_klines(self.symbol, self.interval,
self.pre_count + self.display_count)
self.md.tick_time += self.tick_td
def perf_get_klines_adv(self):
total_interval_count = int((self.end_time - self.start_time) /
self.interval_td) + self.pre_count
print("total_interval_count: %s" % (total_interval_count))
interval_collection = kl.get_kline_collection(self.symbol,
self.interval)
interval_klines = self.md.get_original_klines(
interval_collection,
self.start_time - self.interval_td * self.pre_count, self.end_time)
kl = interval_klines[0]
print("open_time: %s" % (self.md.get_kline_open_time(kl)))
#print("json: %s" % (kl))
ti.EMA(interval_klines, "close", 13)
ti.EMA(interval_klines, "close", 21)
ti.EMA(interval_klines, "close", 55)
ti.BIAS_EMA(interval_klines, 13, 21)
ti.BIAS_EMA(interval_klines, 21, 55)
ti.RSI(interval_klines, "close")
#print(interval_klines[self.pre_count:self.pre_count+2])
#print(interval_klines[-2:])
#pprint(interval_klines[self.pre_count])
#pprint(interval_klines[-1])
for i in range(self.pre_count + 1):
if self.md.get_kline_open_time(
interval_klines[i]) >= self.start_time:
break
interval_idx = i
kl = interval_klines[interval_idx]
print("interval_idx: %s" % (interval_idx))
print("open time: %s" % (self.md.get_kline_open_time(kl)))
#print("json: %s" % (kl))
for i in range(interval_idx, len(interval_klines)):
start_i = i - self.pre_count
if start_i < 0:
start_i = 0
history_kls = interval_klines[start_i:i]
#print(len(history_kls))
interval_open_time = self.md.get_kline_open_time(
interval_klines[i])
#print(interval_open_time)
tick_klines = self.md.get_original_klines(
self.tick_collection, interval_open_time,
interval_open_time + self.interval_td)
new_interval_kl = tick_klines[0]
for tick_kl in tick_klines[1:]:
tick_open_time = self.md.get_kline_open_time(tick_kl)
#print(tick_open_time)
new_interval_kl["close"] = tick_kl["close"]
new_interval_kl["close_time"] = tick_kl["close_time"]
if new_interval_kl["high"] < tick_kl["high"]:
new_interval_kl["high"] = tick_kl["high"]
if new_interval_kl["low"] > tick_kl["low"]:
new_interval_kl["low"] = tick_kl["low"]
cur_kls = history_kls + [new_interval_kl]
ti.EMA(cur_kls, "close", 13)
ti.EMA(cur_kls, "close", 21)
ti.EMA(cur_kls, "close", 55)
ti.BIAS_EMA(cur_kls, 13, 21)
ti.BIAS_EMA(cur_kls, 21, 55)
ti.RSI(cur_kls, "close")
def __get_klines_1min(self, symbol, s_time, e_time):
""" 获取分钟k线 """
return self.get_original_klines(
kl.get_kline_collection(symbol, kl.KLINE_INTERVAL_1MINUTE), s_time,
e_time)
from setup import *
import pandas as pd
from importer import add_common_arguments, split_time_range
if __name__ == "__main__":
parser = add_common_arguments('Binance Importer')
args = parser.parse_args()
# print(args)
if not (args.s and args.k and args.m):
parser.print_help()
exit(1)
symbol = args.s
interval = timedelta(seconds=kl.get_interval_seconds(args.k))
collection = kl.get_kline_collection(symbol, args.k)
#print("collection: ", collection)
db = md.MongoDB(mongo_user, mongo_pwd, args.m, db_url)
db.ensure_index(collection, [("open_time",1)], unique=True)
# 注意，下面代码有隐患，在上午8点前取1d、12h时，最后的一个是不完整的，后续再整改
if args.r:
start_time, end_time = split_time_range(args.r)
else:
# 续接db中最后一条记录，至今天之前
klines = db.find_sort(collection, {}, 'open_time', -1, 1)
if len(klines) > 0:
start_time = (datetime.fromtimestamp(klines[0]["open_time"]/1000) + interval).replace(hour=0,minute=0,second=0,microsecond=0)
else:
start_time = None
return mid_kline
if __name__ == "__main__":
parser = add_common_arguments('fix')
args = parser.parse_args()
# print(args)
if not (args.s and args.r and args.k and args.m):
parser.print_help()
exit(1)
start_time, end_time = split_time_range(args.r)
interval = args.k
collection = kl.get_kline_collection(args.s, interval)
td = kl.get_interval_timedelta(interval)
period = kl.get_interval_seconds(interval)
tick_time = kl.get_open_time(interval, start_time)
if tick_time < start_time:
tick_time = kl.get_open_time(interval, start_time+td)
db = md.MongoDB(mongo_user, mongo_pwd, args.m, db_url)
klines = db.find_sort(collection, {"open_time": {
"$gte": int(start_time.timestamp())*1000,
"$lt": int(end_time.timestamp())*1000}}, 'open_time', 1)
i = 0
miss_count = 0
print(len(klines))
def run_2kls(engine, md, strategy, start_time, end_time, progress_disp=True):
symbol = strategy.config["symbol"]
tick_interval = kl.KLINE_INTERVAL_1MINUTE
tick_collection = kl.get_kline_collection(symbol, tick_interval)
tick_td = kl.get_interval_timedelta(tick_interval)
kline_cfg = strategy.config["kline"]
size = kline_cfg["size"]
master_interval = kline_cfg["interval"]
master_td = kl.get_interval_timedelta(master_interval)
master_original_kls = md.get_original_klines(
kl.get_kline_collection(symbol, master_interval),
start_time - master_td * size, end_time)
micro_interval = kline_cfg["micro_interval"]
micro_td = kl.get_interval_timedelta(micro_interval)
micro_original_kls = md.get_original_klines(
kl.get_kline_collection(symbol, micro_interval),
start_time - micro_td * size, end_time)
if hasattr(strategy, "before_backtest"):
strategy.before_backtest(master_original_kls, micro_original_kls)
for master_start_idx in range(size + 1):
master_start_open_time = md.get_kline_open_time(
master_original_kls[master_start_idx]) + master_td
if master_start_open_time >= start_time:
break
for micro_idx in range(size + 1 + int(master_td / micro_td)):
micro_start_open_time = md.get_kline_open_time(
micro_original_kls[micro_idx]) + micro_td
if micro_start_open_time >= master_start_open_time:
break
pre_tick_cost_time = total_tick_cost_start = datetime.now()
tick_count = 0
tick_klines = []
tick_idx = 0
for master_idx in range(master_start_idx, len(master_original_kls)):
pre_start_i = master_idx - size
if pre_start_i < 0:
pre_start_i = 0
history_master_kls = master_original_kls[pre_start_i:master_idx + 1]
new_master_open_time = md.get_kline_open_time(
history_master_kls[-1]) + master_td
new_master_close_time = new_master_open_time + master_td
new_master_kl = None
#print("new master open time: %s" % (new_master_open_time))
if tick_idx >= len(tick_klines):
tick_klines = md.get_original_klines(
tick_collection, new_master_open_time,
new_master_open_time + 7 * master_td)
tick_idx = 0
while (micro_idx < len(micro_original_kls)):
if md.get_kline_open_time(micro_original_kls[micro_idx]
) + micro_td >= new_master_close_time:
break
if micro_idx > size:
history_micro_kls = micro_original_kls[(micro_idx -
size):micro_idx + 1]
else:
history_micro_kls = micro_original_kls[:micro_idx + 1]
micro_idx += 1
new_micro_open_time = md.get_kline_open_time(
history_micro_kls[-1]) + micro_td
new_micro_close_time = new_micro_open_time + micro_td
new_micro_kl = None
#print("new micro open time: %s" % (new_micro_open_time))
while (tick_idx < len(tick_klines)):
tick_kl = tick_klines[tick_idx]
tick_open_time = md.get_kline_open_time(tick_kl)
if tick_open_time >= new_micro_close_time:
break
tick_idx += 1
engine.log_info("tick_time: %s" %
tick_open_time.strftime("%Y-%m-%d %H:%M:%S"))
#print(tick_open_time)
if not new_master_kl:
new_master_kl = copy.copy(tick_kl)
else:
update_kl(md, new_master_kl, tick_kl)
master_kls = history_master_kls + [new_master_kl]
if not new_micro_kl:
new_micro_kl = copy.copy(tick_kl)
else:
update_kl(md, new_micro_kl, tick_kl)
micro_kls = history_micro_kls + [new_micro_kl]
if md.kline_data_type == kl.KLINE_DATA_TYPE_LIST:
master_kls = kl.trans_from_json_to_list(
master_kls, md.kline_column_names)
micro_kls = kl.trans_from_json_to_list(
micro_kls, md.kline_column_names)
md.tick_time = tick_open_time + tick_td
strategy.on_tick(master_kls, micro_kls)
tick_cost_time = datetime.now()
engine.log_info("tick cost: %s \n\n" %
(tick_cost_time - pre_tick_cost_time))
pre_tick_cost_time = tick_cost_time
tick_count += 1
if progress_disp:
progress = (master_idx + 1 - master_start_idx) / (
len(master_original_kls) - master_start_idx)
sys.stdout.write(
"%s progress: %d%%, cost: %s, open time: %s\r" % (
" " * 36,
progress * 100,
tick_cost_time - total_tick_cost_start,
new_master_open_time.strftime("%Y-%m-%d %H:%M:%S"),
))
sys.stdout.flush()
return tick_count
def run_1kls(engine, md, strategy, start_time, end_time, progress_disp=True):
symbol = strategy.config["symbol"]
tick_interval = kl.KLINE_INTERVAL_1MINUTE
tick_collection = kl.get_kline_collection(symbol, tick_interval)
tick_td = kl.get_interval_timedelta(tick_interval)
kline_cfg = strategy.config["kline"]
size = kline_cfg["size"]
master_interval = kline_cfg["interval"]
master_td = kl.get_interval_timedelta(master_interval)
master_kls = md.get_original_klines(
kl.get_kline_collection(symbol, master_interval),
start_time - master_td * size, end_time)
if hasattr(strategy, "before_backtest"):
strategy.before_backtest(master_kls)
for i in range(size + 1):
if md.get_kline_open_time(master_kls[i]) >= start_time:
break
master_idx = i
pre_tick_cost_time = total_tick_cost_start = datetime.now()
tick_count = 0
for i in range(master_idx, len(master_kls)):
start_i = i - size
if start_i < 0:
start_i = 0
history_master_kls = master_kls[start_i:i]
master_open_time = md.get_kline_open_time(master_kls[i])
tick_klines = md.get_original_klines(tick_collection, master_open_time,
master_open_time + master_td)
for j, tick_kl in enumerate(tick_klines):
tick_open_time = md.get_kline_open_time(tick_kl)
engine.log_info("tick_time: %s" %
tick_open_time.strftime("%Y-%m-%d %H:%M:%S"))
#print(tick_open_time)
if j == 0:
new_master_kl = tick_kl
else:
update_kl(md, new_master_kl, tick_kl)
kls = history_master_kls + [new_master_kl]
if md.kline_data_type == kl.KLINE_DATA_TYPE_LIST:
kls = kl.trans_from_json_to_list(kls, md.kline_column_names)
md.tick_time = tick_open_time + tick_td
strategy.on_tick(kls)
tick_cost_time = datetime.now()
engine.log_info("tick cost: %s \n\n" %
(tick_cost_time - pre_tick_cost_time))
pre_tick_cost_time = tick_cost_time
tick_count += 1
if progress_disp:
progress = (i + 1 - master_idx) / (len(master_kls) - master_idx)
sys.stdout.write(
"%s progress: %d%%, cost: %s, next open time: %s\r" % (
" " * 36,
progress * 100,
tick_cost_time - total_tick_cost_start,
(master_open_time +
master_td).strftime("%Y-%m-%d %H:%M:%S"),
))
sys.stdout.flush()
return tick_count
def download_from_exchange(exchange, db, symbol, kline_type, time_range):
print('%12s %6s ' % (' ', kline_type), end='')
collection = kl.get_kline_collection(symbol, kline_type)
open_time_key = exchange.kline_key_open_time
db.ensure_index(collection, [(open_time_key, 1)], unique=True)
interval = kl.get_interval_timedelta(kline_type)
if time_range:
start_time, end_time = split_time_range(time_range)
else:
# 续接db中最后一条记录，至今天之前
klines = db.find_sort(collection, {}, open_time_key, -1, 1)
if len(klines) > 0:
start_time = (
exchange.get_time_from_data_ts(klines[0][open_time_key]) +
interval)
else:
start_time = exchange.start_time
end_time = datetime.now()
#print(kl.get_open_time(kline_type, end_time))
"""
if start_time.hour != exchange.start_time.hour:
print("open time(%s) hour error! %s open time hour: %s" % (start_time, exchange.name, exchange.start_time.hour))
exit(1)
if end_time.hour < exchange.start_time.hour:
end_time -= timedelta(days=1)
end_time = end_time.replace(hour=exchange.start_time.hour, minute=0, second=0, microsecond=0)
"""
end_time = end_time.replace(minute=0, second=0, microsecond=0)
end_time = kl.get_open_time(kline_type, end_time)
print("time range: %s ~ %s " % (start_time, end_time))
size = exchange.max_count_of_single_download_kl
tmp_time = start_time
while tmp_time < end_time:
size_interval = size * interval
if (tmp_time + size_interval) > end_time:
batch = int((end_time - tmp_time) / interval)
else:
batch = size
# print(batch)
if batch == 0:
break
klines = exchange.get_klines(
symbol,
kline_type,
size=batch,
since=exchange.get_data_ts_from_time(tmp_time))
klines_df = pd.DataFrame(klines, columns=exchange.kline_column_names)
klen = len(klines)
print(" %20s start time: %s %s" % (' ', tmp_time, klen))
for i in range(klen - 1, -1, -1):
last_open_time = exchange.get_time_from_data_ts(
klines_df[open_time_key].values[i])
if last_open_time + interval <= end_time:
break
klines_df = klines_df.drop([i])
db_datalines = klines_df.to_dict('records')
if len(db_datalines) == 0:
break
if not db.insert_many(collection, db_datalines):
for item in db_datalines:
db.insert_one(collection, item)
last_time = exchange.get_time_from_data_ts(
klines_df[open_time_key].values[-1]) + interval
if last_time > tmp_time + batch * interval:
batch = int((last_time - tmp_time) / interval)
tmp_time += batch * interval
def run2(engine, md, strategy, start_time, end_time, progress_disp=True):
""" run advance"""
secs = strategy.config["sec"]
if secs < 60:
secs = 60
td_secs = timedelta(seconds=secs)
symbol = strategy.config["symbol"]
tick_interval = kl.KLINE_INTERVAL_1MINUTE
tick_collection = kl.get_kline_collection(symbol, tick_interval)
tick_td = kl.get_interval_timedelta(tick_interval)
interval = strategy.config["kline"]["interval"]
interval_collection = kl.get_kline_collection(symbol, interval)
interval_td = kl.get_interval_timedelta(interval)
size = strategy.config["kline"]["size"]
interval_klines = md.get_original_klines(interval_collection, start_time - interval_td * size, end_time)
kl_key_open_time = md.kline_key_open_time
for i in range(size+1):
if datetime.fromtimestamp(interval_klines[i][kl_key_open_time]/1000) >= start_time:
break
interval_idx = i
pre_tick_cost_time = total_tick_cost_start = datetime.now()
tick_count = 0
for i in range(interval_idx, len(interval_klines)):
interval_open_time = datetime.fromtimestamp(interval_klines[i][kl_key_open_time]/1000)
start_i = i - size
if start_i < 0:
start_i = 0
history_kls = interval_klines[start_i:i]
#print(len(history_kls))
interval_open_ts = interval_klines[i][kl_key_open_time]
interval_open_time = datetime.fromtimestamp(interval_open_ts/1000)
#print(interval_open_time)
tick_klines = md.get_original_klines(tick_collection, interval_open_time, interval_open_time + interval_td)
for j, tick_kl in enumerate(tick_klines):
tick_open_time = datetime.fromtimestamp(tick_kl[kl_key_open_time]/1000)
engine.log_info("tick_time: %s" % tick_open_time.strftime("%Y-%m-%d %H:%M:%S"))
#print(tick_open_time)
if j == 0:
new_interval_kl = tick_kl
else:
new_interval_kl[md.kline_key_close] = tick_kl[md.kline_key_close]
new_interval_kl[md.kline_key_close_time] = tick_kl[md.kline_key_close_time]
if new_interval_kl[md.kline_key_high] < tick_kl[md.kline_key_high]:
new_interval_kl[md.kline_key_high] = tick_kl[md.kline_key_high]
if new_interval_kl[md.kline_key_low] > tick_kl[md.kline_key_low]:
new_interval_kl[md.kline_key_low] = tick_kl[md.kline_key_low]
kls = history_kls + [new_interval_kl]
if md.kline_data_type == kl.KLINE_DATA_TYPE_LIST:
kls = kl.trans_from_json_to_list(kls, md.kline_column_names)
md.tick_time = tick_open_time + tick_td
strategy.on_tick(kls)
tick_cost_time = datetime.now()
engine.log_info("tick cost: %s \n\n" % (tick_cost_time - pre_tick_cost_time))
pre_tick_cost_time = tick_cost_time
tick_count += 1
if progress_disp:
progress = (i + 1 - interval_idx) / (len(interval_klines) - interval_idx)
sys.stdout.write(
"%s progress: %d%%, cost: %s, next open time: %s\r"
% (
" "*36,
progress * 100,
tick_cost_time - total_tick_cost_start,
(interval_open_time+interval_td).strftime("%Y-%m-%d %H:%M:%S"),
)
)
sys.stdout.flush()
return tick_count
