def __init__(self,
pcontracts,
dt_start="1980-1-1",
dt_end="2100-1-1",
n=None,
spec_date={}): # 'symbol':[,]
"""
Args:
pcontracts (list): list of pcontracts(string)
dt_start (datetime/str): start time of all pcontracts
dt_end (datetime/str): end time of all pcontracts
n (int): last n bars
spec_date (dict): time range for specific pcontracts
"""
self.finished_data = []
pcontracts = map(lambda x: x.upper(), pcontracts)
self.pcontracts = pcontracts
self._combs = []
self._data_manager = DataManager()
# str(PContract): DataWrapper
if settings['source'] == 'csv':
self.pcontracts = self._parse_pcontracts(self.pcontracts)
self._default_pcontract = self.pcontracts[0]
self._all_data, self._max_window = self._load_data(
self.pcontracts, dt_start, dt_end, n, spec_date)
self.context = Context(self._all_data, self._max_window)
def __init__(self):
log.info("Init Backend..")
self._dm = DataManager()
self._engine = ZMQEventEngine('Backend')
self._engine.start()
self._shell_srv = EventRPCServer(self._engine, self.SERVER_FOR_SHELL)
self._ui_srv = EventRPCServer(self._engine, self.SERVER_FOR_UI)
self.register_functions(self._shell_srv)
self.register_functions(self._ui_srv)
def __init__(self, pcontracts,
dt_start="1980-1-1",
dt_end="2100-1-1",
n = None,
spec_date = { }): # 'symbol':[,]
"""
Args:
pcontracts (list): list of pcontracts(string)
dt_start (datetime/str): start time of all pcontracts
dt_end (datetime/str): end time of all pcontracts
n (int): last n bars
spec_date (dict): time range for specific pcontracts
"""
self.finished_data = []
pcontracts = map(lambda x: x.upper(), pcontracts)
self.pcontracts = pcontracts
self._combs = []
self._data_manager = DataManager()
# str(PContract): DataWrapper
self.pcontracts = self._parse_pcontracts(self.pcontracts)
self._all_data, self._max_window = self._load_data(self.pcontracts,
dt_start,
dt_end,
n,
spec_date)
self.context = Context(self._all_data, self._max_window)
def test_local_data2(self):
old_source = ConfigUtil.get("source")
if old_source == "csv":
return
logger.info("***** 数据测试开始 *****")
ConfigUtil.set(source="sqlite")
data_manager = DataManager()
target = data_manager.get_bars("BB.TEST-1.Minute", _DT_START, _DT_END).data
fname = os.path.join(os.getcwd(), "data", "1MINUTE", "TEST", "CC.csv")
source = pd.read_csv(fname, parse_dates="datetime", index_col="datetime")
self.assertFalse(source.equals(target), "本地数据接口负测试失败！")
fname = os.path.join(os.getcwd(), "data", "1MINUTE", "TEST", "BB.csv")
source = pd.read_csv(fname, parse_dates="datetime", index_col="datetime")
self.assertTrue(source.equals(target), "本地数据接口正测试失败！")
logger.info("-- 本地数据接口测试成功 --")
logger.info("***** 数据测试结束 *****\n")
ConfigUtil.set(source=old_source)
def test_csv_source(self):
source_bak = ConfigUtil.get('source')
logger.info('***** 数据测试开始 *****')
ConfigUtil.set(source='csv')
data_manager = DataManager()
target = data_manager.get_bars(
'BB.TEST-1.Minute', _DT_START, _DT_END).data
fname = os.path.join(os.getcwd(), 'data', '1MINUTE', 'TEST', 'CC.csv')
source = pd.read_csv(
fname, parse_dates='datetime', index_col='datetime')
self.assertFalse(source.equals(target), '本地数据接口负测试失败！')
fname = os.path.join(os.getcwd(), 'data', '1MINUTE', 'TEST', 'BB.csv')
source = pd.read_csv(
fname, parse_dates='datetime', index_col='datetime')
self.assertTrue(source.equals(target), '本地数据接口正测试失败！')
logger.info('-- 本地数据接口测试成功 --')
ConfigUtil.set(source=source_bak)
logger.info('***** 数据测试结束 *****\n')
def __init__(self, pcontracts, window_size=0, dt_start=datetime(1980,1,1),
dt_end=datetime(2100,1,1), spec_date = { }): # 'symbol':[,]
series.g_rolling = False if window_size == 0 else True
series.g_window = window_size
self.all_data = OrderedDict() # str(PContract): DataWrapper
self.finished_data = []
pcontracts = map(lambda x: x.upper(), pcontracts)
self.pcontracts = pcontracts
self._combs = []
self._window_size = window_size + 1
self._data_manager = DataManager()
for pcon in pcontracts:
if pcon in spec_date:
dt_start = spec_date[pcon][0]
dt_end = spec_date[pcon][1]
assert(dt_start < dt_end)
self.load_data(pcon, dt_start, dt_end)
self.context = Context(self.all_data)
def test_csv_source(self):
source_bak = ConfigUtil.get('source')
logger.info('***** 数据测试开始 *****')
ConfigUtil.set(source='csv')
data_manager = DataManager()
target = data_manager.get_bars('BB.TEST-1.Minute', _DT_START,
_DT_END).data
fname = os.path.join(os.getcwd(), 'data', '1MINUTE', 'TEST', 'CC.csv')
source = pd.read_csv(fname,
parse_dates='datetime',
index_col='datetime')
self.assertFalse(source.equals(target), '本地数据接口负测试失败！')
fname = os.path.join(os.getcwd(), 'data', '1MINUTE', 'TEST', 'BB.csv')
source = pd.read_csv(fname,
parse_dates='datetime',
index_col='datetime')
self.assertTrue(source.equals(target), '本地数据接口正测试失败！')
logger.info('-- 本地数据接口测试成功 --')
ConfigUtil.set(source=source_bak)
logger.info('***** 数据测试结束 *****\n')
def _load_data(self, pcontracts, dt_start, dt_end, spec_date):
all_data = OrderedDict() # str(PContract): DataWrapper
self._data_manager = DataManager()
for pcon in pcontracts:
if pcon in spec_date:
dt_start = spec_date[pcon][0]
dt_end = spec_date[pcon][1]
assert(dt_start < dt_end)
wrapper = self._data_manager.get_bars(pcon, dt_start, dt_end)
all_data[pcon] = DataContext(wrapper)
return all_data
def __init__(self):
log.info("Init Backend..")
self._dm = DataManager()
self._engine = ZMQEventEngine('Backend')
self._engine.start()
self._shell_srv = EventRPCServer(self._engine,
self.SERVER_FOR_SHELL)
self._ui_srv = EventRPCServer(self._engine,
self.SERVER_FOR_UI)
self.register_functions(self._shell_srv)
self.register_functions(self._ui_srv)
def _load_data(self, pcontracts, dt_start, dt_end, spec_date):
all_data = OrderedDict()
self._data_manager = DataManager()
max_window = -1
for pcon in pcontracts:
if pcon in spec_date:
dt_start = spec_date[pcon][0]
dt_end = spec_date[pcon][1]
assert(dt_start < dt_end)
wrapper = self._data_manager.get_bars(pcon, dt_start, dt_end)
all_data[pcon] = DataContext(wrapper)
max_window = max(max_window, len(wrapper))
return all_data, max_window
def __init__(self, pcontracts, window_size=0, dt_start=datetime(1980,1,1),
dt_end=datetime(2100,1,1), spec_date = { }): # 'symbol':[,]
series.g_rolling = False if window_size == 0 else True
series.g_window = window_size
self.all_data = { } # PContract -> DataWrapper
self.finished_data = []
self.pcontracts = pcontracts
self._combs = []
self._window_size = window_size + 1
self._data_manager = DataManager()
for pcon in pcontracts:
if pcon in spec_date:
dt_start = spec_date[pcon][0]
dt_end = spec_date[pcon][1]
assert(dt_start < dt_end)
self.load_data(pcon, dt_start, dt_end)
self.context = Context(self.all_data)
class ExecuteUnit(object):
""" 策略执行的物理单元，支持多个策略同时运行。
每个执行单元都可能跟踪多个数据(策略用到的周期合约数据集合)。
其中第一个合约为"主合约" 。 每个执行器可加载多个策略,只要数据需求不超过pcontracts。
:ivar dt_begin: 策略执行的时间起点。
:ivar dt_end: 策略执行的时间终点。
:ivar pcontracts: 策略用到的周期合约数据集合。
:ivar _strategies: 策略集合。
:ivar datasource: 数据源。
"""
def __init__(self, pcontracts, window_size=0, dt_start=datetime(1980,1,1),
dt_end=datetime(2100,1,1), spec_date = { }): # 'symbol':[,]
series.g_rolling = False if window_size == 0 else True
series.g_window = window_size
self.all_data = { } # PContract -> DataWrapper
self.finished_data = []
self.pcontracts = pcontracts
self._combs = []
self._window_size = window_size + 1
self._data_manager = DataManager()
for pcon in pcontracts:
if pcon in spec_date:
dt_start = spec_date[pcon][0]
dt_end = spec_date[pcon][1]
assert(dt_start < dt_end)
self.load_data(pcon, dt_start, dt_end)
self.context = Context(self.all_data)
def _init_strategies(self):
for pcon, dcontext in self.all_data.iteritems():
# switch context
self.context.switch_to_contract(pcon)
for i, combination in enumerate(self._combs):
for j, s in enumerate(combination):
self.context.switch_to_strategy(i, j)
s.on_init(self.context)
def add_comb(self, comb, settings = { }):
""" 添加策略组合组合
Args:
comb (list): 一个策略组合
"""
self._combs.append(comb)
if settings:
num_strategy = len(comb)
assert (settings['captial'] > 0)
assert len(settings['ratio']) == num_strategy
assert(sum(settings['ratio']) == 1)
ctxs = []
for i, s in enumerate(comb):
iset = { }
if settings:
iset = { 'captial': settings['captial'] * settings['ratio'][i] }
logger.debug(iset)
ctxs.append(StrategyContext(s.name, iset))
self.context.add_strategy_context(ctxs)
blotters = [ ctx.blotter for ctx in ctxs]
return blotter.Profile(blotters, self.all_data, self.pcontracts[0], len(self._combs)-1)
def run(self):
# 初始化策略自定义时间序列变量
print 'runing strategies..'
self._init_strategies()
print 'on_bars..'
# todo 对单策略优化
has_next = True
# 遍历每个数据轮, 次数为数据的最大长度
for pcon, data in self.all_data.iteritems():
self.context.switch_to_contract(pcon)
self.context.rolling_foward()
while True:
# 遍历数据轮的所有合约
for pcon, data in self.all_data.iteritems():
self.context.switch_to_contract(pcon)
if self.context.time_aligned():
self.context.update_system_vars()
# 组合遍历
for i, combination in enumerate(self._combs):
# 策略遍历
for j, s in enumerate(combination):
self.context.switch_to_strategy(i, j)
self.context.update_strategy_environment(i, j)
self.context.update_user_vars()
s.on_bar(self.context)
# 每轮数据的最后处理
#print "*********"
for i, combination in enumerate(self._combs):
for j, s in enumerate(combination):
self.context.switch_to_strategy(i, j)
# 只有在on_final引用跨合约数据才有保证。
s.on_final(self.context)
self.context.process_trading_events()
self.context.last_date = datetime(2100,1,1)
#
toremove = []
for pcon, data in self.all_data.iteritems():
self.context.switch_to_contract(pcon)
has_next = self.context.rolling_foward()
if not has_next:
toremove.append(pcon)
if toremove:
for key in toremove:
del self.all_data[key]
if len(self.all_data) == 0:
# 策略退出后的处理
for i, combination in enumerate(self._combs):
for j, s in enumerate(combination):
self.context.switch_to_strategy(i, j)
s.on_exit(self.context)
return
def load_data(self, pcontract, dt_start=datetime(1980,1,1), dt_end=datetime(2100,1,1)):
""" 加载周期合约数据
Args:
pcontract (PContract): 周期合约
dt_start(datetime): 开始时间
dt_end(datetime): 结束时间
Returns:
pd.DataFrame. k线数据
"""
try:
return self.all_data[str(pcontract)]
except KeyError:
wrapper = self._data_manager.load_bars(pcontract, dt_start, dt_end, self._window_size)
window_size = len(wrapper.data) if not series.g_rolling else self._window_size
self.all_data[pcontract] = DataContext(wrapper, window_size)
return wrapper
class ExecuteUnit(object):
""" 策略执行的物理单元，支持多个组合同时运行。
"""
def __init__(self,
pcontracts,
dt_start="1980-1-1",
dt_end="2100-1-1",
n=None,
spec_date={}): # 'symbol':[,]
"""
Args:
pcontracts (list): list of pcontracts(string)
dt_start (datetime/str): start time of all pcontracts
dt_end (datetime/str): end time of all pcontracts
n (int): last n bars
spec_date (dict): time range for specific pcontracts
"""
self.finished_data = []
pcontracts = list(map(lambda x: x.upper(), pcontracts))
self.pcontracts = pcontracts
self._contexts = []
self._data_manager = DataManager()
if settings['source'] == 'csv':
self.pcontracts = self._parse_pcontracts(self.pcontracts)
self._all_data, self._max_window = self._load_data(
self.pcontracts, dt_start, dt_end, n, spec_date)
self._all_pcontracts = list(self._all_data.keys())
def _parse_pcontracts(self, pcontracts):
# @TODO test
code2strpcon, exch_period2strpcon = \
self._data_manager.get_code2strpcon()
rst = []
for strpcon in pcontracts:
strpcon = strpcon.upper()
code = strpcon.split('.')[0]
if code == "*":
if strpcon == "*": # '*'
for key, value in six.iteritems(exch_period2strpcon):
rst += value
else:
# "*.xxx"
# "*.xxx_period"
k = strpcon.split('.')[1]
for key, value in six.iteritems(exch_period2strpcon):
if '-' in k:
if k == key:
rst += value
elif k == key.split('-')[0]:
rst += value
else:
try:
pcons = code2strpcon[code]
except IndexError:
raise IndexError # 本地不含该文件
else:
for pcon in pcons:
if '-' in strpcon:
# "xxx.xxx_xxx.xxx"
if strpcon == pcon:
rst.append(pcon)
elif '.' in strpcon:
# "xxx.xxx"
if strpcon == pcon.split('-')[0]:
rst.append(pcon)
elif strpcon == pcon.split('.')[0]:
# "xxx"
rst.append(pcon)
return rst
def add_strategies(self, settings):
for setting in settings:
strategy = setting['strategy']
ctx = Context(self._all_data, strategy.name, setting, strategy,
self._max_window)
ctx.data_ref.default_pcontract = self.pcontracts[0]
self._contexts.append(ctx)
yield (Profile(ctx.marks, ctx.blotter, ctx.data_ref))
def _init_strategies(self):
for s_pcontract in self._all_pcontracts:
for context in self._contexts:
context.data_ref.switch_to_pcontract(s_pcontract)
context.strategy.on_init(context)
def run(self):
def run_strategy(ctx, pcontract_symbols):
for s_pcontract in self._all_pcontracts:
ctx.data_ref.switch_to_pcontract(s_pcontract)
ctx.strategy.on_init(ctx)
while True:
# Feeding data of latest.
toremove = set()
for s_pcontract in pcontract_symbols:
ctx.data_ref.switch_to_pcontract(s_pcontract)
has_next = ctx.data_ref.rolling_forward(
ctx.update_datetime)
if not has_next:
toremove.add(s_pcontract)
if toremove:
for s_pcontract in toremove:
pcontract_symbols.remove(s_pcontract)
# call `on_exit` of strategies
if len(pcontract_symbols) == 0:
ctx.data_ref.switch_to_default_pcontract()
ctx.strategy.on_exit(ctx)
return
for s_pcontract in pcontract_symbols:
ctx.data_ref.switch_to_pcontract(s_pcontract)
if not ctx.data_ref.datetime_aligned(ctx.aligned_dt):
continue
# Update original and derived series data
ctx.data_ref.update_original_vars()
ctx.data_ref.update_derived_vars()
ctx.data_ref.original.has_pending_data = False
# call `on_symbol` of strategies
ctx.on_bar = False
ctx.strategy.on_symbol(ctx)
# call `on_bar` of strategies
ctx.data_ref.switch_to_default_pcontract()
ctx.on_bar = True
# 确保交易状态是基于开盘时间的。
ctx.process_trading_events(at_baropen=True)
ctx.strategy.on_bar(ctx)
if not settings['tick_test']:
# 保证有可能在当根Bar成交
ctx.process_trading_events(at_baropen=False)
ctx.aligned_dt = MAX_DATETIME
ctx.aligned_bar_index += 1
for ctx in self._contexts:
log.info("run strategy {0}".format(ctx.strategy_name))
run_strategy(ctx, deepcopy(self._all_pcontracts))
def _load_data(self, strpcons, dt_start, dt_end, n, spec_date):
all_data = OrderedDict()
max_window = -1
log.info("loading data...")
pcontracts = [PContract.from_string(s) for s in strpcons]
pcontracts = sorted(pcontracts, key=PContract.__str__, reverse=True)
for i, pcon in enumerate(pcontracts):
strpcon = str(pcon)
if strpcon in spec_date:
dt_start = spec_date[strpcon][0]
dt_end = spec_date[strpcon][1]
assert (dt_start < dt_end)
if n:
raw_data = self._data_manager.get_last_bars(strpcon, n)
else:
raw_data = self._data_manager.get_bars(strpcon, dt_start,
dt_end)
if len(raw_data) == 0:
continue
all_data[strpcon] = raw_data
max_window = max(max_window, len(raw_data))
if n:
assert (max_window <= n)
if len(all_data) == 0:
assert (False)
# @TODO raise
return all_data, max_window
class ExecuteUnit(object):
""" 策略执行的物理单元，支持多个组合同时运行。
"""
def __init__(self,
pcontracts,
dt_start="1980-1-1",
dt_end="2100-1-1",
n=None,
spec_date={}): # 'symbol':[,]
"""
Args:
pcontracts (list): list of pcontracts(string)
dt_start (datetime/str): start time of all pcontracts
dt_end (datetime/str): end time of all pcontracts
n (int): last n bars
spec_date (dict): time range for specific pcontracts
"""
self.finished_data = []
pcontracts = map(lambda x: x.upper(), pcontracts)
self.pcontracts = pcontracts
self._combs = []
self._data_manager = DataManager()
# str(PContract): DataWrapper
if settings['source'] == 'csv':
self.pcontracts = self._parse_pcontracts(self.pcontracts)
self._default_pcontract = self.pcontracts[0]
self._all_data, self._max_window = self._load_data(
self.pcontracts, dt_start, dt_end, n, spec_date)
self.context = Context(self._all_data, self._max_window)
def _init_strategies(self):
for pcon, dcontext in self._all_data.iteritems():
# switch context
self.context.switch_to_pcontract(pcon)
for i, combination in enumerate(self._combs):
for j, s in enumerate(combination):
self.context.switch_to_strategy(i, j)
s.on_init(self.context)
@deprecated
def _parse_pcontracts(self, pcontracts):
# @TODO test
code2strpcon, exch_period2strpcon = \
self._data_manager.get_code2strpcon()
rst = []
for strpcon in pcontracts:
strpcon = strpcon.upper()
code = strpcon.split('.')[0]
if code == "*":
if strpcon == "*": # '*'
for key, value in exch_period2strpcon.iteritems():
rst += value
else:
# "*.xxx"
# "*.xxx_period"
k = strpcon.split('.')[1]
for key, value in exch_period2strpcon.iteritems():
if '-' in k:
if k == key:
rst += value
elif k == key.split('-')[0]:
rst += value
else:
try:
pcons = code2strpcon[code]
except IndexError:
raise IndexError # 本地不含该文件
else:
for pcon in pcons:
if '-' in strpcon:
# "xxx.xxx_xxx.xxx"
if strpcon == pcon:
rst.append(pcon)
elif '.' in strpcon:
# "xxx.xxx"
if strpcon == pcon.split('-')[0]:
rst.append(pcon)
elif strpcon == pcon.split('.')[0]:
# "xxx"
rst.append(pcon)
#if strpcon in pcon:
#rst.append(strpcon)
return rst
def add_comb(self, comb, settings):
""" 添加策略组合组合
Args:
comb (list): 一个策略组合
"""
self._combs.append(comb)
num_strategy = len(comb)
if 'capital' not in settings:
settings['capital'] = 1000000.0 # 默认资金
logger.info('BackTesting with default capital 1000000.0.')
assert (settings['capital'] > 0)
if num_strategy == 1:
settings['ratio'] = [1]
elif num_strategy > 1 and 'ratio' not in settings:
settings['ratio'] = [1.0 / num_strategy] * num_strategy
assert ('ratio' in settings)
assert (len(settings['ratio']) == num_strategy)
assert (sum(settings['ratio']) - 1.0 < 0.000001)
assert (num_strategy >= 1)
ctxs = []
for i, s in enumerate(comb):
iset = {}
if settings:
iset = {'capital': settings['capital'] * settings['ratio'][i]}
# logger.debug(iset)
ctxs.append(StrategyContext(s.name, iset))
self.context.add_strategy_context(ctxs)
return blotter.Profile(ctxs, self._all_data, self.pcontracts[0],
len(self._combs) - 1)
def run(self):
# @TODO max_window 可用来显示回测进度
# 初始化策略自定义时间序列变量
logger.info("runing strategies...")
self._init_strategies()
pbar = ProgressBar().start()
# todo 对单策略优化
has_next = True
# 遍历每个数据轮, 次数为数据的最大长度
for pcon, data in self._all_data.iteritems():
self.context.switch_to_pcontract(pcon)
self.context.rolling_forward()
while True:
self.context.on_bar = False
# 遍历数据轮的所有合约
for pcon, data in self._all_data.iteritems():
self.context.switch_to_pcontract(pcon)
if self.context.time_aligned():
self.context.update_system_vars()
# 组合遍历
for i, combination in enumerate(self._combs):
# 策略遍历
for j, s in enumerate(combination):
self.context.switch_to_strategy(i, j)
self.context.update_user_vars()
s.on_symbol(self.context)
# 确保单合约回测的默认值
self.context.switch_to_pcontract(self._default_pcontract)
self.context.on_bar = True
# 遍历组合策略每轮数据的最后处理
tick_test = settings['tick_test']
for i, combination in enumerate(self._combs):
# print self.context.ctx_datetime, "--"
for j, s in enumerate(combination):
self.context.switch_to_strategy(i, j)
# 确保交易状态是基于开盘时间的。
self.context.process_trading_events(at_baropen=True)
s.on_bar(self.context)
if not tick_test:
# 保证有可能在当根Bar成交
self.context.process_trading_events(at_baropen=False)
# print self.context.ctx_datetime
self.context.ctx_datetime = datetime(2100, 1, 1)
self.context.ctx_curbar += 1
if self.context.ctx_curbar <= self._max_window:
pbar.update(self.context.ctx_curbar * 100.0 / self._max_window)
#
toremove = []
for pcon, data in self._all_data.iteritems():
self.context.switch_to_pcontract(pcon)
has_next = self.context.rolling_forward()
if not has_next:
toremove.append(pcon)
if toremove:
for key in toremove:
del self._all_data[key]
if len(self._all_data) == 0:
# 策略退出后的处理
for i, combination in enumerate(self._combs):
for j, s in enumerate(combination):
self.context.switch_to_strategy(i, j)
s.on_exit(self.context)
return
pbar.finish()
def _load_data(self, strpcons, dt_start, dt_end, n, spec_date):
all_data = OrderedDict()
max_window = -1
logger.info("loading data...")
pbar = ProgressBar().start()
pcontracts = [PContract.from_string(s) for s in strpcons]
pcontracts = sorted(pcontracts, reverse=True)
for i, pcon in enumerate(pcontracts):
strpcon = str(pcon)
if strpcon in spec_date:
dt_start = spec_date[strpcon][0]
dt_end = spec_date[strpcon][1]
assert (dt_start < dt_end)
if n:
wrapper = self._data_manager.get_last_bars(strpcon, n)
else:
wrapper = self._data_manager.get_bars(strpcon, dt_start,
dt_end)
if len(wrapper) == 0:
continue
all_data[strpcon] = DataContext(wrapper)
max_window = max(max_window, len(wrapper))
pbar.update(i * 100.0 / len(strpcons))
# progressbar.log('')
if n:
assert (max_window <= n)
pbar.finish()
if len(all_data) == 0:
assert (False)
# @TODO raise
return all_data, max_window
class ExecuteUnit(object):
""" 策略执行的物理单元，支持多个策略同时运行。
每个执行单元都可能跟踪多个数据(策略用到的周期合约数据集合)。
其中第一个合约为"主合约" 。 每个执行器可加载多个策略,只要数据需求不超过pcontracts。
:ivar dt_begin: 策略执行的时间起点。
:ivar dt_end: 策略执行的时间终点。
:ivar pcontracts: [str]策略用到的周期合约数据集合。
:ivar _strategies: 策略集合。
:ivar datasource: 数据源。
"""
def __init__(self,
pcontracts,
window_size=0,
dt_start=datetime(1980, 1, 1),
dt_end=datetime(2100, 1, 1),
spec_date={}): # 'symbol':[,]
series.g_rolling = False if window_size == 0 else True
series.g_window = window_size
self.all_data = OrderedDict() # str(PContract): DataWrapper
self.finished_data = []
pcontracts = map(lambda x: x.upper(), pcontracts)
self.pcontracts = pcontracts
self._combs = []
self._window_size = window_size + 1
self._data_manager = DataManager()
for pcon in pcontracts:
if pcon in spec_date:
dt_start = spec_date[pcon][0]
dt_end = spec_date[pcon][1]
assert (dt_start < dt_end)
self.load_data(pcon, dt_start, dt_end)
self.context = Context(self.all_data)
def _init_strategies(self):
for pcon, dcontext in self.all_data.iteritems():
# switch context
self.context.switch_to_contract(pcon)
for i, combination in enumerate(self._combs):
for j, s in enumerate(combination):
self.context.switch_to_strategy(i, j)
s.on_init(self.context)
def add_comb(self, comb, settings={}):
""" 添加策略组合组合
Args:
comb (list): 一个策略组合
"""
self._combs.append(comb)
if settings:
num_strategy = len(comb)
assert (settings['capital'] > 0)
assert len(settings['ratio']) == num_strategy
assert (sum(settings['ratio']) == 1)
ctxs = []
for i, s in enumerate(comb):
iset = {}
if settings:
iset = {'capital': settings['capital'] * settings['ratio'][i]}
logger.debug(iset)
ctxs.append(StrategyContext(s.name, iset))
self.context.add_strategy_context(ctxs)
blotters = [ctx.blotter for ctx in ctxs]
return blotter.Profile(blotters, self.all_data, self.pcontracts[0],
len(self._combs) - 1)
def run(self):
# 初始化策略自定义时间序列变量
print 'runing strategies..'
self._init_strategies()
print 'on_bars..'
# todo 对单策略优化
has_next = True
# 遍历每个数据轮, 次数为数据的最大长度
for pcon, data in self.all_data.iteritems():
self.context.switch_to_contract(pcon)
self.context.rolling_forward()
while True:
# 遍历数据轮的所有合约
for pcon, data in self.all_data.iteritems():
self.context.switch_to_contract(pcon)
if self.context.time_aligned():
self.context.update_system_vars()
# 组合遍历
for i, combination in enumerate(self._combs):
# 策略遍历
for j, s in enumerate(combination):
self.context.switch_to_strategy(i, j)
#self.context.switch_to_data(i, j)
self.context.update_user_vars()
s.on_symbol(self.context)
## 默认的是第一个合约
self.context.switch_to_contract(self.pcontracts[0])
# 每轮数据的最后处理
for i, combination in enumerate(self._combs):
for j, s in enumerate(combination):
self.context.switch_to_strategy(i, j, True)
# 只有在on_final引用跨合约数据才有保证。
self.context.process_trading_events(append=True)
s.on_bar(self.context)
self.context.process_trading_events(append=False)
self.context.ctx_datetime = datetime(2100, 1, 1)
#
toremove = []
for pcon, data in self.all_data.iteritems():
self.context.switch_to_contract(pcon)
has_next = self.context.rolling_forward()
if not has_next:
toremove.append(pcon)
if toremove:
for key in toremove:
del self.all_data[key]
if len(self.all_data) == 0:
# 策略退出后的处理
for i, combination in enumerate(self._combs):
for j, s in enumerate(combination):
self.context.switch_to_strategy(i, j)
s.on_exit(self.context)
return
#print "*********"
def load_data(self,
strpcon,
dt_start=datetime(1980, 1, 1),
dt_end=datetime(2100, 1, 1)):
""" 加载周期合约数据
Args:
strpcon (str): 周期合约
dt_start(datetime): 开始时间
dt_end(datetime): 结束时间
Returns:
pd.DataFrame. k线数据
"""
strpcon = strpcon.upper()
try:
return self.all_data[strpcon]
except KeyError:
wrapper = self._data_manager.get_bars(strpcon, dt_start, dt_end,
self._window_size)
window_size = len(
wrapper.data) if not series.g_rolling else self._window_size
self.all_data[strpcon] = DataContext(wrapper, window_size)
return wrapper
class ExecuteUnit(object):
""" 策略执行的物理单元，支持多个组合同时运行。
"""
def __init__(self, pcontracts,
dt_start="1980-1-1",
dt_end="2100-1-1",
n = None,
spec_date = { }): # 'symbol':[,]
"""
Args:
pcontracts (list): list of pcontracts(string)
dt_start (datetime/str): start time of all pcontracts
dt_end (datetime/str): end time of all pcontracts
n (int): last n bars
spec_date (dict): time range for specific pcontracts
"""
self.finished_data = []
pcontracts = map(lambda x: x.upper(), pcontracts)
self.pcontracts = pcontracts
self._combs = []
self._data_manager = DataManager()
# str(PContract): DataWrapper
self.pcontracts = self._parse_pcontracts(self.pcontracts)
self._all_data, self._max_window = self._load_data(self.pcontracts,
dt_start,
dt_end,
n,
spec_date)
self.context = Context(self._all_data, self._max_window)
def _init_strategies(self):
for pcon, dcontext in self._all_data.iteritems():
# switch context
self.context.switch_to_contract(pcon)
for i, combination in enumerate(self._combs):
for j, s in enumerate(combination):
self.context.switch_to_strategy(i, j)
s.on_init(self.context)
def _parse_pcontracts(self, pcontracts):
## @TODO test
code2strpcon, exch_period2strpcon = self._data_manager.get_code2strpcon()
rst = []
for strpcon in pcontracts:
strpcon = strpcon.upper()
code = strpcon.split('.')[0]
if code == "*" :
if strpcon == "*" : # '*'
for key, value in exch_period2strpcon.iteritems():
rst += value
else:
# "*.xxx"
# "*.xxx_period"
k = strpcon.split('.')[1]
for key, value in exch_period2strpcon.iteritems():
if '-' in k:
if k == key:
rst += value
elif k == key.split('-')[0]:
rst += value
else:
try:
pcons = code2strpcon[code]
except IndexError:
raise IndexError # 本地不含该文件
else:
for pcon in pcons:
if '-' in strpcon:
# "xxx.xxx_xxx.xxx"
if strpcon == pcon:
rst.append(pcon)
elif '.' in strpcon:
# "xxx.xxx"
if strpcon == pcon.split('-')[0]:
rst.append(pcon)
elif strpcon == pcon.split('.')[0]:
# "xxx"
rst.append(pcon)
#if strpcon in pcon:
#rst.append(strpcon)
return rst
def add_comb(self, comb, settings):
""" 添加策略组合组合
Args:
comb (list): 一个策略组合
"""
self._combs.append(comb)
num_strategy = len(comb)
if 'capital' not in settings:
settings['capital'] = 1000000.0 # 默认资金
logger.info('BackTesting with default capital 1000000.0.' )
assert (settings['capital'] > 0)
if num_strategy == 1:
settings['ratio'] = [1]
elif num_strategy > 1 and 'ratio' not in settings:
settings['ratio'] = [1.0/num_strategy] * num_strategy
assert('ratio' in settings)
assert(len(settings['ratio']) == num_strategy)
assert(sum(settings['ratio']) - 1.0 < 0.000001)
assert(num_strategy>=1)
ctxs = []
for i, s in enumerate(comb):
iset = { }
if settings:
iset = { 'capital': settings['capital'] * settings['ratio'][i] }
#logger.debug(iset)
ctxs.append(StrategyContext(s.name, iset))
self.context.add_strategy_context(ctxs)
blotters = [ ctx.blotter for ctx in ctxs]
return blotter.Profile(blotters, self._all_data, self.pcontracts[0], len(self._combs)-1)
def run(self):
## @TODO max_window 可用来显示回测进度
# 初始化策略自定义时间序列变量
logger.info("runing strategies...")
self._init_strategies()
pbar = ProgressBar().start()
# todo 对单策略优化
has_next = True
tick_test = settings['tick_test']
# 遍历每个数据轮, 次数为数据的最大长度
for pcon, data in self._all_data.iteritems():
self.context.switch_to_contract(pcon)
self.context.rolling_forward()
while True:
self.context.on_bar = False
# 遍历数据轮的所有合约
for pcon, data in self._all_data.iteritems():
self.context.switch_to_contract(pcon)
if self.context.time_aligned():
self.context.update_system_vars()
# 组合遍历
for i, combination in enumerate(self._combs):
# 策略遍历
for j, s in enumerate(combination):
self.context.switch_to_strategy(i, j)
self.context.update_user_vars()
s.on_symbol(self.context)
## 确保单合约回测的默认值
self.context.switch_to_contract(self.pcontracts[0])
self.context.on_bar = True
# 遍历组合策略每轮数据的最后处理
for i, combination in enumerate(self._combs):
for j, s in enumerate(combination):
self.context.switch_to_strategy(i, j, True)
self.context.process_trading_events(at_baropen=True)
s.on_bar(self.context)
if not tick_test:
# 保证有可能在当根Bar成交
self.context.process_trading_events(at_baropen=False)
#print self.context.ctx_datetime
self.context.ctx_datetime = datetime(2100,1,1)
self.context.step += 1
if self.context.step <= self._max_window:
pbar.update(self.context.step*100.0/self._max_window)
#
toremove = []
for pcon, data in self._all_data.iteritems():
self.context.switch_to_contract(pcon)
has_next = self.context.rolling_forward()
if not has_next:
toremove.append(pcon)
if toremove:
for key in toremove:
del self._all_data[key]
if len(self._all_data) == 0:
# 策略退出后的处理
for i, combination in enumerate(self._combs):
for j, s in enumerate(combination):
self.context.switch_to_strategy(i, j)
s.on_exit(self.context)
return
pbar.finish()
def _load_data(self, strpcons, dt_start, dt_end, n, spec_date):
all_data = OrderedDict()
max_window = -1
logger.info("loading data...")
pbar = ProgressBar().start()
for i, pcon in enumerate(strpcons):
#print "load data: %s" % pcon
if pcon in spec_date:
dt_start = spec_date[pcon][0]
dt_end = spec_date[pcon][1]
assert(dt_start < dt_end)
if n:
wrapper = self._data_manager.get_last_bars(pcon, n)
else:
wrapper = self._data_manager.get_bars(pcon, dt_start, dt_end)
if len(wrapper) == 0:
continue
all_data[pcon] = DataContext(wrapper)
max_window = max(max_window, len(wrapper))
pbar.update(i*100.0/len(strpcons))
#progressbar.log('')
if n:
assert(max_window <= n)
pbar.finish()
if len(all_data) == 0:
assert(False)
## @TODO raise
return all_data, max_window
class Backend(BackendInterface):
## @TODO singleton
SERVER_FOR_UI = 'backend4ui'
SERVER_FOR_SHELL = "backend4shell"
def __init__(self):
log.info("Init Backend..")
self._dm = DataManager()
self._engine = ZMQEventEngine('Backend')
self._engine.start()
self._shell_srv = EventRPCServer(self._engine,
self.SERVER_FOR_SHELL)
self._ui_srv = EventRPCServer(self._engine,
self.SERVER_FOR_UI)
self.register_functions(self._shell_srv)
self.register_functions(self._ui_srv)
def register_functions(self, server):
server.register('get_all_contracts', self.get_all_contracts)
server.register('get_all_pcontracts', self.get_all_pcontracts)
server.register('get_pcontract', self.get_pcontract)
server.register('get_strategies', self.get_strategies)
server.register('run_strategy', self.run_strategy)
server.register('run_technical', self.run_technical)
def stop(self):
log.info('Backend stopped.')
self._engine.stop()
def get_all_contracts(self):
def _mk_contract(code, exchage):
s = '%s.%s' % (code, exchage)
return Contract(s)
# 模拟接口
df = self._dm.get_contracts()
contracts = [str(_mk_contract(row['code'], row['exchange'])) for _, row in df.iterrows()]
return serialize_all_contracts(contracts)
#data = ['CC.SHFE-1.MINUTE', 'BB.SHFE-1.MINUTE']
#pcons = [PContract.from_string(d) for d in data]
#contracts = [pcon.contract for pcon in pcons]
#return serialize_all_contracts(contracts)
def get_all_pcontracts(self):
# 模拟接口
data = ['CC.SHFE-1.MINUTE', 'BB.SHFE-1.MINUTE']
pcontracts = [PContract.from_string(d) for d in data]
return serialize_all_pcontracts(pcontracts)
def get_pcontract(self, str_pcontract):
da = self._dm.get_bars(str_pcontract)
return serialize_pcontract_bars(str_pcontract, da.data)
def run_strategy(self, name):
""""""
return
def run_technical(self, name):
return
def get_technicals(self):
""" 获取系统的所有指标。 """
from quantdigger.technicals import get_techs
return get_techs()
def get_strategies(self):
return 'hello'
def get_pcontract(self, str_pcontract):
dm = DataManager()
da = dm.get_bars(str_pcontract)
return serialize_pcontract_bars(str_pcontract, da.data)
class ExecuteUnit(object):
""" 策略执行的物理单元，支持多个组合同时运行。
"""
def __init__(self, pcontracts,
dt_start="1980-1-1",
dt_end="2100-1-1",
spec_date = { }): # 'symbol':[,]
"""
Args:
pcontracts (list): list of pcontracts(string)
dt_start (datetime/str): start time of all pcontracts
dt_end (datetime/str): end time of all pcontracts
spec_date (dict): time range for specific pcontracts
"""
self.finished_data = []
pcontracts = map(lambda x: x.upper(), pcontracts)
self.pcontracts = pcontracts
self._combs = []
self._all_data = self._load_data(self.pcontracts, dt_start, dt_end, spec_date)
self.context = Context(self._all_data)
def _init_strategies(self):
for pcon, dcontext in self._all_data.iteritems():
# switch context
self.context.switch_to_contract(pcon)
for i, combination in enumerate(self._combs):
for j, s in enumerate(combination):
self.context.switch_to_strategy(i, j)
s.on_init(self.context)
def add_comb(self, comb, settings):
""" 添加策略组合组合
Args:
comb (list): 一个策略组合
"""
self._combs.append(comb)
num_strategy = len(comb)
if 'capital' not in settings:
settings['capital'] = 1000000.0 # 默认资金
assert (settings['capital'] > 0)
if num_strategy == 1:
settings['ratio'] = [1]
elif num_strategy > 1 and 'ratio' not in settings:
settings['ratio'] = [1.0/num_strategy] * num_strategy
assert('ratio' in settings)
assert(len(settings['ratio']) == num_strategy)
assert(sum(settings['ratio']) == 1)
assert(num_strategy>=1)
ctxs = []
for i, s in enumerate(comb):
iset = { }
if settings:
iset = { 'capital': settings['capital'] * settings['ratio'][i] }
#logger.debug(iset)
ctxs.append(StrategyContext(s.name, iset))
self.context.add_strategy_context(ctxs)
blotters = [ ctx.blotter for ctx in ctxs]
return blotter.Profile(blotters, self._all_data, self.pcontracts[0], len(self._combs)-1)
def run(self):
# 初始化策略自定义时间序列变量
print 'runing strategies..'
self._init_strategies()
print 'on_bars..'
# todo 对单策略优化
has_next = True
tick_test = settings['tick_test']
# 遍历每个数据轮, 次数为数据的最大长度
for pcon, data in self._all_data.iteritems():
self.context.switch_to_contract(pcon)
self.context.rolling_forward()
while True:
# 遍历数据轮的所有合约
for pcon, data in self._all_data.iteritems():
self.context.switch_to_contract(pcon)
if self.context.time_aligned():
self.context.update_system_vars()
# 组合遍历
for i, combination in enumerate(self._combs):
# 策略遍历
for j, s in enumerate(combination):
self.context.switch_to_strategy(i, j)
self.context.update_user_vars()
s.on_symbol(self.context)
## 默认的是第一个合约
self.context.switch_to_contract(self.pcontracts[0])
# 每轮数据的最后处理
for i, combination in enumerate(self._combs):
for j, s in enumerate(combination):
self.context.switch_to_strategy(i, j, True)
self.context.process_trading_events(append=True)
s.on_bar(self.context)
if not tick_test:
# 保证有可能在当根Bar成交
self.context.process_trading_events(append=False)
self.context.ctx_datetime = datetime(2100,1,1)
#
toremove = []
for pcon, data in self._all_data.iteritems():
self.context.switch_to_contract(pcon)
has_next = self.context.rolling_forward()
if not has_next:
toremove.append(pcon)
if toremove:
for key in toremove:
del self._all_data[key]
if len(self._all_data) == 0:
# 策略退出后的处理
for i, combination in enumerate(self._combs):
for j, s in enumerate(combination):
self.context.switch_to_strategy(i, j)
s.on_exit(self.context)
return
#print "*********"
def _load_data(self, pcontracts, dt_start, dt_end, spec_date):
all_data = OrderedDict() # str(PContract): DataWrapper
self._data_manager = DataManager()
for pcon in pcontracts:
if pcon in spec_date:
dt_start = spec_date[pcon][0]
dt_end = spec_date[pcon][1]
assert(dt_start < dt_end)
wrapper = self._data_manager.get_bars(pcon, dt_start, dt_end)
all_data[pcon] = DataContext(wrapper)
return all_data
class Backend:
SERVER_FOR_UI = 'backend4ui'
SERVER_FOR_SHELL = "backend4shell"
def __init__(self):
log.info("Init Backend..")
self._dm = DataManager()
self._engine = ZMQEventEngine('Backend')
self._engine.start()
self._shell_srv = EventRPCServer(self._engine, self.SERVER_FOR_SHELL)
self._ui_srv = EventRPCServer(self._engine, self.SERVER_FOR_UI)
self.register_functions(self._shell_srv)
self.register_functions(self._ui_srv)
def register_functions(self, server):
server.register('get_all_contracts', self.get_all_contracts)
server.register('get_all_pcontracts', self.get_all_pcontracts)
server.register('get_pcontract', self.get_pcontract)
server.register('get_strategies', self.get_strategies)
server.register('run_strategy', self.run_strategy)
server.register('run_technical', self.run_technical)
server.register('test_speed', self.test_speed)
def stop(self):
log.info('Backend stopped.')
self._engine.stop()
def get_all_contracts(self):
def _mk_contract(code, exchage):
s = '%s.%s' % (code, exchage)
return Contract(s)
# 模拟接口
df = self._dm.get_contracts()
contracts = [
str(_mk_contract(row['code'], row['exchange']))
for _, row in df.iterrows()
]
return serialize_all_contracts(contracts)
def get_all_pcontracts(self):
# 模拟接口
data = ['CC.SHFE-1.MINUTE', 'BB.SHFE-1.MINUTE']
pcontracts = [PContract.from_string(d) for d in data]
return serialize_all_pcontracts(pcontracts)
def get_pcontract(self, str_pcontract):
da = self._dm.get_bars(str_pcontract)
return serialize_pcontract_bars(str_pcontract, da.data)
def run_strategy(self, name):
""""""
return
def run_technical(self, name):
return
def get_technicals(self):
""" 获取系统的所有指标。 """
from quantdigger.technicals import get_techs
return get_techs()
def get_strategies(self):
return 'hello'
def test_speed(self):
return 'hello world!'
