def action_process(d: DataDir, config: Configuration, logger: Logger,
userid: int):
if not os.path.exists(d.user_config_path(userid)):
raise ValueError(f'config file for user {userid} not found')
user_data = DataDir(config)
user_data.add_config(d.user_config_path(userid))
acc: UserAccounts = d.UserAccounts(userid)
# try to convert xls to xlsx
for datafile in d.UserXLSDataFiles(userid):
try:
import pyexcel as p
xlsx_file = datafile + 'x'
logger.info(f'Trying to convert {datafile} to {xlsx_file}')
if os.path.exists(xlsx_file):
logger.info(f'IGNORED (Already exists) {xlsx_file}')
else:
p.save_book_as(file_name=datafile, dest_file_name=xlsx_file)
logger.info(f'Converted')
except ImportError as e:
logger.error('Error during xls to xlsx conversion',
'Error',
exc_info=e)
datafiles: List[str] = d.UserDataFiles(userid)
if len(datafiles) == 0:
logger.error('No xlsx files')
import_tag: str = f'IMP-{datetime.date.today().strftime("%Y-%m-%d")}'
for datafile in d.UserDataFiles(userid):
#datafile_parts:str = os.path.splitext(os.path.basename(datafile))
outputfn = os.path.join(user_data.user_folder_output(userid),
os.path.basename(datafile))
stmt_data = XlsxDriver(filepath=datafile,
output_file_path=outputfn,
first_row=get_config_int(
user_data.config, 'datafile.first_row'))
stmt_data.read()
from execution_context import ExecutionContext
ec = ExecutionContext(stmt_data, acc, list_files('rules', r'.*\.yaml'),
d.UserTaxNumber(userid),
d.user_folder_output(userid))
(total_rows, rows_processed) = ec.execute(import_tag)
stmt_data.close()
logger.info(
f'DONE. Processed {total_rows} rows, prepared {rows_processed} statements with tag {import_tag}'
)
def _handle_modified_date(self, exec_context: ExecutionContext) -> bool:
"""Based on the ModifiedDateDecisionEnum returned by compare_modified date, take
an action.
Args:
exec_context: The execution context
Returns:
True if the action should finish executing, False if not.
"""
choice = self.file_source.compare_modified_date(self.dest_path)
if choice is not None:
if choice == ModifiedDateDecisionEnum.stop_execution:
utils.log_and_raise(
logger.error,
f"User stopped execution at action with key {self.key}",
UserStoppedExecutionException,
errors.AC_USER_STOPPED_EXECUTION,
)
elif choice == ModifiedDateDecisionEnum.skip_this_action:
logger.info(f"Skipped action with key {self.key}")
return False
elif choice == ModifiedDateDecisionEnum.proceed_once:
pass
elif choice == ModifiedDateDecisionEnum.ignore_in_future:
exec_context.skip_modified_date_warning = True
return True
return True
def test_happy_path_github__file_sync_created(self):
test_source_path = "test_source_path"
test_dest_path = "test_dest_path"
test_key = "test_key"
test_dep_keys = ["test_dep_key_one", "test_dep_key_two"]
test_user = "******"
test_repo_name = "test_repo_name"
test_repository = f"www.github.com/{test_user}/{test_repo_name}"
test_app_dir = "/app-dir"
test_file_sync_config = {
const.LOCATION_TYPE_NODE: const.LOCATION_TYPE_GITHUB,
const.SOURCE_FILE_PATH: test_source_path,
const.REPOSITORY: test_repository,
const.DEST_FILE_PATH: test_dest_path,
const.NODE_KEY: test_key,
const.DEPENDENCY: test_dep_keys,
}
file_sync = node_parsers.parse_file_sync(test_file_sync_config, ExecutionContext(pyrsonalizer_directory=test_app_dir))
expected_file_sync = actions.FileSync(
backend=actions.FileSyncBackendType.github,
file_source=actions.GithubFileLocation(repo_url=test_repository, relative_path=test_source_path, app_dir=test_app_dir),
local_path=pathlib.Path(test_dest_path),
overwrite=False,
key=test_key,
dependency_keys=test_dep_keys,
)
def before_trading_(self, event):
with ExecutionContext(EXECUTION_PHASE.BEFORE_TRADING):
self._stage = 'before_trading'
for day_rule, time_rule, func in self._registry:
if day_rule() and time_rule():
with ModifyExceptionFromType(EXC_TYPE.USER_EXC):
func(self._ucontext, None)
self._stage = None
def init(self):
if not self._init:
return
with ExecutionContext(EXECUTION_PHASE.ON_INIT):
with ModifyExceptionFromType(EXC_TYPE.USER_EXC):
self._init(self._user_context)
Environment.get_instance().event_bus.publish_event(Event(EVENT.POST_USER_INIT))
def next_bar_(self, event):
bars = event.bar_dict
with ExecutionContext(EXECUTION_PHASE.SCHEDULED):
self._current_minute = self._minutes_since_midnight(
self._ucontext.now.hour, self._ucontext.now.minute)
for day_rule, time_rule, func in self._registry:
if day_rule() and time_rule():
with ModifyExceptionFromType(EXC_TYPE.USER_EXC):
func(self._ucontext, bars)
self._last_minute = self._current_minute
def test_circular_dependency__raises(self):
node_one = execution_graph.GraphNode(TestAction("1"))
node_two = execution_graph.GraphNode(TestAction("2"))
node_three = execution_graph.GraphNode(TestAction("3"))
node_four = execution_graph.GraphNode(TestAction("4"))
node_one.add_child(node_two)
node_two.add_child(node_three)
node_three.add_child(node_four)
node_four.add_child(node_two)
exec_graph = execution_graph.ExecutionGraph(node_one, [])
with pytest.raises(execution_graph.CircularDependencyException):
exec_graph.execute(ExecutionContext())
def mavg(self, intervals, frequency='1d'):
if frequency == 'day':
frequency = '1d'
if frequency == 'minute':
frequency = '1m'
# copy form history
env = Environment.get_instance()
dt = env.calendar_dt
if (env.config.base.frequency == '1m' and frequency == '1d') or ExecutionContext.phase() == EXECUTION_PHASE.BEFORE_TRADING:
# 在分钟回测获取日线数据, 应该推前一天
dt = env.data_proxy.get_previous_trading_date(env.calendar_dt.date())
bars = env.data_proxy.fast_history(self._instrument.order_book_id, intervals, frequency, 'close', dt)
return bars.mean()
def call(self, context: execution_context.ExecutionContext, arg_ast, kwarg_ast):
args = [arg.execute(context) for arg in arg_ast]
kwargs = {k: v.execute(context) for k, v in kwarg_ast.items()}
result = None
context.add_scope()
if isinstance(self.action, list):
for i, param in enumerate(self.parameters):
context.declare(param.name, 'parameter')
context.assign(param.name, args[i])
for statement in self.action:
try:
statement.execute(context)
except Return as e:
result = e.value
break
else:
result = self.action(context, *args, **kwargs)
context.remove_scope()
return result
def test_happy_path__all_nodes_executed(self):
node_one = execution_graph.GraphNode(TestAction("1"))
node_two = execution_graph.GraphNode(TestAction("2"))
node_three = execution_graph.GraphNode(TestAction("3"))
node_four = execution_graph.GraphNode(TestAction("4"))
node_one.add_child(node_two)
node_one.add_child(node_four)
node_two.add_child(node_three)
node_three.add_child(node_four)
exec_graph = execution_graph.ExecutionGraph(node_one, [])
try:
exec_graph.execute(ExecutionContext())
except:
pytest.fail()
global VISITED_COUNT
assert VISITED_COUNT == 4
VISITED_COUNT = 0
def vwap(self, intervals, frequency='1d'):
if frequency == 'day':
frequency = '1d'
if frequency == 'minute':
frequency = '1m'
# copy form history
env = Environment.get_instance()
dt = env.calendar_dt
if (env.config.base.frequency == '1m' and frequency == '1d') or ExecutionContext.phase() == EXECUTION_PHASE.BEFORE_TRADING:
# 在分钟回测获取日线数据, 应该推前一天
dt = env.data_proxy.get_previous_trading_date(env.calendar_dt.date())
bars = env.data_proxy.fast_history(self._instrument.order_book_id, intervals, frequency, ['close', 'volume'], dt)
sum = bars['volume'].sum()
if sum == 0:
# 全部停牌
return 0
return np.dot(bars['close'], bars['volume']) / sum
def before_trading(self, event):
with ExecutionContext(EXECUTION_PHASE.BEFORE_TRADING):
with ModifyExceptionFromType(EXC_TYPE.USER_EXC):
self._before_trading(self._user_context)
class TestGraphParser:
exec_context = ExecutionContext(pyrsonalizer_directory="./test_app_dir")
def test_path_does_not_exist__raises(self, mock_log_and_raise):
test_path = "path"
with pytest.raises(ValueError):
graph_parser.parse_execution_graph(pathlib.Path(test_path),
self.exec_context)
call_args = mock_log_and_raise.call_args[0]
assert errors.GP_PATH_DOES_NOT_EXIST in call_args
def test_no_class_mapping__raises(self, mock_log_and_raise,
mock_get_config_bad_klass):
test_path = "path"
with pytest.raises(ValueError):
graph_parser.parse_execution_graph(pathlib.Path(test_path),
self.exec_context)
call_args = mock_log_and_raise.call_args[0]
assert errors.GP_NO_CLASS_MAP in call_args
def test_bad_dependency_ref__raises(self, mock_log_and_raise,
mock_get_config_bad_deps):
test_path = "path"
with pytest.raises(KeyError):
graph_parser.parse_execution_graph(pathlib.Path(test_path),
self.exec_context)
call_args = mock_log_and_raise.call_args[0]
assert errors.GP_BAD_DEPENDENCY_REF in call_args
def test_happy_path(self, mock_get_config_happy):
result_graph = None
try:
result_graph = graph_parser.parse_execution_graph(
pathlib.Path("test_path"), self.exec_context)
except BaseException as err:
pytest.fail()
assert isinstance(result_graph._root, execution_graph.GraphNode)
children = result_graph._root.children
assert len(children) == 1
child = children[0]
assert child.value.backend == actions.FileSyncBackendType.local
assert str(child.value.file_source.path) == "source_path"
assert str(child.value.dest_path) == "dest_path"
assert child.value.overwrite is False
def test_happy_path_with_deps(self, mock_get_config_happy_deps_local):
result_graph = None
try:
result_graph = graph_parser.parse_execution_graph(
pathlib.Path("test_path"), self.exec_context)
except BaseException as err:
pytest.fail()
assert isinstance(result_graph._root, execution_graph.GraphNode)
children = result_graph._root.children
assert len(children) == 1
child = children[0]
assert child.value.backend == actions.FileSyncBackendType.local
assert str(child.value.file_source.path) == "source_path"
assert str(child.value.dest_path) == "dest_path"
assert child.value.overwrite is False
assert child.value.key == "key_one"
grandchild = child.children[0]
assert grandchild.value.backend == actions.FileSyncBackendType.local
assert str(grandchild.value.file_source.path) == "source_path"
assert str(grandchild.value.dest_path) == "dest_path"
assert grandchild.value.overwrite is False
assert grandchild.value.key == "key_two"
def handle_tick(self, event):
tick = event.tick
with ExecutionContext(EXECUTION_PHASE.ON_TICK):
with ModifyExceptionFromType(EXC_TYPE.USER_EXC):
self._handle_tick(self._user_context, tick)
def runCmd(cmd): ExecutionContext.run_cmd(cmd)
def after_trading(self, event):
with ExecutionContext(EXECUTION_PHASE.AFTER_TRADING):
with ModifyExceptionFromType(EXC_TYPE.USER_EXC):
self._after_trading(self._user_context)
def history_bars(order_book_id,
bar_count,
frequency,
fields=None,
skip_suspended=True,
include_now=False,
adjust_type='pre'):
"""
获取指定合约的历史行情，同时支持日以及分钟历史数据。不能在init中调用。 注意，该API会自动跳过停牌数据。
日回测获取分钟历史数据：不支持
日回测获取日历史数据
========================= ===================================================
调用时间 返回数据
========================= ===================================================
T日before_trading T-1日day bar
T日handle_bar T日day bar
========================= ===================================================
分钟回测获取日历史数据
========================= ===================================================
调用时间 返回数据
========================= ===================================================
T日before_trading T-1日day bar
T日handle_bar T-1日day bar
========================= ===================================================
分钟回测获取分钟历史数据
========================= ===================================================
调用时间 返回数据
========================= ===================================================
T日before_trading T-1日最后一个minute bar
T日handle_bar T日当前minute bar
========================= ===================================================
:param order_book_id: 合约代码
:type order_book_id: `str`
:param int bar_count: 获取的历史数据数量，必填项
:param str frequency: 获取数据什么样的频率进行。'1d'或'1m'分别表示每日和每分钟，必填项
:param str fields: 返回数据字段。必填项。见下方列表。
========================= ===================================================
fields 字段名
========================= ===================================================
datetime 时间戳
open 开盘价
high 最高价
low 最低价
close 收盘价
volume 成交量
total_turnover 成交额
datetime int类型时间戳
open_interest 持仓量（期货专用）
basis_spread 期现差（股指期货专用）
settlement 结算价（期货日线专用）
prev_settlement 结算价（期货日线专用）
========================= ===================================================
:param bool skip_suspended: 是否跳过停牌数据
:param bool include_now: 是否包含当前数据
:param str adjust_type: 复权类型，默认为前复权 pre；可选 pre, none, post
:return: `ndarray`, 方便直接与talib等计算库对接，效率较history返回的DataFrame更高。
:example:
获取最近5天的日线收盘价序列（策略当前日期为20160706）:
.. code-block:: python3
:linenos:
[In]
logger.info(history_bars('000002.XSHE', 5, '1d', 'close'))
[Out]
[ 8.69 8.7 8.71 8.81 8.81]
"""
order_book_id = assure_order_book_id(order_book_id)
env = Environment.get_instance()
dt = env.calendar_dt
if frequency[-1] == 'm' and env.config.base.frequency == '1d':
raise RQInvalidArgument('can not get minute history in day back test')
if adjust_type not in {'pre', 'post', 'none'}:
raise RuntimeError('invalid adjust_type')
if frequency == '1d':
sys_frequency = Environment.get_instance().config.base.frequency
if ((sys_frequency in ['1m', 'tick'] and not include_now)
or ExecutionContext.phase() == EXECUTION_PHASE.BEFORE_TRADING):
dt = env.data_proxy.get_previous_trading_date(
env.trading_dt.date())
# 当 EXECUTION_PHASE.BEFORE_TRADING 的时候，强制 include_now 为 False
include_now = False
if sys_frequency == "1d":
# 日回测不支持 include_now
include_now = False
return env.data_proxy.history_bars(order_book_id,
bar_count,
frequency,
fields,
dt,
skip_suspended=skip_suspended,
include_now=include_now,
adjust_type=adjust_type,
adjust_orig=env.trading_dt)
def handle_bar(self, event):
bar_dict = event.bar_dict
with ExecutionContext(EXECUTION_PHASE.ON_BAR):
with ModifyExceptionFromType(EXC_TYPE.USER_EXC):
self._handle_bar(self._user_context, bar_dict)
def run(config, source_code=None, user_funcs=None):
env = Environment(config)
persist_helper = None
init_succeed = False
mod_handler = ModHandler()
try:
if source_code is not None:
env.set_strategy_loader(SourceCodeStrategyLoader(source_code))
elif user_funcs is not None:
env.set_strategy_loader(UserFuncStrategyLoader(user_funcs))
else:
env.set_strategy_loader(
FileStrategyLoader(config.base.strategy_file))
env.set_global_vars(GlobalVars())
mod_handler.set_env(env)
mod_handler.start_up()
if not env.data_source:
env.set_data_source(BaseDataSource(config.base.data_bundle_path))
env.set_data_proxy(DataProxy(env.data_source))
Scheduler.set_trading_dates_(env.data_source.get_trading_calendar())
scheduler = Scheduler(config.base.frequency)
mod_scheduler._scheduler = scheduler
env._universe = StrategyUniverse()
_adjust_start_date(env.config, env.data_proxy)
_validate_benchmark(env.config, env.data_proxy)
broker = env.broker
assert broker is not None
env.portfolio = broker.get_portfolio()
env.benchmark_portfolio = create_benchmark_portfolio(env)
event_source = env.event_source
assert event_source is not None
bar_dict = BarMap(env.data_proxy, config.base.frequency)
env.set_bar_dict(bar_dict)
if env.price_board is None:
from core.bar_dict_price_board import BarDictPriceBoard
env.price_board = BarDictPriceBoard()
ctx = ExecutionContext(const.EXECUTION_PHASE.GLOBAL)
ctx._push()
# FIXME
start_dt = datetime.datetime.combine(config.base.start_date,
datetime.datetime.min.time())
env.calendar_dt = start_dt
env.trading_dt = start_dt
env.event_bus.publish_event(Event(EVENT.POST_SYSTEM_INIT))
scope = create_base_scope()
scope.update({"g": env.global_vars})
apis = api_helper.get_apis(config.base.account_list)
scope.update(apis)
scope = env.strategy_loader.load(scope)
if env.config.extra.enable_profiler:
enable_profiler(env, scope)
ucontext = StrategyContext()
user_strategy = Strategy(env.event_bus, scope, ucontext)
scheduler.set_user_context(ucontext)
if not config.extra.force_run_init_when_pt_resume:
with run_with_user_log_disabled(disabled=config.base.resume_mode):
user_strategy.init()
if config.extra.context_vars:
for k, v in six.iteritems(config.extra.context_vars):
setattr(ucontext, k, v)
if config.base.persist:
persist_provider = env.persist_provider
persist_helper = PersistHelper(persist_provider, env.event_bus,
config.base.persist_mode)
persist_helper.register('core', CoreObjectsPersistProxy(scheduler))
persist_helper.register('user_context', ucontext)
persist_helper.register('global_vars', env.global_vars)
persist_helper.register('universe', env._universe)
if isinstance(event_source, Persistable):
persist_helper.register('event_source', event_source)
persist_helper.register('portfolio', env.portfolio)
if env.benchmark_portfolio:
persist_helper.register('benchmark_portfolio',
env.benchmark_portfolio)
for name, module in six.iteritems(env.mod_dict):
if isinstance(module, Persistable):
persist_helper.register('mod_{}'.format(name), module)
# broker will restore open orders from account
if isinstance(broker, Persistable):
persist_helper.register('broker', broker)
persist_helper.restore()
env.event_bus.publish_event(Event(EVENT.POST_SYSTEM_RESTORED))
init_succeed = True
# When force_run_init_when_pt_resume is active,
# we should run `init` after restore persist data
if config.extra.force_run_init_when_pt_resume:
assert config.base.resume_mode == True
with run_with_user_log_disabled(disabled=False):
user_strategy.init()
from core.executor import Executor
Executor(env).run(bar_dict)
# print(env.__dict__)
if env.profile_deco:
output_profile_result(env)
result = mod_handler.tear_down(const.EXIT_CODE.EXIT_SUCCESS)
system_log.debug(_(u"strategy run successfully, normal exit"))
return result
except CustomException as e:
if init_succeed and env.config.base.persist and persist_helper:
persist_helper.persist()
user_detail_log.exception(_(u"strategy execute exception"))
user_system_log.error(e.error)
better_exceptions.excepthook(e.error.exc_type, e.error.exc_val,
e.error.exc_tb)
mod_handler.tear_down(const.EXIT_CODE.EXIT_USER_ERROR, e)
except Exception as e:
if init_succeed and env.config.base.persist and persist_helper:
persist_helper.persist()
exc_type, exc_val, exc_tb = sys.exc_info()
user_exc = create_custom_exception(exc_type, exc_val, exc_tb,
config.base.strategy_file)
better_exceptions.excepthook(exc_type, exc_val, exc_tb)
user_system_log.error(user_exc.error)
code = const.EXIT_CODE.EXIT_USER_ERROR
if not is_user_exc(exc_val):
system_log.exception(_(u"strategy execute exception"))
code = const.EXIT_CODE.EXIT_INTERNAL_ERROR
else:
user_detail_log.exception(_(u"strategy execute exception"))
mod_handler.tear_down(code, user_exc)
import_tag: str = f'IMP-{datetime.date.today().strftime("%Y-%m-%d")}'
for datafile in d.UserDataFiles(userid):
#datafile_parts:str = os.path.splitext(os.path.basename(datafile))
outputfn = os.path.join(user_data.user_folder_output(userid),
os.path.basename(datafile))
stmt_data = XlsxDriver(filepath=datafile,
output_file_path=outputfn,
first_row=get_config_int(
user_data.config, 'datafile.first_row'))
stmt_data.read()
from execution_context import ExecutionContext
ec = ExecutionContext(stmt_data, d.UserAccounts(userid),
list_files('rules', r'.*\.yaml'),
d.UserTaxNumber(userid),
d.user_folder_output(userid))
(total_rows, rows_processed) = ec.execute(import_tag)
stmt_data.close()
logger.info(
f'DONE. Processed {total_rows} rows, prepared {rows_processed} statements with tag {import_tag}'
)
if config['action'].as_str() == 'push':
for userid in d.Users():
if not os.path.exists(d.user_config_path(userid)):
raise ValueError(f'config file for user {userid} not found')
user_data = DataDir(config)
user_data.add_config(d.user_config_path(userid))
output_folder = d.user_folder_output(userid)
if not os.path.exists(output_folder):
def add_native_functions(self,
context: execution_context.ExecutionContext):
for name, native_fn in native_functions.FUNCTIONS.items():
fn = function.Function(name, [], [], native_fn)
context.declare(name, 'native_function')
context.assign(name, fn)