def run_tests(file_path=None):
if file_path is not None:
files = [file_path]
else:
files = [f for f in os.listdir(TEST_DIR) if f.find("test") == 0]
error_map = {}
for filename in files:
try:
r, result_data = run_test(filename)
if r is not None:
error_map[filename.replace(".py", "")] = result_data
except Exception as e:
system_log.exception()
error_map[filename.replace(".py", "")] = e
for filename, result_data in iteritems(error_map):
print(u"*" * 20, u"[{}]did not pass!".format(filename), u"*" * 20)
if isinstance(result_data, Exception):
system_log.error(result_data)
else:
df, old_df, result = result_data
# print("+" * 10, "old test Dataframe: ", "+" * 10)
# print(old_df.drop(result.columns[result.all()], axis=1))
# print("+" * 10, "new test Dataframe: ", "+" * 10)
# print(df.drop(result.columns[result.all()], axis=1))
print(result.all())
print(u"=" * 40)
print(u"[{}|{}] strategies has been passed!".format(len(files) - len(error_map), len(files)))
return len(error_map)
def _exception_handler(e):
user_system_log.exception(_(u"strategy execute exception"))
if not is_user_exc(e.error.exc_val):
system_log.exception(_(u"strategy execute exception"))
return const.EXIT_CODE.EXIT_INTERNAL_ERROR
return const.EXIT_CODE.EXIT_USER_ERROR
def run_tests(file_path=None):
if file_path is not None:
files = [file_path]
else:
files = [f for f in os.listdir(TEST_DIR) if f.find("test") == 0]
error_map = {}
for filename in files:
try:
r, result_data = run_test(filename)
if r is not None:
error_map[filename.replace(".py", "")] = result_data
except Exception as e:
system_log.exception()
error_map[filename.replace(".py", "")] = e
for filename, result_data in iteritems(error_map):
print(u"*" * 20, u"[{}]did not pass!".format(filename), u"*" * 20)
if isinstance(result_data, Exception):
system_log.error(result_data)
else:
df, old_df, result = result_data
# print("+" * 10, "old test Dataframe: ", "+" * 10)
# print(old_df.drop(result.columns[result.all()], axis=1))
# print("+" * 10, "new test Dataframe: ", "+" * 10)
# print(df.drop(result.columns[result.all()], axis=1))
print(result.all())
print(u"=" * 40)
print(u"[{}|{}] strategies has been passed!".format(
len(files) - len(error_map), len(files)))
return len(error_map)
def __getitem__(self, key):
if not isinstance(key, six.string_types):
raise patch_user_exc(ValueError('invalid key {} (use order_book_id please)'.format(key)))
instrument = self._data_proxy.instruments(key)
if instrument is None:
raise patch_user_exc(ValueError('invalid order book id or symbol: {}'.format(key)))
order_book_id = instrument.order_book_id
try:
return self._cache[order_book_id]
except KeyError:
try:
if not self._dt:
return BarObject(instrument, NANDict, self._dt)
if ExecutionContext.phase() == EXECUTION_PHASE.OPEN_AUCTION:
bar = self._data_proxy.get_open_auction_bar(order_book_id, self._dt)
else:
bar = self._data_proxy.get_bar(order_book_id, self._dt, self._frequency)
except PermissionError:
raise
except Exception as e:
system_log.exception(e)
raise patch_user_exc(KeyError(_(u"id_or_symbols {} does not exist").format(key)))
if bar is None:
return BarObject(instrument, NANDict, self._dt)
else:
self._cache[order_book_id] = bar
return bar
def __getitem__(self, key):
if not isinstance(key, six.string_types):
raise patch_user_exc(
ValueError(
'invalid key {} (use order_book_id please)'.format(key)))
instrument = self._data_proxy.instruments(key)
if instrument is None:
raise patch_user_exc(
ValueError('invalid order book id or symbol: {}'.format(key)))
order_book_id = instrument.order_book_id
try:
return self._cache[order_book_id]
except KeyError:
try:
#system_log.debug("order_book_id1=" + order_book_id + ", dt=" + str(self._dt))
bar = self._data_proxy.get_bar(order_book_id, self._dt,
self._frequency)
#system_log.debug("order_book_id2=" + order_book_id + ", dt=" + str(self._dt))
except Exception as e:
system_log.exception(e)
raise patch_user_exc(
KeyError(
_(u"id_or_symbols {} does not exist").format(key)))
if bar is None:
return BarObject(instrument, NANDict, self._dt)
else:
self._cache[order_book_id] = bar
return bar
def quotation_server(redis_url):
"""
[sys_stock_realtime] quotation service, download market data into redis
Multiple RQAlpha instance can use single market data service.
"""
import redis
import time
import json
from .utils import get_realtime_quotes
redis_client = redis.from_url(redis_url)
from rqalpha.data.data_proxy import DataProxy
config = parse_config({}, verify_config=False)
data_source = BaseDataSource(config.base.data_bundle_path)
data_proxy = DataProxy(data_source)
order_book_id_list = sorted(ins.order_book_id for ins in data_proxy.all_instruments("CS"))
def record_market_data(total_df):
for order_book_id, item in total_df.iterrows():
redis_client[order_book_id] = json.dumps(item.to_dict())
while True:
try:
total_df = get_realtime_quotes(order_book_id_list, include_limit=True)
except Exception as e:
system_log.exception("get_realtime_quotes fail. {}", e)
continue
system_log.info("Fetching snapshots, size {}", len(total_df))
record_market_data(total_df)
time.sleep(1)
def _restore_obj(self, key, obj):
state = self._persist_provider.load(key)
system_log.debug('restore {} with state = {}', key, state)
if not state:
return False
try:
obj.set_state(state)
except Exception:
system_log.exception('restore failed: key={} state={}'.format(key, state))
return True
def quotation_worker(self):
while True:
if not is_holiday_today() and is_tradetime_now():
order_book_id_list = sorted([instruments.order_book_id for instruments in self._env.data_proxy.all_instruments("CS", self._env.trading_dt)])
try:
data_board.realtime_quotes_df = get_realtime_quotes(order_book_id_list)
except Exception as e:
system_log.exception(_("get_realtime_quotes fail"))
continue
time.sleep(1)
def tear_down(self, *args):
result = {}
for mod_name, __ in reversed(self._mod_list):
try:
ret = self._mod_dict[mod_name].tear_down(*args)
except Exception as e:
system_log.exception("tear down fail for {}", mod_name)
continue
if ret is not None:
result[mod_name] = ret
return result
def tear_down(self, *args):
result = {}
for mod_name, __ in reversed(self._mod_list):
try:
ret = self._mod_dict[mod_name].tear_down(*args)
except Exception as e:
system_log.exception("tear down fail for {}", mod_name)
continue
if ret is not None:
result[mod_name] = ret
return result
def _exception_handler(e):
better_exceptions.excepthook(e.error.exc_type, e.error.exc_val, e.error.exc_tb)
user_system_log.error(e.error)
if not is_user_exc(e.error.exc_val):
code = const.EXIT_CODE.EXIT_INTERNAL_ERROR
system_log.exception(_(u"strategy execute exception"))
else:
code = const.EXIT_CODE.EXIT_USER_ERROR
user_detail_log.exception(_(u"strategy execute exception"))
return code
def tear_down(self, *args):
result = {}
for mod_name, __ in reversed(self._mod_list):
try:
basic_system_log.debug(_(u"mod tear_down [START] {}").format(mod_name))
ret = self._mod_dict[mod_name].tear_down(*args)
basic_system_log.debug(_(u"mod tear_down [END] {}").format(mod_name))
except Exception as e:
system_log.exception("tear down fail for {}", mod_name)
continue
if ret is not None:
result[mod_name] = ret
return result
def tear_down(self, *args):
result = {}
for mod_name, __ in reversed(self._mod_list):
try:
basic_system_log.debug(_(u"mod tear_down [START] {}").format(mod_name))
ret = self._mod_dict[mod_name].tear_down(*args)
basic_system_log.debug(_(u"mod tear_down [END] {}").format(mod_name))
except Exception as e:
system_log.exception("tear down fail for {}", mod_name)
continue
if ret is not None:
result[mod_name] = ret
return result
def quotation_worker(self):
while True:
if not is_holiday_today() and is_tradetime_now():
order_book_id_list = sorted(Environment.get_instance().data_proxy.all_instruments("CS").order_book_id.tolist())
code_list = [order_book_id_2_tushare_code(code) for code in order_book_id_list]
try:
self._env.data_source.realtime_quotes_df = get_realtime_quotes(code_list)
except Exception as e:
system_log.exception("get_realtime_quotes fail")
continue
time.sleep(1)
def persist(self, *_):
for key, obj in six.iteritems(self._objects):
try:
state = obj.get_state()
if not state:
continue
md5 = hashlib.md5(state).hexdigest()
if self._last_state.get(key) == md5:
continue
self._persist_provider.store(key, state)
except Exception as e:
system_log.exception("PersistHelper.persist fail")
else:
self._last_state[key] = md5
def _init_gateway(self):
self.gateway_type = self._config.gateway_type
if self.gateway_type == 'CTP':
try:
from .vnpy_gateway import RQVNCTPGateway
self.vnpy_gateway = RQVNCTPGateway(
self.event_engine, self.gateway_type,
dict(getattr(self._config, self.gateway_type)))
QueryExecutor.interval = self._config.query_interval
QueryExecutor.start()
except ImportError as e:
system_log.exception("No Gateway named CTP")
else:
system_log.error('No Gateway named {}', self.gateway_type)
def persist(self, *args):
for key, obj in six.iteritems(self._objects):
try:
state = obj.get_state()
if not state:
continue
md5 = hashlib.md5(state).hexdigest()
if self._last_state.get(key) == md5:
continue
self._persist_provider.store(key, state)
except Exception as e:
system_log.exception("PersistHelper.persist fail")
else:
self._last_state[key] = md5
def quotation_server(redis_url):
"""
[stock_realtime] quotation service, download market data into redis
Multiple RQAlpha instance can use single market data service.
"""
import redis
import time
import json
from .utils import get_realtime_quotes
redis_client = redis.from_url(redis_url)
from rqalpha.environment import Environment
from rqalpha.data.data_proxy import DataProxy
# 通过 parse_config 函数 获取 默认 data_bundle 的配置
# 由于 parse_config 会进行配置检查，如果没有设置 account 会提示报错，因此随意设置一个股票初始资金，来保证正确获取默认参数配置
config = parse_config({
'base': {
'accounts': {
'stock': 10000
}
}
})
Environment(config)
data_source = BaseDataSource(config.base.data_bundle_path)
data_proxy = DataProxy(data_source)
order_book_id_list = sorted(ins.order_book_id for ins in data_proxy.all_instruments("CS"))
def record_market_data(total_df):
for order_book_id, item in total_df.iterrows():
redis_client[order_book_id] = json.dumps(item.to_dict())
retry_cnt = 0
while True:
try:
total_df = get_realtime_quotes(order_book_id_list, include_limit=True)
except (OSError, IOError) as e:
system_log.exception("get_realtime_quotes socket error. retry {} {}", retry_cnt, e)
time.sleep(retry_cnt * 2)
retry_cnt += 1
continue
system_log.info("Fetching snapshots, size {}", len(total_df))
record_market_data(total_df)
time.sleep(1)
retry_cnt = 0
def _exception_handler(e):
try:
sys.excepthook(e.error.exc_type, e.error.exc_val, e.error.exc_tb)
except Exception as e:
system_log.exception("hook exception failed")
user_system_log.error(e.error)
if not is_user_exc(e.error.exc_val):
code = const.EXIT_CODE.EXIT_INTERNAL_ERROR
system_log.error(_(u"strategy execute exception"), exc=e)
else:
code = const.EXIT_CODE.EXIT_USER_ERROR
user_detail_log.error(_(u"strategy execute exception"), exc=e)
return code
def _exception_handler(e):
try:
sys.excepthook(e.error.exc_type, e.error.exc_val, e.error.exc_tb)
except Exception as e:
system_log.exception("hook exception failed")
user_system_log.error(e.error)
if not is_user_exc(e.error.exc_val):
code = const.EXIT_CODE.EXIT_INTERNAL_ERROR
system_log.error(_(u"strategy execute exception"), exc=e)
else:
code = const.EXIT_CODE.EXIT_USER_ERROR
user_detail_log.error(_(u"strategy execute exception"), exc=e)
return code
def quotation_server(redis_url):
"""
[sys_stock_realtime] quotation service, download market data into redis
Multiple RQAlpha instance can use single market data service.
"""
import redis
import time
import json
from .utils import get_realtime_quotes
redis_client = redis.from_url(redis_url)
from rqalpha.data.data_proxy import DataProxy
config = parse_config({}, verify_config=False)
data_source = BaseDataSource(config.base.data_bundle_path)
data_proxy = DataProxy(data_source)
order_book_id_list = sorted(ins.order_book_id
for ins in data_proxy.all_instruments("CS"))
def record_market_data(total_df):
for order_book_id, item in total_df.iterrows():
redis_client[order_book_id] = json.dumps(item.to_dict())
while True:
try:
total_df = get_realtime_quotes(order_book_id_list,
include_limit=True)
except Exception as e:
system_log.exception("get_realtime_quotes fail. {}", e)
continue
system_log.info("Fetching snapshots, size {}", len(total_df))
record_market_data(total_df)
time.sleep(1)
def __getitem__(self, key):
if not isinstance(key, six.string_types):
raise patch_user_exc(ValueError('invalid key {} (use order_book_id please)'.format(key)))
instrument = self._data_proxy.instruments(key)
if instrument is None:
raise patch_user_exc(ValueError('invalid order book id or symbol: {}'.format(key)))
order_book_id = instrument.order_book_id
try:
return self._cache[order_book_id]
except KeyError:
try:
if not self._dt:
return BarObject(instrument, NANDict, self._dt)
bar = self._data_proxy.get_bar(order_book_id, self._dt, self._frequency)
except Exception as e:
system_log.exception(e)
raise patch_user_exc(KeyError(_(u"id_or_symbols {} does not exist").format(key)))
if bar is None:
return BarObject(instrument, NANDict, self._dt)
else:
self._cache[order_book_id] = bar
return bar
def run_tests(file_path=None):
if file_path is not None:
files = [file_path]
else:
files = [f for f in os.listdir(TEST_DIR) if f.find("test") == 0]
error_map = {}
for filename in files:
try:
r, result_data = run_test(filename)
if r is not None:
error_map[filename.replace(".py", "")] = result_data
except Exception as e:
system_log.exception()
error_map[filename.replace(".py", "")] = e
for filename, result_data in iteritems(error_map):
print(u"*" * 20, u"[{}]did not pass!".format(filename), u"*" * 20)
if isinstance(result_data, Exception):
system_log.error(result_data)
else:
df, old_df, result = result_data
# print("+" * 10, "old test Dataframe: ", "+" * 10)
# print(old_df.drop(result.columns[result.all()], axis=1))
# print("+" * 10, "new test Dataframe: ", "+" * 10)
# print(df.drop(result.columns[result.all()], axis=1))
if "summary" in df.keys():
df["summary"] = pd.DataFrame([df["summary"]])
old_df["summary"] = pd.DataFrame([old_df["summary"]])
for k in df.keys():
d = old_df[k][~old_df[k].isin(df[k])].dropna()
if not d.empty:
print(k, 'max diff:', d.abs().max(), "\n")
print(result.all())
print(u"=" * 40)
print(u"[{}|{}] strategies has been passed!".format(
len(files) - len(error_map), len(files)))
return len(error_map)
