async def test_heart_beat(self):
e.register("hi", self.on_echo)
await e.start(e.Engine.REDIS,
heart_beat=0.5,
dsn=self.dsn,
start_server=True)
await asyncio.sleep(1)
async def _test_sync_bars(self):
# fixme: recover this test later
config_items = [[{
"frame": "1d",
"start": "2020-01-01",
"delay": 3,
"type": [],
"include": "000001.XSHE,000004.XSHE",
"exclude": "000001.XSHG",
}]]
sync_request = []
async def on_sync_bars(params: dict):
sync_request.append(params)
emit.register(Events.OMEGA_DO_SYNC, on_sync_bars)
for config in config_items:
cfg.omega.sync.bars = config
for frame_config in config:
frame_type = FrameType(frame_config.get("frame"))
sync_params = frame_config
await sync.trigger_bars_sync(frame_type,
sync_params,
force=True)
await asyncio.sleep(0.2)
self.assertDictEqual(
{
"start": "2020-01-01",
"stop": None,
"frame_type": FrameType.DAY
},
sync_request[0],
)
async def init(self, *args):
logger.info("init %s", self.__class__.__name__)
cfg4py.init(get_config_dir(), False)
cfg4py.update_config(self.inherit_cfg)
await aq.create_instance(self.fetcher_impl, **self.params)
await omicron.init(aq)
# listen on omega events
emit.register(Events.OMEGA_DO_SYNC, sq.sync_bars_worker)
await emit.start(emit.Engine.REDIS, dsn=cfg.redis.dsn)
# register route here
app.add_route(self.get_version, "/sys/version")
app.add_route(self.get_security_list_handler, "/quotes/security_list")
app.add_route(self.get_bars_handler, "/quotes/bars")
app.add_route(self.get_bars_batch_handler, "/quotes/bars_batch")
app.add_route(self.get_all_trade_days_handler, "/quotes/all_trade_days")
app.add_route(self.bars_sync_handler, "/jobs/sync_bars", methods=["POST"])
app.add_route(
self.sync_calendar_handler, "/jobs/sync_calendar", methods=["POST"]
)
app.add_route(
self.sync_seurity_list_handler, "jobs/sync_security_list", methods=["POST"]
)
app.add_route(self.get_valuation, "quotes/valuation")
logger.info("<<< init %s process done", self.__class__.__name__)
async def _test_200_validation(self):
# fixme: recover later. All validation/checksum related cases need to be redesigned.
await self.prepare_checksum_data()
errors = set()
async def collect_error(report: tuple):
if report[0] != ValidationError.UNKNOWN:
errors.add(report)
emit.register(Events.OMEGA_VALIDATION_ERROR, collect_error)
codes = ["000001.XSHE"]
await cache.sys.set("jobs.bars_validation.range.start", "20200511")
await cache.sys.set("jobs.bars_validation.range.stop", "20200513")
await omega.core.sanity.do_validation(codes, "20200511", "20200512")
self.assertSetEqual({(0, 20200511, None, None, None, None)}, set(errors))
mock_checksum = {
"000001.XSHE": {
"1d": "7918c38d",
"1m": "15311c54",
"5m": "54f9ac0a",
"15m": "3c2cd435",
"30m": "0cfbf775",
"60m": "d21018b3",
},
"000001.XSHG": {
"1d": "3a24582f",
"1m": "d37d8742",
"5m": "7bb329ad",
"15m": "7c4e48a5",
"30m": "e5db84ef",
"60m": "af4be47d",
},
}
await cache.sys.set("jobs.bars_validation.range.start", "20200511")
await cache.sys.set("jobs.bars_validation.range.stop", "20200513")
with mock.patch(
"omega.core.sanity.calc_checksums",
side_effect=[mock_checksum, mock_checksum],
):
try:
_tmp = tf.day_frames
tf.day_frames = np.array([20200512])
codes = ["000001.XSHE", "000001.XSHG"]
errors = set()
await omega.core.sanity.do_validation(codes, "20200511", "20200512")
self.assertSetEqual(
{
(3, 20200512, "000001.XSHE", "1d", "7918c38d", "7918c38c"),
(3, 20200512, "000001.XSHG", "1m", "d37d8742", "d37d8741"),
},
errors,
)
finally:
tf.day_frames = _tmp
async def test_in_process_engine(self):
e.register('test_in_process', self.on_in_process)
await e.start(e.Engine.IN_PROCESS)
e.register("test_after_start", self.on_in_process)
await asyncio.sleep(0.1)
await e.emit('test_in_process', {"msg": "in-process"})
await e.emit("test_after_start", {"msg": "after-start"})
await asyncio.sleep(0.5)
async def test_aio_redis_engine(self):
await e.start(e.Engine.REDIS, dsn=self.dsn, start_server=True)
e.register('echo', self.on_echo)
await asyncio.sleep(0.5)
await e.emit('echo', {"msg": "new message 1.0"})
# receiver will receive None
await e.emit('echo')
# this will cause no problem. sender can send any message out
await e.emit("not registered")
await asyncio.sleep(1)
async def test_redis_stop(self):
e.register("test_stop", self.on_echo)
await e.start(e.Engine.REDIS,
heart_beat=0.3,
start_server=True,
dsn=self.dsn)
await e.emit("test_stop", {"msg": "check this in log"})
await asyncio.sleep(0.1)
e.unsubscribe("test_stop", self.on_echo)
await e.emit("test_stop", {"msg": "nobody will handle this"})
async def test_unsubscribe(self):
e.register("test_unsub", self.on_echo)
self.assertIn(self.on_echo, e._registry['/test_unsub']['handlers'])
await e.start()
e.unsubscribe("test_unsub", self.on_echo)
self.assertNotIn(self.on_echo, e._registry['/test_unsub']['handlers'])
e.register('test_unsub', self.on_echo)
logger = logging.getLogger('pyemit.emit')
logger.warning = mock.MagicMock(name='warning')
e.unsubscribe('test_unsub', self.test_unsubscribe)
self.assertTrue(
check_log(logger.warning, 'is not registered as handler of'))
async def init(self, *args):
logger.info("init %s", self.__class__.__name__)
cfg4py.init(get_config_dir(), False)
cfg4py.update_config(self.inherit_cfg)
await aq.create_instance(self.fetcher_impl, **self.params)
await omicron.init(aq)
interfaces = Blueprint.group(jobs.bp, quotes.bp, sys.bp)
app.blueprint(interfaces)
# listen on omega events
emit.register(Events.OMEGA_DO_SYNC, syncjobs.sync_bars)
await emit.start(emit.Engine.REDIS, dsn=cfg.redis.dsn)
logger.info("<<< init %s process done", self.__class__.__name__)
async def test_trigger_bars_sync(self):
sync_params = {
"frame": "1d",
"start": "2020-01-01",
"delay": 3,
"cat": [],
"include": "000001.XSHE 000004.XSHE",
"exclude": "000001.XSHG",
}
sync_request = []
async def on_sync_bars(params: dict):
sync_request.append(params)
emit.register(Events.OMEGA_DO_SYNC, on_sync_bars)
with mock.patch("arrow.now", return_value=arrow.get("2020-1-6 10:00")):
await syncjobs.trigger_bars_sync(sync_params, force=True)
await asyncio.sleep(0.2)
self.assertDictEqual(
{
"start": arrow.get("2019-12-31").date(),
"stop": arrow.get("2020-1-3").date(),
"frame_type": FrameType.DAY,
},
sync_request[0],
)
sync_request = []
with mock.patch("arrow.now", return_value=arrow.get("2020-1-6 15:00")):
await syncjobs.trigger_bars_sync(sync_params, force=True)
await asyncio.sleep(0.2)
self.assertDictEqual(
{
"start": arrow.get("2019-12-31").date(),
"stop": arrow.get("2020-1-6").date(),
"frame_type": FrameType.DAY,
},
sync_request[0],
)
async def test_test_signal(self):
plot = Momentum()
results = {}
async def on_trade_signal(results, msg):
results[msg.get('fire_on')] = {
"flag": msg.get('flag'),
"frame_type": msg.get('frame_type')
}
emit.register(Events.sig_trade, functools.partial(on_trade_signal, results))
for frame in tf.get_frames(arrow.get('2020-8-24 11:00'),
arrow.get('2020-8-28 15:00'),
FrameType.MIN30):
frame = tf.int2time(frame)
await plot.evaluate('000001.XSHG', '30m', frame, flag='both')
self.assertDictEqual({
202008261000: {"flag": "short", "frame_type": '30m'},
202008271130: {"flag": "long", "frame_type": '30m'}
}, results)
async def test_error_handling(self):
# handler is not registered
await e.start()
logger = logging.getLogger('pyemit.emit')
logger.warning = mock.MagicMock(name='warning')
await e.emit("test_not_registered")
self.assertTrue(
check_log(logger.warning, 'test_not_registered has no listener'))
# ConnectionClosedError when publish
await e.stop()
await e.start(e.Engine.REDIS, dsn=self.dsn)
e._pub_conn.publish = mock.MagicMock()
import aioredis
e._pub_conn.publish.side_effect = aioredis.errors.ConnectionClosedError(
)
e.register('mock_connection_closed', self.on_echo)
try:
await e.emit("this should raise connectionclosederror")
self.assertTrue(False)
except ConnectionError as error:
logger.exception(error)
self.assertTrue(True)
async def start_validation():
"""
将待校验的证券按CPU个数均匀划分，创建与CPU个数相同的子进程来执行校验。校验的起始时间由数据
库中jobs.bars_validation.range.start和jobs.bars_validation.range.stop来决定，每次校验
结束后，将jobs.bars_validation.range.start更新为校验截止的最后交易日。如果各个子进程报告
的截止交易日不一样（比如发生了异常），则使用最小的交易日。
"""
global validation_errors, no_validation_error_days
validation_errors = []
secs = Securities()
cpu_count = psutil.cpu_count()
# to check if the range is right
pl = cache.sys.pipeline()
pl.get("jobs.bars_validation.range.start")
pl.get("jobs.bars_validation.range.end")
start, end = await pl.execute()
if start is None:
if cfg.omega.validation.start is None:
logger.warning(
"start of validation is not specified, validation aborted.")
return
else:
start = tf.date2int(arrow.get(cfg.omega.validation.start))
else:
start = int(start)
if end is None:
end = tf.date2int(tf.floor(arrow.now().date(), FrameType.DAY))
else:
end = int(end)
assert start <= end
no_validation_error_days = set(tf.day_frames[(tf.day_frames >= start)
& (tf.day_frames <= end)])
# fixme: do validation per frame_type
# fixme: test fail. Rewrite this before 0.6 releases
codes = secs.choose(cfg.omega.sync)
await cache.sys.delete("jobs.bars_validation.scope")
await cache.sys.lpush("jobs.bars_validation.scope", *codes)
logger.info("start validation %s secs from %s to %s.", len(codes), start,
end)
emit.register(Events.OMEGA_VALIDATION_ERROR, on_validation_error)
t0 = time.time()
code = ("from omega.core.sanity import do_validation_process_entry; "
"do_validation_process_entry()")
procs = []
for i in range(cpu_count):
proc = subprocess.Popen([sys.executable, "-c", code], env=os.environ)
procs.append(proc)
timeout = 3600
while timeout > 0:
await asyncio.sleep(2)
timeout -= 2
for proc in procs:
proc.poll()
if all([proc.returncode is not None for proc in procs]):
break
if timeout <= 0:
for proc in procs:
try:
os.kill(proc.pid, signal.SIGTERM)
except Exception:
pass
# set next start point
validation_days = set(tf.day_frames[(tf.day_frames >= start)
& (tf.day_frames <= end)])
diff = validation_days - no_validation_error_days
if len(diff):
last_no_error_day = min(diff)
else:
last_no_error_day = end
await cache.sys.set("jobs.bars_validation.range.start", last_no_error_day)
elapsed = time.time() - t0
logger.info(
"Validation cost %s seconds, validation will start at %s next time",
elapsed,
last_no_error_day,
)
async def test_inprocess_stop(self):
e.register("test_stop", self.on_echo)
await e.start()
await e.stop()
async def start(self):
trigger = FrameTrigger(FrameType.MIN30)
self.scheduler.add_job(self.distribution, trigger)
emit.register(Events.sig_trade, self.on_plot_report)