async def setUp(self) -> None:
os.environ[cfg4py.envar] = 'DEV'
config_dir = get_config_dir()
cfg4py.init(config_dir, False)
await omicron.init()
await emit.start(emit.Engine.REDIS, dsn=cfg.redis.dsn)
def init_test_env():
os.environ[cfg4py.envar] = "DEV"
cfg4py.init(get_config_dir(), False)
# enable postgres for unittest
cfg.postgres.enabled = True
return cfg
async def setUp(self) -> None:
os.environ[cfg4py.envar] = 'DEV'
config_dir = get_config_dir()
cfg4py.init(config_dir)
await omicron.init()
def init():
server_roles = ['PRODUCTION', 'TEST', 'DEV']
if os.environ.get(cfg4py.envar) not in ['PRODUCTION', 'TEST', 'DEV']:
print(f"请设置环境变量{colored(cfg4py.envar, 'red')}为["
f"{colored(server_roles, 'red')}]之一。")
sys.exit(-1)
config_dir = get_config_dir()
cfg4py.init(config_dir, False)
def init_test_env():
import logging
logging.captureWarnings(True)
os.environ[cfg4py.envar] = "DEV"
cfg4py.init(get_config_dir(), False)
# enable postgres for unittest
cfg.postgres.enabled = True
return cfg
async def init(self, model_file: Optional[str] = None):
os.environ[cfg4py.envar] = 'DEV'
self.config_dir = get_config_dir()
cfg4py.init(self.config_dir, False)
await emit.start(engine=emit.Engine.REDIS, dsn=cfg.redis.dsn,
start_server=True)
await omicron.init()
if model_file is not None:
with open(model_file, "rb") as f:
self.model = load(f)
def test_007_cmd_build(self):
cmd = Command()
os.makedirs(self.output)
with unittest.mock.patch("builtins.input", side_effect=["31"]):
cmd.scaffold(self.output)
# normal run
cmd.build(self.output)
cfg = cfg4py.init(self.output, strict=True)
self.assertEqual(cfg.postgres.dsn, "dbname=test user=postgres password=secret")
# path not exists run
shutil.rmtree(self.output)
try:
with unittest.mock.patch(
"sys.exit", lambda *args: early_jump("Path not exists")
):
cmd.build(self.output)
self.assertTrue(False, "sys.exit not triggered")
except SystemExit:
self.assertTrue(True)
# no files in self.output
os.makedirs(self.output, exist_ok=True)
try:
with unittest.mock.patch(
"sys.exit", lambda *args: early_jump("no files in folder")
):
cmd.build(self.output)
self.assertTrue(False, "sys.exit not triggered")
except SystemExit:
self.assertTrue(True)
async def init(app, loop): # noqa
global scheduler
config_dir = get_config_dir()
cfg4py.init(get_config_dir(), False)
logger.info("init omega-jobs process with config at %s", config_dir)
await omicron.init()
await emit.start(emit.Engine.REDIS, dsn=cfg.redis.dsn)
scheduler = AsyncIOScheduler(timezone=cfg.tz)
# sync securities daily
h, m = map(int, cfg.omega.sync.security_list.split(":"))
scheduler.add_job(
functools.partial(sq.trigger_single_worker_sync, "calendar"),
"cron",
hour=h,
minute=m,
)
scheduler.add_job(
functools.partial(sq.trigger_single_worker_sync, "security_list"),
"cron",
hour=h,
minute=m,
)
# sync bars
_add_bars_sync_job()
last_sync = await cache.sys.get("jobs.bars_sync.stop")
if last_sync:
last_sync = arrow.get(last_sync, tzinfo=cfg.tz).timestamp
if not last_sync or time.time() - last_sync >= 24 * 3600:
logger.info("start catch-up quotes sync")
for frame_type in itertools.chain(tf.day_level_frames,
tf.minute_level_frames):
params = sq.read_sync_params(frame_type)
if params:
asyncio.create_task(
sq.trigger_bars_sync(frame_type, params, force=True))
else:
logger.info("%s: less than 24 hours since last sync", last_sync)
scheduler.start()
logger.info("omega jobs finished initialization")
async def init(self, *args):
logger.info("init %s", self.__class__.__name__)
cfg4py.init(get_config_dir(), False)
cfg4py.update_config(self.inherit_cfg)
await aq.create_instance(self.fetcher_impl, **self.params)
await omicron.init(aq)
interfaces = Blueprint.group(jobs.bp, quotes.bp, sys.bp)
app.blueprint(interfaces)
# listen on omega events
emit.register(Events.OMEGA_DO_SYNC, syncjobs.sync_bars)
await emit.start(emit.Engine.REDIS, dsn=cfg.redis.dsn)
logger.info("<<< init %s process done", self.__class__.__name__)
def init_test_env():
os.environ[cfg4py.envar] = "DEV"
src_dir = os.path.dirname(__file__)
config_path = os.path.join(src_dir, "../omicron/config")
# todo: omega-0.6 dev assumes /var/log/zillionare exists by default
# this will cause UT hang if no /var/log/zillionare
os.makedirs("/var/log/zillionare", exist_ok=True)
return cfg4py.init(config_path, False)
def build(self, config_dir: str):
"""Compile configuration files into a python class file, which is used by IDE's
auto-complete function
Args:
config_dir: The folder where your configuration files located
Returns:
"""
if not os.path.exists(config_dir):
print(f"path {config_dir} not exists")
sys.exit(-1)
count = 0
for f in os.listdir(config_dir):
if (f.startswith("default") or f.startswith("dev")
or f.startswith("test") or f.startswith("production")):
print(f"found {f}")
count += 1
if count > 0:
print(f"{count} files found in total")
else:
print("the folder contains no valid configuration files")
sys.exit(-1)
try:
init(config_dir)
sys.path.insert(0, config_dir)
from schema import Config # type: ignore # noqa
output_file = f"{os.path.join(config_dir, 'schema')}"
msg = f"Config file is built with success and saved at {output_file}"
print(msg)
except Exception as e: # pragma: no cover
logging.exception(e)
print("Config file built failure.")
async def start(service: str = ""):
"""启动omega主进程或者任务管理进程
Args:
service: if service is '', then starts fetcher processes.
Returns:
"""
print(f"正在启动zillionare-omega {colored(service, 'green')}...")
config_dir = get_config_dir()
cfg4py.init(config_dir, False)
if service == "":
await _start_jobs()
await _start_fetcher_processes()
elif service == "jobs":
return await _start_jobs()
elif service == "fetcher":
return await _start_fetcher_processes()
else:
print("不支持的服务")
async def _init():
config_dir = get_config_dir()
cfg = cfg4py.init(config_dir, False)
# remove console log, so the output message will looks prettier
remove_console_log_handler()
try:
await emit.start(emit.Engine.REDIS, dsn=cfg.redis.dsn)
except Exception:
print(f"dsn is {cfg.redis.dsn}")
impl = cfg.quotes_fetchers[0]["impl"]
params = cfg.quotes_fetchers[0]["workers"][0]
await AbstractQuotesFetcher.create_instance(impl, **params)
await omicron.init(AbstractQuotesFetcher)
def main(months: str, cats: str, archive_server: str = None):
"""允许将本模块以独立进程运行，以支持多进程
Args:
months (str): 逗号分隔的月列表。格式如202012
cats (str): 逗号分隔的类别列表，如"stock,index"
"""
config_dir = get_config_dir()
cfg = cfg4py.init(config_dir, False)
if archive_server:
cfg.omega.urls.archive = archive_server
months = str(months)
months = [int(x) for x in months.split(",") if x]
cats = [x for x in cats.split(",")]
asyncio.run(_main(months, cats))
async def setup(reset_factory=False, force=False):
"""安装初始化入口
Args:
reset_factory: reset to factory settings
force: if true, force setup no matter if run already
Returns:
"""
msg = """
Zillionare-omega (大富翁)\\n
-------------------------\\n
感谢使用Zillionare-omega -- 高速分布式行情服务器！\\n
"""
print(format_msg(msg))
if not force:
config_file = os.path.join(get_config_dir(), "defaults.yaml")
if os.path.exists(config_file):
print(f"{colored('[PASS]', 'green')} 安装程序已在本机上成功运行")
sys.exit(0)
if reset_factory:
import sh
dst = get_config_dir()
os.makedirs(dst, exist_ok=True)
src = os.path.join(factory_config_dir(), "defaults.yaml")
dst = os.path.join(get_config_dir(), "defaults.yaml")
sh.cp("-r", src, dst)
print_title("Step 1. 检测安装环境...")
settings = load_factory_settings()
if not check_environment():
sys.exit(-1)
print_title("Step 2. 配置日志")
config_logging(settings)
print_title("Step 3. 配置上游服务器")
config_fetcher(settings)
print_title("Step 4. 配置Redis服务器")
await config_redis(settings)
print_title("Step 5. 配置Postgres服务器")
await config_postgres(settings)
save_config(settings)
print_title("Step 6. 下载历史数据")
config_dir = get_config_dir()
cfg4py.init(config_dir, False)
remove_console_log_handler()
await start("fetcher")
await download_archive(None)
print_title("配置已完成。现在为您启动Omega,开启财富之旅！")
await start("jobs")
await status()
async def init(app, loop): # noqa
global scheduler
config_dir = get_config_dir()
cfg4py.init(get_config_dir(), False)
await start_logging()
logger.info("init omega-jobs process with config at %s", config_dir)
await omicron.init()
await emit.start(emit.Engine.REDIS, dsn=cfg.redis.dsn)
scheduler = AsyncIOScheduler(timezone=cfg.tz)
# sync securities daily
h, m = map(int, cfg.omega.sync.security_list.split(":"))
scheduler.add_job(
syncjobs.trigger_single_worker_sync,
"cron",
hour=h,
minute=m,
args=("calendar", ),
name="sync_calendar",
)
scheduler.add_job(
syncjobs.trigger_single_worker_sync,
"cron",
args=("security_list", ),
name="sync_security_list",
hour=h,
minute=m,
)
syncjobs.load_bars_sync_jobs(scheduler)
# sync bars at startup
last_sync = await cache.sys.get("jobs.bars_sync.stop")
if last_sync:
try:
last_sync = arrow.get(last_sync, tzinfo=cfg.tz).timestamp
except ValueError:
logger.warning("failed to parse last_sync: %s", last_sync)
last_sync = None
if not last_sync or time.time() - last_sync >= 24 * 3600:
next_run_time = arrow.now(cfg.tz).shift(minutes=5).datetime
logger.info("start catch-up quotes sync at %s", next_run_time)
for frame_type in itertools.chain(tf.day_level_frames,
tf.minute_level_frames):
params = syncjobs.load_sync_params(frame_type)
if params:
scheduler.add_job(
syncjobs.trigger_bars_sync,
args=(params, True),
name=f"catch-up sync for {frame_type}",
next_run_time=next_run_time,
)
else:
logger.info("%s: less than 24 hours since last sync", last_sync)
scheduler.start()
logger.info("omega jobs finished initialization")
