async def list_stock_pool(plot=None, time_offset: int = 3):
now = arrow.now().date()
start = tf.day_shift(now, -time_offset)
if plot is None:
keys = await cache.sys.keys("plots.*.pool")
else:
keys = [f"plots.{plot}.pool"]
results = []
for key in keys:
recs = await cache.sys.hgetall(key)
data = []
for k, v in recs.items():
_frame, code = k.split(":")
if len(_frame) == 8:
frame = tf.int2date(int(_frame))
else:
frame = tf.int2time(int(_frame))
if arrow.get(frame) < arrow.get(start):
continue
sec = Security(code)
row = {"name": sec.display_name, "code": code, "frame": frame}
row.update(json.loads(v))
data.append(row)
print(f"----------{key.lower()}----------")
df = DataFrame(data=data)
df.set_index('frame', inplace=True)
display(df)
results.append(df)
return results
async def list_momentum_pool(day_offset: int = 1, sort_by='y'):
start = tf.day_shift(arrow.now().date(), -day_offset)
key = f"plots.momentum.pool"
recs = await cache.sys.hgetall(key)
data = []
for k, v in recs.items():
frame, code = k.split(":")
if arrow.get(frame) < arrow.get(start):
continue
sec = Security(code)
v = json.loads(v)
frame_type = FrameType(v.get("frame_type"))
fired = tf.int2time(frame) if frame_type in tf.minute_level_frames else \
tf.int2date(frame)
data.append({
"name": sec.display_name,
"code": code,
"fired": fired,
"frame": frame_type.value,
"y": round(v.get("y"), 2),
"vx": round(v.get("vx"), 1),
"a": round(v.get("a"), 4),
"b": round(v.get("b"), 4),
"err": round(v.get("err"), 4)
})
if len(data) == 0:
print("no data")
else:
df = DataFrame(data)
df.set_index('fired', inplace=True)
display(df.sort_values(sort_by))
async def scan(self,
start: Frame,
end: Frame,
signal_func: Callable,
frame_type: FrameType = FrameType.DAY):
frames = tf.get_frames(start, end, frame_type)
results = []
for frame in frames:
if frame_type in tf.day_level_frames:
frame = tf.int2date(frame)
else:
frame = tf.int2time(frame)
result = await signal_func(frame,
frame_type=frame_type,
win=60,
adv=0.0)
results.extend(result)
df = DataFrame(data=results, columns=['date', 'code', 'pc'])
df.to_csv("/tmp/hrp.csv")
async def scan(self,
start: Frame,
end: Frame,
signal_func: Callable,
frame_type: FrameType = FrameType.DAY):
frames = tf.get_frames(start, end, frame_type)
results = []
for frame in frames:
if frame_type in tf.day_level_frames:
frame = tf.int2date(frame)
else:
frame = tf.int2time(frame)
result = await signal_func(frame, frame_type=frame_type)
results.extend(result)
df = DataFrame(data=results,
columns=[
'date', 'code', 't1', 't2', 't3', 't4', 'slope_60',
'pct', "fired"
])
df.to_csv("/tmp/two.csv")
async def do_validation(secs: List[str] = None,
start: str = None,
end: str = None):
"""对列表secs中指定的证券行情数据按start到end指定的时间范围进行校验
Args:
secs (List[str], optional): [description]. Defaults to None.
start (str, optional): [description]. Defaults to None.
end (str, optional): [description]. Defaults to None.
Returns:
[type]: [description]
"""
logger.info("start validation...")
report = logging.getLogger("validation_report")
cfg = cfg4py.init(get_config_dir(), False)
await emit.start(engine=emit.Engine.REDIS,
dsn=cfg.redis.dsn,
start_server=True)
await omicron.init()
start = int(start
or await cache.sys.get("jobs.bars_validation.range.start"))
if end is None:
end = tf.date2int(arrow.now().date())
else:
end = int(end
or await cache.sys.get("jobs.bars_validation.range.stop"))
if secs is None:
async def get_sec():
return await cache.sys.lpop("jobs.bars_validation.scope")
else:
async def get_sec():
return secs.pop() if len(secs) else None
errors = 0
while code := await get_sec():
try:
for day in tf.day_frames[(tf.day_frames >= start)
& (tf.day_frames <= end)]:
expected = await get_checksum(day)
if expected and expected.get(code):
actual = await calc_checksums(tf.int2date(day), [code])
d1 = actual.get(code)
d2 = expected.get(code)
missing1 = d2.keys() - d1 # local has no checksum
missing2 = d1.keys() - d2 # remote has no checksum
mismatch = {k for k in d1.keys() & d2 if d1[k] != d2[k]}
for k in missing1:
info = (
ValidationError.LOCAL_MISS,
day,
code,
k,
d1.get(k),
d2.get(k),
)
report.info("%s,%s,%s,%s,%s,%s", *info)
await emit.emit(Events.OMEGA_VALIDATION_ERROR, info)
for k in missing2:
info = (
ValidationError.REMOTE_MISS,
day,
code,
k,
d1.get(k),
d2.get(k),
)
report.info("%s,%s,%s,%s,%s,%s", *info)
await emit.emit(Events.OMEGA_VALIDATION_ERROR, info)
for k in mismatch:
info = (
ValidationError.MISMATCH,
day,
code,
k,
d1.get(k),
d2.get(k),
)
report.info("%s,%s,%s,%s,%s,%s", *info)
await emit.emit(Events.OMEGA_VALIDATION_ERROR, info)
else:
logger.error("checksum for %s not found.", day)
info = (ValidationError.NO_CHECKSUM, day, None, None, None,
None)
report.info("%s,%s,%s,%s,%s,%s", *info)
await emit.emit(Events.OMEGA_VALIDATION_ERROR, info)
except Exception as e:
logger.exception(e)
errors += 1
def test_get_frames(self):
days = [
20200117,
20200120,
20200121,
20200122,
20200123,
20200203,
20200204,
20200205,
20200206,
20200207,
20200210,
20200211,
]
for i in range(len(days)):
start = tf.int2date(days[0])
end = tf.int2date(days[i])
actual = tf.get_frames(start, end, FrameType.DAY)
logger.debug(
"get_frames(%s, %s, %s)->%s", start, end, FrameType.DAY, actual
)
self.assertListEqual(days[0 : i + 1], list(actual))
X = [
(202002041030, 1, [202002041030]),
(202002041030, 2, [202002041000, 202002041030]),
(202002041030, 3, [202002031500, 202002041000, 202002041030]),
(202002041030, 4, [202002031430, 202002031500, 202002041000, 202002041030]),
(
202002041030,
5,
[202002031400, 202002031430, 202002031500, 202002041000, 202002041030],
),
(
202002041030,
6,
[
202002031330,
202002031400,
202002031430,
202002031500,
202002041000,
202002041030,
],
),
(
202002041030,
7,
[
202002031130,
202002031330,
202002031400,
202002031430,
202002031500,
202002041000,
202002041030,
],
),
(
202002041030,
8,
[
202002031100,
202002031130,
202002031330,
202002031400,
202002031430,
202002031500,
202002041000,
202002041030,
],
),
(
202002041030,
9,
[
202002031030,
202002031100,
202002031130,
202002031330,
202002031400,
202002031430,
202002031500,
202002041000,
202002041030,
],
),
(
202002041030,
10,
[
202002031000,
202002031030,
202002031100,
202002031130,
202002031330,
202002031400,
202002031430,
202002031500,
202002041000,
202002041030,
],
),
(
202002041030,
11,
[
202001231500,
202002031000,
202002031030,
202002031100,
202002031130,
202002031330,
202002031400,
202002031430,
202002031500,
202002041000,
202002041030,
],
),
]
for i, (end, n, expected) in enumerate(X):
start = tf.int2time(expected[0])
end = tf.int2time(end)
actual = tf.get_frames(start, end, FrameType.MIN30)
logger.debug(
"get_frames(%s, %s, %s)->%s", start, end, FrameType.MIN30, actual
)
self.assertListEqual(expected, actual)
def test_get_frames_by_count(self):
days = [
20200117,
20200120,
20200121,
20200122,
20200123,
20200203,
20200204,
20200205,
20200206,
20200207,
20200210,
20200211,
]
for i in range(len(days)):
end, n = tf.int2date(days[i]), i + 1
expected = days[:n]
actual = tf.get_frames_by_count(end, n, FrameType.DAY)
logger.debug(
"get_frames_by_count(%s, %s, %s)->%s", end, n, FrameType.DAY, actual
)
self.assertListEqual(expected, list(actual))
X = [
(202002041030, 1, [202002041030]),
(202002041030, 2, [202002041000, 202002041030]),
(202002041030, 3, [202002031500, 202002041000, 202002041030]),
(202002041030, 4, [202002031430, 202002031500, 202002041000, 202002041030]),
(
202002041030,
5,
[202002031400, 202002031430, 202002031500, 202002041000, 202002041030],
),
(
202002041030,
6,
[
202002031330,
202002031400,
202002031430,
202002031500,
202002041000,
202002041030,
],
),
(
202002041030,
7,
[
202002031130,
202002031330,
202002031400,
202002031430,
202002031500,
202002041000,
202002041030,
],
),
(
202002041030,
8,
[
202002031100,
202002031130,
202002031330,
202002031400,
202002031430,
202002031500,
202002041000,
202002041030,
],
),
(
202002041030,
9,
[
202002031030,
202002031100,
202002031130,
202002031330,
202002031400,
202002031430,
202002031500,
202002041000,
202002041030,
],
),
(
202002041030,
10,
[
202002031000,
202002031030,
202002031100,
202002031130,
202002031330,
202002031400,
202002031430,
202002031500,
202002041000,
202002041030,
],
),
(
202002041030,
11,
[
202001231500,
202002031000,
202002031030,
202002031100,
202002031130,
202002031330,
202002031400,
202002031430,
202002031500,
202002041000,
202002041030,
],
),
]
for i, (end, n, expected) in enumerate(X):
end = tf.int2time(end)
actual = tf.get_frames_by_count(end, n, FrameType.MIN30)
logger.debug(
"get_frames_by_count(%s, %s, %s)->%s", end, n, FrameType.DAY, actual
)
self.assertListEqual(expected, actual)
actual = tf.get_frames_by_count(datetime.date(2020, 2, 12), 3, FrameType.MONTH)
self.assertListEqual([20191129, 20191231, 20200123], actual.tolist())
actual = tf.get_frames_by_count(datetime.date(2020, 2, 12), 3, FrameType.WEEK)
self.assertListEqual([20200117, 20200123, 20200207], actual.tolist())
async def list_stock_pool(self,
frames: int,
frame_types: List[FrameType] = None):
key = "plots.momentum.pool"
recs = await cache.sys.hgetall(key)
items = []
now = arrow.now()
for k, v in recs.items():
frame, code = k.split(":")
sec = Security(code)
v = json.loads(v)
frame_type = FrameType(v.get("frame_type"))
if frame_type not in frame_types:
continue
latest_frame = tf.floor(now, frame_type)
start = tf.shift(latest_frame, -frames, frame_type)
fired = tf.int2time(frame) if frame_type in tf.minute_level_frames else \
tf.int2date(frame)
if fired < start:
continue
items.append({
"name": sec.display_name,
"code": code,
"fired": str(fired),
"frame": frame_type.value,
"y": round(v.get("y"), 2),
"vx": round(v.get("vx"), 1),
"a": round(v.get("a"), 4),
"b": round(v.get("b"), 4),
"err": round(v.get("err"), 4)
})
return {
"name":
self.display_name,
"plot":
self.name,
"items":
items,
"headers": [{
"text": '名称',
"value": 'name'
}, {
"text": '代码',
"value": 'code'
}, {
"text": '信号时间',
"value": 'fired'
}, {
"text": '预测涨幅',
"value": 'y'
}, {
"text": '动能',
"value": 'a'
}, {
"text": '势能',
"value": 'b'
}, {
"text": '周期',
"value": 'frame'
}, {
"text": '底部距离(周期)',
"value": 'vx'
}, {
"text": '拟合误差',
"value": 'err'
}]
}