async def get_bars(
cls,
sec: str,
end: Frame,
n_bars: int,
frame_type: FrameType,
include_unclosed=True,
) -> np.ndarray:
bars = await cls.get_instance().get_bars(
sec, end, n_bars, frame_type, include_unclosed
)
now = arrow.now()
if type(end) == datetime.date:
if end == now.date():
closed = tf.floor(
datetime.datetime(end.year, end.month, end.day, now.hour),
frame_type,
)
else:
closed = tf.floor(
datetime.datetime(end.year, end.month, end.day, 15), frame_type
)
else:
closed = tf.floor(end, frame_type)
if frame_type in tf.day_level_frames:
end = end.date() # noqa
else:
end = end.replace(second=0, microsecond=0)
if closed != end:
finished = cls._fill_na(bars, n_bars - 1, closed, frame_type)
if bars[-1]["frame"] == end:
remainder = [bars[-1]]
else: # 停牌，或者end当天休市。调用者要自己保证传入的end不在休市中
remainder = np.empty(1, dtype=bars.dtype)
remainder[:] = np.nan
remainder["frame"] = end
logger.warning("证券%s在frame [%s]处于停牌中", sec, end)
else:
finished = cls._fill_na(bars, n_bars, closed, frame_type)
remainder = None
# 只保存已结束的frame数据到数据库
await cache.save_bars(sec, finished, frame_type)
if remainder is None:
return finished
return np.concatenate([finished, remainder])
async def test_get_bars_011(self):
"""分钟级别，中间有停牌，end指定时间未对齐的情况"""
# 600721, ST百花， 2020-4-29停牌一天
sec = "600721.XSHG"
frame_type = FrameType.MIN60
end = arrow.get("2020-04-30 10:32", tzinfo="Asia/Shanghai").datetime
await self.clear_cache(sec, frame_type)
bars = await aq.get_bars(sec, end, 7, frame_type)
print(bars)
self.assertEqual(7, len(bars))
self.assertEqual(arrow.get("2020-04-28 15:00", tzinfo="Asia/Shanghai"),
bars["frame"][0])
self.assertEqual(arrow.get("2020-04-30 10:30", tzinfo="Asia/Shanghai"),
bars["frame"][-2])
self.assertEqual(arrow.get("2020-4-30 10:32", tzinfo="Asia/Shanghai"),
bars["frame"][-1])
self.assertAlmostEqual(5.37, bars["open"][0], places=2)
self.assertAlmostEqual(5.26, bars["open"][-2], places=2)
self.assertAlmostEqual(5.33, bars["open"][-1], places=2)
# 检查cache,10：32未存入cache
cache_len = await cache.security.hlen(f"{sec}:{frame_type.value}")
self.assertEqual(8, cache_len)
bars_2 = await cache.get_bars(sec, tf.floor(end, frame_type), 6,
frame_type)
np.array_equal(bars[:-1], bars_2)
async def test_get_bars_013(self):
"""分钟级别，end指定时间正处在停牌中"""
# 600721, ST百花， 2020-4-29停牌一天
sec = "600721.XSHG"
frame_type = FrameType.MIN60
end = arrow.get("2020-04-29 10:30", tzinfo="Asia/Chongqing").datetime
await self.clear_cache(sec, frame_type)
bars = await aq.get_bars(sec, end, 6, frame_type)
print(bars)
self.assertEqual(6, len(bars))
self.assertEqual(arrow.get("2020-04-27 15:00", tzinfo="Asia/Shanghai"),
bars["frame"][0])
self.assertEqual(arrow.get("2020-04-29 10:30", tzinfo="Asia/Shanghai"),
bars["frame"][-1])
self.assertAlmostEqual(5.47, bars["open"][0], places=2)
self.assertAlmostEqual(5.37, bars["open"][-2], places=2)
self.assertTrue(np.isnan(bars["open"][-1]))
# 检查cache,10：30 已存入cache
cache_len = await cache.security.hlen(f"{sec}:{frame_type.value}")
self.assertEqual(8, cache_len)
bars_2 = await cache.get_bars(sec, tf.floor(end, frame_type), 6,
frame_type)
np.array_equal(bars, bars_2)
def _parse_sync_params(sync_params: dict):
"""
如果sync_params['start'], sync_params['stop']类型为date：
如果frame_type为分钟级，则意味着要取到当天的开始帧和结束帧；
如果frame_type为日期级，则要对齐到已结束的帧日期，并且设置时间为15:00否则某些sdk可能会只取
到上一周期数据（如jqdatasdk 1.8)
如果sync_params['start'], sync_params['stop']类型为datetime：
如果frame_type为分钟级别，则通过tf.floor对齐到已结束的帧
如果frame_type为日线级别，则对齐到上一个已结束的日帧，时间部分重置为15:00
Args:
sync_params:
Returns:
"""
frame_type = sync_params.get("frame_type")
start = sync_params.get("start")
stop = sync_params.get("stop")
if start is None:
raise ValueError("sync_params['start'] must be specified!")
if type(start) not in [datetime.date, datetime.datetime]:
raise TypeError(
"type of sync_params['start'] must be one of ["
"datetime.datetime, datetime.date]"
)
if stop is None:
stop = tf.floor(arrow.now(cfg.tz), frame_type)
if type(stop) not in [datetime.datetime, datetime.date]:
raise TypeError(
"type of sync_params['stop'] must be one of [None, "
"datetime.datetime, datetime.date]"
)
if frame_type in tf.minute_level_frames:
if type(start) is datetime.date:
start = tf.floor(start, FrameType.DAY)
minutes = tf.ticks[frame_type][0]
h, m = minutes // 60, minutes % 60
start = datetime.datetime(
start.year, start.month, start.day, h, m, tzinfo=tz.gettz(cfg.tz)
)
else:
start = tf.floor(start, frame_type)
if type(stop) is datetime.date:
stop = tf.floor(stop, FrameType.DAY)
stop = datetime.datetime(
stop.year, stop.month, stop.day, 15, tzinfo=tz.gettz(cfg.tz)
)
else:
stop = tf.floor(stop, frame_type)
else:
start = tf.floor(start, frame_type)
stop = tf.floor(stop, frame_type)
return frame_type, start, stop
async def get_bars(self,
code: str,
n: int,
frame_type: FrameType,
end_dt: Frame = None):
end_dt = end_dt or arrow.now(tz=cfg.tz)
sec = Security(code)
start = tf.shift(tf.floor(end_dt, frame_type), -n + 1, frame_type)
return await sec.load_bars(start, end_dt, frame_type)
async def test_evaluate(self):
scheduler = AsyncIOScheduler(timezone=cfg.tz)
plot = DuckPlot(scheduler)
secs = Securities().choose(['stock'])
for code in secs:
dt = tf.floor(arrow.get('2020-7-24'), FrameType.DAY)
try:
await plot.evaluate(code, FrameType.DAY, dt=dt)
except Exception as e:
logger.exception(e)
async def plot_ma(code: str,
groups=None,
end: Frame = None,
frame_type: FrameType = FrameType.DAY):
groups = groups or [5, 10, 20, 60, 120]
sec = Security(code)
end = end or tf.floor(arrow.now(), frame_type)
start = tf.shift(end, -(groups[-1] + 19), frame_type)
bars = await sec.load_bars(start, end, frame_type)
for win in groups:
ma = signal.moving_average(bars['close'], win)
plt.plot(ma[-20:])
async def _build_train_data(self, frame_type: FrameType,
n: int, max_error: float = 0.01):
"""
从最近的符合条件的日期开始，遍历股票，提取特征和标签，生成数据集。
Args:
n: 需要采样的样本数
Returns:
"""
watch_win = 5
max_curve_len = 5
max_ma_win = 20
# y_stop = arrow.get('2020-7-24').date()
y_stop = tf.floor(arrow.now(tz=cfg.tz), frame_type)
y_start = tf.shift(y_stop, -watch_win + 1, frame_type)
x_stop = tf.shift(y_start, -1, frame_type)
x_start = tf.shift(x_stop, -(max_curve_len + max_ma_win - 1), frame_type)
data = []
while len(data) < n:
for code in Securities().choose(['stock']):
#for code in ['000601.XSHE']:
try:
sec = Security(code)
x_bars = await sec.load_bars(x_start, x_stop, FrameType.DAY)
y_bars = await sec.load_bars(y_start, y_stop, FrameType.DAY)
# [a, b, axis] * 3
x = self.extract_features(x_bars, max_error)
if len(x) == 0: continue
y = np.max(y_bars['close']) / x_bars[-1]['close'] - 1
if np.isnan(y): continue
feature = [code, tf.date2int(x_stop)]
feature.extend(x)
data.append(feature)
except Exception as e:
logger.warning("Failed to extract features for %s (%s)",
code,
x_stop)
logger.exception(e)
if len(data) >= n:
break
if len(data) % 500 == 0:
logger.info("got %s records.", len(data))
y_stop = tf.day_shift(y_stop, -1)
y_start = tf.day_shift(y_start, -1)
x_stop = tf.day_shift(y_start, -1)
x_start = tf.day_shift(x_start, -1)
return data
async def evaluate(self, code: str, frame_type: Union[FrameType, str], flag: str,
win: int,
c1: float, d1: Frame,
c2: float, d2: Frame,
slip: float = 0.015):
frame_type = FrameType(frame_type)
n1 = tf.count_frames(d1, d2, frame_type)
slp = (c2 - c1) / n1
n2 = tf.count_frames(d2, tf.floor(arrow.now(), frame_type),
frame_type)
c_ = c2 + slp * n2
bars = await self.get_bars(code, 1, frame_type)
if abs(c_ / bars[-1]['close'] - 1) <= slip:
await self.fire_trade_signal(flag, code, bars[-1]['frame'], frame_type)
async def evaluate(self,
code: str,
frame_type: FrameType,
dt: Frame = None,
win=15):
sec = Security(code)
end = dt or arrow.now(tz=cfg.tz)
start = tf.shift(tf.floor(end, frame_type), -win - 20, frame_type)
bars = await sec.load_bars(start, end, frame_type)
if len(bars) < win + 20:
return
# 使用股价重心而不是收盘价来判断走势
o, c = bars[-1]['open'], bars[-1]['close']
feat = features.ma_lines_trend(bars, [5, 10, 20])
ma5, ma10, ma20 = feat["ma5"][0], feat["ma10"][0], feat["ma20"][0]
if np.any(np.isnan(ma5)):
return
mas = np.array([ma5[-1], ma10[-1], ma20[-1]])
# 起涨点：一阳穿三线来确认
if not (np.all(o <= mas) and np.all(c >= mas)):
return
# 三线粘合：三线距离小于self.max_distance
distance = self.distance(ma5[:-1], ma10[:-1], ma20[:-1])
if distance > self.max_distance:
return
# 月线要拉直，走平或者向上
err, (a, b) = signal.polyfit((ma20 / ma20[0]), deg=1)
if err > self.fiterr_ma20 and a < self.fitslp_ma20:
return
logger.info("%s",
f"{sec.display_name}\t{distance:.3f}\t{a:.3f}\t{err:.3f}")
await self.fire("long",
code,
dt,
frame_type=frame_type.value,
distance=distance)
def test_floor(self):
X = [
("2005-01-09", FrameType.DAY, "2005-01-07"),
("2005-01-07", FrameType.DAY, "2005-01-07"),
("2005-01-08 14:00", FrameType.DAY, "2005-1-7"),
("2005-01-07 16:00:00", FrameType.DAY, "2005-01-07"),
("2005-01-07 14:59:00", FrameType.DAY, "2005-01-06"),
("2005-1-10 15:00:00", FrameType.WEEK, "2005-1-7"),
("2005-1-13 15:00:00", FrameType.WEEK, "2005-1-7"),
("2005-1-14 15:00:00", FrameType.WEEK, "2005-1-14"),
("2005-2-1 15:00:00", FrameType.MONTH, "2005-1-31"),
("2005-2-27 15:00:00", FrameType.MONTH, "2005-1-31"),
("2005-2-28 15:00:00", FrameType.MONTH, "2005-2-28"),
("2005-3-1 15:00:00", FrameType.MONTH, "2005-2-28"),
("2005-1-5 09:30", FrameType.MIN1, "2005-1-4 15:00"),
("2005-1-5 09:31", FrameType.MIN1, "2005-1-5 09:31"),
("2005-1-5 09:34", FrameType.MIN5, "2005-1-4 15:00"),
("2005-1-5 09:36", FrameType.MIN5, "2005-1-5 09:35"),
("2005-1-5 09:46", FrameType.MIN15, "2005-1-5 09:45"),
("2005-1-5 10:01", FrameType.MIN30, "2005-1-5 10:00"),
("2005-1-5 10:31", FrameType.MIN60, "2005-1-5 10:30"),
# 如果moment为非交易日，则floor到上一交易日收盘
("2020-11-21 09:32", FrameType.MIN1, "2020-11-20 15:00"),
# 如果moment刚好是frame结束时间，则floor(frame) == frame
("2005-1-5 10:00", FrameType.MIN30, "2005-1-5 10:00"),
]
for i, (moment, frame_type, expected) in enumerate(X):
logger.debug("testing %s", X[i])
frame = arrow.get(moment).datetime
if frame_type in tf.day_level_frames and frame.hour == 0:
frame = frame.date()
actual = tf.floor(frame, frame_type)
expected = arrow.get(expected)
if frame_type in tf.day_level_frames:
expected = arrow.get(expected).date()
else:
expected = arrow.get(expected).datetime
self.assertEqual(expected, actual)
async def get_bars(
cls,
sec: str,
end: Frame,
n_bars: int,
frame_type: FrameType,
include_unclosed=True,
) -> np.ndarray:
"""获取行情数据，并将已结束的周期数据存入缓存。
各种情况：
1. 假设现在时间是2021-2-24日，盘中。此时请求上证指数日线，且`include_unclosed`为
`True`：
```python
get_bars("000001.XSHE", None, 1, FrameType.DAY)
```
得到的数据可能如下：
```
[(datetime.date(2021, 2, 24), 3638.9358, 3645.5288, 3617.44, 3620.3542, ...)]
```
在收盘前不同时间调用，得到的数据除开盘价外，其它都实时在变动。
2. 假设现在时间是2021-2-23日，盘后，此时请求上证指数日线，将得到收盘后固定的价格。
3. 上述请求中，`include_unclosed`参数使用默认值(`True`)。如果取为`False`，仍以示例1
指定的场景为例，则:
```python
get_bars("000001.XSHG", None, 1, FrameType.DAY, False)
```
因为2021-2-24日未收盘，所以获取的最后一条数据是2021-2-23日的。
4. 同样假设现在时间是2021-2-24日盘中，周三。此时获取周K线。在`include_unclosed`分别为
`True`和`False`的情况下：
```
[(datetime.date(2021, 2, 24), 3707.19, 3717.27, 3591.3647, 3592.3977, ...)]
[(datetime.date(2021, 2, 19), 3721.09, 3731.69, 3634.01, 3696.17, ...)]
```
注意这里当`include_unclosed`为True时，返回的周K线是以2021-2-24为Frame的。同样，在盘中
的不同时间取这个数据，除了`open`数值之外，其它都是实时变化的。
5. 如果在已结束的周期中，包含停牌数据，则会对停牌期间的数据进行nan填充，以方便数据使用
者可以较容易地分辨出数据不连贯的原因：哪些是停牌造成的，哪些是非交易日造成的。这种处理
会略微降低数据获取速度，并增加存储空间。
比如下面的请求:
```python
get_bars("000029.XSHE", datetime.date(2020,8,18), 10, FrameType.DAY)
```
将获取到2020-8-5到2020-8-18间共10条数据。但由于期间000029这支股票处于停牌期，所以返回
的10条数据中，数值部分全部填充为np.nan。
注意如果取周线和月线数据，如果当天停牌，但只要周线有数据，则仍能取到。周线（或者月线）的
`frame`将是停牌前一交易日。比如，
```python
sec = "600721.XSHG"
frame_type = FrameType.WEEK
end = arrow.get("2020-4-29 15:00").datetime
bars = await aq.get_bars(sec, end, 3, FrameType.WEEK)
print(bars)
```
2020年4月30日是该周的最后一个交易日。股票600721在4月29日停牌一天。上述请求将得到如下数
据：
```
[(datetime.date(2020, 4, 17), 6.02, 6.69, 5.84, 6.58, ...)
(datetime.date(2020, 4, 24), 6.51, 6.57, 5.68, 5.72, ...)
(datetime.date(2020, 4, 28), 5.7, 5.71, 5.17, 5.36, ...)]
```
停牌发生在日线级别上，但我们的请求发生在周线级别上，所以不会对4/29日进行填充，而是返回
截止到4月29日的数据。
args:
sec: 证券代码
end: 数据截止日
n_bars: 待获取的数据条数
frame_type: 数据所属的周期
include_unclosed: 如果为真，则会包含当end所处的那个Frame的数据，即使当前它还未结束
"""
now = arrow.now(tz=cfg.tz)
end = end or now.datetime
# 如果end超出当前时间，则认为是不合法的。如果用户想取到最新的数据，应该传入None
if type(end) == datetime.date:
if end > now.date():
return None
elif type(end) == datetime.datetime:
if end > now:
return None
bars = await cls.get_instance().get_bars(sec, end, n_bars,
frame_type.value,
include_unclosed)
if len(bars) == 0:
return
# 根据指定的end，计算结束时的frame
last_closed_frame = tf.floor(end, frame_type)
last_frame = bars[-1]["frame"]
# 计算有多少根k线是已结束的
n_closed = n_bars - 1
if frame_type == FrameType.DAY:
# 盘后取日线，返回的一定是全部都已closed的数据
# 盘中取日线，返回的last_frame会是当天的日期，但该日线并未结束
if now.datetime.hour >= 15 or last_frame < now.date():
n_closed = n_bars
else:
# 如果last_frame <= end的上限，则返回的也一定是全部都closed的数据
if last_frame <= tf.floor(end, frame_type):
n_closed = n_bars
remainder = [bars[-1]] if n_closed < n_bars else None
closed_bars = cls._fill_na(bars, n_closed, last_closed_frame,
frame_type)
# 只保存已结束的bar
await cache.save_bars(sec, closed_bars, frame_type)
if remainder is None:
return closed_bars
else:
return np.concatenate([closed_bars, remainder])
async def quick_scan():
# fixme
secs = Securities()
report = logging.getLogger("quickscan")
counters = {}
for sync_config in cfg.omega.sync.bars:
frame = sync_config.get("frame")
start = sync_config.get("start")
if frame is None or start is None:
logger.warning("skipped %s: required fields are [frame, start]",
sync_config)
continue
frame_type = FrameType(frame)
start = arrow.get(start).date()
start = tf.floor(start, FrameType.DAY)
stop = sync_config.get("stop") or arrow.now().date()
if frame_type in tf.minute_level_frames:
minutes = tf.ticks[frame_type][0]
h, m = minutes // 60, minutes % 60
start = datetime.datetime(start.year,
start.month,
start.day,
h,
m,
tzinfo=tz.gettz(cfg.tz))
stop = datetime.datetime(stop.year,
stop.month,
stop.day,
15,
tzinfo=tz.gettz(cfg.tz))
counters[frame] = [0, 0]
codes = secs.choose(sync_config.get("type"))
include = filter(lambda x: x,
sync_config.get("include", "").split(","))
include = map(lambda x: x.strip(" "), include)
codes.extend(include)
exclude = sync_config.get("exclude", "")
exclude = map(lambda x: x.strip(" "), exclude)
codes = set(codes) - set(exclude)
counters[frame][1] = len(codes)
for code in codes:
head, tail = await cache.get_bars_range(code, frame_type)
if head is None or tail is None:
report.info("ENOSYNC,%s,%s", code, frame)
counters[frame][0] = counters[frame][0] + 1
continue
expected = tf.count_frames(head, tail, frame_type)
# 'head', 'tail' should be excluded
actual = (await
cache.security.hlen(f"{code}:{frame_type.value}")) - 2
if actual != expected:
report.info("ELEN,%s,%s,%s,%s,%s,%s", code, frame, expected,
actual, head, tail)
counters[frame][0] = counters[frame][0] + 1
continue
sec = Security(code)
if start != head:
if (type(start) == datetime.date and start > sec.ipo_date
or (type(start) == datetime.datetime
and start.date() > sec.ipo_date)):
report.info("ESTART,%s,%s,%s,%s,%s", code, frame, start,
head, sec.ipo_date)
counters[frame][0] = counters[frame][0] + 1
continue
if tail != stop:
report.info("EEND,%s,%s,%s,%s", code, frame, stop, tail)
counters[frame][0] = counters[frame][0] + 1
return counters
async def start_validation():
"""
将待校验的证券按CPU个数均匀划分，创建与CPU个数相同的子进程来执行校验。校验的起始时间由数据
库中jobs.bars_validation.range.start和jobs.bars_validation.range.stop来决定，每次校验
结束后，将jobs.bars_validation.range.start更新为校验截止的最后交易日。如果各个子进程报告
的截止交易日不一样（比如发生了异常），则使用最小的交易日。
"""
global validation_errors, no_validation_error_days
validation_errors = []
secs = Securities()
cpu_count = psutil.cpu_count()
# to check if the range is right
pl = cache.sys.pipeline()
pl.get("jobs.bars_validation.range.start")
pl.get("jobs.bars_validation.range.end")
start, end = await pl.execute()
if start is None:
if cfg.omega.validation.start is None:
logger.warning(
"start of validation is not specified, validation aborted.")
return
else:
start = tf.date2int(arrow.get(cfg.omega.validation.start))
else:
start = int(start)
if end is None:
end = tf.date2int(tf.floor(arrow.now().date(), FrameType.DAY))
else:
end = int(end)
assert start <= end
no_validation_error_days = set(tf.day_frames[(tf.day_frames >= start)
& (tf.day_frames <= end)])
# fixme: do validation per frame_type
# fixme: test fail. Rewrite this before 0.6 releases
codes = secs.choose(cfg.omega.sync)
await cache.sys.delete("jobs.bars_validation.scope")
await cache.sys.lpush("jobs.bars_validation.scope", *codes)
logger.info("start validation %s secs from %s to %s.", len(codes), start,
end)
emit.register(Events.OMEGA_VALIDATION_ERROR, on_validation_error)
t0 = time.time()
code = ("from omega.core.sanity import do_validation_process_entry; "
"do_validation_process_entry()")
procs = []
for i in range(cpu_count):
proc = subprocess.Popen([sys.executable, "-c", code], env=os.environ)
procs.append(proc)
timeout = 3600
while timeout > 0:
await asyncio.sleep(2)
timeout -= 2
for proc in procs:
proc.poll()
if all([proc.returncode is not None for proc in procs]):
break
if timeout <= 0:
for proc in procs:
try:
os.kill(proc.pid, signal.SIGTERM)
except Exception:
pass
# set next start point
validation_days = set(tf.day_frames[(tf.day_frames >= start)
& (tf.day_frames <= end)])
diff = validation_days - no_validation_error_days
if len(diff):
last_no_error_day = min(diff)
else:
last_no_error_day = end
await cache.sys.set("jobs.bars_validation.range.start", last_no_error_day)
elapsed = time.time() - t0
logger.info(
"Validation cost %s seconds, validation will start at %s next time",
elapsed,
last_no_error_day,
)
async def evaluate(self, code: str, end: Frame):
"""
最近穿越年线的股票，回归到[5,10,20]日均线时，如果均线形态良好，则提示买入
Args:
code:
end:
Returns:
"""
start = tf.shift(tf.floor(end, FrameType.DAY), -26, FrameType.DAY)
sec = Security(code)
bars = await sec.load_bars(start, end, FrameType.DAY)
close = bars['close']
c0 = close[-1]
# 检查接近5日均线，要求5日内强于均线，均线不能向下（或者拐头趋势）
ma = signal.moving_average(close, 5)
err, (a, b, c), (vx, _) = signal.polyfit(ma[-7:] / ma[-7])
t1 = np.all(close[-5:] > ma[-5:])
t2 = err < 3e-3
t3 = a > 5e-4 or (abs(a) < 1e-5 and b > 1e-3)
t4 = (c0 - ma[-1] / c0 < 5e-3)
t5 = vx < 6
logger.debug("%s 5日：%s, (a,b):%s,%s", sec, [t1, t2, t3, t4, t5], a, b)
if all([t1, t2, t3, t4, t5]):
logger.info("fired 5日买入:%s", sec)
await emit.emit("/alpha/signals/long", {
"plot": "crossyear",
"code": code,
"frame": str(end),
"desc": "回探5日线",
"coef": np.round([a, b], 4),
"vx": vx,
"c": c0,
"ma": ma[-5:]
})
return
# 检查接近20日线买点
ma = signal.moving_average(close, 20)
err, (a, b, c), (vx, _) = signal.polyfit(ma[-10:] / ma[-10])
t1 = err < 3e-3
t2 = a > 5e-4 or (abs(a) < 1e-5 and b > 5e-3)
t3 = (c0 - ma[-1]) < 5e-3
t4 = vx < 9
logger.debug("%s 20日：%s, (a,b):%s,%s", sec, [t1, t2, t3, t4], a, b)
if all([t1, t2, t3, t4]):
logger.info("fired 20日买入:%s", sec)
await emit.emit("/alpha/signals/long", {
"plot": "crossyear",
"code": code,
"frame": str(end),
"desc": "回探20日线",
"coef": np.round([a, b], 4),
"vx": vx,
"c": c0,
"ma": ma[-5:]
})
return
# 检查是否存在30分钟买点
start = tf.shift(tf.floor(end, FrameType.MIN30), -30, FrameType.MIN30)
bars = await sec.load_bars(start, end, FrameType.MIN30)
close = bars['close']
ma = signal.moving_average(close, 5)
err, (a, b, c), (vx, _) = signal.polyfit(ma[-7:] / ma[-7])
t1 = err < 3e-3
t2 = a > 5e-4 or (abs(a) < 1e-5 and b > 1e-2)
t3 = vx < 6
logger.debug("%s 30分钟：%s, (a,b)", sec, [t1, t2, t3, t4, t5], a, b)
if all([t1, t2, t3]):
logger.info("fired 30分钟买入:%s", sec)
await emit.emit("/alpha/signals/long", {
"plot": "crossyear",
"code": code,
"frame": str(end),
"desc": "30分钟买点",
"vx": vx,
"c": c0,
"ma": ma[-5:]
})
async def scan(self, end: Frame = None,
frame_type: FrameType = FrameType.DAY,
codes=None,
adv_limit=0.3):
"""
Args:
end:
adv_limit: 不包括在win周期内涨幅超过adv_limit的个股
Returns:
"""
win = 20
secs = Securities()
end = end or tf.floor(arrow.now(), FrameType.DAY)
results = []
holdings = await cache.sys.smembers("holdings")
for i, code in enumerate(secs.choose(['stock'])):
try:
if code in holdings: # 如果已经持仓，则不跟踪评估
continue
sec = Security(code)
if sec.code.startswith('688') or sec.display_name.find('ST') != -1:
continue
start = tf.day_shift(end, -270)
bars = await sec.load_bars(start, end, FrameType.DAY)
close = bars['close']
ma5 = signal.moving_average(close, 5)
ma250 = signal.moving_average(close, 250)
cross, idx = signal.cross(ma5[-win:], ma250[-win:])
cross_day = bars[-win + idx]['frame']
if cross != 1:
continue
ma20 = signal.moving_average(close, 20)
ma120 = signal.moving_average(close, 120)
# 如果上方还有月线和ma120线，则不发出信号，比如广州浪奇 2020-7-23,泛海控股2020-8-3
if close[-1] < ma120[-1] or close[-1] < ma20[-1]:
continue
# 计算20日以来大阳次数。如果不存在大阳线，认为还未到上涨时机，跳过
grl, ggl = features.count_long_body(bars[-20:])
if grl == 0:
continue
#
# # 计算突破以来净余买量（用阳线量减去阴线量来模拟,十字星不计入）
# bsc = bars[-10 + idx:] # bars_since_open: included both side
# ups = bsc[bsc['close'] > (bsc['open'] * 1.01)]
# downs = bsc[bsc['open'] > (bsc['close'] * 0.99)]
# balance = np.sum(ups['volume']) - np.sum(downs['volume'])
# pc = await sec.price_change(cross_day, tf.day_shift(cross_day, 5),
# FrameType.DAY, return_max=True)
#
faf = int(win - idx) # frames after fired
adv = await sec.price_change(tf.day_shift(end, -win), end,
FrameType.DAY, False)
if adv > adv_limit:
continue
logger.info(f"{sec}上穿年线\t{cross_day}\t{faf}")
await cache.sys.hmset_dict("plots.crossyear", {code: json.dumps({
"fired_at": tf.date2int(end),
"cross_day": tf.date2int(cross_day),
"faf": faf,
"grl": grl,
"ggl": ggl,
"status": 0 # 0 - generated by plots 1 - disabled manually
})})
results.append(
[sec.display_name, tf.date2int(end), tf.date2int(cross_day),
faf, grl,
ggl])
except Exception as e:
logger.exception(e)
logger.info("done crossyear scan.")
return results
def parse_sync_params(
frame: Union[str, Frame],
cat: List[str] = None,
start: Union[str, datetime.date] = None,
stop: Union[str, Frame] = None,
delay: int = 0,
include: str = "",
exclude: str = "",
) -> Tuple:
"""按照[使用手册](usage.md#22-如何同步K线数据)中的规则，解析和补全同步参数。
如果`frame_type`为分钟级，则当`start`指定为`date`类型时，自动更正为对应交易日的起始帧；
当`stop`为`date`类型时，自动更正为对应交易日的最后一帧。
Args:
frame (Union[str, Frame]): frame type to be sync. The word ``frame`` is used
here for easy understand by end user. It actually implies "FrameType".
cat (List[str]): which catetories is about to be synced. Should be one of
['stock', 'index']. Defaults to None.
start (Union[str, datetime.date], optional): [description]. Defaults to None.
stop (Union[str, Frame], optional): [description]. Defaults to None.
delay (int, optional): [description]. Defaults to 5.
include (str, optional): which securities should be included, seperated by
space, for example, "000001.XSHE 000004.XSHE". Defaults to empty string.
exclude (str, optional): which securities should be excluded, seperated by
a space. Defaults to empty string.
Returns:
- codes (List[str]): 待同步证券列表
- frame_type (FrameType):
- start (Frame):
- stop (Frame):
- delay (int):
"""
frame_type = FrameType(frame)
if frame_type in tf.minute_level_frames:
if stop:
stop = arrow.get(stop, tzinfo=cfg.tz)
if stop.hour == 0: # 未指定有效的时间帧，使用当日结束帧
stop = tf.last_min_frame(tf.day_shift(stop.date(), 0),
frame_type)
else:
stop = tf.floor(stop, frame_type)
else:
stop = tf.floor(arrow.now(tz=cfg.tz).datetime, frame_type)
if stop > arrow.now(tz=cfg.tz):
raise ValueError(f"请勿将同步截止时间设置在未来: {stop}")
if start:
start = arrow.get(start, tzinfo=cfg.tz)
if start.hour == 0: # 未指定有效的交易帧，使用当日的起始帧
start = tf.first_min_frame(tf.day_shift(start.date(), 0),
frame_type)
else:
start = tf.floor(start, frame_type)
else:
start = tf.shift(stop, -999, frame_type)
else:
stop = (stop and arrow.get(stop).date()) or arrow.now().date()
if stop == arrow.now().date():
stop = arrow.now(tz=cfg.tz)
stop = tf.floor(stop, frame_type)
start = tf.floor(
(start and arrow.get(start).date()), frame_type) or tf.shift(
stop, -1000, frame_type)
secs = Securities()
codes = secs.choose(cat or [])
exclude = map(lambda x: x, exclude.split(" "))
codes = list(set(codes) - set(exclude))
include = list(filter(lambda x: x, include.split(" ")))
codes.extend(include)
return codes, frame_type, start, stop, int(delay)
async def load_bars_batch(cls, codes: List[str], end: Frame, n: int,
frame_type: FrameType) -> AsyncIterator:
"""为一批证券品种加载行情数据
examples:
```
codes = ["000001.XSHE", "000001.XSHG"]
end = arrow.get("2020-08-27").datetime
async for code, bars in Security.load_bars_batch(codes, end, 5, FrameType.DAY):
print(code, bars[-2:])
self.assertEqual(5, len(bars))
self.assertEqual(bars[-1]["frame"], end.date())
if code == "000001.XSHG":
self.assertAlmostEqual(3350.11, bars[-1]["close"], places=2)
```
Args:
codes : 证券列表
end : 结束帧
n : 周期数
frame_type : 帧类型
Returns:
[description]
Yields:
[description]
"""
assert type(end) in (datetime.date, datetime.datetime)
closed_frame = tf.floor(end, frame_type)
if end == closed_frame:
start = tf.shift(closed_frame, -n + 1, frame_type)
cached = [
asyncio.create_task(
cls._get_bars(code, start, closed_frame, frame_type))
for code in codes
]
for fut in asyncio.as_completed(cached):
rec = await fut
yield rec
else:
start = tf.shift(closed_frame, -n + 2, frame_type)
cached = [
asyncio.create_task(
cls._get_bars(code, start, closed_frame, frame_type))
for code in codes
]
recs1 = await asyncio.gather(*cached)
recs2 = await cls._load_bars_batch(codes, end, 1, frame_type)
for code, bars in recs1:
_bars = recs2.get(code)
if _bars is None or len(_bars) != 1:
logger.warning("wrong/emtpy records for %s", code)
continue
yield code, np.append(bars, _bars)
async def load_bars(
self,
start: Frame,
stop: datetime.datetime,
frame_type: FrameType,
fq=True,
turnover=False,
) -> np.ndarray:
"""
加载[`start`, `stop`]间的行情数据到`Security`对象中，并返回行情数据。
这里`start`可以等于`stop`。
为加快速度，对分钟级别的turnover数据，均使用当前周期的成交量除以最新报告期的流通股本数,
注意这样得到的是一个近似值。如果近期有解禁股，则同样的成交量，解禁后的换手率应该小于解
禁前。
Args:
start:
stop:
frame_type:
fq: 是否进行复权处理
turnover: 是否包含turnover数据。
Returns:
"""
self._bars = None
start = tf.floor(start, frame_type)
_stop = tf.floor(stop, frame_type)
assert start <= _stop
head, tail = await cache.get_bars_range(self.code, frame_type)
if not all([head, tail]):
# not cached at all, ensure cache pointers are clear
await cache.clear_bars_range(self.code, frame_type)
n = tf.count_frames(start, _stop, frame_type)
if stop > _stop:
self._bars = await get_bars(self.code, stop, n + 1, frame_type)
else:
self._bars = await get_bars(self.code, _stop, n, frame_type)
if fq:
self.qfq()
if turnover:
await self._add_turnover(frame_type)
return self._bars
if start < head:
n = tf.count_frames(start, head, frame_type)
if n > 0:
_end = tf.shift(head, -1, frame_type)
self._bars = await get_bars(self.code, _end, n, frame_type)
if _stop > tail:
n = tf.count_frames(tail, _stop, frame_type)
if n > 0:
await get_bars(self.code, _stop, n, frame_type)
# now all closed bars in [start, _stop] should exist in cache
n = tf.count_frames(start, _stop, frame_type)
self._bars = await cache.get_bars(self.code, _stop, n, frame_type)
if arrow.get(stop) > arrow.get(_stop):
bars = await get_bars(self.code, stop, 2, frame_type)
if len(bars) == 2 and bars[0]["frame"] == self._bars[-1]["frame"]:
self._bars = np.append(self._bars, bars[1])
if fq:
self.qfq()
if turnover:
await self._add_turnover(frame_type)
return self._bars
async def sync_bars(params: dict):
"""sync bars on signal OMEGA_DO_SYNC received
Args:
params (dict): composed of the following:
```
{
secs (List[str]): 待同步的证券标的.如果为None或者为空，则从数据库中轮询
frame_type (FrameType):k线的帧类型
start (Frame): k线起始时间
stop (Frame): k线结束时间
}
```
Returns:
[type]: [description]
"""
secs, frame_type, start, stop = (
params.get("secs"),
params.get("frame_type"),
params.get("start"),
params.get("stop"),
)
if secs is not None:
logger.info(
"sync bars with %s(%s ~ %s) for given %s secs",
frame_type,
start,
stop,
len(secs),
)
async def get_sec():
return secs.pop() if len(secs) else None
else:
logger.info("sync bars with %s(%s ~ %s) in polling mode", frame_type,
start, stop)
async def get_sec():
return await cache.sys.lpop(key_scope)
key_scope = f"jobs.bars_sync.scope.{frame_type.value}"
if start is None or frame_type is None:
raise ValueError("you must specify a start date/frame_type for sync")
if stop is None:
stop = tf.floor(arrow.now(tz=cfg.tz), frame_type)
while code := await get_sec():
try:
await sync_bars_for_security(code, frame_type, start, stop)
except FetcherQuotaError as e:
logger.warning("Quota exceeded when syncing %s. Sync aborted.",
code)
logger.exception(e)
return # stop the sync
except Exception as e:
logger.warning("Failed to sync %s", code)
logger.exception(e)
async def scan(self,
frame_type: Union[str, FrameType] = FrameType.DAY,
end: Frame = None,
codes: List[str] = None):
logger.info("running momentum scan at %s level", frame_type)
if end is None:
end = arrow.now(cfg.tz).datetime
assert type(end) in (datetime.date, datetime.datetime)
frame_type = FrameType(frame_type)
ft = frame_type.value
codes = codes or Securities().choose(['stock'])
day_bars = {}
async for code, bars in Security.load_bars_batch(
codes, end, 2, FrameType.DAY):
day_bars[code] = bars
if len(day_bars) == 0:
return
async for code, bars in Security.load_bars_batch(
codes, end, 11, frame_type):
if len(bars) < 11:
continue
fired = bars[-1]['frame']
day_bar = day_bars.get(code)
if day_bar is None:
continue
c1, c0 = day_bars.get(code)[-2:]['close']
cmin = min(bars['close'])
# 还处在下跌状态、或者涨太多
if c0 == cmin or (c0 / c1 - 1) > self.baseline(f"up_limit"):
continue
ma5 = signal.moving_average(bars['close'], 5)
err, (a, b, c), (vx, _) = signal.polyfit(ma5[-7:] / ma5[-7])
# 无法拟合，或者动能不足
if err > self.baseline(f"ma5:{ft}:err") or a < self.baseline(
f"ma5:{ft}:a"):
continue
# 时间周期上应该是信号刚出现，还在窗口期内
vx_range = self.baseline(f"ma5:{ft}:vx")
if not vx_range[0] < vx < vx_range[1]:
continue
p = np.poly1d((a, b, c))
y = p(9) / p(6) - 1
# 如果预测未来三周期ma5上涨幅度不够
if y < self.baseline(f"ma5:{ft}:y"):
continue
sec = Security(code)
if frame_type == FrameType.DAY:
start = tf.shift(tf.floor(end, frame_type), -249, frame_type)
bars250 = await sec.load_bars(start, end, frame_type)
ma60 = signal.moving_average(bars250['close'], 60)
ma120 = signal.moving_average(bars250['close'], 120)
ma250 = signal.moving_average(bars250['close'], 250)
# 上方无均线压制
if (c0 > ma60[-1]) and (c0 > ma120[-1]) and (c0 > ma250[-1]):
logger.info("%s, %s, %s, %s, %s, %s", sec, round(a, 4),
round(b, 4), round(vx, 1),
round(c0 / c1 - 1, 3), round(y, 3))
await self.enter_stock_pool(code,
fired,
frame_type,
a=a,
b=b,
err=err,
y=y,
vx=self.fit_win - vx)
elif frame_type == FrameType.WEEK:
await self.enter_stock_pool(code,
fired,
frame_type,
a=a,
b=b,
err=err,
y=y,
vx=self.fit_win - vx)
elif frame_type == FrameType.MIN30:
await self.fire_trade_signal('long',
code,
fired,
frame_type,
a=a,
b=b,
err=err,
y=y,
vx=self.fit_win - vx)
async def list_stock_pool(self,
frames: int,
frame_types: List[FrameType] = None):
key = "plots.momentum.pool"
recs = await cache.sys.hgetall(key)
items = []
now = arrow.now()
for k, v in recs.items():
frame, code = k.split(":")
sec = Security(code)
v = json.loads(v)
frame_type = FrameType(v.get("frame_type"))
if frame_type not in frame_types:
continue
latest_frame = tf.floor(now, frame_type)
start = tf.shift(latest_frame, -frames, frame_type)
fired = tf.int2time(frame) if frame_type in tf.minute_level_frames else \
tf.int2date(frame)
if fired < start:
continue
items.append({
"name": sec.display_name,
"code": code,
"fired": str(fired),
"frame": frame_type.value,
"y": round(v.get("y"), 2),
"vx": round(v.get("vx"), 1),
"a": round(v.get("a"), 4),
"b": round(v.get("b"), 4),
"err": round(v.get("err"), 4)
})
return {
"name":
self.display_name,
"plot":
self.name,
"items":
items,
"headers": [{
"text": '名称',
"value": 'name'
}, {
"text": '代码',
"value": 'code'
}, {
"text": '信号时间',
"value": 'fired'
}, {
"text": '预测涨幅',
"value": 'y'
}, {
"text": '动能',
"value": 'a'
}, {
"text": '势能',
"value": 'b'
}, {
"text": '周期',
"value": 'frame'
}, {
"text": '底部距离(周期)',
"value": 'vx'
}, {
"text": '拟合误差',
"value": 'err'
}]
}
