async def save_bars(self,
sec: str,
bars: np.ndarray,
frame_type: FrameType,
sync_mode: int = 1):
"""
为每条k线记录生成一个ID，将时间：id存入该sec对应的ordered set
code:frame_type -> {
20191204: [date, o, l, h, c, v]::json
20191205: [date, o, l, h, c, v]::json
head: date or datetime
tail: date or datetime
}
:param sec: the full qualified code of a security or index
:param bars: the data to save
:param frame_type: use this to decide which store to use
:param sync_mode: 1 for update, 2 for overwrite
:return:
"""
if bars is None or len(bars) == 0:
return
head, tail = await self.get_bars_range(sec, frame_type)
if not (head and tail) or sync_mode == 2:
await self._save_bars(sec, bars, frame_type)
return
if (tf.shift(bars["frame"][-1], 1, frame_type) < head
or tf.shift(bars["frame"][0], -1, frame_type) > tail):
# don't save to database, otherwise the data is not continuous
logger.warning(
"discrete bars found, code: %s, db(%s, %s), bars(%s,%s)",
sec,
head,
tail,
bars["frame"][0],
bars["frame"][-1],
)
return
# both head and tail exist, only save bars out of database's range
bars_to_save = bars[(bars["frame"] < head) | (bars["frame"] > tail)]
if len(bars_to_save) == 0:
return
await self._save_bars(
sec,
bars_to_save,
frame_type,
min(head, bars["frame"][0]),
max(tail, bars["frame"][-1]),
)
async def _build_train_data(self, frame_type: FrameType,
n: int, max_error: float = 0.01):
"""
从最近的符合条件的日期开始，遍历股票，提取特征和标签，生成数据集。
Args:
n: 需要采样的样本数
Returns:
"""
watch_win = 5
max_curve_len = 5
max_ma_win = 20
# y_stop = arrow.get('2020-7-24').date()
y_stop = tf.floor(arrow.now(tz=cfg.tz), frame_type)
y_start = tf.shift(y_stop, -watch_win + 1, frame_type)
x_stop = tf.shift(y_start, -1, frame_type)
x_start = tf.shift(x_stop, -(max_curve_len + max_ma_win - 1), frame_type)
data = []
while len(data) < n:
for code in Securities().choose(['stock']):
#for code in ['000601.XSHE']:
try:
sec = Security(code)
x_bars = await sec.load_bars(x_start, x_stop, FrameType.DAY)
y_bars = await sec.load_bars(y_start, y_stop, FrameType.DAY)
# [a, b, axis] * 3
x = self.extract_features(x_bars, max_error)
if len(x) == 0: continue
y = np.max(y_bars['close']) / x_bars[-1]['close'] - 1
if np.isnan(y): continue
feature = [code, tf.date2int(x_stop)]
feature.extend(x)
data.append(feature)
except Exception as e:
logger.warning("Failed to extract features for %s (%s)",
code,
x_stop)
logger.exception(e)
if len(data) >= n:
break
if len(data) % 500 == 0:
logger.info("got %s records.", len(data))
y_stop = tf.day_shift(y_stop, -1)
y_start = tf.day_shift(y_start, -1)
x_stop = tf.day_shift(y_start, -1)
x_start = tf.day_shift(x_start, -1)
return data
async def copy(self,
code: str,
frame_type: Union[str, FrameType],
frame: Union[str, Frame],
ma_wins=None):
frame_type = FrameType(frame_type)
frame = arrow.get(frame, tzinfo=cfg.tz)
ma_wins = ma_wins or [5, 10, 20]
fit_win = 7
stop = frame
start = tf.shift(stop, -fit_win - max(ma_wins), frame_type)
sec = Security(code)
bars = await sec.load_bars(start, stop, frame_type)
features = []
for win in ma_wins:
ma = signal.moving_average(bars['close'], win)
if len(ma) < fit_win:
raise ValueError(
f"{sec.display_name} doesn't have enough bars for "
f"extracting features")
err, (a, b, c), (vx,
_) = signal.polyfit(ma[-fit_win:] / ma[-fit_win])
features.append((err, (a, b, c), vx))
return features
async def evaluate(self,
code: str,
frame_type: FrameType,
dt: Frame = None):
logger.debug("测试%s, 参数:%s %s", code, frame_type, dt)
win = 10
sec = Security(code)
end = dt or arrow.now(tz=cfg.tz).datetime
start = tf.shift(dt, -29, frame_type)
bars = await sec.load_bars(start, end, frame_type)
feat = features.ma_lines_trend(bars, [5, 10, 20])
ma5, ma10 = feat["ma5"][0], feat["ma10"][0]
# 判断是否存在vcross
vcrossed, (idx0, idx1) = signal.vcross(ma5[-win:], ma10[-win:])
if not vcrossed:
return
else:
dt1, dt2 = bars[-win:][idx0]['frame'], bars[-win:][idx1]['frame']
logger.info("%s vcross(5,10): %s, %s", sec, dt1, dt2)
# 月线要向上且形成支撑作用
err, a, b, vx, fit_win, war = feat["ma20"][1]
if err < self.err_20 and war > self.war_20:
logger.info("%s月线向上，war:%s", code, war)
await emit.emit(Events.sig_long, {
"code": code,
"plot": self.name,
"desc": f"{sec.display_name}发出老鸭头信号"
})
async def scan(self, stop: Frame = None):
start = tf.shift(stop, -26, FrameType.WEEK)
ERR = {5: 0.008, 10: 0.004, 20: 0.004}
for code in Securities().choose(['stock']):
#for code in ['002150.XSHE']:
sec = Security(code)
bars = await sec.load_bars(start, stop, FrameType.WEEK)
if bars[-1]['frame'] != stop:
raise ValueError("")
t1, t2, t3 = False, False, False
params = []
for win in [5, 10, 20]:
ma = signal.moving_average(bars['close'], win)
err, (a, b, c), (vx, _) = signal.polyfit(ma[-7:] / ma[-7])
if err > ERR[win]:
continue
p = np.poly1d((a, b, c))
slp3 = round(p(9) / p(6) - 1, 2)
params.append(np.round([slp3, a, b], 4))
if win == 5:
t1 = slp3 >= 0.03 and a > 0.005
if win == 10:
t2 = slp3 >= 0.02 and (b > abs(10 * a) or a > 0.0005)
if win == 20:
t3 = slp3 >= -1e-6 and a >= 0
if all([t1, t2, t3]):
print(sec.display_name, params)
async def test_004_fq(self):
"""测试复权"""
sec = Security("002320.XSHE")
start = arrow.get("2020-05-06").date()
stop = tf.shift(start, -249, FrameType.DAY)
start, stop = stop, start
# bars with no fq
bars1 = await sec.load_bars(start, stop, FrameType.DAY, fq=False)
bars2 = await sec.load_bars(start, stop, FrameType.DAY)
self.assertEqual(250, len(bars1))
expected1 = [
[
arrow.get("2019-04-24").date(),
16.26,
16.38,
15.76,
16.00,
5981087.0,
9.598480e07,
3.846000,
],
[
arrow.get("2020-05-06").date(),
10.94,
11.22,
10.90,
11.15,
22517883.0,
2.488511e08,
8.849346,
],
]
expected2 = [
[
arrow.get("2019-04-24").date(),
7.07,
7.12,
6.85,
6.95,
13762015.0,
9.598480e07,
3.846000,
],
[
arrow.get("2020-05-06").date(),
10.94,
11.22,
10.90,
11.15,
22517883.0,
2.488511e08,
8.849346,
],
]
self.assert_bars_equal(expected2, bars2)
self.assert_bars_equal(expected1, bars1)
async def summary(code: str,
end: Frame,
frame: FrameType = FrameType.DAY,
win: int = 7):
sec = Security(code)
start = tf.shift(end, -(60 + 19), frame)
bars = await sec.load_bars(start, end, frame)
end_dt = bars['date'].iat[-1]
ma5 = np.array(signal.moving_average(bars['close'], 5))
ma10 = np.array(signal.moving_average(bars['close'], 10))
ma20 = np.array(signal.moving_average(bars['close'], 20))
ma60 = np.array(signal.moving_average(bars['close'], 60))
_ma5, _ma10, _ma20 = ma5[-win:], ma10[-win:], ma20[-win:]
xlen = 20
ax_x = [i for i in range(xlen - win, xlen + 1)] # 回归线的坐标
# 5日均线及拟合点、预测点
color = '#b08080'
lw = 0.5
err5, coef5, vertex5 = signal.polyfit(_ma5)
vx, vy = vertex5
vx = xlen - win + vx
plt.plot(vx, vy * 0.995, "^", color=color) # 5日低点
plt.plot(ma5[-xlen:], color=color, linewidth=lw) # 均线
p5 = np.poly1d(coef5)
y5 = [p5(i) for i in range(win + 1)]
c0 = bars['close'].iat[-1]
pred_c = y5[-1] * 5 - bars['close'][-4:].sum()
plt.gcf().text(0.15, 0.85,
f"{code} {end_dt} ^{100 * (pred_c / c0 - 1):.02f}%")
plt.plot(ax_x, y5, 'o', color=color, mew=0.25, ms=2.5) # 回归均线
# 10日均线及回归线
color = '#00ff80'
err10, coef10, vertex10 = signal.polyfit(_ma10)
p10 = np.poly1d(coef10)
y10 = [p10(i) for i in range(win + 1)]
plt.plot(ma10[-xlen:], color=color, linewidth=lw)
plt.plot(ax_x, y10, 'o', color=color, mew=0.25, ms=2.5)
# 20日均线及回归线
color = '#00ffff'
err20, coef20, vertex20 = signal.polyfit(_ma20)
p20 = np.poly1d(coef20)
y20 = [p20(i) for i in range(win + 1)]
plt.plot(ma20[-xlen:], color=color, linewidth=lw)
plt.plot(ax_x, y20, 'o', color=color, mew=0.25, ms=2.5)
# 60日均线
color = "#2222ff"
plt.plot(ma60[-xlen:], color=color, linewidth=lw)
async def get_bars(self,
code: str,
n: int,
frame_type: FrameType,
end_dt: Frame = None):
end_dt = end_dt or arrow.now(tz=cfg.tz)
sec = Security(code)
start = tf.shift(tf.floor(end_dt, frame_type), -n + 1, frame_type)
return await sec.load_bars(start, end_dt, frame_type)
async def plot_ma(code: str,
groups=None,
end: Frame = None,
frame_type: FrameType = FrameType.DAY):
groups = groups or [5, 10, 20, 60, 120]
sec = Security(code)
end = end or tf.floor(arrow.now(), frame_type)
start = tf.shift(end, -(groups[-1] + 19), frame_type)
bars = await sec.load_bars(start, end, frame_type)
for win in groups:
ma = signal.moving_average(bars['close'], win)
plt.plot(ma[-20:])
def test_shift_min1(self):
X = [
("2020-03-26 09:31", 0, "2020-03-26 09:31"),
("2020-03-26 09:31", 1, "2020-03-26 09:32"),
("2020-03-26 11:30", 0, "2020-03-26 11:30"),
("2020-03-26 11:30", 1, "2020-03-26 13:01"),
("2020-03-26 11:30", 2, "2020-03-26 13:02"),
("2020-03-26 15:00", 0, "2020-03-26 15:00"),
("2020-03-26 15:00", 1, "2020-03-27 09:31"),
("2020-03-26 15:00", 241, "2020-03-30 09:31"),
]
for i, (start, offset, expected) in enumerate(X):
logger.debug("testing %s", X[i])
actual = tf.shift(arrow.get(start, tzinfo=cfg.tz), offset, FrameType.MIN1)
self.assertEqual(arrow.get(expected, tzinfo=cfg.tz).datetime, actual)
def test_shift_min5(self):
X = [
("2020-03-26 09:35", 0, "2020-03-26 09:35"),
("2020-03-26 09:35", 1, "2020-03-26 09:40"),
("2020-03-26 09:35", 2, "2020-03-26 09:45"),
("2020-03-26 11:30", 0, "2020-03-26 11:30"),
("2020-03-26 11:30", 1, "2020-03-26 13:05"),
("2020-03-26 11:30", 2, "2020-03-26 13:10"),
("2020-03-26 15:00", 0, "2020-03-26 15:00"),
("2020-03-26 15:00", 1, "2020-03-27 09:35"),
("2020-03-26 15:00", 49, "2020-03-30 09:35"),
]
for i, (start, offset, expected) in enumerate(X):
logger.info("testing %s", X[i])
actual = tf.shift(arrow.get(start, tzinfo=cfg.tz), offset,
FrameType.MIN5)
self.assertEqual(arrow.get(expected, tzinfo=cfg.tz), actual)
async def evaluate(self,
code: str,
frame_type: FrameType,
dt: Frame = None,
win=15):
sec = Security(code)
end = dt or arrow.now(tz=cfg.tz)
start = tf.shift(tf.floor(end, frame_type), -win - 20, frame_type)
bars = await sec.load_bars(start, end, frame_type)
if len(bars) < win + 20:
return
# 使用股价重心而不是收盘价来判断走势
o, c = bars[-1]['open'], bars[-1]['close']
feat = features.ma_lines_trend(bars, [5, 10, 20])
ma5, ma10, ma20 = feat["ma5"][0], feat["ma10"][0], feat["ma20"][0]
if np.any(np.isnan(ma5)):
return
mas = np.array([ma5[-1], ma10[-1], ma20[-1]])
# 起涨点：一阳穿三线来确认
if not (np.all(o <= mas) and np.all(c >= mas)):
return
# 三线粘合：三线距离小于self.max_distance
distance = self.distance(ma5[:-1], ma10[:-1], ma20[:-1])
if distance > self.max_distance:
return
# 月线要拉直，走平或者向上
err, (a, b) = signal.polyfit((ma20 / ma20[0]), deg=1)
if err > self.fiterr_ma20 and a < self.fitslp_ma20:
return
logger.info("%s",
f"{sec.display_name}\t{distance:.3f}\t{a:.3f}\t{err:.3f}")
await self.fire("long",
code,
dt,
frame_type=frame_type.value,
distance=distance)
def get_next_fire_time(
self,
previous_fire_time: Union[datetime.date, datetime.datetime],
now: Union[datetime.date, datetime.datetime],
):
ft = self.frame_type
next_tick = now
next_frame = tf.ceiling(now, ft)
while next_tick <= now:
if ft in tf.day_level_frames:
next_tick = tf.combine_time(next_frame, 15) + self.jitter
else:
next_tick = next_frame + self.jitter
if next_tick > now:
return next_tick
else:
next_frame = tf.shift(next_frame, 1, ft)
def test_shift_min15(self):
X = [
["2020-03-26 09:45", 0, "2020-03-26 09:45"],
["2020-03-26 09:45", 5, "2020-03-26 11:00"],
["2020-03-26 09:45", 8, "2020-03-26 13:15"],
["2020-03-27 10:45", 14, "2020-03-30 10:15"],
["2020-03-26 13:15", -9, "2020-03-25 15:00"],
["2020-03-26 13:15", -18, "2020-03-25 11:15"],
["2020-03-26 13:15", -34, "2020-03-24 11:15"],
]
fmt = "YYYY-MM-DD HH:mm"
for i, (start, offset, expected) in enumerate(X):
logger.debug("testing %s", X[i])
actual = tf.shift(
arrow.get(start, fmt, tzinfo=cfg.tz), offset, FrameType.MIN15
)
self.assertEqual(arrow.get(expected, fmt, tzinfo=cfg.tz).datetime, actual)
async def predict_ma(code: str,
frame_type: FrameType = FrameType.DAY,
end: Frame = None):
"""
预测ma5、ma10、ma20的下一个数据
Args:
code:
frame_type:
end:
Returns:
"""
sec = Security(code)
start = tf.shift(end, -29, frame_type)
bars = await sec.load_bars(start, end, frame_type)
c0 = bars['close'][-1]
target = [c0 * (1 + f / 100) for f in range(-3, 3)]
for c in target:
close = np.append(bars['close'], c)
ma5 = signal.moving_average(close, 5)
ma10 = signal.moving_average(close, 10)
ma20 = signal.moving_average(close, 20)
fig = plt.figure()
axes = plt.subplot(111)
axes.plot([i for i in range(27)],
close[-27:],
color='#000000',
linewidth=0.5) # len(ma5) == 27
axes.plot(ma5)
axes.plot([i for i in range(5, len(ma10) + 5)], ma10)
axes.text(26, ma10[-1], f"{ma10[-1]:.2f}")
axes.plot([i for i in range(15, len(ma20) + 15)], ma20)
axes.text(26, c, f"{c:.2f}")
axes.text(
0.5, 3450,
f"{100 * (c / c0 - 1):.2f}% {c:.2f} {ma5[-1]:.2f} {ma10[-1]:.2f}")
async def fire_long(self,
end: Frame,
frame_type: FrameType.DAY,
win=60,
adv=0.03):
secs = Securities()
results = []
for code in secs.choose(['stock']):
#for code in ['601238.XSHG']:
sec = Security(code)
if sec.name.find("ST") != -1 or sec.code.startswith("688"):
continue
start = tf.shift(end, -win + 1, frame_type)
bars = await sec.load_bars(start, end, frame_type)
ilow = np.argmin(bars['low'])
if ilow > win // 2: #创新低及后面的反弹太近，信号不可靠
continue
low = bars['low'][ilow]
last = bars['low'][-5:]
if np.count_nonzero((last > low) & (last < low * 1.02)) < 3:
# 对新低的测试不够
continue
c1, c0 = bars['close'][-2:]
# 今天上涨幅度是否大于adv?
if c0 / c1 - 1 < adv:
continue
# 是否站上5日线10日线？
ma5 = signal.moving_average(bars['close'], 5)
ma10 = signal.moving_average(bars['close'], 10)
if c0 < ma5[-1] or c0 < ma10[-1]: continue
price_change = await sec.price_change(end, tf.day_shift(end, 5),
frame_type)
print(f"FIRED:{end}\t{code}\t{price_change:.2f}")
results.append([end, code, price_change])
return results
async def copy(self,
code: str,
frame_type: FrameType,
end: Frame,
ma_win=5):
fit_win = 7
sec = Security(code)
start = tf.shift(end, -(ma_win + fit_win), frame_type)
bars = await sec.load_bars(start, end, frame_type)
ma = signal.moving_average(bars['close'], ma_win)
err, (a, b, c), (vx, _) = signal.polyfit(ma[-fit_win:] / ma[-fit_win])
p = np.poly1d((a, b, c))
slp3 = p(fit_win + 2) / p(fit_win - 1) - 1
print(
f"{sec.display_name}({code})\t{err:.4f}\t{a:.4f}\t{b:.4f}\t{vx:.1f}\
\t{slp3:.2f}")
self.ref_lines[f"ma{ma_win}"] = {
"err": err,
"coef": (a, b),
"vx": vx,
"slp3": slp3
}
async def evaluate(self, code: str, end: Frame):
"""
最近穿越年线的股票，回归到[5,10,20]日均线时，如果均线形态良好，则提示买入
Args:
code:
end:
Returns:
"""
start = tf.shift(tf.floor(end, FrameType.DAY), -26, FrameType.DAY)
sec = Security(code)
bars = await sec.load_bars(start, end, FrameType.DAY)
close = bars['close']
c0 = close[-1]
# 检查接近5日均线，要求5日内强于均线，均线不能向下（或者拐头趋势）
ma = signal.moving_average(close, 5)
err, (a, b, c), (vx, _) = signal.polyfit(ma[-7:] / ma[-7])
t1 = np.all(close[-5:] > ma[-5:])
t2 = err < 3e-3
t3 = a > 5e-4 or (abs(a) < 1e-5 and b > 1e-3)
t4 = (c0 - ma[-1] / c0 < 5e-3)
t5 = vx < 6
logger.debug("%s 5日：%s, (a,b):%s,%s", sec, [t1, t2, t3, t4, t5], a, b)
if all([t1, t2, t3, t4, t5]):
logger.info("fired 5日买入:%s", sec)
await emit.emit("/alpha/signals/long", {
"plot": "crossyear",
"code": code,
"frame": str(end),
"desc": "回探5日线",
"coef": np.round([a, b], 4),
"vx": vx,
"c": c0,
"ma": ma[-5:]
})
return
# 检查接近20日线买点
ma = signal.moving_average(close, 20)
err, (a, b, c), (vx, _) = signal.polyfit(ma[-10:] / ma[-10])
t1 = err < 3e-3
t2 = a > 5e-4 or (abs(a) < 1e-5 and b > 5e-3)
t3 = (c0 - ma[-1]) < 5e-3
t4 = vx < 9
logger.debug("%s 20日：%s, (a,b):%s,%s", sec, [t1, t2, t3, t4], a, b)
if all([t1, t2, t3, t4]):
logger.info("fired 20日买入:%s", sec)
await emit.emit("/alpha/signals/long", {
"plot": "crossyear",
"code": code,
"frame": str(end),
"desc": "回探20日线",
"coef": np.round([a, b], 4),
"vx": vx,
"c": c0,
"ma": ma[-5:]
})
return
# 检查是否存在30分钟买点
start = tf.shift(tf.floor(end, FrameType.MIN30), -30, FrameType.MIN30)
bars = await sec.load_bars(start, end, FrameType.MIN30)
close = bars['close']
ma = signal.moving_average(close, 5)
err, (a, b, c), (vx, _) = signal.polyfit(ma[-7:] / ma[-7])
t1 = err < 3e-3
t2 = a > 5e-4 or (abs(a) < 1e-5 and b > 1e-2)
t3 = vx < 6
logger.debug("%s 30分钟：%s, (a,b)", sec, [t1, t2, t3, t4, t5], a, b)
if all([t1, t2, t3]):
logger.info("fired 30分钟买入:%s", sec)
await emit.emit("/alpha/signals/long", {
"plot": "crossyear",
"code": code,
"frame": str(end),
"desc": "30分钟买点",
"vx": vx,
"c": c0,
"ma": ma[-5:]
})
async def save_bars(self,
sec: str,
bars: np.ndarray,
frame_type: FrameType,
sync_mode: int = 1):
"""将行情数据存入缓存
在redis cache中的数据以如下方式存储
```text
"000001.XSHE:30m" -> {
# frame open low high close volume amount factor
"20200805": "13.82 13.85 13.62 13.76 144020313.0 1980352978.34 120.77"
"20200806": "13.82 13.96 13.65 13.90 135251068.0 1868047342.49 120.77"
"head": "20200805"
"tail": "20200806"
}
```
这里的amount即对应frame的成交额；factor为复权因子
args:
sec: the full qualified code of a security or index
bars: the data to save
frame_type: use this to decide which store to use
sync_mode: 1 for update, 2 for overwrite
"""
if bars is None or len(bars) == 0:
return
head, tail = await self.get_bars_range(sec, frame_type)
if not (head and tail) or sync_mode == 2:
await self._save_bars(sec, bars, frame_type)
return
if (tf.shift(bars["frame"][-1], 1, frame_type) < head
or tf.shift(bars["frame"][0], -1, frame_type) > tail):
# don't save to database, otherwise the data is not continuous
logger.warning(
"discrete bars found, code: %s, db(%s, %s), bars(%s,%s)",
sec,
head,
tail,
bars["frame"][0],
bars["frame"][-1],
)
return
# both head and tail exist, only save bars out of database's range
bars_to_save = bars[(bars["frame"] < head) | (bars["frame"] > tail)]
if len(bars_to_save) == 0:
return
await self._save_bars(
sec,
bars_to_save,
frame_type,
min(head, bars["frame"][0]),
max(tail, bars["frame"][-1]),
)
async def sync_bars_for_security(
code: str,
frame_type: FrameType,
start: Union[datetime.date, datetime.datetime],
stop: Union[None, datetime.date, datetime.datetime],
):
counters = 0
# 取数据库中该frame_type下该code的k线起始点
head, tail = await cache.get_bars_range(code, frame_type)
if not all([head, tail]):
await cache.clear_bars_range(code, frame_type)
n_bars = tf.count_frames(start, stop, frame_type)
bars = await aq.get_bars(code, stop, n_bars, frame_type)
if bars is not None and len(bars):
logger.debug(
"sync %s(%s), from %s to %s: actual got %s ~ %s (%s)",
code,
frame_type,
start,
head,
bars[0]["frame"],
bars[-1]["frame"],
len(bars),
)
counters = len(bars)
return
if start < head:
n = tf.count_frames(start, head, frame_type) - 1
if n > 0:
_end_at = tf.shift(head, -1, frame_type)
bars = await aq.get_bars(code, _end_at, n, frame_type)
if bars is not None and len(bars):
counters += len(bars)
logger.debug(
"sync %s(%s), from %s to %s: actual got %s ~ %s (%s)",
code,
frame_type,
start,
head,
bars[0]["frame"],
bars[-1]["frame"],
len(bars),
)
if bars["frame"][-1] != _end_at:
logger.warning(
"discrete frames found:%s, bars[-1](%s), "
"head(%s)",
code,
bars["frame"][-1],
head,
)
if stop > tail:
n = tf.count_frames(tail, stop, frame_type) - 1
if n > 0:
bars = await aq.get_bars(code, stop, n, frame_type)
if bars is not None and len(bars):
logger.debug(
"sync %s(%s), from %s to %s: actual got %s ~ %s (%s)",
code,
frame_type,
tail,
stop,
bars[0]["frame"],
bars[-1]["frame"],
len(bars),
)
counters += len(bars)
if bars["frame"][0] != tf.shift(tail, 1, frame_type):
logger.warning(
"discrete frames found: %s, tail(%s), bars[0]("
"%s)",
code,
tail,
bars["frame"][0],
)
def test_shift(self):
mom = arrow.get("2020-1-20")
self.assertEqual(tf.shift(mom, 1, FrameType.DAY), tf.day_shift(mom, 1))
self.assertEqual(tf.shift(mom, 1, FrameType.WEEK), tf.week_shift(mom, 1))
self.assertEqual(tf.shift(mom, 1, FrameType.MONTH), tf.month_shift(mom, 1))
async def visualize(self, code: Union[str, List[str]], frame: Union[str,
Frame],
frame_type: Union[str, FrameType]):
"""
将code列表中的股票的动量特征图象化
Args:
code:
frame:
frame_type:
Returns:
"""
import matplotlib.pyplot as plt
if isinstance(code, str):
code = [code]
col = 4
row = len(code) // col + 1
plt.figure(figsize=(5 * row * col, 7))
plt.subplots_adjust(wspace=0.2, hspace=0.2)
fit_win = 7
colors = {"5": '#808080', "10": '#00cc80', "20": '#00ccff'}
frame = arrow.get(frame)
frame_type = FrameType(frame_type)
for i, code in enumerate(code):
_code = code.split(".")[0]
start = tf.shift(frame, -25, frame_type)
bars = await Security(code).load_bars(start, frame, frame_type)
plt.subplot(len(code) // col + 1, col, i + 1)
y_lim = 0
text = ""
for win in [5, 10, 20]:
ma = signal.moving_average(bars['close'], win)
_ma = ma[-fit_win:]
plt.plot(_ma, color=colors[f"{win}"])
err, (a, b, c), (vx, _) = signal.polyfit(_ma / _ma[0])
p = np.poly1d((a * _ma[0], b * _ma[0], c * _ma[0]))
y = p(fit_win + 2) / p(fit_win - 1) - 1
y_lim = max(y_lim, np.max(_ma))
if win == 5:
text = f"{_code} a:{a:.4f} b:{b:.4f} vx:{vx:.1f} y:{y:.2f}"
if err < self.baseline(f"ma{win}:{frame_type.value}:err"):
# 如果拟合在误差范围内，则画出拟合线
plt.plot([p(i) for i in range(len(_ma))],
"--",
color=colors[f"{win}"])
plt.plot([p(i) for i in range(len(_ma))],
"o",
color=colors[f"{win}"])
if 0 < vx < fit_win:
plt.plot([vx], p(vx), 'x')
plt.plot(0, y_lim * 1.035)
plt.text(0.1, y_lim * 1.02, text, color='r')
async def sync_bars_for_security(
code: str,
frame_type: FrameType,
start: Union[datetime.date, datetime.datetime],
stop: Union[None, datetime.date, datetime.datetime],
):
counters = 0
logger.info("syncing quotes for %s", code)
# 取数据库中该frame_type下该code的k线起始点
head, tail = await cache.get_bars_range(code, frame_type)
if not all([head, tail]):
await cache.clear_bars_range(code, frame_type)
n_bars = tf.count_frames(start, stop, frame_type)
bars = await aq.get_bars(code, stop, n_bars, frame_type)
counters = len(bars)
logger.info("finished sync %s(%s), %s bars synced", code, frame_type,
counters)
return
if start < head:
n = tf.count_frames(start, head, frame_type) - 1
if n > 0:
_end_at = tf.shift(head, -1, frame_type)
bars = await aq.get_bars(code, _end_at, n, frame_type)
counters += len(bars)
logger.debug(
"sync %s level bars of %s to %s: expected: %s, actual %s",
frame_type,
code,
_end_at,
n,
len(bars),
)
if len(bars) and bars["frame"][-1] != _end_at:
logger.warning(
"discrete frames found:%s, bars[-1](%s), "
"head(%s)",
code,
bars["frame"][-1],
head,
)
if stop > tail:
n = tf.count_frames(tail, stop, frame_type) - 1
if n > 0:
bars = await aq.get_bars(code, stop, n, frame_type)
logger.debug(
"sync %s level bars of %s to %s: expected: %s, actual %s",
frame_type,
code,
stop,
n,
len(bars),
)
counters += len(bars)
if bars["frame"][0] != tf.shift(tail, 1, frame_type):
logger.warning(
"discrete frames found: %s, tail(%s), bars[0]("
"%s)",
code,
tail,
bars["frame"][0],
)
logger.info("finished sync %s(%s), %s bars synced", code, frame_type,
counters)
async def list_stock_pool(self,
frames: int,
frame_types: List[FrameType] = None):
key = "plots.momentum.pool"
recs = await cache.sys.hgetall(key)
items = []
now = arrow.now()
for k, v in recs.items():
frame, code = k.split(":")
sec = Security(code)
v = json.loads(v)
frame_type = FrameType(v.get("frame_type"))
if frame_type not in frame_types:
continue
latest_frame = tf.floor(now, frame_type)
start = tf.shift(latest_frame, -frames, frame_type)
fired = tf.int2time(frame) if frame_type in tf.minute_level_frames else \
tf.int2date(frame)
if fired < start:
continue
items.append({
"name": sec.display_name,
"code": code,
"fired": str(fired),
"frame": frame_type.value,
"y": round(v.get("y"), 2),
"vx": round(v.get("vx"), 1),
"a": round(v.get("a"), 4),
"b": round(v.get("b"), 4),
"err": round(v.get("err"), 4)
})
return {
"name":
self.display_name,
"plot":
self.name,
"items":
items,
"headers": [{
"text": '名称',
"value": 'name'
}, {
"text": '代码',
"value": 'code'
}, {
"text": '信号时间',
"value": 'fired'
}, {
"text": '预测涨幅',
"value": 'y'
}, {
"text": '动能',
"value": 'a'
}, {
"text": '势能',
"value": 'b'
}, {
"text": '周期',
"value": 'frame'
}, {
"text": '底部距离(周期)',
"value": 'vx'
}, {
"text": '拟合误差',
"value": 'err'
}]
}
async def load_bars_batch(cls, codes: List[str], end: Frame, n: int,
frame_type: FrameType) -> AsyncIterator:
"""为一批证券品种加载行情数据
examples:
```
codes = ["000001.XSHE", "000001.XSHG"]
end = arrow.get("2020-08-27").datetime
async for code, bars in Security.load_bars_batch(codes, end, 5, FrameType.DAY):
print(code, bars[-2:])
self.assertEqual(5, len(bars))
self.assertEqual(bars[-1]["frame"], end.date())
if code == "000001.XSHG":
self.assertAlmostEqual(3350.11, bars[-1]["close"], places=2)
```
Args:
codes : 证券列表
end : 结束帧
n : 周期数
frame_type : 帧类型
Returns:
[description]
Yields:
[description]
"""
assert type(end) in (datetime.date, datetime.datetime)
closed_frame = tf.floor(end, frame_type)
if end == closed_frame:
start = tf.shift(closed_frame, -n + 1, frame_type)
cached = [
asyncio.create_task(
cls._get_bars(code, start, closed_frame, frame_type))
for code in codes
]
for fut in asyncio.as_completed(cached):
rec = await fut
yield rec
else:
start = tf.shift(closed_frame, -n + 2, frame_type)
cached = [
asyncio.create_task(
cls._get_bars(code, start, closed_frame, frame_type))
for code in codes
]
recs1 = await asyncio.gather(*cached)
recs2 = await cls._load_bars_batch(codes, end, 1, frame_type)
for code, bars in recs1:
_bars = recs2.get(code)
if _bars is None or len(_bars) != 1:
logger.warning("wrong/emtpy records for %s", code)
continue
yield code, np.append(bars, _bars)
async def load_bars(
self,
start: Frame,
stop: datetime.datetime,
frame_type: FrameType,
fq=True,
turnover=False,
) -> np.ndarray:
"""
加载[`start`, `stop`]间的行情数据到`Security`对象中，并返回行情数据。
这里`start`可以等于`stop`。
为加快速度，对分钟级别的turnover数据，均使用当前周期的成交量除以最新报告期的流通股本数,
注意这样得到的是一个近似值。如果近期有解禁股，则同样的成交量，解禁后的换手率应该小于解
禁前。
Args:
start:
stop:
frame_type:
fq: 是否进行复权处理
turnover: 是否包含turnover数据。
Returns:
"""
self._bars = None
start = tf.floor(start, frame_type)
_stop = tf.floor(stop, frame_type)
assert start <= _stop
head, tail = await cache.get_bars_range(self.code, frame_type)
if not all([head, tail]):
# not cached at all, ensure cache pointers are clear
await cache.clear_bars_range(self.code, frame_type)
n = tf.count_frames(start, _stop, frame_type)
if stop > _stop:
self._bars = await get_bars(self.code, stop, n + 1, frame_type)
else:
self._bars = await get_bars(self.code, _stop, n, frame_type)
if fq:
self.qfq()
if turnover:
await self._add_turnover(frame_type)
return self._bars
if start < head:
n = tf.count_frames(start, head, frame_type)
if n > 0:
_end = tf.shift(head, -1, frame_type)
self._bars = await get_bars(self.code, _end, n, frame_type)
if _stop > tail:
n = tf.count_frames(tail, _stop, frame_type)
if n > 0:
await get_bars(self.code, _stop, n, frame_type)
# now all closed bars in [start, _stop] should exist in cache
n = tf.count_frames(start, _stop, frame_type)
self._bars = await cache.get_bars(self.code, _stop, n, frame_type)
if arrow.get(stop) > arrow.get(_stop):
bars = await get_bars(self.code, stop, 2, frame_type)
if len(bars) == 2 and bars[0]["frame"] == self._bars[-1]["frame"]:
self._bars = np.append(self._bars, bars[1])
if fq:
self.qfq()
if turnover:
await self._add_turnover(frame_type)
return self._bars
async def scan(self,
frame_type: Union[str, FrameType] = FrameType.DAY,
end: Frame = None,
codes: List[str] = None):
logger.info("running momentum scan at %s level", frame_type)
if end is None:
end = arrow.now(cfg.tz).datetime
assert type(end) in (datetime.date, datetime.datetime)
frame_type = FrameType(frame_type)
ft = frame_type.value
codes = codes or Securities().choose(['stock'])
day_bars = {}
async for code, bars in Security.load_bars_batch(
codes, end, 2, FrameType.DAY):
day_bars[code] = bars
if len(day_bars) == 0:
return
async for code, bars in Security.load_bars_batch(
codes, end, 11, frame_type):
if len(bars) < 11:
continue
fired = bars[-1]['frame']
day_bar = day_bars.get(code)
if day_bar is None:
continue
c1, c0 = day_bars.get(code)[-2:]['close']
cmin = min(bars['close'])
# 还处在下跌状态、或者涨太多
if c0 == cmin or (c0 / c1 - 1) > self.baseline(f"up_limit"):
continue
ma5 = signal.moving_average(bars['close'], 5)
err, (a, b, c), (vx, _) = signal.polyfit(ma5[-7:] / ma5[-7])
# 无法拟合，或者动能不足
if err > self.baseline(f"ma5:{ft}:err") or a < self.baseline(
f"ma5:{ft}:a"):
continue
# 时间周期上应该是信号刚出现，还在窗口期内
vx_range = self.baseline(f"ma5:{ft}:vx")
if not vx_range[0] < vx < vx_range[1]:
continue
p = np.poly1d((a, b, c))
y = p(9) / p(6) - 1
# 如果预测未来三周期ma5上涨幅度不够
if y < self.baseline(f"ma5:{ft}:y"):
continue
sec = Security(code)
if frame_type == FrameType.DAY:
start = tf.shift(tf.floor(end, frame_type), -249, frame_type)
bars250 = await sec.load_bars(start, end, frame_type)
ma60 = signal.moving_average(bars250['close'], 60)
ma120 = signal.moving_average(bars250['close'], 120)
ma250 = signal.moving_average(bars250['close'], 250)
# 上方无均线压制
if (c0 > ma60[-1]) and (c0 > ma120[-1]) and (c0 > ma250[-1]):
logger.info("%s, %s, %s, %s, %s, %s", sec, round(a, 4),
round(b, 4), round(vx, 1),
round(c0 / c1 - 1, 3), round(y, 3))
await self.enter_stock_pool(code,
fired,
frame_type,
a=a,
b=b,
err=err,
y=y,
vx=self.fit_win - vx)
elif frame_type == FrameType.WEEK:
await self.enter_stock_pool(code,
fired,
frame_type,
a=a,
b=b,
err=err,
y=y,
vx=self.fit_win - vx)
elif frame_type == FrameType.MIN30:
await self.fire_trade_signal('long',
code,
fired,
frame_type,
a=a,
b=b,
err=err,
y=y,
vx=self.fit_win - vx)
def parse_sync_params(
frame: Union[str, Frame],
cat: List[str] = None,
start: Union[str, datetime.date] = None,
stop: Union[str, Frame] = None,
delay: int = 0,
include: str = "",
exclude: str = "",
) -> Tuple:
"""按照[使用手册](usage.md#22-如何同步K线数据)中的规则，解析和补全同步参数。
如果`frame_type`为分钟级，则当`start`指定为`date`类型时，自动更正为对应交易日的起始帧；
当`stop`为`date`类型时，自动更正为对应交易日的最后一帧。
Args:
frame (Union[str, Frame]): frame type to be sync. The word ``frame`` is used
here for easy understand by end user. It actually implies "FrameType".
cat (List[str]): which catetories is about to be synced. Should be one of
['stock', 'index']. Defaults to None.
start (Union[str, datetime.date], optional): [description]. Defaults to None.
stop (Union[str, Frame], optional): [description]. Defaults to None.
delay (int, optional): [description]. Defaults to 5.
include (str, optional): which securities should be included, seperated by
space, for example, "000001.XSHE 000004.XSHE". Defaults to empty string.
exclude (str, optional): which securities should be excluded, seperated by
a space. Defaults to empty string.
Returns:
- codes (List[str]): 待同步证券列表
- frame_type (FrameType):
- start (Frame):
- stop (Frame):
- delay (int):
"""
frame_type = FrameType(frame)
if frame_type in tf.minute_level_frames:
if stop:
stop = arrow.get(stop, tzinfo=cfg.tz)
if stop.hour == 0: # 未指定有效的时间帧，使用当日结束帧
stop = tf.last_min_frame(tf.day_shift(stop.date(), 0),
frame_type)
else:
stop = tf.floor(stop, frame_type)
else:
stop = tf.floor(arrow.now(tz=cfg.tz).datetime, frame_type)
if stop > arrow.now(tz=cfg.tz):
raise ValueError(f"请勿将同步截止时间设置在未来: {stop}")
if start:
start = arrow.get(start, tzinfo=cfg.tz)
if start.hour == 0: # 未指定有效的交易帧，使用当日的起始帧
start = tf.first_min_frame(tf.day_shift(start.date(), 0),
frame_type)
else:
start = tf.floor(start, frame_type)
else:
start = tf.shift(stop, -999, frame_type)
else:
stop = (stop and arrow.get(stop).date()) or arrow.now().date()
if stop == arrow.now().date():
stop = arrow.now(tz=cfg.tz)
stop = tf.floor(stop, frame_type)
start = tf.floor(
(start and arrow.get(start).date()), frame_type) or tf.shift(
stop, -1000, frame_type)
secs = Securities()
codes = secs.choose(cat or [])
exclude = map(lambda x: x, exclude.split(" "))
codes = list(set(codes) - set(exclude))
include = list(filter(lambda x: x, include.split(" ")))
codes.extend(include)
return codes, frame_type, start, stop, int(delay)
async def test_002_load_bars(self):
sec = Security("000001.XSHE")
start = arrow.get("2020-01-03").date()
stop = arrow.get("2020-1-16").date()
frame_type = FrameType.DAY
expected = [
[
arrow.get("2020-01-03").date(),
16.94,
17.31,
16.92,
17.18,
1.11619481e8,
1914495474.63,
118.73,
],
[
stop, 16.52, 16.57, 16.2, 16.33, 1.02810467e8, 1678888507.83,
118.73
],
]
logger.info("scenario: no cache")
await cache.clear_bars_range(sec.code, frame_type)
bars = await sec.load_bars(start, start, frame_type)
self.assert_bars_equal([expected[0]], bars)
bars = await sec.load_bars(start, stop, frame_type)
self.assert_bars_equal(expected, bars)
logger.info("scenario: load from cache")
bars = await sec.load_bars(start, stop, frame_type)
self.assert_bars_equal(expected, bars)
logger.info("scenario: partial data fetch: head")
await cache.set_bars_range(sec.code,
frame_type,
start=arrow.get("2020-01-07").date())
bars = await sec.load_bars(start, stop, frame_type)
self.assert_bars_equal(expected, bars)
logger.info("scenario: partial data fetch: tail")
await cache.set_bars_range(sec.code,
frame_type,
end=arrow.get("2020-01-14").date())
bars = await sec.load_bars(start, stop, frame_type)
self.assert_bars_equal(expected, bars)
logger.info("scenario: 1min level backward")
frame_type = FrameType.MIN1
start = arrow.get("2020-05-06 15:00:00", tzinfo=cfg.tz).datetime
await cache.clear_bars_range(sec.code, frame_type)
stop = tf.shift(start, -249, frame_type)
start, stop = stop, start
bars = await sec.load_bars(start, stop, frame_type)
# fmt:off
expected = [[
arrow.get('2020-04-30 14:51:00', tzinfo=cfg.tz).datetime, 13.99,
14., 13.98, 13.99, 281000., 3931001., 118.725646
],
[
arrow.get('2020-05-06 15:00:00',
tzinfo=cfg.tz).datetime, 13.77, 13.77, 13.77,
13.77, 1383400.0, 19049211.45000005, 118.725646
]]
# fmt:on
self.assert_bars_equal(expected, bars)
logger.info("scenario: 30 min level")
frame_type = FrameType.MIN15
start = arrow.get("2020-05-06 10:15:00", tzinfo=cfg.tz).datetime
await cache.clear_bars_range(sec.code, frame_type)
stop = arrow.get("2020-05-06 15:00:00", tzinfo=cfg.tz).datetime
bars = await sec.load_bars(start, stop, frame_type)
# fmt: off
expected = [[
arrow.get('2020-05-06 10:15:00', tzinfo=cfg.tz).datetime, 13.67,
13.74, 13.66, 13.72, 8341905., 1.14258451e+08, 118.725646
],
[
arrow.get('2020-05-06 15:00:00',
tzinfo=cfg.tz).datetime, 13.72, 13.77, 13.72,
13.77, 7053085., 97026350.76999998, 118.725646
]]
# fmt: on
self.assert_bars_equal(expected, bars)
async def fire_long(self,
end: Frame = None,
overlap_win=10,
frame_type: FrameType = FrameType.MIN30):
"""
寻找开多仓信号
Args:
Returns:
"""
result = []
end = end or arrow.now().datetime
secs = Securities()
for code in secs.choose(['stock']):
#for code in ['600139.XSHG']:
try:
sec = Security(code)
start = tf.shift(end, -(60 + overlap_win - 1), frame_type)
bars = await sec.load_bars(start, end, frame_type)
mas = {}
for win in [5, 10, 20, 60]:
ma = signal.moving_average(bars['close'], win)
mas[f"{win}"] = ma
# 收盘价高于各均线值
c1, c0 = bars['close'][-2:]
t1 = c0 > mas["5"][-1] and c0 > mas["10"][-1] and c0 > mas["20"][-1] \
and c0 > mas["60"][-1]
# 60均线斜率向上
slope_60, err = signal.slope(mas["60"][-10:])
if err is None or err > 5e-4:
continue
t2 = slope_60 >= 5e-4
# 均线粘合
diff = np.abs(mas["5"][-6:-1] -
mas["10"][-6:-1]) / mas["10"][-6:-1]
overlap_5_10 = np.count_nonzero(diff < 5e-3)
t3 = overlap_5_10 > 3
diff = np.abs(mas["10"][-10:] -
mas["60"][-10:]) / mas["60"][-10:]
overlap_10_60 = np.count_nonzero(diff < 5e-3)
t4 = overlap_10_60 > 5
price_change = await sec.price_change(
end, tf.shift(end, 8, frame_type), frame_type)
result.append(
[end, code, t1, t2, t3, t4, slope_60, price_change, True])
if t1 and t2 and t3 and t4:
print("FIRED:", [
end, code, t1, t2, t3, t4, slope_60, price_change, True
])
except Exception as e:
pass
return result
