async def get_bars( cls, sec: str, end: Frame, n_bars: int, frame_type: FrameType, include_unclosed=True, ) -> np.ndarray: bars = await cls.get_instance().get_bars( sec, end, n_bars, frame_type, include_unclosed ) now = arrow.now() if type(end) == datetime.date: if end == now.date(): closed = tf.floor( datetime.datetime(end.year, end.month, end.day, now.hour), frame_type, ) else: closed = tf.floor( datetime.datetime(end.year, end.month, end.day, 15), frame_type ) else: closed = tf.floor(end, frame_type) if frame_type in tf.day_level_frames: end = end.date() # noqa else: end = end.replace(second=0, microsecond=0) if closed != end: finished = cls._fill_na(bars, n_bars - 1, closed, frame_type) if bars[-1]["frame"] == end: remainder = [bars[-1]] else: # 停牌,或者end当天休市。调用者要自己保证传入的end不在休市中 remainder = np.empty(1, dtype=bars.dtype) remainder[:] = np.nan remainder["frame"] = end logger.warning("证券%s在frame [%s]处于停牌中", sec, end) else: finished = cls._fill_na(bars, n_bars, closed, frame_type) remainder = None # 只保存已结束的frame数据到数据库 await cache.save_bars(sec, finished, frame_type) if remainder is None: return finished return np.concatenate([finished, remainder])
async def test_get_bars_011(self): """分钟级别,中间有停牌,end指定时间未对齐的情况""" # 600721, ST百花, 2020-4-29停牌一天 sec = "600721.XSHG" frame_type = FrameType.MIN60 end = arrow.get("2020-04-30 10:32", tzinfo="Asia/Shanghai").datetime await self.clear_cache(sec, frame_type) bars = await aq.get_bars(sec, end, 7, frame_type) print(bars) self.assertEqual(7, len(bars)) self.assertEqual(arrow.get("2020-04-28 15:00", tzinfo="Asia/Shanghai"), bars["frame"][0]) self.assertEqual(arrow.get("2020-04-30 10:30", tzinfo="Asia/Shanghai"), bars["frame"][-2]) self.assertEqual(arrow.get("2020-4-30 10:32", tzinfo="Asia/Shanghai"), bars["frame"][-1]) self.assertAlmostEqual(5.37, bars["open"][0], places=2) self.assertAlmostEqual(5.26, bars["open"][-2], places=2) self.assertAlmostEqual(5.33, bars["open"][-1], places=2) # 检查cache,10:32未存入cache cache_len = await cache.security.hlen(f"{sec}:{frame_type.value}") self.assertEqual(8, cache_len) bars_2 = await cache.get_bars(sec, tf.floor(end, frame_type), 6, frame_type) np.array_equal(bars[:-1], bars_2)
async def test_get_bars_013(self): """分钟级别,end指定时间正处在停牌中""" # 600721, ST百花, 2020-4-29停牌一天 sec = "600721.XSHG" frame_type = FrameType.MIN60 end = arrow.get("2020-04-29 10:30", tzinfo="Asia/Chongqing").datetime await self.clear_cache(sec, frame_type) bars = await aq.get_bars(sec, end, 6, frame_type) print(bars) self.assertEqual(6, len(bars)) self.assertEqual(arrow.get("2020-04-27 15:00", tzinfo="Asia/Shanghai"), bars["frame"][0]) self.assertEqual(arrow.get("2020-04-29 10:30", tzinfo="Asia/Shanghai"), bars["frame"][-1]) self.assertAlmostEqual(5.47, bars["open"][0], places=2) self.assertAlmostEqual(5.37, bars["open"][-2], places=2) self.assertTrue(np.isnan(bars["open"][-1])) # 检查cache,10:30 已存入cache cache_len = await cache.security.hlen(f"{sec}:{frame_type.value}") self.assertEqual(8, cache_len) bars_2 = await cache.get_bars(sec, tf.floor(end, frame_type), 6, frame_type) np.array_equal(bars, bars_2)
def _parse_sync_params(sync_params: dict): """ 如果sync_params['start'], sync_params['stop']类型为date: 如果frame_type为分钟级,则意味着要取到当天的开始帧和结束帧; 如果frame_type为日期级,则要对齐到已结束的帧日期,并且设置时间为15:00否则某些sdk可能会只取 到上一周期数据(如jqdatasdk 1.8) 如果sync_params['start'], sync_params['stop']类型为datetime: 如果frame_type为分钟级别,则通过tf.floor对齐到已结束的帧 如果frame_type为日线级别,则对齐到上一个已结束的日帧,时间部分重置为15:00 Args: sync_params: Returns: """ frame_type = sync_params.get("frame_type") start = sync_params.get("start") stop = sync_params.get("stop") if start is None: raise ValueError("sync_params['start'] must be specified!") if type(start) not in [datetime.date, datetime.datetime]: raise TypeError( "type of sync_params['start'] must be one of [" "datetime.datetime, datetime.date]" ) if stop is None: stop = tf.floor(arrow.now(cfg.tz), frame_type) if type(stop) not in [datetime.datetime, datetime.date]: raise TypeError( "type of sync_params['stop'] must be one of [None, " "datetime.datetime, datetime.date]" ) if frame_type in tf.minute_level_frames: if type(start) is datetime.date: start = tf.floor(start, FrameType.DAY) minutes = tf.ticks[frame_type][0] h, m = minutes // 60, minutes % 60 start = datetime.datetime( start.year, start.month, start.day, h, m, tzinfo=tz.gettz(cfg.tz) ) else: start = tf.floor(start, frame_type) if type(stop) is datetime.date: stop = tf.floor(stop, FrameType.DAY) stop = datetime.datetime( stop.year, stop.month, stop.day, 15, tzinfo=tz.gettz(cfg.tz) ) else: stop = tf.floor(stop, frame_type) else: start = tf.floor(start, frame_type) stop = tf.floor(stop, frame_type) return frame_type, start, stop
async def get_bars(self, code: str, n: int, frame_type: FrameType, end_dt: Frame = None): end_dt = end_dt or arrow.now(tz=cfg.tz) sec = Security(code) start = tf.shift(tf.floor(end_dt, frame_type), -n + 1, frame_type) return await sec.load_bars(start, end_dt, frame_type)
async def test_evaluate(self): scheduler = AsyncIOScheduler(timezone=cfg.tz) plot = DuckPlot(scheduler) secs = Securities().choose(['stock']) for code in secs: dt = tf.floor(arrow.get('2020-7-24'), FrameType.DAY) try: await plot.evaluate(code, FrameType.DAY, dt=dt) except Exception as e: logger.exception(e)
async def plot_ma(code: str, groups=None, end: Frame = None, frame_type: FrameType = FrameType.DAY): groups = groups or [5, 10, 20, 60, 120] sec = Security(code) end = end or tf.floor(arrow.now(), frame_type) start = tf.shift(end, -(groups[-1] + 19), frame_type) bars = await sec.load_bars(start, end, frame_type) for win in groups: ma = signal.moving_average(bars['close'], win) plt.plot(ma[-20:])
async def _build_train_data(self, frame_type: FrameType, n: int, max_error: float = 0.01): """ 从最近的符合条件的日期开始,遍历股票,提取特征和标签,生成数据集。 Args: n: 需要采样的样本数 Returns: """ watch_win = 5 max_curve_len = 5 max_ma_win = 20 # y_stop = arrow.get('2020-7-24').date() y_stop = tf.floor(arrow.now(tz=cfg.tz), frame_type) y_start = tf.shift(y_stop, -watch_win + 1, frame_type) x_stop = tf.shift(y_start, -1, frame_type) x_start = tf.shift(x_stop, -(max_curve_len + max_ma_win - 1), frame_type) data = [] while len(data) < n: for code in Securities().choose(['stock']): #for code in ['000601.XSHE']: try: sec = Security(code) x_bars = await sec.load_bars(x_start, x_stop, FrameType.DAY) y_bars = await sec.load_bars(y_start, y_stop, FrameType.DAY) # [a, b, axis] * 3 x = self.extract_features(x_bars, max_error) if len(x) == 0: continue y = np.max(y_bars['close']) / x_bars[-1]['close'] - 1 if np.isnan(y): continue feature = [code, tf.date2int(x_stop)] feature.extend(x) data.append(feature) except Exception as e: logger.warning("Failed to extract features for %s (%s)", code, x_stop) logger.exception(e) if len(data) >= n: break if len(data) % 500 == 0: logger.info("got %s records.", len(data)) y_stop = tf.day_shift(y_stop, -1) y_start = tf.day_shift(y_start, -1) x_stop = tf.day_shift(y_start, -1) x_start = tf.day_shift(x_start, -1) return data
async def evaluate(self, code: str, frame_type: Union[FrameType, str], flag: str, win: int, c1: float, d1: Frame, c2: float, d2: Frame, slip: float = 0.015): frame_type = FrameType(frame_type) n1 = tf.count_frames(d1, d2, frame_type) slp = (c2 - c1) / n1 n2 = tf.count_frames(d2, tf.floor(arrow.now(), frame_type), frame_type) c_ = c2 + slp * n2 bars = await self.get_bars(code, 1, frame_type) if abs(c_ / bars[-1]['close'] - 1) <= slip: await self.fire_trade_signal(flag, code, bars[-1]['frame'], frame_type)
async def evaluate(self, code: str, frame_type: FrameType, dt: Frame = None, win=15): sec = Security(code) end = dt or arrow.now(tz=cfg.tz) start = tf.shift(tf.floor(end, frame_type), -win - 20, frame_type) bars = await sec.load_bars(start, end, frame_type) if len(bars) < win + 20: return # 使用股价重心而不是收盘价来判断走势 o, c = bars[-1]['open'], bars[-1]['close'] feat = features.ma_lines_trend(bars, [5, 10, 20]) ma5, ma10, ma20 = feat["ma5"][0], feat["ma10"][0], feat["ma20"][0] if np.any(np.isnan(ma5)): return mas = np.array([ma5[-1], ma10[-1], ma20[-1]]) # 起涨点:一阳穿三线来确认 if not (np.all(o <= mas) and np.all(c >= mas)): return # 三线粘合:三线距离小于self.max_distance distance = self.distance(ma5[:-1], ma10[:-1], ma20[:-1]) if distance > self.max_distance: return # 月线要拉直,走平或者向上 err, (a, b) = signal.polyfit((ma20 / ma20[0]), deg=1) if err > self.fiterr_ma20 and a < self.fitslp_ma20: return logger.info("%s", f"{sec.display_name}\t{distance:.3f}\t{a:.3f}\t{err:.3f}") await self.fire("long", code, dt, frame_type=frame_type.value, distance=distance)
def test_floor(self): X = [ ("2005-01-09", FrameType.DAY, "2005-01-07"), ("2005-01-07", FrameType.DAY, "2005-01-07"), ("2005-01-08 14:00", FrameType.DAY, "2005-1-7"), ("2005-01-07 16:00:00", FrameType.DAY, "2005-01-07"), ("2005-01-07 14:59:00", FrameType.DAY, "2005-01-06"), ("2005-1-10 15:00:00", FrameType.WEEK, "2005-1-7"), ("2005-1-13 15:00:00", FrameType.WEEK, "2005-1-7"), ("2005-1-14 15:00:00", FrameType.WEEK, "2005-1-14"), ("2005-2-1 15:00:00", FrameType.MONTH, "2005-1-31"), ("2005-2-27 15:00:00", FrameType.MONTH, "2005-1-31"), ("2005-2-28 15:00:00", FrameType.MONTH, "2005-2-28"), ("2005-3-1 15:00:00", FrameType.MONTH, "2005-2-28"), ("2005-1-5 09:30", FrameType.MIN1, "2005-1-4 15:00"), ("2005-1-5 09:31", FrameType.MIN1, "2005-1-5 09:31"), ("2005-1-5 09:34", FrameType.MIN5, "2005-1-4 15:00"), ("2005-1-5 09:36", FrameType.MIN5, "2005-1-5 09:35"), ("2005-1-5 09:46", FrameType.MIN15, "2005-1-5 09:45"), ("2005-1-5 10:01", FrameType.MIN30, "2005-1-5 10:00"), ("2005-1-5 10:31", FrameType.MIN60, "2005-1-5 10:30"), # 如果moment为非交易日,则floor到上一交易日收盘 ("2020-11-21 09:32", FrameType.MIN1, "2020-11-20 15:00"), # 如果moment刚好是frame结束时间,则floor(frame) == frame ("2005-1-5 10:00", FrameType.MIN30, "2005-1-5 10:00"), ] for i, (moment, frame_type, expected) in enumerate(X): logger.debug("testing %s", X[i]) frame = arrow.get(moment).datetime if frame_type in tf.day_level_frames and frame.hour == 0: frame = frame.date() actual = tf.floor(frame, frame_type) expected = arrow.get(expected) if frame_type in tf.day_level_frames: expected = arrow.get(expected).date() else: expected = arrow.get(expected).datetime self.assertEqual(expected, actual)
async def get_bars( cls, sec: str, end: Frame, n_bars: int, frame_type: FrameType, include_unclosed=True, ) -> np.ndarray: """获取行情数据,并将已结束的周期数据存入缓存。 各种情况: 1. 假设现在时间是2021-2-24日,盘中。此时请求上证指数日线,且`include_unclosed`为 `True`: ```python get_bars("000001.XSHE", None, 1, FrameType.DAY) ``` 得到的数据可能如下: ``` [(datetime.date(2021, 2, 24), 3638.9358, 3645.5288, 3617.44, 3620.3542, ...)] ``` 在收盘前不同时间调用,得到的数据除开盘价外,其它都实时在变动。 2. 假设现在时间是2021-2-23日,盘后,此时请求上证指数日线,将得到收盘后固定的价格。 3. 上述请求中,`include_unclosed`参数使用默认值(`True`)。如果取为`False`,仍以示例1 指定的场景为例,则: ```python get_bars("000001.XSHG", None, 1, FrameType.DAY, False) ``` 因为2021-2-24日未收盘,所以获取的最后一条数据是2021-2-23日的。 4. 同样假设现在时间是2021-2-24日盘中,周三。此时获取周K线。在`include_unclosed`分别为 `True`和`False`的情况下: ``` [(datetime.date(2021, 2, 24), 3707.19, 3717.27, 3591.3647, 3592.3977, ...)] [(datetime.date(2021, 2, 19), 3721.09, 3731.69, 3634.01, 3696.17, ...)] ``` 注意这里当`include_unclosed`为True时,返回的周K线是以2021-2-24为Frame的。同样,在盘中 的不同时间取这个数据,除了`open`数值之外,其它都是实时变化的。 5. 如果在已结束的周期中,包含停牌数据,则会对停牌期间的数据进行nan填充,以方便数据使用 者可以较容易地分辨出数据不连贯的原因:哪些是停牌造成的,哪些是非交易日造成的。这种处理 会略微降低数据获取速度,并增加存储空间。 比如下面的请求: ```python get_bars("000029.XSHE", datetime.date(2020,8,18), 10, FrameType.DAY) ``` 将获取到2020-8-5到2020-8-18间共10条数据。但由于期间000029这支股票处于停牌期,所以返回 的10条数据中,数值部分全部填充为np.nan。 注意如果取周线和月线数据,如果当天停牌,但只要周线有数据,则仍能取到。周线(或者月线)的 `frame`将是停牌前一交易日。比如, ```python sec = "600721.XSHG" frame_type = FrameType.WEEK end = arrow.get("2020-4-29 15:00").datetime bars = await aq.get_bars(sec, end, 3, FrameType.WEEK) print(bars) ``` 2020年4月30日是该周的最后一个交易日。股票600721在4月29日停牌一天。上述请求将得到如下数 据: ``` [(datetime.date(2020, 4, 17), 6.02, 6.69, 5.84, 6.58, ...) (datetime.date(2020, 4, 24), 6.51, 6.57, 5.68, 5.72, ...) (datetime.date(2020, 4, 28), 5.7, 5.71, 5.17, 5.36, ...)] ``` 停牌发生在日线级别上,但我们的请求发生在周线级别上,所以不会对4/29日进行填充,而是返回 截止到4月29日的数据。 args: sec: 证券代码 end: 数据截止日 n_bars: 待获取的数据条数 frame_type: 数据所属的周期 include_unclosed: 如果为真,则会包含当end所处的那个Frame的数据,即使当前它还未结束 """ now = arrow.now(tz=cfg.tz) end = end or now.datetime # 如果end超出当前时间,则认为是不合法的。如果用户想取到最新的数据,应该传入None if type(end) == datetime.date: if end > now.date(): return None elif type(end) == datetime.datetime: if end > now: return None bars = await cls.get_instance().get_bars(sec, end, n_bars, frame_type.value, include_unclosed) if len(bars) == 0: return # 根据指定的end,计算结束时的frame last_closed_frame = tf.floor(end, frame_type) last_frame = bars[-1]["frame"] # 计算有多少根k线是已结束的 n_closed = n_bars - 1 if frame_type == FrameType.DAY: # 盘后取日线,返回的一定是全部都已closed的数据 # 盘中取日线,返回的last_frame会是当天的日期,但该日线并未结束 if now.datetime.hour >= 15 or last_frame < now.date(): n_closed = n_bars else: # 如果last_frame <= end的上限,则返回的也一定是全部都closed的数据 if last_frame <= tf.floor(end, frame_type): n_closed = n_bars remainder = [bars[-1]] if n_closed < n_bars else None closed_bars = cls._fill_na(bars, n_closed, last_closed_frame, frame_type) # 只保存已结束的bar await cache.save_bars(sec, closed_bars, frame_type) if remainder is None: return closed_bars else: return np.concatenate([closed_bars, remainder])
async def quick_scan(): # fixme secs = Securities() report = logging.getLogger("quickscan") counters = {} for sync_config in cfg.omega.sync.bars: frame = sync_config.get("frame") start = sync_config.get("start") if frame is None or start is None: logger.warning("skipped %s: required fields are [frame, start]", sync_config) continue frame_type = FrameType(frame) start = arrow.get(start).date() start = tf.floor(start, FrameType.DAY) stop = sync_config.get("stop") or arrow.now().date() if frame_type in tf.minute_level_frames: minutes = tf.ticks[frame_type][0] h, m = minutes // 60, minutes % 60 start = datetime.datetime(start.year, start.month, start.day, h, m, tzinfo=tz.gettz(cfg.tz)) stop = datetime.datetime(stop.year, stop.month, stop.day, 15, tzinfo=tz.gettz(cfg.tz)) counters[frame] = [0, 0] codes = secs.choose(sync_config.get("type")) include = filter(lambda x: x, sync_config.get("include", "").split(",")) include = map(lambda x: x.strip(" "), include) codes.extend(include) exclude = sync_config.get("exclude", "") exclude = map(lambda x: x.strip(" "), exclude) codes = set(codes) - set(exclude) counters[frame][1] = len(codes) for code in codes: head, tail = await cache.get_bars_range(code, frame_type) if head is None or tail is None: report.info("ENOSYNC,%s,%s", code, frame) counters[frame][0] = counters[frame][0] + 1 continue expected = tf.count_frames(head, tail, frame_type) # 'head', 'tail' should be excluded actual = (await cache.security.hlen(f"{code}:{frame_type.value}")) - 2 if actual != expected: report.info("ELEN,%s,%s,%s,%s,%s,%s", code, frame, expected, actual, head, tail) counters[frame][0] = counters[frame][0] + 1 continue sec = Security(code) if start != head: if (type(start) == datetime.date and start > sec.ipo_date or (type(start) == datetime.datetime and start.date() > sec.ipo_date)): report.info("ESTART,%s,%s,%s,%s,%s", code, frame, start, head, sec.ipo_date) counters[frame][0] = counters[frame][0] + 1 continue if tail != stop: report.info("EEND,%s,%s,%s,%s", code, frame, stop, tail) counters[frame][0] = counters[frame][0] + 1 return counters
async def start_validation(): """ 将待校验的证券按CPU个数均匀划分,创建与CPU个数相同的子进程来执行校验。校验的起始时间由数据 库中jobs.bars_validation.range.start和jobs.bars_validation.range.stop来决定,每次校验 结束后,将jobs.bars_validation.range.start更新为校验截止的最后交易日。如果各个子进程报告 的截止交易日不一样(比如发生了异常),则使用最小的交易日。 """ global validation_errors, no_validation_error_days validation_errors = [] secs = Securities() cpu_count = psutil.cpu_count() # to check if the range is right pl = cache.sys.pipeline() pl.get("jobs.bars_validation.range.start") pl.get("jobs.bars_validation.range.end") start, end = await pl.execute() if start is None: if cfg.omega.validation.start is None: logger.warning( "start of validation is not specified, validation aborted.") return else: start = tf.date2int(arrow.get(cfg.omega.validation.start)) else: start = int(start) if end is None: end = tf.date2int(tf.floor(arrow.now().date(), FrameType.DAY)) else: end = int(end) assert start <= end no_validation_error_days = set(tf.day_frames[(tf.day_frames >= start) & (tf.day_frames <= end)]) # fixme: do validation per frame_type # fixme: test fail. Rewrite this before 0.6 releases codes = secs.choose(cfg.omega.sync) await cache.sys.delete("jobs.bars_validation.scope") await cache.sys.lpush("jobs.bars_validation.scope", *codes) logger.info("start validation %s secs from %s to %s.", len(codes), start, end) emit.register(Events.OMEGA_VALIDATION_ERROR, on_validation_error) t0 = time.time() code = ("from omega.core.sanity import do_validation_process_entry; " "do_validation_process_entry()") procs = [] for i in range(cpu_count): proc = subprocess.Popen([sys.executable, "-c", code], env=os.environ) procs.append(proc) timeout = 3600 while timeout > 0: await asyncio.sleep(2) timeout -= 2 for proc in procs: proc.poll() if all([proc.returncode is not None for proc in procs]): break if timeout <= 0: for proc in procs: try: os.kill(proc.pid, signal.SIGTERM) except Exception: pass # set next start point validation_days = set(tf.day_frames[(tf.day_frames >= start) & (tf.day_frames <= end)]) diff = validation_days - no_validation_error_days if len(diff): last_no_error_day = min(diff) else: last_no_error_day = end await cache.sys.set("jobs.bars_validation.range.start", last_no_error_day) elapsed = time.time() - t0 logger.info( "Validation cost %s seconds, validation will start at %s next time", elapsed, last_no_error_day, )
async def evaluate(self, code: str, end: Frame): """ 最近穿越年线的股票,回归到[5,10,20]日均线时,如果均线形态良好,则提示买入 Args: code: end: Returns: """ start = tf.shift(tf.floor(end, FrameType.DAY), -26, FrameType.DAY) sec = Security(code) bars = await sec.load_bars(start, end, FrameType.DAY) close = bars['close'] c0 = close[-1] # 检查接近5日均线,要求5日内强于均线,均线不能向下(或者拐头趋势) ma = signal.moving_average(close, 5) err, (a, b, c), (vx, _) = signal.polyfit(ma[-7:] / ma[-7]) t1 = np.all(close[-5:] > ma[-5:]) t2 = err < 3e-3 t3 = a > 5e-4 or (abs(a) < 1e-5 and b > 1e-3) t4 = (c0 - ma[-1] / c0 < 5e-3) t5 = vx < 6 logger.debug("%s 5日:%s, (a,b):%s,%s", sec, [t1, t2, t3, t4, t5], a, b) if all([t1, t2, t3, t4, t5]): logger.info("fired 5日买入:%s", sec) await emit.emit("/alpha/signals/long", { "plot": "crossyear", "code": code, "frame": str(end), "desc": "回探5日线", "coef": np.round([a, b], 4), "vx": vx, "c": c0, "ma": ma[-5:] }) return # 检查接近20日线买点 ma = signal.moving_average(close, 20) err, (a, b, c), (vx, _) = signal.polyfit(ma[-10:] / ma[-10]) t1 = err < 3e-3 t2 = a > 5e-4 or (abs(a) < 1e-5 and b > 5e-3) t3 = (c0 - ma[-1]) < 5e-3 t4 = vx < 9 logger.debug("%s 20日:%s, (a,b):%s,%s", sec, [t1, t2, t3, t4], a, b) if all([t1, t2, t3, t4]): logger.info("fired 20日买入:%s", sec) await emit.emit("/alpha/signals/long", { "plot": "crossyear", "code": code, "frame": str(end), "desc": "回探20日线", "coef": np.round([a, b], 4), "vx": vx, "c": c0, "ma": ma[-5:] }) return # 检查是否存在30分钟买点 start = tf.shift(tf.floor(end, FrameType.MIN30), -30, FrameType.MIN30) bars = await sec.load_bars(start, end, FrameType.MIN30) close = bars['close'] ma = signal.moving_average(close, 5) err, (a, b, c), (vx, _) = signal.polyfit(ma[-7:] / ma[-7]) t1 = err < 3e-3 t2 = a > 5e-4 or (abs(a) < 1e-5 and b > 1e-2) t3 = vx < 6 logger.debug("%s 30分钟:%s, (a,b)", sec, [t1, t2, t3, t4, t5], a, b) if all([t1, t2, t3]): logger.info("fired 30分钟买入:%s", sec) await emit.emit("/alpha/signals/long", { "plot": "crossyear", "code": code, "frame": str(end), "desc": "30分钟买点", "vx": vx, "c": c0, "ma": ma[-5:] })
async def scan(self, end: Frame = None, frame_type: FrameType = FrameType.DAY, codes=None, adv_limit=0.3): """ Args: end: adv_limit: 不包括在win周期内涨幅超过adv_limit的个股 Returns: """ win = 20 secs = Securities() end = end or tf.floor(arrow.now(), FrameType.DAY) results = [] holdings = await cache.sys.smembers("holdings") for i, code in enumerate(secs.choose(['stock'])): try: if code in holdings: # 如果已经持仓,则不跟踪评估 continue sec = Security(code) if sec.code.startswith('688') or sec.display_name.find('ST') != -1: continue start = tf.day_shift(end, -270) bars = await sec.load_bars(start, end, FrameType.DAY) close = bars['close'] ma5 = signal.moving_average(close, 5) ma250 = signal.moving_average(close, 250) cross, idx = signal.cross(ma5[-win:], ma250[-win:]) cross_day = bars[-win + idx]['frame'] if cross != 1: continue ma20 = signal.moving_average(close, 20) ma120 = signal.moving_average(close, 120) # 如果上方还有月线和ma120线,则不发出信号,比如广州浪奇 2020-7-23,泛海控股2020-8-3 if close[-1] < ma120[-1] or close[-1] < ma20[-1]: continue # 计算20日以来大阳次数。如果不存在大阳线,认为还未到上涨时机,跳过 grl, ggl = features.count_long_body(bars[-20:]) if grl == 0: continue # # # 计算突破以来净余买量(用阳线量减去阴线量来模拟,十字星不计入) # bsc = bars[-10 + idx:] # bars_since_open: included both side # ups = bsc[bsc['close'] > (bsc['open'] * 1.01)] # downs = bsc[bsc['open'] > (bsc['close'] * 0.99)] # balance = np.sum(ups['volume']) - np.sum(downs['volume']) # pc = await sec.price_change(cross_day, tf.day_shift(cross_day, 5), # FrameType.DAY, return_max=True) # faf = int(win - idx) # frames after fired adv = await sec.price_change(tf.day_shift(end, -win), end, FrameType.DAY, False) if adv > adv_limit: continue logger.info(f"{sec}上穿年线\t{cross_day}\t{faf}") await cache.sys.hmset_dict("plots.crossyear", {code: json.dumps({ "fired_at": tf.date2int(end), "cross_day": tf.date2int(cross_day), "faf": faf, "grl": grl, "ggl": ggl, "status": 0 # 0 - generated by plots 1 - disabled manually })}) results.append( [sec.display_name, tf.date2int(end), tf.date2int(cross_day), faf, grl, ggl]) except Exception as e: logger.exception(e) logger.info("done crossyear scan.") return results
def parse_sync_params( frame: Union[str, Frame], cat: List[str] = None, start: Union[str, datetime.date] = None, stop: Union[str, Frame] = None, delay: int = 0, include: str = "", exclude: str = "", ) -> Tuple: """按照[使用手册](usage.md#22-如何同步K线数据)中的规则,解析和补全同步参数。 如果`frame_type`为分钟级,则当`start`指定为`date`类型时,自动更正为对应交易日的起始帧; 当`stop`为`date`类型时,自动更正为对应交易日的最后一帧。 Args: frame (Union[str, Frame]): frame type to be sync. The word ``frame`` is used here for easy understand by end user. It actually implies "FrameType". cat (List[str]): which catetories is about to be synced. Should be one of ['stock', 'index']. Defaults to None. start (Union[str, datetime.date], optional): [description]. Defaults to None. stop (Union[str, Frame], optional): [description]. Defaults to None. delay (int, optional): [description]. Defaults to 5. include (str, optional): which securities should be included, seperated by space, for example, "000001.XSHE 000004.XSHE". Defaults to empty string. exclude (str, optional): which securities should be excluded, seperated by a space. Defaults to empty string. Returns: - codes (List[str]): 待同步证券列表 - frame_type (FrameType): - start (Frame): - stop (Frame): - delay (int): """ frame_type = FrameType(frame) if frame_type in tf.minute_level_frames: if stop: stop = arrow.get(stop, tzinfo=cfg.tz) if stop.hour == 0: # 未指定有效的时间帧,使用当日结束帧 stop = tf.last_min_frame(tf.day_shift(stop.date(), 0), frame_type) else: stop = tf.floor(stop, frame_type) else: stop = tf.floor(arrow.now(tz=cfg.tz).datetime, frame_type) if stop > arrow.now(tz=cfg.tz): raise ValueError(f"请勿将同步截止时间设置在未来: {stop}") if start: start = arrow.get(start, tzinfo=cfg.tz) if start.hour == 0: # 未指定有效的交易帧,使用当日的起始帧 start = tf.first_min_frame(tf.day_shift(start.date(), 0), frame_type) else: start = tf.floor(start, frame_type) else: start = tf.shift(stop, -999, frame_type) else: stop = (stop and arrow.get(stop).date()) or arrow.now().date() if stop == arrow.now().date(): stop = arrow.now(tz=cfg.tz) stop = tf.floor(stop, frame_type) start = tf.floor( (start and arrow.get(start).date()), frame_type) or tf.shift( stop, -1000, frame_type) secs = Securities() codes = secs.choose(cat or []) exclude = map(lambda x: x, exclude.split(" ")) codes = list(set(codes) - set(exclude)) include = list(filter(lambda x: x, include.split(" "))) codes.extend(include) return codes, frame_type, start, stop, int(delay)
async def load_bars_batch(cls, codes: List[str], end: Frame, n: int, frame_type: FrameType) -> AsyncIterator: """为一批证券品种加载行情数据 examples: ``` codes = ["000001.XSHE", "000001.XSHG"] end = arrow.get("2020-08-27").datetime async for code, bars in Security.load_bars_batch(codes, end, 5, FrameType.DAY): print(code, bars[-2:]) self.assertEqual(5, len(bars)) self.assertEqual(bars[-1]["frame"], end.date()) if code == "000001.XSHG": self.assertAlmostEqual(3350.11, bars[-1]["close"], places=2) ``` Args: codes : 证券列表 end : 结束帧 n : 周期数 frame_type : 帧类型 Returns: [description] Yields: [description] """ assert type(end) in (datetime.date, datetime.datetime) closed_frame = tf.floor(end, frame_type) if end == closed_frame: start = tf.shift(closed_frame, -n + 1, frame_type) cached = [ asyncio.create_task( cls._get_bars(code, start, closed_frame, frame_type)) for code in codes ] for fut in asyncio.as_completed(cached): rec = await fut yield rec else: start = tf.shift(closed_frame, -n + 2, frame_type) cached = [ asyncio.create_task( cls._get_bars(code, start, closed_frame, frame_type)) for code in codes ] recs1 = await asyncio.gather(*cached) recs2 = await cls._load_bars_batch(codes, end, 1, frame_type) for code, bars in recs1: _bars = recs2.get(code) if _bars is None or len(_bars) != 1: logger.warning("wrong/emtpy records for %s", code) continue yield code, np.append(bars, _bars)
async def load_bars( self, start: Frame, stop: datetime.datetime, frame_type: FrameType, fq=True, turnover=False, ) -> np.ndarray: """ 加载[`start`, `stop`]间的行情数据到`Security`对象中,并返回行情数据。 这里`start`可以等于`stop`。 为加快速度,对分钟级别的turnover数据,均使用当前周期的成交量除以最新报告期的流通股本数, 注意这样得到的是一个近似值。如果近期有解禁股,则同样的成交量,解禁后的换手率应该小于解 禁前。 Args: start: stop: frame_type: fq: 是否进行复权处理 turnover: 是否包含turnover数据。 Returns: """ self._bars = None start = tf.floor(start, frame_type) _stop = tf.floor(stop, frame_type) assert start <= _stop head, tail = await cache.get_bars_range(self.code, frame_type) if not all([head, tail]): # not cached at all, ensure cache pointers are clear await cache.clear_bars_range(self.code, frame_type) n = tf.count_frames(start, _stop, frame_type) if stop > _stop: self._bars = await get_bars(self.code, stop, n + 1, frame_type) else: self._bars = await get_bars(self.code, _stop, n, frame_type) if fq: self.qfq() if turnover: await self._add_turnover(frame_type) return self._bars if start < head: n = tf.count_frames(start, head, frame_type) if n > 0: _end = tf.shift(head, -1, frame_type) self._bars = await get_bars(self.code, _end, n, frame_type) if _stop > tail: n = tf.count_frames(tail, _stop, frame_type) if n > 0: await get_bars(self.code, _stop, n, frame_type) # now all closed bars in [start, _stop] should exist in cache n = tf.count_frames(start, _stop, frame_type) self._bars = await cache.get_bars(self.code, _stop, n, frame_type) if arrow.get(stop) > arrow.get(_stop): bars = await get_bars(self.code, stop, 2, frame_type) if len(bars) == 2 and bars[0]["frame"] == self._bars[-1]["frame"]: self._bars = np.append(self._bars, bars[1]) if fq: self.qfq() if turnover: await self._add_turnover(frame_type) return self._bars
async def sync_bars(params: dict): """sync bars on signal OMEGA_DO_SYNC received Args: params (dict): composed of the following: ``` { secs (List[str]): 待同步的证券标的.如果为None或者为空,则从数据库中轮询 frame_type (FrameType):k线的帧类型 start (Frame): k线起始时间 stop (Frame): k线结束时间 } ``` Returns: [type]: [description] """ secs, frame_type, start, stop = ( params.get("secs"), params.get("frame_type"), params.get("start"), params.get("stop"), ) if secs is not None: logger.info( "sync bars with %s(%s ~ %s) for given %s secs", frame_type, start, stop, len(secs), ) async def get_sec(): return secs.pop() if len(secs) else None else: logger.info("sync bars with %s(%s ~ %s) in polling mode", frame_type, start, stop) async def get_sec(): return await cache.sys.lpop(key_scope) key_scope = f"jobs.bars_sync.scope.{frame_type.value}" if start is None or frame_type is None: raise ValueError("you must specify a start date/frame_type for sync") if stop is None: stop = tf.floor(arrow.now(tz=cfg.tz), frame_type) while code := await get_sec(): try: await sync_bars_for_security(code, frame_type, start, stop) except FetcherQuotaError as e: logger.warning("Quota exceeded when syncing %s. Sync aborted.", code) logger.exception(e) return # stop the sync except Exception as e: logger.warning("Failed to sync %s", code) logger.exception(e)
async def scan(self, frame_type: Union[str, FrameType] = FrameType.DAY, end: Frame = None, codes: List[str] = None): logger.info("running momentum scan at %s level", frame_type) if end is None: end = arrow.now(cfg.tz).datetime assert type(end) in (datetime.date, datetime.datetime) frame_type = FrameType(frame_type) ft = frame_type.value codes = codes or Securities().choose(['stock']) day_bars = {} async for code, bars in Security.load_bars_batch( codes, end, 2, FrameType.DAY): day_bars[code] = bars if len(day_bars) == 0: return async for code, bars in Security.load_bars_batch( codes, end, 11, frame_type): if len(bars) < 11: continue fired = bars[-1]['frame'] day_bar = day_bars.get(code) if day_bar is None: continue c1, c0 = day_bars.get(code)[-2:]['close'] cmin = min(bars['close']) # 还处在下跌状态、或者涨太多 if c0 == cmin or (c0 / c1 - 1) > self.baseline(f"up_limit"): continue ma5 = signal.moving_average(bars['close'], 5) err, (a, b, c), (vx, _) = signal.polyfit(ma5[-7:] / ma5[-7]) # 无法拟合,或者动能不足 if err > self.baseline(f"ma5:{ft}:err") or a < self.baseline( f"ma5:{ft}:a"): continue # 时间周期上应该是信号刚出现,还在窗口期内 vx_range = self.baseline(f"ma5:{ft}:vx") if not vx_range[0] < vx < vx_range[1]: continue p = np.poly1d((a, b, c)) y = p(9) / p(6) - 1 # 如果预测未来三周期ma5上涨幅度不够 if y < self.baseline(f"ma5:{ft}:y"): continue sec = Security(code) if frame_type == FrameType.DAY: start = tf.shift(tf.floor(end, frame_type), -249, frame_type) bars250 = await sec.load_bars(start, end, frame_type) ma60 = signal.moving_average(bars250['close'], 60) ma120 = signal.moving_average(bars250['close'], 120) ma250 = signal.moving_average(bars250['close'], 250) # 上方无均线压制 if (c0 > ma60[-1]) and (c0 > ma120[-1]) and (c0 > ma250[-1]): logger.info("%s, %s, %s, %s, %s, %s", sec, round(a, 4), round(b, 4), round(vx, 1), round(c0 / c1 - 1, 3), round(y, 3)) await self.enter_stock_pool(code, fired, frame_type, a=a, b=b, err=err, y=y, vx=self.fit_win - vx) elif frame_type == FrameType.WEEK: await self.enter_stock_pool(code, fired, frame_type, a=a, b=b, err=err, y=y, vx=self.fit_win - vx) elif frame_type == FrameType.MIN30: await self.fire_trade_signal('long', code, fired, frame_type, a=a, b=b, err=err, y=y, vx=self.fit_win - vx)
async def list_stock_pool(self, frames: int, frame_types: List[FrameType] = None): key = "plots.momentum.pool" recs = await cache.sys.hgetall(key) items = [] now = arrow.now() for k, v in recs.items(): frame, code = k.split(":") sec = Security(code) v = json.loads(v) frame_type = FrameType(v.get("frame_type")) if frame_type not in frame_types: continue latest_frame = tf.floor(now, frame_type) start = tf.shift(latest_frame, -frames, frame_type) fired = tf.int2time(frame) if frame_type in tf.minute_level_frames else \ tf.int2date(frame) if fired < start: continue items.append({ "name": sec.display_name, "code": code, "fired": str(fired), "frame": frame_type.value, "y": round(v.get("y"), 2), "vx": round(v.get("vx"), 1), "a": round(v.get("a"), 4), "b": round(v.get("b"), 4), "err": round(v.get("err"), 4) }) return { "name": self.display_name, "plot": self.name, "items": items, "headers": [{ "text": '名称', "value": 'name' }, { "text": '代码', "value": 'code' }, { "text": '信号时间', "value": 'fired' }, { "text": '预测涨幅', "value": 'y' }, { "text": '动能', "value": 'a' }, { "text": '势能', "value": 'b' }, { "text": '周期', "value": 'frame' }, { "text": '底部距离(周期)', "value": 'vx' }, { "text": '拟合误差', "value": 'err' }] }