def do_validation_process_entry(): try: t0 = time.time() asyncio.run(do_validation()) logger.info("validation finished in %s seconds", time.time() - t0) return 0 except Exception as e: logger.warning("validation exit due to exception:") logger.exception(e) return -1
async def calc_checksums(day: datetime.date, codes: List) -> dict: """ Args: day: codes: Returns: 返回值为以code为键,该证券对应的{周期:checksum}的集合为值的集合 """ end_time = arrow.get(day, tzinfo=cfg.tz).replace(hour=15) checksums = {} for i, code in enumerate(codes): try: checksum = {} d = await cache.get_bars_raw_data(code, day, 1, FrameType.DAY) if d: checksum[f"{FrameType.DAY.value}"] = xxhash.xxh32_hexdigest(d) d = await cache.get_bars_raw_data(code, end_time, 240, FrameType.MIN1) if d: checksum[f"{FrameType.MIN1.value}"] = xxhash.xxh32_hexdigest(d) d = await cache.get_bars_raw_data(code, end_time, 48, FrameType.MIN5) if d: checksum[f"{FrameType.MIN5.value}"] = xxhash.xxh32_hexdigest(d) d = await cache.get_bars_raw_data(code, end_time, 16, FrameType.MIN15) if d: checksum[f"{FrameType.MIN15.value}"] = xxhash.xxh32_hexdigest( d) d = await cache.get_bars_raw_data(code, end_time, 8, FrameType.MIN30) if d: checksum[f"{FrameType.MIN30.value}"] = xxhash.xxh32_hexdigest( d) d = await cache.get_bars_raw_data(code, end_time, 4, FrameType.MIN60) if d: checksum[f"{FrameType.MIN60.value}"] = xxhash.xxh32_hexdigest( d) checksums[code] = checksum except Exception as e: logger.exception(e) if (i + 1) % 500 == 0: logger.info("calc checksum progress: %s/%s", i + 1, len(codes)) return checksums
async def do_validation(secs: List[str] = None, start: str = None, end: str = None): """对列表secs中指定的证券行情数据按start到end指定的时间范围进行校验 Args: secs (List[str], optional): [description]. Defaults to None. start (str, optional): [description]. Defaults to None. end (str, optional): [description]. Defaults to None. Returns: [type]: [description] """ logger.info("start validation...") report = logging.getLogger("validation_report") cfg = cfg4py.init(get_config_dir(), False) await emit.start(engine=emit.Engine.REDIS, dsn=cfg.redis.dsn, start_server=True) await omicron.init() start = int(start or await cache.sys.get("jobs.bars_validation.range.start")) if end is None: end = tf.date2int(arrow.now().date()) else: end = int(end or await cache.sys.get("jobs.bars_validation.range.stop")) if secs is None: async def get_sec(): return await cache.sys.lpop("jobs.bars_validation.scope") else: async def get_sec(): return secs.pop() if len(secs) else None errors = 0 while code := await get_sec(): try: for day in tf.day_frames[(tf.day_frames >= start) & (tf.day_frames <= end)]: expected = await get_checksum(day) if expected and expected.get(code): actual = await calc_checksums(tf.int2date(day), [code]) d1 = actual.get(code) d2 = expected.get(code) missing1 = d2.keys() - d1 # local has no checksum missing2 = d1.keys() - d2 # remote has no checksum mismatch = {k for k in d1.keys() & d2 if d1[k] != d2[k]} for k in missing1: info = ( ValidationError.LOCAL_MISS, day, code, k, d1.get(k), d2.get(k), ) report.info("%s,%s,%s,%s,%s,%s", *info) await emit.emit(Events.OMEGA_VALIDATION_ERROR, info) for k in missing2: info = ( ValidationError.REMOTE_MISS, day, code, k, d1.get(k), d2.get(k), ) report.info("%s,%s,%s,%s,%s,%s", *info) await emit.emit(Events.OMEGA_VALIDATION_ERROR, info) for k in mismatch: info = ( ValidationError.MISMATCH, day, code, k, d1.get(k), d2.get(k), ) report.info("%s,%s,%s,%s,%s,%s", *info) await emit.emit(Events.OMEGA_VALIDATION_ERROR, info) else: logger.error("checksum for %s not found.", day) info = (ValidationError.NO_CHECKSUM, day, None, None, None, None) report.info("%s,%s,%s,%s,%s,%s", *info) await emit.emit(Events.OMEGA_VALIDATION_ERROR, info) except Exception as e: logger.exception(e) errors += 1