def inform(action: EmailInformer, entity_ids, target_date, title, entity_provider, entity_type, em_group, em_group_over_write): msg = "no targets" if entity_ids: entities = get_entities(provider=entity_provider, entity_type=entity_type, entity_ids=entity_ids, return_type="domain") if em_group: try: codes = [entity.code for entity in entities] add_to_eastmoney(codes=codes, entity_type=entity_type, group=em_group, over_write=em_group_over_write) except Exception as e: action.send_message( zvt_config["email_username"], f"{target_date} {title} error", f"{target_date} {title} error: {e}", ) infos = [f"{entity.name}({entity.code})" for entity in entities] msg = "\n".join(infos) + "\n" logger.info(msg) action.send_message(zvt_config["email_username"], f"{target_date} {title}", msg)
def record_stock_data(data_provider="em", entity_provider="em", sleeping_time=2): # A股标的 run_data_recorder(domain=Stock, data_provider=data_provider, force_update=False) # A股后复权行情 run_data_recorder( domain=Stock1dHfqKdata, data_provider=data_provider, entity_provider=entity_provider, day_data=True, sleeping_time=sleeping_time, ) # 板块(概念,行业) run_data_recorder(domain=Block, data_provider="eastmoney", force_update=False) # 板块行情(概念,行业) run_data_recorder(domain=Block1dKdata, data_provider="em", day_data=True, sleeping_time=sleeping_time) # 报告新概念和行业 email_action = EmailInformer() list_date = next_date(current_date(), -90) df = Block.query_data( filters=[Block.category == BlockCategory.concept.value, Block.list_date >= list_date], index="entity_id" ) # add them to eastmoney try: add_to_eastmoney(codes=df["code"], entity_type="block", group="新概念", over_write=False) except Exception as e: email_action.send_message( zvt_config["email_username"], f"report_concept error", "report_concept error:{}".format(e) )
def run_data_recorder( domain: Type[Mixin], entity_provider=None, data_provider=None, entity_ids=None, retry_times=10, sleeping_time=10, **recorder_kv, ): logger.info( f" record data: {domain.__name__}, entity_provider: {entity_provider}, data_provider: {data_provider}" ) while retry_times > 0: email_action = EmailInformer() try: domain.record_data(entity_ids=entity_ids, provider=data_provider, sleeping_time=sleeping_time, **recorder_kv) msg = f"record {domain.__name__} success" logger.info(msg) email_action.send_message(zvt_config["email_username"], msg, msg) break except Exception as e: logger.exception("report error:{}".format(e)) time.sleep(60 * 3) retry_times = retry_times - 1 if retry_times == 0: email_action.send_message( zvt_config["email_username"], f"record {domain.__name__} error", f"record {domain.__name__} error: {e}", )
def report_targets( factor_cls: Type[Factor], entity_provider, data_provider, title, entity_type="stock", informer: EmailInformer = None, em_group=None, em_group_over_write=True, filter_by_volume=True, adjust_type=None, start_timestamp="2019-01-01", **factor_kv, ): logger.info( f"entity_provider: {entity_provider}, data_provider: {data_provider}, entity_type: {entity_type}, start_timestamp: {start_timestamp}" ) error_count = 0 while error_count <= 10: try: if not adjust_type: adjust_type = default_adjust_type(entity_type=entity_type) target_date = get_latest_kdata_date(provider=data_provider, entity_type=entity_type, adjust_type=adjust_type) logger.info(f"target_date :{target_date}") current_entity_pool = None if filter_by_volume: # 成交量 vol_df = get_top_volume_entities( entity_type=entity_type, start_timestamp=next_date(target_date, -30), end_timestamp=target_date, adjust_type=adjust_type, pct=0.4, data_provider=data_provider, ) current_entity_pool = vol_df.index.tolist() logger.info( f"current_entity_pool({len(current_entity_pool)}): {current_entity_pool}" ) kdata_schema = get_kdata_schema(entity_type, level=IntervalLevel.LEVEL_1DAY, adjust_type=adjust_type) filters = [] if "turnover_threshold" in factor_kv: filters = filters + [ kdata_schema.turnover >= factor_kv.get("turnover_threshold") ] if "turnover_rate_threshold" in factor_kv: filters = filters + [ kdata_schema.turnover_rate >= factor_kv.get("turnover_rate_threshold") ] if filters: filters = filters + [kdata_schema.timestamp == target_date] kdata_df = kdata_schema.query_data( provider=data_provider, filters=filters, columns=["entity_id", "timestamp"], index="entity_id") if current_entity_pool: current_entity_pool = set(current_entity_pool) & set( kdata_df.index.tolist()) else: current_entity_pool = kdata_df.index.tolist() if "entity_ids" in factor_kv: if current_entity_pool: current_entity_pool = set(current_entity_pool) & set( factor_kv.pop("entity_ids")) else: current_entity_pool = set(factor_kv.pop("entity_ids")) # add the factor my_selector = TargetSelector(start_timestamp=start_timestamp, end_timestamp=target_date, select_mode=SelectMode.condition_or) entity_schema = get_entity_schema(entity_type=entity_type) tech_factor = factor_cls( entity_schema=entity_schema, entity_provider=entity_provider, provider=data_provider, entity_ids=current_entity_pool, start_timestamp=start_timestamp, end_timestamp=target_date, adjust_type=adjust_type, **factor_kv, ) my_selector.add_factor(tech_factor) my_selector.run() long_stocks = my_selector.get_open_long_targets( timestamp=target_date) inform( informer, entity_ids=long_stocks, target_date=target_date, title=title, entity_provider=entity_provider, entity_type=entity_type, em_group=em_group, em_group_over_write=em_group_over_write, ) break except Exception as e: logger.exception("report error:{}".format(e)) time.sleep(60 * 3) error_count = error_count + 1 if error_count == 10: informer.send_message( zvt_config["email_username"], f"report {entity_type}{factor_cls.__name__} error", f"report {entity_type}{factor_cls.__name__} error: {e}", )
def report_top_entities( entity_provider, data_provider, periods=None, ignore_new_stock=True, ignore_st=True, entity_ids=None, entity_type="stock", adjust_type=None, top_count=30, turnover_threshold=100000000, turnover_rate_threshold=0.02, informer: EmailInformer = None, em_group=None, em_group_over_write=True, return_type=TopType.positive, ): error_count = 0 while error_count <= 10: try: if periods is None: periods = [7, 30, 365] if not adjust_type: adjust_type = default_adjust_type(entity_type=entity_type) kdata_schema = get_kdata_schema(entity_type=entity_type, adjust_type=adjust_type) entity_schema = get_entity_schema(entity_type=entity_type) target_date = get_latest_kdata_date(provider=data_provider, entity_type=entity_type, adjust_type=adjust_type) filter_entity_ids = get_entity_ids_by_filter( provider=entity_provider, ignore_st=ignore_st, ignore_new_stock=ignore_new_stock, entity_schema=entity_schema, target_date=target_date, entity_ids=entity_ids, ) if not filter_entity_ids: msg = f"{entity_type} no entity_ids selected" logger.error(msg) informer.send_message(zvt_config["email_username"], "report_top_stats error", msg) return filter_turnover_df = kdata_schema.query_data( filters=[ kdata_schema.turnover >= turnover_threshold, kdata_schema.turnover_rate >= turnover_rate_threshold, ], provider=data_provider, start_timestamp=target_date, index="entity_id", columns=["entity_id", "code"], ) if filter_entity_ids: filter_entity_ids = set(filter_entity_ids) & set( filter_turnover_df.index.tolist()) else: filter_entity_ids = filter_turnover_df.index.tolist() if not filter_entity_ids: msg = f"{entity_type} no entity_ids selected" logger.error(msg) informer.send_message(zvt_config["email_username"], "report_top_stats error", msg) return logger.info( f"{entity_type} filter_entity_ids size: {len(filter_entity_ids)}" ) filters = [kdata_schema.entity_id.in_(filter_entity_ids)] selected = [] for i, period in enumerate(periods): interval = period if target_date.weekday() + 1 < interval: interval = interval + 2 start = next_date(target_date, -interval) positive_df, negative_df = get_top_performance_entities( entity_type=entity_type, start_timestamp=start, kdata_filters=filters, pct=1, show_name=True, entity_provider=entity_provider, data_provider=data_provider, return_type=return_type, ) if return_type == TopType.positive: df = positive_df else: df = negative_df selected = selected + df.index[:top_count].tolist() selected = list(dict.fromkeys(selected)) inform( informer, entity_ids=selected, target_date=target_date, title=f"{entity_type} {em_group}({len(selected)})", entity_provider=entity_provider, entity_type=entity_type, em_group=em_group, em_group_over_write=em_group_over_write, ) break except Exception as e: logger.exception("report error:{}".format(e)) time.sleep(30) error_count = error_count + 1
def report_targets( factor_cls: Type[Factor], entity_provider, data_provider, title, entity_type="stock", em_group=None, em_group_over_write=True, filter_by_volume=True, adjust_type=None, start_timestamp="2019-01-01", **factor_kv, ): logger.info( f"entity_provider: {entity_provider}, data_provider: {data_provider}, entity_type: {entity_type}, start_timestamp: {start_timestamp}" ) error_count = 0 while error_count <= 10: email_action = EmailInformer() try: if entity_type == "stock" and not adjust_type: adjust_type = AdjustType.hfq target_date = get_latest_kdata_date(provider=data_provider, entity_type=entity_type, adjust_type=adjust_type) logger.info(f"target_date :{target_date}") current_entity_pool = None if filter_by_volume: # 成交量 vol_df = get_top_volume_entities( entity_type=entity_type, start_timestamp=next_date(target_date, -30), end_timestamp=target_date, adjust_type=adjust_type, pct=0.4, ) current_entity_pool = vol_df.index.tolist() logger.info( f"current_entity_pool({len(current_entity_pool)}): {current_entity_pool}" ) # add the factor my_selector = TargetSelector(start_timestamp=start_timestamp, end_timestamp=target_date, select_mode=SelectMode.condition_or) entity_schema = get_entity_schema(entity_type=entity_type) tech_factor = factor_cls( entity_schema=entity_schema, entity_provider=entity_provider, provider=data_provider, entity_ids=current_entity_pool, start_timestamp=start_timestamp, end_timestamp=target_date, adjust_type=adjust_type, **factor_kv, ) my_selector.add_factor(tech_factor) my_selector.run() long_stocks = my_selector.get_open_long_targets( timestamp=target_date) msg = "no targets" if long_stocks: entities = get_entities(provider=entity_provider, entity_type=entity_type, entity_ids=long_stocks, return_type="domain") if em_group: try: codes = [entity.code for entity in entities] add_to_eastmoney(codes=codes, entity_type=entity_type, group=em_group, over_write=em_group_over_write) except Exception as e: email_action.send_message( zvt_config["email_username"], f"report {entity_type}{factor_cls.__name__} error", f"report {entity_type}{factor_cls.__name__} error: {e}", ) infos = [ f"{entity.name}({entity.code})" for entity in entities ] msg = "\n".join(infos) + "\n" logger.info(msg) email_action.send_message(zvt_config["email_username"], f"{target_date} {title}", msg) break except Exception as e: logger.exception("report error:{}".format(e)) time.sleep(60 * 3) error_count = error_count + 1 if error_count == 10: email_action.send_message( zvt_config["email_username"], f"report {entity_type}{factor_cls.__name__} error", f"report {entity_type}{factor_cls.__name__} error: {e}", )
def report_top_stats( entity_provider, data_provider, periods=[7, 30, 180, 365], ignore_new_stock=True, entity_type="stock", adjust_type=None, top_count=30, turnover_threshold=100000000, turnover_rate_threshold=0.02, em_group_over_write=True, ): if not adjust_type: adjust_type = default_adjust_type(entity_type=entity_type) kdata_schema = get_kdata_schema(entity_type=entity_type, adjust_type=adjust_type) entity_schema = get_entity_schema(entity_type=entity_type) latest_day = kdata_schema.query_data(provider=data_provider, order=kdata_schema.timestamp.desc(), limit=1, return_type="domain") current_timestamp = latest_day[0].timestamp email_action = EmailInformer() # 至少上市一年 filter_entity_ids = [] if ignore_new_stock: pre_year = next_date(current_timestamp, -365) entity_ids = get_entity_ids( provider=entity_provider, entity_schema=entity_schema, filters=[entity_schema.timestamp <= pre_year]) if not entity_ids: msg = f"{entity_type} no entity_ids listed one year" logger.error(msg) email_action.send_message(zvt_config["email_username"], "report_top_stats error", msg) return filter_entity_ids = entity_ids filter_turnover_df = kdata_schema.query_data( filters=[ kdata_schema.turnover >= turnover_threshold, kdata_schema.turnover_rate >= turnover_rate_threshold, ], provider=data_provider, start_timestamp=current_timestamp, index="entity_id", columns=["entity_id", "code"], ) if filter_entity_ids: filter_entity_ids = set(filter_entity_ids) & set( filter_turnover_df.index.tolist()) else: filter_entity_ids = filter_turnover_df.index.tolist() if not filter_entity_ids: msg = f"{entity_type} no entity_ids selected" logger.error(msg) email_action.send_message(zvt_config["email_username"], "report_top_stats error", msg) return logger.info( f"{entity_type} filter_entity_ids size: {len(filter_entity_ids)}") filters = [kdata_schema.entity_id.in_(filter_entity_ids)] stats = [] ups = [] downs = [] for period in periods: start = next_date(current_timestamp, -period) df, _ = get_top_performance_entities( entity_type=entity_type, start_timestamp=start, filters=filters, pct=1, show_name=True, entity_provider=entity_provider, data_provider=data_provider, ) df.rename(columns={"score": f"score_{period}"}, inplace=True) ups.append(tabulate(df.iloc[:top_count], headers="keys")) downs.append(tabulate(df.iloc[-top_count:], headers="keys")) stats.append(tabulate(df.describe(), headers="keys")) # 最近一个月和一周最靓仔的 if period == 7 or period == 30: try: codes = [ decode_entity_id(entity_id)[2] for entity_id in df.index[:top_count] ] add_to_eastmoney(codes=codes, entity_type=entity_type, group="最靓仔", over_write=em_group_over_write) except Exception as e: logger.exception(e) email_action.send_message( zvt_config["email_username"], f"report_top_stats error", "report_top_stats error:{}".format(e)) # 一年内跌幅最大的 if period == 365: try: codes = [ decode_entity_id(entity_id)[2] for entity_id in df.index[-top_count:] ] add_to_eastmoney(codes=codes, entity_type=entity_type, group="谁有我惨", over_write=em_group_over_write) except Exception as e: logger.exception(e) email_action.send_message( zvt_config["email_username"], f"report_top_stats error", "report_top_stats error:{}".format(e)) msg = "\n" for s in stats: msg = msg + s + "\n" email_action.send_message(zvt_config["email_username"], f"{current_timestamp} {entity_type}统计报告", msg) msg = "\n" for up in ups: msg = msg + up + "\n" email_action.send_message(zvt_config["email_username"], f"{current_timestamp} {entity_type}涨幅统计报告", msg) msg = "\n" for down in downs: msg = msg + down + "\n" email_action.send_message(zvt_config["email_username"], f"{current_timestamp} {entity_type}跌幅统计报告", msg)