def record_stock_data(data_provider="em", entity_provider="em", sleeping_time=2):
# A股标的
run_data_recorder(domain=Stock, data_provider=data_provider, force_update=False)
# A股后复权行情
run_data_recorder(
domain=Stock1dHfqKdata,
data_provider=data_provider,
entity_provider=entity_provider,
day_data=True,
sleeping_time=sleeping_time,
)
# 板块(概念，行业)
run_data_recorder(domain=Block, data_provider="eastmoney", force_update=False)
# 板块行情(概念，行业)
run_data_recorder(domain=Block1dKdata, data_provider="em", day_data=True, sleeping_time=sleeping_time)
# 报告新概念和行业
email_action = EmailInformer()
list_date = next_date(current_date(), -90)
df = Block.query_data(
filters=[Block.category == BlockCategory.concept.value, Block.list_date >= list_date], index="entity_id"
)
# add them to eastmoney
try:
add_to_eastmoney(codes=df["code"], entity_type="block", group="新概念", over_write=False)
except Exception as e:
email_action.send_message(
zvt_config["email_username"], f"report_concept error", "report_concept error:{}".format(e)
)
def inform(action: EmailInformer, entity_ids, target_date, title,
entity_provider, entity_type, em_group, em_group_over_write):
msg = "no targets"
if entity_ids:
entities = get_entities(provider=entity_provider,
entity_type=entity_type,
entity_ids=entity_ids,
return_type="domain")
if em_group:
try:
codes = [entity.code for entity in entities]
add_to_eastmoney(codes=codes,
entity_type=entity_type,
group=em_group,
over_write=em_group_over_write)
except Exception as e:
action.send_message(
zvt_config["email_username"],
f"{target_date} {title} error",
f"{target_date} {title} error: {e}",
)
infos = [f"{entity.name}({entity.code})" for entity in entities]
msg = "\n".join(infos) + "\n"
logger.info(msg)
action.send_message(zvt_config["email_username"], f"{target_date} {title}",
msg)
def run_data_recorder(
domain: Type[Mixin],
entity_provider=None,
data_provider=None,
entity_ids=None,
retry_times=10,
sleeping_time=10,
**recorder_kv,
):
logger.info(
f" record data: {domain.__name__}, entity_provider: {entity_provider}, data_provider: {data_provider}"
)
while retry_times > 0:
email_action = EmailInformer()
try:
domain.record_data(entity_ids=entity_ids,
provider=data_provider,
sleeping_time=sleeping_time,
**recorder_kv)
msg = f"record {domain.__name__} success"
logger.info(msg)
email_action.send_message(zvt_config["email_username"], msg, msg)
break
except Exception as e:
logger.exception("report error:{}".format(e))
time.sleep(60 * 3)
retry_times = retry_times - 1
if retry_times == 0:
email_action.send_message(
zvt_config["email_username"],
f"record {domain.__name__} error",
f"record {domain.__name__} error: {e}",
)
def record_stock_data(data_provider="em",
entity_provider="em",
sleeping_time=2):
# A股标的
run_data_recorder(domain=Stock,
data_provider=data_provider,
force_update=False)
# A股后复权行情
run_data_recorder(
domain=Stock1dHfqKdata,
data_provider=data_provider,
entity_provider=entity_provider,
day_data=True,
sleeping_time=sleeping_time,
)
# 板块(概念，行业)
run_data_recorder(domain=Block,
entity_provider=entity_provider,
data_provider=entity_provider,
force_update=False)
# 板块行情(概念，行业)
run_data_recorder(
domain=Block1dKdata,
entity_provider=entity_provider,
data_provider=entity_provider,
day_data=True,
sleeping_time=sleeping_time,
)
# 报告新概念和行业
email_action = EmailInformer()
df = Block.query_data(
filters=[Block.category == BlockCategory.concept.value],
order=Block.list_date.desc(),
index="entity_id",
limit=5,
)
inform(
action=email_action,
entity_ids=df.index.tolist(),
target_date=current_date(),
title="report 新概念",
entity_provider=entity_provider,
entity_type="block",
em_group="关注板块",
em_group_over_write=True,
)
from apscheduler.schedulers.background import BackgroundScheduler
from examples.report_utils import report_targets
from zvt import init_log
from zvt.api.kdata import get_latest_kdata_date
from zvt.api.selector import get_mini_and_small_stock, get_middle_and_big_stock
from zvt.contract import AdjustType
from zvt.factors import VolumeUpMaFactor
from zvt.informer import EmailInformer
logger = logging.getLogger(__name__)
sched = BackgroundScheduler()
email_informer = EmailInformer()
@sched.scheduled_job("cron", hour=17, minute=0, day_of_week="mon-fri")
def report_vol_up():
target_date = get_latest_kdata_date(entity_type="stock", adjust_type=AdjustType.hfq, provider="em")
entity_ids = get_mini_and_small_stock(timestamp=target_date, provider="em")
report_targets(
factor_cls=VolumeUpMaFactor,
entity_provider="em",
data_provider="em",
informer=email_informer,
em_group="年线股票",
title="放量突破(半)年线小市值股票",
entity_type="stock",
def record_dragon_tiger(data_provider="em",
entity_provider="em",
sleeping_time=2):
# 龙虎榜数据
run_data_recorder(
domain=DragonAndTiger,
data_provider=data_provider,
entity_provider=entity_provider,
day_data=True,
sleeping_time=sleeping_time,
)
email_action = EmailInformer()
# recent year
start_timestamp = next_date(current_date(), -400)
# 最近一年牛x的营业部
players = get_big_players(start_timestamp=start_timestamp)
# 最近30天有牛x的营业部上榜的个股
recent_date = next_date(current_date(), -30)
selected = []
for player in players:
filters = [
or_(
and_(DragonAndTiger.dep1 == player,
DragonAndTiger.dep1_rate >= 5),
and_(DragonAndTiger.dep2 == player,
DragonAndTiger.dep2_rate >= 5),
and_(DragonAndTiger.dep3 == player,
DragonAndTiger.dep3_rate >= 5),
and_(DragonAndTiger.dep4 == player,
DragonAndTiger.dep4_rate >= 5),
and_(DragonAndTiger.dep5 == player,
DragonAndTiger.dep5_rate >= 5),
)
]
df = DragonAndTiger.query_data(
start_timestamp=recent_date,
filters=filters,
columns=[
DragonAndTiger.timestamp, DragonAndTiger.entity_id,
DragonAndTiger.code, DragonAndTiger.name
],
index="entity_id",
)
selected = selected + df.index.tolist()
if selected:
selected = list(set(selected))
target_date = get_latest_kdata_date(provider=data_provider,
entity_type="stock",
adjust_type="hfq")
df = Stock1dHfqKdata.query_data(
provider=data_provider,
entity_ids=selected,
filters=[
Stock1dHfqKdata.turnover_rate > 0.02,
Stock1dHfqKdata.timestamp == to_pd_timestamp(target_date),
Stock1dHfqKdata.turnover > 300000000,
],
index=["entity_id"],
)
inform(
action=email_action,
entity_ids=df.index.tolist(),
target_date=current_date(),
title="report 龙虎榜",
entity_provider=entity_provider,
entity_type="stock",
em_group="重要指数",
em_group_over_write=False,
)
def report_targets(
factor_cls: Type[Factor],
entity_provider,
data_provider,
title,
entity_type="stock",
informer: EmailInformer = None,
em_group=None,
em_group_over_write=True,
filter_by_volume=True,
adjust_type=None,
start_timestamp="2019-01-01",
**factor_kv,
):
logger.info(
f"entity_provider: {entity_provider}, data_provider: {data_provider}, entity_type: {entity_type}, start_timestamp: {start_timestamp}"
)
error_count = 0
while error_count <= 10:
try:
if not adjust_type:
adjust_type = default_adjust_type(entity_type=entity_type)
target_date = get_latest_kdata_date(provider=data_provider,
entity_type=entity_type,
adjust_type=adjust_type)
logger.info(f"target_date :{target_date}")
current_entity_pool = None
if filter_by_volume:
# 成交量
vol_df = get_top_volume_entities(
entity_type=entity_type,
start_timestamp=next_date(target_date, -30),
end_timestamp=target_date,
adjust_type=adjust_type,
pct=0.4,
data_provider=data_provider,
)
current_entity_pool = vol_df.index.tolist()
logger.info(
f"current_entity_pool({len(current_entity_pool)}): {current_entity_pool}"
)
kdata_schema = get_kdata_schema(entity_type,
level=IntervalLevel.LEVEL_1DAY,
adjust_type=adjust_type)
filters = []
if "turnover_threshold" in factor_kv:
filters = filters + [
kdata_schema.turnover >=
factor_kv.get("turnover_threshold")
]
if "turnover_rate_threshold" in factor_kv:
filters = filters + [
kdata_schema.turnover_rate >=
factor_kv.get("turnover_rate_threshold")
]
if filters:
filters = filters + [kdata_schema.timestamp == target_date]
kdata_df = kdata_schema.query_data(
provider=data_provider,
filters=filters,
columns=["entity_id", "timestamp"],
index="entity_id")
if current_entity_pool:
current_entity_pool = set(current_entity_pool) & set(
kdata_df.index.tolist())
else:
current_entity_pool = kdata_df.index.tolist()
if "entity_ids" in factor_kv:
if current_entity_pool:
current_entity_pool = set(current_entity_pool) & set(
factor_kv.pop("entity_ids"))
else:
current_entity_pool = set(factor_kv.pop("entity_ids"))
# add the factor
my_selector = TargetSelector(start_timestamp=start_timestamp,
end_timestamp=target_date,
select_mode=SelectMode.condition_or)
entity_schema = get_entity_schema(entity_type=entity_type)
tech_factor = factor_cls(
entity_schema=entity_schema,
entity_provider=entity_provider,
provider=data_provider,
entity_ids=current_entity_pool,
start_timestamp=start_timestamp,
end_timestamp=target_date,
adjust_type=adjust_type,
**factor_kv,
)
my_selector.add_factor(tech_factor)
my_selector.run()
long_stocks = my_selector.get_open_long_targets(
timestamp=target_date)
inform(
informer,
entity_ids=long_stocks,
target_date=target_date,
title=title,
entity_provider=entity_provider,
entity_type=entity_type,
em_group=em_group,
em_group_over_write=em_group_over_write,
)
break
except Exception as e:
logger.exception("report error:{}".format(e))
time.sleep(60 * 3)
error_count = error_count + 1
if error_count == 10:
informer.send_message(
zvt_config["email_username"],
f"report {entity_type}{factor_cls.__name__} error",
f"report {entity_type}{factor_cls.__name__} error: {e}",
)
def report_top_entities(
entity_provider,
data_provider,
periods=None,
ignore_new_stock=True,
ignore_st=True,
entity_ids=None,
entity_type="stock",
adjust_type=None,
top_count=30,
turnover_threshold=100000000,
turnover_rate_threshold=0.02,
informer: EmailInformer = None,
em_group=None,
em_group_over_write=True,
return_type=TopType.positive,
):
error_count = 0
while error_count <= 10:
try:
if periods is None:
periods = [7, 30, 365]
if not adjust_type:
adjust_type = default_adjust_type(entity_type=entity_type)
kdata_schema = get_kdata_schema(entity_type=entity_type,
adjust_type=adjust_type)
entity_schema = get_entity_schema(entity_type=entity_type)
target_date = get_latest_kdata_date(provider=data_provider,
entity_type=entity_type,
adjust_type=adjust_type)
filter_entity_ids = get_entity_ids_by_filter(
provider=entity_provider,
ignore_st=ignore_st,
ignore_new_stock=ignore_new_stock,
entity_schema=entity_schema,
target_date=target_date,
entity_ids=entity_ids,
)
if not filter_entity_ids:
msg = f"{entity_type} no entity_ids selected"
logger.error(msg)
informer.send_message(zvt_config["email_username"],
"report_top_stats error", msg)
return
filter_turnover_df = kdata_schema.query_data(
filters=[
kdata_schema.turnover >= turnover_threshold,
kdata_schema.turnover_rate >= turnover_rate_threshold,
],
provider=data_provider,
start_timestamp=target_date,
index="entity_id",
columns=["entity_id", "code"],
)
if filter_entity_ids:
filter_entity_ids = set(filter_entity_ids) & set(
filter_turnover_df.index.tolist())
else:
filter_entity_ids = filter_turnover_df.index.tolist()
if not filter_entity_ids:
msg = f"{entity_type} no entity_ids selected"
logger.error(msg)
informer.send_message(zvt_config["email_username"],
"report_top_stats error", msg)
return
logger.info(
f"{entity_type} filter_entity_ids size: {len(filter_entity_ids)}"
)
filters = [kdata_schema.entity_id.in_(filter_entity_ids)]
selected = []
for i, period in enumerate(periods):
interval = period
if target_date.weekday() + 1 < interval:
interval = interval + 2
start = next_date(target_date, -interval)
positive_df, negative_df = get_top_performance_entities(
entity_type=entity_type,
start_timestamp=start,
kdata_filters=filters,
pct=1,
show_name=True,
entity_provider=entity_provider,
data_provider=data_provider,
return_type=return_type,
)
if return_type == TopType.positive:
df = positive_df
else:
df = negative_df
selected = selected + df.index[:top_count].tolist()
selected = list(dict.fromkeys(selected))
inform(
informer,
entity_ids=selected,
target_date=target_date,
title=f"{entity_type} {em_group}({len(selected)})",
entity_provider=entity_provider,
entity_type=entity_type,
em_group=em_group,
em_group_over_write=em_group_over_write,
)
break
except Exception as e:
logger.exception("report error:{}".format(e))
time.sleep(30)
error_count = error_count + 1
def report_targets(
factor_cls: Type[Factor],
entity_provider,
data_provider,
title,
entity_type="stock",
em_group=None,
em_group_over_write=True,
filter_by_volume=True,
adjust_type=None,
start_timestamp="2019-01-01",
**factor_kv,
):
logger.info(
f"entity_provider: {entity_provider}, data_provider: {data_provider}, entity_type: {entity_type}, start_timestamp: {start_timestamp}"
)
error_count = 0
while error_count <= 10:
email_action = EmailInformer()
try:
if entity_type == "stock" and not adjust_type:
adjust_type = AdjustType.hfq
target_date = get_latest_kdata_date(provider=data_provider,
entity_type=entity_type,
adjust_type=adjust_type)
logger.info(f"target_date :{target_date}")
current_entity_pool = None
if filter_by_volume:
# 成交量
vol_df = get_top_volume_entities(
entity_type=entity_type,
start_timestamp=next_date(target_date, -30),
end_timestamp=target_date,
adjust_type=adjust_type,
pct=0.4,
)
current_entity_pool = vol_df.index.tolist()
logger.info(
f"current_entity_pool({len(current_entity_pool)}): {current_entity_pool}"
)
# add the factor
my_selector = TargetSelector(start_timestamp=start_timestamp,
end_timestamp=target_date,
select_mode=SelectMode.condition_or)
entity_schema = get_entity_schema(entity_type=entity_type)
tech_factor = factor_cls(
entity_schema=entity_schema,
entity_provider=entity_provider,
provider=data_provider,
entity_ids=current_entity_pool,
start_timestamp=start_timestamp,
end_timestamp=target_date,
adjust_type=adjust_type,
**factor_kv,
)
my_selector.add_factor(tech_factor)
my_selector.run()
long_stocks = my_selector.get_open_long_targets(
timestamp=target_date)
msg = "no targets"
if long_stocks:
entities = get_entities(provider=entity_provider,
entity_type=entity_type,
entity_ids=long_stocks,
return_type="domain")
if em_group:
try:
codes = [entity.code for entity in entities]
add_to_eastmoney(codes=codes,
entity_type=entity_type,
group=em_group,
over_write=em_group_over_write)
except Exception as e:
email_action.send_message(
zvt_config["email_username"],
f"report {entity_type}{factor_cls.__name__} error",
f"report {entity_type}{factor_cls.__name__} error: {e}",
)
infos = [
f"{entity.name}({entity.code})" for entity in entities
]
msg = "\n".join(infos) + "\n"
logger.info(msg)
email_action.send_message(zvt_config["email_username"],
f"{target_date} {title}", msg)
break
except Exception as e:
logger.exception("report error:{}".format(e))
time.sleep(60 * 3)
error_count = error_count + 1
if error_count == 10:
email_action.send_message(
zvt_config["email_username"],
f"report {entity_type}{factor_cls.__name__} error",
f"report {entity_type}{factor_cls.__name__} error: {e}",
)
def report_top_stats(
entity_provider,
data_provider,
periods=[7, 30, 180, 365],
ignore_new_stock=True,
entity_type="stock",
adjust_type=None,
top_count=30,
turnover_threshold=100000000,
turnover_rate_threshold=0.02,
em_group_over_write=True,
):
if not adjust_type:
adjust_type = default_adjust_type(entity_type=entity_type)
kdata_schema = get_kdata_schema(entity_type=entity_type,
adjust_type=adjust_type)
entity_schema = get_entity_schema(entity_type=entity_type)
latest_day = kdata_schema.query_data(provider=data_provider,
order=kdata_schema.timestamp.desc(),
limit=1,
return_type="domain")
current_timestamp = latest_day[0].timestamp
email_action = EmailInformer()
# 至少上市一年
filter_entity_ids = []
if ignore_new_stock:
pre_year = next_date(current_timestamp, -365)
entity_ids = get_entity_ids(
provider=entity_provider,
entity_schema=entity_schema,
filters=[entity_schema.timestamp <= pre_year])
if not entity_ids:
msg = f"{entity_type} no entity_ids listed one year"
logger.error(msg)
email_action.send_message(zvt_config["email_username"],
"report_top_stats error", msg)
return
filter_entity_ids = entity_ids
filter_turnover_df = kdata_schema.query_data(
filters=[
kdata_schema.turnover >= turnover_threshold,
kdata_schema.turnover_rate >= turnover_rate_threshold,
],
provider=data_provider,
start_timestamp=current_timestamp,
index="entity_id",
columns=["entity_id", "code"],
)
if filter_entity_ids:
filter_entity_ids = set(filter_entity_ids) & set(
filter_turnover_df.index.tolist())
else:
filter_entity_ids = filter_turnover_df.index.tolist()
if not filter_entity_ids:
msg = f"{entity_type} no entity_ids selected"
logger.error(msg)
email_action.send_message(zvt_config["email_username"],
"report_top_stats error", msg)
return
logger.info(
f"{entity_type} filter_entity_ids size: {len(filter_entity_ids)}")
filters = [kdata_schema.entity_id.in_(filter_entity_ids)]
stats = []
ups = []
downs = []
for period in periods:
start = next_date(current_timestamp, -period)
df, _ = get_top_performance_entities(
entity_type=entity_type,
start_timestamp=start,
filters=filters,
pct=1,
show_name=True,
entity_provider=entity_provider,
data_provider=data_provider,
)
df.rename(columns={"score": f"score_{period}"}, inplace=True)
ups.append(tabulate(df.iloc[:top_count], headers="keys"))
downs.append(tabulate(df.iloc[-top_count:], headers="keys"))
stats.append(tabulate(df.describe(), headers="keys"))
# 最近一个月和一周最靓仔的
if period == 7 or period == 30:
try:
codes = [
decode_entity_id(entity_id)[2]
for entity_id in df.index[:top_count]
]
add_to_eastmoney(codes=codes,
entity_type=entity_type,
group="最靓仔",
over_write=em_group_over_write)
except Exception as e:
logger.exception(e)
email_action.send_message(
zvt_config["email_username"], f"report_top_stats error",
"report_top_stats error:{}".format(e))
# 一年内跌幅最大的
if period == 365:
try:
codes = [
decode_entity_id(entity_id)[2]
for entity_id in df.index[-top_count:]
]
add_to_eastmoney(codes=codes,
entity_type=entity_type,
group="谁有我惨",
over_write=em_group_over_write)
except Exception as e:
logger.exception(e)
email_action.send_message(
zvt_config["email_username"], f"report_top_stats error",
"report_top_stats error:{}".format(e))
msg = "\n"
for s in stats:
msg = msg + s + "\n"
email_action.send_message(zvt_config["email_username"],
f"{current_timestamp} {entity_type}统计报告", msg)
msg = "\n"
for up in ups:
msg = msg + up + "\n"
email_action.send_message(zvt_config["email_username"],
f"{current_timestamp} {entity_type}涨幅统计报告", msg)
msg = "\n"
for down in downs:
msg = msg + down + "\n"
email_action.send_message(zvt_config["email_username"],
f"{current_timestamp} {entity_type}跌幅统计报告", msg)