def run():
args = env_utils.get_args()
ts_code = args.code
to_date = args.date
if to_date is None:
to_date = date_utils.get_current_dt()
fetch_broker_recommend.fetch(to_date)
fetch_trade_cal.fetch(to_date)
fetch_dividend.update_dividend_to(to_date)
init_ts_basic.init()
basic_list = []
session: Session = db_client.get_session()
if ts_code is not None and ts_code != '':
basic_list = session.query(TsBasic).filter(
TsBasic.ts_code == ts_code).all()
else:
basic_list = session.query(TsBasic).all()
session.close()
for basic in basic_list:
if env_utils.parallel():
threadpool.submit(fetch_by_code,
ts_code=basic.ts_code,
to_date=to_date)
else:
fetch_by_code(ts_code=basic.ts_code, to_date=to_date)
threadpool.join()
strategy.generate_strategy_pool(to_date)
def fetch_by_date(to_date: str = date_utils.get_current_dt()):
to_date = trade_date_service.get_next_trade_date(to_date)
from_date = fetch_from_date()
trade_date_service.refresh_cache(trade_date_service.get_previous_trade_date(from_date), to_date)
return fetch.common_fetch_report(TsExpress, 'fetch_express',
TsExpress.ts_code, TsExpress.mq_ann_date,
from_date=from_date, to_date=to_date)
def common_fetch(exchange: str, to_date: str = date_utils.get_current_dt()):
"""
按交易所获取交易日历
:param exchange: SSE上交所,SZSE深交所
:param to_date: 获取截止日期
:return:
"""
from_date = fetch_from_date(exchange)
while from_date <= to_date:
next_date = date_utils.format_delta(from_date, 1000)
if next_date > to_date:
next_date = to_date
for cnt in range(2):
log.info('To fetch trade calendar %s %s~%s' %
(exchange, from_date, next_date))
try:
df: DataFrame = ts_client.fetch_trade_cal(exchange=exchange,
start_date=from_date,
end_date=next_date)
if not df.empty:
db_client.store_dataframe(df, TsTradeCal.__tablename__)
log.info('Successfully save trade cal: %s~%s' %
(from_date, to_date))
break
except Exception as e:
log.exception(
'Calling TuShare too fast. Will sleep 1 minutes...',
exc_info=e)
time.sleep(60)
ts_client.init_token()
from_date = date_utils.format_delta(next_date, 1)
def fetch_by_period(to_date: str = date_utils.get_current_dt()):
to_period = date_utils.next_period(to_date)
from_period = fetch_from_period()
return fetch.common_fetch_report_by_period(TsCashFlow,
'fetch_cash_flow',
TsCashFlow.end_date,
from_period=from_period,
to_period=to_period)
def calculate_by_code(ts_code: str, to_date: str = date_utils.get_current_dt()):
session: Session = db_client.get_session()
basic: TsBasic = session.query(TsBasic).filter(TsBasic.ts_code == ts_code).one()
session.close()
if basic is None:
log.error("Cant find ts_basic of %s" % ts_code)
return
calculate_and_insert(ts_code, basic.name, to_date)
def fetch_by_period(to_date: str = date_utils.get_current_dt()):
to_period = date_utils.next_period(to_date)
from_period = fetch_from_period()
return fetch.common_fetch_report_by_period(TsFinaIndicator,
'fetch_fina_indicator',
TsFinaIndicator.end_date,
from_period=from_period,
to_period=to_period)
def fetch_by_period(to_date: str = date_utils.get_current_dt()):
to_period = date_utils.next_period(to_date)
from_period = fetch_from_period()
return fetch.common_fetch_report_by_period(TsBalanceSheet,
'fetch_balance_sheet',
TsBalanceSheet.end_date,
from_period=from_period,
to_period=to_period)
def calculate_and_insert(ts_code: str, share_name: str, to_date: str = date_utils.get_current_dt()):
result_list = calculate_one(ts_code, share_name, to_date)
if len(result_list) > 0:
start_time = time.time()
db_client.batch_insert(result_list)
log.info("Insert %s for %s: %s seconds" % (MqDailyMetric.__tablename__, ts_code, time.time() - start_time))
else:
log.info('Nothing to insert into %s %s' % (MqDailyMetric.__tablename__, ts_code))
def fetch_by_period(to_date: str = date_utils.get_current_dt()):
to_period = date_utils.next_period(to_date)
from_period = fetch_from_period()
return fetch.common_fetch_report_by_period(TsIncome,
'fetch_income',
TsIncome.end_date,
from_period=from_period,
to_period=to_period)
def common_fetch_data(ts_code: str,
api_name: str,
table: Table,
date_field: str,
code_field: str,
to_date: str = date_utils.get_current_dt(),
to_do: bool = True,
page_size: int = 1000):
"""
:param ts_code: 股票编码
:param api_name: 调用tsclient的方法名
:param table: sqlalchemy的表定义
:param date_field: 对应发布日期的字段名 用于获取该类型数据在DB中最新日期
:param code_field: 对应股票编码的字段名
:param to_date: 数据要获取到哪天
:param to_do: 是否要进行此次获取
:param page_size: 每次获取多少条数据
:return:
"""
if not to_do:
return False, None
# 财报的 end_date 是第二天，不过目前是没有那么快有数据的
to_date = trade_date_service.get_next_trade_date(to_date)
from_date = fetch_from_date(date_field, code_field, ts_code)
from_date_to_ret = None
while from_date < to_date:
stock_data = None
next_date = date_utils.format_delta(from_date, page_size)
if next_date >= to_date:
next_date = to_date
for cnt in range(2):
log.info('To fetch %s of stock %s %s~%s' %
(table.__tablename__, ts_code, from_date, next_date))
try:
param = {
'ts_code': ts_code,
'start_date': from_date,
'end_date': next_date
}
stock_data = ts_client.fetch_data_frame(api_name, **param)
break
except Exception as e:
log.exception(
'Calling TuShare too fast. Will sleep 1 minutes...',
exc_info=e)
time.sleep(60)
ts_client.init_token()
if stock_data is not None and not stock_data.empty:
db_client.store_dataframe(stock_data, table.__tablename__)
if from_date_to_ret is None:
from_date_to_ret = from_date
log.info('Successfully save %s of stock %s %s~%s' %
(table.__tablename__, ts_code, from_date, next_date))
from_date = date_utils.format_delta(next_date, 1)
return True, from_date_to_ret
def fetch_by_period(to_date: str = date_utils.get_current_dt()):
to_period = date_utils.period_delta(date_utils.next_period(to_date),
1) # 年报和一季度的预报可能同时出
from_period = fetch_from_period()
return fetch.common_fetch_report_by_period(TsForecast,
'fetch_forecast',
TsForecast.end_date,
from_period=from_period,
to_period=to_period)
def fetch_by_date(to_date: str = date_utils.get_current_dt()):
to_date = trade_date_service.get_next_trade_date(to_date)
from_date = fetch_from_date()
return fetch.common_fetch_report(TsForecast,
'fetch_forecast',
TsForecast.ts_code,
TsForecast.mq_ann_date,
from_date=from_date,
to_date=to_date)
def run():
"""
获取最新的报告，并更新对应指标
注意不要跟daily.py同时运行，避免冲突
:return:
"""
current = get_current_dt()
next_dt = trade_date.get_next_trade_date(current)
fetch_forecast(current, next_dt)
def run(to_date: str = date_utils.get_current_dt()):
dt_code_map = {}
# 股票基础信息
init_ts_basic.init()
# 交易日历
fetch_trade_cal.fetch(to_date)
trade_date_service.refresh_cache(fetch_data_start_date,
date_utils.format_delta(to_date, 30))
to_fetch_list = [
# 同花顺指数
fetch_ths_index.run,
# 券商金股
fetch_broker_recommend.fetch,
# 分红记录
fetch_dividend.update_dividend_to,
# 每日指标
fetch_daily_basic.fetch_by_date,
# 每日交易
fetch_daily_bar.fetch_by_date,
# 复权因子
fetch_adj_factor.fetch_by_date,
# 涨跌停价格
fetch_stk_limit.fetch_by_date,
# 利润表
fetch_income.fetch_by_period,
# 资产负债表
fetch_balance_sheet.fetch_by_period,
# 现金流量表
fetch_cash_flow.fetch_by_period,
# 预报
fetch_forecast.fetch_by_period,
# 快报
fetch_express.fetch_by_period,
# 财务指标
fetch_fina_indicator.fetch_by_period
]
if env_utils.parallel():
fl = []
for func in to_fetch_list:
fl.append(threadpool.submit(func, to_date=to_date))
for f in fl:
m = f.result()
if m is not None:
fetch.merge_from_map(dt_code_map, m)
else:
for func in to_fetch_list:
m = func(to_date=to_date)
if m is not None:
fetch.merge_from_map(dt_code_map, m)
return dt_code_map
def cal_by_code(ts_code, to_date=date_utils.get_current_dt(), clear_from_date=None):
if ts_code is None:
log.error('None ts_code to calculate')
return
if clear_from_date is not None:
remove_after_fetch(ts_code, clear_from_date)
clear_after_fetch.clear(ts_code)
cal_mq_quarter.calculate_by_code(ts_code=ts_code, to_date=to_date)
cal_mq_daily.calculate_by_code(ts_code=ts_code, to_date=to_date)
cal_mq_daily_price.calculate_by_code(ts_code=ts_code, to_date=to_date)
cal_message.calculate_by_code(ts_code=ts_code, to_date=to_date)
def recalculate(ts_code: str, to_date=date_utils.get_current_dt(), from_date=None):
if ts_code is None:
log.error('None ts_code to recalculate')
return
if from_date is not None:
remove_after_fetch(ts_code, from_date)
cal_by_code(ts_code, to_date)
else:
cal_mq_quarter.recalculate_by_code(ts_code=ts_code, to_date=to_date)
cal_mq_daily.recalculate_by_code(ts_code=ts_code, to_date=to_date)
cal_mq_daily_price.recalculate_by_code(ts_code=ts_code, to_date=to_date)
cal_message.recalculate_by_code(ts_code=ts_code, to_date=to_date)
def fetch_by_code(to_date: str = date_utils.get_current_dt()):
session: Session = db_client.get_session()
basic_list = session.query(TsBasic).all()
session.close()
for basic in basic_list: # type: TsBasic
ts_code = basic.ts_code
from_date = fetch_from_date(ts_code)
fetch.common_fetch(TsDailyTradeInfo,
'fetch_daily_bar',
ts_code=ts_code,
from_date=from_date,
to_date=to_date)
def run():
args = env_utils.get_args()
ts_code = args.code
to_date = args.date
if to_date is None:
to_date = date_utils.get_current_dt()
dt_code_map = fetch_by_date.run(to_date)
calculate.daily_run(dt_code_map=dt_code_map,
to_date=to_date,
ts_code=ts_code)
cal_trade_amount.run(to_date=to_date)
strategy.generate_strategy_pool(to_date)
def recalculate_by_code(ts_code: str,
to_date: str = date_utils.get_current_dt()):
"""
删除相关数据然后重算
:param ts_code: 股票编码
:param to_date: 重刷到哪天
:return:
"""
session: Session = db_client.get_session()
session.query(MqDailyPrice).filter(
MqDailyPrice.ts_code == ts_code).delete()
session.close()
calculate_by_code(ts_code, to_date)
def fetch_broker_recommend(self, month):
"""
券商月度金股
https://tushare.pro/document/2?doc_id=267
:param month: 月份
:return:
"""
qpm = 5
if not self.__rt.get('broker_recommend', seconds_per_minute / qpm, qpm):
raise Exception('broker_recommend %d per minute, waiting' % qpm)
df: DataFrame = self.__pro.broker_recommend(month=month)
df['insert_dt'] = date_utils.get_current_dt()
df.drop('name', axis=1, inplace=True)
return df
def cal_by_ths_index_code(index_code,
index_name,
to_date: str = date_utils.get_current_dt()):
from_date = get_from_date(index_code)
history_avl = get_history_percent(index_code, from_date)
s: Session = db_client.get_session()
member_list = s.query(ThsMember).filter(
ThsMember.ts_code == index_code).all()
s.close()
share_code_list = [m.code for m in member_list]
result_list = []
while from_date <= to_date:
log.info('Calculating %s %s %s' %
(MqIndexTradeAmount.__tablename__, index_code, from_date))
s: Session = db_client.get_session()
trade_list = s.query(TsDailyTradeInfo) \
.filter(TsDailyTradeInfo.trade_date == from_date) \
.all()
s.close()
if len(trade_list) > 0:
total_amount = Decimal(0)
target_amount = Decimal(0)
for i in trade_list: # type: TsDailyTradeInfo
total_amount += i.amount
if i.ts_code in share_code_list:
target_amount += i.amount
val = float(decimal_utils.div(target_amount, total_amount))
history_percent = get_history_high_percent(history_avl, val)
history_avl.add(val)
result_list.append(
MqIndexTradeAmount(index_code=index_code,
index_name=index_name,
trade_date=from_date,
amount=target_amount,
percent=val,
history_high_ratio=history_percent))
from_date = date_utils.format_delta(from_date, 1)
if len(result_list) >= 1000:
db_client.batch_insert(result_list)
result_list = []
db_client.batch_insert(result_list)
def calculate_by_code(ts_code, to_date: str = date_utils.get_current_dt()):
session: Session = db_client.get_session()
basic: TsBasic = session.query(TsBasic).filter(
TsBasic.ts_code == ts_code).one()
session.close()
if basic is None:
log.error("Cant find ts_basic of %s" % ts_code)
return
report_message_list = generate_report_message_by_code(
ts_code, basic.name, to_date)
if len(report_message_list) > 0:
start = time.time()
db_client.batch_insert(report_message_list)
log.info("Insert mq_message for %s: %s seconds" %
(ts_code, time.time() - start))
else:
log.info('Nothing to insert into mq_message %s' % ts_code)
def generate_report_message_by_code(
ts_code: str, share_name: str, to_date: str = date_utils.get_current_dt()):
result_list = []
from_date = mq_calculate_start_date
session = db_client.get_session()
last_one: MqMessage = session.query(MqMessage) \
.filter(MqMessage.ts_code == ts_code, MqMessage.msg_type <= 3) \
.order_by(MqMessage.pub_date.desc()).limit(1).all()
if len(last_one) > 0:
from_date = date_utils.format_delta(last_one[0].pub_date, 1)
quarter_store = mq_quarter_store.init_quarter_store_by_date(
ts_code, from_date)
session.close()
while from_date <= to_date: # type: MqQuarterBasic
latest = quarter_store.find_latest(ts_code,
mq_quarter_metric_enum.dprofit.name,
from_date)
if latest is None:
from_date = date_utils.format_delta(from_date, 1)
continue
for i in range(5):
period = date_utils.period_delta(latest.period, -i)
dprofit = quarter_store.find_period_exact(
ts_code, mq_quarter_metric_enum.dprofit.name, period,
from_date)
if dprofit is None:
continue
# 只保留官方财报
if not ((dprofit.report_type & (1 << mq_report_type.report)) > 0 or \
(dprofit.report_type & (1 << mq_report_type.forecast)) > 0 or \
(dprofit.report_type & (1 << mq_report_type.express)) > 0):
continue
result_list.append(
MqMessage(ts_code=ts_code,
msg_type=mq_message_type.report,
message=get_report_message_content(
dprofit, share_name),
pub_date=date_utils.format_delta(from_date, 1)))
from_date = date_utils.format_delta(from_date, 1)
return result_list
def run(to_date: str = date_utils.get_current_dt()):
s: Session = db_client.get_session()
s.query(ThsIndex).delete()
index_df = ts_client.ths_index()
if index_df.empty:
return
db_client.store_dataframe(index_df, ThsIndex.__tablename__)
log.info('Successfully save %s' % ThsIndex.__tablename__)
index_code_list = index_df['ts_code'].unique()
for index_code in index_code_list:
s.query(ThsMember).filter(ThsMember.ts_code == index_code).delete()
member_df = ts_client.ths_member(index_code)
if member_df.empty:
continue
db_client.store_dataframe(member_df, ThsMember.__tablename__)
log.info('Successfully save %s %s' %
(ThsMember.__tablename__, index_code))
def fetch(to_date: str = date_utils.get_current_dt()):
m = date_utils.month_delta(to_date[0:6], 1)
fm = fetch_from_month()
while fm <= m:
for cnt in range(2):
log.info('To fetch broker recommend %s' % fm)
try:
df: DataFrame = ts_client.fetch_broker_recommend(month=fm)
if not df.empty:
db_client.store_dataframe(df,
TsBrokerRecommend.__tablename__)
log.info('Successfully save %s: %s' %
(TsBrokerRecommend.__tablename__, fm))
break
except Exception as e:
log.exception(
'Calling TuShare too fast. Will sleep 1 minutes...',
exc_info=e)
time.sleep(60)
ts_client.init_token()
fm = date_utils.month_delta(fm, 1)
def daily_run(dt_code_map={}, to_date=date_utils.get_current_dt(), ts_code=None):
basic_list = []
session: Session = db_client.get_session()
if ts_code is not None and ts_code != '':
basic_list = session.query(TsBasic).filter(TsBasic.ts_code == ts_code).all()
else:
basic_list = session.query(TsBasic).all()
session.close()
f_list = []
for basic in basic_list:
ts_code = basic.ts_code
clear_from_date = dt_code_map[ts_code] if ts_code in dt_code_map else None
if env_utils.parallel():
f_list.append(
threadpool.submit(cal_by_code, ts_code=ts_code, to_date=to_date, clear_from_date=clear_from_date))
else:
cal_by_code(ts_code=ts_code, to_date=to_date, clear_from_date=clear_from_date)
for f in f_list:
f.result()
def cal_top_amount_share(to_date: str = date_utils.get_current_dt()):
from_date = get_from_date(mq_index.top_amount_code)
history_avl = get_history_percent(mq_index.top_amount_code, from_date)
result_list = []
while from_date <= to_date:
log.info('Calculating %s %s %s' %
(MqIndexTradeAmount.__tablename__, mq_index.top_amount_code,
from_date))
s: Session = db_client.get_session()
trade_list = s.query(TsDailyTradeInfo) \
.filter(TsDailyTradeInfo.trade_date == from_date) \
.order_by(TsDailyTradeInfo.amount.desc()) \
.all()
s.close()
if len(trade_list) > 0:
top_to_count = math.floor(len(trade_list) * 0.05)
total_amount = Decimal(0)
top_amount = Decimal(0)
for i in range(len(trade_list)):
total_amount += trade_list[i].amount
if i <= top_to_count:
top_amount += trade_list[i].amount
val = float(decimal_utils.div(top_amount, total_amount))
history_percent = get_history_high_percent(history_avl, val)
history_avl.add(val)
result_list.append(
MqIndexTradeAmount(index_code=mq_index.top_amount_code,
index_name=mq_index.top_amount_name,
trade_date=from_date,
amount=top_amount,
percent=val,
history_high_ratio=history_percent))
from_date = date_utils.format_delta(from_date, 1)
if len(result_list) >= 1000:
db_client.batch_insert(result_list)
result_list = []
db_client.batch_insert(result_list)
def cal_ths_index(to_date: str = date_utils.get_current_dt()):
"""
计算同花顺指数相关的成交量占比，根据配置控制是否并发计算
:param to_date:
:return:
"""
s: Session = db_client.get_session()
index_list = s.query(ThsIndex).all()
s.close()
ignore_index_list = [
'885338.TI', '885867.TI', '885694.TI', '885520.TI', '885745.TI',
'885598.TI', '883303.TI', '883300.TI', '883302.TI', '885796.TI',
'885873.TI', '885869.TI', '885803.TI', '885916.TI', '885907.TI',
'885906.TI', '885905.TI', '885893.TI', '885639.TI'
]
if env_utils.parallel():
f_list = []
for index in index_list: # type: ThsIndex
if index.ts_code in ignore_index_list:
continue
f_list.append(
threadpool.submit(cal_by_ths_index_code,
index_code=index.ts_code,
index_name=index.name,
to_date=to_date))
for f in f_list:
f.result()
else:
for index in index_list: # type: ThsIndex
if index.ts_code in ignore_index_list:
continue
cal_by_ths_index_code(index_code=index.ts_code,
index_name=index.name,
to_date=to_date)
from_date=from_date,
to_date=to_date)
def fetch_from_period():
s: Session = db_client.get_session()
result = s.query(func.max(TsCashFlow.end_date)) \
.all()
s.close()
from_date = fetch_data_start_period
if len(result) > 0 and not result[0][0] is None:
from_date = date_utils.period_delta(result[0][0], -4)
return from_date
def fetch_by_period(to_date: str = date_utils.get_current_dt()):
to_period = date_utils.next_period(to_date)
from_period = fetch_from_period()
return fetch.common_fetch_report_by_period(TsCashFlow,
'fetch_cash_flow',
TsCashFlow.end_date,
from_period=from_period,
to_period=to_period)
if __name__ == '__main__':
trade_date_service.refresh_cache(
fetch_data_start_date,
date_utils.format_delta(date_utils.get_current_dt(), 30))
print(fetch_by_date())
to_date = trade_date_service.get_next_trade_date(to_date)
from_date = fetch_from_date()
trade_date_service.refresh_cache(trade_date_service.get_previous_trade_date(from_date), to_date)
return fetch.common_fetch_report(TsExpress, 'fetch_express',
TsExpress.ts_code, TsExpress.mq_ann_date,
from_date=from_date, to_date=to_date)
def fetch_from_period():
s: Session = db_client.get_session()
result = s.query(func.max(TsExpress.end_date)) \
.all()
s.close()
from_date = fetch_data_start_period
if len(result) > 0 and not result[0][0] is None:
from_date = date_utils.period_delta(result[0][0], -4)
return from_date
def fetch_by_period(to_date: str = date_utils.get_current_dt()):
to_period = date_utils.next_period(to_date)
from_period = fetch_from_period()
return fetch.common_fetch_report_by_period(
TsExpress, 'fetch_express', TsExpress.end_date,
from_period=from_period, to_period=to_period)
if __name__ == '__main__':
trade_date_service.refresh_cache(fetch_data_start_date, date_utils.format_delta(date_utils.get_current_dt(), 30))
print(fetch_by_date())