def flush_forecast():
"""刷新业绩预告"""
f10 = THSF10()
sess = get_session()
flush(sess, f10)
sess.close()
f10.browser.quit()
def flush_index_info():
"""刷新指数基本信息"""
f10 = THSF10()
df = f10.get_index_info()
df.sort_values('code', inplace=True)
sess = get_session()
for _, row in df.iterrows():
origin = row['code']
code = _map_to(origin)
s = IndexInfo(code=code)
s.title = row['title']
s.url = row['url']
s.origin_code = origin
sess.add(s)
try:
sess.commit()
logger.info('新增代码：{}'.format(code))
except IntegrityError:
sess.rollback()
logger.info('代码：{}已经存在'.format(code))
continue
# 关闭会话
sess.close()
# 关闭浏览器
f10.browser.quit()
def flush(codes, end):
for code in codes:
sess = get_session()
start = _get_start_date(sess, code)
if start is not None and start > end:
logger.info('代码：{} 无需刷新'.format(code))
continue
try:
df = history_data_reader.read(code=code,
start=start,
end=end,
is_index=True)
# 按日期排序（升序）
df.sort_index(inplace=True)
except ValueError:
# 当开始日期大于结束日期时，触发值异常
logger.info('无法获取网页数据。代码：{}，开始日期：{}, 结束日期：{}'.format(
code, start, end))
continue
objs = _gen(df)
sess.add_all(objs)
sess.commit()
logger.info('代码：{}, 新增{}行'.format(code, len(objs)))
sess.close()
if len(objs):
log_to_db(IndexDaily.__tablename__,
True,
len(objs),
Action.INSERT,
code,
start=df.index[0].date(),
end=df.index[-1].date())
else:
log_to_db(IndexDaily.__tablename__, False, 0, Action.INSERT, code)
def batch_flush(self, codes):
"""刷新批量股票"""
codes = ensure_list(codes)
for code in codes:
sess = get_session()
self.flush(sess, code)
sess.close()
def main():
trading = is_trading_reader.read(oneday=now.date())
if trading:
sess = get_session()
codes = get_all_codes(False)
flush(sess, codes)
sess.close()
else:
logger.info('{}非交易时段'.format(now))
def last_update_date(code):
"""类给定代码最后更新日期"""
sess = get_session()
dt = sess.query(func.max(Adjustment.last_updated)).filter(
Adjustment.code == code).scalar()
if dt:
return dt.date()
else:
# 返回一个尽量早的日期
return pd.Timestamp('1900').date()
def existed(class_, code=None, date_=None):
"""判定是否存在指定代码、日期的实例"""
sess = get_session()
stmt = sess.query(class_)
if code is not None:
stmt = stmt.filter(class_.code == code)
if date_ is not None:
# 确保类型一致
date_ = pd.Timestamp(date_).date()
stmt = stmt.filter(class_.date == date_)
res = sess.query(stmt.exists()).scalar()
sess.close()
return res
def flush_index_daily():
"""刷新指数日线数据"""
sess = get_session()
end = sess.query(func.max(TradingCalendar.date)).filter(
TradingCalendar.is_trading == True).scalar()
if end is None:
raise NotImplementedError('尚未初始化交易日历数据！')
query = sess.query(IndexInfo).order_by(IndexInfo.code)
codes = [x.code for x in query.all()]
sess.close()
if len(codes) < 10:
raise NotImplementedError('尚未刷新股票代码数据！')
flush(codes, end)
def flush_treasury():
sess = get_session()
start = get_start(sess)
if start is None:
# 读取本地及网络数据
df = read_data()
elif start > pd.Timestamp('today').date():
return
else:
# 读取自开始日期的数据
df = fetch_treasury_data_from(start)
insert(sess, df)
sess.close()
def flush(codes):
today = pd.Timestamp('today').date()
for code in codes:
sess = get_session()
lud = last_update_date(code)
if lud == today:
logger.info('代码：{} 无需更新'.format(code))
continue
last_date = sess.query(func.max(Adjustment.date)).filter(
Adjustment.code == code).scalar()
if last_date:
start = last_date + timedelta(days=1)
else:
ipo = sess.query(Issue.A004_上市日期).filter(
Issue.code == code).filter(Issue.A004_上市日期.isnot(None)).scalar()
# 如尚未上市，则继续下一个代码
if not ipo:
logger.info('代码：{} 无数据'.format(code))
continue
else:
start = ipo
# 读取全部分红派息表网络数据
df = adjustment_reader.read(code)
# 如存在分红派息数据
if df is not None:
try:
# 如分红、派息均无效，则去除
df = df.dropna(0, 'all', subset=['amount', 'ratio'])
except KeyError:
continue
data = df[df.index.date >= start]
# 如没有有效数据，则继续下一个代码
if data.empty:
continue
data.sort_index(inplace=True)
to_adds = _gen(code, data)
sess.add_all(to_adds)
sess.commit()
logger.info('代码：{}, 添加{}行'.format(code, len(to_adds)))
log_to_db(Adjustment.__tablename__, True,
len(to_adds), Action.INSERT, code,
start=data.index[0].date(),
end=data.index[-1].date())
else:
logger.info('代码：{}, 无数据'.format(code))
log_to_db(Adjustment.__tablename__, True, 0, Action.INSERT, code)
sess.close()
def flush_stockdaily(codes=None, init=False):
"""
刷新股票日线数据
说明：
如初始化则包含所有曾经上市的股票代码，含已经退市
否则仅包含当前在市的股票代码
"""
sess = get_session()
end = sess.query(func.max(TradingCalendar.date)).filter(
TradingCalendar.is_trading == True).scalar()
sess.close()
if end is None:
raise NotImplementedError('尚未初始化交易日历数据！')
if init or codes is None:
codes = get_all_codes(init)
else:
codes = ensure_list(codes)
# 删除无效数据
_delete()
flush(codes, end)
def get_all_codes(init=False):
"""
获取所有股票代码列表
Parameters
----------
init : bool
是否为初始化时使用
初始化时包含所有曾经上市的股票代码；而日常使用则仅仅包含在市代码
默认为否
Returns
-------
res : list
股票代码列表
Example
-------
>>> get_all_codes(True)
['000001', '000002', '000003', '000004', '000005',......]
>>> get_all_codes(False)
['000001', '000002', '000004', '000005', '000006', '000007',......]
"""
sess = get_session()
if init:
query = sess.query(Issue).order_by(Issue.code).filter(
Issue.A004_上市日期.isnot(None))
codes = [x.code for x in query.all()]
else:
stmt = sess.query(Issue.code).order_by(Issue.code).filter(
Issue.A004_上市日期.isnot(None))
query = sess.query(Stock).order_by(Stock.code).filter(
Stock.latest_status.is_(Status.in_trading)).filter(
Stock.code.in_(stmt))
codes = [x.code for x in query.all()]
if len(codes) == 0:
raise NotImplementedError('可能未完成股票代码或发行数据初始化！')
sess.close()
return codes
def flush(dates):
for day in dates:
sess = get_session()
# 此时day为元组
d = day[0]
has_data = existed(Margin, date_=d)
# 确保不重复添加
if not has_data:
df = fetch_margin_data(d)
to_adds = _gen(df)
sess.add_all(to_adds)
sess.commit()
logger.info('日期：{}, 新增{}行'.format(d, len(to_adds)))
log_to_db(Margin.__tablename__,
True,
len(to_adds),
Action.INSERT,
start=d,
end=d)
else:
logger.info('日期：{}, 数据已经存在'.format(d))
sess.close()
def flush_dealdetail(codes=None, init=False):
"""
刷新股票分时交易数据
说明：
如初始化则包含所有曾经上市的股票代码，含已经退市
否则仅包含当前在市的股票代码
"""
sess = get_session()
if init:
# 最多前溯2周?
start = datetime.today().date() - timedelta(days=14)
else:
start = datetime.today().date() - timedelta(days=1)
if init or codes is None:
codes = get_all_codes(init)
else:
codes = ensure_list(codes)
dates = sess.query(
TradingCalendar.date).filter(TradingCalendar.date >= start).filter(
TradingCalendar.is_trading == True).all()
sess.close()
flush(codes, dates)
def flush(codes, dates):
for day in dates:
# 此时day为元组
d = day[0]
for code in codes:
sess = get_session()
# 必须限定成交量>0
traded = sess.query(StockDaily.code).filter(
StockDaily.code == code).filter(StockDaily.date == d).filter(
StockDaily.A006_成交量 > 0).scalar()
# 股票当天存在交易，而非停牌
if traded:
try:
has_data, = sess.query(DealDetail.code).filter(
DealDetail.code == code).filter(
DealDetail.date == d).first()
except TypeError:
has_data = None
# 确保不重复添加
if not has_data:
df = fetch_cjmx(code, d)
to_adds = _gen(df)
sess.add_all(to_adds)
sess.commit()
logger.info('股票：{}，日期：{}, 新增{}行'.format(
code, d, len(to_adds)))
log_to_db(DealDetail.__tablename__, True, len(to_adds),
Action.INSERT, code, d, d)
else:
logger.info('股票：{}，日期：{} 已经刷新'.format(code, d))
log_to_db(DealDetail.__tablename__, True, 0, Action.INSERT,
code, d, d)
else:
logger.info('股票：{}，日期：{} 停牌'.format(code, d))
log_to_db(DealDetail.__tablename__, True, 0, Action.INSERT,
code, d, d)
sess.close()
def flush(codes, end):
for code in codes:
sess = get_session()
last_date = sess.query(func.max(
StockDaily.date)).filter(StockDaily.code == code).scalar()
if last_date is None:
start = None
else:
start = last_date + timedelta(days=1)
if start > end:
logger.info('代码：{} 数据无需刷新'.format(code))
continue
try:
df = history_data_reader.read(code=code,
start=start,
end=end,
is_index=False)
# 按日期排序（升序）
df.sort_index(inplace=True)
except ValueError:
# 当开始日期大于结束日期时，触发值异常
logger.info('无法获取网页数据。代码：{}，开始日期：{}, 结束日期：{}'.format(
code, start, end))
continue
to_adds = _gen(code, df)
if len(to_adds):
sess.add_all(to_adds)
sess.commit()
logger.info('代码：{}, 开始日期：{}, 结束日期：{} 添加{}行'.format(
code, start, end, len(to_adds)))
log_to_db(StockDaily.__tablename__, True, len(to_adds),
Action.INSERT, code, start, end)
else:
logger.info('代码：{}，开始日期：{}, 结束日期：{} 无数据'.format(code, start, end))
log_to_db(StockDaily.__tablename__, True, 0, Action.INSERT, code,
start, end)
sess.close()
def flush_stock_issue():
"""刷新沒有上市日期的股票"""
sess = get_session()
f10 = THSF10()
query = sess.query(Stock).order_by(Stock.code).filter(
Stock.latest_status.is_(Status.in_trading))
codes = [x.code for x in query.all()]
for code in codes:
ipo = sess.query(Issue.A004_上市日期).filter(Issue.code == code).scalar()
if ipo is None:
issue = get_issue(f10, code)
if issue.A004_上市日期:
sess.add(issue)
sess.commit()
logger.info('刷新股票代码：{}'.format(code))
else:
logger.info('股票代码：{} 尚未上市'.format(code))
else:
logger.info('股票代码：{} 发行信息无需更新'.format(code))
# 关闭会话
sess.close()
# 退出浏览器
f10.browser.quit()
def flush_margin(init=False):
"""
刷新融资融券
说明：
如初始化，则从开始融资融券日期循环，否则自数据库内最后一日开始。
"""
sess = get_session()
if init:
start = MARGIN_START
else:
last_date = sess.query(func.max(Margin.date)).scalar()
if last_date is None:
start = MARGIN_START
else:
start = last_date + timedelta(days=1)
dates = sess.query(
TradingCalendar.date).filter(TradingCalendar.date >= start).filter(
TradingCalendar.is_trading == True).order_by(
TradingCalendar.date.asc()).all()
sess.close()
flush(dates)
def flush_global_news():
sess = get_session()
data = fetch_globalnews()
_insert(sess, data)
sess.close()
def main():
sess = get_session()
delete_old(sess)
sess.close()
