def incremental_get_fi(self):
"""
增量获取财务指标
:return:
"""
today = datetime.datetime.now().strftime('%Y%m%d')
latest_fi = stockdb.get_collection(
cons.S_FINANCIAL_INDICATOR).find().sort("end_date", -1).limit(1)
latest_fi = list(latest_fi)
if latest_fi:
df = ts_pro.stock_basic()
all_stocks = []
for idx, row in df.iterrows():
all_stocks.append(row['ts_code'])
latest_end_date = latest_fi[0]['end_date']
stocks_with_latest_fi = stockdb.get_collection(
cons.S_FINANCIAL_INDICATOR).find({"end_date": latest_end_date})
# 数据库里已有最新财报的股票
done_stocks = [s['ts_code'] for s in list(stocks_with_latest_fi)]
# 尚未有最新财报的股票
pending_stocks = []
for stock in all_stocks:
if stock not in done_stocks:
pending_stocks.append(stock)
for s in pending_stocks:
df = ts_pro.fina_indicator(ts_code=s, period=latest_end_date)
if df.empty:
logging.info('最新财报尚未披露: [{}]'.format(s))
else:
stockdb.get_collection(
cons.S_FINANCIAL_INDICATOR).insert_many(
json.loads(df.T.to_json()).values())
def get_history_basic(self, start, end):
"""
获取全部股票历史上某一时间段的基本面指标并保存
:return:
"""
start_day = datetime.datetime.strptime(start, '%Y%m%d')
end_day = datetime.datetime.strptime(end, '%Y%m%d')
while start_day <= end_day:
trade_date = start_day.strftime('%Y%m%d')
df = ts_pro.daily_basic(ts_code='', trade_date=trade_date)
if df.empty:
logging.info('Result is empty for {}'.format(trade_date))
else:
exist = self.check_if_exist(trade_date)
if exist:
logging.info(
'Data already exists for {}, will delete it before insert new data'
.format(trade_date))
stockdb.get_collection(cons.S_DAILY_BASIC).delete_many(
{"trade_date": trade_date})
rst = stockdb.get_collection(cons.S_DAILY_BASIC).insert_many(
json.loads(df.T.to_json()).values())
if rst:
logging.info(
'Insert daily basic for {} successfully'.format(
trade_date))
else:
logging.error(
'Insert daily basic for {} failed'.format(trade_date))
start_day = start_day + datetime.timedelta(days=1)
def get_all_fin_ind(self, start, end):
"""
获取一个时间段内全部股票的财务报表指标并保存
:param start: 开始时间
:param end: 结束时间
:return:
"""
stocks = ts_pro.stock_basic()
for idx, row in stocks.iterrows():
try:
ts_code = row['ts_code']
df = ts_pro.fina_indicator(ts_code=ts_code,
start_date=start,
end_date=end)
stockdb.get_collection(cons.S_FINANCIAL_INDICATOR).insert_many(
json.loads(df.T.to_json()).values())
result = 'succeed'
except Exception as e:
logging.error(
'Failed to get financial indicator of {}: {}'.format(
row['ts_code'], e))
result = 'failed'
finally:
logging.info('{}: {}'.format(row['ts_code'], result))
# 限频
time.sleep(1)
def printStocksInfo(stocks):
p = PrettyTable()
p.field_names = ['ts_code', 'name', 'industry', 'financial date', 'roe', 'pe_ttm']
for s in stocks:
rst = list(stockdb.get_collection(cons.S_STOCK_LIST).find({"ts_code": s}).limit(1))
if rst:
name = rst[0]['name']
industry = rst[0]['industry']
else:
name = 'N/A'
industry = 'N/A'
rst = list(stockdb.get_collection(cons.S_FINANCIAL_INDICATOR).find({'ts_code': s}).sort("end_date", -1).limit(1))
if rst:
financial_date = rst[0]['end_date']
roe = rst[0]['roe']
else:
financial_date = 'N/A'
roe = 'N/A'
rst = list(stockdb.get_collection(cons.S_DAILY_BASIC).find({'ts_code': s}).sort('trade_date', -1).limit(1))
if rst:
pe_ttm = rst[0]['pe_ttm']
else:
pe_ttm = 'N/A'
p.add_row([s, name, industry, financial_date, roe, pe_ttm])
print(p)
def profit_keep_growing(count, q_or_y='quarter', deduct=True):
"""
Get the stocks whose net profit keep growing for <count> years/quarters
:param count: 连续多少年或者季度
:param q_or_y: quarter or year, 年或者季度
:param deduct: 净利润是否扣非，True为扣非，False为不扣非
:return: stock list
"""
results = []
stocks = stockdb.get_collection(cons.S_STOCK_LIST).find(projection={
'_id': False,
'ts_code': True,
'name': True,
'industry': True
})
for s in stocks:
ts_code = s['ts_code']
if q_or_y == 'year':
hist_ind = stockdb.get_collection(
cons.S_FINANCIAL_INDICATOR).find(filter={
'ts_code': ts_code,
'end_date': {
"$regex": "1231"
}
}).sort('end_date', -1).limit(count)
else:
hist_ind = stockdb.get_collection(
cons.S_FINANCIAL_INDICATOR).find(filter={
'ts_code': ts_code
}).sort('end_date', -1).limit(count)
hist_ind = list(hist_ind)
if deduct:
result = isSortedAndPositive(hist_ind,
key='dt_netprofit_yoy',
reverse=True)
else:
result = isSortedAndPositive(hist_ind,
key='netprofit_yoy',
reverse=True)
if result:
results.append(s['ts_code'])
rs = stockdb.get_collection(cons.S_DAILY_BASIC).find({
"ts_code":
s['ts_code']
}).sort("trade_date", -1)
if rs:
pe_ttm = rs[0]['pe_ttm']
else:
pe_ttm = 'N/A'
print('{} {} {} {} {} {}'.format(s['ts_code'], s['name'],
s['industry'],
hist_ind[0]['end_date'],
hist_ind[0]['roe'], pe_ttm))
return results
def get_today_basic(self):
"""
获取全部股票每日的基本面指标并保存
:param date:
:return:
"""
now = datetime.datetime.now()
if now.hour < 17:
logging.info('今天的数据未完成更新，暂停获取')
return
today = now.strftime('%Y%m%d')
df = ts_pro.daily_basic(ts_code='', trade_date=today)
stockdb.get_collection(cons.S_DAILY_BASIC).insert_many(
json.loads(df.T.to_json()).values())
def check_if_exist(self, day):
"""
Check if the daily basic data already exists in db for a given day
:param day:
:return:
"""
rst = stockdb.get_collection(cons.S_DAILY_BASIC).count_documents(
{'trade_date': day})
if rst == 0 or rst is None:
return False
return True
def growing_for_years(indicator, years):
"""
某个指标连续几年递增
:param indicator: 财务指标
:param years: 连续多少年
:return: 满足条件的股票代码列表
"""
results = []
stocks = stockdb.get_collection(cons.S_STOCK_LIST).find(projection={'_id': False, 'ts_code': True, 'name': True, 'industry': True})
for s in stocks:
ts_code = s['ts_code']
hist_fi = stockdb.get_collection(cons.S_FINANCIAL_INDICATOR).find({'ts_code': ts_code, 'end_date': {"$regex":"1231"}}).sort('end_date', -1).limit(years)
hist_fi = list(hist_fi)
if hist_fi:
rst = is_sorted(hist_fi, key=indicator, reverse=True)
if rst:
# 大于0
rst = is_greater_than(hist_fi, key=indicator, value=0)
if rst:
print(s['ts_code'])
results.append(s['ts_code'])
return results
def refresh_stock_list(self):
stocks = ts_pro.stock_basic()
if not stocks.empty:
stockdb.get_collection(cons.S_STOCK_LIST).drop()
stockdb.get_collection(cons.S_STOCK_LIST).insert_many(
json.loads(stocks.T.to_json()).values())