def write_info_to_database(session: DBSession, rows):
rows = [format_row(info_json) for info_json in rows]
for row in rows:
rowDict = zip(header(), row)
rowDict = {a[0]: a[1] for a in rowDict}
session.merge(FundInfo1(**rowDict))
session.commit()
def write_info_to_database(session: DBSession, rows):
obj_list = []
for row in rows:
try:
row['基金代码'] = row['code']
row['净值日期'] = row['date']
# row['单位净值'] = float(row['单位净值'])
# row['累计净值'] = float(row['累计净值'])
del row['code']
del row['date']
obj_list.append(FundTrend(**row))
except Exception as e:
print(e)
session.bulk_save_objects(obj_list)
session.commit()
def crawl_info(self, response):
info = get_fund_info(pq(response.text))
if not info['_success']:
return {'_success': False}
session = DBSession()
session.merge(
FundInfoJson(code=info['代码'],
type='20201111',
json=json.dumps(info)))
session.merge(
FundProc(code=info['代码'],
info_status=0,
info_retry=0,
info_mdf=get_today_dt()))
session.commit()
session.close()
return {'_success': True}
def on_start(self):
pengding_list = get_pending_info_fund()
session = DBSession()
for fund_detail in pengding_list:
if fund_detail['info_obsolete']:
self.crawl(
get_fund_info_url(fund_detail['code']),
callback=self.crawl_info,
fetch_type='js',
age=24 * 60 * 60 * 3,
auto_recrawl=True,
retries=1,
)
def read_json_from_database(session: DBSession, code):
return [
json.loads(r.json)
for r in session.query(FundTrendJson).filter(FundTrendJson.code == code).all()
]
def write_info_to_database(session: DBSession, rows):
obj_list = []
for row in rows:
try:
row['基金代码'] = row['code']
row['净值日期'] = row['date']
# row['单位净值'] = float(row['单位净值'])
# row['累计净值'] = float(row['累计净值'])
del row['code']
del row['date']
obj_list.append(FundTrend(**row))
except Exception as e:
print(e)
session.bulk_save_objects(obj_list)
session.commit()
if __name__ == '__main__':
session = DBSession()
init_db()
fund_list = session.query(Fund).all()
for fund in tqdm(fund_list):
try:
fund_trends = read_json_from_database(session, code=fund.code)
write_info_to_database(session, fund_trends)
except Exception as e:
print(e)
'阶段涨幅_近1年': to_float(get_rise_data('区间回报', '近1年')),
'阶段涨幅_近2年': to_float(get_rise_data('区间回报', '近2年')),
'阶段涨幅_近3年': to_float(get_rise_data('区间回报', '近3年')),
'风险_年化夏普比率_1年': to_float(get_risk_data('年化夏普比率', '1年')),
'风险_年化夏普比率_2年': to_float(get_risk_data('年化夏普比率', '2年')),
'风险_年化夏普比率_3年': to_float(get_risk_data('年化夏普比率', '3年')),
}
for 基金信息 in tqdm(基金信息列表):
try:
行数据 = 构造基金基础信息(基金信息)
行数据 = 构造基金基础信息(基金信息)
数据库会话.merge(基金基础信息(**行数据))
数据库会话.commit()
except:
traceback.print_exc()
if __name__ == '__main__':
session = DBSession()
init_db()
info_jsons = read_info_json_from_database(session)
# 更新基金行业(session, info_jsons)
# 更新基金经理(session, info_jsons)
# 更新基金交易风格(session, info_jsons)
更新基金基础信息(session, info_jsons)
# abc = to_float('-123.abc.45亿%')
# print(abc)
# session.close()