def validate_data():
db = mongodb_utility.connect_db(db_name="list_company")
sz_df = mongodb_utility.export2df(database=db, collection="sz_stock")
sh_df = mongodb_utility.export2df(database=db, collection="sh_stock")
fzb_df = mongodb_utility.export2df(database=db, collection="fzb")
# llb_df = mongodb_utility.export2df(database=db,collection="llb")
# lrb_df = mongodb_utility.export2df(database=db,collection="lrb")
# missing_fzb_sz = sz_df[~sz_df['_id'].isin(fzb_df['机构ID'])]
missing_fzb_sh = sh_df[~sh_df['_id'].isin(fzb_df['机构ID'])]
missing_sh_a = missing_fzb_sh[pd.Series(
missing_fzb_sh['A股上市日期']).notnull()][["_id", "A股上市日期"]]
# print(missing_sh_a["A股上市日期"])
# missing_sh_a["A股上市日期"] = missing_sh_a["A股上市日期"].apply(lambda x: x.strip())
missing_sh_a["A股上市日期"] = missing_sh_a["A股上市日期"].apply(
lambda x: str(x.strip())[0:4])
missing_sh_2017 = missing_sh_a[missing_sh_a["A股上市日期"] < "2017"].copy()
print(missing_sh_2017.count())
missing_sh_2017["max_year"] = "2016"
missing_sh_2017["A股上市日期"] = "2016"
download_list = missing_sh_2017.values.tolist()
"""download missing data
"""
url = 'http://www.cninfo.com.cn/cninfo-new/data/download'
# frDownloader.getFR(url=url, stock_list=download_list)
# missing_fzb_sh.to_csv("missing_fzb_sh.csv")
# print(missing_fzb_sh.count())
return
def validate_stock(data_base=None, collection="fzb", stock_table="sh_stock"):
# db = mongodb_utility.connect_db(db_name="list_company")
stock_df = mongodb_utility.export2df(database=data_base,
collection=stock_table)
report_df = mongodb_utility.export2df(database=data_base,
collection=collection)
missing_stock = stock_df[~stock_df['_id'].isin(report_df['机构ID'])]
# exclude B stock
missing_stock_a = missing_stock[pd.Series(
missing_stock['A股上市日期']).notnull()][["_id", "A股上市日期"]]
missing_stock_a["A股上市日期"] = missing_stock_a["A股上市日期"].apply(
lambda x: str(x.strip()))
return missing_stock_a
def validate_reports_download(table1='fzb', table2='lrb', auto_download=False):
db = mongodb_utility.connect_db(db_name="list_company")
df1 = mongodb_utility.export2df(database=db, collection=table1)
df2 = mongodb_utility.export2df(database=db, collection=table2)
t1_notin_t2 = df1[~df1['_id'].isin(df2['_id'])][['_id', '机构ID', '报告年度']]
t1_notin_t2['报告年度'] = t1_notin_t2['报告年度'].apply(
lambda x: str(x.strip())[0:4])
t1_notin_t2 = t1_notin_t2.drop_duplicates(['机构ID', '报告年度'])
missing_t2 = t1_notin_t2[['机构ID', '报告年度']].copy()
missing_t2['max_year'] = missing_t2['报告年度']
download_list = missing_t2.values.tolist()
print(len(download_list))
if auto_download:
url = 'http://www.cninfo.com.cn/cninfo-new/data/download'
frDownloader.getFR(url=url, stock_list=download_list, types=[table2])
return download_list
def update_sh_b_data(update_year="2016"):
url = 'http://www.cninfo.com.cn/cninfo-new/data/download'
"""get stock list"""
db = mongodb_utility.connect_db(db_name="list_company")
sh_df = mongodb_utility.export2df(database=db, collection="sh_stock")
"""download sh B stock data"""
sh_df_b = sh_df[Series(sh_df['A股上市日期']).isnull()][["_id"]]
sh_df_b["_id"] = sh_df_b["_id"].apply(lambda x: str(int(x)))
sh_df_b["min_year"] = update_year
sh_df_b["max_year"] = update_year
shb_list = sh_df_b.values.tolist()
frDownloader.getFR(url=url, stock_list=shb_list)
def validate_bw_reports():
db = mongodb_utility.connect_db(db_name="list_company")
fzb_df = mongodb_utility.export2df(database=db, collection="fzb")
llb_df = mongodb_utility.export2df(database=db, collection="llb")
lrb_df = mongodb_utility.export2df(database=db, collection="lrb")
missing_fzb_in_lrb = fzb_df[~fzb_df['_id'].isin(lrb_df['_id'])][[
'_id', '机构ID', '报告年度'
]]
print(missing_fzb_in_lrb)
missing_fzb_in_lrb.to_csv('missing_fzb_lrb.csv')
missing_fzb_in_llb = fzb_df[~fzb_df['_id'].isin(llb_df['_id'])][[
'_id', '机构ID', '报告年度'
]]
missing_fzb_in_llb.to_csv('missing_fzb_llb.csv')
missing_lrb_in_fzb = lrb_df[~lrb_df['_id'].isin(fzb_df['_id'])][[
'_id', '机构ID', '报告年度'
]]
missing_lrb_in_fzb.to_csv('missing_lrb_fzb.csv')
missing_llb_in_fzb = llb_df[~llb_df['_id'].isin(fzb_df['_id'])][[
'_id', '机构ID', '报告年度'
]]
missing_llb_in_fzb.to_csv('missing_llb_fzb.csv')
missing_lrb_in_llb = lrb_df[~lrb_df['_id'].isin(llb_df['_id'])][[
'_id', '机构ID', '报告年度'
]]
missing_lrb_in_llb.to_csv('missing_lrb_llb.csv')
missing_llb_in_lrb = llb_df[~llb_df['_id'].isin(lrb_df['_id'])][[
'_id', '机构ID', '报告年度'
]]
missing_llb_in_lrb.to_csv('missing_llb_lrb.csv')