def check_integrity_daily_data_table(base_name: str, engine, pro,
skipping_year_start: []):
"""
:param skipping_year:
:param base_name:
:param engine:
:return:
"""
missing = {}
codes = get_ts_code_and_list_date(engine)
years = dbUtil.getYears()
table_range = {}
if (len(skipping_year_start) > 0):
for i in skipping_year_start:
if (years.get(i) != None):
years.pop(i)
for start_year in years.keys():
table_range[int(str(start_year) +
"0101")] = int(str(years.get(start_year)) + "1231")
for table_start in table_range.keys():
table_end = table_range.get(table_start)
for code in codes['ts_code']:
list_date = int(codes[codes['ts_code'] == code]['list_date'])
if list_date > table_end:
# if list date after end of table, there should be no lines here
pass
elif table_start <= list_date <= table_end:
check_integrity_worker(base_name=base_name,
table_name=base_name + "_" +
str(table_start)[0:4] + "_" +
str(table_end)[0:4],
code=code,
start_date=list_date,
end_date=table_end,
engine=engine)
else:
# list date < table_start
check_integrity_worker(base_name=base_name,
table_name=base_name + "_" +
str(table_start)[0:4] + "_" +
str(table_end)[0:4],
code=code,
start_date=table_start,
end_date=table_end,
engine=engine)
# create formatter
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
# add formatter to ch
ch.setFormatter(formatter)
fi.setFormatter(formatter)
# add ch to logger
logger.addHandler(ch)
logger.addHandler(fi)
engine = create_engine(config_service.getDefaultDB())
conn = engine.connect()
metadata = MetaData()
years = dbUtil.getYears()
for i in years.keys():
getTableMeta(i, metadata)
metadata.create_all(engine)
# df = get_ts_code_and_list_date(engine)
df = get_ts_code(engine)
# update_bulk_fina_indicator_by_period_and_ts_code(base_name='fina_indicator', engine=engine, pro=pro, codes=df, start_date='19950101',
# end_date=datetime.date.today().strftime("%Y%m%d"))
update_bulk_fina_indicator_by_ts_code_and_insert_by_year(
base_name='fina_indicator',
engine=engine,
pro=pro,
codes=df,
sharding_column='ann_date')