def check_trade_dates(self, lost_day, ts_code):
# 数据库缺少日期
check_dates = lost_day
# 获取最长时间段
start, end = tmdt.get_early_and_late_date(check_dates)
# 缺少的最长时间段交易日期数据
ts_trade_df = TsTool.instance().ts_get_trade_date(start, end)
# # 保存交易日期数据，等验证完数据以后，需要保存到数据库
# self.ts_trade_df = ts_trade_df
# 获取check_data数据库
# checkMysqlTool = CheckTradeDateMysqlTool()
check_db_df = CheckTradeDateMysqlTool.instance().get_data_from_db_by_date_list(ts_code, lost_day)
CheckTradeDateMysqlTool.instance().disconnect_db()
ts_trade_df_dates = list(ts_trade_df.cal_date)
check_db_df_dates = list(check_db_df.cal_date)
# need_check_dates 是checks数据库没有验证过的数据
need_check_dates = list(set(ts_trade_df_dates) - set(check_db_df_dates))
# 需要验证的时间段的基础数据在ts_trade_df中,其中包含is_opne==0的数据
basic_merge_trade_date_df = ts_trade_df[ts_trade_df['cal_date'].isin(need_check_dates)]
basic_merge_trade_date_df['ts_code'] = ts_code
# check_db_df中包含已经验证有数据的内容，这部分内容还有保存到oms数据库,需要添加到下载列表
# 本地check存在需要下载的数据
check_box_need_download_dates = list(check_db_df[check_db_df.has_data == 1].cal_date)
return basic_merge_trade_date_df, need_check_dates, check_box_need_download_dates
def ts_get_oms_price(self, ts_code, start_date, end_date):
# get tick data from tushare
# 一次最多获取7000条数据
# 假设从20140112-20130111，从20140112开始往后获取7000条数据，最多到20131129，中间跨度最多30个交易日
# 所以我们保守可以一次下载15天，最多不超过30天
'''
:param ts_code: symbol 000000.SZ
:param start_date: YYYYMMDD
:param end_date: YYYYMMDD
:return: df
'''
if self.check_token() == False:
return None
end_date = tmd.date_add_days(end_date, 1)
try:
df1 = ts.pro_bar(ts_code=ts_code,
start_date=start_date,
end_date=end_date,
asset='E',
freq='1min')
except Exception as msg:
print(msg)
return None
return df1
def step_repeat_download(self, ts_code, need_check_dates,
need_download_dates):
# new_download_list = tmdt.sort_date_list(need_check_dates + need_download_dates)
new_download_list = need_check_dates + need_download_dates
download_task_list = tmdt.incise_date_into_block(new_download_list, 25)
tmjs.saveTasksJsonFile(ts_code, download_task_list)
return download_task_list
def get_need_check_dates(self, lost_day, ts_code):
# 数据库缺少日期
check_dates = [d.strftime('%Y%m%d') for d in list(lost_day)]
# 获取最长时间段
se = tmdt.get_early_and_late_date(check_dates)
# 缺少的最长时间段交易日期数据
ts_trade_df = TsTool().ts_get_trade_date(se[0], se[1])
# # 保存交易日期数据，等验证完数据以后，需要保存到数据库
# self.ts_trade_df = ts_trade_df
# 获取check_data数据库
self.checkMysqlTool = CheckTradeDateMysqlTool()
check_db_df = self.checkMysqlTool.get_data_from_db_by_date_list(
ts_code, lost_day)
ts_trade_df_dates = list(ts_trade_df.cal_date)
check_db_df_dates = list(check_db_df.cal_date)
# need_check_dates 是checks数据库没有验证过的数据
self.need_check_dates = list(
set(ts_trade_df_dates) - set(check_db_df_dates))
# 需要验证的时间段的基础数据在ts_trade_df中,其中包含is_opne==0的数据
self.basic_merge_trade_date_df = ts_trade_df[
ts_trade_df['cal_date'].isin(self.need_check_dates)]
self.basic_merge_trade_date_df['ts_code'] = ts_code
# check_db_df中包含已经验证有数据的内容，这部分内容还有保存到oms数据库,需要添加到下载列表
need_download_dates = list(
check_db_df[check_db_df.has_data == 1].cal_date)
# 本地check存在需要下载的数据
self.need_download_dates = need_download_dates
return (self.basic_merge_trade_date_df, self.need_check_dates,
self.need_download_dates)
def get_lost_data_from_oms_db(self, start, end, ts_code):
# checked
'''
:param start: YYYYMMDD
:param end: YYYYMMDD
:param ts_code: str
:return: list
'''
OmsMysqlTool.instance().create_table(ts_code)
has_data_day = OmsMysqlTool.instance().get_exist_trade_date_index(ts_code, start, end)
everyday = tmdt.get_everyday(start, end)
lost_day = list(set(everyday) - set(has_data_day))
OmsMysqlTool.instance().disconnect_db()
lost_day.sort()
return lost_day
def get_lost_data_from_oms_db(self, start, end, ts_code):
'''
:param start: YYYYMMDD
:param end: YYYYMMDD
:param ts_code: str
:return: list
'''
self.omsMysqlTool = OmsMysqlTool()
self.omsMysqlTool.create_table(ts_code)
has_data_day = self.omsMysqlTool.get_exist_trade_date_index(
ts_code, start, end)
everyday = tmdt.get_everyday(start, end)
lost_day = list(set(everyday) - set(has_data_day))
self.omsMysqlTool.disconnect_db()
return lost_day