def get_stock_basic_one(self):
now = time.strftime("%Y-%m-%d %H:%M:%S")
paras = {
"data_type": "stock_basic",
"data_name": "股票基本信息",
"data_source": "tusharepro",
"collect_start_time": now,
"collect_status": "R"
}
log_id = mysql_script.record_log(paras, flag='before')
self.mysql.exec(
mysql_script.truncate_table_common.format("stock_basic"))
self.tshelper.stock_basic_mysql_one("L")
self.tshelper.stock_basic_mysql_one("D")
self.tshelper.stock_basic_mysql_one("P")
today = datetime.datetime.now()
end_date = today.strftime('%Y-%m-%d')
end_date = datetime.datetime.strptime(end_date, '%Y-%m-%d').date()
now = time.strftime("%Y-%m-%d %H:%M:%S")
paras = {
"data_end_date": str(end_date),
"collect_end_time": now,
"collect_log": f"完成股票基础数据更新",
"collect_status": "S",
"id": log_id
}
mysql_script.record_log(paras, flag='after')
print("--------tusharepro股票基本信息更新完成------------")
def get_stock_history_pro(self):
sql = "select ts_code from stock_basic"
stock_basic_df = pd.read_sql(sql,
self.engine,
index_col=None,
coerce_float=True,
params=None,
parse_dates=None,
columns=None,
chunksize=None)
now = time.strftime("%Y-%m-%d %H:%M:%S")
paras = {
"data_type": "tushare_history_pro",
"data_name": "tusharepro交易数据",
"data_source": "tusharepro",
"collect_start_time": now,
"collect_status": "R"
}
log_id = mysql_script.record_log(paras, flag='before')
for ts_code in stock_basic_df['ts_code']:
time.sleep(1)
self.tshelper.get_history_pro(ts_code, "19880101", "20081231")
self.tshelper.get_history_pro(ts_code, "20090101", "20200411")
now = time.strftime("%Y-%m-%d %H:%M:%S")
paras = {
"data_end_date": "2020-04-12",
"collect_end_time": now,
"collect_log": "一次完成到2020-04-12的所有交易数据",
"collect_status": "S",
"id": log_id
}
mysql_script.record_log(paras, flag='after')
def update_block_data(self):
data_type = 'ths_block_member'
today = datetime.datetime.now()
today_str = today.strftime('%Y-%m-%d')
last_data_end_date_record = self.mysql.select(
"select max(data_end_date) from collect_log where data_type = %s",
data_type)
last_data_end_date = last_data_end_date_record[0][
0] # last_data_end_date是日期类型
if last_data_end_date == None:
last_data_end_date = datatime_util.str2date(
today_str) - datetime.timedelta(days=1)
end_date = datetime.datetime.strptime(
today_str, '%Y-%m-%d').date() # end_date是日期类型
if last_data_end_date >= end_date: # 日志记录的
print("今天的数据已经更新完成，不必重复执行!")
return
result1 = self.thsHelper.get_block_gn()
result2 = self.thsHelper.get_block_dy()
result3 = self.thsHelper.get_block_zjhhy()
result4 = self.thsHelper.get_block_thshy()
block_count = 0
member_count = 0
if result1 is not None:
block_count = block_count + result1[0]
member_count = member_count + result1[1]
if result2 is not None:
block_count = block_count + result2[0]
member_count = member_count + result2[1]
if result3 is not None:
block_count = block_count + result3[0]
member_count = member_count + result3[1]
if result4 is not None:
block_count = block_count + result4[0]
member_count = member_count + result4[1]
if block_count != 0 or member_count != 0:
now = time.strftime("%Y-%m-%d %H:%M:%S") # now是字符串
today = time.strftime("%Y-%m-%d")
paras = {
"data_type": data_type,
"data_name": "ths板块成分股",
"data_source": "ths",
"collect_start_time": now,
"data_end_date": today,
"collect_end_time": now,
"collect_log":
f"完成{data_type}的数据采集,更新板块信息{block_count}条，更新板块成分数据{block_count}条",
"collect_status": "S"
}
print(paras)
mysql_script.record_log(paras)
else:
print("=======没有要更新的板块数据========")
def update_day_attach(self, trade_date=""):
data_type = 'ths_day_attach'
today = datetime.datetime.now()
today_str = today.strftime('%Y-%m-%d')
if len(trade_date) == 0:
trade_date = time.strftime("%Y%m%d") # 如果没指定交易日期则用当天日期
last_data_end_date_record = self.mysql.select(
"select max(data_end_date) from collect_log where data_type = %s",
data_type)
last_data_end_date = last_data_end_date_record[0][
0] # last_data_end_date是日期类型
if last_data_end_date == None:
last_data_end_date = datatime_util.str2date(
today_str) - datetime.timedelta(days=1)
end_date = datetime.datetime.strptime(
today_str, '%Y-%m-%d').date() # end_date是日期类型
if last_data_end_date >= end_date: # 日志记录的
print("今天的数据已经更新完成，不必重复执行!")
return
now = time.strftime("%Y-%m-%d %H:%M:%S") # now是字符串
paras = {
"data_type": data_type,
"data_name": "股票日线附加数据",
"data_source": "ths",
"collect_start_time": now,
"collect_status": "R"
}
log_id = mysql_script.record_log(paras, flag='before')
result = self.thsHelper.get_day_attach(trade_date)
now = time.strftime("%Y-%m-%d %H:%M:%S")
paras = {
"data_end_date": str(end_date),
"collect_end_time": now,
"collect_log": f"完成{end_date}的数据采集,更新股票{result}条",
"collect_status": "S",
"id": log_id
}
mysql_script.record_log(paras, flag='after')
print("--------更新每日股票附加数据完成------------")
def test_record_log(self):
now = time.strftime("%Y-%m-%d %H:%M:%S")
print(now)
paras = {
"data_type": "tushare_history_all",
"data_name": "tushare交易数据，两个接口合并",
"data_source": "tusharepro+tushare",
"collect_start_time": now,
"collect_status": "R"
}
paras = {
"data_end_date": "2020-04-03",
"collect_end_time": now,
"collect_log": "一次完成2017-10-09到2020-04-03的所有交易数据",
"collect_status": "S",
"id": 1
}
mysql_script.record_log(paras, False)
def get_init_minite1_simple(self):
last_data_end_date = self.mysql.select(
"select max(data_end_date) from collect_log where data_type = %s",
'tdx_minline1_simple')
if last_data_end_date[0][0] == None:
start_date = datatime_util.str2date('2020-01-13')
else:
start_date = last_data_end_date[0][0] + datetime.timedelta(days=1)
today = datetime.datetime.now()
end_date = today.strftime('%Y-%m-%d')
end_date = datetime.datetime.strptime(end_date, '%Y-%m-%d').date()
if start_date > end_date:
print("今天的数据已经更新完成，不必重复执行!")
return
now = time.strftime("%Y-%m-%d %H:%M:%S")
paras = {
"data_type": "tdx_minline1_simple",
"data_name": "tdx1分钟数据",
"data_source": "tdx_local",
"collect_start_time": now,
"collect_status": "R"
}
log_id = mysql_script.record_log(paras, flag='before')
file_util.traversal_dir(
config.tdx_local_sh_minline1,
self.tdx_local_helper.read_tdx_local_minline_simple)
file_util.traversal_dir(
config.tdx_local_sz_minline1,
self.tdx_local_helper.read_tdx_local_minline_simple)
now = time.strftime("%Y-%m-%d %H:%M:%S")
paras = {
"data_end_date": str(end_date),
"collect_end_time": now,
"collect_log": f"一次性完成从{start_date}到{end_date}的数据采集",
"collect_status": "S",
"id": log_id
}
mysql_script.record_log(paras, flag='after')
print("--------tdx1分钟数据初始化完成------------")
def get_stock_history(self):
#last_data_end_date = self.mysql.select("select max(data_end_date) from collect_log where data_type = %s",'tushare_history_all')
#start_date = last_data_end_date[0][0] + datetime.timedelta(days=1)
today = datetime.datetime.now()
end_date = today.strftime('%Y-%m-%d')
end_date = datetime.datetime.strptime(end_date, '%Y-%m-%d').date()
now = time.strftime("%Y-%m-%d %H:%M:%S")
paras = {
"data_type": "tushare_history_all",
"data_name": "tushare交易数据，两个接口合并",
"data_source": "tusharepro+tushare",
"collect_start_time": now,
"collect_status": "R"
}
log_id = mysql_script.record_log(paras, flag='before')
sql = "select ts_code from stock_basic"
stock_basic_df = pd.read_sql(sql,
self.engine,
index_col=None,
coerce_float=True,
params=None,
parse_dates=None,
columns=None,
chunksize=None)
for ts_code in stock_basic_df['ts_code']:
print(ts_code)
##filename = config.tushare_csv_home + "day/" + ts_code + ".csv"
##if os.path.isfile(filename): continue
time.sleep(0.8)
self.tshelper.get_history_phase(ts_code)
start_date = "2017-10-11"
now = time.strftime("%Y-%m-%d %H:%M:%S")
paras = {
"data_end_date": str(end_date),
"collect_end_time": now,
"collect_log": f"完成从{start_date}到{end_date}的数据采集",
"collect_status": "S",
"id": log_id
}
mysql_script.record_log(paras, flag='after')
def get_history_pro_by_date(self):
# start_date = self.mysql.select("select max(data_end_date) from collect_log where data_type = 'tushare_history_all'")
# start_date = self.mysql.select("select * from collect_log where data_type = %s",['tushare_history_all'])
# start_date = self.mysql.select("select * from collect_log ")
last_data_end_date = self.mysql.select(
"select max(data_end_date) from collect_log where data_type = %s",
'tushare_history_pro')
start_date = last_data_end_date[0][0] + datetime.timedelta(days=1)
start_date_bak = start_date
today = datetime.datetime.now()
end_date = today.strftime('%Y-%m-%d')
end_date = datetime.datetime.strptime(end_date, '%Y-%m-%d').date()
if start_date > end_date:
print("今天的数据已经更新完成，不必重复执行!")
return
now = time.strftime("%Y-%m-%d %H:%M:%S")
paras = {
"data_type": "tushare_history_pro",
"data_name": "tusharepro交易数据",
"data_source": "tusharepro",
"collect_start_time": now,
"collect_status": "R"
}
log_id = mysql_script.record_log(paras, flag='before')
while start_date <= end_date:
start_date_pro = start_date.strftime('%Y%m%d')
# print(start_date_pro)
self.tshelper.get_history_pro_by_date(start_date_pro)
start_date = start_date + datetime.timedelta(days=1)
now = time.strftime("%Y-%m-%d %H:%M:%S")
paras = {
"data_end_date": str(end_date),
"collect_end_time": now,
"collect_log": f"完成从{start_date_bak}到{end_date}的数据追加",
"collect_status": "S",
"id": log_id
}
mysql_script.record_log(paras, flag='after')
print("--------tusharepro日数据追加完成-----------------")
def get_day_index_all(self):
last_data_end_date = self.mysql.select(
"select max(data_end_date) from collect_log where data_type = %s",
'tushare_index')
if last_data_end_date[0][0] == None:
start_date = datatime_util.str2date("2017-10-16")
else:
start_date = last_data_end_date[0][0] + datetime.timedelta(days=1)
today = datetime.datetime.now()
end_date = today.strftime('%Y-%m-%d')
end_date = datetime.datetime.strptime(end_date, '%Y-%m-%d').date()
if start_date > end_date:
print("今天的数据已经更新完成，不必重复执行!")
return
now = time.strftime("%Y-%m-%d %H:%M:%S")
paras = {
"data_type": "tushare_index",
"data_name": "tushare大盘指数",
"data_source": "tushare",
"collect_start_time": now,
"collect_status": "R"
}
log_id = mysql_script.record_log(paras, flag='before')
self.tshelper.get_day_index_all(start_date, end_date)
now = time.strftime("%Y-%m-%d %H:%M:%S")
paras = {
"data_end_date": str(end_date),
"collect_end_time": now,
"collect_log": f"完成从{start_date}到{end_date}的数据采集",
"collect_status": "S",
"id": log_id
}
mysql_script.record_log(paras, flag='after')
print("--------tushare_index大盘指数追加完成------------")
def update_block_member(self):
data_type = 'tdx_block_member'
today = datetime.datetime.now()
today_str = today.strftime('%Y-%m-%d')
last_data_end_date_record = self.mysql.select(
"select max(data_end_date) from collect_log where data_type = %s",
data_type)
last_data_end_date = last_data_end_date_record[0][
0] # last_data_end_date是日期类型
if last_data_end_date == None:
last_data_end_date = datatime_util.str2date(
today_str) - datetime.timedelta(days=1)
end_date = datetime.datetime.strptime(
today_str, '%Y-%m-%d').date() # end_date是日期类型
if last_data_end_date >= end_date: # 日志记录的
print("今天的数据已经更新完成，不必重复执行!")
return
result = self.tdxhelper.update_block_member()
if result is not None:
now = time.strftime("%Y-%m-%d %H:%M:%S") # now是字符串
today = time.strftime("%Y-%m-%d")
paras = {
"data_type": data_type,
"data_name": "tdx板块成分股",
"data_source": "tdx",
"collect_start_time": now,
"data_end_date": today,
"collect_end_time": now,
"collect_log":
f"完成{data_type}的数据采集,更新板块信息{result[0]}条，更新板块成分数据{result[1]}条",
"collect_status": "S"
}
print(paras)
mysql_script.record_log(paras)
else:
print("=======没有要更新的板块成分股========")
def append_minite1_all(self,
default_start_date="",
file_modify_time_flag=""):
data_type = 'tdx_minline1'
today = datetime.datetime.now()
today_str = today.strftime('%Y-%m-%d')
last_data_end_date_record = self.mysql.select(
"select max(data_end_date) from collect_log where data_type = %s",
data_type)
last_data_end_date = last_data_end_date_record[0][
0] #last_data_end_date是日期类型
if last_data_end_date == None:
last_data_end_date = datatime_util.str2date(
default_start_date) - datetime.timedelta(days=1)
end_date = datetime.datetime.strptime(
today_str, '%Y-%m-%d').date() #end_date是日期类型
if last_data_end_date >= end_date: #日志记录的
print("今天的数据已经更新完成，不必重复执行!")
return
records = self.mysql.select(
"select ts_code from stock_basic where list_status='L'")
if len(records) == 0: return
now = time.strftime("%Y-%m-%d %H:%M:%S") #now是字符串
paras = {
"data_type": data_type,
"data_name": "tdx1分钟数据",
"data_source": "tdx",
"collect_start_time": now,
"collect_status": "R"
}
log_id = mysql_script.record_log(paras, flag='before')
market = {'SZ': 0, 'SH': 1}
for stock in records:
ts_code = stock[0]
filename = config.tdx_csv_minline1_all + ts_code + ".csv"
if os.path.isfile(filename):
#获取文件更新时间，更新时间在某个时间之后的忽略
if len(file_modify_time_flag) > 0:
mtime = os.stat(filename).st_mtime
file_modify_time = time.strftime(
'%Y-%m-%d',
time.localtime(mtime)) #file_modify_time是字符串
if file_modify_time > file_modify_time_flag: continue
#获取csv文件中数据的最后日期，如果该日期大于或等于end_date则说明数据已经完成采集，忽略该文件；
#如果csv文件中数据的最后日期大于从数据库日志中获取的上次更新日期last_data_end_date，则修改last_data_end_date为文件中的最后日期
df = pd.read_csv(filename)
last_file_end_date = df.max()[
'trade_date'] #last_file_end_date是int类型
if last_file_end_date >= int(str(end_date).replace('-', '')):
continue
if last_file_end_date >= int(
str(last_data_end_date).replace('-', '')):
start_date = datetime.date(
year=int((str(last_file_end_date))[0:4]),
month=int((str(last_file_end_date))[4:6]),
day=int((str(last_file_end_date)
)[6:8])) + datetime.timedelta(days=1)
else:
start_date = last_data_end_date + datetime.timedelta(
days=1)
else:
start_date = last_data_end_date + datetime.timedelta(days=1)
self.tdxhelper.get_minute1_data(7, market[ts_code[7:9]],
ts_code[0:6], str(start_date),
str(end_date))
now = time.strftime("%Y-%m-%d %H:%M:%S")
paras = {
"data_end_date": str(end_date),
"collect_end_time": now,
"collect_log": f"完成{end_date}的数据采集",
"collect_status": "S",
"id": log_id
}
mysql_script.record_log(paras, flag='after')
print("--------追加tdx1分钟数据初始化完成------------")
