def get_source_data(file_path: Path, freq: str) -> pd.DataFrame:
if not file_path.exists():
logger.info(f"{str(file_path)} is not exists!")
return pd.DataFrame()
if file_path.parent.parent.name == "ds":
if freq == "day":
reader = TdxExHqDailyBarReader()
else:
reader = TdxLCMinBarReader()
else:
# Todo 对股票数据进行后复权
if freq == "day":
reader = TdxDailyBarReader()
else:
reader = TdxLCMinBarReader()
try:
df = reader.get_df(str(file_path))
except:
logger.warning(f"Cant read {str(file_path)} ")
return pd.DataFrame()
df = df.reset_index()
df["date"] = df["date"].astype(str).astype(np.datetime64)
return df
def tdxread5m(market, code): # .5 minutes
market = str(market)
code = str(code)
reader = TdxLCMinBarReader()
#reader = TdxDailyBarReader()
df = reader.get_df(panfu + market + '/fzline/' + market + code + '.lc5')
return df
def __init__(self):
self.day_reader = TdxDailyBarReader()
self.minline_reader = TdxLCMinBarReader()
self.block_reader = BlockReader()
# pandas数据显示设置
pd.set_option('display.max_columns', None) # 显示所有列
def _get_source_data(self, file_path: Path) -> pd.DataFrame:
if file_path.parent.parent.name == "ds":
if self.freq == "day":
reader = TdxExHqDailyBarReader()
else:
reader = TdxLCMinBarReader()
else:
if self.freq == "day":
reader = TdxDailyBarReader()
else:
reader = TdxLCMinBarReader()
try:
df = reader.get_df(str(file_path))
except:
logger.warning(f"Cant read {str(file_path)} ")
return pd.DataFrame()
instrument = self.tdx_files_info[self.tdx_files_info["file_path"] ==
file_path]['instrument'].iloc[0]
if instrument in ['future', 'option']:
df.rename(columns={
'amount': "position",
"jiesuan": "settle"
},
inplace=True)
if "hk_stock_amount" in df.columns:
del df["hk_stock_amount"]
df = df.reset_index()
df[self.date_field_name] = df[self.date_field_name].astype(str).astype(
np.datetime64)
return df
def tdxreadm(code): # 1 minutes
market = marketcode(code)
code = str(code)
reader = TdxLCMinBarReader()
#reader = TdxDailyBarReader()
df = reader.get_df(panfu + '\\' + market + '\\minline\\' + market + code +
'.lc1')
return df
def fzline(self, symbol=None):
'''
获取5分钟线
:param symbol:
:return: pd.dataFrame or None
'''
symbol = self.find_path(symbol, subdir='fzline', ext=['lc5', '5'])
reader = TdxLCMinBarReader()
if symbol is not None:
return reader.get_df(symbol)
return None
def main(input, output, datatype):
"""
通达信数据文件读取
"""
if datatype == 'daily':
reader = TdxDailyBarReader()
elif datatype == 'ex_daily':
reader = TdxExHqDailyBarReader()
elif datatype == 'lc':
reader = TdxLCMinBarReader()
elif datatype == 'gbbq':
reader = GbbqReader()
elif datatype == 'block':
reader = BlockReader()
else:
reader = TdxMinBarReader()
try:
df = reader.get_df(input)
if (output):
click.echo("写入到文件 : " + output)
df.to_csv(output)
else:
print(df)
except Exception as e:
print(str(e))
def main(input, output, datatype):
"""
通达信数据文件读取
"""
if datatype == 'daily':
reader = TdxDailyBarReader()
elif datatype == 'ex_daily':
reader = TdxExHqDailyBarReader()
elif datatype == 'lc':
reader = TdxLCMinBarReader()
elif datatype == 'gbbq':
reader = GbbqReader()
elif datatype == 'block':
reader = BlockReader()
elif datatype == 'customblock':
reader = CustomerBlockReader()
elif datatype == 'history_financial' or datatype == 'hf':
reader = HistoryFinancialReader()
else:
reader = TdxMinBarReader()
try:
df = reader.get_df(input)
if output:
click.echo("写入到文件 : " + output)
df.to_csv(output)
else:
print(df)
except Exception as e:
print(str(e))
def minute(self, symbol=None, suffix=1):
'''
获取1,5分钟线
:param suffix:
:param symbol:
:return: pd.dataFrame or None
'''
subdir = 'fzline' if str(suffix) == '5' else 'minline'
suffix = ['lc5', '5'] if str(suffix) == '5' else ['lc1', '1']
symbol = self.find_path(symbol, subdir=subdir, ext=suffix)
reader = TdxLCMinBarReader()
if symbol is not None:
return reader.get_df(symbol)
return None
class TdxLocalHelper:
BlockReader_TYPE_FLAT = 0
BlockReader_TYPE_GROUP = 1
def __init__(self):
self.day_reader = TdxDailyBarReader()
self.minline_reader = TdxLCMinBarReader()
self.block_reader = BlockReader()
# pandas数据显示设置
pd.set_option('display.max_columns', None) # 显示所有列
#pd.set_option('display.max_rows', None) # 显示所有行
# 解析日线文件数据
#返回值：date:open,high,low,close,amount,volume
def read_tdx_local_day(self):
#df = reader.get_df(config.tdx_local_sz_day + "sz000001.day") sh000001.day
df = self.day_reader.get_df(config.tdx_local_sh_day + "sh000001.day")
print(df)
# 批量格式化日期字段，去掉分隔符
def format_date(self, date):
return ((str(date))[0:10]).replace('-', '')
# 解析1分钟和5分钟数据
#返回值：date: open,high,low,close,amount,volume
#csv格式：code,ts_code,trade_date(缩写）,trade_time,time_index,open,high,low,close,amount,volume
def read_tdx_local_minline_all(self, full_path,filename=""):
#df = reader.get_df(config.tdx_local_sz_minline1 + "sz399001.lc1")
#df = reader.get_df(config.tdx_local_sz_minline5 + "sz399001.lc5")
#df = self.minline_reader.get_df(config.tdx_local_sh_minline1 + "sh600300.lc1")
code = filename[2:8]
ts_code = code + "." + filename[0:2].upper()
df = self.minline_reader.get_df(full_path)
df.insert(0, 'trade_time', df.index)
df.insert(0, 'trade_date', df.index.floor('D'))
df.insert(0, 'ts_code', ts_code)
df.insert(0, 'code', code)
df.reset_index(drop=True,inplace=True) #参考：https://zhuanlan.zhihu.com/p/110819220?from_voters_page=true
df.insert(4, 'time_index', df.index)
#df['trade_time'] = pd.to_datetime(df['trade_time'], infer_datetime_format=True).dt.normalize() # strftime('%m/%d/%Y') # format='%m/%d/%Y').dt.date
#df['trade_time'] = df['trade_time'].apply(lambda x: x.strftime('%H:%M:%S'))
df['trade_time'] = pd.to_datetime(df['trade_time'], format='%H:%M:%S').dt.strftime('%H:%M:%S')
df['time_index'] = df['trade_time'].apply(lambda x: datatime_util.stockTradeTime2Index(x))
df['trade_date'] = df['trade_date'].apply(lambda x: self.format_date(x))
df['open'] = df['open'].apply(lambda x: round(x,2))
df['high'] = df['high'].apply(lambda x: round(x, 2))
df['low'] = df['low'].apply(lambda x: round(x, 2))
df['close'] = df['close'].apply(lambda x: round(x, 2))
csv_filename = config.tdx_csv_minline1_all + ts_code + ".csv"
if os.path.isfile(csv_filename):
os.remove(csv_filename)
df.to_csv(csv_filename, index=False, mode='w', header=True, sep=',', encoding="utf_8_sig")
else:
df.to_csv(csv_filename, index=False, mode='w', header=True, sep=',', encoding="utf_8_sig")
# 解析1分钟和5分钟数据，输出简单形式，只返回price和vol
# 返回值：date: open,high,low,close,amount,volume
# csv格式：code,ts_code,trade_date(缩写）,trade_time,time_index,price,volume
def read_tdx_local_minline_simple(self, full_path, filename=""):
# df = reader.get_df(config.tdx_local_sz_minline1 + "sz399001.lc1")
# df = reader.get_df(config.tdx_local_sz_minline5 + "sz399001.lc5")
# df = self.minline_reader.get_df(config.tdx_local_sh_minline1 + "sh600300.lc1")
code = filename[2:8]
ts_code = code + "." + filename[0:2].upper()
df = self.minline_reader.get_df(full_path)
df.insert(0, 'trade_time', df.index)
df.insert(0, 'trade_date', df.index.floor('D'))
df.insert(0, 'ts_code', ts_code)
df.insert(0, 'code', code)
df.reset_index(drop=True, inplace=True) # 参考：https://zhuanlan.zhihu.com/p/110819220?from_voters_page=true
df.insert(4, 'time_index', df.index)
# df['trade_time'] = pd.to_datetime(df['trade_time'], infer_datetime_format=True).dt.normalize() # strftime('%m/%d/%Y') # format='%m/%d/%Y').dt.date
# df['trade_time'] = df['trade_time'].apply(lambda x: x.strftime('%H:%M:%S'))
df['trade_time'] = pd.to_datetime(df['trade_time'], format='%H:%M:%S').dt.strftime('%H:%M:%S')
df['time_index'] = df['trade_time'].apply(lambda x: datatime_util.stockTradeTime2Index(x))
df['trade_date'] = df['trade_date'].apply(lambda x: self.format_date(x))
#df['open'] = df['open'].apply(lambda x: round(x, 2))
#df['high'] = df['high'].apply(lambda x: round(x, 2))
#df['low'] = df['low'].apply(lambda x: round(x, 2))
df['close'] = df['close'].apply(lambda x: round(x, 2))
df.rename(columns={'close': 'price'}, inplace=True)
df.drop(['open', 'high', 'low', 'amount'], axis=1, inplace=True)
csv_filename = config.tdx_csv_minline1_simple + ts_code + ".csv"
if os.path.isfile(csv_filename):
os.remove(csv_filename)
df.to_csv(csv_filename, index=False, mode='w', header=True, sep=',', encoding="utf_8_sig")
else:
df.to_csv(csv_filename, index=False, mode='w', header=True, sep=',', encoding="utf_8_sig")
# 解析板块数据
#扁平格式返回值：blockname,block_type,code_index,code
def read_tdx_local_block(self):
##指数板块 风格板块 概念板块 一般板块
block_filename = ["block_zs.dat", "block_fg.dat", "block_gn.dat", "block.dat"]
for block in block_filename:
df = self.block_reader.get_df(config.tdx_local_block + block) # 默认扁平格式
df_group = self.block_reader.get_df(config.tdx_local_block + block, self.BlockReader_TYPE_GROUP) #分组格式
filename = config.tdx_csv_block + block[0:-4] + ".csv"
filename_group = config.tdx_csv_block + block[0:-4] + "_group" + ".csv"
if os.path.isfile(filename):
df.to_csv(filename, index=False, mode='a', header=False, sep=',', encoding="utf_8_sig")
df_group.to_csv(filename_group, index=False, mode='a', header=False, sep=',', encoding="utf_8_sig")
else:
df.to_csv(filename, index=False, mode='w', header=True, sep=',', encoding="utf_8_sig")
df_group.to_csv(filename_group, index=False, mode='w', header=True, sep=',', encoding="utf_8_sig")
# 解析分时图数据
def read_tdx_local_fst(self):
reader = TdxMinBarReader() #这个reader不能解析分时图文件
# df = reader.get_df(config.tdx_local_sz_minline + "sz399001.lc1")
df = reader.get_df(config.tdx_local_fst + "sh20200417.tfz")
print(df)
# 解析分时图文件,没有解析出来
"""
通达信的zst的数据记录是每6508个字节为一天的数据,
每26个字节为一个分钟的记录,这26个字节是这样分配的,
时间占两个字节化为十进制为570的话表示9:30分(570/60=9.5)
下一个是占四个字节的叫现价,
再下四个字节叫均价,
另外还有两个字节为该分钟成交量(现在有可能已经改为四个字节),
剩下的14个字节是预留的,
"""
def parse_fst_file(self):
full_path = config.tdx_local_fst + "sh20200417.tfz"
filesize = os.path.getsize(full_path) # 文件字节数
print("filesize为: %s" % (filesize))
if filesize == 0: return
#print(chardet.detect(open(full_path, mode='rb').read())) #查看文件编码格式
file = open(full_path, "rb")
try:
i = 0
while True:
print("游标位置：", file.tell())
stock_date = file.read(2)
cur_price = file.read(4)
arr_price = file.read(4)
vol = file.read(4)
stock_reservation = file.read(12)
stock_date = unpack("h", stock_date)
cur_price = unpack("l", cur_price)
arr_price = unpack("l", arr_price)
vol = unpack("l", vol)
#stock_reservation = unpack("s", stock_reservation)
print(stock_date)
print(cur_price)
print(arr_price)
print(vol)
print(stock_reservation)
i = i + 1
if i == 2:break
for line in file:
result = chardet.detect(line)
print("code: ", result)
buf_size = len(line)
rec_count = buf_size // 32
begin = 0
end = 32
print("行内容：", line)
print("buf_size：", buf_size)
print("rec_count：", rec_count)
a = unpack('IIIIIfII', line[begin:end])
print("解码后的数据0: %s" % (str(a[0])))
print("解码后的数据1: %s" % (str(a[1])))
print("解码后的数据2: %s" % (str(a[2])))
break
finally:
file.close()
def get_bar(code, start=None, end=None, freq='day', exchange=None):
"""
股票成交量 volume 单位是100股
"""
code = code.upper()
standard_freq = parse_frequency_str(freq)
try:
tdx_code = _get_tdx_code_from_security_dataframe(code, exchange)
except:
util_log_info("Can't get tdx_code from {}".format(code))
return
if standard_freq in ['D', 'w', 'M', 'Q', 'Y']:
file_path = _generate_path(code, 'D', tdx_code)
elif standard_freq in ['1min', '5min', '30min', '60min']:
file_path = _generate_path(code, '5min', tdx_code)
elif standard_freq in ['1min']:
file_path = _generate_path(code, '1min', tdx_code)
else:
util_log_info('Not supported frequency {}'.format(freq))
return
if not os.path.exists(file_path):
util_log_info('=={}== {} file is not exists!'.format(code, file_path))
return
# 统一freq的数据结构
if tdx_code in ['sh', 'sz']:
if standard_freq in ['D', 'w', 'M', 'Q', 'Y']:
reader = TdxDailyBarReader()
df = reader.get_df(file_path)
elif standard_freq in ['1min', '5min', '30min', '60min']:
reader = TdxLCMinBarReader()
df = reader.get_df(file_path)
else:
util_log_info('Not supported frequency {}'.format(freq))
return
else:
if standard_freq in ['D', 'w', 'M', 'Q', 'Y']:
reader = TdxExHqDailyBarReader()
df = reader.get_df(file_path)
elif standard_freq in ['1min', '5min', '30min', '60min']:
reader = TdxLCMinBarReader()
df = reader.get_df(file_path)
else:
util_log_info('Not supported frequency {}'.format(freq))
return
if len(df) < 1:
return
recorder = SECURITY_DATAFRAME.loc[code]
if isinstance(recorder, pd.DataFrame):
instrument = recorder.loc[recorder['tdx_code'] == tdx_code].loc[
code, 'instrument']
exchange = recorder.loc[recorder['tdx_code'] == tdx_code].loc[
code, 'exchange']
else:
instrument = recorder['instrument']
exchange = recorder['exchange']
if instrument in ['future', 'option']:
df.rename(columns={
'amount': "position",
"jiesuan": "settle"
},
inplace=True)
if start:
start = pd.to_datetime(start)
df = df[df.index >= start]
if end:
end = pd.to_datetime(end)
df = df[df.index <= end]
df['date'] = df.index
df = df.assign(code=code, exchange=exchange)
if standard_freq in ['w', 'M', 'Q', 'Y']:
df = resample_from_daily_data(df, standard_freq)
return df