def main(input, output, datatype):
"""
通达信数据文件读取
"""
if datatype == 'daily':
reader = TdxDailyBarReader()
elif datatype == 'ex_daily':
reader = TdxExHqDailyBarReader()
elif datatype == 'lc':
reader = TdxLCMinBarReader()
elif datatype == 'gbbq':
reader = GbbqReader()
elif datatype == 'block':
reader = BlockReader()
elif datatype == 'customblock':
reader = CustomerBlockReader()
elif datatype == 'history_financial' or datatype == 'hf':
reader = HistoryFinancialReader()
else:
reader = TdxMinBarReader()
try:
df = reader.get_df(input)
if output:
click.echo("写入到文件 : " + output)
df.to_csv(output)
else:
print(df)
except Exception as e:
print(str(e))
def __init__(self):
self.day_reader = TdxDailyBarReader()
self.minline_reader = TdxLCMinBarReader()
self.block_reader = BlockReader()
# pandas数据显示设置
pd.set_option('display.max_columns', None) # 显示所有列
def daily(self, symbol=None):
'''
获取日线数据
:param symbol:
:return: pd.dataFrame or None
'''
reader = TdxDailyBarReader()
vipdoc = self.find_path(symbol=symbol, subdir='lday', ext='day')
if vipdoc is not None:
return reader.get_df(vipdoc)
return None
def main(input, output, datatype):
"""
通达信数据文件读取
"""
if datatype == 'daily':
reader = TdxDailyBarReader()
elif datatype == 'ex_daily':
reader = TdxExHqDailyBarReader()
elif datatype == 'lc':
reader = TdxLCMinBarReader()
elif datatype == 'gbbq':
reader = GbbqReader()
elif datatype == 'block':
reader = BlockReader()
else:
reader = TdxMinBarReader()
try:
df = reader.get_df(input)
if (output):
click.echo("写入到文件 : " + output)
df.to_csv(output)
else:
print(df)
except Exception as e:
print(str(e))
def main(input, output, datatype):
"""
通达信数据文件读取
"""
reader = TdxDailyBarReader()
try:
df = reader.get_df(input)
if (output):
click.echo("写入到文件 : " + output)
df.to_csv(output)
else:
print(df)
except Exception as e:
print(str(e))
def get_source_data(file_path: Path, freq: str) -> pd.DataFrame:
if not file_path.exists():
logger.info(f"{str(file_path)} is not exists!")
return pd.DataFrame()
if file_path.parent.parent.name == "ds":
if freq == "day":
reader = TdxExHqDailyBarReader()
else:
reader = TdxLCMinBarReader()
else:
# Todo 对股票数据进行后复权
if freq == "day":
reader = TdxDailyBarReader()
else:
reader = TdxLCMinBarReader()
try:
df = reader.get_df(str(file_path))
except:
logger.warning(f"Cant read {str(file_path)} ")
return pd.DataFrame()
df = df.reset_index()
df["date"] = df["date"].astype(str).astype(np.datetime64)
return df
def _get_source_data(self, file_path: Path) -> pd.DataFrame:
if file_path.parent.parent.name == "ds":
if self.freq == "day":
reader = TdxExHqDailyBarReader()
else:
reader = TdxLCMinBarReader()
else:
if self.freq == "day":
reader = TdxDailyBarReader()
else:
reader = TdxLCMinBarReader()
try:
df = reader.get_df(str(file_path))
except:
logger.warning(f"Cant read {str(file_path)} ")
return pd.DataFrame()
instrument = self.tdx_files_info[self.tdx_files_info["file_path"] ==
file_path]['instrument'].iloc[0]
if instrument in ['future', 'option']:
df.rename(columns={
'amount': "position",
"jiesuan": "settle"
},
inplace=True)
if "hk_stock_amount" in df.columns:
del df["hk_stock_amount"]
df = df.reset_index()
df[self.date_field_name] = df[self.date_field_name].astype(str).astype(
np.datetime64)
return df
def read_file(self):
# 读取数据
# print ("read_file, {}".format(self.file))
reader = TdxDailyBarReader()
self.df = reader.get_df(self.file)
# print (self.df.index)
# 更新UI
self.lineEdit.setText(os.path.splitext(os.path.basename(self.file))[0])
# 初始化日期
start = self.df.index[0]
end = self.df.index[-1]
self.dateEdit_start.setDate(start)
self.dateEdit_start.setMinimumDate(start)
self.dateEdit_start.setMaximumDate(end)
self.dateEdit_end.setDate(end)
self.dateEdit_end.setMinimumDate(start)
self.dateEdit_end.setMaximumDate(end)
# print ("read_file end, {}, {}".format(start, end))
self.read_ticker(start, end)
class TdxLocalHelper:
BlockReader_TYPE_FLAT = 0
BlockReader_TYPE_GROUP = 1
def __init__(self):
self.day_reader = TdxDailyBarReader()
self.minline_reader = TdxLCMinBarReader()
self.block_reader = BlockReader()
# pandas数据显示设置
pd.set_option('display.max_columns', None) # 显示所有列
#pd.set_option('display.max_rows', None) # 显示所有行
# 解析日线文件数据
#返回值：date:open,high,low,close,amount,volume
def read_tdx_local_day(self):
#df = reader.get_df(config.tdx_local_sz_day + "sz000001.day") sh000001.day
df = self.day_reader.get_df(config.tdx_local_sh_day + "sh000001.day")
print(df)
# 批量格式化日期字段，去掉分隔符
def format_date(self, date):
return ((str(date))[0:10]).replace('-', '')
# 解析1分钟和5分钟数据
#返回值：date: open,high,low,close,amount,volume
#csv格式：code,ts_code,trade_date(缩写）,trade_time,time_index,open,high,low,close,amount,volume
def read_tdx_local_minline_all(self, full_path,filename=""):
#df = reader.get_df(config.tdx_local_sz_minline1 + "sz399001.lc1")
#df = reader.get_df(config.tdx_local_sz_minline5 + "sz399001.lc5")
#df = self.minline_reader.get_df(config.tdx_local_sh_minline1 + "sh600300.lc1")
code = filename[2:8]
ts_code = code + "." + filename[0:2].upper()
df = self.minline_reader.get_df(full_path)
df.insert(0, 'trade_time', df.index)
df.insert(0, 'trade_date', df.index.floor('D'))
df.insert(0, 'ts_code', ts_code)
df.insert(0, 'code', code)
df.reset_index(drop=True,inplace=True) #参考：https://zhuanlan.zhihu.com/p/110819220?from_voters_page=true
df.insert(4, 'time_index', df.index)
#df['trade_time'] = pd.to_datetime(df['trade_time'], infer_datetime_format=True).dt.normalize() # strftime('%m/%d/%Y') # format='%m/%d/%Y').dt.date
#df['trade_time'] = df['trade_time'].apply(lambda x: x.strftime('%H:%M:%S'))
df['trade_time'] = pd.to_datetime(df['trade_time'], format='%H:%M:%S').dt.strftime('%H:%M:%S')
df['time_index'] = df['trade_time'].apply(lambda x: datatime_util.stockTradeTime2Index(x))
df['trade_date'] = df['trade_date'].apply(lambda x: self.format_date(x))
df['open'] = df['open'].apply(lambda x: round(x,2))
df['high'] = df['high'].apply(lambda x: round(x, 2))
df['low'] = df['low'].apply(lambda x: round(x, 2))
df['close'] = df['close'].apply(lambda x: round(x, 2))
csv_filename = config.tdx_csv_minline1_all + ts_code + ".csv"
if os.path.isfile(csv_filename):
os.remove(csv_filename)
df.to_csv(csv_filename, index=False, mode='w', header=True, sep=',', encoding="utf_8_sig")
else:
df.to_csv(csv_filename, index=False, mode='w', header=True, sep=',', encoding="utf_8_sig")
# 解析1分钟和5分钟数据，输出简单形式，只返回price和vol
# 返回值：date: open,high,low,close,amount,volume
# csv格式：code,ts_code,trade_date(缩写）,trade_time,time_index,price,volume
def read_tdx_local_minline_simple(self, full_path, filename=""):
# df = reader.get_df(config.tdx_local_sz_minline1 + "sz399001.lc1")
# df = reader.get_df(config.tdx_local_sz_minline5 + "sz399001.lc5")
# df = self.minline_reader.get_df(config.tdx_local_sh_minline1 + "sh600300.lc1")
code = filename[2:8]
ts_code = code + "." + filename[0:2].upper()
df = self.minline_reader.get_df(full_path)
df.insert(0, 'trade_time', df.index)
df.insert(0, 'trade_date', df.index.floor('D'))
df.insert(0, 'ts_code', ts_code)
df.insert(0, 'code', code)
df.reset_index(drop=True, inplace=True) # 参考：https://zhuanlan.zhihu.com/p/110819220?from_voters_page=true
df.insert(4, 'time_index', df.index)
# df['trade_time'] = pd.to_datetime(df['trade_time'], infer_datetime_format=True).dt.normalize() # strftime('%m/%d/%Y') # format='%m/%d/%Y').dt.date
# df['trade_time'] = df['trade_time'].apply(lambda x: x.strftime('%H:%M:%S'))
df['trade_time'] = pd.to_datetime(df['trade_time'], format='%H:%M:%S').dt.strftime('%H:%M:%S')
df['time_index'] = df['trade_time'].apply(lambda x: datatime_util.stockTradeTime2Index(x))
df['trade_date'] = df['trade_date'].apply(lambda x: self.format_date(x))
#df['open'] = df['open'].apply(lambda x: round(x, 2))
#df['high'] = df['high'].apply(lambda x: round(x, 2))
#df['low'] = df['low'].apply(lambda x: round(x, 2))
df['close'] = df['close'].apply(lambda x: round(x, 2))
df.rename(columns={'close': 'price'}, inplace=True)
df.drop(['open', 'high', 'low', 'amount'], axis=1, inplace=True)
csv_filename = config.tdx_csv_minline1_simple + ts_code + ".csv"
if os.path.isfile(csv_filename):
os.remove(csv_filename)
df.to_csv(csv_filename, index=False, mode='w', header=True, sep=',', encoding="utf_8_sig")
else:
df.to_csv(csv_filename, index=False, mode='w', header=True, sep=',', encoding="utf_8_sig")
# 解析板块数据
#扁平格式返回值：blockname,block_type,code_index,code
def read_tdx_local_block(self):
##指数板块 风格板块 概念板块 一般板块
block_filename = ["block_zs.dat", "block_fg.dat", "block_gn.dat", "block.dat"]
for block in block_filename:
df = self.block_reader.get_df(config.tdx_local_block + block) # 默认扁平格式
df_group = self.block_reader.get_df(config.tdx_local_block + block, self.BlockReader_TYPE_GROUP) #分组格式
filename = config.tdx_csv_block + block[0:-4] + ".csv"
filename_group = config.tdx_csv_block + block[0:-4] + "_group" + ".csv"
if os.path.isfile(filename):
df.to_csv(filename, index=False, mode='a', header=False, sep=',', encoding="utf_8_sig")
df_group.to_csv(filename_group, index=False, mode='a', header=False, sep=',', encoding="utf_8_sig")
else:
df.to_csv(filename, index=False, mode='w', header=True, sep=',', encoding="utf_8_sig")
df_group.to_csv(filename_group, index=False, mode='w', header=True, sep=',', encoding="utf_8_sig")
# 解析分时图数据
def read_tdx_local_fst(self):
reader = TdxMinBarReader() #这个reader不能解析分时图文件
# df = reader.get_df(config.tdx_local_sz_minline + "sz399001.lc1")
df = reader.get_df(config.tdx_local_fst + "sh20200417.tfz")
print(df)
# 解析分时图文件,没有解析出来
"""
通达信的zst的数据记录是每6508个字节为一天的数据,
每26个字节为一个分钟的记录,这26个字节是这样分配的,
时间占两个字节化为十进制为570的话表示9:30分(570/60=9.5)
下一个是占四个字节的叫现价,
再下四个字节叫均价,
另外还有两个字节为该分钟成交量(现在有可能已经改为四个字节),
剩下的14个字节是预留的,
"""
def parse_fst_file(self):
full_path = config.tdx_local_fst + "sh20200417.tfz"
filesize = os.path.getsize(full_path) # 文件字节数
print("filesize为: %s" % (filesize))
if filesize == 0: return
#print(chardet.detect(open(full_path, mode='rb').read())) #查看文件编码格式
file = open(full_path, "rb")
try:
i = 0
while True:
print("游标位置：", file.tell())
stock_date = file.read(2)
cur_price = file.read(4)
arr_price = file.read(4)
vol = file.read(4)
stock_reservation = file.read(12)
stock_date = unpack("h", stock_date)
cur_price = unpack("l", cur_price)
arr_price = unpack("l", arr_price)
vol = unpack("l", vol)
#stock_reservation = unpack("s", stock_reservation)
print(stock_date)
print(cur_price)
print(arr_price)
print(vol)
print(stock_reservation)
i = i + 1
if i == 2:break
for line in file:
result = chardet.detect(line)
print("code: ", result)
buf_size = len(line)
rec_count = buf_size // 32
begin = 0
end = 32
print("行内容：", line)
print("buf_size：", buf_size)
print("rec_count：", rec_count)
a = unpack('IIIIIfII', line[begin:end])
print("解码后的数据0: %s" % (str(a[0])))
print("解码后的数据1: %s" % (str(a[1])))
print("解码后的数据2: %s" % (str(a[2])))
break
finally:
file.close()
def tdxread(code): #day data
market = marketcode(code)
#market = str(market)
reader = TdxDailyBarReader()
data = reader.get_df(panfu + market + '\\lday\\' + market + code + '.day')
return data
def __init__(self):
self.__lc_min_bar_reader = lc_min_bar_reader.TdxLCMinBarReader()
self.__daily_bar_reader = TdxDailyBarReader()
self.__block_reader = block_reader.BlockReader()
self.__tdx_api = TdxHq_API()
self.__cfg_provider = ConfigProvider.ConfigProvider()
class TDX_GW(object):
__lc_min_bar_reader = None
__daily_bar_reader = None
__tdx_api = None
__connected_ip = '119.147.212.81'
__connected_port = 7709
def __init__(self):
self.__lc_min_bar_reader = lc_min_bar_reader.TdxLCMinBarReader()
self.__daily_bar_reader = TdxDailyBarReader()
self.__block_reader = block_reader.BlockReader()
self.__tdx_api = TdxHq_API()
self.__cfg_provider = ConfigProvider.ConfigProvider()
def get_local_stock_bars(self, file_path: str, stock_date_type: StockDataType):
if stock_date_type == StockDataType.ONE_MIN or \
stock_date_type == StockDataType.FIVE_MINS:
# start = time.time()
# df = self.__lc_min_bar_reader.get_df(file_path)
data = self.__lc_min_bar_reader.parse_data_by_file(file_path)
df = pd.DataFrame(data=data)
# df = df['date', 'open', 'high', 'low', 'close', 'amount', 'volume']
# print(f"TDX get 1min bar time spent: {(time.time() - start) * 1000} ms")
return df[['date', 'open', 'high', 'low', 'close', 'amount', 'volume']]
elif stock_date_type == StockDataType.DAILY:
data = self.__daily_bar_reader.get_df(file_path).reset_index()
data['date'] = data['date'].dt.strftime('%Y-%m-%d')
return data[['date', 'open', 'high', 'low', 'close', 'amount', 'volume']]
else:
raise UnimplementedException
def get_local_stock_bars_raw_data(self, file_path: str, stock_date_type: StockDataType):
if stock_date_type == StockDataType.ONE_MIN or \
stock_date_type == StockDataType.FIVE_MINS:
return self.__lc_min_bar_reader.parse_data_by_file(file_path)
elif stock_date_type == StockDataType.DAILY:
return self.__daily_bar_reader.parse_data_by_file(file_path)
else:
raise UnimplementedException
def get_local_block(self):
file_path = self.__cfg_provider.get_tdx_block_directory_path()
return self.__block_reader.get_df(file_path, BlockReader_TYPE_GROUP)
def get_realtime_stock_1min_bars(self, market: str, stock_id: str):
with self.__tdx_api.connect(self.__connected_ip, self.__connected_port):
market_code = self.__get_market_code(market)
df = self.__tdx_api.to_df(
self.__tdx_api.get_security_bars(8, market_code, stock_id, 0, 10)) # 返回DataFrame
return df
def get_realtime_stocks_quotes(self, stock_ids: []):
stock_list = []
for id in stock_ids:
stock_list.append((com_utils.get_stock_market(id), id))
with self.__tdx_api.connect(self.__connected_ip, self.__connected_port):
return self.__tdx_api.get_security_quotes(stock_list)
def get_history_minute_time_data(self, market: str, stock_id: str, date: int):
with self.__tdx_api.connect(self.__connected_ip, self.__connected_port):
market_code = self.__get_market_code(market)
df = self.__tdx_api.to_df(
self.__tdx_api.get_history_minute_time_data(market_code, stock_id, date))
return df
def get_xdxr_info(self, market: str, stock_id: str):
with self.__tdx_api.connect(self.__connected_ip, self.__connected_port):
market_code = self.__get_market_code(market)
df = self.__tdx_api.to_df(
self.__tdx_api.get_xdxr_info(market_code, stock_id))
return df
def test(self):
with self.__tdx_api.connect(self.__connected_ip, self.__connected_port):
return self.__tdx_api.get_history_minute_time_data(0, '000001', '2019-05-05')
def __get_market_code(self, market: str):
if market == cfg.MARKET.SHANGHAI:
return 1
elif market == cfg.MARKET.SHENZHEN:
return 0
else:
raise UnimplementedException
def is_tdx_local_data_ready_for(self, dt):
file = os.path.join(self.root, 'vipdoc', 'sz', 'lday', 'sz399001.day')
reader = TdxDailyBarReader()
df = reader.get_df(file)
return dt.strftime('%Y-%m-%d') in df.index
def read_tdx_file():
source = os.getcwd() + "/tdx_file/"
reader = TdxDailyBarReader()
df = reader.get_df(source + "sz000001.day")
print(df)
df.to_csv(source + "/sz000001.csv")
def get_bar(code, start=None, end=None, freq='day', exchange=None):
"""
股票成交量 volume 单位是100股
"""
code = code.upper()
standard_freq = parse_frequency_str(freq)
try:
tdx_code = _get_tdx_code_from_security_dataframe(code, exchange)
except:
util_log_info("Can't get tdx_code from {}".format(code))
return
if standard_freq in ['D', 'w', 'M', 'Q', 'Y']:
file_path = _generate_path(code, 'D', tdx_code)
elif standard_freq in ['1min', '5min', '30min', '60min']:
file_path = _generate_path(code, '5min', tdx_code)
elif standard_freq in ['1min']:
file_path = _generate_path(code, '1min', tdx_code)
else:
util_log_info('Not supported frequency {}'.format(freq))
return
if not os.path.exists(file_path):
util_log_info('=={}== {} file is not exists!'.format(code, file_path))
return
# 统一freq的数据结构
if tdx_code in ['sh', 'sz']:
if standard_freq in ['D', 'w', 'M', 'Q', 'Y']:
reader = TdxDailyBarReader()
df = reader.get_df(file_path)
elif standard_freq in ['1min', '5min', '30min', '60min']:
reader = TdxLCMinBarReader()
df = reader.get_df(file_path)
else:
util_log_info('Not supported frequency {}'.format(freq))
return
else:
if standard_freq in ['D', 'w', 'M', 'Q', 'Y']:
reader = TdxExHqDailyBarReader()
df = reader.get_df(file_path)
elif standard_freq in ['1min', '5min', '30min', '60min']:
reader = TdxLCMinBarReader()
df = reader.get_df(file_path)
else:
util_log_info('Not supported frequency {}'.format(freq))
return
if len(df) < 1:
return
recorder = SECURITY_DATAFRAME.loc[code]
if isinstance(recorder, pd.DataFrame):
instrument = recorder.loc[recorder['tdx_code'] == tdx_code].loc[
code, 'instrument']
exchange = recorder.loc[recorder['tdx_code'] == tdx_code].loc[
code, 'exchange']
else:
instrument = recorder['instrument']
exchange = recorder['exchange']
if instrument in ['future', 'option']:
df.rename(columns={
'amount': "position",
"jiesuan": "settle"
},
inplace=True)
if start:
start = pd.to_datetime(start)
df = df[df.index >= start]
if end:
end = pd.to_datetime(end)
df = df[df.index <= end]
df['date'] = df.index
df = df.assign(code=code, exchange=exchange)
if standard_freq in ['w', 'M', 'Q', 'Y']:
df = resample_from_daily_data(df, standard_freq)
return df
