def worker(q, hy_map, hy_code):
index_market, index_code = hy_map[hy_code]["market"], hy_map[hy_code][
"code"]
stock_lst = hy_map[hy_code]["stock"]
download_lst = [dict(market=index_market, code=index_code, index=True)]
download_lst.extend(stock_lst)
# 下载指数 historical data
try:
server_addr = q.get(timeout=2)
except Exception as e:
print("没有可用的服务器")
return
api = TdxHq_API(heartbeat=True, auto_retry=True, multithread=True)
try:
if api.connect(*server_addr[1:]):
for item in download_lst:
market = item["market"]
code = item["code"]
isindex = item.get("index", False)
download(api, market, code, years, isindex=isindex)
else:
print("连接失败")
except Exception as e:
print(e)
traceback.print_exc()
print("进程下载失败")
finally:
api.disconnect()
q.put(server_addr)
return
def __hq_ping(self, ip, port):
api = TdxHq_API()
with api.connect(ip, port, time_out=0.7):
assert len(api.get_security_list(0, 1)) > 800
api.disconnect()
return True
return False
def main():
manager = Manager()
q = manager.Queue()
if not path.exists(stock_data_path):
os.makedirs(stock_data_path)
if not path.exists(index_data_path):
os.makedirs(index_data_path)
hy_map = get_hy(tdxzs_path, tdxhy_path)
print("总共行业数量: %s" % len(hy_map))
pool = ProcessPoolExecutor(process_pool_size)
api = TdxHq_API(heartbeat=True, auto_retry=True, multithread=True)
print("寻找可用服务器...")
server_count = 0
for data_server in random.choices(hq_hosts, k=4):
try:
if api.connect(*data_server[1:]):
q.put(data_server)
server_count += 1
except Exception:
print("连接服务器失败")
finally:
api.disconnect()
print("可用服务器: %s" % server_count)
if server_count < 1:
print("没有可用服务器")
print("开始下载数据...")
fut_lst = []
for hy_code in hy_map:
fut_lst.append(pool.submit(worker, q, hy_map, hy_code))
# print(len(fut_lst))
wait(fut_lst)
print(fut_lst[0].result())
def get_a_quote_by_tdx(code_list):
"""
获取A股市场行情，该接口只获取A股市场行情
其他的行情使用拓展行情接口
code_list：数字或者字符串格式的list，调用def standard_tdx_code(code_list)来格式化市场信息及code
"""
if len(code_list) == 0:
return pd.DataFrame(columns=stock_tdx_columns)
# 一次最多获取50只股票的实时行情，如果code_list 多于50只，则50只股票取一次行情，然后进行拼接
tdx_api = TdxHq_API()
if len(code_list) >= 50:
tmp = div_list(code_list, 50)
df_tmp = pd.DataFrame(columns=stock_tdx_columns)
if tdx_api.connect('119.147.212.81', 7709):
for tmp_list in tmp:
tmp_list = standard_tdx_code(tmp_list)
tdx_data = tdx_api.to_df(tdx_api.get_security_quotes(tmp_list))
df_tmp = pd.concat([df_tmp, tdx_data])
df_tmp = df_tmp[stock_tdx_columns]
else:
tmp = standard_tdx_code(code_list)
if tdx_api.connect('119.147.212.81', 7709):
tdx_data = tdx_api.to_df(tdx_api.get_security_quotes(tmp))
df_tmp = tdx_data[stock_tdx_columns]
tdx_api.disconnect()
return process_tdx_price(df_tmp)
def get_stock_count():
api = TdxHq_API()
if api.connect('119.147.212.81', 7709):
print(api.get_security_count(0)) #参数为市场代码
print(api.get_security_list(0, 0)) #第一个参数为市场代码，第二个参数为起始位置
print(api.get_security_count(1))
print(api.to_df(api.get_security_list(1, 0)))
api.disconnect()
def _getKLineData(self):
api = TdxHq_API(auto_retry=True)
if api.connect('119.147.212.81', 7709):
self._KLineData = api.to_df(
api.get_index_bars(9, 1, self._stockCode, 0,
self._stockOberCount)) # 返回DataFrame
# print(self._KLineData)
api.disconnect()
def get_company_info():
api = TdxHq_API()
if api.connect('119.147.212.81', 7709):
print(api.get_company_info_category(TDXParams.MARKET_SZ,
'000001')) #查询公司信息目录,参数：市场代码， 股票代码
api.get_company_info_content(0, '000001',
os.getcwd() + "/tdx_file/" + '000001.txt',
0, 100) #读取公司信息详情，参数文件路径不知干什么
print(api.get_finance_info(0, '000001')) #读取财务信息
api.disconnect()
def __hq_tick(self, code):
api = TdxHq_API()
ip, port = self.__hq.get()
with api.connect(ip, port):
df = api.get_security_quotes(code)
if df is not None:
df = api.to_df(df)
api.disconnect()
self.__hq.put((ip, port))
return df
def get_block_info():
api = TdxHq_API()
if api.connect('119.147.212.81', 7709):
data = api.get_and_parse_block_info("block.dat") #一般板块
#print(api.get_and_parse_block_info("block_zs.dat")) #指数板块
#print(api.get_and_parse_block_info("block_fg.dat")) #风格板块
#print(api.get_and_parse_block_info("block_gn.dat")) #概念板块
datadf = api.to_df(data)
print(datadf)
api.disconnect()
def get_kline_data():
api = TdxHq_API()
if api.connect('119.147.212.81', 7709):
data = api.get_security_bars(
9, 0, '000001', 0,
10) # 返回普通list,五个参数分别为：category（k线),市场代码(深市),股票代码,开始时间,记录条数
print(data)
data = api.to_df(api.get_security_bars(9, 0, '000001', 0,
10)) # 返回DataFrame
print(data)
api.disconnect()
def quote_TDX(ds_tuple):
api = TdxHq_API()
if api.connect('119.147.212.81', 7709):
MainCodeData = api.get_security_quotes([(1, code)])
n = ceil(len(ds_tuple) / 80)
for i in range(n):
if i == 0:
quot = api.get_security_quotes(ds_tuple[i * 80:(i + 1) *
80]) #每次最多请求80个数据？
else:
quot_add = api.get_security_quotes(
ds_tuple[i * 80:(i + 1) * 80])
quot = quot + quot_add
api.disconnect()
return quot, MainCodeData
def __hq_list(self, market):
assert self.__hq.qsize() > 0
api = TdxHq_API()
ip, port = self.__hq.get()
with api.connect(ip, port):
df = list()
for start in range(0, api.get_security_count(market=market), 1000):
df.append(api.to_df(api.get_security_list(market, start)))
api.disconnect()
self.__hq.put((ip, port))
df = pandas.concat(df, sort=False).assign(sse=market)
df = df[[
'code', 'volunit', 'decimal_point', 'name', 'pre_close', 'sse'
]].dropna()
df = df.assign(sse='sh' if market == CODE_MARKET_SH else 'sz',
sec=get_code_type(df.code.tolist(), market))
return df
return None
class SP(object):
def __init__(self,userid="account4",server="http://192.168.118.1:5000"):
self.server = server
self.userid = userid
self.api = TdxHq_API(heartbeat=True)
self.trader = None
self.trading = False
self.TDX_IP_SETS = STOCK_IP_SETS
def connect(self):
for ip in self.TDX_IP_SETS:
try:
if self.api.connect(ip, 7709):
return
except:
pass
def disconnect(self):
logger.info("[DISCONNECT]:start disconnect !!!!!")
self.api.disconnect()
logger.info("[DISCONNECT]:disconnect finished !!!!!")
def initial(self):
'''每天初始化设置
'''
logger.info("[INITIAL]:start initial !!!!!")
logger.info("[INITIAL]:try to create connect... ")
self.connect()
self.trader = trade(UserID=self.userid,api=self.api,mock=False,server=self.server)
logger.info("[INITIAL]:connect successful!")
self.trading = self.judgetradeday()
logger.info("[INITIAL]:initial finished !!!!!")
def judgetradeday(self):
today = datetime.datetime.today().date().strftime('%Y-%m-%d')
df = ts.trade_cal()
return df[(df["calendarDate"]==today)].isOpen.values[0]
def run(self):
if not self.trading:return
logger.info("[RUN]:start run !!!!!")
self.trader.autobuy()
logger.info("[RUN]:run finished !!!!!")
class PYTDXService():
"""pytdx数据服务类"""
def __init__(self):
"""Constructor"""
self.connected = False # 数据服务连接状态
self.hq_api = None # 行情API
def connect_api(self):
"""连接API"""
# 连接增强行情API并检查连接情况
try:
if not self.connected:
host = SETTINGS["TDX_HOST"]
port = SETTINGS["TDX_PORT"]
self.hq_api = TdxHq_API()
self.hq_api.connect(host, port)
self.connected = True
return True
except Exception:
raise ConnectionError("pytdx连接错误")
def get_realtime_data(self, symbol: str):
"""获取股票实时数据"""
try:
symbols = self.generate_symbols(symbol)
df = self.hq_api.to_df(self.hq_api.get_security_quotes(symbols))
return df
except Exception:
raise ValueError("股票数据获取失败")
@staticmethod
def generate_symbols(symbol: str):
"""组装symbols数据，pytdx接收的是以市场代码和标的代码组成的元祖的list"""
new_symbols = []
code, exchange = symbol.split('.')
new_symbols.append((exchange_map[exchange], code))
return new_symbols
def close(self):
"""数据服务关闭"""
self.connected = False
self.hq_api.disconnect()
class tdxApi:
def __init__(self):
self.ipset = [(v[1], v[2]) for v in hq_hosts]
random.shuffle(self.ipset)
self.ippool = AvailableIPPool(TdxHq_API, self.ipset)
self.primary_ip, _ = self.ippool.sync_get_top_n(2)
self.api = TdxHq_API(multithread=True, heartbeat=True, auto_retry=True)
def connect(self):
self.api.connect(self.primary_ip[0], self.primary_ip[1])
def release(self):
self.api.disconnect()
def getQuotes(self, stock_list):
datas = self.api.get_security_quotes(stock_list)
if datas is None:
return None
else:
return self.api.to_df(datas)
def getList(self, market, index):
df = self.api.to_df(self.api.get_security_list(market, index))
return df
def getCount(self, market=0):
return self.api.get_security_count(market)
def getMinQuotes(self, market, stock_code):
datas = self.api.get_minute_time_data(market, stock_code)
if datas is None:
return None
return self.api.to_df(datas)
def getMinQuotesHis(self, market, stock_code, date):
datas = self.api.get_history_minute_time_data(market, stock_code, date)
if datas is None:
return None
return self.api.to_df(datas)
class SP(object):
def __init__(self, userid="account4", server="http://192.168.118.1:5000"):
self.server = server
self.userid = userid
self.api = TdxHq_API(heartbeat=True)
self.trader = None
self.trading = False
self.TDX_IP_SETS = STOCK_IP_SETS
def connect(self):
for ip in self.TDX_IP_SETS:
try:
if self.api.connect(ip, 7709):
return
except:
pass
def disconnect(self):
logger.info("[DISCONNECT]:start disconnect !!!!!")
self.api.disconnect()
logger.info("[DISCONNECT]:disconnect finished !!!!!")
def initial(self):
'''每天初始化设置
'''
logger.info("[INITIAL]:start initial !!!!!")
logger.info("[INITIAL]:try to create connect... ")
self.connect()
self.trader = trade(UserID=self.userid,
api=self.api,
mock=False,
server=self.server)
logger.info("[INITIAL]:connect successful!")
logger.info("[INITIAL]:initial finished !!!!!")
def run(self):
logger.info("[RUN]:start run !!!!!")
self.trader.autobuy()
logger.info("[RUN]:run finished !!!!!")
def __hq_bars(self, code, offset, frequency=9, index=False):
assert self.__hq.qsize() > 0
api = TdxHq_API()
if index is True:
market_func = get_index_market
bars_func = api.get_index_bars
else:
market_func = get_code_market
bars_func = api.get_security_bars
ip, port = self.__hq.get()
with api.connect(ip, port):
df = list()
for _code in code:
market = market_func(_code)
for _start, _count in offset:
df.append(
api.to_df(
bars_func(frequency, market, _code, _start,
_count)).assign(code=_code))
api.disconnect()
self.__hq.put((ip, port))
if len(df) < 1:
return None
return pandas.concat(df, sort=False)
class Engine:
def __init__(self, *args, **kwargs):
if kwargs.pop('best_ip', False):
self.ip = self.best_ip
else:
self.ip = '14.17.75.71'
self.ip = kwargs.pop('ip', '14.17.75.71')
self.thread_num = kwargs.pop('thread_num', 1)
if not PY2 and self.thread_num != 1:
self.use_concurrent = True
else:
self.use_concurrent = False
self.api = TdxHq_API(args, kwargs)
if self.use_concurrent:
self.apis = [
TdxHq_API(args, kwargs) for i in range(self.thread_num)
]
self.executor = ThreadPoolExecutor(self.thread_num)
def connect(self):
self.api.connect(self.ip)
if self.use_concurrent:
for api in self.apis:
api.connect(self.ip)
return self
def __enter__(self):
return self
def exit(self):
self.api.disconnect()
if self.use_concurrent:
for api in self.apis:
api.disconnect()
def __exit__(self, exc_type, exc_val, exc_tb):
self.api.disconnect()
if self.use_concurrent:
for api in self.apis:
api.disconnect()
def quotes(self, code):
code = [code] if not isinstance(code, list) else code
code = self.security_list[self.security_list.code.isin(
code)].index.tolist()
data = [
self.api.to_df(
self.api.get_security_quotes(code[80 * pos:80 * (pos + 1)]))
for pos in range(int(len(code) / 80) + 1)
]
return pd.concat(data)
# data = data[['code', 'open', 'high', 'low', 'price']]
# data['datetime'] = datetime.datetime.now()
# return data.set_index('code', drop=False, inplace=False)
def stock_quotes(self):
code = self.stock_list.index.tolist()
if self.use_concurrent:
res = {
self.executor.submit(self.apis[pos % self.thread_num].get_security_quotes,
code[80 * pos:80 * (pos + 1)]) \
for pos in range(int(len(code) / 80) + 1)}
return pd.concat([self.api.to_df(dic.result()) for dic in res])
else:
data = [
self.api.to_df(
self.api.get_security_quotes(code[80 * pos:80 *
(pos + 1)]))
for pos in range(int(len(code) / 80) + 1)
]
return pd.concat(data)
@lazyval
def security_list(self):
return pd.concat([
pd.concat([
self.api.to_df(self.api.get_security_list(
j, i * 1000)).assign(sse=0 if j == 0 else 1).set_index(
['sse', 'code'], drop=False)
for i in range(int(self.api.get_security_count(j) / 1000) + 1)
],
axis=0) for j in range(2)
],
axis=0)
@lazyval
def stock_list(self):
aa = map(stock_filter, self.security_list.index.tolist())
return self.security_list[list(aa)]
@lazyval
def best_ip(self):
return select_best_ip()
@lazyval
def concept(self):
return self.api.to_df(
self.api.get_and_parse_block_info(TDXParams.BLOCK_GN))
@lazyval
def index(self):
return self.api.to_df(
self.api.get_and_parse_block_info(TDXParams.BLOCK_SZ))
@lazyval
def fengge(self):
return self.api.to_df(
self.api.get_and_parse_block_info(TDXParams.BLOCK_FG))
@lazyval
def customer_block(self):
return CustomerBlockReader().get_df(CUSTOMER_BLOCK_PATH)
@lazyval
def gbbq(self):
df = GbbqReader().get_df(GBBQ_PATH).query('category == 1')
df['datetime'] = pd.to_datetime(df['datetime'], format='%Y%m%d')
return df
def get_security_type(self, code):
if code in self.security_list.code.values:
return self.security_list[self.security_list.code ==
code]['sse'].as_matrix()[0]
else:
raise SecurityNotExists()
def get_security_bars(self, code, freq, index=False):
if index:
exchange = self.get_security_type(code)
func = self.api.get_index_bars
else:
exchange = get_stock_type(code)
func = self.api.get_security_bars
df = pd.DataFrame()
if freq in ['1d', 'day']:
freq = 9
elif freq in ['1m', 'min']:
freq = 8
else:
raise Exception("1d and 1m frequency supported only")
res = []
start = 0
while True:
data = func(freq, exchange, code, start, 800)
if not data:
break
res = data + res
start += 800
df = self.api.to_df(res).drop(
['year', 'month', 'day', 'hour', 'minute'], axis=1)
df['datetime'] = pd.to_datetime(df.datetime)
df['code'] = code
return df.set_index('datetime')
def _get_transaction(self, code, date):
res = []
start = 0
while True:
data = self.api.get_history_transaction_data(
get_stock_type(code), code, start, 2000, date)
if not data:
break
start += 2000
res = data + res
if len(res) == 0:
return pd.DataFrame()
df = self.api.to_df(res).assign(date=date)
df.index = pd.to_datetime(str(date) + " " + df["time"])
df['code'] = code
return df.drop("time", axis=1)
def time_and_price(self, code):
start = 0
res = []
exchange = self.get_security_type(code)
while True:
data = self.api.get_transaction_data(exchange, code, start, 2000)
if not data:
break
res = data + res
start += 2000
df = self.api.to_df(res)
df.time = pd.to_datetime(
str(pd.to_datetime('today').date()) + " " + df['time'])
df.loc[0, 'time'] = df.time[1]
return df.set_index('time')
@classmethod
def minute_bars_from_transaction(cls, transaction, freq):
if transaction.empty:
return pd.DataFrame()
data = transaction['price'].resample(freq,
label='right',
closed='left').ohlc()
data['volume'] = transaction['vol'].resample(freq,
label='right',
closed='left').sum()
data['code'] = transaction['code'][0]
return fillna(data)
def get_k_data(self, code, start, end, freq):
if isinstance(start, str) or isinstance(end, str):
start = pd.Timestamp(start)
end = pd.Timestamp(end)
sessions = pd.date_range(start, end)
trade_days = map(int, sessions.strftime("%Y%m%d"))
if freq == '1m':
freq = '1 min'
if freq == '1d':
freq = '24 H'
res = []
for trade_day in trade_days:
df = Engine.minute_bars_from_transaction(
self._get_transaction(code, trade_day), freq)
if df.empty:
continue
res.append(df)
if len(res) != 0:
return pd.concat(res)
return pd.DataFrame()
class TDXData(DataSource):
def __init__(self, db_interface: DBInterface = None, host: str = None, port: int = None):
super().__init__(db_interface)
if host is None:
conf = get_global_config()
host = conf['tdx_server']['host']
port = conf['tdx_server']['port']
self.api = TdxHq_API()
self.host = host
self.port = port
self._factor_param = utils.load_param('tdx_param.json')
self.stock_ticker = StockTickers(db_interface)
def connect(self):
self.api.connect(self.host, self.port)
def __enter__(self):
self.connect()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.api.disconnect()
def update_stock_minute(self):
"""更新股票分钟行情"""
table_name = '股票分钟行情'
db_timestamp = self.db_interface.get_latest_timestamp(table_name, dt.datetime(2015, 1, 1))
start_date = self.calendar.offset(db_timestamp.date(), 1)
end_date = dt.datetime.today()
dates = self.calendar.select_dates(start_date, end_date)
for date in dates:
self.get_stock_minute(date)
def get_stock_minute(self, date: dt.datetime) -> None:
"""获取 ``date`` 的股票分钟行情"""
minute_data = self._get_stock_minute(date)
auction_time = date + dt.timedelta(hours=9, minutes=25)
auction_db_data = self.db_interface.read_table('股票集合竞价数据', columns=['成交价', '成交量', '成交额'], dates=auction_time)
df = self.left_shift_minute_data(minute_data=minute_data, auction_db_data=auction_db_data)
self.db_interface.insert_df(df, '股票分钟行情')
def _get_stock_minute(self, date: dt.datetime) -> pd.DataFrame:
num_days = self.calendar.days_count(date, dt.date.today())
start_index = num_days * 60 * 4
tickers = self.stock_ticker.ticker(date)
storage = []
with tqdm(tickers) as pbar:
for ticker in tickers:
pbar.set_description(f'下载 {ticker} 在 {date} 的分钟数据')
code, market = self._split_ticker(ticker)
data = self.api.get_security_bars(category=8, market=market, code=code, start=start_index, count=240)
data = self._formatting_data(data, ticker)
storage.append(data)
pbar.update()
df = pd.concat(storage)
return df
def _formatting_data(self, info: OrderedDict, ticker: str) -> pd.DataFrame:
df = pd.DataFrame(info)
df['datetime'] = df['datetime'].apply(self.str2datetime)
df = df.drop(['year', 'month', 'day', 'hour', 'minute'], axis=1).rename(self._factor_param['行情数据'], axis=1)
df['ID'] = ticker
df = df.set_index(['DateTime', 'ID'], drop=True)
return df
@staticmethod
def _split_ticker(ticker: str) -> [str, int]:
code, market_str = ticker.split('.')
market = 0 if market_str == 'SZ' else 1
return code, market
@staticmethod
def str2datetime(date: str) -> dt.datetime:
return dt.datetime.strptime(date, '%Y-%m-%d %H:%M')
class PytdxApi:
"""TDX数据服务类"""
def __init__(self):
"""Constructor"""
self.connect_status: bool = False
self.login_status: bool = False
self.hq_api = None # 行情API
self.conc_code_num = 50 # 并发获取行情的股票个数
# 行情订阅
self.active = False
self.run_subscribe = Thread(target=self.get_realtime_data)
self.symbols = list()
self.symbols_split = list()
def connect_api(self, host: str = "", port: int = 0):
"""连接行情api"""
# 连接行情API并检查连接情况
try:
if not self.connect_status:
self.hq_api = TdxHq_API()
self.hq_api.connect(host, port)
self.connect_status = True
self.login_status = True
self.subscribe_start()
except Exception as e:
return e
def get_realtime_quotes(self, quotes_list: list):
"""获取实时行情数据"""
data = self.hq_api.get_security_quotes(quotes_list)
return data
def get_realtime_data(self):
"""获取实时行情切片"""
try:
while self.active:
if not self.symbols_split:
sleep(1)
continue
data = list()
for symbols in self.symbols_split:
d = self.get_realtime_quotes(symbols)
data.extend(d)
self.on_tick_data(data)
sleep(2)
except:
error = dict()
error["error_id"] = "pytdx"
error["error_msg"] = "行情订阅失败"
self.on_error(error)
def get_transaction_count(self, market: int) -> int:
"""
查询市场标的数量
"""
return self.hq_api.get_security_count(market)
def get_transaction_list(self, market: int, start: int) -> list:
"""查询市场标的列表"""
return self.hq_api.get_security_list(market, start)
def subscribe_start(self):
"""启动行情订阅"""
self.active = True
self.run_subscribe.start()
def subscribe(self, symbol: Any):
"""订阅行情数据"""
if isinstance(symbol, tuple):
if symbol not in self.symbols:
self.symbols.append(symbol)
elif isinstance(symbol, list):
for s in symbol:
if s not in self.symbols:
self.symbols.append(s)
else:
error = dict()
error["error_id"] = "pytdx"
error["error_msg"] = f"订阅标的代码格式不正确{symbol}"
self.on_error(error)
return
symbol_split = self.get_code_split()
self.symbols_split = copy(symbol_split)
def subscribe_close(self):
"""关闭订阅"""
if self.active:
self.active = False
self.run_subscribe.join()
def get_transaction_info(self):
"""获取所有合约信息"""
for exchange in list(exchange_map.values()):
count = self.get_transaction_count(exchange)
for c in range(0, count, 1000):
symbols = self.get_transaction_list(exchange, c)
for symbol in symbols:
symbol["exchange"] = exchange
if symbol["code"][:2] in ["60", "30", "688", "00"]:
symbol["product"] = 3
else:
symbol["product"] = 2
self.on_contract_info(symbol, False)
self.on_contract_info({"exchange": exchange}, True)
def get_all_stock(self):
"""获取所有股票数据"""
stocks = list()
for exchange in list(exchange_map.values()):
count = self.get_transaction_count(exchange)
for s in range(0, count, 1000):
d = self.get_transaction_list(exchange, s)
stocks.extend(d)
l = len(stocks) - 1
for i, stock in enumerate(stocks):
if stock["code"][:1] in ["60", "30", "688", "00"]:
if i == l:
self.on_contract_info(stock, True)
else:
self.on_contract_info(stock, False)
def on_contract_info(self, data: dict, last: bool) -> None:
""""""
pass
def on_tick_data(self, data):
"""订阅数据处理"""
pass
def on_error(self, err):
"""接口错误处理"""
pass
@staticmethod
def generate_symbols(symbols: list):
"""组装symbols数据，pytdx接收的是以市场代码和标的代码组成的元祖的list"""
new_symbols = []
for symbol in symbols:
code, exchange = symbol.split(".")
new_symbol = (exchange_map[exchange], code)
new_symbols.append(new_symbol)
return new_symbols
@staticmethod
def get_fast_ip():
"""获取最快IP"""
host = "210.51.39.201"
port = 7709
return host, port
@staticmethod
def check_symbol(symbol: str):
"""检查标的格式"""
if symbol:
code, market = symbol.split(".")
market = exchange_map.get(market)
return code, market
else:
return False
def get_code_split(self):
"""获得切割好的股票代码段"""
code_split_list = []
for i in range(0, len(self.symbols) + 1, self.conc_code_num):
code_split = self.symbols[i : i + self.conc_code_num]
code_split_list.append(code_split)
return code_split_list
def exit(self):
"""数据服务关闭"""
# 关闭订阅
self.subscribe_close()
# 关闭接口
self.login_status = False
self.connect_status = False
self.hq_api.disconnect()
self.hq_api = None
class Engine:
concurrent_thread_count = 50
def __init__(self, *args, **kwargs):
if 'ip' in kwargs:
self.ip = kwargs.pop('ip')
else:
if kwargs.pop('best_ip', False):
self.ip = self.best_ip
else:
self.ip = '14.17.75.71'
if 'concurrent_thread_count' in kwargs:
self.concurrent_thread_count = kwargs.pop(
'concurrent_thread_count', 50)
self.thread_num = kwargs.pop('thread_num', 1)
self.api = TdxHq_API(args, kwargs, raise_exception=True)
def connect(self):
self.api.connect(self.ip)
return self
def __enter__(self):
return self
def exit(self):
self.api.disconnect()
def __exit__(self, exc_type, exc_val, exc_tb):
self.api.disconnect()
def quotes(self, code):
code = [code] if not isinstance(code, list) else code
code = self.security_list[self.security_list.code.isin(
code)].index.tolist()
data = [
self.api.to_df(
self.api.get_security_quotes(code[80 * pos:80 * (pos + 1)]))
for pos in range(int(len(code) / 80) + 1)
]
return pd.concat(data)
# data = data[['code', 'open', 'high', 'low', 'price']]
# data['datetime'] = datetime.datetime.now()
# return data.set_index('code', drop=False, inplace=False)
def stock_quotes(self):
code = self.stock_list.index.tolist()
data = [
self.api.to_df(
self.api.get_security_quotes(code[80 * pos:80 * (pos + 1)]))
for pos in range(int(len(code) / 80) + 1)
]
return pd.concat(data)
@lazyval
def security_list(self):
return pd.concat([
pd.concat([
self.api.to_df(self.api.get_security_list(
j, i * 1000)).assign(sse=0 if j == 0 else 1).set_index(
['sse', 'code'], drop=False)
for i in range(int(self.api.get_security_count(j) / 1000) + 1)
],
axis=0) for j in range(2)
],
axis=0)
@lazyval
def stock_list(self):
aa = map(stock_filter, self.security_list.index.tolist())
return self.security_list[list(aa)]
@lazyval
def best_ip(self):
return select_best_ip()
@lazyval
def concept(self):
return self.api.to_df(
self.api.get_and_parse_block_info(TDXParams.BLOCK_GN))
@lazyval
def index(self):
return self.api.to_df(
self.api.get_and_parse_block_info(TDXParams.BLOCK_SZ))
@lazyval
def fengge(self):
return self.api.to_df(
self.api.get_and_parse_block_info(TDXParams.BLOCK_FG))
@lazyval
def block(self):
return self.api.to_df(
self.api.get_and_parse_block_info(TDXParams.BLOCK_DEFAULT))
@lazyval
def customer_block(self):
return CustomerBlockReader().get_df(CUSTOMER_BLOCK_PATH)
def xdxr(self, code):
df = self.api.to_df(
self.api.get_xdxr_info(self.get_security_type(code), code))
if df.empty:
return df
df['datetime'] = pd.to_datetime((df.year * 10000 + df.month * 100 +
df.day).apply(lambda x: str(x)))
return df.drop(['year', 'month', 'day'], axis=1).set_index('datetime')
@lazyval
def gbbq(self):
df = GbbqReader().get_df(GBBQ_PATH).query('category == 1')
df['datetime'] = pd.to_datetime(df['datetime'], format='%Y%m%d')
return df
def get_security_type(self, code):
if code in self.security_list.code.values:
return self.security_list[self.security_list.code ==
code]['sse'].as_matrix()[0]
else:
raise SecurityNotExists()
@retry(3)
def get_security_bars(self, code, freq, start=None, end=None, index=False):
if index:
exchange = self.get_security_type(code)
func = self.api.get_index_bars
else:
exchange = get_stock_type(code)
func = self.api.get_security_bars
if start:
start = start.tz_localize(None)
if end:
end = end.tz_localize(None)
if freq in ['1d', 'day']:
freq = 9
elif freq in ['1m', 'min']:
freq = 8
else:
raise Exception("1d and 1m frequency supported only")
res = []
pos = 0
while True:
data = func(freq, exchange, code, pos, 800)
if not data:
break
res = data + res
pos += 800
if start and pd.to_datetime(data[0]['datetime']) < start:
break
try:
df = self.api.to_df(res).drop(
['year', 'month', 'day', 'hour', 'minute'], axis=1)
df['datetime'] = pd.to_datetime(df.datetime)
df.set_index('datetime', inplace=True)
if freq == 9:
df.index = df.index.normalize()
except ValueError: # 未上市股票，无数据
logger.warning("no k line data for {}".format(code))
# return pd.DataFrame({
# 'amount': [0],
# 'close': [0],
# 'open': [0],
# 'high': [0],
# 'low': [0],
# 'vol': [0],
# 'code': code
# },
# index=[start]
# )
return pd.DataFrame()
close = [df.close.values[-1]]
if start:
df = df.loc[lambda df: start <= df.index]
if end:
df = df.loc[lambda df: df.index.normalize() <= end]
if df.empty:
# return pd.DataFrame({
# 'amount': [0],
# 'close': close,
# 'open': close,
# 'high': close,
# 'low': close,
# 'vol': [0],
# 'code': code
# },
# index=[start]
# )
return df
else:
if int(df['vol'][-1]) <= 0 and end == df.index[-1] and len(
df) == 1: # 成交量为0，当天返回的是没开盘的数据
return pd.DataFrame()
df['code'] = code
return df
def _get_transaction(self, code, date):
res = []
start = 0
while True:
data = self.api.get_history_transaction_data(
get_stock_type(code), code, start, 2000, date)
if not data:
break
start += 2000
res = data + res
if len(res) == 0:
return pd.DataFrame()
df = self.api.to_df(res).assign(date=date)
df.loc[0, 'time'] = df.time[1]
df.index = pd.to_datetime(str(date) + " " + df["time"])
df['code'] = code
return df.drop("time", axis=1)
def time_and_price(self, code):
start = 0
res = []
exchange = self.get_security_type(code)
while True:
data = self.api.get_transaction_data(exchange, code, start, 2000)
if not data:
break
res = data + res
start += 2000
df = self.api.to_df(res)
df.time = pd.to_datetime(
str(pd.to_datetime('today').date()) + " " + df['time'])
df.loc[0, 'time'] = df.time[1]
return df.set_index('time')
@classmethod
def minute_bars_from_transaction(cls, transaction, freq):
if transaction.empty:
return pd.DataFrame()
mask = transaction.index < transaction.index[0].normalize(
) + pd.Timedelta('12 H')
def resample(transaction):
if transaction.empty:
return pd.DataFrame()
data = transaction['price'].resample(freq,
label='right',
closed='left').ohlc()
data['volume'] = transaction['vol'].resample(freq,
label='right',
closed='left').sum()
data['code'] = transaction['code'][0]
return data
morning = resample(transaction[mask])
afternoon = resample(transaction[~mask])
if morning.empty and afternoon.empty:
return pd.DataFrame()
if not afternoon.empty:
morning.index.values[-1] = afternoon.index[0] - pd.Timedelta(
'1 min')
df = pd.concat([morning, afternoon])
return fillna(df)
def _get_k_data(self, code, freq, sessions):
trade_days = map(int, sessions.strftime("%Y%m%d"))
if freq == '1m':
freq = '1 min'
if freq == '1d':
freq = '24 H'
res = []
concurrent_count = self.concurrent_thread_count
jobs = []
for trade_day in trade_days:
# df = Engine.minute_bars_from_transaction(self._get_transaction(code, trade_day), freq)
reqevent = gevent.spawn(Engine.minute_bars_from_transaction,
self._get_transaction(code, trade_day),
freq)
jobs.append(reqevent)
if len(jobs) >= concurrent_count:
gevent.joinall(jobs, timeout=30)
for j in jobs:
if j.value is not None and not j.value.empty:
res.append(j.value)
jobs.clear()
gevent.joinall(jobs, timeout=30)
for j in jobs:
if j.value is not None and not j.value.empty:
res.append(j.value)
jobs.clear()
if len(res) != 0:
return pd.concat(res)
return pd.DataFrame()
def get_k_data(self, code, start, end, freq, check=True):
if isinstance(start, str) or isinstance(end, str):
start = pd.Timestamp(start)
end = pd.Timestamp(end)
if check:
daily_bars = self.get_security_bars(code, '1d', start, end)
if daily_bars is None or daily_bars.empty:
return daily_bars
sessions = daily_bars.index
else:
sessions = pd.bdate_range(start,
end,
weekmask='Mon Tue Wed Thu Fri')
df = self._get_k_data(code, freq, sessions)
def check_df(freq, df, daily_bars):
if freq == '1m':
need_check = pd.DataFrame({
'open':
df['open'].resample('1D').first(),
'high':
df['high'].resample('1D').max(),
'low':
df['low'].resample('1D').min(),
'close':
df['close'].resample('1D').last(),
'volume':
df['volume'].resample('1D').sum()
}).dropna()
else:
need_check = df
if daily_bars.shape[0] != need_check.shape[0]:
logger.warning("{} merged {}, expected {}".format(
code, need_check.shape[0], daily_bars.shape[0]))
need_check = fillna(
need_check.reindex(daily_bars.index, copy=False))
diff = daily_bars[['open',
'close']] == need_check[['open', 'close']]
res = (diff.open) & (diff.close)
sessions = res[res == False].index
return sessions
if not df.empty:
if check:
sessions = check_df(freq, df, daily_bars)
if sessions.shape[0] != 0:
logger.info(
"fixing data for {}-{} with sessions: {}".format(
code, freq, sessions))
fix = self._get_k_data(code, freq, sessions)
df.loc[fix.index] = fix
return df
return df
class SP(object):
def __init__(self,userid="account4",rate=1.2,products="1",limit=2,total=1000000,mock=True,server="http://192.168.118.1:65000"):
if products == "1":
self.products = {'880414': {'args': (3, 12, 90), 'stocklst': {}}, '880456': {'args': (6, 28, 40), 'stocklst': {}},
'880476': {'args': (5, 20, 90), 'stocklst': {}}, '880440': {'args': (4, 8, 85), 'stocklst': {}},
'880424': {'args': (4, 16, 110),'stocklst': {}}, '880448': {'args': (3, 14, 60), 'stocklst': {}},
'880454': {'args': (4, 8, 85), 'stocklst': {}}, '880493': {'args': (3, 24, 55), 'stocklst': {}},
'880344': {'args': (4, 4, 30), 'stocklst': {}}, '880301': {'args': (6, 20, 30), 'stocklst': {}},
'880464': {'args': (4, 14, 50), 'stocklst': {}}, '880459': {'args': (3, 16, 80), 'stocklst': {}},
'880380': {'args': (5, 22, 70), 'stocklst': {}}, '880472': {'args': (3, 32, 170),'stocklst': {}},
'880421': {'args': (3, 16, 45), 'stocklst': {}}, '880471': {'args': (4, 12, 40), 'stocklst': {}},
'880453': {'args': (6, 14, 30), 'stocklst': {}}, '880350': {'args': (4, 26, 30), 'stocklst': {}},
'880447': {'args': (5, 20, 170),'stocklst': {}}, '880351': {'args': (3, 6, 65), 'stocklst': {}},
'880390': {'args': (3, 14, 65), 'stocklst': {}}, '880406': {'args': (4, 16, 50), 'stocklst': {}},
'880305': {'args': (3, 14, 95), 'stocklst': {}}, '880492': {'args': (3, 30, 105),'stocklst': {}},
'880387': {'args': (5, 20, 70), 'stocklst': {}}, '880418': {'args': (4, 16, 100),'stocklst': {}},
'880367': {'args': (5, 14, 110),'stocklst': {}}, '880398': {'args': (5, 26, 100),'stocklst': {}},
'880437': {'args': (4, 8, 85), 'stocklst': {}}, '880474': {'args': (4, 24, 45), 'stocklst': {}},
'880324': {'args': (6, 22, 75), 'stocklst': {}}, '880335': {'args': (3, 12, 75), 'stocklst': {}},
'880372': {'args': (6, 20, 105),'stocklst': {}}, '880491': {'args': (4, 16, 20), 'stocklst': {}},
'880490': {'args': (3, 26, 110),'stocklst': {}}, '880431': {'args': (7, 12, 40), 'stocklst': {}},
'880432': {'args': (3, 10, 110),'stocklst': {}}, '880318': {'args': (4, 20, 90), 'stocklst': {}},
'880497': {'args': (3, 8, 40), 'stocklst': {}}, '880494': {'args': (3, 30, 110),'stocklst': {}},
'880400': {'args': (3, 10, 90), 'stocklst': {}}, '880360': {'args': (3, 16, 70), 'stocklst': {}},
'880310': {'args': (3, 12, 20), 'stocklst': {}}, '880423': {'args': (5, 16, 30), 'stocklst': {}},
'880430': {'args': (7, 12, 25), 'stocklst': {}}, '880422': {'args': (6, 8, 35), 'stocklst': {}},
'880465': {'args': (4, 12, 95), 'stocklst': {}}, '880482': {'args': (3, 20, 180),'stocklst': {}},
'880355': {'args': (3, 18, 45), 'stocklst': {}}, '880473': {'args': (3, 6, 40), 'stocklst': {}},
'880446': {'args': (3, 8, 45), 'stocklst': {}}, '880452': {'args': (3, 32, 90), 'stocklst': {}},
'880455': {'args': (5, 6, 20), 'stocklst': {}}, '880399': {'args': (3, 8, 85), 'stocklst': {}},
'880330': {'args': (5, 12, 80), 'stocklst': {}}, '880489': {'args': (3, 18, 190),'stocklst': {}}}
elif products == "2": #资金比较少是，选择配置部分行业
self.products = {'880414': {'args': (3, 12, 90), 'stocklst': {}}, '880456': {'args': (6, 28, 40), 'stocklst': {}},
'880476': {'args': (5, 20, 90), 'stocklst': {}}, '880440': {'args': (4, 8, 85), 'stocklst': {}},
'880424': {'args': (4, 16, 110),'stocklst': {}}, '880448': {'args': (3, 14, 60), 'stocklst': {}},
'880454': {'args': (4, 8, 85), 'stocklst': {}}, '880493': {'args': (3, 24, 55), 'stocklst': {}},
'880344': {'args': (4, 4, 30), 'stocklst': {}}, '880301': {'args': (6, 20, 30), 'stocklst': {}},
'880464': {'args': (4, 14, 50), 'stocklst': {}}, '880459': {'args': (3, 16, 80), 'stocklst': {}},
'880380': {'args': (5, 22, 70), 'stocklst': {}}, '880472': {'args': (3, 32, 170),'stocklst': {}},
'880421': {'args': (3, 16, 45), 'stocklst': {}}, '880471': {'args': (4, 12, 40), 'stocklst': {}},
'880453': {'args': (6, 14, 30), 'stocklst': {}}, '880350': {'args': (4, 26, 30), 'stocklst': {}},
'880447': {'args': (5, 20, 170),'stocklst': {}}, '880351': {'args': (3, 6, 65), 'stocklst': {}},
'880390': {'args': (3, 14, 65), 'stocklst': {}}, '880406': {'args': (4, 16, 50), 'stocklst': {}},
'880305': {'args': (3, 14, 95), 'stocklst': {}}, '880492': {'args': (3, 30, 105),'stocklst': {}},
'880387': {'args': (5, 20, 70), 'stocklst': {}}, '880418': {'args': (4, 16, 100),'stocklst': {}},
}
self.server = server
self.datatype = 1
self.userid = userid
self.limit = limit
self.api = TdxHq_API(heartbeat=True)
self.trader = None
self.trading = False
self.mock = mock
self.rate = rate #持仓杠杆率
self.total = total #默认持仓资金
self.TDX_IP_SETS = STOCK_IP_SETS
self.file_incon = FILE_INCON
self.file_tdxhy = FILE_TDXHY
self.file_tdxzs = FILE_TDXZS
def connect(self):
for ip in self.TDX_IP_SETS:
try:
if self.api.connect(ip, 7709):
return
except:
pass
def getdata(self,product,market=1,number=5000,pn=400):
data = []
for i in range(int(number/pn)+1):
temp = self.api.get_index_bars(self.datatype, market, product, (int(number/pn)-i)*pn,pn)
if not temp or len(temp)<pn:
self.connect()
for _ in range(2):
temp = self.api.get_index_bars(self.datatype, market, product, (int(number/pn)-i)*pn,pn)
if not temp or len(temp)<pn:
logger.info("record not reach the limit!")
else:
break
data += temp
df = self.api.to_df(data)[["open","close","high","low","datetime"]]
df.set_index("datetime",inplace=True,drop=False)
return df
def disconnect(self):
logger.info("[DISCONNECT]:statrt disconnect!!!!!")
if self.istradeday:
self.api.disconnect()
logger.info("[DISCONNECT]:disconnect finished!!!!!")
def updatetotal(self):
'''更新总资金
'''
accountinfo,holdlists = self.trader.position()
nhg = holdlists[holdlists[u"证券代码"].map(lambda x:x in ["SZRQ88","SHRQ88"])][u"最新市值"].sum()
self.total = accountinfo.ix["总资产"]["人民币"]-nhg
return self.total
def set_permoney(self):
'''单个品种资金上限
'''
self.permoney = self.total * self.rate /len(self.products)
return self.permoney
def _get_incon(self,):
'''获取行业分类代码
'''
f= open(self.file_incon, "rb")
data = f.read()
strings = data.decode("gbk", 'ignore').rstrip("\x00").replace("\r\n","\n")
data = strings.split("######")
rst = {}
for hystr in data:
key = re.findall(r'#.*',hystr)
if key == ['#TDXNHY']:
hylst = hystr.replace("#TDXNHY","").strip("\n").split("\n")
for item in hylst:
k,v = item.split("|")
rst[k] = [v]
return rst
def _get_tdxhy(self,islocal=True):
'''获取股票和行业对应列表
'''
if islocal:
stocklist = HY_WEIGHT.keys()
else:
stocklist = list(ts.get_stock_basics().index) #获取全市场股票代码
rst = self._get_incon()
f= open(self.file_tdxhy, "rb")
data = f.read().decode("gbk", 'ignore').rstrip("\x00").replace("\r\n","\n").strip("\n").split("\n")
for i in data:
_,code,tdxhy,_,_ = i.split("|")
if tdxhy != "T00" and code in stocklist:
rst[tdxhy].append(code)
return rst
def _get_tdxzs(self,islocal=True):
'''生成通达性版块代码对应股票列表
'''
dct = {}
rst = self._get_tdxhy(islocal=islocal)
f= open(self.file_tdxzs, "rb")
data = f.read().decode("gbk", 'ignore').rstrip("\x00").replace("\r\n","\n").strip("\n").split("\n")
for i in data:
name,code,_,_,_,hy = i.split("|")
code = int(code)
if 880301<=code and 880497>=code and hy in rst.keys() :
k = hy[:5]
if not dct.__contains__(k):
dct[k] = {"name":"","code":"","stocklist":[]}
if k==hy:
dct[k]["name"] = name
dct[k]["code"] = code
dct[k]["stocklist"].extend(rst[hy][1:])
return dct
def get_tdxhy_list(self,islocal=True):
'''获取通达信行业板块指数对应的股票列表
'''
return self._get_tdxzs(islocal)
def get_weight(self,htlist={},islocal=True):
'''获取行业板块个股权重，流动市值为权重系数
备注：回测是为方便处理，以最后一天的权重系数作为历史上的权重
'''
if islocal:
self.weight = HY_WEIGHT
else:
if not htlist:
htlist = self.get_tdxhy_list(islocal)
tasks = []
for v in htlist.values():
tasks.append(gevent.spawn(self.get_latest_ltsz,v["stocklist"]))
gevent.joinall(tasks)
return self.weight
def get_latest_ltsz(self,stocks=[]):
'''获取最新流通市值,千万为单位，取整
'''
unit = 10000000
for code in stocks:
mk = self._select_market_code(code)
print(mk,code)
try:
ltgb = self.api.get_finance_info(mk,code)["liutongguben"]
price = self.api.get_security_bars(4,mk,code,0,1)[0]["close"]
ltsz = int(ltgb*price/unit)
self.weight[code] = ltsz
except:
print("*****",code)
return
def set_instrument(self):
'''设置交易股票
'''
func = lambda x :1 if x.startswith("6") else 0
weight = self.get_weight()
for v in self.get_tdxhy_list().values():
code = str(v["code"])
if not self.products.__contains__(code):continue
stocks = [(i,weight[i]) for i in v["stocklist"]]
limit_stocks = sorted(stocks,key=lambda x:x[1],reverse=True)[:self.limit]
total = sum([i[1] for i in limit_stocks])
for i in limit_stocks:
market = func(i[0])
price = self.api.get_security_bars(4,market,i[0],0,1)[0]["close"]
number = int(self.permoney*i[1]/total/price/100)*100
self.products[code]["stocklst"][i[0]] = number
return self.products
def judgetradeday(self,):
today = datetime.datetime.today().date().strftime('%Y-%m-%d')
df = ts.trade_cal()
return df[(df["calendarDate"]==today)].isOpen.values[0]
@property
def istradeday(self):
if not hasattr(self, "_istradeday"):
self._istradeday = self.judgetradeday()
return self._istradeday
def initial(self):
'''每天初始化设置
'''
logger.info("[INITIAL]:start initial !!!!!!")
if not self.istradeday:
self.trading = False
return
self.trading = True
logger.info("[INITIAL]:try to create connect... ")
self.connect()
self.trader = trade(UserID=self.userid,api=self.api,mock=self.mock,server=self.server)
logger.info("[INITIAL]:connect successful!")
logger.info("[INITIAL]:initial account info...")
self.updatetotal() #更新账户总资金
self.set_permoney() #设置单个品种资金上限
logger.info("[INITIAL]:set per product money limit:{}".format(self.permoney))
self.set_instrument() #设置交易股票和手数
logger.info("[INITIAL]:set stock list succcessful !!!")
logger.info("[INITIAL]:initial finished!!!!!!")
def handledata(self,df,args=[]):
df.loc[:,"number"] = range(df.shape[0])
s,m,l = args
for i in args:#5 15 60 D
key = str(5*i)
df.loc[:,key+"high"] = df["high"].rolling(i).max()
df.loc[:,key+"low"] = df["low"].rolling(i).min()
df.loc[:,key+"atr"] = (df[key+"high"]-df[key+"low"]).rolling(10*i).mean()
df.loc[:,key+"med"] = (df[key+"high"]+df[key+"low"])/2
df.loc[:,key+"HH"] = df[key+"med"] + 1.5*df[key+"atr"]
df.loc[:,key+"LL"] = df[key+"med"] - 1.5*df[key+"atr"]
df.loc[:,key+'HHmax'] = df[key+'HH'].rolling(10*i).max()
df.loc[:,key+'LLmin'] = df[key+'LL'].rolling(10*i).min()
df.loc[df[key+'HH']>=df[key+'HHmax'],key+'hmark'] = df["number"]
df.loc[df[key+'LL']<=df[key+'LLmin'],key+'lmark'] = df["number"]
df[key+'hmark'].fillna(method="ffill",inplace=True)
df[key+'hmark'].fillna(0,inplace=True)
df[key+'lmark'].fillna(method="ffill",inplace=True)
df[key+'lmark'].fillna(0,inplace=True)
df.loc[:,key+'UP'] = df[key+'hmark'] >= df[key+'lmark']
# debuginfo.append({key+'hmark':df.iloc[-1][key+'hmark'],key+'lmark':df.iloc[-1][key+'lmark']})
df.fillna(method="ffill",inplace=True)
df.dropna(inplace=True)
# logger.info("trademessage:{}".format(debuginfo))
result = (df.iloc[-1][str(5*l)+"UP"] >0)&(df.iloc[-1][str(5*s)+"UP"]>0)
result |= (df.iloc[-1][str(5*l)+"UP"]>0)&(df.iloc[-1][str(5*m)+"UP"]>0)
result |= (df.iloc[-1][str(5*l)+"UP"]<=0)&(df.iloc[-1][str(5*m)+"UP"]>0)&(df.iloc[-1][str(5*s)+"UP"]>0)
return result
def sync(self,idx,director=True):
stocks = self.products[idx]["stocklst"]
for stock,number in stocks.items():
if not director: number = 0 #空信号,清仓
#判断现有持仓
try:
code = stock
h_number = self.hd_df.ix[code]["证券数量"]
except:
h_number = 0
logger.info("[RUN]:{},{},{}".format(stock,h_number,number))
#补仓差
cangcha = int((number-h_number)/100)*100
if h_number>0 and abs(cangcha)/h_number<0.2: #如果有持仓，同时仓差小于10% 不进行更改，为了处理频繁加减仓达到问题
continue
if cangcha>0:
logger.info("[RUN]:buy code:{}, number:{}".format(stock,number-h_number))
self.buy(stock,cangcha)
elif cangcha<0:
couldsell = self.hd_df.ix[code]["可卖数量"]
logger.info("[RUN]:sell code:{}, number:{},couldsell:{}".format(stock,h_number-number,couldsell))
if couldsell >0:
self.sell(stock,min(-cangcha,couldsell))
#
def buy(self,stock,number):
self.trader.buy(stock, number)
def sell(self,stock,number):
self.trader.sell(stock, number)
def check_position(self,status):
'''检查仓位情况
'''
self.handleposition()
handlelist = []
for ins,v in status.items():
if ins not in self.g_df.index :
handlelist.append(ins)
elif self.g_df.ix[ins]["Position"] != v["number"]:
handlelist.append(ins)
return handlelist
def handleposition(self):
'''计算多仓，空仓，以及昨仓和今仓
'''
self.trader.cancelorder() #先尝试撤单
_,holdlists = self.trader.position()
self.hd_df = holdlists
if holdlists.shape[0]>0:
self.hd_df.set_index("证券代码",inplace=True)
self.hd_df.index = self.hd_df.index.astype(np.str).map(lambda x:x if len(x)>=6 else "0"*(6-len(x))+x)
return self.hd_df
def run(self):
logger.info("[RUN]:start run !!!!!")
if not self.trading:
return
self.handleposition()
rst = {}
for idx in list(self.products.keys()):
director = self.handledata(self.getdata(idx,market=1),self.products[idx]["args"]) #用指数出信号
logger.info("[RUN]:trademessage: block:{}, director:{}".format(idx,director))
self.sync(idx,director)
rst[idx] = {"up":director,"number":self.products[idx]["stocklst"],"product":idx}
logger.info("[RUN]:lastest position status:{}".format(rst))
logger.info("[RUN]:run finished !!!!!")
def __call__(self):
total_count = 0
try:
connect = sqlite3.connect(self.sqlitefile)
except Exception as e:
#self.queue.put([self.msg_name, str(e), -1, 0, total_count])
self.queue.put([self.msg_name, 'INFO', str(e), 0, 0])
self.queue.put([self.msg_name, '', 0, None, total_count])
return
try:
download_dir = self.dest_dir + "/downloads"
if not os.path.lexists(download_dir):
os.makedirs(download_dir)
self.queue.put([self.msg_name, '正在下载钱龙权息信息...', 0, 0, 0])
net_file = urllib.request.urlopen(
'http://www.qianlong.com.cn/download/history/weight.rar',
timeout=60)
buffer = net_file.read()
self.queue.put(
[self.msg_name, '钱龙权息信息下载完成，正在校验是否存在更新...', 0, 0, 0])
new_md5 = hashlib.md5(buffer).hexdigest()
dest_filename = download_dir + '/weight.rar'
old_md5 = None
if os.path.exists(dest_filename):
with open(dest_filename, 'rb') as oldfile:
old_md5 = hashlib.md5(oldfile.read()).hexdigest()
#如果没变化不需要解压导入
if new_md5 != old_md5:
with open(dest_filename, 'wb') as file:
file.write(buffer)
self.queue.put([self.msg_name, '正在解压钱龙权息信息...', 0, 0, 0])
x = os.system('unrar x -o+ -inul {} {}'.format(
dest_filename, download_dir))
if x != 0:
raise Exception("无法找到unrar命令！")
self.queue.put([self.msg_name, '正在导入钱龙权息数据...', 0, 0, 0])
total_count = qianlong_import_weight(connect,
download_dir + '/weight',
'SH')
total_count += qianlong_import_weight(connect,
download_dir + '/weight',
'SZ')
self.queue.put(
[self.msg_name, '导入钱龙权息数据完毕!', 0, 0, total_count])
else:
self.queue.put([self.msg_name, '钱龙权息数据无变化', 0, 0, 0])
hosts = search_best_tdx()
api = TdxHq_API()
api.connect(hosts[0][2], hosts[0][3])
self.queue.put([self.msg_name, '下载通达信权息信息(上证)...', 0, 0, 0])
x = pytdx_import_finance(connect, api, "SH")
self.queue.put([self.msg_name, '下载通达信权息信息(深证)...', 0, 0, 0])
x += pytdx_import_finance(connect, api, "SZ")
self.queue.put([self.msg_name, '导入通达信权息信息完毕!', 0, 0, x])
api.disconnect()
except Exception as e:
#self.queue.put([self.msg_name, str(e), -1, 0, total_count])
self.queue.put([self.msg_name, 'INFO', str(e), 0, 0])
finally:
connect.commit()
connect.close()
self.queue.put([self.msg_name, '', 0, None, total_count])
net_file = urllib.request.urlopen('http://www.qianlong.com.cn/download/history/weight.rar', timeout=60)
dest_filename = dest_dir + '/weight.rar'
with open(dest_filename, 'wb') as file:
file.write(net_file.read())
print("下载完成，正在解压...")
os.system('unrar x -o+ -inul {} {}'.format(dest_filename, dest_dir))
print("解压完成，正在导入...")
add_count = qianlong_import_weight(connect, dest_dir + '/weight', 'SH')
add_count += qianlong_import_weight(connect, dest_dir + '/weight', 'SZ')
print("导入数量：", add_count)
print("\n导入上证分笔数据")
#add_count = import_trans(connect, 'SH', quotations, api, dest_dir, max_days=2, progress=ProgressBar)
print("\n导入数量：", add_count)
print("\n导入上证分时数据")
add_count = import_time(connect, 'SZ', quotations, api, dest_dir, max_days=3, progress=ProgressBar)
print("\n导入数量：", add_count)
"""
api.disconnect()
connect.close()
endtime = time.time()
print("\nTotal time:")
print("%.2fs" % (endtime - starttime))
print("%.2fm" % ((endtime - starttime) / 60))
def __call__(self):
total_count = 0
try:
connect = sqlite3.connect(self.sqlitefile)
except Exception as e:
#self.queue.put([self.msg_name, str(e), -1, 0, total_count])
self.queue.put([self.msg_name, 'INFO', str(e), 0, 0])
self.queue.put([self.msg_name, '', 0, None, total_count])
return
try:
download_dir = self.dest_dir + "/downloads"
if not os.path.lexists(download_dir):
os.makedirs(download_dir)
self.queue.put([self.msg_name, '正在下载钱龙权息信息...', 0, 0, 0])
net_file = urllib.request.urlopen('http://www.qianlong.com.cn/download/history/weight.rar', timeout=60)
buffer = net_file.read()
self.queue.put([self.msg_name, '钱龙权息信息下载完成，正在校验是否存在更新...', 0, 0, 0])
new_md5 = hashlib.md5(buffer).hexdigest()
dest_filename = download_dir + '/weight.rar'
old_md5 = None
if os.path.exists(dest_filename):
with open(dest_filename, 'rb') as oldfile:
old_md5 = hashlib.md5(oldfile.read()).hexdigest()
#如果没变化不需要解压导入
if new_md5 != old_md5:
with open(dest_filename, 'wb') as file:
file.write(buffer)
self.queue.put([self.msg_name, '正在解压钱龙权息信息...', 0, 0, 0])
x = os.system('unrar x -o+ -inul {} {}'.format(dest_filename, download_dir))
if x != 0:
raise Exception("无法找到unrar命令！")
self.queue.put([self.msg_name, '正在导入钱龙权息数据...', 0, 0, 0])
total_count = qianlong_import_weight(connect, download_dir + '/weight', 'SH')
total_count += qianlong_import_weight(connect, download_dir + '/weight', 'SZ')
self.queue.put([self.msg_name, '导入钱龙权息数据完毕!', 0, 0, total_count])
else:
self.queue.put([self.msg_name, '钱龙权息数据无变化', 0, 0, 0])
hosts = search_best_tdx()
api = TdxHq_API()
api.connect(hosts[0][2], hosts[0][3])
self.queue.put([self.msg_name, '下载通达信权息信息(上证)...', 0, 0, 0])
x = pytdx_import_finance(connect, api, "SH")
self.queue.put([self.msg_name, '下载通达信权息信息(深证)...', 0, 0, 0])
x += pytdx_import_finance(connect, api, "SZ")
self.queue.put([self.msg_name, '导入通达信权息信息完毕!', 0, 0, x])
api.disconnect()
except Exception as e:
#self.queue.put([self.msg_name, str(e), -1, 0, total_count])
self.queue.put([self.msg_name, 'INFO', str(e), 0, 0])
finally:
connect.commit()
connect.close()
self.queue.put([self.msg_name, '', 0, None, total_count])
def __call__(self):
total_count = 0
try:
if self.config.getboolean('hdf5', 'enable', fallback=True):
sqlite_file = "{}/stock.db".format(self.config['hdf5']['dir'])
connect = sqlite3.connect(sqlite_file, timeout=1800)
pytdx_import_weight = pytdx_import_weight_to_sqlite
self.logger.debug('use sqlite import weight')
else:
db_config = {
'user': self.config['mysql']['usr'],
'password': self.config['mysql']['pwd'],
'host': self.config['mysql']['host'],
'port': self.config['mysql']['port']
}
connect = mysql.connector.connect(**db_config)
pytdx_import_weight = pytdx_import_weight_to_mysql
self.logger.debug('use mysql import weight')
except Exception as e:
#self.queue.put([self.msg_name, str(e), -1, 0, total_count])
self.queue.put([self.msg_name, 'INFO', str(e), 0, 0])
self.queue.put([self.msg_name, '', 0, None, total_count])
return
try:
"""
download_dir = self.dest_dir + "/downloads"
if not os.path.lexists(download_dir):
os.makedirs(download_dir)
self.queue.put([self.msg_name, '正在下载钱龙权息信息...', 0, 0, 0])
net_file = urllib.request.urlopen(
'http://www.qianlong.com.cn/download/history/weight.rar', timeout=60
)
buffer = net_file.read()
self.queue.put([self.msg_name, '钱龙权息信息下载完成，正在校验是否存在更新...', 0, 0, 0])
new_md5 = hashlib.md5(buffer).hexdigest()
dest_filename = download_dir + '/weight.rar'
old_md5 = None
if os.path.exists(dest_filename):
with open(dest_filename, 'rb') as oldfile:
old_md5 = hashlib.md5(oldfile.read()).hexdigest()
#如果没变化不需要解压导入
if new_md5 != old_md5:
with open(dest_filename, 'wb') as file:
file.write(buffer)
self.queue.put([self.msg_name, '正在解压钱龙权息信息...', 0, 0, 0])
x = os.system('unrar x -o+ -inul {} {}'.format(dest_filename, download_dir))
if x != 0:
raise Exception("无法找到unrar命令！")
self.queue.put([self.msg_name, '正在导入钱龙权息数据...', 0, 0, 0])
total_count = qianlong_import_weight(connect, download_dir + '/weight', 'SH')
total_count += qianlong_import_weight(connect, download_dir + '/weight', 'SZ')
self.queue.put([self.msg_name, '导入钱龙权息数据完毕!', 0, 0, total_count])
else:
self.queue.put([self.msg_name, '钱龙权息数据无变化', 0, 0, 0])
"""
hosts = search_best_tdx()
api = TdxHq_API()
api.connect(hosts[0][2], hosts[0][3])
self.queue.put([self.msg_name, '正在导入权息数据...', 0, 0, 0])
total_count = pytdx_import_weight(api, connect, "SH")
total_count += pytdx_import_weight(api, connect, "SZ")
self.queue.put([self.msg_name, '导入权息数据完毕!', 0, 0, total_count])
#self.queue.put([self.msg_name, '下载通达信财务信息(上证)...', 0, 0, 0])
#x = pytdx_import_finance(connect, api, "SH")
#self.queue.put([self.msg_name, '下载通达信财务信息(深证)...', 0, 0, 0])
#x += pytdx_import_finance(connect, api, "SZ")
#self.queue.put([self.msg_name, '导入通达信财务信息完毕!', 0, 0, x])
api.disconnect()
except Exception as e:
self.logger.error(e)
#self.queue.put([self.msg_name, str(e), -1, 0, total_count])
self.queue.put([self.msg_name, 'INFO', str(e), 0, 0])
finally:
connect.commit()
connect.close()
self.queue.put([self.msg_name, '', 0, None, total_count])
'changqifuzhai': '长期负债',
'zibengongjijin': '资本公积金',
'jingzichan': '净资产',
'zhuyingshouru': '主营收入',
'zhuyinglirun': '主营利润',
'yingshouzhangkuan': '应收账款',
'yingyelirun': '营业利润',
'touzishouyu': '投资收益',
'jingyingxianjinliu': '经营现金流',
'zongxianjinliu': '总现金流',
'cunhuo': '存货',
'lirunzonghe': '利润总和',
'shuihoulirun': '税后利润',
'jinglirun': '净利润',
'weifenpeilirun': '未分配利润',
'meigujingzichan': '每股净资产',
'baoliu2': 'baoliu2'
})
#print finance_df
#finance_df.to_csv(r"..\all_other_data\test_finance_info.csv", encoding="gbk", quoting=csv.QUOTE_NONE, index=False)
# string_columns = ['代码']
# finance_df[string_columns] = finance_df[string_columns].applymap(
# lambda x: '=""' if type(x) is float else '="' + str(x) + '"')
finance_df.to_csv(r"..\all_other_data\all_finance_info.csv",
encoding="gbk",
quoting=csv.QUOTE_NONE,
index=False)
TDXHQ.disconnect()
class TdxHelper:
ip_list = [{
'ip': '119.147.212.81',
'port': 7709
}, {
'ip': '60.12.136.250',
'port': 7709
}]
def __init__(self):
#连接tdx接口
self.api = TdxHq_API()
if not self.api.connect('60.12.136.250', 7709):
print("服务器连接失败！")
# pandas数据显示设置
pd.set_option('display.max_columns', None) # 显示所有列
#pd.set_option('display.max_rows', None) # 显示所有行
# mysql对象
self.mysql = mysqlHelper(config.mysql_host, config.mysql_username,
bluedothe.mysql_password, config.mysql_dbname)
# pandas的mysql对象
self.engine = create_engine(
f'mysql+pymysql://{config.mysql_username}:{bluedothe.mysql_password}@{config.mysql_host}/{config.mysql_dbname}?charset=utf8'
)
#断开tdx接口连接
def close_connect(self):
self.api.disconnect()
#获取k线，最后一个参数day,说明需要获取的数量，本接口只获取从最近交易日往前的数据
#输入参数：五个参数分别为：category（k线),市场代码(0:深圳，1:上海),股票代码,开始位置(从最近交易日向前取，0表示最近交易日),返回的记录条数
#K线种类: 0 5分钟K线; 1 15分钟K线; 2 30分钟K线; 3 1小时K线; 4 日K线;5 周K线;6 月K线;7 1分钟;8 1分钟K线; 9 日K线;10 季K线;11 年K线
#返回值：open,close,high,low,vol,amount,year,month,day,hour,minute,datetime
# csv格式：code,ts_code,trade_date(缩写）,trade_time,time_index,open,high,low,close,amount,volume
def get_security_bars(self, category, market, code, start=0, count=240):
dict = {0: 'SZ', 1: 'SH'}
ts_code = code + "." + dict[market]
order = [
'code', 'ts_code', 'trade_date', 'trade_time', 'time_index',
'open', 'high', 'low', 'close', 'amount', 'volume'
]
#df = self.api.get_security_bars(9, 0, '000001', 0, 10) # 返回普通list
df = self.api.to_df(
self.api.get_security_bars(category, market, code, start,
count)) # 返回DataFrame
if df.empty: return df
df.insert(0, 'ts_code', ts_code)
df.insert(0, 'code', code)
df['trade_time'] = df['datetime'].apply(lambda x: str(x)[11:19])
df['time_index'] = df['trade_time'].apply(
lambda x: datatime_util.stockTradeTime2Index(x))
df['trade_date'] = df['datetime'].apply(
lambda x: (str(x)[0:10]).replace('-', ''))
df.rename(columns={'vol': 'volume'}, inplace=True)
df.drop(['year', 'month', 'day', 'hour', 'minute', 'datetime'],
axis=1,
inplace=True)
df['volume'] = df['volume'].apply(lambda x: int(x)) #取整
df.loc[df['amount'] == 5.877471754111438e-39,
'amount'] = 0 #列值根据条件筛选后修改为0
df = df[order]
filename = config.tdx_csv_minline1_all + ts_code + ".csv"
if os.path.isfile(filename):
df.to_csv(filename,
index=False,
mode='a',
header=False,
sep=',',
encoding="utf_8_sig")
else:
df.to_csv(filename,
index=False,
mode='w',
header=True,
sep=',',
encoding="utf_8_sig")
print("新增加的一分钟all股票数据：", filename)
# 获取1分钟k线，最后一个参数说明需要获取的数量，本接口只获取从最近交易日往前的数据
# 输入参数：五个参数分别为：category（k线),市场代码(0:深圳，1:上海),股票代码,开始位置(从最近交易日向前取，0表示最近交易日),返回的记录条数
# K线种类: 0 5分钟K线; 1 15分钟K线; 2 30分钟K线; 3 1小时K线; 4 日K线;5 周K线;6 月K线;7 1分钟;8 1分钟K线; 9 日K线;10 季K线;11 年K线
# 返回值：open,close,high,low,vol,amount,year,month,day,hour,minute,datetime
# csv格式：code,ts_code,trade_date(缩写）,trade_time,time_index,open,high,low,close,amount,volume
def get_security_bars_minute1(self, category, market, code, start, count):
dict = {0: 'SZ', 1: 'SH'}
ts_code = code + "." + dict[market]
order = [
'code', 'ts_code', 'trade_date', 'trade_time', 'time_index',
'open', 'high', 'low', 'close', 'amount', 'volume'
]
# df = self.api.get_security_bars(9, 0, '000001', 0, 10) # 返回普通list
df = self.api.to_df(
self.api.get_security_bars(category, market, code, start,
count)) # 返回DataFrame
if df.empty: return
df.insert(0, 'ts_code', ts_code)
df.insert(0, 'code', code)
df['trade_time'] = df['datetime'].apply(lambda x: str(x)[11:19])
df['time_index'] = df['trade_time'].apply(
lambda x: datatime_util.stockTradeTime2Index(x))
df['trade_date'] = df['datetime'].apply(
lambda x: (str(x)[0:10]).replace('-', ''))
df.rename(columns={'vol': 'volume'}, inplace=True)
df.drop(['year', 'month', 'day', 'hour', 'minute', 'datetime'],
axis=1,
inplace=True)
df['volume'] = df['volume'].apply(lambda x: int(x)) # 取整
df.loc[df['amount'] == 5.877471754111438e-39,
'amount'] = 0 # 列值根据条件筛选后修改为0
df = df[order]
#过滤掉停牌的数据，在tdx中，停牌股票也能取到数据，价格是前一交易日的收盘价，所以只能用成交量或成交金额为0来判断
#1按日期分组后取出成交量为0的日期；2循环过滤掉成交量为0的日期的数据。
dfg = df.groupby(by='trade_date').mean() #分组
dfg['trade_date'] = dfg.index
dfg = dfg[dfg.volume == 0] #条件过滤，保留满足条件的数据
for trade_date in dfg['trade_date'].values:
df = df[(df['trade_date'] != trade_date)] # 每个条件要用括号()括起来
return df
#可以获取多只股票的行情信息
#返回值：market,code,active1,price,last_close,open,high,low,reversed_bytes0,reversed_bytes1,vol,cur_vol,amount,s_vol,
#reversed_bytes2,reversed_bytes3,bid1,ask1,bid_vol1,ask_vol1,bid2,ask2,bid_vol2,ask_vol2,bid3,ask3,bid_vol3,ask_vol3,bid4,
#ask4,bid_vol4,ask_vol4,bid5,ask5,bid_vol5,ask_vol5,reversed_bytes4,reversed_bytes5,reversed_bytes6,reversed_bytes7,
#reversed_bytes8,reversed_bytes9,active2
def get_security_quotes(self):
df = self.api.to_df(
self.api.get_security_quotes([(0, '000001'), (1, '600300')]))
print(df)
# 获取市场股票数量
#返回值：value
def get_security_count(self):
df = self.api.to_df(self.api.get_security_count(0)) #0 - 深圳， 1 - 上海
print(df)
# 获取股票列表，返回值里面除了股票，还有国债等
#返回值：code,volunit,decimal_point,name,pre_close
def get_security_list(self):
df = self.api.to_df(self.api.get_security_list(
0, 10000)) # 市场代码, 起始位置 如： 0,0 或 1,100
print(df)
# 获取指数k线
#输入参数同股票k线接口
# 返回值：open,close,high,low,vol,amount,year,month,day,hour,minute,datetime,up_count down_count
def get_index_bars(self):
index_dict_cn = {
"上证指数": "999999",
"深证成指": "399001",
"中小板指": "399005",
"创业板指": "399006",
"深证综指": "399106",
"上证50": "000016",
"沪深300": "000300"
}
index_dict = {
"sh": "999999",
"sz": "399001",
"zxb": "399005",
"cyb": "399006",
"szz": "399106",
"sz50": "000016",
"hs300": "000300"
}
for key in index_dict.keys():
df = self.api.to_df(
self.api.get_index_bars(9, 1, index_dict[key], 0, 2))
print(df)
# 查询分时行情，最近交易日的数据，一分钟一条记录
#返回值：price,vol
def get_minute_time_data(self):
df = self.api.to_df(self.api.get_minute_time_data(
1, '600300')) #市场代码， 股票代码
print(df)
# 查询历史分时行情
# 返回值：price,vol
def get_history_minute_time_data(self):
df = self.api.to_df(
self.api.get_history_minute_time_data(TDXParams.MARKET_SH,
'603887',
20200420)) #市场代码， 股票代码，时间
print(df)
# 查询分笔成交，最近交易日数据
#返回值：time，price，vol，num，buyorsell
def get_transaction_data(self):
df = self.api.to_df(
self.api.get_transaction_data(TDXParams.MARKET_SZ, '000001', 0,
30)) #市场代码， 股票代码，起始位置， 数量
print(df)
# 查询历史分笔成交
#返回值:time,price,vol,buyorsell
def get_history_transaction_data(self):
df = self.api.to_df(
self.api.get_history_transaction_data(
TDXParams.MARKET_SZ, '000001', 0, 10,
20170209)) #市场代码， 股票代码，起始位置，日期 数量
print(df)
# 查询公司信息目录,返回的不是具体数据
#返回值:name,filename,start,length
def get_company_info_category(self):
df = self.api.to_df(
self.api.get_company_info_category(TDXParams.MARKET_SZ,
'000001')) #市场代码， 股票代码
print(df)
# 读取公司信息详情
#返回值:value
def get_company_info_content(self):
df = self.api.to_df(
self.api.get_company_info_content(
0, '000001', '000001.txt', 0,
1000)) #市场代码， 股票代码, 文件名, 起始位置， 数量
print(df)
# 读取除权除息信息
#返回值:year,month,day,category,name,fenhong,peigujia,songzhuangu,peigu
def get_xdxr_info(self):
df = self.api.to_df(self.api.get_xdxr_info(1, '600300')) #市场代码， 股票代码
print(df)
# 读取财务信息
#返回值:market,code,liutongguben,province,industry,updated_date,ipo_date,zongguben,guojiagu,faqirenfarengu,farengu,bgu,hgu,zhigonggu,
#zongzichan,liudongzichan,gudingzichan,wuxingzichan,gudongrenshu,liudongfuzhai,changqifuzhai,zibengongjijin,jingzichan,zhuyingshouru,
#zhuyinglirun,yingshouzhangkuan,yingyelirun,touzishouyu,jingyingxianjinliu,zongxianjinliu,cunhuo,lirunzonghe,shuihoulirun,jinglirun,weifenlirun,baoliu1,baoliu2
def get_finance_info(self):
df = self.api.to_df(self.api.get_finance_info(1,
'600300')) #市场代码， 股票代码
print(df)
# 读取k线信息
# 返回值:value
def get_k_data(self):
df = self.api.to_df(
self.api.get_k_data('600300', '2017-07-03',
'2017-07-10')) #股票代码， 开始时间， 结束时间
print(df)
# 读取板块信息
#返回值：blockname, block_type, code_index, code
""" BLOCK_SZ = "block_zs.dat";BLOCK_FG = "block_fg.dat";BLOCK_GN = "block_gn.dat";BLOCK_DEFAULT = "block.dat" """
def get_and_parse_block_info(self):
##指数板块 风格板块 概念板块 一般板块
block_filename = [
"block_zs.dat", "block_fg.dat", "block_gn.dat", "block.dat"
]
for block in block_filename:
df = self.api.to_df(
self.api.get_and_parse_block_info(block)) #板块文件名称
filename = config.tdx_csv_block + block[0:-4] + ".csv"
if os.path.isfile(filename):
os.remove(filename)
df.to_csv(filename,
index=False,
mode='w',
header=True,
sep=',',
encoding="utf_8_sig")
else:
df.to_csv(filename,
index=False,
mode='w',
header=True,
sep=',',
encoding="utf_8_sig")
# 读取板块信息，多个类型封装到一个df对象中返回
# 返回值：data_source, block_category, block_type, block_name, block_code, ts_code, create_time
def update_block_member(self):
##指数板块 风格板块 概念板块 一般板块
#block_filename = ["block_zs.dat", "block_fg.dat", "block_gn.dat", "block.dat"]
block_filename = ["block_zs.dat", "block_fg.dat",
"block_gn.dat"] #block.dat中的数据都包含在其他版块里了，这个可以去掉
data_source = "tdx"
dfall = None
for block in block_filename:
df = self.api.to_df(
self.api.get_and_parse_block_info(block)) # 板块文件名称
df['data_source'] = data_source
if block == "block.dat":
df['block_category'] = data_source + ".yb"
else:
df['block_category'] = data_source + "." + block[6:8]
df['block_type'] = df['block_type'].map(lambda x: str(x))
df['block_type'] = df['block_category'].str.cat(
df['block_type'], sep=".") #, sep = "."
df['block_code'] = "" #使用pd直接插入到数据库时，字段不能是None值
df['ts_code'] = df['code'].apply(lambda x: x + ".SH"
if x[0:1] == "6" else x + ".SZ")
if (dfall is not None) and (not dfall.empty):
dfall = dfall.append(df, ignore_index=True)
else:
dfall = df
if (dfall is None) or (dfall.empty): return None
dfall.rename(columns={'blockname': 'block_name'}, inplace=True)
dfall['create_time'] = time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time.time()))
dfall = dfall[[
'data_source', 'block_category', 'block_type', 'block_name',
'block_code', 'ts_code', 'create_time'
]] #列重排序
#分组统计
dfg = dfall.groupby(by=[
'data_source', 'block_category', 'block_type', 'block_name',
'block_code'
],
as_index=False).count() # 分组求每组数量
dfg.rename(columns={'ts_code': 'member_count'},
inplace=True) #ts_code列数值为汇总值，需要重命名
dfg['create_time'] = time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time.time())) #create_time列数值为汇总值，需要重新赋值
delete_condition = f"data_source = '{data_source}'"
mysql_script.df2db_update(delete_condition=delete_condition,
block_basic_df=dfg,
block_member_df=dfall)
return (len(dfg), len(dfall))
#获取一段时间的1分钟数据，因为每次调用接口只能返回3天的分钟数据（240*3)，需要分多次调用
#返回值：0没有提取到数据；1提取到数据
def get_minute1_data(self, category, market, code, start_date, end_date):
init_start_date = start_date.replace('-', '')
init_end_date = end_date.replace('-', '')
day = datatime_util.diffrentPeriod(datatime_util.DAILY, start_date,
end_date)
df = self.get_security_bars_minute1(category, market, code, 0,
240 * 3) # 返回DataFrame
if df is None or df.empty:
print('{0}没有交易数据'.format(code))
return 0
print(market, '--', code, '--', start_date, '--', end_date)
#print("最大值：",df.groupby('datetime').max())
#print(df.describe()) #df数据统计
data_start_date = df.min()['trade_date']
data_end_date = df.max()['trade_date']
start_date = start_date.replace('-', '')
end_date = end_date.replace('-', '')
if data_end_date < start_date or end_date < data_start_date:
print("采集时间在数据范围之外，退出函数")
return 0
elif end_date > data_end_date:
end_date = data_end_date
if start_date < data_start_date:
#最近三天的数据中，去掉无用的数据后即是最终数据
#需要取的数据还有三天前的数据，需要继续向前取
n = (day - 3) // 3
m = (day - 3) % 3
for i in range(0, n):
dfn = self.get_security_bars_minute1(category, market, code,
240 * 3 * (i + 1),
240 * 3) # 返回DataFrame
if (dfn is not None) and (not dfn.empty):
df = dfn.append(df, ignore_index=True)
if m > 0:
dfn = self.get_security_bars_minute1(category, market, code,
240 * 3 * (n + 1),
240 * m)
if (dfn is not None) and (not dfn.empty):
df = dfn.append(df, ignore_index=True)
df = df.sort_values(by=['trade_date', 'time_index'],
axis=0,
ascending=True)
#过滤掉start_date, end_date之外的数据
df = df[(df['trade_date'] >= str(init_start_date)) &
(df['trade_date'] <= str(init_end_date))] #每个条件要用括号()括起来
dict = {0: 'SZ', 1: 'SH'}
ts_code = code + "." + dict[market]
filename = config.tdx_csv_minline1_all + ts_code + ".csv"
if os.path.isfile(filename):
df.to_csv(filename,
index=False,
mode='a',
header=False,
sep=',',
encoding="utf_8_sig")
print("更新一分钟all股票数据：", filename)
else:
df.to_csv(filename,
index=False,
mode='w',
header=True,
sep=',',
encoding="utf_8_sig")
print("新增加的一分钟all股票数据：", filename)
print("下载完成，正在解压...")
os.system('unrar x -o+ -inul {} {}'.format(dest_filename, dest_dir))
print("解压完成，正在导入...")
add_count = qianlong_import_weight(connect, dest_dir + '/weight', 'SH')
add_count += qianlong_import_weight(connect, dest_dir + '/weight', 'SZ')
print("导入数量：", add_count)
"""
print("\n导入上证分笔数据")
#add_count = import_trans(connect, 'SH', quotations, api, dest_dir, max_days=2, progress=ProgressBar)
print("\n导入数量：", add_count)
print("\n导入上证分时数据")
add_count = import_time(connect,
'SZ',
quotations,
api,
dest_dir,
max_days=3,
progress=ProgressBar)
print("\n导入数量：", add_count)
api.disconnect()
connect.close()
endtime = time.time()
print("\nTotal time:")
print("%.2fs" % (endtime - starttime))
print("%.2fm" % ((endtime - starttime) / 60))
class SP(object):
def __init__(self,
userid="account1",
number=2,
mock=True,
server="http://192.168.118.1:65000"):
''' 根据实际账户资金，以股指1手为基本对冲单位进行配置。
'''
self.products = {
'000905': {
'args': (3, 6, 65),
'stocklst': {
"510500": 0
}
}
}
self.datatype = 0
self.userid = userid
self.api = TdxHq_API(heartbeat=True)
self.trader = None
self.trading = False
self.mock = mock
self.permoney = 200 #指数一个点对应的金额
self.number = number #指数的倍数
self.server = server
self.TDX_IP_SETS = STOCK_IP_SETS
if mock: self.judgerate = 100
else: self.judgerate = 10
def connect(self):
for ip in self.TDX_IP_SETS:
try:
if self.api.connect(ip, 7709):
return
except:
pass
def getdata(self, product, market=1, number=10000, pn=400):
data = []
for i in range(int(number / pn) + 1):
temp = self.api.get_index_bars(self.datatype, market, product,
(int(number / pn) - i) * pn, pn)
if not temp or len(temp) < pn:
for _ in range(2):
temp = self.api.get_index_bars(self.datatype, market,
product,
(int(number / pn) - i) * pn,
pn)
if not temp or len(temp) < pn:
logger.info("record not reach the limit!")
else:
break
try:
data += temp
except:
self.connect()
df = self.api.to_df(data)[["open", "close", "high", "low", "datetime"]]
df.set_index("datetime", inplace=True, drop=False)
return df
def disconnect(self):
logger.info("[DISCONNECT]:statrt disconnect!!!!!")
if self.istradeday:
self.api.disconnect()
logger.info("[DISCONNECT]:disconnect finished!!!!!")
def set_number(self):
'''设置日内交易的单次手数
'''
price = self.api.get_security_quotes([(1, '510500')])[0]["last_close"]
index_price = self.api.get_security_quotes([(1, '000905')
])[0]["last_close"]
self.products["000905"]["stocklst"]["510500"] = int(
index_price * self.number * self.permoney / price /
self.judgerate) * 100
def judgetradeday(self, ):
today = datetime.datetime.today().date().strftime('%Y-%m-%d')
df = ts.trade_cal()
return df[(df["calendarDate"] == today)].isOpen.values[0]
@property
def istradeday(self):
if not hasattr(self, "_istradeday"):
self._istradeday = self.judgetradeday()
return self._istradeday
def initial(self):
'''每天初始化设置
'''
logger.info("[INITIAL]:start initial !!!!")
if not self.istradeday:
self.trading = False
return
self.trading = True
logger.info("[INITIAL]:try to create connect... ")
self.connect()
self.trader = trade(UserID=self.userid,
api=self.api,
mock=self.mock,
server=self.server)
logger.info("[INITIAL]:connect successful!")
self.set_number() #设置手数
logger.info("[INITIAL]:initial finished !!!!!")
def handledata(self, df, args=[]):
df.loc[:, "number"] = range(df.shape[0])
s, m, l = args
for i in args: #5 15 60 D
key = str(5 * i)
df.loc[:, key + "high"] = df["high"].rolling(i).max()
df.loc[:, key + "low"] = df["low"].rolling(i).min()
df.loc[:, key + "atr"] = (df[key + "high"] -
df[key + "low"]).rolling(10 * i).mean()
df.loc[:, key + "med"] = (df[key + "high"] + df[key + "low"]) / 2
df.loc[:, key + "HH"] = df[key + "med"] + 1.5 * df[key + "atr"]
df.loc[:, key + "LL"] = df[key + "med"] - 1.5 * df[key + "atr"]
df.loc[:, key + 'HHmax'] = df[key + 'HH'].rolling(10 * i).max()
df.loc[:, key + 'LLmin'] = df[key + 'LL'].rolling(10 * i).min()
df.loc[df[key + 'HH'] >= df[key + 'HHmax'],
key + 'hmark'] = df["number"]
df.loc[df[key + 'LL'] <= df[key + 'LLmin'],
key + 'lmark'] = df["number"]
df[key + 'hmark'].fillna(method="ffill", inplace=True)
df[key + 'hmark'].fillna(0, inplace=True)
df[key + 'lmark'].fillna(method="ffill", inplace=True)
df[key + 'lmark'].fillna(0, inplace=True)
df.loc[:, key + 'UP'] = df[key + 'hmark'] >= df[key + 'lmark']
# debuginfo.append({key+'hmark':df.iloc[-1][key+'hmark'],key+'lmark':df.iloc[-1][key+'lmark']})
df.fillna(method="ffill", inplace=True)
df.dropna(inplace=True)
# logger.info("trademessage:{}".format(debuginfo))
result = (df.iloc[-1][str(5 * l) + "UP"] >
0) & (df.iloc[-1][str(5 * s) + "UP"] > 0)
result |= (df.iloc[-1][str(5 * l) + "UP"] >
0) & (df.iloc[-1][str(5 * m) + "UP"] > 0)
result |= (df.iloc[-1][str(5 * l) + "UP"] <= 0) & (df.iloc[-1][
str(5 * m) + "UP"] > 0) & (df.iloc[-1][str(5 * s) + "UP"] > 0)
return result
def sync(self, idx, director=True):
stocks = self.products[idx]["stocklst"]
for stock, number in stocks.items():
if not director: number = 2 * number #空信号,清仓
else: number = 4 * number
#判断现有持仓
try:
h_number = self.hd_df.ix[stock]["证券数量"]
except:
h_number = 0
#补仓差
cangcha = int((number - h_number) / 100) * 100
if h_number > 0 and abs(
cangcha
) / number < 0.05: #如果有持仓，同时仓差小于5% 不进行更改，为了处理频繁加减仓达到问题
return
if cangcha > 0:
logger.info("[RUN]:buy code:{}, number:{}".format(
stock, number - h_number))
self.buy(stock, cangcha)
elif cangcha < 0:
logger.info("[RUN]:sell code:{}, number:{}".format(
stock, h_number - number))
lastholdnumber = self.hd_df.ix[stock]["可卖数量"]
self.sell(stock, min(-cangcha, lastholdnumber))
def buy(self, stock, number, jump=0.02):
self.trader.buy(stock, number, jump=jump)
def sell(self, stock, number, jump=0.02):
self.trader.sell(stock, number, jump=jump)
def check_position(self, status):
'''检查仓位情况
'''
self.handleposition()
handlelist = []
for ins, v in status.items():
if ins not in self.g_df.index:
handlelist.append(ins)
elif self.g_df.ix[ins]["Position"] != v["number"]:
handlelist.append(ins)
return handlelist
def handleposition(self):
'''计算多仓，空仓，以及昨仓和今仓
'''
self.trader.cancelorder() #先尝试撤单
_, holdlists = self.trader.position()
self.hd_df = holdlists
if holdlists.shape[0] > 0:
self.hd_df.set_index("证券代码", inplace=True)
self.hd_df.index = self.hd_df.index.astype(np.str)
return self.hd_df
def run(self):
logger.info("[RUN]:start run !!!!!")
if not self.trading:
return
self.handleposition()
rst = {}
for idx in list(self.products.keys()):
director = self.handledata(self.getdata(idx, market=1),
self.products[idx]["args"]) #用指数出信号
logger.info("[RUN]:trademessage: block:{}, director:{}".format(
idx, director))
self.sync(idx, director)
rst[idx] = {
"up": director,
"number": self.products[idx]["stocklst"],
"product": idx
}
logger.info("[RUN]:lastest position status:{}".format(rst))
logger.info("[RUN]:run finished !!!!!")
def nhg(self):
'''尾盘自动进行逆回购
'''
if not self.mock:
self.trader.autobuy()
class PYTDXService():
"""pytdx数据服务类"""
def __init__(self):
"""Constructor"""
self.connected = False # 数据服务连接状态
self.hq_api = None # 行情API
def connect_api(self):
"""连接API"""
# 连接增强行情API并检查连接情况
try:
if not self.connected:
host = SETTINGS["TDX_HOST"]
port = SETTINGS["TDX_PORT"]
self.hq_api = TdxHq_API()
self.hq_api.connect(host, port)
self.connected = True
return True
except Exception:
raise ConnectionError("pytdx连接错误")
def get_realtime_data(self, symbol: str):
"""获取股票实时数据"""
try:
symbols = self.generate_symbols(symbol)
df = self.hq_api.to_df(self.hq_api.get_security_quotes(symbols))
return df
except Exception:
raise ValueError("股票数据获取失败")
def get_history_transaction_data(self, symbol, date):
"""
查询历史分笔数据
get_history_transaction_data(TDXParams.MARKET_SZ, '000001', 0, 10, 20170209)
参数：市场代码, 股票代码, 起始位置, 数量, 日期
输出[time, price, vol, buyorsell(0:buy, 1:sell, 2:平)]
"""
# 获得标的
code, market = self.check_symbol(symbol)
# 设置参数
check_date = int(date)
count = 2000
data_list = []
position = [6000, 4000, 2000, 0]
for start in position:
data = self.hq_api.to_df(
self.hq_api.get_history_transaction_data(
market, code, start, count, check_date))
data_list.append(data)
df = pd.concat(data_list)
df.drop_duplicates(inplace=True)
return df
@staticmethod
def generate_symbols(symbol: str):
"""组装symbols数据，pytdx接收的是以市场代码和标的代码组成的元祖的list"""
new_symbols = []
code, exchange = symbol.split('.')
new_symbols.append((exchange_map[exchange], code))
return new_symbols
@staticmethod
def check_symbol(symbol: str):
"""检查标的格式"""
if symbol:
code, market = symbol.split('.')
market = exchange_map.get(market)
return code, market
else:
return False
def close(self):
"""数据服务关闭"""
self.connected = False
self.hq_api.disconnect()
class StockMod(object):
def __init__(self,debug=False):
# 初始化tdx
self.debug = debug
oplogs("Loading stock mod...")
self.api = TdxHq_API(heartbeat=(not debug),auto_retry=True)
self.connect_to_tdx()
self.load_stocklist()
self.monitor_queue = dict()
def __del__(self):
oplogs("Unloading stock mod...")
self.api.disconnect()
#self.stop_jobs()
def connect_to_tdx(self):
random.shuffle(HZ_HOST_IPS)
for ip in HZ_HOST_IPS:
if self.api.connect(ip, 7709):
oplogs("Connected to %s success" %ip)
return
else:
oplogs("Connected to %s failed!!" %ip)
def load_stocklist(self):
try:
oplogs("get_stock_basics() ...")
self.stocklist = ts.get_stock_basics()
oplogs("get_stock_basics() load from remote success")
except:
# 如果加载远程股票列表失败, 从本地列表读取
oplogs("get_stock_basics() timeout, use local cached file instead")
cur_path=os.path.dirname(os.path.realpath(__file__))
stocklist_path=os.path.join(cur_path,'all.csv')
df = pd.read_csv(stocklist_path, dtype={'code':'object'},encoding='GBK')
df = df.set_index('code')
self.stocklist = df
# remove space in stock name
self.stocklist['name'] = self.stocklist['name'].str.replace(' ','')
self.stocklist.drop(self.stocklist[self.stocklist.name == '机器人'].index, inplace=True)
df_index = pd.DataFrame([['999999','上证指数'], ['399001','深证成指'], ['399006','创业板指']], index=['999999','399001','399006'], columns=['code', 'name'])
self.stocklist = self.stocklist.append(df_index)
# create dict and list to shorten search time
self.stockdict = self.stocklist.to_dict()["name"]
self.stockrdict = {value:key for key,value in self.stockdict.items()}
self.stockcode = self.stocklist.index.to_list()
self.stockname = self.stocklist.name.to_list()
def get_market(self, code):
# 计算市场代码,1=sh, 0=sz
market = 0
if code == '999999':
return 1
elif code in ['399001','399006']:
return 0
elif code[0] == '6':
market = 1
return market
def get_stock_price(self, code):
if code not in self.stockcode:
return ""
market = self.get_market(code)
oplogs("get_stock_price called,code=%s,market=%s" %(code,market))
price = self.api.get_security_quotes([(market,code)])
last_close = price[0]['last_close']
current_price = price[0]['price']
if last_close >0:
change = (current_price/last_close - 1)*100
else:
change = 0
result = "[%s]%.2f 涨幅:%.2f%%" %(self.stockdict[code],current_price,change )
return result
def get_stock_price_by_name(self, stockname):
if stockname in self.stockname:
return self.get_stock_price(self.stockrdict[stockname])
return ""
def get_fast_alarms(self):
pass
def scan_stock(self, group, msg):
info = msg.lower()
for code in self.stockcode:
if code in info:
self.add_to_monitor_group(group, code)
for name in self.stockname:
if name in info:
return self.add_to_monitor_group(group, self.stockrdict[name])
def isvalid_stock(self, stock):
if stock.isdigit():
return stock in self.stockcode
else:
return stock in self.stockname
def extract_stock_from_msg(self, msg):
stocklist = []
for code in self.stockcode:
if code in msg:
stocklist.append(code)
for name in self.stockname:
if name in msg:
stocklist.append(self.stockrdict[name])
return stocklist
def add_to_monitor_group(self,group,stock):
if stock.isdigit():
code = stock
else:
code = self.stockrdict[stock]
if group in self.monitor_queue.keys():
if code not in self.monitor_queue[group]:
self.monitor_queue[group].append(code)
else:
self.monitor_queue[group] = [code]
oplogs("add_to_monitor_group called:%s %s" %(group, stock))
def get_group_stock_price(self, group):
if group in self.monitor_queue.keys():
stocks = []
for code in self.monitor_queue[group]:
price, change = self.get_stock_daily(code)
stocks.append([code,price,change])
sorted_alarms = sorted(stocks, key=lambda x: x[2],reverse=True)
result = "本群关注：\n"
for record in sorted_alarms:
if record[1] == 0:
result += "[%s]停盘\n" %self.stockdict[record[0]]
else:
newprice = "%.2f" %record[1]
newchange = "%.2f%%" %record[2]
result += "[%s]%s:%s\n" %(self.stockdict[record[0]],
newprice, newchange)
return result
return ""
def get_stock_daily(self, code):
# 计算当日涨跌
market = self.get_market(code)
data_day = self.api.get_security_quotes([(market,code)]) # get daily changes
if self.debug:
oplogs("get_stock_daily called")
#print(self.api.to_df(data_day).ix[:,0:8])
last_close = data_day[0]['last_close']
current_price = data_day[0]['price']
if last_close >0: #正常情况
change = (current_price/last_close - 1)*100
else:
change = 0
return current_price, change
def get_stock_min(self, code, min=5):
market = self.get_market(code)
data_min = self.api.get_security_bars(0,market, code, 0, 1) # 5 min K bar
if self.debug:
oplogs("get_stock_min called")
#print(self.api.to_df(data_min).ix[:,0:8])
open = data_min[0]['open']
close = data_min[0]['close']
high = data_min[0]['high']
low = data_min[0]['low']
if low >0 and open>0:
# 波动范围
change_high_low = (high-low)/low*100
# 涨跌幅
change_close_open = (close-open)/open*100
else:
change_high_low = 0
change_close_open = 0
return change_high_low, change_close_open
def get_alarms(self,valve=3.0):
# timeslice , slice in time series, default =5 min
# range, monitor change range, default = 3%. Actual change range = abs((high-low)/low)
groupalarms = dict()
for (key, value) in self.monitor_queue.items():
# 获取单个group监控的股票报警
alarms = []
for code in value:
# 当日涨跌
price, change = self.get_stock_daily(code)
if price <= 0: # 停盘不监控
continue
# 5分钟涨跌
highlow, closeopen = self.get_stock_min(code)
# 判断波动幅度是否超过阈值
if abs(highlow) >=valve:
alarm = [self.stockdict[code],price,change,highlow,closeopen]
alarms.append(alarm)
# 按5分钟涨跌幅排序
sorted_alarms = sorted(alarms, key=lambda x: (x[4],x[2]),reverse=True)
# 打包所有group的报警列表
groupalarms[key] = sorted_alarms
return groupalarms
def del_from_list(self, group, stock):
# group, group name
# stock, stock name
# remove stock from group
if stock.isdigit():
code = stock
else:
code = self.stockrdict[stock]
if group in self.monitor_queue.keys():
if code in self.monitor_queue[group]:
self.monitor_queue[group].remove(code)
oplogs("del_from_list called:%s %s" %(group, stock))
def test(self):
data = self.api.get_security_bars(0,1, '600477', 0, 48) # 5 min K bar
df = self.api.to_df(data)
df['change_high_low'] = (df['high']-df['low'])/df['low']*100
df['change_close_open'] = (df['close']-df['open'])/df['open']*100
print(df)
