def on_error(self, error):
"""
处理错误信息
"""
QA_util_log_info("Error: " + str(error))
error = gzip.decompress(error).decode()
QA_util_log_info(error)
def publish_bar(self, symbol):
QA_util_log_info('=================================')
QA_util_log_info('publish')
QA_util_log_info('=================================')
print(self.data[symbol])
self.pro.pub(json.dumps(self.data[symbol]))
self.is_send = True
def upcoming_data(self, new_tick):
# print(new_tick)
curtime = new_tick['datetime']
time = curtime
try:
if new_tick['datetime'][17:19] == '00' and len(new_tick['datetime']) == 19:
print(True)
old_data = self.update_bar(new_tick)
self.last_volume = new_tick['volume']
self.publish_bar(new_tick['symbol'])
self.pro_realtimemin.pub(json.dumps(old_data))
self.data[new_tick['symbol']] = {}
self.data[new_tick['symbol']]['datetime'] = time
elif new_tick['datetime'][17:19] == '00' and len(new_tick['datetime']) > 19:
if self.is_send:
self.is_send = False
else:
self.publish_bar(new_tick['symbol'])
QA_util_log_info('xxx')
self.create_new(new_tick)
self.pro_realtimemin.pub(json.dumps(
self.data[new_tick['symbol']]))
QA_util_log_info(self.data)
else:
try:
self.update_bar(new_tick)
except:
self.create_new(new_tick)
self.pro_realtimemin.pub(json.dumps(
self.data[new_tick['symbol']]))
except Exception as e:
print(e)
def download_day_data_from_tushare(trade_date='20190102'):
trade_date = trade_date.replace('-', '') #兼容设置以防日期格式为2001-10-20格式
lastEx = None
retry = 10
for _ in range(retry):
try:
df_daily = pro.query('daily', trade_date=trade_date)
df_daily_basic = pro.query('daily_basic', trade_date=trade_date)
df_factor = pro.query('adj_factor', trade_date=trade_date)
break
except Exception as ex:
lastEx = ex
QA_util_log_info("[{}]TuSharePro数据异常: {}, retrying...".format(
trade_date, ex))
else:
QA_util_log_info("[{}]TuSharePro异常: {}, retried {} times".format(
trade_date, lastEx, retry))
return None
df = pd.merge(df_daily, df_factor, how='left')
res = pd.merge(df, df_daily_basic, how='left').sort_values(by='ts_code')
res['code'] = res['ts_code'].apply(lambda x: x[:6]) #x[7:9].lower()
res['trade_date'] = pd.to_datetime(
res['trade_date'],
format='%Y-%m-%d').dt.tz_localize(None).dt.tz_localize('Asia/Shanghai')
return res
def QA_Update(
update_dict={
DATABASE_NAME.STOCK_LIST: {
DATASOURCE.TDX: None
},
DATABASE_NAME.STOCK_XDXR: {
DATASOURCE.TDX: None
},
DATABASE_NAME.STOCK_INFO: {
DATASOURCE.TDX: None
},
DATABASE_NAME.STOCK_BLOCK: {
DATASOURCE.TDX: None
},
DATABASE_NAME.STOCK_DAY: {
DATASOURCE.TDX: None
},
#
DATABASE_NAME.FUTURE_LIST: {
DATASOURCE.TDX: None
},
DATABASE_NAME.FUTURE_DAY: {
DATASOURCE.TDX: None
},
DATABASE_NAME.FUTURE_MIN: {
DATASOURCE.JQDATA: {
'data_type': ['1min', '5min', '15min', '30min', '60min']
}
},
},
ui_log=None,
ui_progress=None):
'''
2019/04/28
TODO: 对不同的financial包增加裁定，当前只使用通达信financial包
注：股票的分钟数据从其他数据源对接
期货的分钟数据由tick合成
:param update_dict:
:return:
'''
QA_util_log_info(
'QUANTAXIS {} Include these DataBases: {}'.format(
QAVERSION, DATABASE_NAME_ALL), ui_log)
QA_util_log_info('DataBases update this time: {}'.format(update_dict),
ui_log)
for update_database in update_dict.keys():
update_package = list(update_dict[update_database].keys())[0]
if update_dict[update_database][update_package] == None:
QA_SU_update_single(database_name=update_database,
package=update_package,
ui_log=ui_log)
else:
for update_data_type in update_dict[update_database][
update_package]['data_type']:
QA_SU_update_single(database_name=update_database,
package=update_package,
data_type=update_data_type,
ui_log=ui_log)
def debug(self):
self.running_mode = 'backtest'
self.database = pymongo.MongoClient(mongo_ip).QUANTAXIS
user = QA_User(username="******", password='******')
port = user.new_portfolio(self.portfolio)
self.acc = port.new_accountpro(account_cookie=self.strategy_id,
init_cash=self.init_cash,
commission_coeff=self.commission_coeff,
allow_t0=False,
allow_margin=False,
allow_sellopen=False,
market_type=self.market_type,
running_environment=self.market_type,
frequence=self.frequence)
#self.positions = self.acc.get_position(self.code)
QA_util_log_info(self.acc)
QA_util_log_info(self.acc.market_type)
data = QA.QA_quotation(self.code,
self.start,
self.end,
source=QA.DATASOURCE.MONGO,
frequence=self.frequence,
market=self.market_type,
output=QA.OUTPUT_FORMAT.DATASTRUCT)
data.data.progress_apply(self.x1, axis=1)
def publish_bar(self, InstrumentID):
QA_util_log_info('=================================')
QA_util_log_info('publish bar')
QA_util_log_info('=================================')
print(self.data)
self.pro.pub(json.dumps(self.data[InstrumentID]))
self.is_send = True
def __init__(self, market_type, frequence, start, end, code_list, commission_fee):
super().__init__(market_type, frequence, start, end, code_list, commission_fee)
self.user = QA_User()
mastrategy = MAStrategy()
self.portfolio, self.account = self.user.register_account(mastrategy)
QA_util_log_info(self.user.get_portfolio(self.portfolio).accounts)
QA_util_log_info(self.user.get_portfolio(
self.portfolio).get_account(self.account).cash)
def QA_user_sign_in(name, password, client):
coll = client.quantaxis.user_list
if (coll.find({'username': name, 'password': password}).count() > 0):
QA_util_log_info('success login! your username is:' + str(name))
return True
else:
QA_util_log_info('Failed to login,please check your password ')
return False
def QA_user_sign_up(name, password, client):
coll = client.quantaxis.user_list
if (coll.find({'username': name}).count() > 0):
QA_util_log_info('user name is already exist')
return False
else:
coll.insert({'username': name, 'password': password})
QA_util_log_info('✅Success sign in! please login ')
return True
def insert_order(self, account_id, amount, amount_model, time, code, price,
order_model, towards, market_type, frequence,
broker_name):
flag = False
if order_model in [ORDER_MODEL.CLOSE, ORDER_MODEL.NEXT_OPEN]:
_price = self.query_data_no_wait(broker_name=broker_name,
frequence=frequence,
market_type=market_type,
code=code,
start=time)
if _price is not None and len(_price) > 0:
price = float(_price[0][4])
flag = True
else:
QA_util_log_info('MARKET WARING: SOMEING WRONG WITH ORDER \n ')
QA_util_log_info(
'code {} date {} price {} order_model {} amount_model {}'.
format(code, time, price, order_model, amount_model))
elif order_model is ORDER_MODEL.MARKET:
_price = self.query_data_no_wait(broker_name=broker_name,
frequence=frequence,
market_type=market_type,
code=code,
start=time)
if _price is not None and len(_price) > 0:
price = float(_price[0][1])
flag = True
else:
QA_util_log_info('MARKET WARING: SOMEING WRONG WITH ORDER \n ')
QA_util_log_info(
'code {} date {} price {} order_model {} amount_model {}'.
format(code, time, price, order_model, amount_model))
elif order_model is ORDER_MODEL.LIMIT:
# if price > self.last_query_data[0][2] or price < self.last_query_data[0][3]:
flag = True
if flag:
order = self.get_account(account_id).send_order(
amount=amount,
amount_model=amount_model,
time=time,
code=code,
price=price,
order_model=order_model,
towards=towards)
self.event_queue.put(
QA_Task(
worker=self.broker[self.get_account(account_id).broker],
engine=self.get_account(account_id).broker,
event=QA_Event(event_type=BROKER_EVENT.RECEIVE_ORDER,
order=order,
callback=self.on_insert_order)))
else:
pass
def QA_user_sign_in(name, password, client):
coll = client.user_list
cursor = coll.find({'username': name, 'password': password})
if (cursor.count() > 0):
QA_util_log_info('success login! your username is:' + str(name))
return cursor
else:
QA_util_log_info('Failed to login,please check your password ')
return None
def plot_datastruct(__stock_hq_base, code=None):
if code is None:
path_name = '.' + os.sep + 'QA_' + __stock_hq_base.type + \
'_codepackage_' + __stock_hq_base.if_fq + '.html'
kline = Kline('CodePackage_' + __stock_hq_base.if_fq + '_' +
__stock_hq_base.type,
width=1360,
height=700,
page_title='QUANTAXIS')
data_splits = __stock_hq_base.splits()
for i_ in range(len(data_splits)):
data = []
axis = []
for dates, row in data_splits[i_].data.iterrows():
open, high, low, close = row[1:5]
datas = [open, close, low, high]
axis.append(dates[0])
data.append(datas)
kline.add(__stock_hq_base.code[i_],
axis,
data,
mark_point=["max", "min"],
is_datazoom_show=True,
datazoom_orient='horizontal')
kline.render(path_name)
webbrowser.open(path_name)
QA_util_log_info('The Pic has been saved to your path: %s' % path_name)
else:
data = []
axis = []
for dates, row in __stock_hq_base.select_code(code).data.iterrows():
open, high, low, close = row[1:5]
datas = [open, close, low, high]
axis.append(dates[0])
data.append(datas)
path_name = '.' + os.sep + 'QA_' + __stock_hq_base.type + \
'_' + code + '_' + __stock_hq_base.if_fq + '.html'
kline = Kline(code + '__' + __stock_hq_base.if_fq + '__' +
__stock_hq_base.type,
width=1360,
height=700,
page_title='QUANTAXIS')
kline.add(code,
axis,
data,
mark_point=["max", "min"],
is_datazoom_show=True,
datazoom_orient='horizontal')
kline.render(path_name)
webbrowser.open(path_name)
QA_util_log_info('The Pic has been saved to your path: %s' % path_name)
def QA_user_sign_in(username, password):
"""用户登陆
不使用 QAUSER库
只返回 TRUE/FALSE
"""
#user = QA_User(name= name, password=password)
cursor = DATABASE.user.find_one(
{'username': username, 'password': password})
if cursor is None:
QA_util_log_info('SOMETHING WRONG')
QA_util_log_info('....test..... success!!!')
return False
else:
return True
def login(self, user_name, password):
'''
login to a database
todo： fix 返回 是否成功
:param user_name: 连接 mongodb 的用户名
:param password: 连接 mongodb 的密码
:return: Boolean 是否成功连接
'''
if self.setting.login(user_name, password):
QA_util_log_info('SUCCESS')
return True
else:
QA_util_log_info('FAILD')
return False
def QA_plot_save_html(pic_handle, path, if_open_web):
"""
explanation:
将绘图结果保存至指定位置
params:
* pic_handle ->:
meaning: 绘图
type: null
optional: [null]
* path ->:
meaning: 保存地址
type: null
optional: [null]
* if_open_web ->:
meaning: 是否调用浏览器打开
type: bool
optional: [null]
return:
None
demonstrate:
Not described
output:
Not described
"""
pic_handle.render(path)
if if_open_web:
webbrowser.open(path)
else:
pass
QA_util_log_info('The Pic has been saved to your path: %s' % path)
def run(self, event):
if event.event_type is MARKET_EVENT.QUERY_DATA:
self.order_handler.run(event)
try:
data = self.fetcher[(event.market_type,
event.frequence)](code=event.code,
start=event.start,
end=event.end).values[0]
if 'vol' in data.keys() and 'volume' not in data.keys():
data['volume'] = data['vol']
elif 'vol' not in data.keys() and 'volume' in data.keys():
data['vol'] = data['volume']
return data
except Exception as e:
QA_util_log_info('MARKET_ENGING ERROR: {}'.format(e))
return None
elif event.event_type is MARKET_EVENT.QUERY_ORDER:
self.order_handler.run(event)
elif event.event_type is BROKER_EVENT.RECEIVE_ORDER:
self.order_handler.run(event)
elif event.event_type is BROKER_EVENT.TRADE:
event = self.order_handler.run(event)
event.message = 'trade'
if event.callback:
event.callback(event)
def QA_plot_save_html(pic_handle, path, if_open_web):
pic_handle.render(path)
if if_open_web:
webbrowser.open(path)
else:
pass
QA_util_log_info('The Pic has been saved to your path: %s' % path)
def QA_fetch_stock_day_pg(code=['000001','000002'],start_date='19000101',end_date='20500118',data='*'):
name_biao='stock_day'
if isinstance(code,list):
code="','".join(code)
mes='select '+ data+' from '+name_biao+" where trade_date >= date_trunc('day',timestamp '"+start_date+"') and trade_date <= date_trunc('day',timestamp '"+end_date+"') and code in ('"+code+"');"
try:
t=time.time()
res=load_data_from_postgresql(mes=mes)
t1=time.time()
QA_util_log_info('load '+ name_biao+ ' success,take '+str(round(t1-t,2))+' S')
except Exception as e:
print(e)
res=None
else:
QA_util_log_info('code type is not list, please cheack it.')
return res
def get_market(self, order):
"""get_market func
[description]
Arguments:
order {orders} -- [description]
Returns:
[type] -- [description]
"""
# 首先判断是否在_quotation里面
if (pd.Timestamp(order.datetime), order.code) in self._quotation.keys():
return self._quotation[(pd.Timestamp(order.datetime), order.code)]
else:
try:
data = self.fetcher[(order.market_type,
order.frequence)](
code=order.code,
start=order.datetime,
end=order.datetime,
format='json',
frequence=order.frequence
)[0]
if 'vol' in data.keys() and 'volume' not in data.keys():
data['volume'] = data['vol']
elif 'vol' not in data.keys() and 'volume' in data.keys():
data['vol'] = data['volume']
return data
except Exception as e:
QA_util_log_info('MARKET_ENGING ERROR: {}'.format(e))
return None
def QA_save_stock_list_pg():
stock_list_l = pro.stock_basic(
exchange_id='',
is_hs='',
list_status='L',
fields=
'ts_code,symbol,name,area,industry,fullname,enname,market,exchange,curr_type,list_status,list_date,delist_date,is_hs'
)
stock_list_D = pro.stock_basic(
exchange_id='',
is_hs='',
list_status='D',
fields=
'ts_code,symbol,name,area,industry,fullname,enname,market,exchange,curr_type,list_status,list_date,delist_date,is_hs'
)
stock_list_P = pro.stock_basic(
exchange_id='',
is_hs='',
list_status='P',
fields=
'ts_code,symbol,name,area,industry,fullname,enname,market,exchange,curr_type,list_status,list_date,delist_date,is_hs'
)
stock_list = pd.concat([stock_list_l, stock_list_D], axis=0)
stock_list = pd.concat([stock_list, stock_list_P], axis=0)
try:
t = time.time()
save_data_to_postgresql('stock_list', stock_list)
t1 = time.time()
QA_util_log_info('save stock_list data success,take ' +
str(round(t1 - t, 2)) + ' S')
except Exception as e:
print(e)
def QA_util_to_datetime(time):
if len(str(time)) == 10:
_time = '{} 00:00:00'.format(time)
elif len(str(time)) == 19:
_time = str(time)
else:
QA_util_log_info('WRONG DATETIME FORMAT {}'.format(time))
return datetime.datetime.strptime(_time, '%Y-%m-%d %H:%M:%S')
def after_success(self):
QA_util_log_info(self.account.history_table)
risk = QA_Risk(self.account, benchmark_code='000300',
benchmark_type=MARKET_TYPE.INDEX_CN)
print(risk().T)
fig=risk.plot_assets_curve()
fig.show()
def send_message(self, message_dict, message_txt=''):
"""
发送消息请求
"""
data = json.dumps(message_dict).encode()
QA_util_log_info("Sending Message: {:s}".format(message_txt))
self.__ws.send(data)
def after_success(self):
QA_util_log_info(self.account.history_table)
risk = QA_Risk(self.account, benchmark_code='000300',
benchmark_type=MARKET_TYPE.INDEX_CN)
print(risk().T)
self.account.save()
risk.save()
def Tick(self, symbol):
"""
Tick 计数器
"""
if symbol in self.__summary:
self.__summary[symbol] = self.__summary[symbol] + 1
else:
self.__summary[symbol] = 1
if (datetime.now() - self.__next).total_seconds() > 0:
QA_util_log_info("Tick message counter @ %s" %
datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
for symbol in self.__summary:
QA_util_log_info("on %s got %d ticks" %
(symbol, self.__summary[symbol]))
self.__summary.clear()
self.__next = datetime.now() + timedelta(seconds=self.__countdown)
def after_success(self):
QA_util_log_info(self.account.history_table)
risk = QA_Risk(self.account,
benchmark_code='000300',
benchmark_type=MARKET_TYPE.INDEX_CN)
print(risk().T)
self.user.save()
risk.save()
risk.plot_assets_curve()
print(risk.profit_construct)
def run_backtest(shell_cmd):
shell_cmd = 'python "{}"'.format(shell_cmd)
cmd = shlex.split(shell_cmd)
p = subprocess.Popen(cmd,
shell=False,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
while p.poll() is None:
line = p.stdout.readline()
line = line.strip()
if line:
QA_util_log_info(line)
#print('QUANTAXIS: [{}]'.format(line))
if p.returncode == 0:
QA_util_log_info('backtest run success')
else:
QA_util_log_info('Subprogram failed')
return p.returncode
def after_success(self):
QA_util_log_info(self.account.history_table)
risk = QA_Risk(self.account)
print(risk.assets)
print('annualize_return : {} %'.format(risk.annualize_return))
print('max_dropback : {} %'.format(risk.max_dropback))
print('profit : {} %'.format(risk.profit))
print('volatility : {}'.format(risk.volatility))
self.account.save()
def QA_fetch_stock_list_pg(name_biao='stock_list'):
mes='select * from '+name_biao+";"
try:
t=time.time()
res=load_data_from_postgresql(mes=mes)
t1=time.time()
QA_util_log_info('load '+ name_biao+ ' success,take '+str(round(t1-t,2))+' S')
except Exception as e:
print(e)
res=None
return res
