def holdlist_format(self,
date=None,
prctype='close',
outdir=None,
source='wind'):
""" 提取标准格式, 与get_totval 平行，不会互相调用 """
if date is None:
date = dt.datetime.today()
if source == 'wind':
####### 万得数据源 #######
w.start()
if date is None:
date = dt.datetime.today()
elif source == 'gm':
###### 掘金数据源 #######
md.init('18201141877', 'Wqxl7309')
if prctype == 'settle':
prctype = 'settle_price' # 转为掘金格式
holdnum = self.get_holdnum(date=date)
holding = pd.DataFrame()
for strat in self._logdir:
stratinfo = strat.split('_')
cttype = stratinfo[1].upper()
montype = stratinfo[0]
name = RawHoldingFutures.get_contracts_ours(date=date,
cttype=cttype)[montype]
code = RawHoldingStocks.addfix(name)
num = holdnum[strat]
multi = self._multiplier[cttype]
if source == 'wind':
############# wind 数据源
prc = w.wsd('.'.join([name, 'CFE']), prctype, date,
date).Data[0][0]
elif source == 'gm':
######### 掘金数据
lastbar = md.get_last_n_dailybars(
symbol='.'.join(['CFFEX', name]),
n=1,
end_time=date.strftime('%Y-%m-%d'))[0]
prc = eval('.'.join(['lastbar', prctype]))
#### 紧急措施 手动
# prc = 6069.2 if name=='IC1707' else 5880.6
holdlist = pd.DataFrame(
[[code, name, num, multi, prc]],
columns=['code', 'name', 'num', 'multi', 'prc'])
#holdlist['val'] = holdlist['num']*holdlist['multi']*holdlist['prc']
holding = holding.append(holdlist, ignore_index=True)
holding = holding[holding['num'] != 0]
if outdir:
holding.to_csv(outdir, header=True, index=False)
else:
return holding
def get_data( begin_time, end_time, period ):
"""
:param from_time:
:param to_time:
:return:
"""
# df = ts.get_hist_data(code, start=begin_time, end=end_time)
# df = ts.get_hist_data('600848')
ret = md.init( "13601380996", "it@iZ23psatkqsZ" )
# df4 = tt.get_bars( "SHFE.ru1709", 60, "2016-09-05 09:00:00", "2016-11-04 23:30:00" )
# df5 = tt.get_bars( "SHFE.ru1709", 60, "2016-11-05 09:00:00", "2017-01-04 23:30:00" )
# df6 = tt.get_bars( "SHFE.ru1709", 60, "2017-01-05 09:00:00", "2017-03-04 23:30:00" )
# df7 = tt.get_bars( "SHFE.ru1709", 60, "2017-03-05 09:00:00", "2017-05-04 23:30:00" )
# df8 = tt.get_bars( "SHFE.ru1709", 60, "2017-05-25 09:00:00", "2017-07-06 23:30:00" )
# df8 = tt.get_bars( "CZCE.CF709", 600, "2017-03-25 09:00:00", "2017-07-06 23:30:00" )
df8 = tt.get_bars( "CZCE.ZC709", 900, "2017-02-25 09:00:00", "2017-07-06 23:30:00" )
# df = pd.concat( [df4, df5, df6, df7, df8] )
df = pd.concat( [df8] )
print( len( df ) )
return df
def extract_tick_data(self, session_start, session_end, target_contract):
return_tick_dicts = []
ret_value = md.init("*****@*****.**", "btbxl@121208")
current_loop_start = session_start
while (True):
test_tick_list = md.get_ticks(
target_contract,
current_loop_start.strftime(
stdafx.my_datetime.my_quant_time_format),
session_end.strftime(stdafx.my_datetime.my_quant_time_format))
if test_tick_list.__len__() <= 1: #如果已经取不出数据，则跳出
break
current_final_tick = test_tick_list[test_tick_list.__len__() - 1]
current_final_tick_moment = stdafx.my_datetime.utc_float2datetime(
current_final_tick.utc_time)
current_loop_start = current_final_tick_moment #更新下一圈提数据的开始时刻
test_tick_list.pop()
return_tick_dicts.extend(test_tick_list)
if current_loop_start >= session_end: #如果下圈开始时间已经等于最终时间，则结束
break
pass
tick_series = [
my_tick(tick_to_dict(each_tick)) for each_tick in return_tick_dicts
]
self.tick_profitability_walk(tick_series)
my_tick_dict_series = [each.__dict__ for each in tick_series]
pickle_filename = self.get_pickle_filename(target_contract,
session_start, session_end)
pickle_file = open(pickle_filename, mode='wb')
pickle.dump(my_tick_dict_series, pickle_file, -1)
pickle_file.close()
return my_tick_dict_series
def __init__(self, username, password):
self._username = username
self._password = password
md.init(username=username, password=password)
def OneDayMinData_MD(date,stklst):
md.init('18201141877','Wqxl7309')
datestr= dt.datetime.strptime(str(date),'%Y%m%d')
datestr = dt.datetime.strftime(datestr,'%Y-%m-%d')
starttime = ' '.join([datestr,'09:30:00'])
endtime = ' '.join([datestr,'15:00:00'])
# day base obj
objday = StocksMinDB(configpath=r'E:\stocks_data_min\StocksMinDB\configs')
objday.connectDB('stocks_data_min_by_day')
daycols = [c.lower() for c in ByDay.colinfo if c!='STKID']
dayconn = objday._getConn_()
# stk base obj
objstk = StocksMinDB(configpath=r'E:\stocks_data_min\StocksMinDB\configs')
objstk.connectDB('stocks_data_min_by_stock')
stkcols = [c.lower() for c in ByStock.colinfo]
stkconn = objstk._getConn_()
# data to extract
cols = ['date','time','stkcd','open','high','low','close','volume','amount']
alldata = pd.DataFrame()
for stk in stklst:
szstk = False
if stk>=600000:
stkcd = '.'.join(['SHSE',str(stk)])
else:
szstk = True
stkstr = str(stk)
stkcd = '.'.join(['SZSE','0'*(6-len(stkstr))+stkstr])
stkbars = md.get_bars(stkcd,60,starttime,endtime)
stkdata = [[date,utc2local(bar.utc_endtime,'%H%M'),stk,bar.open,bar.high,bar.low,bar.close,bar.volume,bar.amount] for bar in stkbars]
if szstk and stkdata[-1][1]==1500 and stkdata[-2][1]>=1457:
tms = stkdata[-2][1]+1
cls = stkdata[-2][6]
for tm in range(tms,1460):
stkdata.insert(-1,[date,tm,stk,cls,cls,cls,cls,0,0])
stkdata = pd.DataFrame(stkdata,columns=cols)
stkdata['volamtflag'] = 0
stkdata.loc[stkdata['amount']>(stkdata['volume']+100)*stkdata['high'],'volamtflag'] = 1
stkdata.loc[stkdata['amount']<(stkdata['volume']-100)*stkdata['low'],'volamtflag'] = 2
stkdata.loc[(stkdata['amount']==0) & (stkdata['volume']>0),'volamtflag'] = 3
stkdata.loc[(stkdata['amount']==0) & (stkdata['volume']==0),'volamtflag'] = 4
alldata = alldata.append(stkdata,ignore_index=True)
stktable = '_'.join(['stkmin',stkcd[:2].lower()+stkcd.split('.')[1]])
objstk.update_db(conn=stkconn,
dbname='stocks_data_min_by_stock',
data=stkdata.loc[:,stkcols].values,
tablename=stktable,
colinfo=ByStock.colinfo,
prmkey=ByStock.prmkey,
if_exist='append',
chunksize=1000
)
# update day base
daytable = 'stkmin_{0}'.format(date)
daycolinfo = dict([itm for itm in ByDay.colinfo.items() if itm[0]!='STKID'])
objday.update_db(conn=dayconn,
dbname='stocks_data_min_by_day',
data=alldata.loc[:,daycols].values,
tablename=daytable,
colinfo=daycolinfo,
prmkey=ByDay.prmkey,
if_exist='append',
chunksize=1000
)
# coding:utf-8
__author__ = 'song.lu'
from dateutil.parser import parse
from gmsdk import md
import pandas as pd
import numpy as np
import time
md.init('cloud.myquant.cn:8000', 'demo', 'demo')
r = md.get_bars(
'CFFEX.IF1506',
60,
'2015-05-27 09:15:00',
'2015-05-27 15:30:00',
)
st = time.time()
times = []
closep = []
openp = []
highp = []
lowp = []
volume = []
for i in r:
times.append(parse(str(i.bar_time)))
closep.append(i.close)
def OneDayMinData_MD(date, stklst):
md.init('18201141877', 'Wqxl7309')
datestr = dt.datetime.strptime(str(date), '%Y%m%d')
datestr = dt.datetime.strftime(datestr, '%Y-%m-%d')
starttime = ' '.join([datestr, '09:30:00'])
endtime = ' '.join([datestr, '15:00:00'])
# day base obj
objday = StocksMinDB(configpath=r'E:\stocks_data_min\StocksMinDB\configs')
objday.connectDB('stocks_data_min_by_day')
daycols = [c.lower() for c in ByDay.colinfo if c != 'STKID']
dayconn = objday._getConn_()
# stk base obj
objstk = StocksMinDB(configpath=r'E:\stocks_data_min\StocksMinDB\configs')
objstk.connectDB('stocks_data_min_by_stock')
stkcols = [c.lower() for c in ByStock.colinfo]
stkconn = objstk._getConn_()
# data to extract
cols = [
'date', 'time', 'stkcd', 'open', 'high', 'low', 'close', 'volume',
'amount'
]
alldata = pd.DataFrame()
for stk in stklst:
szstk = False
if stk >= 600000:
stkcd = '.'.join(['SHSE', str(stk)])
else:
szstk = True
stkstr = str(stk)
stkcd = '.'.join(['SZSE', '0' * (6 - len(stkstr)) + stkstr])
stkbars = md.get_bars(stkcd, 60, starttime, endtime)
stkdata = [[
date,
utc2local(bar.utc_endtime, '%H%M'), stk, bar.open, bar.high,
bar.low, bar.close, bar.volume, bar.amount
] for bar in stkbars]
if szstk and stkdata[-1][1] == 1500 and stkdata[-2][1] >= 1457:
tms = stkdata[-2][1] + 1
cls = stkdata[-2][6]
for tm in range(tms, 1460):
stkdata.insert(-1, [date, tm, stk, cls, cls, cls, cls, 0, 0])
stkdata = pd.DataFrame(stkdata, columns=cols)
stkdata['volamtflag'] = 0
stkdata.loc[stkdata['amount'] >
(stkdata['volume'] + 100) * stkdata['high'],
'volamtflag'] = 1
stkdata.loc[stkdata['amount'] <
(stkdata['volume'] - 100) * stkdata['low'],
'volamtflag'] = 2
stkdata.loc[(stkdata['amount'] == 0) & (stkdata['volume'] > 0),
'volamtflag'] = 3
stkdata.loc[(stkdata['amount'] == 0) & (stkdata['volume'] == 0),
'volamtflag'] = 4
alldata = alldata.append(stkdata, ignore_index=True)
stktable = '_'.join(
['stkmin', stkcd[:2].lower() + stkcd.split('.')[1]])
objstk.update_db(conn=stkconn,
dbname='stocks_data_min_by_stock',
data=stkdata.loc[:, stkcols].values,
tablename=stktable,
colinfo=ByStock.colinfo,
prmkey=ByStock.prmkey,
if_exist='append',
chunksize=1000)
# update day base
daytable = 'stkmin_{0}'.format(date)
daycolinfo = dict(
[itm for itm in ByDay.colinfo.items() if itm[0] != 'STKID'])
objday.update_db(conn=dayconn,
dbname='stocks_data_min_by_day',
data=alldata.loc[:, daycols].values,
tablename=daytable,
colinfo=daycolinfo,
prmkey=ByDay.prmkey,
if_exist='append',
chunksize=1000)
# coding:utf-8
from gmsdk import md, to_dict
import time
import datetime
import pymongo
# 读取期货列表
f_list = list()
with open('list.txt', 'r') as f:
lines = f.readlines()
for line in lines:
i = line.split(' ')
f_list.append(i[0])
# 初始化行情服务器
ret = md.init('13063638616', '940304', mode=1)
print ret
##########################
'''
设置数据库
'''
conn = pymongo.MongoClient('139.196.6.151', 26666)
db = conn['test_center']
qh_data_one_min = db['qh_data_one_min']
qh_data_five_min = db['qh_data_five_min']
qh_data_five_daily = db['qh_data_five_daily']
###############################
end = datetime.datetime.now()
start_m = end.month - 1
start = datetime.datetime(end.year, start_m, 1, 9, 0, 0)
start_e = datetime.datetime(end.year, start_m, 1, 15, 0, 0)
def min_data(daima,start_date,end_date,n):
def count_split_days(start_date,end_date):
if datetime.datetime(int(start_date[0:4]),int(start_date[4:6]),int(start_date[6:]))<datetime.datetime(2015,1,5):
num = (datetime.datetime(int(end_date[0:4]),int(end_date[4:6]),int(end_date[6:]))-datetime.datetime(2015,1,5)).days
else:
num = (datetime.datetime(int(end_date[0:4]),int(end_date[4:6]),int(end_date[6:]))-datetime.datetime(int(start_date[0:4]),int(start_date[4:6]),int(start_date[6:]))).days
return num
from gmsdk import md
ret = md.init("yourname", "password")
if daima[0:2]=='60':
daima_new = str('SHSE.')+str(daima)
else:
daima_new = str('SZSE.')+str(daima)
mins = n*60
strtime = []
openp = []
high = []
low = []
close = []
volume = []
amount = []
days_internal = 120
LoopNum = divmod(count_split_days(start_date,end_date),days_internal)[0]+1
left = divmod(count_split_days(start_date,end_date),days_internal)[1]
bars_set = []
n = 0
for ln in range(0,LoopNum):
if n == 0:
bars = md.get_bars(daima_new,mins,str(datetime.datetime(int(start_date[0:4]),int(start_date[4:6]),int(start_date[6:]))+datetime.timedelta(days=days_internal*(ln)))[0:10]+' 09:00:00',str(datetime.datetime(int(start_date[0:4]),int(start_date[4:6]),int(start_date[6:]))+datetime.timedelta(days=days_internal*(ln+1)))[0:10]+' 15:30:00')
n = n+1
else:
if ln == LoopNum-1:
bars = md.get_bars(daima_new,mins,str(datetime.datetime(int(start_date[0:4]),int(start_date[4:6]),int(start_date[6:]))+datetime.timedelta(days=days_internal*(ln)+1))[0:10]+' 09:00:00',end_date[0:4]+'-'+end_date[4:6]+'-'+end_date[6:]+' 15:30:00')
n = n+1
else:
bars = md.get_bars(daima_new,mins,str(datetime.datetime(int(start_date[0:4]),int(start_date[4:6]),int(start_date[6:]))+datetime.timedelta(days=days_internal*(ln)+1))[0:10]+' 09:00:00',str(datetime.datetime(int(start_date[0:4]),int(start_date[4:6]),int(start_date[6:]))+datetime.timedelta(days=days_internal*(ln+1)))[0:10]+' 15:30:00')
n = n+1
bars_set.append(bars)
for SignalBars in bars_set:
for b in SignalBars:
strtime.append(b.strtime)
openp.append(b.open)
high.append(b.high)
low.append(b.low)
close.append(b.close)
volume.append(b.volume)
amount.append(b.amount)
mins = pd.DataFrame({'strtime':strtime,
'open':openp,
'close':close,
'high':high,
'low':low,
'volume':volume,
'amount':amount})
return mins
def min_data(daima, start_date, end_date, n):
def count_split_days(start_date, end_date):
if datetime.datetime(int(start_date[0:4]), int(start_date[4:6]),
int(start_date[6:])) < datetime.datetime(
2015, 1, 5):
num = (datetime.datetime(int(end_date[0:4]), int(end_date[4:6]),
int(end_date[6:])) -
datetime.datetime(2015, 1, 5)).days
else:
num = (datetime.datetime(int(end_date[0:4]), int(end_date[4:6]),
int(end_date[6:])) -
datetime.datetime(int(start_date[0:4]), int(
start_date[4:6]), int(start_date[6:]))).days
return num
from gmsdk import md
ret = md.init("yourname", "password")
if daima[0:2] == '60':
daima_new = str('SHSE.') + str(daima)
else:
daima_new = str('SZSE.') + str(daima)
mins = n * 60
strtime = []
openp = []
high = []
low = []
close = []
volume = []
amount = []
days_internal = 120
LoopNum = divmod(count_split_days(start_date, end_date),
days_internal)[0] + 1
left = divmod(count_split_days(start_date, end_date), days_internal)[1]
bars_set = []
n = 0
for ln in range(0, LoopNum):
if n == 0:
bars = md.get_bars(
daima_new, mins,
str(
datetime.datetime(int(start_date[0:4]), int(
start_date[4:6]), int(start_date[6:])) +
datetime.timedelta(days=days_internal * (ln)))[0:10] +
' 09:00:00',
str(
datetime.datetime(int(start_date[0:4]), int(
start_date[4:6]), int(start_date[6:])) +
datetime.timedelta(days=days_internal *
(ln + 1)))[0:10] + ' 15:30:00')
n = n + 1
else:
if ln == LoopNum - 1:
bars = md.get_bars(
daima_new, mins,
str(
datetime.datetime(int(start_date[
0:4]), int(start_date[4:6]), int(start_date[6:])) +
datetime.timedelta(days=days_internal *
(ln) + 1))[0:10] + ' 09:00:00',
end_date[0:4] + '-' + end_date[4:6] + '-' + end_date[6:] +
' 15:30:00')
n = n + 1
else:
bars = md.get_bars(
daima_new, mins,
str(
datetime.datetime(int(start_date[
0:4]), int(start_date[4:6]), int(start_date[6:])) +
datetime.timedelta(days=days_internal *
(ln) + 1))[0:10] + ' 09:00:00',
str(
datetime.datetime(int(start_date[
0:4]), int(start_date[4:6]), int(start_date[6:])) +
datetime.timedelta(days=days_internal *
(ln + 1)))[0:10] + ' 15:30:00')
n = n + 1
bars_set.append(bars)
for SignalBars in bars_set:
for b in SignalBars:
strtime.append(b.strtime)
openp.append(b.open)
high.append(b.high)
low.append(b.low)
close.append(b.close)
volume.append(b.volume)
amount.append(b.amount)
mins = pd.DataFrame({
'strtime': strtime,
'open': openp,
'close': close,
'high': high,
'low': low,
'volume': volume,
'amount': amount
})
return mins
from gmsdk import md
ret = md.init("13601380996",
"it@iZ23psatkqsZ",
2,
"SHFE.RU1709.bar.60")
ticks = md.get_ticks("SHSE.600000,SZSE.000001",
"2017-06-01 09:30:00",
"2017-06-01 09:31:00")
print(ticks)
ret = md.init("13601380996",
"it@iZ23psatkqsZ",
4,
"SHFE.ru1709.tick")
ti = md.get_ticks("SHFE.ru1709.tick","2017-06-01 09:30:00","2017-06-01 09:31:00")
print("aaa!")
print(ti)
# -*- coding: utf-8 -*-
import datetime
from gmsdk import md, to_dict
import pandas as pd
md.init('13601380996', 'it@iZ23psatkqsZ')
CFFEX = ['IF', 'IH', 'IC', 'T', 'TF']
CZCE = ['CF', 'FG', 'MA', 'RM', 'SR', 'TA', 'ZC']
SHFE = ['AL', 'BU', 'CU', 'HC', 'NI', 'RB', 'RU', 'SN', 'ZN']
DCE = ['C', 'CS', 'I', 'J', 'JD', 'JM', 'L', 'M', 'P', 'PP', 'V', 'Y']
def mtsymbol_list(symbol_list):
z = len(symbol_list)
ret = ''
for i in range(z):
ret = ret + symbol_list[i] + ','
ret = ret[:len(ret) - 1]
return ret
def to_pd(var, index):
ret = []
for i in var:
ret.append(to_dict(i))
ret = pd.DataFrame(ret)
ret = ret.set_index(index)
return ret
def get_shse():
var = md.get_instruments('SHSE', 1, 0)
# -*- coding: utf-8 -*-
import datetime
from gmsdk import md,to_dict
import pandas as pd
md.init('*****@*****.**','tquant121')
CFFEX = ['IF','IH','IC','T','TF']
CZCE =['CF','FG','MA','RM','SR','TA','ZC']
SHFE = ['AL','BU','CU','HC','NI','RB','RU','SN','ZN']
DCE=['C','CS','I','J','JD','JM','L','M','P','PP','V','Y']
def mtsymbol_list(symbol_list):
z = len (symbol_list)
ret = ''
for i in range(z):
ret = ret + symbol_list[i] +','
ret = ret[:len(ret)-1]
return ret
def to_pd(var,index):
ret =[]
for i in var:
ret.append(to_dict(i))
ret = pd.DataFrame (ret)
ret = ret.set_index(index)
return ret
def get_shse( ):
var =md.get_instruments('SHSE', 1, 0)
return to_pd(var,'symbol')
def get_szse():
var =md.get_instruments('SZSE', 1, 0)
return to_pd(var,'symbol')
def get_shfe():
var = md.get_instruments('SHFE', 4, 1)
# -*- coding: utf-8 -*-
import datetime
from gmsdk import md,to_dict
import pandas as pd
md.init('*****@*****.**','it@iZ23psatkqsZ')
CFFEX = ['IF','IH','IC','T','TF']
CZCE =['CF','FG','MA','RM','SR','TA','ZC']
SHFE = ['AL','BU','CU','HC','NI','RB','RU','SN','ZN']
DCE=['C','CS','I','J','JD','JM','L','M','P','PP','V','Y']
def mtsymbol_list(symbol_list):
z = len (symbol_list)
ret = ''
for i in range(z):
ret = ret + symbol_list[i] +','
ret = ret[:len(ret)-1]
return ret
def to_pd(var,index):
ret =[]
for i in var:
ret.append(to_dict(i))
ret = pd.DataFrame (ret)
ret = ret.set_index(index)
return ret
def get_shse( ):
var =md.get_instruments('SHSE', 1, 0)
return to_pd(var,'symbol')
def get_szse():
var =md.get_instruments('SZSE', 1, 0)
return to_pd(var,'symbol')
def get_shfe():
var = md.get_instruments('SHFE', 4, 1)
from dateutil.parser import parse
from gmsdk import md
import pandas as pd
import numpy as np
import time
# st = time.time()
md.init('cloud.myquant.cn:8000', '18616622990', '034863342')
class GMPrice(object):
def __init__(self, symbol=None, bar_types=None, begin_time=None, end_time=None):
self.symbol = symbol
self.bar_type = bar_types
self.begin_time = begin_time
self.end_time = end_time
def __var_df(self, symbol=None, bar_type=None, begin_time=None, end_time=None, method=''):
symbol = symbol if symbol else self.symbol
bar_type = bar_type if bar_type else self.bar_type
begin_time = begin_time if begin_time else self.begin_time
end_time = end_time if end_time else self.end_time
a = md.__getattribute__(method)
if method in ['get_bars']:
r = a(symbol, bar_type, begin_time, end_time)
elif method in ['get_dailybars', 'get_ticks']:
r = a(symbol, begin_time, end_time)
elif method in ['get_last_ticks', 'get_last_dailybars']:
r = a(symbol)
# ret = self.__df_trans(r)
# -*- coding: utf-8 -*-
import datetime
import pandas as pd
from gmsdk import md, to_dict
md.init('13382753152', '940809')
CFFEX = ['IF', 'IH', 'IC', 'T', 'TF']
CZCE = ['CF', 'FG', 'MA', 'RM', 'SR', 'TA', 'ZC']
SHFE = ['AL', 'BU', 'CU', 'HC', 'NI', 'RB', 'RU', 'SN', 'ZN']
DCE = ['C', 'CS', 'I', 'J', 'JD', 'JM', 'L', 'M', 'P', 'PP', 'V', 'Y']
def mtsymbol_list(symbol_list):
z = len(symbol_list)
ret = ''
for i in range(z):
ret = ret + symbol_list[i] + ','
ret = ret[:len(ret) - 1]
return ret
def to_pd(var, index):
ret = []
for i in var:
ret.append(to_dict(i))
ret = pd.DataFrame(ret)
ret = ret.set_index(index)
return ret
def __init__(self,configPath = r'E:\stocks_data_min\StocksMinDB\configs'):
self.byDayDb = StocksMinDB(configpath=configPath,corenum=1)
self.byDayDb.connectDB('stocks_data_min_by_day')
self.byStkDb = StocksMinDB(configpath=configPath,corenum=1)
self.byStkDb.connectDB('stocks_data_min_by_stock')
md.init('18201141877','Wqxl7309')
def updatePal(palPath=None):
start = time.time()
md.init('18201141877', 'Wqxl7309')
if not w.isconnected():
w.start()
palPath = r'E:\bqfcts\bqfcts\data\Paltest' if palPath is None else palPath
tempFilePath = os.path.join(palPath,'temp_files')
if not os.path.exists(tempFilePath):
os.mkdir(tempFilePath)
matName = 'data_20150701_now.mat'
savedPal = h5py.File(os.path.join(palPath,matName))
# print(read_cell(savedPal,'sec_names'))
nextTrd = dt.datetime.strptime(str(int(savedPal['nexttrd'][0][0])),'%Y%m%d')
nextTrdStr = nextTrd.strftime('%Y-%m-%d')
updateTime = dt.datetime(nextTrd.year,nextTrd.month,nextTrd.day,15,30,0)
if updateTime > dt.datetime.now():
print('not update time yet')
return
else:
availableDateStr = md.get_last_dailybars('SHSE.000001')[0].strtime[:10]
if int(availableDateStr.replace('-','')) <= int(nextTrdStr.replace('-','')):
print('new data not avaliable yet')
return
else:
print('will update from {0} to {1}'.format(nextTrdStr,availableDateStr))
betweenDays = [tdt.strtime[:10] for tdt in md.get_calendar('SHSE',nextTrdStr,availableDateStr)]
if nextTrdStr!=availableDateStr: # 避免同一日期重复
betweenDays.append(availableDateStr)
betweenDaysNumber = [int(tdt.replace('-','')) for tdt in betweenDays]
newDateNum = len(betweenDaysNumber)
# 更新前 先备份数据
backupPath = os.path.join(palPath,'backup')
cpResult = os.system(r'COPY {0} {1} /Y'.format(os.path.join(palPath,matName),os.path.join(backupPath,matName)))
assert cpResult==0,'backup failed'
gmDateFmt = 'yyyy-mm-dd'
# update indice
indiceNames = ['sh','hs300','zz500','sz50']
indiceCodes = ['000001','000300','000905','000016']
symbols = ','.join(['SHSE.{}'.format(sbl) for sbl in indiceCodes])
indiceBars = md.get_dailybars(symbols,nextTrdStr,availableDateStr)
for dumi,idx in enumerate(indiceNames):
bars = indiceBars[dumi::4]
idxret = np.array([bar.close for bar in bars])/np.array([bar.pre_close for bar in bars]) - 1
idxArray = np.array([betweenDaysNumber,
[bar.open for bar in bars],
[bar.high for bar in bars],
[bar.low for bar in bars],
[bar.close for bar in bars],
[bar.volume for bar in bars],
[bar.amount for bar in bars],
idxret
])
# newIndex = np.column_stack([savedPal['index_{}'.format(idx)][:], idxArray])
pd.DataFrame(np.transpose(idxArray)).to_csv(os.path.join(tempFilePath,'index_{}.csv'.format(idx)),index=False,header=False)
# update stock info
nCut = savedPal['N_cut'][0][0] # 6000
nEnd = savedPal['N_end'][0][0] # last end date id ex.6732
stockNames = read_cell(savedPal, 'stockname')
savedStkcdsGM = ['.'.join([stk[-2:]+'SE',stk[:6]]) for stk in stockNames]
savedStkNum = len(stockNames)
listedStkcdsWind = w.wset('sectorconstituent','date={};sectorid=a001010100000000'.format(availableDateStr)).Data[1]
newStkcdsWind = sorted(list(set(listedStkcdsWind) - set(stockNames)))
if newStkcdsWind:
stockNames.extend( newStkcdsWind )
newStkIpos = [int(tdt.strftime('%Y%m%d')) for tdt in w.wss(newStkcdsWind, 'ipo_date').Data[0]]
newIpoIds = [(w.tdayscount(nextTrd,str(ipo)).Data[0][0]+nEnd) for ipo in newStkIpos]
newStockip = pd.DataFrame([[int(newStkcdsWind[dumi][:6]), newStkIpos[dumi], newIpoIds[dumi],0,0,0,0,0] for dumi in range(len(newStkcdsWind))])
newStockip.to_csv( os.path.join(tempFilePath,'stockip.csv'),index=False,header=False )
else:
pd.DataFrame([]).to_csv(os.path.join(tempFilePath, 'stockip.csv'), index=False, header=False)
newStkcdsGm = ['.'.join([stk[-2:]+'SE',stk[:6]]) for stk in newStkcdsWind]
allStkcdsGM = savedStkcdsGM + newStkcdsGm # 全体股票包含已退市 与pal行数相同
# allSecNames = pd.DataFrame(w.wss(stockNames,'sec_name').Data[0])
allInstruments = md.get_instruments('SZSE', 1, 0) + md.get_instruments('SHSE', 1, 0)
allInstrumentsDF = pd.DataFrame([[inds.symbol, inds.sec_name] for inds in allInstruments],columns=['symbol','sec_name']).set_index('symbol')
allSecNames = allInstrumentsDF.loc[allStkcdsGM,'sec_name']
allSecNames.to_csv( os.path.join(tempFilePath, 'sec_names.csv'), index=False, header=False )
pd.DataFrame(newStkcdsWind).to_csv( os.path.join(tempFilePath, 'stockname.csv'), index=False, header=False )
# update trade info
pages = ['date','open','high','low','close','volume','amount','pctchg','flow_a_share','total_share','adjfct','adjprc','isst']
newPal = {}
for page in pages:
newPal[page] = pd.DataFrame(np.zeros([len(allStkcdsGM), newDateNum]),index=allStkcdsGM,columns=betweenDays)
lastPal = pd.DataFrame(savedPal['Pal'][:,-1,:],columns=savedStkcdsGM)
barsDaily = md.get_dailybars(','.join(allStkcdsGM), nextTrdStr, availableDateStr)
for bar in barsDaily:
tdt = bar.strtime[:10]
stk = '.'.join([bar.exchange,bar.sec_id])
newPal['date'].loc[stk, tdt] = int(tdt.replace('-',''))
newPal['open'].loc[stk, tdt] = bar.open
newPal['high'].loc[stk, tdt] = bar.high
newPal['low'].loc[stk, tdt] = bar.low
newPal['close'].loc[stk, tdt] = bar.close
newPal['volume'].loc[stk, tdt] = bar.volume
newPal['amount'].loc[stk, tdt] = bar.amount
newPal['pctchg'].loc[stk, tdt] = bar.close/bar.pre_close - 1
# 计算自算复权因子 ： 前一日收盘价*(1+当日收益率)/当日收盘价 s.t. （当日收盘价*当日复权因子）/前一日收盘价 = 1+ret
# 若当日没有交易 ： 沿用前一日 复权因子 循环外处理
# 若前一日没有交易 前一日收盘价 特殊处理：
# 当日有交易 ： 取停牌前最后一个交易日的 收盘价
# 当日没交易 没有退市 ： 沿用前一日复权因子 循环外处理
# 当日没交易 已经退市 ： 沿用前一日复权因子 循环外处理
# 若新股上市第一天 ： 复权因子为1
if stk in newStkcdsGm:
newPal['adjfct'].loc[stk, tdt] = 1
else:
noTrdLast = (lastPal.loc[0, stk] == 0) if tdt == nextTrdStr else (newPal['date'].loc[stk, betweenDays[betweenDays.index(tdt) - 1]] == 0)
if noTrdLast: # 前一日没交易 今日有交易（否则不应出现在bars里面）
lastBar = md.get_last_n_dailybars(stk, 2, end_time=tdt)[-1]
newPal['adjfct'].loc[stk, tdt] = lastPal.loc[15, stk] * lastBar.close * (1 + newPal['pctchg'].loc[stk, tdt]) / bar.close
else:
preClose = lastPal.loc[4,stk] if tdt==nextTrdStr else newPal['close'].loc[stk,betweenDays[betweenDays.index(tdt)-1]]
newPal['adjfct'].loc[stk, tdt] = lastPal.loc[15, stk] * preClose * (1 + newPal['pctchg'].loc[stk, tdt]) / bar.close
for dumi,tdt in enumerate(betweenDays):
idx = newPal['adjfct'].loc[:,tdt]==0
idx = idx.values
if tdt==nextTrdStr:
newPal['adjfct'].loc[idx[:savedStkNum], tdt] = lastPal.loc[15,:].values[idx[:savedStkNum]]
else:
newPal['adjfct'].loc[idx, tdt] = newPal['adjfct'].loc[idx, betweenDays[dumi-1]]
newPal['adjprc'] = newPal['adjfct']*newPal['close']
shareBar = md.get_share_index(','.join(allStkcdsGM), nextTrdStr, availableDateStr)
for bar in shareBar:
tdt = bar.pub_date
stk = bar.symbol
newPal['flow_a_share'].loc[stk, tdt] = bar.flow_a_share
newPal['total_share'].loc[stk, tdt] = bar.total_share
isST = np.array([int('ST' in sn) for sn in allSecNames.values])
newPal['isst'] = pd.DataFrame(np.repeat(np.reshape(isST,(isST.shape[0],1)),len(betweenDays),axis=1), index=allStkcdsGM, columns=betweenDays)
for page in newPal:
newPal[page].to_csv(os.path.join(tempFilePath,'{}.csv'.format(page)),index=False,header=False )
print('Pal temp files update finished with {0} stocks and {1} days in {2} seconds '.format(len(newStkcdsWind),len(betweenDays),time.time() - start))
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from gmsdk import md, to_dict
md.init('*****@*****.**', '123456')
# 提取tick数据
r = md.get_ticks(
'SHSE.600000',
'2016-01-04 09:00:00',
'2016-01-04 12:00:00',
)
print('get_ticks: ', len(r))
#提取tick快照
r = md.get_last_ticks('SHSE.600000')
print('get_last_ticks: ', len(r))
#提取最新N笔tick数据
r = md.get_last_n_ticks(
'SHSE.600000', 10)
print('get_last_n_ticks(10): ', len(r))
#提起一分钟分时数据(freq=60 seconds)
r = md.get_bars(
'SHSE.600000',
60,
'2015-12-01 09:30:00',
'2015-12-10 09:31:00',
)
print('get_bars: ', len(r))
def data_subscribe(source):
""" 数据源订阅 , pool_columns 提供需要订阅的字段，需要更新 undl_pool_info """
COLNUM = len(gv.POOL_COLUMNS)
if source == 'wind':
# 定义数据源对应 callback 函数
def wind_callback(indata):
if indata.ErrorCode != 0:
raise Exception('Error in callback with ErrorCode %d' %
indata.ErrorCode) # 实际使用时，为防止中断可改为log输出
for dumi in range(len(indata.Codes)):
fieldlen = len(indata.Fields)
if fieldlen == COLNUM: # 只有在所有field都有数据的时候才存储
tempdata = []
for dumj in range(fieldlen):
tempdata.append(indata.Data[dumj][dumi])
gv.UNDL_POOL_INFO[indata.Codes[dumi]] = tempdata
w.start()
underlyings = list(gv.UNDL_POOL['total'])
w.wsq(','.join(underlyings),
','.join(gv.POOL_COLUMNS),
func=wind_callback)
print('data subscribed with source %s' % source)
elif source == 'goldmine':
vars = {'rt_last': 'tick.last_price', 'rt_time': 'tick.str_time'}
def on_tick(tick):
tempdata = []
for col in gv.POOL_COLUMNS:
tempdata.append(eval(vars[col]))
gv.UNDL_POOL_INFO[addfix(tick.sec_id)] = tempdata
# 提取当前资产池中的代码，并转换为gm所需格式
underlyings = list(gv.UNDL_POOL['total'])
ret = md.init(username="******", password="******", mode=3)
if ret != 0:
raise Exception('Error in initiation with ErrorCode %d' % ret)
ret = md.subscribe(','.join(underlyings))
if ret != 0:
raise Exception('Error in subscribe with ErrorCode %d' % ret)
# 添加回调函数
md.ev_tick += on_tick
# 初始填充POOL，确保不会出现 NAN
fillundl = ','.join(underlyings).replace('.tick', '')
ticks = md.get_last_ticks(fillundl)
for tick in ticks:
tempdata = []
for col in gv.POOL_COLUMNS:
tempdata.append(eval(vars[col]))
#UNDL_POOL_INFO[addfix(tick.sec_id)] = tempdata
gv.UNDL_POOL_INFO[RawHoldingStocks.addfix(tick.sec_id).addfix(
tick.sec_id)] = tempdata
# 加入线程
data_thread = threading.Thread(target=md.run)
if not gv.PRE_THREADS.get(source): # 如果是第一次建立线程则创建，否则只要重新订阅
gv.PRE_THREADS[source] = data_thread
data_thread.start()
else:
ret = md.resubscribe(','.join(underlyings))
if ret != 0:
raise Exception('Error in subscribe with ErrorCode %d' % ret)
print(threading.enumerate())
print('data subscribed with source %s' % source)
elif source == 'goldmine_snapshot':
vars = {'rt_last': 'tick.last_price', 'rt_time': 'tick.str_time'}
def pull_ticks():
underlyings = list(gv.UNDL_POOL['total'])
ticks = md.get_last_ticks(','.join(underlyings))
for tick in ticks:
tempdata = []
for col in gv.POOL_COLUMNS:
tempdata.append(eval(vars[col]))
#UNDL_POOL_INFO[addfix(tick.sec_id)] = tempdata
gv.UNDL_POOL_INFO[RawHoldingStocks.addfix(
tick.sec_id)] = tempdata
ret = md.init(username="******", password="******", mode=1)
if ret != 0:
raise Exception('Error in initiation with ErrorCode %d' % ret)
data_thread = NewThread(target=pull_ticks)
if not gv.PRE_THREADS.get(source): # 如果是第一次建立线程则创建，否则先关闭老线程，再开启新线程
gv.PRE_THREADS[source] = data_thread
else:
gv.PRE_THREADS[source].stop()
gv.PRE_THREADS[source] = data_thread
data_thread.start()
print(threading.enumerate())
print('data subscribed with source %s' % source)
elif source == 'simulation':
def simugen(pathtype='Brownian', step=1):
""" 模拟行情数据生成器 """
colnum = len(gv.POOL_COLUMNS)
holdings = gv.UNDL_POOL['total']
for undl in holdings:
if undl not in gv.UNDL_POOL_INFO:
gv.UNDL_POOL_INFO[undl] = np.random.rand(1, colnum)[0]
else:
if pathtype == 'Geometric':
trend = np.random.randn(1, colnum)[0]
sigma = np.random.rand(1, colnum)[0]
gv.UNDL_POOL_INFO[undl] *= np.exp(
trend * step + sigma * np.sqrt(step) *
np.random.randn(1, colnum))[0]
else:
gv.UNDL_POOL_INFO[undl] += np.sqrt(
step) * np.random.randn(1, colnum)[0]
data_thread = NewThread(target=simugen)
if not gv.PRE_THREADS.get(source): # 如果是第一次建立线程则创建，否则先关闭老线程，再开启新线程
gv.PRE_THREADS[source] = data_thread
else:
gv.PRE_THREADS[source].stop()
gv.PRE_THREADS[source] = data_thread
data_thread.start()
print(threading.enumerate())
print('data subscribed with source: %s' % source)
else:
print('No source infomation provided, can not subscribe!')