def add_instrument(self, name):
self.tick_data[name] = []
dtypes = [(field, dtype_map[field]) for field in day_data_list]
self.day_data[name] = data_handler.DynamicRecArray(dtype = dtypes)
dtypes = [(field, dtype_map[field]) for field in min_data_list]
self.min_data[name] = {1: data_handler.DynamicRecArray(dtype = dtypes)}
self.cur_day[name] = dict([(item, 0) for item in day_data_list])
self.cur_min[name] = dict([(item, 0) for item in min_data_list])
self.day_data_func[name] = []
self.min_data_func[name] = {}
self.cur_min[name]['datetime'] = datetime.datetime.fromordinal(self.scur_day.toordinal())
self.cur_min[name]['date'] = self.scur_day
self.cur_day[name]['date'] = self.scur_day
def run_loop_sim(self):
dra = dh.DynamicRecArray(dataframe=self.df)
sim_data = dra.data
nlen = len(dra)
self.scur_day = sim_data['date'][0]
for n in range(1, nlen-1):
self.timestamp = datetime.datetime.utcfromtimestamp(sim_data['datetime'][n].astype('O')/1e9)
self.traded_vol = self.traded_cost = self.closeout_pnl = 0
self.traded_price = sim_data['open'][n]
sim_data['pos'][n] = sim_data['pos'][n - 1]
if not self.check_data_invalid(sim_data, n-1):
if (n >= nlen - 2) or (sim_data['contract'][n]!=sim_data['contract'][n+1]):
for tradepos in self.positions:
self.close_tradepos(tradepos, sim_data['open'][n])
self.positions = []
else:
self.on_bar(sim_data, n - 1)
self.check_curr_pos(sim_data, n)
sim_data['pos'][n] += self.traded_vol
sim_data['cost'][n] = self.traded_cost
sim_data['closeout'][n] = self.closeout_pnl
sim_data['traded_price'][n] = self.traded_price
if self.scur_day != sim_data['date'][n+1]:
self.daily_initialize(sim_data, n)
self.scur_day = sim_data['date'][n + 1]
#pos = pd.Series(sim_data['pos'], index = self.df.index, name = 'pos')
#closeout = pd.Series(sim_data['closeout'], index = self.df.index, name = 'closeout')
#tp = pd.Series(sim_data['traded_price'], index=self.df.index, name='traded_price')
#cost = pd.Series(sim_data['cost'], index=self.df.index, name='cost')
#out_df = pd.concat([self.df.open, self.df.high, self.df.low, self.df.close, self.df.date, self.df.min_id, pos, tp, cost, closeout], \
# join='outer', axis = 1)
out_df = pd.DataFrame(sim_data)
return out_df, self.closed_trades
def __init__(self, config, agent = None):
self.load_config(config)
self.underliers = self.products.keys()
self.option_dict = self.get_option_dict(self.products)
self.option_insts = self.option_dict.values()
self.instIDs = self.underliers + self.option_insts
self.underlying = [None] * len(self.instIDs)
self.expiry_map = {}
self.inst_map = dict([(instID, i) for i, instID in enumerate(self.instIDs)])
self.risk_table = dh.DynamicRecArray( \
dtype = [('name', '|S50'), ('product', '|S50'), \
('under', '|S50'), ('cont_mth', 'i8'), ('otype', '|S50'), \
('strike', 'f8'), ('multiple', 'i8'), ('df', 'f8'), \
('margin_long', 'f8'), ('margin_short', 'f8'), ('under_price', 'f8'), \
('pos_long', 'i8'), ('pos_short', 'i8'), ('out_long', 'i8'), ('out_short', 'i8'), \
('pv', 'f8'), ('delta', 'f8'), ('gamma', 'f8'), ('vega', 'f8'), ('theta', 'f8'), \
('ppv', 'f8'), ('pdelta', 'f8'), ('pgamma', 'f8'), ('pvega', 'f8'), ('ptheta', 'f8')], \
nlen = len(self.instIDs))
self.risk_table.data['name'] = self.instIDs
self.risk_table.data['under'] = self.risk_table.data['name']
self.risk_table.data['df'] = 1.0
self.agent = agent
self.folder = ''
self.submitted_pos = dict([(inst, []) for inst in self.instIDs])
self.hedge_config = {'delta_algo': 'ExecAlgo1DFixT', 'delta_args': {'time_period': 50, 'tick_num': 1}, \
'vega_algo': 'ExecAlgo1DFixT', 'vega_args': {'time_period': 50, 'tick_num': 1}, }
def run_loop_sim(self):
sim_data = dh.DynamicRecArray(dataframe=self.df)
nlen = len(sim_data)
positions = []
closed_trades = []
tradeid = 0
curr_date = None
for n in range(nlen - 3):
cost = 0
pos = sim_data['pos'][n]
if sim_data['ma'][n] == 0 or sim_data['chan_h'][
n] == 0 or sim_data['dopen'][n] == 0:
continue
if curr_date != sim_data['date'][n]:
curr_date = sim_data['date'][n]
dopen = sim_data['dopen'][n]
rng = max(self.min_rng * dopen, self.k * sim_data['tr'][n])
buytrig = dopen + rng
selltrig = dopen - rng
if sim_data['ma'][n] > dopen:
buytrig += self.f * rng
else:
selltrig -= self.f * rng
continue
if self.price_mode == 'TP':
ref_long = ref_short = (sim_data['close'][n] +
sim_data['high'][n] +
sim_data['low'][n]) / 3.0
elif self.price_mode == 'HL':
ref_long = sim_data['high'][n]
ref_short = sim_data['low'][n]
elif self.price_mode == 'CL':
ref_long = ref_short = sim_data['close'][n]
else:
ref_long = ref_short = sim_data['open'][n + 1]
target_pos = (ref_long > buytrig) - (ref_short < selltrig)
if len(positions) > 0:
need_close = (self.close_daily or
(curr_date == sim_data['date'][-1])) and (
sim_data['min_id'][n] >= config['exit_min'])
for tradepos in positions:
ep = sim_data['low'][n] if tradepos.pos > 0 else sim_data[
'high'][n]
if need_close or tradepos.check_exit(
sim_data['open'][n + 1],
0) or (tradepos.pos * target_pos < 0):
tradepos.close(
sim_data['open'][n + 1] -
self.offset * misc.sign(tradepos.pos),
sim_data['datetime'][n + 1])
tradepos.exit_tradeid = tradeid
tradeid += 1
pos -= tradepos.pos
cost += abs(tradepos.pos) * (
self.offset + sim_data['open'][n + 1] * self.tcost)
closed_trades.append(tradepos)
elif self.pos_update:
tradepos.update_price(ep)
positions = [pos for pos in positions if not pos.is_closed]
if need_close:
continue
if target_pos != 0:
if (not self.use_chan) or (
((ref_long > sim_data['chanh'][n]) and target_pos > 0) or
((ref_short < sim_data['chanl'][n]) and target_pos < 0)):
new_pos = self.pos_class(
[sim_data['contract'][n]], [1], self.unit * target_pos,
sim_data['open'][n + 1] + target_pos * self.offset,
buytrig, **self.pos_args)
tradeid += 1
new_pos.entry_tradeid = tradeid
new_pos.open(
sim_data['open'][n + 1] + target_pos * self.offset,
sim_data['datetime'][n + 1])
positions.append(new_pos)
pos += self.unit * target_pos
cost += abs(target_pos) * (
self.offset + sim_data['open'][n + 1] * self.tcost)
sim_data['cost'][n + 1] = cost
sim_data['pos'][n + 1] = pos
out_df = pd.concat([])
return out_df, closed_trades
def run_sim(config):
mdf = config['mdf']
close_daily = config['close_daily']
offset = config['offset']
k = config['param'][0]
win = config['param'][1]
multiplier = config['param'][2]
f = config['param'][3]
price_mode = config.get('price_mode', 'TP')
pos_update = config['pos_update']
pos_class = config['pos_class']
pos_args = config['pos_args']
proc_func = config['proc_func']
proc_args = config['proc_args']
chan_func = config['chan_func']
chan_high = eval(chan_func['high']['func'])
chan_low = eval(chan_func['low']['func'])
tcost = config['trans_cost']
unit = config['unit']
SL = config['stoploss']
min_rng = config['min_range']
chan = config['chan']
use_chan = config['use_chan']
no_trade_set = config['no_trade_set']
pos_freq = config.get('pos_freq', 1)
xdf = proc_func(mdf, **proc_args)
if win == -1:
tr = pd.concat(
[xdf.high - xdf.low,
abs(xdf.close - xdf.close.shift(1))],
join='outer',
axis=1).max(axis=1)
elif win == 0:
tr = pd.concat(
[(pd.rolling_max(xdf.high, 2) - pd.rolling_min(xdf.close, 2)) *
multiplier,
(pd.rolling_max(xdf.close, 2) - pd.rolling_min(xdf.low, 2)) *
multiplier, xdf.high - xdf.close, xdf.close - xdf.low],
join='outer',
axis=1).max(axis=1)
else:
tr = pd.concat([
pd.rolling_max(xdf.high, win) - pd.rolling_min(xdf.close, win),
pd.rolling_max(xdf.close, win) - pd.rolling_min(xdf.low, win)
],
join='outer',
axis=1).max(axis=1)
xdf['TR'] = tr
xdf['chan_h'] = chan_high(xdf['high'], chan, **chan_func['high']['args'])
xdf['chan_l'] = chan_low(xdf['low'], chan, **chan_func['low']['args'])
xdata = pd.concat([
xdf['TR'].shift(1), xdf['MA'].shift(1), xdf['chan_h'].shift(1),
xdf['chan_l'].shift(1), xdf['open']
],
axis=1,
keys=['tr', 'ma', 'chanh', 'chanl', 'dopen']).fillna(0)
df = mdf.join(xdata, how='left').fillna(method='ffill')
df['pos'] = 0
df['cost'] = 0
df['traded_price'] = df['open']
sim_data = dh.DynamicRecArray(dataframe=df)
nlen = len(sim_data)
positions = []
closed_trades = []
tradeid = 0
curr_date = None
buytrig = selltrig = 0.0
for n in range(nlen - 3):
cost = 0
pos = sim_data['pos'][n]
if sim_data['ma'][n] == 0 or sim_data['chan_h'][n] == 0 or sim_data[
'dopen'][n] == 0:
continue
if curr_date != sim_data['date'][n]:
curr_date = sim_data['date'][n]
dopen = sim_data['dopen'][n]
rng = max(min_rng * dopen, k * sim_data['tr'][n])
buytrig = dopen + rng
selltrig = dopen - rng
if sim_data['ma'][n] > dopen:
buytrig += f * rng
else:
selltrig -= f * rng
continue
if price_mode == 'TP':
ref_long = ref_short = (sim_data['close'][n] + sim_data['high'][n]
+ sim_data['low'][n]) / 3.0
elif price_mode == 'HL':
ref_long = sim_data['high'][n]
ref_short = sim_data['low'][n]
elif price_mode == 'CL':
ref_long = ref_short = sim_data['close'][n]
else:
ref_long = ref_short = sim_data['open'][n + 1]
target_pos = (ref_long > buytrig) - (ref_short < selltrig)
if len(positions) > 0:
need_close = (close_daily or
(curr_date == sim_data['date'][-1])) and (
sim_data['min_id'][n] >= config['exit_min'])
for tradepos in positions:
ep = sim_data['low'][n] if tradepos.pos > 0 else sim_data[
'high'][n]
if need_close or tradepos.check_exit(
sim_data['open'][n + 1],
0) or (tradepos.pos * target_pos < 0):
tradepos.close(
sim_data['open'][n + 1] -
offset * misc.sign(tradepos.pos),
sim_data['datetime'][n + 1])
tradepos.exit_tradeid = tradeid
tradeid += 1
pos -= tradepos.pos
cost += abs(tradepos.pos) * (
offset + sim_data['open'][n + 1] * tcost)
closed_trades.append(tradepos)
elif pos_update:
tradepos.update_price(ep)
positions = [pos for pos in positions if not pos.is_closed]
if need_close:
continue
if target_pos != 0:
if (not use_chan) or (
((ref_long > sim_data['chanh'][n]) and target_pos > 0) or
((ref_short < sim_data['chanl'][n]) and target_pos < 0)):
new_pos = pos_class(
[sim_data['contract'][n]], [1], unit * target_pos,
sim_data['open'][n + 1] + target_pos * offset, buytrig,
**pos_args)
tradeid += 1
new_pos.entry_tradeid = tradeid
new_pos.open(sim_data['open'][n + 1] + target_pos * offset,
sim_data['datetime'][n + 1])
positions.append(new_pos)
pos += unit * target_pos
cost += abs(target_pos) * (offset +
sim_data['open'][n + 1] * tcost)
sim_data['cost'][n + 1] = cost
sim_data['pos'][n + 1] = pos
out_df = pd.concat([])
return out_df, closed_trades
def prepare_data_env(self, inst, mid_day=True):
if self.instruments[inst].ptype == instrument.ProductType.Option:
return
if self.daily_data_days > 0 or mid_day:
#self.logger.debug('Updating historical daily data for %s' % self.scur_day.strftime('%Y-%m-%d'))
daily_start = workdays.workday(self.scur_day,
-self.daily_data_days, CHN_Holidays)
daily_end = self.scur_day
ddf = mysqlaccess.load_daily_data_to_df('fut_daily',
inst,
daily_start,
daily_end,
index_col=None)
if len(ddf) > 0:
self.instruments[inst].price = ddf['close'].iloc[-1]
self.instruments[inst].last_update = 0
self.instruments[inst].prev_close = ddf['close'].iloc[-1]
for fobj in self.day_data_func[inst]:
ts = fobj.sfunc(ddf)
if type(ts).__name__ == 'Series':
if ts.name in ddf.columns:
self.logger.warning(
'TimeSeries name %s is already in the columns for inst = %s'
% (ts.name, inst))
ddf[ts.name] = ts
elif type(ts).__name__ == 'DataFrame':
for col_name in ts.columns:
if col_name in ddf.columns:
self.logger.warning(
'TimeSeries name %s is already in the columns for inst = %s'
% (col_name, inst))
ddf[col_name] = ts[col_name]
self.day_data[inst] = data_handler.DynamicRecArray(dataframe=ddf)
if self.min_data_days > 0 or mid_day:
#self.logger.debug('Updating historical min data for %s' % self.scur_day.strftime('%Y-%m-%d'))
d_start = workdays.workday(self.scur_day, -self.min_data_days,
CHN_Holidays)
d_end = self.scur_day
min_start = int(self.instruments[inst].start_tick_id / 1000)
min_end = int(self.instruments[inst].last_tick_id / 1000) + 1
mdf = mysqlaccess.load_min_data_to_df('fut_min',
inst,
d_start,
d_end,
minid_start=min_start,
minid_end=min_end,
database='blueshale',
index_col=None)
mdf = backtest.cleanup_mindata(mdf,
self.instruments[inst].product,
index_col=None)
mdf['bar_id'] = self.conv_bar_id(mdf['min_id'], inst)
if len(mdf) > 0:
min_date = mdf['date'].iloc[-1]
if (len(self.day_data[inst]) == 0) or (
min_date > self.day_data[inst].data['date'][-1]):
ddf = data_handler.conv_ohlc_freq(mdf, 'd', index_col=None)
self.cur_day[inst]['open'] = float(ddf.open[-1])
self.cur_day[inst]['close'] = float(ddf.close[-1])
self.cur_day[inst]['high'] = float(ddf.high[-1])
self.cur_day[inst]['low'] = float(ddf.low[-1])
self.cur_day[inst]['volume'] = int(ddf.volume[-1])
self.cur_day[inst]['openInterest'] = int(
ddf.openInterest[-1])
self.cur_min[inst]['datetime'] = pd.datetime(
*mdf['datetime'].iloc[-1].timetuple()[0:-3])
self.cur_min[inst]['date'] = mdf['date'].iloc[-1]
self.cur_min[inst]['open'] = float(mdf['open'].iloc[-1])
self.cur_min[inst]['close'] = float(mdf['close'].iloc[-1])
self.cur_min[inst]['high'] = float(mdf['high'].iloc[-1])
self.cur_min[inst]['low'] = float(mdf['low'].iloc[-1])
self.cur_min[inst]['volume'] = self.cur_day[inst]['volume']
self.cur_min[inst]['openInterest'] = self.cur_day[inst][
'openInterest']
self.cur_min[inst]['min_id'] = int(mdf['min_id'].iloc[-1])
self.cur_min[inst]['bar_id'] = self.conv_bar_id(
self.cur_min[inst]['min_id'], inst)
self.instruments[inst].price = float(mdf['close'].iloc[-1])
self.instruments[inst].last_update = 0
#self.logger.debug('inst=%s tick data loaded for date=%s' % (inst, min_date))
if 1 not in self.min_data_func[inst]:
self.min_data[inst][1] = data_handler.DynamicRecArray(
dataframe=mdf)
for m in sorted(self.min_data_func[inst]):
if m != 1:
bar_func = lambda ts: self.conv_bar_id(ts, inst)
mdf_m = data_handler.conv_ohlc_freq(
mdf,
str(m) + 'min',
index_col=None,
bar_func=bar_func,
extra_cols=['bar_id'])
else:
mdf_m = mdf
for fobj in self.min_data_func[inst][m]:
ts = fobj.sfunc(mdf_m)
if type(ts).__name__ == 'Series':
if ts.name in mdf_m.columns:
self.logger.warning(
'TimeSeries name %s is already in the columns for inst = %s'
% (ts.name, inst))
mdf_m[ts.name] = ts
elif type(ts).__name__ == 'DataFrame':
for col_name in ts.columns:
if col_name in mdf_m.columns:
self.logger.warning(
'TimeSeries name %s is already in the columns for inst = %s'
% (col_name, inst))
mdf_m[col_name] = ts[col_name]
self.min_data[inst][m] = data_handler.DynamicRecArray(
dataframe=mdf_m)
def plot_indicators(df, ind_fields, ind_levels):
ccolors = ['blue', 'red', 'yellow', 'black', 'green']
ra = dh.DynamicRecArray(dataframe = df)
xdf = ra.data
indx = np.array(range(len(df)))
plt.ion()
fig = plt.figure(facecolor='white')
ax0 = plt.subplot2grid((6, 4), (1, 0), rowspan=4, colspan=4)
finance.candlestick2_ohlc(ax0, xdf['open'], xdf['high'], xdf['low'], xdf['close'], \
width=0.6, colorup='g', colordown='r')
c = 0
for field in ind_fields[0]:
ax0.plot(indx, xdf[field], 'g^', label=field)
c = (c + 1) % len(ccolors)
ax0.grid(True, color='b')
ax0.xaxis.set_major_locator(mticker.MaxNLocator(10))
ax0.yaxis.label.set_color("b")
ax0.spines['bottom'].set_color("#5998ff")
ax0.spines['top'].set_color("#5998ff")
ax0.spines['left'].set_color("#5998ff")
ax0.spines['right'].set_color("#5998ff")
ax0.tick_params(axis='y', colors='b')
plt.gca().yaxis.set_major_locator(mticker.MaxNLocator(prune='upper'))
ax0.tick_params(axis='x', colors='b')
plt.ylabel('Price Chart')
ax0v = ax0.twinx()
ax0v.fill_between(indx, 0, xdf['volume'], facecolor='#00ffe8', alpha=.4)
ax0v.axes.yaxis.set_ticklabels([])
ax0v.grid(False)
ax0v.set_ylim(0, xdf['volume'].max())
ax0v.spines['bottom'].set_color("#5998ff")
ax0v.spines['top'].set_color("#5998ff")
ax0v.spines['left'].set_color("#5998ff")
ax0v.spines['right'].set_color("#5998ff")
ax0v.tick_params(axis='x', colors='b')
ax0v.tick_params(axis='y', colors='b')
ax1 = plt.subplot2grid((6, 4), (0, 0), sharex=ax0, rowspan=1, colspan=4)
indCol = '#c1f9f7'
posCol = '#386d13'
negCol = '#8f2020'
for field in ind_fields[1]:
ax1.plot(indx, xdf[field], indCol, linewidth=1)
ax1.axhline(ind_levels[1][1], color=negCol)
ax1.axhline(ind_levels[1][0], color=posCol)
ax1.fill_between(indx, xdf[ind_fields[1][0]], ind_levels[1][0], \
where=(xdf[ind_fields[1][0]] >= ind_levels[1][0]), facecolor=negCol, edgecolor=negCol, alpha=0.5)
ax1.fill_between(indx, xdf[ind_fields[1][0]], ind_levels[1][1], \
where=(xdf[ind_fields[1][0]] <= ind_levels[1][1]), facecolor=posCol, edgecolor=posCol, alpha=0.5)
ax1.set_yticks([ind_levels[1][1], ind_levels[1][0]])
ax1.yaxis.label.set_color("b")
ax1.spines['bottom'].set_color("#5998ff")
ax1.spines['top'].set_color("#5998ff")
ax1.spines['left'].set_color("#5998ff")
ax1.spines['right'].set_color("#5998ff")
ax1.tick_params(axis='y', colors='b')
ax1.tick_params(axis='x', colors='b')
plt.ylabel(ind_fields[1][0])
ax1.set_title('%s' % ind_fields[1][0])
ax2 = plt.subplot2grid((6, 4), (5, 0), sharex=ax0, rowspan=1, colspan=4)
for field in ind_fields[2]:
ax2.plot(indx, xdf[field], color = ccolors[c], lw=0.6)
c = (c + 1) % len(ccolors)
ax2.spines['bottom'].set_color("#5998ff")
ax2.spines['top'].set_color("#5998ff")
ax2.spines['left'].set_color("#5998ff")
ax2.spines['right'].set_color("#5998ff")
ax2.tick_params(axis='y', colors='b')
ax2.tick_params(axis='x', colors='b')
for ax in ax0, ax0v, ax1, ax2:
for label in ax.get_xticklabels():
label.set_visible(False)
ax1.yaxis.set_major_locator(MyLocator(5, prune='both'))
ax2.yaxis.set_major_locator(MyLocator(5, prune='both'))
plt.show()
