def prepare_data_env(self, inst, mid_day = True):
if self.instruments[inst].ptype == instrument.ProductType.Option:
return
if self.daily_data_days > 0 or mid_day:
self.logger.debug('Updating historical daily data for %s' % self.scur_day.strftime('%Y-%m-%d'))
daily_start = workdays.workday(self.scur_day, -self.daily_data_days, CHN_Holidays)
daily_end = self.scur_day
self.day_data[inst] = mysqlaccess.load_daily_data_to_df('fut_daily', inst, daily_start, daily_end)
df = self.day_data[inst]
if len(df) > 0:
self.instruments[inst].price = df['close'][-1]
self.instruments[inst].last_update = 0
self.instruments[inst].prev_close = df['close'][-1]
for fobj in self.day_data_func[inst]:
ts = fobj.sfunc(df)
df[ts.name]= pd.Series(ts, index=df.index)
if self.min_data_days > 0 or mid_day:
self.logger.debug('Updating historical min data for %s' % self.scur_day.strftime('%Y-%m-%d'))
d_start = workdays.workday(self.scur_day, -self.min_data_days, CHN_Holidays)
d_end = self.scur_day
min_start = int(self.instruments[inst].start_tick_id/1000)
min_end = int(self.instruments[inst].last_tick_id/1000)+1
mindata = mysqlaccess.load_min_data_to_df('fut_min', inst, d_start, d_end, minid_start=min_start, minid_end=min_end, database = 'blueshale')
mindata = backtest.cleanup_mindata(mindata, self.instruments[inst].product)
self.min_data[inst][1] = mindata
if len(mindata)>0:
min_date = mindata.index[-1].date()
if (len(self.day_data[inst].index)==0) or (min_date > self.day_data[inst].index[-1]):
ddf = data_handler.conv_ohlc_freq(mindata, 'd')
self.cur_day[inst]['open'] = float(ddf.open[-1])
self.cur_day[inst]['close'] = float(ddf.close[-1])
self.cur_day[inst]['high'] = float(ddf.high[-1])
self.cur_day[inst]['low'] = float(ddf.low[-1])
self.cur_day[inst]['volume'] = int(ddf.volume[-1])
self.cur_day[inst]['openInterest'] = int(ddf.openInterest[-1])
self.cur_min[inst]['datetime'] = pd.datetime(*mindata.index[-1].timetuple()[0:-3])
self.cur_min[inst]['open'] = float(mindata.ix[-1,'open'])
self.cur_min[inst]['close'] = float(mindata.ix[-1,'close'])
self.cur_min[inst]['high'] = float(mindata.ix[-1,'high'])
self.cur_min[inst]['low'] = float(mindata.ix[-1,'low'])
self.cur_min[inst]['volume'] = self.cur_day[inst]['volume']
self.cur_min[inst]['openInterest'] = self.cur_day[inst]['openInterest']
self.cur_min[inst]['min_id'] = int(mindata.ix[-1,'min_id'])
self.instruments[inst].price = float(mindata.ix[-1,'close'])
self.instruments[inst].last_update = 0
self.logger.debug('inst=%s tick data loaded for date=%s' % (inst, min_date))
for m in self.min_data_func[inst]:
if m != 1:
self.min_data[inst][m] = data_handler.conv_ohlc_freq(self.min_data[inst][1], str(m)+'min')
df = self.min_data[inst][m]
for fobj in self.min_data_func[inst][m]:
ts = fobj.sfunc(df)
df[ts.name]= pd.Series(ts, index=df.index)
def fisher_swing(asset, start_date, end_date, freqs, windows, config):
nearby = config['nearby']
rollrule = config['rollrule']
file_prefix = config['file_prefix'] + '_' + asset + '_'
df = misc.nearby(asset,
nearby,
start_date,
end_date,
rollrule,
'm',
need_shift=True)
df = backtest.cleanup_mindata(df, asset)
output = {}
for ix, freq in enumerate(freqs):
xdf = dh.conv_ohlc_freq(df, freq)
for iy, win in enumerate(windows):
idx = ix * 10 + iy
config['win'] = win
config['freq'] = freq
(res, closed_trades, ts) = fisher_swing_sim(df, xdf, config)
output[idx] = res
print 'saving results for scen = %s' % str(idx)
all_trades = {}
for i, tradepos in enumerate(closed_trades):
all_trades[i] = strat.tradepos2dict(tradepos)
fname = file_prefix + str(idx) + '_trades.csv'
trades = pd.DataFrame.from_dict(all_trades).T
trades.to_csv(fname)
fname = file_prefix + str(idx) + '_dailydata.csv'
ts.to_csv(fname)
fname = file_prefix + 'stats.csv'
res = pd.DataFrame.from_dict(output)
res.to_csv(fname)
return
def r_breaker( asset, start_date, end_date, scenarios, freqs, config):
nearby = config['nearby']
rollrule = config['rollrule']
start_d = misc.day_shift(start_date, '-1b')
file_prefix = config['file_prefix'] + '_' + asset + '_'
ddf = misc.nearby(asset, nearby, start_date, end_date, rollrule, 'd', need_shift=True)
mdf = misc.nearby(asset, nearby, start_date, end_date, rollrule, 'm', need_shift=True)
mdf = backtest.cleanup_mindata(mdf, asset)
#ddf = dh.conv_ohlc_freq(mdf, 'D')
output = {}
for ix, freq in enumerate(freqs):
if freq !='1min':
df = dh.conv_ohlc_freq(mdf, freq)
else:
df = mdf
for iy, k in enumerate(scenarios):
idx = ix*10+iy
config['k'] = k
(res, closed_trades, ts) = r_breaker_sim( ddf, df, config)
output[idx] = res
print 'saving results for scen = %s' % str(idx)
all_trades = {}
for i, tradepos in enumerate(closed_trades):
all_trades[i] = strat.tradepos2dict(tradepos)
fname = file_prefix + str(idx) + '_trades.csv'
trades = pd.DataFrame.from_dict(all_trades).T
trades.to_csv(fname)
fname = file_prefix + str(idx) + '_dailydata.csv'
ts.to_csv(fname)
fname = file_prefix + 'stats.csv'
res = pd.DataFrame.from_dict(output)
res.to_csv(fname)
return
def get_cont_data(asset,
start_date,
end_date,
freq='1m',
nearby=1,
rollrule='-10b'):
if nearby == 0:
mdf = mysqlaccess.load_min_data_to_df('fut_min',
asset,
start_date,
end_date,
minid_start=300,
minid_end=2114,
database='hist_data')
mdf['contract'] = asset
else:
mdf = misc.nearby(asset,
nearby,
start_date,
end_date,
rollrule,
'm',
need_shift=True,
database='hist_data')
mdf = backtest.cleanup_mindata(mdf, asset)
xdf = dh.conv_ohlc_freq(mdf,
freq,
extra_cols=['contract'],
bar_func=dh.bar_conv_func2)
return xdf
def aberration( asset, start_date, end_date, freqs, windows, config):
nearby = config['nearby']
rollrule = config['rollrule']
file_prefix = config['file_prefix'] + '_' + asset + '_'
df = misc.nearby(asset, nearby, start_date, end_date, rollrule, 'm', need_shift=True)
df = backtest.cleanup_mindata(df, asset)
output = {}
for ix, freq in enumerate(freqs):
xdf = dh.conv_ohlc_freq(df, freq)
for iy, win in enumerate(windows):
idx = ix*10+iy
config['win'] = win
(res, closed_trades, ts) = aberration_sim( xdf, config)
output[idx] = res
print 'saving results for scen = %s' % str(idx)
all_trades = {}
for i, tradepos in enumerate(closed_trades):
all_trades[i] = strat.tradepos2dict(tradepos)
fname = file_prefix + str(idx) + '_trades.csv'
trades = pd.DataFrame.from_dict(all_trades).T
trades.to_csv(fname)
fname = file_prefix + str(idx) + '_dailydata.csv'
ts.to_csv(fname)
fname = file_prefix + 'stats.csv'
res = pd.DataFrame.from_dict(output)
res.to_csv(fname)
return
def spd_ratiovol_by_product(products, start_d, end_d, periods=12, tenor='-1m'):
cont_mth, exch = dbaccess.prod_main_cont_exch(products)
contlist = contract_range(products, exch, cont_mth, start_d, end_d)
exp_dates = [get_opt_expiry(cont, inst2contmth(cont)) for cont in contlist]
data = {
'is_dtime': True,
'data_column': 'close',
'data_freq': '30min',
'xs': [0.5, 0.25, 0.75],
'xs_names': ['atm', 'v25', 'v75'],
'xs_func': 'bs_delta_to_strike',
'rehedge_period': 1,
'term_tenor': tenor,
'database': 'hist_data'
}
option_input = {
'otype': True,
'ir': 0.0,
'end_vol': 0.0,
'ref_vol': 0.5,
'pricer_func': 'bsopt.BSOpt',
'delta_func': 'bsopt.BSDelta',
'is_dtime': data['is_dtime'],
}
freq = data['data_freq']
for cont, expiry in zip(contlist, exp_dates):
expiry_d = expiry.date()
if expiry_d > end_d:
break
p_str = '-' + str(int(tenor[1:-1]) * periods) + tenor[-1]
d_start = day_shift(expiry_d, p_str)
cnx = dbaccess.connect(**dbaccess.dbconfig)
if freq == 'd':
df = dbaccess.load_daily_data_to_df(cnx,
'fut_daily',
cont,
d_start,
expiry_d,
index_col=None)
else:
mdf = dbaccess.load_min_data_to_df(cnx,
'fut_min',
cont,
d_start,
expiry_d,
minid_start=300,
minid_end=2115,
index_col=None)
mdf = cleanup_mindata(mdf, products, index_col=None)
mdf['bar_id'] = dh.bar_conv_func2(mdf['min_id'])
df = dh.conv_ohlc_freq(mdf,
freq,
bar_func=dh.bar_conv_func2,
extra_cols=['bar_id'],
index_col=None)
cnx.close()
option_input['expiry'] = expiry
data['dataframe'] = df
vol_df = realized_termstruct(option_input, data)
print cont, expiry_d, vol_df
def fisher_swing(asset, start_date, end_date, freqs, windows, config):
nearby = config["nearby"]
rollrule = config["rollrule"]
file_prefix = config["file_prefix"] + "_" + asset + "_"
df = misc.nearby(asset, nearby, start_date, end_date, rollrule, "m", need_shift=True)
df = backtest.cleanup_mindata(df, asset)
output = {}
for ix, freq in enumerate(freqs):
xdf = dh.conv_ohlc_freq(df, freq)
for iy, win in enumerate(windows):
idx = ix * 10 + iy
config["win"] = win
config["freq"] = freq
(res, closed_trades, ts) = fisher_swing_sim(df, xdf, config)
output[idx] = res
print "saving results for scen = %s" % str(idx)
all_trades = {}
for i, tradepos in enumerate(closed_trades):
all_trades[i] = strat.tradepos2dict(tradepos)
fname = file_prefix + str(idx) + "_trades.csv"
trades = pd.DataFrame.from_dict(all_trades).T
trades.to_csv(fname)
fname = file_prefix + str(idx) + "_dailydata.csv"
ts.to_csv(fname)
fname = file_prefix + "stats.csv"
res = pd.DataFrame.from_dict(output)
res.to_csv(fname)
return
def process_data(self, mdf):
if self.freq == 1:
xdf = mdf
else:
freq_str = str(self.freq) + "min"
xdf = dh.conv_ohlc_freq(mdf, freq_str, extra_cols = ['contract'])
xdf['band_wth'] = self.band_func(xdf, n = self.boll_len) * self.band_ratio
xdf['band_mid'] = self.ma_func(xdf, n=self.boll_len)
xdf['band_up'] = xdf['band_mid'] + xdf['band_wth']
xdf['band_dn'] = xdf['band_mid'] - xdf['band_wth']
if (self.chan_len > 0):
xdf['chan_h'] = dh.DONCH_H(xdf, n = self.chan_len, field = 'high')
xdf['chan_l'] = dh.DONCH_L(xdf, n = self.chan_len, field = 'low')
else:
xdf['chan_h'] = -10000000
xdf['chan_l'] = 10000000
xdata = pd.concat([xdf['band_up'].shift(1), xdf['band_dn'].shift(1), \
xdf['band_mid'].shift(1), xdf['band_wth'].shift(1), \
xdf['chan_h'].shift(1), xdf['chan_l'].shift(1)], axis=1, \
keys=['band_up','band_dn', 'band_mid', 'band_wth', \
'chan_h', 'chan_l'])
self.df = mdf.join(xdata, how = 'left').fillna(method='ffill')
self.df['datetime'] = self.df.index
self.df['cost'] = 0.0
self.df['pos'] = 0.0
self.df['closeout'] = 0.0
self.df['traded_price'] = self.df['open']
def fix_daily_data(contlist, sdate, edate):
for inst in contlist:
ddf = mysqlaccess.load_daily_data_to_df('fut_daily', inst, sdate,
edate)
mdf = mysqlaccess.load_min_data_to_df('fut_min',
inst,
sdate,
edate,
minid_start=300,
minid_end=2115)
dailydata = data_handler.conv_ohlc_freq(mdf, 'D')
for dd in dailydata.index:
d = dd.date()
dslice = dailydata.ix[dd]
if d not in ddf.index:
ddata = {}
ddata['date'] = d
ddata['open'] = float(dslice.open)
ddata['close'] = float(dslice.close)
ddata['high'] = float(dslice.high)
ddata['low'] = float(dslice.low)
ddata['volume'] = int(dslice.volume)
ddata['openInterest'] = int(dslice.openInterest)
print inst, ddata
mysqlaccess.insert_daily_data(inst, ddata)
def process_data(self, df):
if self.freq == 1:
xdf = df
else:
freq_str = str(self.freq) + "min"
xdf = dh.conv_ohlc_freq(df, freq_str, extra_cols=['contract'])
xdf['band_wth'] = self.band_func(
xdf, n=self.boll_len).fillna(method='bfill')
xdf['band_mid'] = self.ma_func(xdf, n=self.boll_len)
xdf['band_up'] = xdf['band_mid'] + xdf['band_wth'] * self.band_ratio
xdf['band_dn'] = xdf['band_mid'] - xdf['band_wth'] * self.band_ratio
xdf['filter_ind'] = self.filter_func(xdf, self.filter_len)
xdf['chan_h'] = dh.DONCH_H(xdf, self.channel)
xdf['chan_l'] = dh.DONCH_L(xdf, self.channel)
xdf['tr_stop_h'] = xdf['chan_h'] - (
xdf['band_wth'] * self.SL /
self.tick_base).astype('int') * self.tick_base
xdf['tr_stop_l'] = xdf['chan_l'] + (
xdf['band_wth'] * self.SL /
self.tick_base).astype('int') * self.tick_base
xdata = pd.concat([xdf['band_up'].shift(1), xdf['band_dn'].shift(1), \
xdf['tr_stop_h'].shift(1), xdf['tr_stop_l'].shift(1), xdf['filter_ind'].shift(1)], axis=1, \
keys=['band_up','band_dn', 'tr_stop_h', 'tr_stop_l', 'filter_ind'])
self.df = df.join(xdata, how='left').fillna(method='ffill').dropna()
self.df['cost'] = 0.0
self.df['pos'] = 0.0
self.df['closeout'] = 0.0
self.df['traded_price'] = self.df['open']
def get_cont_data(asset, start_date, end_date, freq = '1m', nearby = 1, rollrule = '-10b'):
cnx = dbaccess.connect(**dbaccess.hist_dbconfig)
if nearby == 0:
mdf = dbaccess.load_min_data_to_df(cnx, 'fut_min', asset, start_date, end_date, minid_start = 300, minid_end = 2114, database = 'hist_data')
mdf['contract'] = asset
else:
mdf = misc.nearby(asset, nearby, start_date, end_date, rollrule, 'm', need_shift=True, database = 'hist_data')
mdf = misc.cleanup_mindata(mdf, asset)
xdf = dh.conv_ohlc_freq(mdf, freq, extra_cols = ['contract'], bar_func = dh.bar_conv_func2)
return xdf
def hist_cso_by_product(prodcode, start_d, end_d, periods = 24, tenor = '-1w', max_spd = 2, writeDB = False, mode = 'n'):
cont_mth, exch = dbaccess.prod_main_cont_exch(prodcode)
contlist, _ = contract_range(prodcode, exch, cont_mth, start_d, end_d)
exp_dates = [get_opt_expiry(cont, inst2contmth(cont)) for cont in contlist]
if mode == 'n':
xs_func = 'bachelier_delta_to_strike'
pricer_func = 'bsopt.BSFwdNormal'
delta_func = 'bsopt.BSFwdNormalDelta'
else:
xs_func = 'bs_delta_to_strike'
pricer_func = 'bsopt.BSOpt'
delta_func = 'bsopt.BSDelta'
data = {'is_dtime': True,
'data_column': 'close',
'data_freq': '30min',
'xs': [0.5, 0.25, 0.75],
'xs_names': ['atm', 'v25', 'v75'],
'xs_func': xs_func,
'rehedge_period': 1,
'term_tenor': tenor,
'database': 'hist_data'
}
option_input = {'otype': True,
'ir': 0.0,
'end_vol': 0.0,
'ref_vol': 0.5,
'pricer_func': pricer_func,
'delta_func': delta_func,
'is_dtime': data['is_dtime'],
}
freq = data['data_freq']
dbconfig = copy.deepcopy(**dbaccess.hist_dbconfig)
dbconfig['database'] = data['database']
cnx = dbaccess.connect(**dbconfig)
for cont, expiry in zip(contlist, exp_dates):
expiry_d = expiry.date()
if expiry_d > end_d:
break
p_str = '-' + str(int(tenor[1:-1]) * periods) + tenor[-1]
d_start = day_shift(expiry_d, p_str)
if freq == 'd':
df = dbaccess.load_daily_data_to_df(cnx, 'fut_daily', cont, d_start, expiry_d, index_col = None)
else:
mdf = dbaccess.load_min_data_to_df(cnx, 'fut_min', cont, d_start, expiry_d, minid_start=300,
minid_end=2115, index_col = None)
mdf = cleanup_mindata(mdf, prodcode, index_col = None)
mdf['bar_id'] = dh.bar_conv_func2(mdf['min_id'])
df = dh.conv_ohlc_freq(mdf, freq, bar_func=dh.bar_conv_func2, extra_cols=['bar_id'], index_col = None)
cnx.close()
option_input['expiry'] = expiry
data['dataframe'] = df
vol_df = realized_termstruct(option_input, data)
print cont, expiry_d, vol_df
def run_vec_sim(self):
xdf = dh.conv_ohlc_freq(self.df, self.freq, extra_cols=['contract'])
for idx, win in enumerate(self.win_list):
xdf['MA' + str(idx + 1)] = self.ma_func(xdf, win).shift(1)
if self.use_chan:
xdf['chan_high'] = self.chan_high(xdf, self.channel,
**self.high_args).shift(2)
xdf['chan_low'] = self.chan_low(xdf, self.channel,
**self.low_args).shift(2)
else:
xdf['chan_high'] = pd.Series(index=xdf.index)
xdf['chan_low'] = pd.Series(index=xdf.index)
xdf['prev_close'] = xdf['close'].shift(1)
xdf['close_ind'] = np.isnan(xdf['close'].shift(-1))
if self.close_daily:
daily_end = (xdf['date'] != xdf['date'].shift(-1))
xdf['close_ind'] = xdf['close_ind'] | daily_end
long_signal = pd.Series(np.nan, index=xdf.index)
last = len(self.win_list)
long_flag = (xdf['MA1'] >= xdf['MA2']) & (xdf['MA1'] >=
xdf['MA' + str(last)])
if self.use_chan:
long_flag = long_flag & (xdf['open'] >= xdf['chan_high'])
long_signal[long_flag] = 1
cover_flag = (xdf['MA1'] < xdf['MA' + str(last)])
if self.use_chan:
cover_flag = cover_flag | (xdf['open'] < xdf['chan_low'])
long_signal[cover_flag] = 0
long_signal[xdf['close_ind']] = 0
long_signal = long_signal.fillna(method='ffill').fillna(0)
short_signal = pd.Series(np.nan, index=xdf.index)
short_flag = (xdf['MA1'] <= xdf['MA2']) & (xdf['MA1'] <=
xdf['MA' + str(last)])
if self.use_chan:
short_flag = short_flag & (xdf['open'] <= xdf['chan_low'])
cover_flag = (xdf['MA1'] > xdf['MA' + str(last)])
if self.use_chan:
cover_flag = cover_flag | (xdf['open'] > xdf['chan_low'])
short_signal[cover_flag] = 0
short_signal[xdf['close_ind']] = 0
short_signal = short_signal.fillna(method='ffill').fillna(0)
if len(xdf[(long_signal > 0) & (short_signal < 0)]) > 0:
print xdf[(long_signal > 0) & (short_signal < 0)]
print "something wrong with the position as long signal and short signal happen the same time"
xdf['pos'] = long_signal + short_signal
xdf['cost'] = abs(xdf['pos'] - xdf['pos'].shift(1)) * (
self.offset + xdf['open'] * self.tcost)
xdf['cost'] = xdf['cost'].fillna(0.0)
xdf['traded_price'] = xdf.open + (xdf['pos'] -
xdf['pos'].shift(1)) * self.offset
closed_trades = simdf_to_trades1(xdf, slippage=self.offset)
return (xdf, closed_trades)
def process_data(self, mdf):
if self.freq == 1:
xdf = mdf
else:
freq_str = str(self.freq) + "min"
xdf = dh.conv_ohlc_freq(mdf, freq_str, extra_cols=['contract'])
xdf['ATR'] = dh.ATR(xdf, n=self.atr_len)
xdf['ATRMA'] = dh.MA(xdf, n=self.atrma_len, field='ATR')
xdf['RSI'] = dh.RSI(xdf, n=self.rsi_len)
self.df = xdf
self.df['datetime'] = self.df.index
self.df['cost'] = 0.0
self.df['pos'] = 0.0
self.df['traded_price'] = self.df['open']
def process_data(self, mdf):
if self.freq == 1:
xdf = mdf
else:
freq_str = str(self.freq) + "min"
xdf = dh.conv_ohlc_freq(mdf, freq_str, extra_cols = ['contract'])
xdf['ATR'] = dh.ATR(xdf, n = self.atr_len)
xdf['ATRMA'] = dh.MA(xdf, n=self.atrma_len, field = 'ATR')
xdf['RSI'] = dh.RSI(xdf, n = self.rsi_len)
self.df = xdf
self.df['datetime'] = self.df.index
self.df['closeout'] = 0.0
self.df['cost'] = 0.0
self.df['pos'] = 0.0
self.df['traded_price'] = self.df['open']
def chanbreak_sim(mdf, config):
freq = config['freq']
str_freq = str(freq) + 'Min'
xdf = dh.conv_ohlc_freq(mdf, str_freq)
tcost = config['trans_cost']
offset = config['offset']
win = config['win']
chan_func = config['channel_func']
upper_chan_func = eval(chan_func[0])
lower_chan_func = eval(chan_func[1])
entry_chan = win[0]
exit_chan = win[1]
exit_min = config.get('exit_min', 2057)
close_daily = config.get('close_daily', False)
stoploss = config.get('stoploss', 2.0)
xdf['H1'] = upper_chan_func(xdf, entry_chan)
xdf['L1'] = lower_chan_func(xdf, entry_chan)
xdf['H2'] = upper_chan_func(xdf, exit_chan)
xdf['L2'] = lower_chan_func(xdf, exit_chan)
xdf['ATR'] = dh.ATR(xdf, entry_chan)
xdata = pd.concat([xdf['H1'], xdf['L1'], xdf['H2'], xdf['L2'], xdf['ATR']], \
axis=1, keys=['H1', 'L1', 'H2', 'L2', 'ATR']).shift(1)
mdf = mdf.join(xdata, how='left').fillna(method='ffill')
mdf['close_ind'] = np.isnan(mdf['close'].shift(-3))
if close_daily:
mdf['close_ind'] = (mdf['min_id'] >= exit_min) | mdf['close_ind']
long_signal = pd.Series(np.nan, index=mdf.index)
long_signal[(mdf['open'] >= mdf['H1'])] = 1
long_signal[(mdf['open'] <= mdf['L2'])] = 0
if stoploss > 0:
long_signal[(mdf['open'] <= mdf['H1'] - mdf['ATR'] * stoploss)] = 0
long_signal[mdf['close_ind']] = 0
long_signal = long_signal.fillna(method='ffill').fillna(0)
short_signal = pd.Series(np.nan, index=mdf.index)
short_signal[(mdf['open'] <= mdf['L1'])] = -1
short_signal[(mdf['open'] >= mdf['H2'])] = 0
if stoploss > 0:
short_signal[(mdf['open'] >= mdf['L1'] + mdf['ATR'] * stoploss)] = 0
short_signal[mdf['close_ind']] = 0
short_signal = short_signal.fillna(method='ffill').fillna(0)
mdf['pos'] = long_signal + short_signal
mdf['cost'] = abs(mdf['pos'] - mdf['pos'].shift(1)) * (offset +
mdf['open'] * tcost)
mdf['cost'] = mdf['cost'].fillna(0.0)
mdf['traded_price'] = mdf.open + (mdf['pos'] -
mdf['pos'].shift(1)) * offset
closed_trades = backtest.simdf_to_trades1(mdf, slippage=offset)
return (mdf, closed_trades)
def process_data(self, mdf):
if self.freq == 1:
xdf = mdf
else:
freq_str = str(self.freq) + "min"
xdf = dh.conv_ohlc_freq(mdf, freq_str, extra_cols=['contract'])
ma_ribbon = dh.MA_RIBBON(xdf, self.ma_list)
self.df = xdf
for malen in self.ma_list:
self.df['EMA' + str(malen)] = ma_ribbon['EMA_CLOSE_' + str(malen)]
self.df['RIBBON_CORR'] = ma_ribbon['MARIBBON_CORR']
self.df['RIBBON_PVAL'] = ma_ribbon['MARIBBON_PVAL']
self.df['closeout'] = 0.0
self.df['cost'] = 0.0
self.df['pos'] = 0.0
self.df['traded_price'] = self.df['open']
def fix_daily_data(contlist, sdate, edate):
for inst in contlist:
ddf = mysqlaccess.load_daily_data_to_df('fut_daily', inst, sdate, edate)
mdf = mysqlaccess.load_min_data_to_df('fut_min', inst, sdate, edate, minid_start=300, minid_end = 2115)
dailydata = data_handler.conv_ohlc_freq(mdf, 'D')
for dd in dailydata.index:
d = dd.date()
dslice = dailydata.ix[dd]
if d not in ddf.index:
ddata = {}
ddata['date'] = d
ddata['open'] = float(dslice.open)
ddata['close'] = float(dslice.close)
ddata['high'] = float(dslice.high)
ddata['low'] = float(dslice.low)
ddata['volume'] = int(dslice.volume)
ddata['openInterest'] = int(dslice.openInterest)
print inst, ddata
mysqlaccess.insert_daily_data(inst, ddata)
def process_data(self, df):
if self.freq == 1:
xdf = df
else:
freq_str = str(self.freq) + "min"
xdf = dh.conv_ohlc_freq(df, freq_str, extra_cols = ['contract'])
xdf['band_wth'] = self.band_func(xdf, n = self.boll_len).fillna(method = 'bfill')
xdf['band_mid'] = self.ma_func(xdf, n=self.boll_len)
xdf['band_up'] = xdf['band_mid'] + xdf['band_wth'] * self.band_ratio
xdf['band_dn'] = xdf['band_mid'] - xdf['band_wth'] * self.band_ratio
xdf['filter_ind'] = self.filter_func(xdf, self.filter_len)
xdf['chan_h'] = dh.DONCH_H(xdf, self.channel)
xdf['chan_l'] = dh.DONCH_L(xdf, self.channel)
xdf['tr_stop_h'] = xdf['chan_h'] - (xdf['band_wth'] * self.SL / self.tick_base).astype('int') * self.tick_base
xdf['tr_stop_l'] = xdf['chan_l'] + (xdf['band_wth'] * self.SL / self.tick_base).astype('int') * self.tick_base
xdata = pd.concat([xdf['band_up'].shift(1), xdf['band_dn'].shift(1), \
xdf['tr_stop_h'].shift(1), xdf['tr_stop_l'].shift(1), xdf['filter_ind'].shift(1)], axis=1, \
keys=['band_up','band_dn', 'tr_stop_h', 'tr_stop_l', 'filter_ind'])
self.df = df.join(xdata, how = 'left').fillna(method='ffill').dropna()
self.df['cost'] = 0.0
self.df['pos'] = 0.0
self.df['closeout'] = 0.0
self.df['traded_price'] = self.df['open']
def chanbreak_sim(mdf, ddf, config):
freq = config['freq']
str_freq = str(freq) + 'Min'
xdf = dh.conv_ohlc_freq(mdf, str_freq)
start_equity = config['capital']
tcost = config['trans_cost']
unit = config['unit']
k = config['scaler']
marginrate = config['marginrate']
offset = config['offset']
win = config['win']
chan_func = config['channel_func']
upper_chan_func = chan_func[0]
lower_chan_func = chan_func[1]
entry_chan = win
exit_chan = int(entry_chan / k[1])
xdf['H1'] = upper_chan_func(xdf, entry_chan).shift(1)
xdf['L1'] = lower_chan_func(xdf, entry_chan).shift(1)
xdf['H2'] = upper_chan_func(xdf, exit_chan).shift(1)
xdf['L2'] = lower_chan_func(xdf, exit_chan).shift(1)
ddf['ATR'] = dh.ATR(ddf, entry_chan)
ll = mdf.shape[0]
mdf['pos'] = pd.Series([0] * ll, index=mdf.index)
mdf['cost'] = pd.Series([0] * ll, index=mdf.index)
curr_pos = []
closed_trades = []
end_d = mdf.index[-1].date()
tradeid = 0
x_idx = 0
max_idx = len(xdf.index)
for idx, dd in enumerate(mdf.index):
mslice = mdf.ix[dd]
min_id = agent.get_min_id(dd)
d = dd.date()
dslice = ddf.ix[d]
while (x_idx < max_idx - 1) and (xdf.index[x_idx + 1] < dd):
x_idx += 1
xslice = xdf.iloc[x_idx]
if len(curr_pos) == 0:
pos = 0
else:
pos = curr_pos[0].pos
mdf.ix[dd, 'pos'] = pos
if np.isnan(dslice.ATR):
continue
if (min_id >= config['exit_min']):
if (pos != 0) and (d == end_d):
curr_pos[0].close(mslice.close - misc.sign(pos) * offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
mdf.ix[dd,
'cost'] -= abs(pos) * (offset + mslice.close * tcost)
continue
else:
if (pos != 0):
curr_pos[0].trail_update(mslice.close)
if curr_pos[0].trail_check(mslice.close, dslice.ATR * k[0]):
curr_pos[0].close(mslice.close - misc.sign(pos) * offset,
dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
pos = 0
curr_pos = []
if ((mslice.close >= xslice.H2) and
(pos < 0)) or ((mslice.close <= xslice.L2) and (pos > 0)):
curr_pos[0].close(mslice.close - misc.sign(pos) * offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
mdf.ix[dd,
'cost'] -= abs(pos) * (offset + mslice.close * tcost)
pos = 0
if ((mslice.close >= xslice.H1) and
(pos <= 0)) or ((mslice.close <= xslice.L1) and (pos >= 0)):
if (pos == 0):
target_pos = (mslice.close >= xslice.H1) * unit - (
mslice.close <= xslice.L1) * unit
new_pos = strat.TradePos([mslice.contract], [1],
target_pos, mslice.close,
mslice.close)
tradeid += 1
new_pos.entry_tradeid = tradeid
new_pos.open(mslice.close + misc.sign(target_pos) * offset,
dd)
curr_pos.append(new_pos)
mdf.ix[dd,
'cost'] -= abs(target_pos) * (offset +
mslice.close * tcost)
mdf.ix[dd, 'pos'] = pos
else:
print "something wrong with position=%s, close =%s, upBnd=%s, lowBnd=%s" % (
pos, mslice.close, xslice.H1, xslice.L1)
(res_pnl, ts) = backtest.get_pnl_stats(mdf, start_equity, marginrate, 'm')
res_trade = backtest.get_trade_stats(closed_trades)
res = dict(res_pnl.items() + res_trade.items())
return (res, closed_trades, ts)
def bband_chan_sim( mdf, config):
offset = config['offset']
exit_min = config['exit_min']
param = config['param']
bar_func = config.get('bar_conv_func', 'dh.bar_conv_func2')
bar_func = eval(bar_func)
std_func = eval(config['std_func'])
ma_func = eval(config['ma_func'])
close_daily = config['close_daily']
win = param[0]
k = param[1]
chan = param[2]
stop_ratio = config.get('exit_stop', 0.0)
k_e = k * stop_ratio
if chan > 0:
chan_func = config['chan_func']
tcost = config['trans_cost']
unit = config['unit']
freq = config['freq']
xdf = dh.conv_ohlc_freq(mdf, freq, extra_cols = ['contract'], bar_func = bar_func)
xdf['boll_ma'] = ma_func(xdf, win).shift(1)
xdf['boll_std'] = std_func(xdf, win).shift(1)
xdf['upbnd'] = xdf['boll_ma'] + xdf['boll_std'] * k
xdf['lowbnd'] = xdf['boll_ma'] - xdf['boll_std'] * k
xdf['up_exit'] = xdf['boll_ma'] + xdf['boll_std'] * k_e
xdf['dn_exit'] = xdf['boll_ma'] - xdf['boll_std'] * k_e
if chan > 0:
xdf['chan_h'] = eval(chan_func['high']['func'])(xdf, chan, **chan_func['high']['args']).shift(1)
xdf['chan_l'] = eval(chan_func['low']['func'])(xdf, chan, **chan_func['low']['args']).shift(1)
else:
xdf['chan_h'] = -10000000
xdf['chan_l'] = 10000000
xdf['high_band'] = xdf[['chan_h', 'upbnd']].max(axis=1)
xdf['low_band'] = xdf[['chan_l', 'lowbnd']].min(axis=1)
xdata = pd.concat([xdf['high_band'], xdf['low_band'],
xdf['up_exit'], xdf['dn_exit'], xdf['boll_ma']],
axis=1, keys=['high_band','low_band', 'up_exit', 'dn_exit', 'boll_ma'])
mdf = mdf.join(xdata, how = 'left').fillna(method='ffill')
# mdf['prev_close'] = mdf['close'].shift(1)
mdf['close_ind'] = np.isnan(mdf['close'].shift(-3))
if close_daily:
mdf['close_ind'] = (mdf['min_id'] >= exit_min) | mdf['close_ind']
long_signal = pd.Series(np.nan, index = mdf.index)
long_signal[(mdf['open']>=mdf['high_band']) & (mdf['boll_ma'].notnull())] = 1
long_signal[(mdf['open']<=mdf['dn_exit']) & (mdf['boll_ma'].notnull())] = 0
long_signal[mdf['close_ind']] = 0
long_signal = long_signal.fillna(method='ffill').fillna(0)
short_signal = pd.Series(np.nan, index = mdf.index)
short_signal[(mdf['open']<=mdf['low_band']) & (mdf['boll_ma'].notnull())] = -1
short_signal[(mdf['open']>=mdf['up_exit']) & (mdf['boll_ma'].notnull())] = 0
short_signal[mdf['close_ind']] = 0
short_signal = short_signal.fillna(method='ffill').fillna(0)
if len(mdf[(long_signal>0) & (short_signal<0)])>0:
print mdf[(long_signal > 0) & (short_signal < 0)]
print "something wrong with the position as long signal and short signal happen the same time"
mdf['pos'] = long_signal + short_signal
mdf['cost'] = abs(mdf['pos'] - mdf['pos'].shift(1)) * (offset + mdf['open'] * tcost)
mdf['cost'] = mdf['cost'].fillna(0.0)
mdf['traded_price'] = mdf.open + (mdf['pos'] - mdf['pos'].shift(1)) * offset
closed_trades = backtest.simdf_to_trades1(mdf, slippage = offset )
return (mdf, closed_trades)
def aberration_sim(mdf, config):
start_equity = config['capital']
marginrate = config['marginrate']
offset = config['offset']
pos_class = config['pos_class']
pos_args = config['pos_args']
param = config['param']
tick_base = config['tick_base']
close_daily = config['close_daily']
win = param[0]
k = param[1]
chan = param[2]
if chan > 0:
chan_func = config['chan_func']
tcost = config['trans_cost']
unit = config['unit']
freq = config['freq']
#freq_min = int(freq[:-3])
mode = config.get('mode', 'close')
xdf = dh.conv_ohlc_freq(mdf, freq)
boll_ma = dh.MA(xdf, win).shift(1)
boll_std = dh.STDDEV(xdf, win).shift(1)
upbnd = np.ceil((boll_ma + boll_std * k) / tick_base) * tick_base
lowbnd = np.floor((boll_ma - boll_std * k) / tick_base) * tick_base
boll_ma = np.floor(boll_ma / tick_base + 0.5) * tick_base
xdata = [boll_ma, boll_std, upbnd, lowbnd, xdf['min_id']]
xkeys = ['ma', 'stdev', 'upbnd', 'lowbnd', 'xmin_id']
if chan > 0:
chan_h = eval(chan_func['high']['func'])(xdf, chan,
**chan_func['high']['args'])
chan_l = eval(chan_func['low']['func'])(xdf, chan,
**chan_func['low']['args'])
xdata = xdata + [chan_h, chan_l]
xkeys = xkeys + ['chan_h', 'chan_l']
xdf = pd.concat(xdata, axis=1, keys=xkeys).fillna(0)
mdf = mdf.join(xdf, how='left').fillna(method='ffill')
mdf['is_new'] = (mdf['xmin_id'].shift(1) != mdf['xmin_id'])
ll = mdf.shape[0]
mdf['pos'] = pd.Series([0] * ll, index=mdf.index)
mdf['cost'] = pd.Series([0] * ll, index=mdf.index)
curr_pos = []
closed_trades = []
start_d = mdf.date[0]
end_d = mdf.date[-1]
tradeid = 0
for idx, dd in enumerate(mdf.index):
mslice = mdf.ix[dd]
min_id = mslice.min_id
d = mslice.date
if len(curr_pos) == 0:
pos = 0
start_pos = 0
else:
pos = curr_pos[0].pos
start_pos = pos
mdf.ix[dd, 'pos'] = pos
if mslice.ma == 0:
continue
upbnd = mslice.upbnd
lowbnd = mslice.lowbnd
if chan > 0:
if mslice.chan_h == 0:
continue
else:
upbnd = max(mslice.chan_h, upbnd)
lowbnd = min(mslice.chan_l, lowbnd)
if min_id >= config['exit_min']:
if (pos != 0) and ((d == end_d) or close_daily):
curr_pos[0].close(mslice.close - misc.sign(pos) * offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
mdf.ix[dd,
'cost'] -= abs(pos) * (offset + mslice.close * tcost)
pos = 0
else:
if mode == 'close':
high_trig = mslice.close
low_trig = mslice.close
trade_price = mslice.close
else:
high_trig = mslice.high
low_trig = mslice.low
trade_price = mslice.open
if ((high_trig >= mslice.ma) and
(pos < 0)) or ((low_trig <= mslice.ma) and (pos > 0)):
if pos < 0:
exec_price = max(trade_price, mslice.ma)
else:
exec_price = min(trade_price, mslice.ma)
curr_pos[0].close(exec_price - misc.sign(pos) * offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
mdf.ix[dd, 'cost'] -= abs(pos) * (offset + exec_price * tcost)
pos = 0
if ((high_trig > upbnd) and (pos <= 0)) or ((low_trig < lowbnd) and
(pos >= 0)):
target_pos = (high_trig > upbnd) * unit - (low_trig <
lowbnd) * unit
if target_pos == 0:
print "the min bar hit both up and low bounds, need to think about how to handle it", start_pos, mslice
if mslice.close > mslice.ma:
target_pos = (high_trig > upbnd) * unit
target = max(trade_price, upbnd)
else:
trade_pos = -(low_trig < lowbnd) * unit
target = min(trade_price, lowbnd)
else:
target = (high_trig > upbnd) * max(trade_price, upbnd) + (
low_trig < lowbnd) * min(trade_price, lowbnd)
new_pos = pos_class([mslice.contract], [1], target_pos, target,
target, **pos_args)
tradeid += 1
new_pos.entry_tradeid = tradeid
new_pos.open(target + misc.sign(target_pos) * offset, dd)
curr_pos.append(new_pos)
pos = target_pos
mdf.ix[dd,
'cost'] -= abs(target_pos) * (offset + target * tcost)
mdf.ix[dd, 'pos'] = pos
return (mdf, closed_trades)
def asctrend_sim(mdf, config):
offset = config['offset']
tcost = config['trans_cost']
bar_func = config.get('bar_conv_func', 'dh.bar_conv_func2')
param = config['param']
freq = config['freq']
close_daily = config['close_daily']
xdf = dh.conv_ohlc_freq(mdf,
freq,
extra_cols=['contract'],
bar_func=eval(bar_func))
wpr_period = param[0]
wpr_level = param[1]
rsi_sig = config['rsi_sig']
wpr_sig = config.get('wpr_sig', True)
wpr = dh.WPR(xdf, wpr_period)
wpr_buy = 50 + wpr_level
wpr_sell = 50 - wpr_level
wpr_signal = pd.Series(0, index=wpr.index)
if wpr_sig:
wpr_signal[dh.CROSSOVER(wpr, wpr_buy, 1)] = 1
wpr_signal[dh.CROSSOVER(wpr, wpr_sell, -1)] = -1
else:
wpr_signal[(wpr >= wpr_buy)] = 1
wpr_signal[(wpr <= wpr_sell)] = -1
xdf['wpr_signal'] = wpr_signal.shift(1).fillna(0)
rsi_period = param[2]
rsi_offset = param[3]
rsi_buy = 50 + rsi_offset
rsi_sell = 50 - rsi_offset
rsi = dh.RSI(xdf, n=rsi_period)
rsi_signal = pd.Series(0, index=rsi.index)
if rsi_sig:
rsi_signal[dh.CROSSOVER(rsi, rsi_buy, 1)] = 1
rsi_signal[dh.CROSSOVER(rsi, rsi_sell, -1)] = -1
else:
rsi_signal[(rsi >= rsi_buy)] = 1
rsi_signal[(rsi <= rsi_sell)] = -1
xdf['rsi_signal'] = rsi_signal.shift(1).fillna(0)
if len(param) > 4:
sar_step = param[4]
sar_max = param[5]
else:
sar_step = 0.005
sar_max = 0.02
sar = dh.SAR(xdf, incr=sar_step, maxaf=sar_max)
sar_signal = pd.Series(0, index=sar.index)
sar_signal[(sar < xdf['close'])] = 1
sar_signal[(sar > xdf['close'])] = -1
xdf['sar_signal'] = sar_signal.shift(1)
xdf['sar_stop'] = sar.shift(1)
xdf['close_ind'] = np.isnan(xdf['close'].shift(-1))
if close_daily:
daily_end = (xdf['date'] != xdf['date'].shift(-1))
xdf['close_ind'] = xdf['close_ind'] | daily_end
long_signal = pd.Series(np.nan, index=xdf.index)
short_signal = pd.Series(np.nan, index=xdf.index)
long_signal[(xdf['wpr_signal'] > 0) & (xdf['rsi_signal'] > 0) &
(xdf['sar_signal'] > 0)] = 1
long_signal[(xdf['wpr_signal'] < 0) | (xdf['rsi_signal'] < 0)] = 0
long_signal[xdf['close_ind']] = 0
long_signal = long_signal.fillna(method='ffill').fillna(0)
short_signal[(xdf['wpr_signal'] < 0) & (xdf['rsi_signal'] < 0) &
(xdf['sar_signal'] < 0)] = -1
short_signal[(xdf['wpr_signal'] > 0) | (xdf['rsi_signal'] > 0)] = 0
short_signal[xdf['close_ind']] = 0
short_signal = short_signal.fillna(method='ffill').fillna(0)
if len(xdf[(long_signal > 0) & (short_signal < 0)]) > 0:
print xdf[(long_signal > 0) & (short_signal < 0)]
print "something wrong with the position as long signal and short signal happen the same time"
xdf['pos'] = long_signal + short_signal
xdf['cost'] = abs(xdf['pos'] - xdf['pos'].shift(1)) * (offset +
xdf['open'] * tcost)
xdf['cost'] = xdf['cost'].fillna(0.0)
xdf['traded_price'] = xdf.open + (xdf['pos'] -
xdf['pos'].shift(1)) * offset
closed_trades = backtest.simdf_to_trades1(xdf, slippage=offset)
return (xdf, closed_trades)
def prepare_data_env(self, inst, mid_day = True):
if self.instruments[inst].ptype == instrument.ProductType.Option:
return
self.db_conn = dbaccess.connect(**dbaccess.dbconfig)
if self.daily_data_days > 0 or mid_day:
#self.logger.debug('Updating historical daily data for %s' % self.scur_day.strftime('%Y-%m-%d'))
daily_start = workdays.workday(self.scur_day, -self.daily_data_days, CHN_Holidays)
daily_end = self.scur_day
ddf = dbaccess.load_daily_data_to_df(self.db_conn, 'fut_daily', inst, daily_start, daily_end, index_col = None)
if len(ddf) > 0:
self.instruments[inst].price = self.instruments[inst].mid_price = ddf['close'].iloc[-1]
self.instruments[inst].last_update = 0
self.instruments[inst].prev_close = ddf['close'].iloc[-1]
for fobj in self.day_data_func[inst]:
ts = fobj.sfunc(ddf)
if type(ts).__name__ == 'Series':
if ts.name in ddf.columns:
self.logger.warning('TimeSeries name %s is already in the columns for inst = %s' % (ts.name, inst))
ddf[ts.name]= ts
elif type(ts).__name__ == 'DataFrame':
for col_name in ts.columns:
if col_name in ddf.columns:
self.logger.warning('TimeSeries name %s is already in the columns for inst = %s' % (col_name, inst))
ddf[col_name] = ts[col_name]
self.day_data[inst] = data_handler.DynamicRecArray(dataframe = ddf)
if self.min_data_days > 0 or mid_day:
#self.logger.debug('Updating historical min data for %s' % self.scur_day.strftime('%Y-%m-%d'))
d_start = workdays.workday(self.scur_day, -self.min_data_days, CHN_Holidays)
d_end = self.scur_day
min_start = int(self.instruments[inst].start_tick_id/1000)
min_end = int(self.instruments[inst].last_tick_id/1000)+1
mdf = dbaccess.load_min_data_to_df(self.db_conn, 'fut_min', inst, d_start, d_end, minid_start=min_start, minid_end=min_end, index_col = None)
mdf = cleanup_mindata(mdf, self.instruments[inst].product, index_col = None)
mdf['bar_id'] = self.conv_bar_id(mdf['min_id'])
if len(mdf)>0:
min_date = mdf['date'].iloc[-1]
if (len(self.day_data[inst])==0) or (min_date > self.day_data[inst].data['date'][-1]):
ddf = data_handler.conv_ohlc_freq(mdf, 'd', index_col = None)
self.cur_day[inst]['open'] = float(ddf.open[-1])
self.cur_day[inst]['close'] = float(ddf.close[-1])
self.cur_day[inst]['high'] = float(ddf.high[-1])
self.cur_day[inst]['low'] = float(ddf.low[-1])
self.cur_day[inst]['volume'] = int(ddf.volume[-1])
self.cur_day[inst]['openInterest'] = int(ddf.openInterest[-1])
self.cur_min[inst]['datetime'] = pd.datetime(*mdf['datetime'].iloc[-1].timetuple()[0:-3])
self.cur_min[inst]['date'] = mdf['date'].iloc[-1]
self.cur_min[inst]['open'] = float(mdf['open'].iloc[-1])
self.cur_min[inst]['close'] = float(mdf['close'].iloc[-1])
self.cur_min[inst]['high'] = float(mdf['high'].iloc[-1])
self.cur_min[inst]['low'] = float(mdf['low'].iloc[-1])
self.cur_min[inst]['volume'] = self.cur_day[inst]['volume']
self.cur_min[inst]['openInterest'] = self.cur_day[inst]['openInterest']
self.cur_min[inst]['min_id'] = int(mdf['min_id'].iloc[-1])
self.cur_min[inst]['bar_id'] = self.conv_bar_id(self.cur_min[inst]['min_id'])
self.instruments[inst].price = self.instruments[inst].mid_price = float(mdf['close'].iloc[-1])
self.instruments[inst].last_update = 0
#self.logger.debug('inst=%s tick data loaded for date=%s' % (inst, min_date))
if 1 not in self.min_data_func[inst]:
self.min_data[inst][1] = data_handler.DynamicRecArray(dataframe = mdf)
for m in sorted(self.min_data_func[inst]):
if m != 1:
mdf_m = data_handler.conv_ohlc_freq(mdf, str(m)+'min', index_col = None, bar_func = self.conv_bar_id, extra_cols = ['bar_id'])
else:
mdf_m = mdf
for fobj in self.min_data_func[inst][m]:
ts = fobj.sfunc(mdf_m)
if type(ts).__name__ == 'Series':
if ts.name in mdf_m.columns:
self.logger.warning('TimeSeries name %s is already in the columns for inst = %s' % (ts.name, inst))
mdf_m[ts.name]= ts
elif type(ts).__name__ == 'DataFrame':
for col_name in ts.columns:
if col_name in mdf_m.columns:
self.logger.warning('TimeSeries name %s is already in the columns for inst = %s' % (col_name, inst))
mdf_m[col_name] = ts[col_name]
self.min_data[inst][m] = data_handler.DynamicRecArray(dataframe = mdf_m)
#print inst, self.min_data[inst][m].data['date'][-1] < self.cur_min[inst]['date']
self.db_conn.close()
def fisher_swing_sim(mdf, config):
pos_class = config['pos_class']
pos_args = config['pos_args']
pos_update = config.get('pos_update', False)
offset = config['offset']
close_daily = config['close_daily']
tcost = config['trans_cost']
unit = config['unit']
freq = config['freq']
param = config['param']
fisher_n = param[0]
bband_n = param[1]
bband_std = param[2]
heiken_n = param[3]
xdf = dh.conv_ohlc_freq(mdf, freq, extra_cols=['contract'])
fisher = dh.FISHER(xdf, win[0])
xdf['FISHER_I'] = fisher['FISHER_I'].shift(1)
bbands_stop = dh.BBANDS_STOP(xdf, win[1], 1.0)
xdf['BBSTOP_upper'] = bbands_stop['BBSTOP_upper'].shift(1)
xdf['BBSTOP_lower'] = bbands_stop['BBSTOP_lower'].shift(1)
xdf['BBSTOP_trend'] = bbands_stop['BBSTOP_trend'].shift(1)
ha_df = dh.HEIKEN_ASHI(xdf, win[2]).shift(1)
xdf['HAopen'] = ha_df['HAopen'].shift(1)
xdf['HAclose'] = ha_df['HAclose'].shift(1)
xdf['prev_close'] = xdf['close'].shift(1)
xdf['close_ind'] = np.isnan(xdf['close'].shift(-1))
if close_daily:
daily_end = (xdf['date'] != xdf['date'].shift(-1))
xdf['close_ind'] = xdf['close_ind'] | daily_end
xdf['pos'] = 0
xdf['cost'] = 0
xdf['traded_price'] = xdf.open
curr_pos = []
closed_trades = []
tradeid = 0
for idx, dd in enumerate(xdf.index):
mslice = xdf.loc[dd]
min_id = mslice.min_id
d = dd.date()
if len(curr_pos) == 0:
pos = 0
else:
pos = curr_pos[0].pos
xdf.set_value(dd, 'pos', pos)
if np.isnan(mslice.BBSTOP_lower) or np.isnan(
mslice.FISHER_I) or np.isnan(mslice.HAclose):
continue
stop_loss = (pos > 0) and ((mslice.open < mslice.BBSTOP_lower) or
(mslice.FISHER_I < 0))
stop_loss = stop_loss or ((pos < 0) and
((mslice.open > mslice.BBSTOP_upper) or
(mslice.FISHER_I > 0)))
start_long = (mslice.FISHER_I > 0) and (
mslice.HAclose > mslice.HAopen) and (mslice.BBSTOP_trend > 0)
start_short = (mslice.FISHER_I < 0) and (
mslice.HAclose < mslice.HAopen) and (mslice.BBSTOP_trend < 0)
if mslice.close_ind:
if pos != 0:
curr_pos[0].close(mslice.open - misc.sign(pos) * offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
xdf.set_value(
dd, 'cost',
xdf.at[dd, 'cost'] - abs(pos) * (mslice.open * tcost))
xdf.set_value(dd, 'traded_price',
mslice.open - misc.sign(pos) * offset)
pos = 0
else:
if stop_loss:
curr_pos[0].close(mslice.open - misc.sign(pos) * offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
xdf.set_value(
dd, 'cost',
xdf.at[dd, 'cost'] - abs(pos) * (mslice.open * tcost))
xdf.set_value(dd, 'traded_price',
mslice.open - misc.sign(pos) * offset)
pos = 0
pos = (start_long == True) * unit - (start_short == True) * unit
if abs(pos) > 0:
#target = (start_long == True) * mslice.close +(start_short == True) * mslice.close
new_pos = pos_class([mslice.contract], [1], pos, mslice.open,
mslice.open, **pos_args)
tradeid += 1
new_pos.entry_tradeid = tradeid
new_pos.open(mslice.open + misc.sign(pos) * offset, dd)
curr_pos.append(new_pos)
xdf.set_value(
dd, 'cost',
xdf.at[dd, 'cost'] - abs(pos) * (mslice.open * tcost))
xdf.set_value(dd, 'traded_price',
mslice.open + misc.sign(pos) * offset)
xdf.ix[dd, 'pos'] = pos
return (xdf, closed_trades)
def MA_sim(mdf, config):
offset = config['offset']
use_chan = config['use_chan']
pos_class = config['pos_class']
pos_args = config['pos_args']
pos_update = config.get('pos_update', False)
stoploss = config.get('stoploss', 0.0)
win_list = config['param']
MA_func = eval(config['MA_func'])
close_daily = config['close_daily']
tcost = config['trans_cost']
unit = config['unit']
freq = config['freq']
chan_ratio = config['channel_ratio']
channel = int(chan_ratio * win_list[-1])
xdf = dh.conv_ohlc_freq(mdf, freq, extra_cols=['contract'])
for idx, win in enumerate(win_list):
xdf['MA' + str(idx)] = MA_func(xdf, win).shift(1)
if use_chan:
xdf['chan_high'] = eval(config['channel_func'][0])(
xdf, channel, **config['channel_args'][0]).shift(2)
xdf['chan_low'] = eval(config['channel_func'][1])(
xdf, channel, **config['channel_args'][1]).shift(2)
else:
xdf['chan_high'] = pd.Series(index=xdf.index)
xdf['chan_low'] = pd.Series(index=xdf.index)
tot_MA = len(win_list)
xdf['prev_close'] = xdf['close'].shift(1)
xdf['close_ind'] = np.isnan(xdf['close'].shift(-1))
if close_daily:
daily_end = (xdf['date'] != xdf['date'].shift(-1))
xdf['close_ind'] = xdf['close_ind'] | daily_end
ll = xdf.shape[0]
xdf['pos'] = 0
xdf['cost'] = 0
xdf['traded_price'] = xdf.open
curr_pos = []
closed_trades = []
tradeid = 0
for idx, dd in enumerate(xdf.index):
mslice = xdf.loc[dd]
ma_list = [
getattr(mslice, 'MA' + str(i)) for i in range(len(win_list))
]
ma_short = ma_list[0]
ma_last = ma_list[-1]
if len(curr_pos) == 0:
pos = 0
else:
pos = curr_pos[0].pos
xdf.set_value(dd, 'pos', pos)
if np.isnan(getattr(mslice, "MA" + str(tot_MA - 1))):
continue
if mslice.close_ind:
if pos != 0:
curr_pos[0].close(mslice.open - misc.sign(pos) * offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
xdf.set_value(
dd, 'cost',
xdf.at[dd, 'cost'] - abs(pos) * (mslice.open * tcost))
xdf.set_value(dd, 'traded_price',
mslice.open - misc.sign(pos) * offset)
pos = 0
else:
if (((ma_short >= ma_last) or
(mslice.open >= mslice.chan_high)) and
(pos < 0)) or (((ma_short <= ma_last) or
(mslice.open <= mslice.chan_low)) and
(pos > 0)):
curr_pos[0].close(mslice.open - misc.sign(pos) * offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
xdf.set_value(
dd, 'cost',
xdf.at[dd, 'cost'] - abs(pos) * (mslice.open * tcost))
xdf.set_value(dd, 'traded_price',
mslice.open - misc.sign(pos) * offset)
pos = 0
if (((ma_short >= ma_list[1]) and (ma_short >= ma_last) and ((use_chan == False) or (mslice.open>=mslice.chan_high))) or \
((ma_short <= ma_list[1]) and (ma_short <= ma_last) and ((use_chan == False) or (mslice.open<=mslice.chan_low)))) and (pos==0):
target_pos = ((ma_short >= ma_list[1]) and (ma_short >= ma_last) and ((use_chan == False) or (mslice.open>=mslice.chan_high))) * unit \
- ((ma_short <= ma_list[1]) and (ma_short <= ma_last) and ((use_chan == False) or (mslice.open<=mslice.chan_low))) * unit
new_pos = pos_class([mslice.contract], [1], target_pos,
mslice.open, mslice.open, **pos_args)
tradeid += 1
new_pos.entry_tradeid = tradeid
new_pos.open(mslice.open + misc.sign(target_pos) * offset, dd)
curr_pos.append(new_pos)
pos = target_pos
xdf.set_value(
dd, 'cost', xdf.at[dd, 'cost'] - abs(target_pos) *
(mslice.open * tcost))
xdf.set_value(dd, 'traded_price',
mslice.open + misc.sign(target_pos) * offset)
xdf.set_value(dd, 'pos', pos)
return (xdf, closed_trades)
def MA_sim(mdf, config):
offset = config['offset']
pos_class = config['pos_class']
pos_args = config['pos_args']
pos_update = config.get('pos_update', False)
stoploss = config.get('stoploss', 0.0)
ma_list = config['ma_list']
param = config['param']
corr_entry = param[0]
corr_exit = param[1]
pval_entry = param[2]
pval_exit = param[3]
if len(param) >= 5:
channel = param[4]
else:
channel = 0
if channel == 0:
use_chan = False
else:
use_chan = True
close_daily = config['close_daily']
tcost = config['trans_cost']
unit = config['unit']
freq = config['freq']
if int(freq[:-3]) == 1:
xdf = mdf
else:
xdf = dh.conv_ohlc_freq(mdf, freq, extra_cols=['contract'])
if use_chan:
xdf['chan_high'] = eval(config['channel_func'][0])(
xdf, channel, **config['channel_args'][0]).shift(2)
xdf['chan_low'] = eval(config['channel_func'][1])(
xdf, channel, **config['channel_args'][1]).shift(2)
else:
xdf['chan_high'] = pd.Series(index=xdf.index)
xdf['chan_low'] = pd.Series(index=xdf.index)
ma_ribbon = dh.MA_RIBBON(xdf, ma_list)
xdf["ribbon_corr"] = ma_ribbon["MARIBBON_CORR"].shift(1)
xdf["ribbon_pval"] = ma_ribbon["MARIBBON_PVAL"].shift(1)
xdf["ribbon_dist"] = ma_ribbon["MARIBBON_DIST"].shift(1)
xdf['prev_close'] = xdf['close'].shift(1)
xdf['close_ind'] = np.isnan(xdf['close'].shift(-1))
if close_daily:
daily_end = (xdf['date'] != xdf['date'].shift(-1))
xdf['close_ind'] = xdf['close_ind'] | daily_end
ll = xdf.shape[0]
xdf['pos'] = 0
xdf['cost'] = 0
xdf['traded_price'] = xdf.open
curr_pos = []
closed_trades = []
tradeid = 0
for idx, dd in enumerate(xdf.index):
mslice = xdf.loc[dd]
if len(curr_pos) == 0:
pos = 0
else:
pos = curr_pos[0].pos
xdf.set_value(dd, 'pos', pos)
if np.isnan(mslice.ribbon_corr):
continue
if mslice.close_ind:
if pos != 0:
curr_pos[0].close(mslice.open - misc.sign(pos) * offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
xdf.set_value(
dd, 'cost',
xdf.at[dd, 'cost'] - abs(pos) * (mslice.open * tcost))
xdf.set_value(dd, 'traded_price',
mslice.open - misc.sign(pos) * offset)
pos = 0
else:
if (((mslice.ribbon_corr >= -corr_exit) or (mslice.ribbon_pval >= pval_exit)) and (pos<0)) or \
(((mslice.ribbon_corr <= corr_exit) or (mslice.ribbon_pval >= pval_exit)) and (pos>0)):
curr_pos[0].close(mslice.open - misc.sign(pos) * offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
xdf.set_value(
dd, 'cost',
xdf.at[dd, 'cost'] - abs(pos) * (mslice.open * tcost))
xdf.set_value(dd, 'traded_price',
mslice.open - misc.sign(pos) * offset)
pos = 0
if (pos==0) and (mslice.ribbon_corr >= corr_entry) and (mslice.ribbon_pval < pval_entry) \
and ((use_chan == False) or (mslice.open >= mslice.chan_high)):
target_pos = unit
elif (pos==0) and (mslice.ribbon_corr <= -corr_entry) and (mslice.ribbon_pval < pval_entry) \
and ((use_chan == False) or (mslice.open <= mslice.chan_low)):
target_pos = -unit
else:
target_pos = 0
if target_pos != 0:
new_pos = pos_class([mslice.contract], [1], target_pos,
mslice.open, mslice.open, **pos_args)
tradeid += 1
new_pos.entry_tradeid = tradeid
new_pos.open(mslice.open + misc.sign(target_pos) * offset, dd)
curr_pos.append(new_pos)
pos = target_pos
xdf.set_value(
dd, 'cost', xdf.at[dd, 'cost'] - abs(target_pos) *
(mslice.open * tcost))
xdf.set_value(dd, 'traded_price',
mslice.open + misc.sign(target_pos) * offset)
xdf.set_value(dd, 'pos', pos)
return (xdf, closed_trades)
def rbreaker_sim( mdf, config):
if 'ddf' in config:
ddf = config['ddf']
else:
freq = config['freq']
ddf = dh.conv_ohlc_freq(mdf, freq)
start_equity = config['capital']
tcost = config['trans_cost']
unit = config['unit']
pos_class = config['pos_class']
pos_args = config['pos_args']
close_daily = config['close_daily']
marginrate = config['marginrate']
offset = config['offset']
min_rng = config['min_rng']
k = config['params']
a = k[0]
b = k[1]
c = k[2]
ddf['range'] = ddf.high - ddf.low
ddf['ssetup'] = ddf.high + a*(ddf.close - ddf.low)
ddf['bsetup'] = ddf.low - a*(ddf.high - ddf.close)
ddf['senter'] = (1+b)*(ddf.high+ddf.close)/2.0 - b * ddf.low
ddf['benter'] = (1+b)*(ddf.low+ddf.close)/2.0 - b * ddf.high
ddf['bbreak'] = ddf.ssetup + c * (ddf.ssetup - ddf.bsetup)
ddf['sbreak'] = ddf.bsetup - c * (ddf.ssetup - ddf.bsetup)
ddf = ddf.shift(1)
ll = mdf.shape[0]
mdf['pos'] = pd.Series([0]*ll, index = mdf.index)
mdf['cost'] = pd.Series([0]*ll, index = mdf.index)
curr_pos = []
closed_trades = []
start_d = mdf.index[0].date()
end_d = mdf.index[-1].date()
prev_d = start_d - datetime.timedelta(days=1)
tradeid = 0
cur_high = 0
cur_low = 0
rev_flag = True
dslice = None
num_trades = 0
for dd in mdf.index:
mslice = mdf.ix[dd]
min_id = mslice.min_id
d = mslice.date
if (prev_d < d):
num_trades = 0
cur_high = mslice.high
cur_low = mslice.low
if d not in ddf.index:
print d, dd
continue
else:
dslice = ddf.ix[d]
if np.isnan(dslice.bbreak):
continue
if dslice.range < min_rng * dslice.close:
rev_flag = False
else:
rev_flag = True
prev_d = d
else:
cur_high = max(cur_high, mslice.high)
cur_low = min(cur_low, mslice.low)
if len(curr_pos) == 0:
pos = 0
else:
pos = curr_pos[0].pos
mdf.ix[dd, 'pos'] = pos
if (min_id <= config['start_min']):
continue
if (min_id >= config['exit_min']):
if (pos != 0) and (close_daily or (d == end_d)):
curr_pos[0].close(mslice.close - misc.sign(pos) * offset , dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
mdf.ix[dd, 'cost'] -= abs(pos) * (offset + mslice.close*tcost)
pos = 0
else:
if num_trades >=3:
continue
if ((mslice.high >= dslice.bbreak) and (pos <= 0)) or ((mslice.low <= dslice.sbreak) and (pos >= 0)):
rev_flag = False
if (mslice.close >= dslice.bbreak):
direction = 1
else:
direction = -1
if pos != 0:
curr_pos[0].close(mslice.close + direction*offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
mdf.ix[dd, 'cost'] -= abs(pos) * (offset + mslice.close*tcost)
new_pos = pos_class([mslice.contract], [1], unit*direction, mslice.close, 0, **pos_args)
tradeid += 1
new_pos.entry_tradeid = tradeid
new_pos.open(mslice.close + offset*direction, dd)
curr_pos.append(new_pos)
pos = unit*direction
mdf.ix[dd, 'cost'] -= abs(direction) * (offset + mslice.close*tcost)
num_trades += 1
elif rev_flag and (((cur_high < dslice.bbreak) and (cur_high >= dslice.ssetup) and (mslice.close <= dslice.senter) and (pos >=0)) or \
((cur_low > dslice.sbreak) and (cur_low <= dslice.bsetup) and (mslice.close >= dslice.benter) and (pos <=0))):
if len(curr_pos) > 0:
curr_pos[0].close(mslice.close-misc.sign(pos)*offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
mdf.ix[dd, 'cost'] -= abs(pos) * (offset + mslice.close*tcost)
new_pos = pos_class([mslice.contract], [1], -unit*misc.sign(pos), mslice.close, 0, **pos_args)
tradeid += 1
new_pos.entry_tradeid = tradeid
new_pos.open(mslice.close + offset*misc.sign(pos), dd)
curr_pos.append(new_pos)
pos = -unit*misc.sign(pos)
mdf.ix[dd, 'cost'] -= abs(pos) * (offset + mslice.close*tcost)
num_trades += 1
mdf.ix[dd, 'pos'] = pos
return (mdf, closed_trades)
def chanbreak_sim(mdf, config):
freq = config['freq']
str_freq = str(freq) + 'Min'
xdf = dh.conv_ohlc_freq(mdf, str_freq)
tcost = config['trans_cost']
unit = config['unit']
offset = config['offset']
win = config['win']
pos_class = config['pos_class']
pos_args = config['pos_args']
chan_func = config['channel_func']
upper_chan_func = chan_func[0]
lower_chan_func = chan_func[1]
entry_chan = win[0]
exit_chan = win[1]
xdf['H1'] = upper_chan_func(xdf, entry_chan)
xdf['L1'] = lower_chan_func(xdf, entry_chan)
xdf['H2'] = upper_chan_func(xdf, exit_chan)
xdf['L2'] = lower_chan_func(xdf, exit_chan)
xdf['ATR'] = dh.ATR(xdf, entry_chan)
xdata = pd.concat([xdf['H1'], xdf['L1'], xdf['H2'], xdf['L2'], xdf['ATR'], xdf['high'], xdf['low']], \
axis=1, keys=['H1', 'L1', 'H2', 'L2', 'ATR', 'xhigh', 'xlow']).shift(1)
ll = mdf.shape[0]
mdf = mdf.join(xdata, how='left').fillna(method='ffill')
mdf['pos'] = pd.Series([0] * ll, index=mdf.index)
mdf['cost'] = pd.Series([0] * ll, index=mdf.index)
curr_pos = []
closed_trades = []
end_d = mdf.index[-1].date()
tradeid = 0
for idx, dd in enumerate(mdf.index):
mslice = mdf.ix[dd]
min_id = mslice.min_id
cnt_id = count_min_id(dd)
if len(curr_pos) == 0:
pos = 0
else:
pos = curr_pos[0].pos
mdf.ix[dd, 'pos'] = pos
#if np.isnan(mslice.ATR):
# continue
if (min_id >= config['exit_min']):
if (pos != 0) and (dd.date() == end_d):
curr_pos[0].close(mslice.close - misc.sign(pos) * offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
pos = 0
mdf.ix[dd,
'cost'] -= abs(pos) * (offset + mslice.close * tcost)
continue
else:
if (pos != 0):
if (cnt_id % freq) == 0:
curr_pos[0].update_bar(mslice)
check_price = (pos > 0) * mslice.low + (pos < 0) * mslice.high
if curr_pos[0].check_exit(check_price, 0):
curr_pos[0].close(mslice.close - misc.sign(pos) * offset,
dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
pos = 0
curr_pos = []
if ((mslice.high >= mslice.H2) and
(pos < 0)) or ((mslice.low <= mslice.L2) and (pos > 0)):
curr_pos[0].close(mslice.close - misc.sign(pos) * offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
mdf.ix[dd,
'cost'] -= abs(pos) * (offset + mslice.close * tcost)
pos = 0
if ((mslice.high >= mslice.H1) and
(pos <= 0)) or ((mslice.low <= mslice.L1) and (pos >= 0)):
if (pos == 0):
pos = (mslice.high >=
mslice.H1) * unit - (mslice.low <= mslice.L1) * unit
exit_target = (mslice.high >= mslice.H1) * mslice.L2 + (
mslice.low <= mslice.L1) * mslice.H2
new_pos = pos_class([mslice.contract], [1], pos,
mslice.close, exit_target, 1,
**pos_args)
tradeid += 1
new_pos.entry_tradeid = tradeid
new_pos.open(mslice.close + misc.sign(pos) * offset, dd)
curr_pos.append(new_pos)
mdf.ix[dd, 'cost'] -= abs(pos) * (offset +
mslice.close * tcost)
mdf.ix[dd, 'pos'] = pos
return (mdf, closed_trades)
def layer_sim(mdf, config):
tcost = config['trans_cost']
unit = config['unit']
offset = config['offset']
pos_class = config['pos_class']
pos_args = config['pos_args']
freq = config['freq']
signal_func = config['signal_func']
signal_args = config['signal_args']
signal_level = config['signal_level']
close_daily = config['close_daily']
for idx, (f, sfunc, sargs, slvl) in enumerate(
zip(freq, signal_func, signal_args, signal_level)):
df = dh.conv_ohlc_freq(mdf, f, extra_cols=['contract'])
if idx == 0:
xdf = df
farg = tuple(sargs)
func = eval(sfunc[0])
sigout = func(df, *farg)
sig_field = sfunc[1] % farg
signal = sigout[sig_field]
long_lvl = slvl[0] + slvl[1]
short_lvl = slvl[0] - slvl[1]
if idx > 0:
long_ind = (signal > long_lvl)
short_ind = (signal < short_lvl)
else:
long_ind = (signal > long_lvl) & (signal.shift(1) <= long_lvl)
short_ind = (signal < short_lvl) & (signal.shift(1) >= short_lvl)
xdata = pd.concat([signal, long_ind, short_ind],
axis=1,
keys=[
'signal' + str(idx), 'long_ind' + str(idx),
'short_ind' + str(idx)
]).shift(1)
#print xdata
xdf = xdf.join(xdata, how='left').fillna(method='ffill')
xdf['close_ind'] = np.isnan(xdf['close'].shift(-1))
if close_daily:
daily_end = (xdf['date'] != xdf['date'].shift(-1))
xdf['close_ind'] = xdf['close_ind'] | daily_end
xdf['pos'] = 0
xdf['cost'] = 0
xdf['traded_price'] = xdf.open
curr_pos = []
closed_trades = []
tradeid = 0
for idx, dd in enumerate(xdf.index):
mslice = xdf.ix[dd]
if len(curr_pos) == 0:
pos = 0
else:
pos = curr_pos[0].pos
signal_indlist = [
np.isnan(getattr(mslice, 'signal' + str(i)))
for i in range(len(freq))
]
long_indlist = [
getattr(mslice, 'long_ind' + str(i)) for i in range(len(freq))
]
short_indlist = [
getattr(mslice, 'short_ind' + str(i)) for i in range(len(freq))
]
if True in signal_indlist:
continue
if mslice.close_ind:
if pos != 0:
curr_pos[0].close(mslice.open - misc.sign(pos) * offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
xdf.set_value(
dd, 'cost',
xdf.at[dd, 'cost'] - abs(pos) * (mslice.open * tcost))
xdf.set_value(dd, 'traded_price',
mslice.open - misc.sign(pos) * offset)
pos = 0
else:
if (pos != 0):
direction = 0
long_ind = (True in long_indlist)
short_ind = (True in short_indlist)
if ((pos < 0) and long_ind) or ((pos > 0) and short_ind):
curr_pos[0].close(mslice.open - misc.sign(pos) * offset,
dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
xdf.set_value(
dd, 'cost',
xdf.at[dd, 'cost'] - abs(pos) * (mslice.open * tcost))
xdf.set_value(dd, 'traded_price',
mslice.open - misc.sign(pos) * offset)
pos = 0
long_ind = (False not in long_indlist)
short_ind = (False not in short_indlist)
if (long_ind or short_ind) and (pos == 0):
target_pos = long_ind * unit - short_ind * unit
new_pos = pos_class([mslice.contract], [1], target_pos,
mslice.open, mslice.open, **pos_args)
tradeid += 1
new_pos.entry_tradeid = tradeid
new_pos.open(mslice.open + misc.sign(target_pos) * offset, dd)
curr_pos.append(new_pos)
#print dd, target_pos, len(curr_pos), curr_pos[0].pos, curr_pos #[getattr(mslice, 'long_ind' + str(i)) for i in range(len(freq))]
pos = target_pos
xdf.set_value(
dd, 'cost', xdf.at[dd, 'cost'] - abs(target_pos) *
(mslice.open * tcost))
xdf.set_value(dd, 'traded_price',
mslice.open + misc.sign(target_pos) * offset)
xdf.set_value(dd, 'pos', pos)
return (xdf, closed_trades)
def bband_chan_sim(mdf, config):
offset = config['offset']
pos_class = config['pos_class']
pos_args = config['pos_args']
pos_update = config.get('pos_update', False)
stoploss = config.get('stoploss', 0.0)
param = config['param']
bar_func = config.get('bar_conv_func', 'dh.bar_conv_func2')
bar_func = eval(bar_func)
std_func = eval(config['std_func'])
ma_func = eval(config['ma_func'])
#tick_base = config['tick_base']
close_daily = config['close_daily']
win = param[0]
k = param[1]
chan = param[2]
stop_ratio = config.get('exit_stop', 0.0)
k_e = k * stop_ratio
if chan > 0:
chan_func = config['chan_func']
tcost = config['trans_cost']
unit = config['unit']
freq = config['freq']
xdf = dh.conv_ohlc_freq(mdf,
freq,
extra_cols=['contract'],
bar_func=bar_func)
xdf['boll_ma'] = ma_func(xdf, win).shift(1)
xdf['boll_std'] = std_func(xdf, win).shift(1)
xdf['upbnd'] = xdf['boll_ma'] + xdf['boll_std'] * k
xdf['lowbnd'] = xdf['boll_ma'] - xdf['boll_std'] * k
xdf['up_exit'] = xdf['boll_ma'] + xdf['boll_std'] * k_e
xdf['dn_exit'] = xdf['boll_ma'] - xdf['boll_std'] * k_e
if chan > 0:
xdf['chan_h'] = eval(chan_func['high']['func'])(
xdf, chan, **chan_func['high']['args']).shift(2)
xdf['chan_l'] = eval(chan_func['low']['func'])(
xdf, chan, **chan_func['low']['args']).shift(2)
else:
xdf['chan_h'] = -10000000
xdf['chan_l'] = 10000000
xdf['high_band'] = xdf[['chan_h', 'upbnd']].max(axis=1)
xdf['low_band'] = xdf[['chan_l', 'lowbnd']].min(axis=1)
xdf['prev_close'] = xdf['close'].shift(1)
xdf['close_ind'] = np.isnan(xdf['close'].shift(-1))
if close_daily:
daily_end = (xdf['date'] != xdf['date'].shift(-1))
xdf['close_ind'] = xdf['close_ind'] | daily_end
long_signal = pd.Series(np.nan, index=xdf.index)
long_signal[(xdf['open'] >= xdf['high_band'])
& (xdf['boll_ma'].notnull())] = 1
long_signal[(xdf['open'] <= xdf['dn_exit'])
& (xdf['boll_ma'].notnull())] = 0
long_signal[xdf['close_ind']] = 0
long_signal = long_signal.fillna(method='ffill').fillna(0)
short_signal = pd.Series(np.nan, index=xdf.index)
short_signal[(xdf['open'] <= xdf['low_band'])
& (xdf['boll_ma'].notnull())] = -1
short_signal[(xdf['open'] >= xdf['up_exit'])
& (xdf['boll_ma'].notnull())] = 0
short_signal[xdf['close_ind']] = 0
short_signal = short_signal.fillna(method='ffill').fillna(0)
if len(xdf[(long_signal > 0) & (short_signal < 0)]) > 0:
print xdf[(long_signal > 0) & (short_signal < 0)]
print "something wrong with the position as long signal and short signal happen the same time"
xdf['pos'] = long_signal + short_signal
xdf['cost'] = abs(xdf['pos'] - xdf['pos'].shift(1)) * (offset +
xdf['open'] * tcost)
xdf['cost'] = xdf['cost'].fillna(0.0)
xdf['traded_price'] = xdf.open + (xdf['pos'] -
xdf['pos'].shift(1)) * offset
closed_trades = backtest.simdf_to_trades1(xdf, slippage=offset)
return (xdf, closed_trades)
def chanbreak_sim( mdf, config):
freq = config['freq']
str_freq = str(freq) + 'Min'
xdf = dh.conv_ohlc_freq(mdf, str_freq)
start_equity = config['capital']
tcost = config['trans_cost']
unit = config['unit']
#k = config['scaler']
marginrate = config['marginrate']
offset = config['offset']
win = config['win']
pos_class = config['pos_class']
pos_args = config['pos_args']
chan_func = config['channel_func']
upper_chan_func = chan_func[0]
lower_chan_func = chan_func[1]
entry_chan = win[0]
exit_chan = win[1]
xdf['H1'] = upper_chan_func(xdf, entry_chan)
xdf['L1'] = lower_chan_func(xdf, entry_chan)
xdf['H2'] = upper_chan_func(xdf, exit_chan)
xdf['L2'] = lower_chan_func(xdf, exit_chan)
xdf['ATR'] = dh.ATR(xdf, entry_chan)
xdata = pd.concat([xdf['H1'], xdf['L1'], xdf['H2'], xdf['L2'], xdf['ATR'], xdf['high'], xdf['low']], \
axis=1, keys=['H1', 'L1', 'H2', 'L2', 'ATR', 'xhigh', 'xlow']).shift(1)
ll = mdf.shape[0]
mdf = mdf.join(xdata, how = 'left').fillna(method='ffill')
mdf['pos'] = pd.Series([0]*ll, index = mdf.index)
mdf['cost'] = pd.Series([0]*ll, index = mdf.index)
curr_pos = []
closed_trades = []
end_d = mdf.index[-1].date()
tradeid = 0
x_idx = 0
max_idx = len(xdf.index)
for idx, dd in enumerate(mdf.index):
mslice = mdf.ix[dd]
min_id = mslice.min_id
cnt_id = count_min_id(dd)
if len(curr_pos) == 0:
pos = 0
else:
pos = curr_pos[0].pos
mdf.ix[dd, 'pos'] = pos
#if np.isnan(mslice.ATR):
# continue
if (min_id >=config['exit_min']):
if (pos!=0) and (dd.date() == end_d):
curr_pos[0].close(mslice.close - misc.sign(pos) * offset , dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
pos = 0
mdf.ix[dd, 'cost'] -= abs(pos) * (offset + mslice.close*tcost)
continue
else:
if (pos !=0):
if (cnt_id % freq) == 0:
curr_pos[0].update_bar(mslice)
check_price = (pos>0) * mslice.low + (pos<0) * mslice.high
if curr_pos[0].check_exit(check_price, 0):
curr_pos[0].close(mslice.close - misc.sign(pos) * offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
pos = 0
curr_pos = []
if ((mslice.high >= mslice.H2) and (pos<0)) or ((mslice.low <= mslice.L2) and (pos>0)):
curr_pos[0].close(mslice.close - misc.sign(pos) * offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
mdf.ix[dd, 'cost'] -= abs(pos) * (offset + mslice.close*tcost)
pos = 0
if ((mslice.high >= mslice.H1) and (pos<=0)) or ((mslice.low <= mslice.L1) and (pos>=0)):
if (pos ==0 ):
pos = (mslice.high >= mslice.H1) * unit -(mslice.low <= mslice.L1) * unit
exit_target = (mslice.high >= mslice.H1) * mslice.L2 + (mslice.low <= mslice.L1) * mslice.H2
new_pos = pos_class([mslice.contract], [1], pos, mslice.close, exit_target, 1, **pos_args)
tradeid += 1
new_pos.entry_tradeid = tradeid
new_pos.open(mslice.close + misc.sign(pos)*offset, dd)
curr_pos.append(new_pos)
mdf.ix[dd, 'cost'] -= abs(pos) * (offset + mslice.close*tcost)
mdf.ix[dd, 'pos'] = pos
(res_pnl, ts) = backtest.get_pnl_stats( mdf, start_equity, marginrate, 'm')
res_trade = backtest.get_trade_stats( closed_trades )
res = dict( res_pnl.items() + res_trade.items())
return (res, closed_trades, ts)
def bband_chan_sim(mdf, config):
offset = config['offset']
pos_class = config['pos_class']
pos_args = config['pos_args']
pos_update = config.get('pos_update', False)
stoploss = config.get('stoploss', 0.0)
param = config['param']
bar_func = eval(config['bar_conv_func'])
std_func = eval(config['std_func'])
#tick_base = config['tick_base']
close_daily = config['close_daily']
win = param[0]
k = param[1]
chan = param[2]
stop_ratio = config.get('exit_stop', 0.0)
k_e = k * stop_ratio
if chan > 0:
chan_func = config['chan_func']
tcost = config['trans_cost']
unit = config['unit']
freq = config['freq']
xdf = dh.conv_ohlc_freq(mdf,
freq,
extra_cols=['contract'],
bar_func=bar_func)
xdf['boll_ma'] = dh.MA(xdf, win).shift(1)
xdf['boll_std'] = std_func(xdf, win).shift(1)
xdf['upbnd'] = xdf['boll_ma'] + xdf['boll_std'] * k
xdf['lowbnd'] = xdf['boll_ma'] - xdf['boll_std'] * k
xdf['up_exit'] = xdf['boll_ma'] + xdf['boll_std'] * k_e
xdf['dn_exit'] = xdf['boll_ma'] - xdf['boll_std'] * k_e
if chan > 0:
xdf['chan_h'] = eval(chan_func['high']['func'])(
xdf, chan, **chan_func['high']['args']).shift(2)
xdf['chan_l'] = eval(chan_func['low']['func'])(
xdf, chan, **chan_func['low']['args']).shift(2)
else:
xdf['chan_h'] = 0
xdf['chan_l'] = 10000000
xdf['high_band'] = xdf[['chan_h', 'upbnd']].max(axis=1)
xdf['low_band'] = xdf[['chan_l', 'lowbnd']].min(axis=1)
xdf['prev_close'] = xdf['close'].shift(1)
xdf['close_ind'] = np.isnan(xdf['close'].shift(-1))
if close_daily:
daily_end = (xdf['date'] != xdf['date'].shift(-1))
xdf['close_ind'] = xdf['close_ind'] | daily_end
ll = xdf.shape[0]
xdf['pos'] = 0
xdf['cost'] = 0
xdf['traded_price'] = xdf.open
curr_pos = []
closed_trades = []
tradeid = 0
for idx, dd in enumerate(xdf.index):
mslice = xdf.loc[dd]
if len(curr_pos) == 0:
pos = 0
else:
pos = curr_pos[0].pos
xdf.set_value(dd, 'pos', pos)
if np.isnan(mslice.boll_ma) or np.isnan(mslice.chan_h):
continue
if mslice.close_ind:
if pos != 0:
curr_pos[0].close(mslice.open - misc.sign(pos) * offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
xdf.set_value(
dd, 'cost',
xdf.at[dd, 'cost'] - abs(pos) * (mslice.open * tcost))
xdf.set_value(dd, 'traded_price',
mslice.open - misc.sign(pos) * offset)
pos = 0
else:
if ((mslice.open > mslice.up_exit) and
(pos < 0)) or ((mslice.open < mslice.dn_exit) and (pos > 0)):
curr_pos[0].close(mslice.open - misc.sign(pos) * offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
xdf.set_value(
dd, 'cost',
xdf.at[dd, 'cost'] - abs(pos) * (mslice.open * tcost))
xdf.set_value(dd, 'traded_price',
mslice.open - misc.sign(pos) * offset)
pos = 0
if ((mslice.open >= mslice.high_band) or
(mslice.open <= mslice.low_band)) and (pos == 0):
target_pos = (mslice.open >= mslice.high_band) * unit - (
mslice.open <= mslice.low_band) * unit
new_pos = pos_class([mslice.contract], [1], target_pos,
mslice.open, mslice.open, **pos_args)
tradeid += 1
new_pos.entry_tradeid = tradeid
new_pos.open(mslice.open + misc.sign(target_pos) * offset, dd)
curr_pos.append(new_pos)
pos = target_pos
xdf.set_value(
dd, 'cost', xdf.at[dd, 'cost'] - abs(target_pos) *
(mslice.open * tcost))
xdf.set_value(dd, 'traded_price',
mslice.open + misc.sign(target_pos) * offset)
if pos_update and pos != 0:
if curr_pos[0].check_exit(mslice.open, stoploss * mslice.boll_std):
curr_pos[0].close(mslice.open - misc.sign(pos) * offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
xdf.set_value(
dd, 'cost',
xdf.at[dd, 'cost'] - abs(pos) * (mslice.open * tcost))
xdf.set_value(dd, 'traded_price',
mslice.open - misc.sign(pos) * offset)
pos = 0
else:
curr_pos[0].update_bar(mslice)
xdf.set_value(dd, 'pos', pos)
return (xdf, closed_trades)
def MA_sim(mdf, config):
offset = config['offset']
ma_list = config['ma_list']
param = config['param']
corr_entry = param[0]
corr_exit = param[1]
pval_entry = param[2]
pval_exit = param[3]
if len(param) >= 5:
channel = param[4]
else:
channel = 0
if channel == 0:
use_chan = False
else:
use_chan = True
close_daily = config['close_daily']
tcost = config['trans_cost']
freq = config['freq']
if int(freq[:-3]) == 1:
xdf = mdf
else:
xdf = dh.conv_ohlc_freq(mdf, freq, extra_cols=['contract'])
if use_chan:
xdf['chan_high'] = eval(config['channel_func'][0])(
xdf, channel, **config['channel_args'][0]).shift(2)
xdf['chan_low'] = eval(config['channel_func'][1])(
xdf, channel, **config['channel_args'][1]).shift(2)
else:
xdf['chan_high'] = pd.Series(index=xdf.index)
xdf['chan_low'] = pd.Series(index=xdf.index)
ma_ribbon = dh.MA_RIBBON(xdf, ma_list)
ribbon_corr = ma_ribbon["MARIBBON_CORR"].shift(1)
ribbon_pval = ma_ribbon["MARIBBON_PVAL"].shift(1)
ribbon_dist = ma_ribbon["MARIBBON_DIST"].shift(1)
xdf['prev_close'] = xdf['close'].shift(1)
xdf['close_ind'] = np.isnan(xdf['close'].shift(-1))
if close_daily:
daily_end = (xdf['date'] != xdf['date'].shift(-1))
xdf['close_ind'] = xdf['close_ind'] | daily_end
long_signal = pd.Series(np.nan, index=xdf.index)
long_flag = (ribbon_corr >= corr_entry) & (ribbon_pval < pval_entry)
if use_chan:
long_flag = long_flag & (xdf['open'] >= xdf['chan_high'])
long_signal[long_flag] = 1
long_signal[(ribbon_corr < corr_exit) | (ribbon_pval > pval_exit)] = 0
long_signal[xdf['close_ind']] = 0
long_signal = long_signal.fillna(method='ffill').fillna(0)
short_signal = pd.Series(np.nan, index=xdf.index)
short_flag = (ribbon_corr <= -corr_entry) & (ribbon_pval < pval_entry)
if use_chan:
short_flag = short_flag & (xdf['open'] <= xdf['chan_low'])
short_signal[short_flag] = -1
short_signal[(ribbon_corr > -corr_exit) | (ribbon_pval > pval_exit)] = 0
short_signal[xdf['close_ind']] = 0
short_signal = short_signal.fillna(method='ffill').fillna(0)
if len(xdf[(long_signal > 0) & (short_signal < 0)]) > 0:
print xdf[(long_signal > 0) & (short_signal < 0)]
print "something wrong with the position as long signal and short signal happen the same time"
xdf['pos'] = long_signal + short_signal
xdf['cost'] = abs(xdf['pos'] - xdf['pos'].shift(1)) * (offset +
xdf['open'] * tcost)
xdf['cost'] = xdf['cost'].fillna(0.0)
xdf['traded_price'] = xdf.open + (xdf['pos'] -
xdf['pos'].shift(1)) * offset
closed_trades = backtest.simdf_to_trades1(xdf, slippage=offset)
return (xdf, closed_trades)
def bband_chan_sim( mdf, config):
start_equity = config['capital']
marginrate = config['marginrate']
offset = config['offset']
pos_class = config['pos_class']
pos_args = config['pos_args']
param = config['param']
#tick_base = config['tick_base']
close_daily = config['close_daily']
win = param[0]
k = param[1]
chan = param[2]
if chan > 0:
chan_func = config['chan_func']
tcost = config['trans_cost']
filter_func = config['filter_func']['func']
filter_args = config['filter_func']['args']
filter_param = config['filter_func']['param']
unit = config['unit']
freq = config['freq']
xdf = dh.conv_ohlc_freq(mdf, freq)
xdf['boll_ma'] = dh.MA(xdf, win).shift(1)
boll_std = dh.STDEV(xdf, win).shift(1)
xdf['upbnd'] = xdf['boll_ma'] + boll_std * k
xdf['lowbnd'] = xdf['boll_ma'] - boll_std * k
if chan > 0:
xdf['chan_h'] = eval(chan_func['high']['func'])(xdf, chan, **chan_func['high']['args']).shift(1)
xdf['chan_l'] = eval(chan_func['low']['func'])(xdf, chan, **chan_func['low']['args']).shift(1)
else:
xdf['chan_h'] = 0
xdf['chan_l'] = 10000000
xdf['high_band'] = xdf[['chan_h', 'upbnd']].max(axis=1)
xdf['low_band'] = xdf[['chan_l', 'lowbnd']].min(axis=1)
xdf['filter'] = eval(filter_func)(xdf, filter_param[0], **filter_args)
xdf['filter_ma'] = pd.rolling_mean(xdf['filter'], filter_param[1])
xdf['filter_ind'] = xdf['filter_ma'] >= filter_param[2]
xdf['prev_close'] = xdf['close'].shift(1)
xdf['close_ind'] = np.isnan(xdf['close'].shift(-1))
if close_daily:
daily_end = (xdf['date']!=xdf['date'].shift(-1))
xdf['close_ind'] = xdf['close_ind'] | daily_end
ll = xdf.shape[0]
xdf['pos'] = 0
xdf['cost'] = 0
xdf['traded_price'] = xdf.open
curr_pos = []
closed_trades = []
tradeid = 0
for idx, dd in enumerate(xdf.index):
mslice = xdf.loc[dd]
if len(curr_pos) == 0:
pos = 0
else:
pos = curr_pos[0].pos
xdf.set_value(dd, 'pos', pos)
if np.isnan(mslice.boll_ma) or np.isnan(mslice.chan_h):
continue
if mslice.close_ind:
if pos!=0:
curr_pos[0].close(mslice.open - misc.sign(pos) * offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
xdf.set_value(dd, 'cost', xdf.at[dd, 'cost'] - abs(pos) * ( mslice.open * tcost))
xdf.set_value(dd, 'traded_price', mslice.open - misc.sign(pos) * offset)
pos = 0
else:
if ((mslice.open > mslice.boll_ma) and (pos<0)) or ((mslice.open < mslice.boll_ma) and (pos>0)):
curr_pos[0].close(mslice.open - misc.sign(pos) * offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
xdf.set_value(dd, 'cost', xdf.at[dd, 'cost'] - abs(pos) * (mslice.open * tcost))
xdf.set_value(dd, 'traded_price', mslice.open - misc.sign(pos) * offset)
pos = 0
if ((mslice.open >= mslice.high_band) or (mslice.open <= mslice.low_band)) and (pos==0):
target_pos = ( mslice.open >= mslice.high_band) * unit - (mslice.open <= mslice.low_band) * unit
new_pos = pos_class([mslice.contract], [1], target_pos, mslice.open, mslice.open, **pos_args)
tradeid += 1
new_pos.entry_tradeid = tradeid
new_pos.open(mslice.open + misc.sign(target_pos)*offset, dd)
curr_pos.append(new_pos)
pos = target_pos
xdf.set_value(dd, 'cost', xdf.at[dd, 'cost'] - abs(target_pos) * (mslice.open * tcost))
xdf.set_value(dd, 'traded_price', mslice.open + misc.sign(target_pos)*offset)
xdf.set_value(dd, 'pos', pos)
return (xdf, closed_trades)
def psar_chan_sim( mdf, config):
marginrate = config['marginrate']
offset = config['offset']
tcost = config['trans_cost']
unit = config['unit']
stoploss = config['stoploss']
bar_func = config.get('bar_conv_func', 'dh.bar_conv_func2')
param = config['param']
freq = config['freq']
use_HL = config.get('use_HL',False)
use_chan = config['use_chan']
chan_func = config['chan_func']
chan_args = config['chan_args']
chan = param[0]
sar_param = param[1]
pos_update = config.get('pos_update', False)
pos_class = config['pos_class']
pos_args = config['pos_args']
close_daily = config['close_daily']
xdf = dh.conv_ohlc_freq(mdf, freq, extra_cols = ['contract'], bar_func = eval(bar_func))
if use_chan:
xdf['chanH'] = eval(chan_func[0])(xdf, chan, **chan_args[0]).shift(1)
xdf['chanL'] = eval(chan_func[1])(xdf, chan, **chan_args[1]).shift(1)
else:
xdf['chanH'] = 0
xdf['chanL'] = 10000000
psar_data = dh.PSAR(xdf, **sar_param)
xdf['psar_dir'] = psar_data['PSAR_DIR'].shift(1)
xdf['psar_val'] = psar_data['PSAR_VAL'].shift(1)
xdf['prev_high'] = xdf['high'].shift(1)
xdf['prev_low'] = xdf['low'].shift(1)
xdf['prev_close'] = xdf['close'].shift(1)
xdf['close_ind'] = np.isnan(xdf['close'].shift(-1))
if close_daily:
daily_end = (xdf['date']!=xdf['date'].shift(-1))
xdf['close_ind'] = xdf['close_ind'] | daily_end
xdf['pos'] = 0
xdf['cost'] = 0
xdf['traded_price'] = xdf.open
curr_pos = []
closed_trades = []
tradeid = 0
for idx, dd in enumerate(xdf.index):
mslice = xdf.loc[dd]
if len(curr_pos) == 0:
pos = 0
else:
pos = curr_pos[0].pos
xdf.set_value(dd, 'pos', pos)
if np.isnan(mslice.chanH) or np.isnan(mslice.chanL):
continue
if use_HL:
buy_trig = (mslice.prev_high >= mslice.chanH) and (mslice.psar_dir > 0)
sell_trig = (mslice.prev_low <= mslice.chanL) and (mslice.psar_dir < 0)
long_close = (mslice.prev_low <= mslice.chanL) or (mslice.psar_dir < 0)
short_close = (mslice.prev_high >= mslice.chanH) or (mslice.psar_dir > 0)
else:
buy_trig = (mslice.open >= mslice.chanH) and (mslice.psar_dir > 0)
sell_trig = (mslice.open <= mslice.chanL) and (mslice.psar_dir < 0)
long_close = (mslice.open <= mslice.chanL) or (mslice.psar_dir < 0)
short_close = (mslice.open >= mslice.chanH) or (mslice.psar_dir > 0)
if mslice.close_ind:
if (pos != 0):
curr_pos[0].close(mslice.open - misc.sign(pos) * offset , dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
xdf.set_value(dd, 'cost', xdf.at[dd, 'cost'] - abs(pos) * ( mslice.open * tcost))
xdf.set_value(dd, 'traded_price', mslice.open - misc.sign(pos) * offset)
pos = 0
else:
if (short_close and (pos > 0)) or (long_close and (pos < 0)):
curr_pos[0].close(mslice.open - misc.sign(pos) * offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
xdf.set_value(dd, 'cost', xdf.at[dd, 'cost'] - abs(pos) * (mslice.open * tcost))
xdf.set_value(dd, 'traded_price', mslice.open - misc.sign(pos) * offset)
pos = 0
if (buy_trig or sell_trig) and (pos==0):
target_pos = buy_trig * unit - sell_trig * unit
new_pos = pos_class([mslice.contract], [1], target_pos, mslice.open, mslice.open, **pos_args)
tradeid += 1
new_pos.entry_tradeid = tradeid
new_pos.open(mslice.open + misc.sign(target_pos)*offset, dd)
curr_pos.append(new_pos)
pos = target_pos
xdf.set_value(dd, 'cost', xdf.at[dd, 'cost'] - abs(target_pos) * (mslice.open * tcost))
xdf.set_value(dd, 'traded_price', mslice.open + misc.sign(target_pos)*offset)
#if pos_update and pos != 0:
# if curr_pos[0].check_exit(mslice.open, stoploss * mslice.boll_std):
# curr_pos[0].close(mslice.open - misc.sign(pos) * offset, dd)
# tradeid += 1
# curr_pos[0].exit_tradeid = tradeid
# closed_trades.append(curr_pos[0])
# curr_pos = []
# xdf.set_value(dd, 'cost', xdf.at[dd, 'cost'] - abs(pos) * (mslice.open * tcost))
# xdf.set_value(dd, 'traded_price', mslice.open - misc.sign(pos) * offset)
# pos = 0
# else:
# curr_pos[0].update_bar(mslice)
xdf.set_value(dd, 'pos', pos)
return (xdf, closed_trades)
def chanbreak_sim( mdf, ddf, config):
freq = config['freq']
str_freq = str(freq) + 'Min'
xdf = dh.conv_ohlc_freq(mdf, str_freq)
start_equity = config['capital']
tcost = config['trans_cost']
unit = config['unit']
k = config['scaler']
marginrate = config['marginrate']
offset = config['offset']
win = config['win']
chan_func = config['channel_func']
upper_chan_func = chan_func[0]
lower_chan_func = chan_func[1]
entry_chan = win
exit_chan = int(entry_chan/k[1])
xdf['H1'] = upper_chan_func(xdf, entry_chan).shift(1)
xdf['L1'] = lower_chan_func(xdf, entry_chan).shift(1)
xdf['H2'] = upper_chan_func(xdf, exit_chan).shift(1)
xdf['L2'] = lower_chan_func(xdf, exit_chan).shift(1)
ddf['ATR'] = dh.ATR(ddf, entry_chan)
ll = mdf.shape[0]
mdf['pos'] = pd.Series([0]*ll, index = mdf.index)
mdf['cost'] = pd.Series([0]*ll, index = mdf.index)
curr_pos = []
closed_trades = []
end_d = mdf.index[-1].date()
tradeid = 0
x_idx = 0
max_idx = len(xdf.index)
for idx, dd in enumerate(mdf.index):
mslice = mdf.ix[dd]
min_id = agent.get_min_id(dd)
d = dd.date()
dslice = ddf.ix[d]
while (x_idx<max_idx-1) and (xdf.index[x_idx + 1] < dd):
x_idx += 1
xslice = xdf.iloc[x_idx]
if len(curr_pos) == 0:
pos = 0
else:
pos = curr_pos[0].pos
mdf.ix[dd, 'pos'] = pos
if np.isnan(dslice.ATR):
continue
if (min_id >=config['exit_min']):
if (pos!=0) and (d == end_d):
curr_pos[0].close(mslice.close - misc.sign(pos) * offset , dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
mdf.ix[dd, 'cost'] -= abs(pos) * (offset + mslice.close*tcost)
continue
else:
if (pos !=0):
curr_pos[0].trail_update(mslice.close)
if curr_pos[0].trail_check(mslice.close, dslice.ATR*k[0]):
curr_pos[0].close(mslice.close - misc.sign(pos) * offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
pos = 0
curr_pos = []
if ((mslice.close >= xslice.H2) and (pos<0)) or ((mslice.close <= xslice.L2) and (pos>0)):
curr_pos[0].close(mslice.close - misc.sign(pos) * offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
mdf.ix[dd, 'cost'] -= abs(pos) * (offset + mslice.close*tcost)
pos = 0
if ((mslice.close>=xslice.H1) and (pos<=0)) or ((mslice.close <= xslice.L1) and (pos>=0)):
if (pos ==0 ):
target_pos = (mslice.close>=xslice.H1) * unit -(mslice.close<=xslice.L1) * unit
new_pos = strat.TradePos([mslice.contract], [1], target_pos, mslice.close, mslice.close)
tradeid += 1
new_pos.entry_tradeid = tradeid
new_pos.open(mslice.close + misc.sign(target_pos)*offset, dd)
curr_pos.append(new_pos)
mdf.ix[dd, 'cost'] -= abs(target_pos) * (offset + mslice.close*tcost)
mdf.ix[dd, 'pos'] = pos
else:
print "something wrong with position=%s, close =%s, upBnd=%s, lowBnd=%s" % ( pos, mslice.close, xslice.H1, xslice.L1)
(res_pnl, ts) = backtest.get_pnl_stats( mdf, start_equity, marginrate, 'm')
res_trade = backtest.get_trade_stats( closed_trades )
res = dict( res_pnl.items() + res_trade.items())
return (res, closed_trades, ts)
def asctrend_sim(mdf, config):
offset = config['offset']
tcost = config['trans_cost']
unit = config['unit']
stoploss = config['stoploss']
bar_func = config.get('bar_conv_func', 'dh.bar_conv_func2')
param = config['param']
freq = config['freq']
pos_update = config.get('pos_update', False)
pos_class = config['pos_class']
pos_args = config['pos_args']
close_daily = config['close_daily']
xdf = dh.conv_ohlc_freq(mdf,
freq,
extra_cols=['contract'],
bar_func=eval(bar_func))
asc_period = param[0]
asc_risk = param[1]
rsi_sig = config['rsi_sig']
asctrend = dh.ASCTREND(xdf, asc_period, risk=asc_risk)
xdf['asc_signal'] = asctrend["ASCSIG_%s" % str(asc_period)].shift(1)
xdf['asc_stop'] = asctrend["ASCSTOP_%s" % str(asc_period)].shift(1)
rsi_period = param[2]
rsi_offset = param[3]
rsi_buy = 50 + rsi_offset
rsi_sell = 50 - rsi_offset
rsi = dh.RSI(xdf, n=rsi_period)
rsi_signal = pd.Series(0, index=rsi.index)
if rsi_sig:
rsi_signal[(rsi >= rsi_buy) & (rsi.shift(1) < rsi_buy)] = 1
rsi_signal[(rsi <= rsi_sell) & (rsi.shift(1) > rsi_sell)] = -1
else:
rsi_signal[(rsi >= rsi_buy)] = 1
rsi_signal[(rsi <= rsi_sell)] = -1
xdf['rsi_signal'] = rsi_signal.shift(1)
if len(param) > 4:
sar_step = param[4]
sar_max = param[5]
else:
sar_step = 0.005
sar_max = 0.02
sar = dh.SAR(xdf, incr=sar_step, maxaf=sar_max)
sar_signal = pd.Series(0, index=sar.index)
sar_signal[(sar < xdf['close'])] = 1
sar_signal[(sar > xdf['close'])] = -1
xdf['sar_signal'] = sar_signal.shift(1)
xdf['sar_stop'] = sar.shift(1)
xdf['close_ind'] = np.isnan(xdf['close'].shift(-1))
if close_daily:
daily_end = (xdf['date'] != xdf['date'].shift(-1))
xdf['close_ind'] = xdf['close_ind'] | daily_end
xdf['pos'] = 0
xdf['cost'] = 0
xdf['traded_price'] = xdf.open
curr_pos = []
closed_trades = []
tradeid = 0
for idx, dd in enumerate(xdf.index):
mslice = xdf.loc[dd]
if len(curr_pos) == 0:
pos = 0
else:
pos = curr_pos[0].pos
xdf.set_value(dd, 'pos', pos)
if np.isnan(mslice.sar_stop) or np.isnan(mslice.asc_stop):
continue
buy_trig = (mslice.asc_signal > 0) and (mslice.rsi_signal >
0) and (mslice.sar_signal > 0)
sell_trig = (mslice.asc_signal < 0) and (mslice.rsi_signal <
0) and (mslice.sar_signal < 0)
buy_close = (mslice.asc_signal < 0) or (mslice.rsi_signal < 0)
sell_close = (mslice.asc_signal > 0) or (mslice.rsi_signal > 0)
if mslice.close_ind:
if (pos != 0):
curr_pos[0].close(mslice.open - misc.sign(pos) * offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
xdf.set_value(
dd, 'cost',
xdf.at[dd, 'cost'] - abs(pos) * (mslice.open * tcost))
xdf.set_value(dd, 'traded_price',
mslice.open - misc.sign(pos) * offset)
pos = 0
else:
if pos_update and pos != 0:
curr_pos[0].update_price(mslice.asc_stop)
pos_close = curr_pos[0].check_exit(mslice.open, 0)
else:
pos_close = False
if (buy_close and (pos > 0)) or (sell_close and
(pos < 0)) or pos_close:
curr_pos[0].close(mslice.open - misc.sign(pos) * offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
xdf.set_value(
dd, 'cost',
xdf.at[dd, 'cost'] - abs(pos) * (mslice.open * tcost))
xdf.set_value(dd, 'traded_price',
mslice.open - misc.sign(pos) * offset)
pos = 0
if (buy_trig or sell_trig) and (pos == 0):
target_pos = buy_trig * unit - sell_trig * unit
new_pos = pos_class([mslice.contract], [1], target_pos,
mslice.open, mslice.asc_stop, **pos_args)
tradeid += 1
new_pos.entry_tradeid = tradeid
new_pos.open(mslice.open + misc.sign(target_pos) * offset, dd)
curr_pos.append(new_pos)
pos = target_pos
xdf.set_value(
dd, 'cost', xdf.at[dd, 'cost'] - abs(target_pos) *
(mslice.open * tcost))
xdf.set_value(dd, 'traded_price',
mslice.open + misc.sign(target_pos) * offset)
xdf.set_value(dd, 'pos', pos)
return (xdf, closed_trades)
def bband_chan_sim(mdf, config):
start_equity = config['capital']
marginrate = config['marginrate']
offset = config['offset']
pos_class = config['pos_class']
pos_args = config['pos_args']
param = config['param']
#tick_base = config['tick_base']
close_daily = config['close_daily']
win = param[0]
k = param[1]
chan = param[2]
if chan > 0:
chan_func = config['chan_func']
tcost = config['trans_cost']
filter_func = config['filter_func']['func']
filter_args = config['filter_func']['args']
filter_param = config['filter_func']['param']
unit = config['unit']
freq = config['freq']
xdf = dh.conv_ohlc_freq(mdf, freq)
xdf['boll_ma'] = dh.MA(xdf, win).shift(1)
boll_std = dh.STDEV(xdf, win).shift(1)
xdf['upbnd'] = xdf['boll_ma'] + boll_std * k
xdf['lowbnd'] = xdf['boll_ma'] - boll_std * k
if chan > 0:
xdf['chan_h'] = eval(chan_func['high']['func'])(
xdf, chan, **chan_func['high']['args']).shift(1)
xdf['chan_l'] = eval(chan_func['low']['func'])(
xdf, chan, **chan_func['low']['args']).shift(1)
else:
xdf['chan_h'] = 0
xdf['chan_l'] = 10000000
xdf['high_band'] = xdf[['chan_h', 'upbnd']].max(axis=1)
xdf['low_band'] = xdf[['chan_l', 'lowbnd']].min(axis=1)
xdf['filter'] = eval(filter_func)(xdf, filter_param[0], **filter_args)
xdf['filter_ma'] = pd.rolling_mean(xdf['filter'], filter_param[1])
xdf['filter_ind'] = xdf['filter_ma'] >= filter_param[2]
xdf['prev_close'] = xdf['close'].shift(1)
xdf['close_ind'] = np.isnan(xdf['close'].shift(-1))
if close_daily:
daily_end = (xdf['date'] != xdf['date'].shift(-1))
xdf['close_ind'] = xdf['close_ind'] | daily_end
ll = xdf.shape[0]
xdf['pos'] = 0
xdf['cost'] = 0
xdf['traded_price'] = xdf.open
curr_pos = []
closed_trades = []
tradeid = 0
for idx, dd in enumerate(xdf.index):
mslice = xdf.loc[dd]
if len(curr_pos) == 0:
pos = 0
else:
pos = curr_pos[0].pos
xdf.set_value(dd, 'pos', pos)
if np.isnan(mslice.boll_ma) or np.isnan(mslice.chan_h):
continue
if mslice.close_ind:
if pos != 0:
curr_pos[0].close(mslice.open - misc.sign(pos) * offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
xdf.set_value(
dd, 'cost',
xdf.at[dd, 'cost'] - abs(pos) * (mslice.open * tcost))
xdf.set_value(dd, 'traded_price',
mslice.open - misc.sign(pos) * offset)
pos = 0
else:
if ((mslice.open > mslice.boll_ma) and
(pos < 0)) or ((mslice.open < mslice.boll_ma) and (pos > 0)):
curr_pos[0].close(mslice.open - misc.sign(pos) * offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
xdf.set_value(
dd, 'cost',
xdf.at[dd, 'cost'] - abs(pos) * (mslice.open * tcost))
xdf.set_value(dd, 'traded_price',
mslice.open - misc.sign(pos) * offset)
pos = 0
if ((mslice.open >= mslice.high_band) or
(mslice.open <= mslice.low_band)) and (pos == 0):
target_pos = (mslice.open >= mslice.high_band) * unit - (
mslice.open <= mslice.low_band) * unit
new_pos = pos_class([mslice.contract], [1], target_pos,
mslice.open, mslice.open, **pos_args)
tradeid += 1
new_pos.entry_tradeid = tradeid
new_pos.open(mslice.open + misc.sign(target_pos) * offset, dd)
curr_pos.append(new_pos)
pos = target_pos
xdf.set_value(
dd, 'cost', xdf.at[dd, 'cost'] - abs(target_pos) *
(mslice.open * tcost))
xdf.set_value(dd, 'traded_price',
mslice.open + misc.sign(target_pos) * offset)
xdf.set_value(dd, 'pos', pos)
return (xdf, closed_trades)
def prepare_data_env(self, inst, mid_day=True):
if self.instruments[inst].ptype == instrument.ProductType.Option:
return
if self.daily_data_days > 0 or mid_day:
self.logger.debug('Updating historical daily data for %s' %
self.scur_day.strftime('%Y-%m-%d'))
daily_start = workdays.workday(self.scur_day,
-self.daily_data_days, CHN_Holidays)
daily_end = self.scur_day
self.day_data[inst] = mysqlaccess.load_daily_data_to_df(
'fut_daily', inst, daily_start, daily_end)
df = self.day_data[inst]
print inst
if len(df) > 0:
self.instruments[inst].price = df['close'][-1]
self.instruments[inst].last_update = 0
self.instruments[inst].prev_close = df['close'][-1]
for fobj in self.day_data_func[inst]:
ts = fobj.sfunc(df)
df[ts.name] = pd.Series(ts, index=df.index)
if self.min_data_days > 0 or mid_day:
self.logger.debug('Updating historical min data for %s' %
self.scur_day.strftime('%Y-%m-%d'))
d_start = workdays.workday(self.scur_day, -self.min_data_days,
CHN_Holidays)
d_end = self.scur_day
min_start = int(self.instruments[inst].start_tick_id / 1000)
min_end = int(self.instruments[inst].last_tick_id / 1000) + 1
mindata = mysqlaccess.load_min_data_to_df('fut_min',
inst,
d_start,
d_end,
minid_start=min_start,
minid_end=min_end,
database='blueshale')
mindata = backtest.cleanup_mindata(mindata,
self.instruments[inst].product)
self.min_data[inst][1] = mindata
if len(mindata) > 0:
min_date = mindata.index[-1].date()
if (len(self.day_data[inst].index)
== 0) or (min_date > self.day_data[inst].index[-1]):
ddf = data_handler.conv_ohlc_freq(mindata, 'd')
self.cur_day[inst]['open'] = float(ddf.open[-1])
self.cur_day[inst]['close'] = float(ddf.close[-1])
self.cur_day[inst]['high'] = float(ddf.high[-1])
self.cur_day[inst]['low'] = float(ddf.low[-1])
self.cur_day[inst]['volume'] = int(ddf.volume[-1])
self.cur_day[inst]['openInterest'] = int(
ddf.openInterest[-1])
self.cur_min[inst]['datetime'] = pd.datetime(
*mindata.index[-1].timetuple()[0:-3])
self.cur_min[inst]['open'] = float(mindata.ix[-1, 'open'])
self.cur_min[inst]['close'] = float(mindata.ix[-1,
'close'])
self.cur_min[inst]['high'] = float(mindata.ix[-1, 'high'])
self.cur_min[inst]['low'] = float(mindata.ix[-1, 'low'])
self.cur_min[inst]['volume'] = self.cur_day[inst]['volume']
self.cur_min[inst]['openInterest'] = self.cur_day[inst][
'openInterest']
self.cur_min[inst]['min_id'] = int(mindata.ix[-1,
'min_id'])
self.instruments[inst].price = float(mindata.ix[-1,
'close'])
self.instruments[inst].last_update = 0
self.logger.debug('inst=%s tick data loaded for date=%s' %
(inst, min_date))
for m in self.min_data_func[inst]:
if m != 1:
self.min_data[inst][m] = data_handler.conv_ohlc_freq(
self.min_data[inst][1],
str(m) + 'min')
df = self.min_data[inst][m]
for fobj in self.min_data_func[inst][m]:
ts = fobj.sfunc(df)
df[ts.name] = pd.Series(ts, index=df.index)
def dual_thrust_sim(mdf, config):
(pos_class, pos_args) = config['pos_param']
update_freq = config.get('update_freq', 0)
ddf = config['ddf']
close_daily = config['close_daily']
offset = config['offset']
k = config['param'][0]
win = config['param'][1]
multiplier = config['param'][2]
f = config['param'][3]
chan = config['chan']
chan_func = config['chan_func']
use_chan = config['use_chan']
tcost = config['trans_cost']
unit = config['unit']
SL = config['stoploss']
min_rng = config['min_range']
if win == -1:
tr = pd.concat([ddf.high - ddf.low, ddf.close - ddf.close.shift(1)],
join='outer',
axis=1).max(axis=1).shift(1)
elif win == 0:
tr = pd.concat(
[(pd.rolling_max(ddf.high, 2) - pd.rolling_min(ddf.close, 2)) *
multiplier,
(pd.rolling_max(ddf.close, 2) - pd.rolling_min(ddf.low, 2)) *
multiplier, ddf.high - ddf.close, ddf.close - ddf.low],
join='outer',
axis=1).max(axis=1).shift(1)
else:
tr = pd.concat([
pd.rolling_max(ddf.high, win) - pd.rolling_min(ddf.close, win),
pd.rolling_max(ddf.close, win) - pd.rolling_min(ddf.low, win)
],
join='outer',
axis=1).max(axis=1).shift(1)
ddf['TR'] = tr
ddf['ATR'] = dh.ATR(ddf, win)
if use_chan:
ddf['CH_H'] = eval(chan_func['high']['func'])(
ddf, chan, **chan_func['high']['args']).shift(1)
ddf['CH_L'] = eval(chan_func['low']['func'])(
ddf, chan, **chan_func['low']['args']).shift(1)
if update_freq > 1:
xdf = dh.conv_ohlc_freq(mdf, str(update_freq) + 'Min')
else:
xdf = mdf
xdata = pd.concat([xdf['open'], xdf['high'], xdf['low'], xdf['close']],
axis=1,
keys=['xopen', 'xhigh', 'xlow', 'xclose'])
mdf = mdf.join(xdata, how='left').fillna(method='ffill')
ll = mdf.shape[0]
mdf['pos'] = 0
mdf['cost'] = 0
start_d = ddf.index[0]
end_d = mdf.index[-1].date()
prev_d = start_d - datetime.timedelta(days=1)
curr_pos = []
closed_trades = []
tradeid = 0
for idx, dd in enumerate(mdf.index):
mslice = mdf.ix[dd]
min_id = mslice.min_id
d = mslice.date
dslice = ddf.ix[d]
if np.isnan(dslice.TR) or (mslice.close == 0):
continue
if use_chan and np.isnan(dslice.CH_H):
continue
if len(curr_pos) == 0:
pos = 0
else:
pos = curr_pos[0].pos
mdf.ix[dd, 'pos'] = pos
d_open = dslice.open
if (d_open <= 0):
continue
rng = max(min_rng * d_open, k * dslice.TR)
if (prev_d < d):
d_open = mslice.open
else:
d_open = dslice.open
prev_d = d
buytrig = d_open + rng
selltrig = d_open - rng
stoploss = SL * dslice.ATR
tmp_args = pos_args.copy()
if 'reset_margin' in pos_args:
tmp_args['reset_margin'] = dslice.ATR * pos_args['reset_margin']
stoploss = tmp_args['reset_margin'] * stoploss
close_pos = False
if pos != 0:
if ((pos > 0) and
(mslice.high < buytrig)) or ((pos < 0) and
(mslice.low > selltrig)):
close_pos = curr_pos[0].check_exit(mslice.close, stoploss)
if (close_pos == False) and (update_freq > 0) and (
(mslice.min_id + 1) % update_freq == 0):
xslice = conv_xslice(mslice)
curr_pos[0].update_bar(xslice)
if close_pos or ((mslice.min_id >= config['exit_min']) and
(close_daily or (d == end_d))):
curr_pos[0].close(mslice.close - misc.sign(pos) * offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
mdf.ix[dd,
'cost'] -= abs(pos) * (offset + mslice.close * tcost)
pos = 0
if mslice.min_id < config['exit_min']:
if (mslice.high >= buytrig) and (pos <= 0):
if len(curr_pos) > 0:
curr_pos[0].close(mslice.close + offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
mdf.ix[dd, 'cost'] -= abs(pos) * (offset +
mslice.close * tcost)
if (use_chan == False) or (use_chan and
(mslice.high >= dslice.CH_H)):
new_pos = eval(pos_class)([mslice.contract], [1], unit,
mslice.close + offset,
mslice.close + offset,
**tmp_args)
tradeid += 1
new_pos.entry_tradeid = tradeid
new_pos.open(mslice.close + offset, dd)
curr_pos.append(new_pos)
pos = unit
mdf.ix[dd, 'cost'] -= abs(pos) * (offset +
mslice.close * tcost)
elif (mslice.low <= selltrig) and (pos >= 0):
if len(curr_pos) > 0:
curr_pos[0].close(mslice.close - offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
mdf.ix[dd, 'cost'] -= abs(pos) * (offset +
mslice.close * tcost)
if (use_chan == False) or (use_chan and
(mslice.low <= dslice.CH_L)):
new_pos = eval(pos_class)([mslice.contract], [1], -unit,
mslice.close - offset,
mslice.close - offset,
**tmp_args)
tradeid += 1
new_pos.entry_tradeid = tradeid
new_pos.open(mslice.close - offset, dd)
curr_pos.append(new_pos)
pos = -unit
mdf.ix[dd, 'cost'] -= abs(pos) * (offset +
mslice.close * tcost)
mdf.ix[dd, 'pos'] = pos
return (mdf, closed_trades)
def psar_chan_sim(mdf, config):
marginrate = config['marginrate']
offset = config['offset']
tcost = config['trans_cost']
unit = config['unit']
stoploss = config['stoploss']
bar_func = config.get('bar_conv_func', 'dh.bar_conv_func2')
param = config['param']
freq = config['freq']
use_HL = config.get('use_HL', False)
use_chan = config['use_chan']
chan_func = config['chan_func']
chan_args = config['chan_args']
chan = param[0]
sar_param = param[1]
pos_update = config.get('pos_update', False)
pos_class = config['pos_class']
pos_args = config['pos_args']
close_daily = config['close_daily']
xdf = dh.conv_ohlc_freq(mdf,
freq,
extra_cols=['contract'],
bar_func=eval(bar_func))
if use_chan:
xdf['chanH'] = eval(chan_func[0])(xdf, chan, **chan_args[0]).shift(1)
xdf['chanL'] = eval(chan_func[1])(xdf, chan, **chan_args[1]).shift(1)
else:
xdf['chanH'] = 0
xdf['chanL'] = 10000000
psar_data = dh.PSAR(xdf, **sar_param)
xdf['psar_dir'] = psar_data['PSAR_DIR'].shift(1)
xdf['psar_val'] = psar_data['PSAR_VAL'].shift(1)
xdf['prev_high'] = xdf['high'].shift(1)
xdf['prev_low'] = xdf['low'].shift(1)
xdf['prev_close'] = xdf['close'].shift(1)
xdf['close_ind'] = np.isnan(xdf['close'].shift(-1))
if close_daily:
daily_end = (xdf['date'] != xdf['date'].shift(-1))
xdf['close_ind'] = xdf['close_ind'] | daily_end
xdf['pos'] = 0
xdf['cost'] = 0
xdf['traded_price'] = xdf.open
curr_pos = []
closed_trades = []
tradeid = 0
for idx, dd in enumerate(xdf.index):
mslice = xdf.loc[dd]
if len(curr_pos) == 0:
pos = 0
else:
pos = curr_pos[0].pos
xdf.set_value(dd, 'pos', pos)
if np.isnan(mslice.chanH) or np.isnan(mslice.chanL):
continue
if use_HL:
buy_trig = (mslice.prev_high >= mslice.chanH) and (mslice.psar_dir
> 0)
sell_trig = (mslice.prev_low <= mslice.chanL) and (mslice.psar_dir
< 0)
long_close = (mslice.prev_low <= mslice.chanL) or (mslice.psar_dir
< 0)
short_close = (mslice.prev_high >=
mslice.chanH) or (mslice.psar_dir > 0)
else:
buy_trig = (mslice.open >= mslice.chanH) and (mslice.psar_dir > 0)
sell_trig = (mslice.open <= mslice.chanL) and (mslice.psar_dir < 0)
long_close = (mslice.open <= mslice.chanL) or (mslice.psar_dir < 0)
short_close = (mslice.open >= mslice.chanH) or (mslice.psar_dir >
0)
if mslice.close_ind:
if (pos != 0):
curr_pos[0].close(mslice.open - misc.sign(pos) * offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
xdf.set_value(
dd, 'cost',
xdf.at[dd, 'cost'] - abs(pos) * (mslice.open * tcost))
xdf.set_value(dd, 'traded_price',
mslice.open - misc.sign(pos) * offset)
pos = 0
else:
if (short_close and (pos > 0)) or (long_close and (pos < 0)):
curr_pos[0].close(mslice.open - misc.sign(pos) * offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
xdf.set_value(
dd, 'cost',
xdf.at[dd, 'cost'] - abs(pos) * (mslice.open * tcost))
xdf.set_value(dd, 'traded_price',
mslice.open - misc.sign(pos) * offset)
pos = 0
if (buy_trig or sell_trig) and (pos == 0):
target_pos = buy_trig * unit - sell_trig * unit
new_pos = pos_class([mslice.contract], [1], target_pos,
mslice.open, mslice.open, **pos_args)
tradeid += 1
new_pos.entry_tradeid = tradeid
new_pos.open(mslice.open + misc.sign(target_pos) * offset, dd)
curr_pos.append(new_pos)
pos = target_pos
xdf.set_value(
dd, 'cost', xdf.at[dd, 'cost'] - abs(target_pos) *
(mslice.open * tcost))
xdf.set_value(dd, 'traded_price',
mslice.open + misc.sign(target_pos) * offset)
#if pos_update and pos != 0:
# if curr_pos[0].check_exit(mslice.open, stoploss * mslice.boll_std):
# curr_pos[0].close(mslice.open - misc.sign(pos) * offset, dd)
# tradeid += 1
# curr_pos[0].exit_tradeid = tradeid
# closed_trades.append(curr_pos[0])
# curr_pos = []
# xdf.set_value(dd, 'cost', xdf.at[dd, 'cost'] - abs(pos) * (mslice.open * tcost))
# xdf.set_value(dd, 'traded_price', mslice.open - misc.sign(pos) * offset)
# pos = 0
# else:
# curr_pos[0].update_bar(mslice)
xdf.set_value(dd, 'pos', pos)
return (xdf, closed_trades)
def dual_thrust_sim( mdf, config):
(pos_class, pos_args) = config['pos_param']
update_freq = config.get('update_freq', 0)
ddf = config['ddf']
close_daily = config['close_daily']
offset = config['offset']
k = config['param'][0]
win = config['param'][1]
multiplier = config['param'][2]
f = config['param'][3]
chan = config['chan']
chan_func = config['chan_func']
use_chan = config['use_chan']
tcost = config['trans_cost']
unit = config['unit']
SL = config['stoploss']
min_rng = config['min_range']
if win == -1:
tr= pd.concat([ddf.high - ddf.low, ddf.close - ddf.close.shift(1)],
join='outer', axis=1).max(axis=1).shift(1)
elif win == 0:
tr = pd.concat([(pd.rolling_max(ddf.high, 2) - pd.rolling_min(ddf.close, 2))*multiplier,
(pd.rolling_max(ddf.close, 2) - pd.rolling_min(ddf.low, 2))*multiplier,
ddf.high - ddf.close,
ddf.close - ddf.low],
join='outer', axis=1).max(axis=1).shift(1)
else:
tr= pd.concat([pd.rolling_max(ddf.high, win) - pd.rolling_min(ddf.close, win),
pd.rolling_max(ddf.close, win) - pd.rolling_min(ddf.low, win)],
join='outer', axis=1).max(axis=1).shift(1)
ddf['TR'] = tr
ddf['ATR'] = dh.ATR(ddf, win)
if use_chan:
ddf['CH_H'] = eval(chan_func['high']['func'])(ddf, chan, **chan_func['high']['args']).shift(1)
ddf['CH_L'] = eval(chan_func['low']['func'])(ddf, chan, **chan_func['low']['args']).shift(1)
if update_freq > 1:
xdf = dh.conv_ohlc_freq(mdf, str(update_freq)+'Min')
else:
xdf = mdf
xdata = pd.concat([xdf['open'], xdf['high'], xdf['low'], xdf['close']], axis=1, keys=['xopen','xhigh', 'xlow', 'xclose'])
mdf = mdf.join(xdata, how = 'left').fillna(method='ffill')
ll = mdf.shape[0]
mdf['pos'] = 0
mdf['cost'] = 0
start_d = ddf.index[0]
end_d = mdf.index[-1].date()
prev_d = start_d - datetime.timedelta(days=1)
curr_pos = []
closed_trades = []
tradeid = 0
for idx, dd in enumerate(mdf.index):
mslice = mdf.ix[dd]
min_id = mslice.min_id
d = mslice.date
dslice = ddf.ix[d]
if np.isnan(dslice.TR) or (mslice.close == 0):
continue
if use_chan and np.isnan(dslice.CH_H):
continue
if len(curr_pos) == 0:
pos = 0
else:
pos = curr_pos[0].pos
mdf.ix[dd, 'pos'] = pos
d_open = dslice.open
if (d_open <= 0):
continue
rng = max(min_rng * d_open, k * dslice.TR)
if (prev_d < d):
d_open = mslice.open
else:
d_open = dslice.open
prev_d = d
buytrig = d_open + rng
selltrig = d_open - rng
stoploss = SL * dslice.ATR
tmp_args = pos_args.copy()
if 'reset_margin' in pos_args:
tmp_args['reset_margin'] = dslice.ATR * pos_args['reset_margin']
stoploss = tmp_args['reset_margin'] * stoploss
close_pos = False
if pos != 0:
if ((pos > 0) and (mslice.high < buytrig)) or ((pos < 0) and (mslice.low > selltrig)):
close_pos = curr_pos[0].check_exit(mslice.close, stoploss)
if (close_pos == False) and (update_freq > 0) and ((mslice.min_id + 1) % update_freq == 0):
xslice = conv_xslice(mslice)
curr_pos[0].update_bar(xslice)
if close_pos or ((mslice.min_id >= config['exit_min']) and (close_daily or (d == end_d))):
curr_pos[0].close(mslice.close - misc.sign(pos) * offset , dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
mdf.ix[dd, 'cost'] -= abs(pos) * (offset + mslice.close*tcost)
pos = 0
if mslice.min_id < config['exit_min']:
if (mslice.high >= buytrig) and (pos <=0 ):
if len(curr_pos) > 0:
curr_pos[0].close(mslice.close+offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
mdf.ix[dd, 'cost'] -= abs(pos) * (offset + mslice.close*tcost)
if (use_chan == False) or (use_chan and (mslice.high >= dslice.CH_H)):
new_pos = eval(pos_class)([mslice.contract], [1], unit, mslice.close + offset, mslice.close + offset, **tmp_args)
tradeid += 1
new_pos.entry_tradeid = tradeid
new_pos.open(mslice.close + offset, dd)
curr_pos.append(new_pos)
pos = unit
mdf.ix[dd, 'cost'] -= abs(pos) * (offset + mslice.close*tcost)
elif (mslice.low <= selltrig) and (pos >=0 ):
if len(curr_pos) > 0:
curr_pos[0].close(mslice.close-offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
mdf.ix[dd, 'cost'] -= abs(pos) * (offset + mslice.close*tcost)
if (use_chan == False) or (use_chan and (mslice.low <= dslice.CH_L)):
new_pos = eval(pos_class)([mslice.contract], [1], -unit, mslice.close - offset, mslice.close - offset, **tmp_args)
tradeid += 1
new_pos.entry_tradeid = tradeid
new_pos.open(mslice.close - offset, dd)
curr_pos.append(new_pos)
pos = -unit
mdf.ix[dd, 'cost'] -= abs(pos) * (offset + mslice.close*tcost)
mdf.ix[dd, 'pos'] = pos
return (mdf, closed_trades)
def layer_sim( mdf, config):
tcost = config['trans_cost']
unit = config['unit']
offset = config['offset']
pos_class = config['pos_class']
pos_args = config['pos_args']
freq = config['freq']
signal_func = config['signal_func']
signal_args = config['signal_args']
signal_level = config['signal_level']
close_daily = config['close_daily']
for idx, (f, sfunc, sargs, slvl) in enumerate(zip(freq, signal_func, signal_args, signal_level)):
df = dh.conv_ohlc_freq(mdf, f, extra_cols=['contract'])
if idx == 0:
xdf = df
farg = tuple(sargs)
func = eval(sfunc[0])
sigout = func(df, *farg)
sig_field = sfunc[1] % farg
signal = sigout[sig_field]
long_lvl = slvl[0] + slvl[1]
short_lvl = slvl[0] - slvl[1]
if idx > 0:
long_ind = (signal > long_lvl)
short_ind = (signal < short_lvl)
else:
long_ind = (signal > long_lvl) & (signal.shift(1) <= long_lvl)
short_ind = (signal < short_lvl) & (signal.shift(1) >= short_lvl)
xdata = pd.concat([signal, long_ind, short_ind], axis = 1, keys = ['signal'+str(idx), 'long_ind'+str(idx), 'short_ind'+str(idx)]).shift(1)
#print xdata
xdf = xdf.join(xdata, how = 'left').fillna(method='ffill')
xdf['close_ind'] = np.isnan(xdf['close'].shift(-1))
if close_daily:
daily_end = (xdf['date']!=xdf['date'].shift(-1))
xdf['close_ind'] = xdf['close_ind'] | daily_end
xdf['pos'] = 0
xdf['cost'] = 0
xdf['traded_price'] = xdf.open
curr_pos = []
closed_trades = []
tradeid = 0
for idx, dd in enumerate(xdf.index):
mslice = xdf.ix[dd]
if len(curr_pos) == 0:
pos = 0
else:
pos = curr_pos[0].pos
signal_indlist = [np.isnan(getattr(mslice, 'signal' + str(i))) for i in range(len(freq))]
long_indlist = [getattr(mslice, 'long_ind' + str(i)) for i in range(len(freq))]
short_indlist = [ getattr(mslice, 'short_ind'+str(i)) for i in range(len(freq))]
if True in signal_indlist:
continue
if mslice.close_ind:
if pos!=0:
curr_pos[0].close(mslice.open - misc.sign(pos) * offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
xdf.set_value(dd, 'cost', xdf.at[dd, 'cost'] - abs(pos) * ( mslice.open * tcost))
xdf.set_value(dd, 'traded_price', mslice.open - misc.sign(pos) * offset)
pos = 0
else:
if (pos !=0):
direction = 0
long_ind = (True in long_indlist)
short_ind = (True in short_indlist)
if ((pos < 0) and long_ind) or ((pos > 0) and short_ind):
curr_pos[0].close(mslice.open - misc.sign(pos) * offset, dd)
tradeid += 1
curr_pos[0].exit_tradeid = tradeid
closed_trades.append(curr_pos[0])
curr_pos = []
xdf.set_value(dd, 'cost', xdf.at[dd, 'cost'] - abs(pos) * (mslice.open * tcost))
xdf.set_value(dd, 'traded_price', mslice.open - misc.sign(pos) * offset)
pos = 0
long_ind = (False not in long_indlist)
short_ind = (False not in short_indlist)
if (long_ind or short_ind) and (pos == 0):
target_pos = long_ind * unit - short_ind * unit
new_pos = pos_class([mslice.contract], [1], target_pos, mslice.open, mslice.open, **pos_args)
tradeid += 1
new_pos.entry_tradeid = tradeid
new_pos.open(mslice.open + misc.sign(target_pos)*offset, dd)
curr_pos.append(new_pos)
#print dd, target_pos, len(curr_pos), curr_pos[0].pos, curr_pos #[getattr(mslice, 'long_ind' + str(i)) for i in range(len(freq))]
pos = target_pos
xdf.set_value(dd, 'cost', xdf.at[dd, 'cost'] - abs(target_pos) * (mslice.open * tcost))
xdf.set_value(dd, 'traded_price', mslice.open + misc.sign(target_pos)*offset)
xdf.set_value(dd, 'pos', pos)
return (xdf, closed_trades)
def prepare_data_env(self, inst, mid_day=True):
if self.instruments[inst].ptype == instrument.ProductType.Option:
return
if self.daily_data_days > 0 or mid_day:
#self.logger.debug('Updating historical daily data for %s' % self.scur_day.strftime('%Y-%m-%d'))
daily_start = workdays.workday(self.scur_day,
-self.daily_data_days, CHN_Holidays)
daily_end = self.scur_day
ddf = mysqlaccess.load_daily_data_to_df('fut_daily',
inst,
daily_start,
daily_end,
index_col=None)
if len(ddf) > 0:
self.instruments[inst].price = ddf['close'].iloc[-1]
self.instruments[inst].last_update = 0
self.instruments[inst].prev_close = ddf['close'].iloc[-1]
for fobj in self.day_data_func[inst]:
ts = fobj.sfunc(ddf)
if type(ts).__name__ == 'Series':
if ts.name in ddf.columns:
self.logger.warning(
'TimeSeries name %s is already in the columns for inst = %s'
% (ts.name, inst))
ddf[ts.name] = ts
elif type(ts).__name__ == 'DataFrame':
for col_name in ts.columns:
if col_name in ddf.columns:
self.logger.warning(
'TimeSeries name %s is already in the columns for inst = %s'
% (col_name, inst))
ddf[col_name] = ts[col_name]
self.day_data[inst] = data_handler.DynamicRecArray(dataframe=ddf)
if self.min_data_days > 0 or mid_day:
#self.logger.debug('Updating historical min data for %s' % self.scur_day.strftime('%Y-%m-%d'))
d_start = workdays.workday(self.scur_day, -self.min_data_days,
CHN_Holidays)
d_end = self.scur_day
min_start = int(self.instruments[inst].start_tick_id / 1000)
min_end = int(self.instruments[inst].last_tick_id / 1000) + 1
mdf = mysqlaccess.load_min_data_to_df('fut_min',
inst,
d_start,
d_end,
minid_start=min_start,
minid_end=min_end,
database='blueshale',
index_col=None)
mdf = backtest.cleanup_mindata(mdf,
self.instruments[inst].product,
index_col=None)
mdf['bar_id'] = self.conv_bar_id(mdf['min_id'], inst)
if len(mdf) > 0:
min_date = mdf['date'].iloc[-1]
if (len(self.day_data[inst]) == 0) or (
min_date > self.day_data[inst].data['date'][-1]):
ddf = data_handler.conv_ohlc_freq(mdf, 'd', index_col=None)
self.cur_day[inst]['open'] = float(ddf.open[-1])
self.cur_day[inst]['close'] = float(ddf.close[-1])
self.cur_day[inst]['high'] = float(ddf.high[-1])
self.cur_day[inst]['low'] = float(ddf.low[-1])
self.cur_day[inst]['volume'] = int(ddf.volume[-1])
self.cur_day[inst]['openInterest'] = int(
ddf.openInterest[-1])
self.cur_min[inst]['datetime'] = pd.datetime(
*mdf['datetime'].iloc[-1].timetuple()[0:-3])
self.cur_min[inst]['date'] = mdf['date'].iloc[-1]
self.cur_min[inst]['open'] = float(mdf['open'].iloc[-1])
self.cur_min[inst]['close'] = float(mdf['close'].iloc[-1])
self.cur_min[inst]['high'] = float(mdf['high'].iloc[-1])
self.cur_min[inst]['low'] = float(mdf['low'].iloc[-1])
self.cur_min[inst]['volume'] = self.cur_day[inst]['volume']
self.cur_min[inst]['openInterest'] = self.cur_day[inst][
'openInterest']
self.cur_min[inst]['min_id'] = int(mdf['min_id'].iloc[-1])
self.cur_min[inst]['bar_id'] = self.conv_bar_id(
self.cur_min[inst]['min_id'], inst)
self.instruments[inst].price = float(mdf['close'].iloc[-1])
self.instruments[inst].last_update = 0
#self.logger.debug('inst=%s tick data loaded for date=%s' % (inst, min_date))
if 1 not in self.min_data_func[inst]:
self.min_data[inst][1] = data_handler.DynamicRecArray(
dataframe=mdf)
for m in sorted(self.min_data_func[inst]):
if m != 1:
bar_func = lambda ts: self.conv_bar_id(ts, inst)
mdf_m = data_handler.conv_ohlc_freq(
mdf,
str(m) + 'min',
index_col=None,
bar_func=bar_func,
extra_cols=['bar_id'])
else:
mdf_m = mdf
for fobj in self.min_data_func[inst][m]:
ts = fobj.sfunc(mdf_m)
if type(ts).__name__ == 'Series':
if ts.name in mdf_m.columns:
self.logger.warning(
'TimeSeries name %s is already in the columns for inst = %s'
% (ts.name, inst))
mdf_m[ts.name] = ts
elif type(ts).__name__ == 'DataFrame':
for col_name in ts.columns:
if col_name in mdf_m.columns:
self.logger.warning(
'TimeSeries name %s is already in the columns for inst = %s'
% (col_name, inst))
mdf_m[col_name] = ts[col_name]
self.min_data[inst][m] = data_handler.DynamicRecArray(
dataframe=mdf_m)
import data_handler as dh
import pandas as pd
import numpy as np
import strategy as strat
import datetime
import backtest
def aberration( asset, start_date, end_date, freqs, windows, config):
nearby = config['nearby']
rollrule = config['rollrule']
file_prefix = config['file_prefix'] + '_' + asset + '_'
df = misc.nearby(asset, nearby, start_date, end_date, rollrule, 'm', need_shift=True)
output = {}
for ix, freq in enumerate(freqs):
xdf = dh.conv_ohlc_freq(df, freq):
for iy, win in enumerate(windows):
idx = ix*10+iy
config['win'] = win
(res, closed_trades, ts) = aberration_sim( xdf, mdf, , config)
output[idx] = res
print 'saving results for scen = %s' % str(idx)
all_trades = {}
for i, tradepos in enumerate(closed_trades):
all_trades[i] = strat.tradepos2dict(tradepos)
fname = file_prefix + str(idx) + '_trades.csv'
trades = pd.DataFrame.from_dict(all_trades).T
trades.to_csv(fname)
fname = file_prefix + str(idx) + '_dailydata.csv'
ts.to_csv(fname)
fname = file_prefix + 'stats.csv'