def TTM_squeeze(self, df_price, df_high, df_low, N, K, e, a, m):
sym_list = df_price.columns
timestamps = df_price.index
df_bb_ma = copy.deepcopy(df_price)
df_bb_ma = df_bb_ma * np.NAN
df_bb_u = copy.deepcopy(df_bb_ma)
#df_bb_u = df_bb_u * np.NAN
df_bb_l = copy.deepcopy(df_bb_ma)
#df_bb_l = df_bb_l * np.NAN
df_kch_u = copy.deepcopy(df_price)
df_kch_u = df_kch_u * np.NAN
df_kch_l = copy.deepcopy(df_price)
df_kch_l = df_kch_l * np.NAN
df_kch_m = self.ema(df_price,e)
df_atr = self.atr(df_price,df_high,df_low,a,m)
for sym in sym_list:
df_bb_ma[sym] = pd.rolling_mean(df_price[sym],window=N)
df_bb_u[sym] = pd.rolling_std(df_price[sym],window=N)
df_bb_l[sym] = pd.rolling_std(df_price[sym],window=N)
df_bb_u[sym] = df_bb_ma[sym] + (K*df_bb_u[sym])
df_bb_l[sym] = df_bb_ma[sym] - (K*df_bb_l[sym])
for t_stamp in range(0,len(timestamps)):
df_kch_u[sym].ix[t_stamp] = df_kch_m[sym].ix[t_stamp] + df_atr[sym].ix[t_stamp]
df_kch_l[sym].ix[t_stamp] = df_kch_m[sym].ix[t_stamp] - df_atr[sym].ix[t_stamp]
return df_bb_ma, df_bb_u, df_bb_l, df_kch_m, df_kch_u, df_kch_l
def volatility(self, n, freq=None, which='close', ann=True, model='ln', min_periods=1):
"""Return the annualized volatility series. N is the number of lookback periods.
:param n: int, number of lookback periods
:param freq: resample frequency or None
:param which: price series to use
:param ann: If True then annualize
:param model: {'ln', 'pct', 'bbg'}
ln - use logarithmic price changes
pct - use pct price changes
bbg - use logarithmic price changes but Bloomberg uses actual business days
:return:
"""
if model not in ('bbg', 'ln', 'pct'):
raise ValueError('model must be one of (bbg, ln, pct), not %s' % model)
px = self.frame[which]
px = px if not freq else px.resample(freq, how='last')
if model == 'bbg' and periods_in_year(px) == 252:
# Bloomberg uses business days, so need to convert and reindex
orig = px.index
px = px.resample('B').ffill()
chg = np.log(px / px.shift(1))
chg[chg.index - orig] = np.nan
vol = pd.rolling_std(chg, n, min_periods=min_periods).reindex(orig)
return vol if not ann else vol * np.sqrt(260)
else:
chg = px.pct_change() if model == 'pct' else np.log(px / px.shift(1))
vol = pd.rolling_std(chg, n, min_periods=min_periods)
return vol if not ann else vol * np.sqrt(periods_in_year(vol))
def find_events_using_bollingerBandIndicator(ls_symbols, d_data):
'''Find event using bollinger band'''
df_close = d_data['close']
ts_market = df_close['SPY']
event_count = 0
df_events = copy.deepcopy(df_close)
df_events = df_events * np.NAN
ldt_timestamps = df_close.index
spyPrice = df_close['SPY']
spyMean = pd.rolling_mean(spyPrice, 20)
spyStd = pd.rolling_std(spyPrice, 20)
spyBollinger = (spyPrice-spyMean)/spyStd
for s_sym in ls_symbols:
symprice = df_close[s_sym]
mean = pd.rolling_mean(symprice, 20)
std = pd.rolling_std(symprice, 20)
bollingerVals = (symprice-mean)/std
for i in range(1, len(ldt_timestamps)):
if(bollingerVals.ix[ldt_timestamps[i]] <= -2.0 and bollingerVals.ix[ldt_timestamps[i-1]] >= -2.0 and spyBollinger.ix[ldt_timestamps[i]] >= 1.5):
df_events[s_sym].ix[ldt_timestamps[i]] = 1
event_count += 1
print ("Total event number is %s."%(event_count))
return df_events
def generateTradeSigs(self,
windowLength,
entryScale,
exitScale,
ewma_par = None,
reg_params = None):
#option to specifiy reg params if not using those from OLSfit
if reg_params == None:
self['spread'] = self.results.resid
else:
self['spread'] = self[self.yInd] - reg_params[1]*self[self.xInd] - reg_params[0]
#We want to scale entry and exit params around the moving average - if ewma parameter given we do this
if ewma_par == None:
self['trend'] = 0
else:
self['trend'] = pd.ewma(self['spread'], ewma_par)
trend = self['trend']
#contruct bollingers based around trend
self['entryUpper'] = trend + entryScale*pd.rolling_std(self['spread'],windowLength)
self['entryLower'] = trend - entryScale*pd.rolling_std(self['spread'],windowLength)
self['exitUpper'] = trend + exitScale*pd.rolling_std(self['spread'],windowLength)
self['exitLower'] = trend - exitScale*pd.rolling_std(self['spread'],windowLength)
def volcone(symbol):
symbols = [symbol]
data = dict((sym, DataReader(sym, "yahoo")) for sym in symbols)
panel = pd.Panel(data).swapaxes('items', 'minor')
close_px = panel['Close']
#rets = close_px / close_px.shift(1) - 1
rets = np.log(close_px / close_px.shift(1) )
#annualize
rets *= np.sqrt(252)
std_30 = pd.rolling_std(rets, 30, min_periods=30)
std_60 = pd.rolling_std(rets, 60, min_periods=60)
std_90 = pd.rolling_std(rets, 90, min_periods=90)
std_120 = pd.rolling_std(rets, 120, min_periods=120)
a=std_30.describe()
b=std_60.describe()
c=std_90.describe()
d=std_120.describe()
for s in symbols:
cone = pd.DataFrame(a[s],columns=['30'])
cone.insert(1,'120',d[s])
cone.insert(1,'90',c[s])
cone.insert(1,'60',b[s])
cone=cone.T
del cone['count']
del cone['std']
cone.plot(title='%s Volatility Cone'%(s))
plt.grid(True)
plt.show()
def main():
try: os.makedirs('figs')
except: pass
df = pd.read_csv('./training_log.txt')
df['rps'] = df.reward / df.n_steps
df['pps'] = df.pnl / df.n_steps
df['rps_mmean'] = pd.rolling_mean(df.rps, 10)
df['rps_mstd'] = pd.rolling_std(df.rps, 10)
df['pps_mmean'] = pd.rolling_mean(df.pps, 10)
df['pps_mstd'] = pd.rolling_std(df.pps, 10)
fig, axes = plt.subplots(2, sharex=True)
fig.suptitle('Training results')
axes[0].plot(df.rps)
axes[0].plot(df.rps_mmean, label='mmean_10')
axes[0].plot(df.rps_mstd, label='mstd_10')
axes[0].set_ylabel('Mean reward per step [-]')
axes[0].legend(loc=4, prop={'size': 6})
axes[1].plot(df.pps)
axes[1].plot(df.pps_mmean, label='mmean_10')
axes[1].plot(df.pps_mstd, label='mstd_10')
axes[1].set_ylabel('PnL per Step [Currency]')
axes[1].set_xlabel('Episode [-]')
axes[1].legend(loc=4, prop={'size': 6})
plt.savefig('./figs/train.pdf', dpi=1000)
plt.close()
print 'Done'
def convert_test_runs_list_to_time_series_dict(test_runs_list, resample):
test_runs = []
for test_run in test_runs_list:
tr = test_run.to_dict()
# Populate dict
start_time = test_run.start_time
if start_time and test_run.start_time_microsecond:
start_time = start_time.replace(
microsecond=test_run.start_time_microsecond)
tr['start_time'] = start_time
tr.pop('start_time_microsecond')
if test_run.stop_time:
stop_time = test_run.stop_time
if test_run.stop_time_microsecond:
stop_time = stop_time.replace(
microsecond=test_run.stop_time_microsecond)
tr['stop_time'] = stop_time
tr['run_time'] = read_subunit.get_duration(start_time,
tr.pop('stop_time'))
tr.pop('stop_time_microsecond')
tr.pop('id')
tr.pop('test_id')
test_runs.append(tr)
df = pd.DataFrame(test_runs).set_index('start_time')
df.index = pd.DatetimeIndex(df.index)
# Add rolling mean and std dev of run_time to datafram
df['avg_run_time'] = pd.rolling_mean(df['run_time'], 20)
df['stddev_run_time'] = pd.rolling_std(df['run_time'], 20)
# Resample numeric data for the run_time graph from successful runs
numeric_df = df[df['status'] == 'success'].resample(
base.resample_matrix[resample], how='mean')
# Drop duplicate or invalid colums
del(numeric_df['run_id'])
del(df['run_time'])
# Interpolate missing data
numeric_df['run_time'] = numeric_df.interpolate(method='time', limit=20)
# Add rolling mean and std dev of run_time to datafram
numeric_df['avg_run_time'] = pd.rolling_mean(numeric_df['run_time'], 20)
numeric_df['stddev_run_time'] = pd.rolling_std(numeric_df['run_time'], 20)
# Convert the dataframes to a dict
numeric_dict = dict(
(date.isoformat(),
{
'run_time': run_time,
'avg_run_time': avg,
'std_dev_run_time': stddev,
}) for date, run_time, avg, stddev in zip(
numeric_df.index, numeric_df.run_time, numeric_df.avg_run_time,
numeric_df.stddev_run_time))
temp_dict = dict(
(date.isoformat(),
{
'run_id': run_id,
'status': status,
}) for date, run_id, status in zip(df.index, df.run_id, df.status))
return {'numeric': numeric_dict, 'data': temp_dict}
def compute_insample_data_ML4T(start_date,end_date):
dates=pd.date_range(start_date,end_date)
prices=get_data(['ML4T-399'],dates,True)
prices=prices.drop('SPY',axis=1)
#calculating volatility
daily_returns = prices.copy()
daily_returns[1:] = (prices[1:]/prices[:-1].values)-1.0
daily_returns.ix[0,:]=0
vol=pd.rolling_std(daily_returns,window=10)
vol1=vol[9:-5]
#calculating momentum
momentum=prices.copy()
momentum[9:] = (prices[9:]/prices[:-9].values)-1.0
momentum1=momentum[9:-5]
#calculating bollinger values
sma = pd.rolling_mean(prices,window=10)
sma1 = sma.dropna()
std=pd.rolling_std(prices,window=10)
std1=std[9:]
bb_prices=prices[9:]
bb_value=(bb_prices - sma1)/(2*std1)
bb_value1=bb_value[0:-5]
#shifting prices
shifted_prices=prices.shift(-5)
future_prices = prices.copy()
prices1=prices[9:-5]
future_return=(shifted_prices/prices) - 1.0
future_return1=future_return[9:-5]
vol_array=vol1.values
momentum_array=momentum1.values
bb_value_array=bb_value1.values
data=np.concatenate((vol_array,momentum_array,bb_value_array,future_return1), axis=1)
predY = knn_learner(data)
predY_df_return=pd.DataFrame(predY,index=future_return1.index,columns=['predY'])
predY_df_price=prices1*(predY_df_return.values+1)
future_prices=prices1*(future_return1.values+1)
predY_df_price=predY_df_price.rename(columns={'ML4T-399':'Predicted Y'})
ax=predY_df_price.plot(title="Sine Data Training Y/Price/Predicted Y[2008-2009]")
future_prices = future_prices.rename(columns={'ML4T-399':'Training Y'})
prices1 = prices1.rename(columns={'ML4T-399':'Price'})
future_prices.plot(ax=ax)
prices1.plot(ax=ax)
plt.ylim((0,100))
start_date='2008-01-15'
end_date='2009-12-23'
print
print "Sine Data In Sample Statistics"
print
my_strategy_ML4T(prices1,predY_df_price,start_date,end_date,"ML4T_insample_orders","Sine Data In Sample Entries/Exits","Sine Data In Sample Backtest")
start_date='2010-01-01'
end_date='2010-12-31'
compute_outsample_data_ML4T(data,start_date,end_date)
def get_indicators(start_date, end_date, symbols):
"""Simulate and assess the performance of a stock portfolio."""
# Read in adjusted closing prices for given symbols, date range
dates = pd.date_range(start_date, end_date)
prices_all = get_data(symbols, dates) # automatically adds SPY
prices = prices_all[symbols] # only portfolio symbols
# prices_SPY = prices_all['SPY'] # only SPY, for comparison later
sym = symbols[1]
x1 = (prices[sym] - pd.rolling_mean(prices[sym], 20)) / (2 * pd.rolling_std(prices[sym], 20))
x1_dis = pd.cut(x1, 10, labels=False)
x2 = prices[sym].pct_change(20)
x2_dis = pd.cut(x2, 10, labels=False)
x3 = pd.rolling_std(prices[sym].pct_change(1), 20)
x3_dis = pd.cut(x3, 10, labels=False)
# return pd.concat([x1_,x2_0,x3_0], axis=1).dropna(), prices
tempdf = pd.concat([x1_dis, x2_dis, x3_dis], axis=1).dropna()
tempdf.columns = ["x1", "x2", "x3"]
print tempdf.dtypes
tempdf["holding"] = np.random.randint(0, 3, size=len(tempdf))
# 0 = no position , 1 = negative positin 2 =holding long
tempdf["s"] = 1000 * tempdf["holding"] + 100 * tempdf["x3"] + 10 * tempdf["x2"] + 1 * tempdf["x1"]
print tempdf.head(50)
return tempdf, prices
def generateTradeSigs(self, windowLength, entryScale, exitScale):
resid = self.results.resid
self.core_data['spread'] = self.results.resid
self.core_data['entryUpper'] = entryScale*pandas.rolling_std(resid,windowLength,min_periods=10)
self.core_data['entryLower'] = -entryScale*pandas.rolling_std(resid,windowLength,min_periods=10)
self.core_data['exitUpper'] = exitScale*pandas.rolling_std(resid,windowLength,min_periods=10)
self.core_data['exitLower'] = -exitScale*pandas.rolling_std(resid,windowLength,min_periods=10)
def _boll(df,periods=21,column=None,include=True,str=None,detail=False,output=None,**boll_kwargs):
study='BOLL'
df,_df,column=validate(df,column)
_df['SMA']=pd.rolling_mean(df[column],periods)
_df['UPPER']=_df['SMA']+pd.rolling_std(df[column],periods)*boll_std
_df['LOWER']=_df['SMA']-pd.rolling_std(df[column],periods)*boll_std
str=str if str else '{name}({period})'
return rename(df,_df,study,periods,column,include,str,detail,output=output)
def compute_insample_data_IBM(start_date, end_date):
dates = pd.date_range(start_date, end_date)
prices = get_data(["IBM"], dates, True)
prices = prices.drop("SPY", axis=1)
# calculating volatility
daily_returns = prices.copy()
daily_returns[1:] = (prices[1:] / prices[:-1].values) - 1.0
daily_returns.ix[0, :] = 0
vol = pd.rolling_std(daily_returns, window=10)
vol1 = vol[9:-5]
# calculating momentum
momentum = prices.copy()
momentum[9:] = (prices[9:] / prices[:-9].values) - 1.0
momentum1 = momentum[9:-5]
# calculating bollinger values
sma = pd.rolling_mean(prices, window=10)
sma1 = sma.dropna()
std = pd.rolling_std(prices, window=10)
std1 = std[9:]
bb_prices = prices[9:]
bb_value = (bb_prices - sma1) / (2 * std1)
bb_value1 = bb_value[0:-5]
# shifting prices
shifted_prices = prices.shift(-5)
future_prices = prices.copy()
prices1 = prices[9:-5]
future_return = (shifted_prices / prices) - 1.0
future_return1 = future_return[9:-5]
vol_array = vol1.values
momentum_array = momentum1.values
bb_value_array = bb_value1.values
data = np.concatenate((vol_array, momentum_array, bb_value_array, future_return1), axis=1)
predY = knn_learner(data)
predY_df_return = pd.DataFrame(predY, index=future_return1.index, columns=["predY"])
predY_df_price = prices1 * (predY_df_return.values + 1)
future_prices = prices1 * (future_return1.values + 1)
predY_df_price = predY_df_price.rename(columns={"IBM": "Predicted Y"})
ax = predY_df_price.plot(title="Training Y/Price/Predicted Y: 2008-2009")
future_prices = future_prices.rename(columns={"IBM": "Training Y"})
prices1 = prices1.rename(columns={"IBM": "Price"})
future_prices.plot(ax=ax)
prices1.plot(ax=ax)
start_date = "2008-01-15"
end_date = "2009-12-23"
my_strategy_IBM(prices1, predY_df_price, start_date, end_date, "IBM_insample_orders", "Strategy 2008-2009")
start_date = "2010-01-01"
end_date = "2010-12-31"
compute_outsample_data(data, start_date, end_date)
def compute_outsample_data_IBM(traindata,start_date,end_date):
dates=pd.date_range(start_date,end_date)
prices=get_data(['IBM'],dates,True)
prices=prices.drop('SPY',axis=1)
#calculating volatility
daily_returns = prices.copy()
daily_returns[1:] = (prices[1:]/prices[:-1].values)-1.0
daily_returns.ix[0,:]=0
vol=pd.rolling_std(daily_returns,window=10)
vol1=vol[9:-5]
#calculating momentum
momentum=prices.copy()
momentum[9:] = (prices[9:]/prices[:-9].values)-1.0
momentum1=momentum[9:-5]
#calculating bollinger bands value
sma = pd.rolling_mean(prices,window=10)
sma1 = sma.dropna()
std=pd.rolling_std(prices,window=10)
std1=std[9:]
bb_prices=prices[9:]
bb_value=(bb_prices - sma1)/(2*std1)
bb_value1=bb_value[0:-5]
#calculating 5 day shifted prices
shifted_prices=prices.shift(-5)
future_prices = prices.copy()
prices1=prices[9:-5]
future_return=(shifted_prices/prices) - 1.0
future_return1=future_return[9:-5]
vol_array=vol1.values
momentum_array=momentum1.values
bb_value_array=bb_value1.values
data=np.concatenate((vol_array,momentum_array,bb_value_array,future_return1), axis=1)
#getting predicted Y from knn
print
print "KNN Learner Statistics"
print
predY = knn_learner_test(traindata,data)
predY_df_return=pd.DataFrame(predY,index=future_return1.index,columns=['predY'])
predY_df_price=prices1*(predY_df_return.values+1)
future_prices=prices1*(future_return1.values+1)
predY_df_price=predY_df_price.rename(columns={'IBM':'Predicted Y'})
future_prices = future_prices.rename(columns={'IBM':'Training Y'})
prices1 = prices1.rename(columns={'IBM':'Price'})
start_date='2010-01-15'
end_date='2010-12-23'
print
print "IBM Out of Sample Satistics"
print
my_strategy_IBM(prices1,predY_df_price,start_date,end_date,"IBM_outsample_orders","IBM Data Out of Sample Entries/Exits","IBM Data Out of Sample Backtest")
def find_events(ls_symbols, d_data):
''' Finding the event dataframe '''
# Using actual close for better approximation
df_close = d_data['actual_close']
ts_market = df_close['SPY']
# Time stamps for the event range
ldt_timestamps = df_close.index
# Getting the numpy ndarray of close prices.
na_price = df_close
print na_price
# Getting rolling mean of prices
mean = pd.rolling_mean(na_price, 20)
meanspy = pd.rolling_mean(ts_market, 20)
# Getting the rolling std of prices
std = pd.rolling_std(na_price, 20)
stdspy = pd.rolling_std(ts_market, 20)
# Upper and lower band
upper = mean + std
lower = mean - std
# Bollinger value
bol = (na_price - mean) / std
boldf = pd.DataFrame(bol, ldt_timestamps)
bolspy = (ts_market - meanspy) / stdspy
bolspydf = pd.DataFrame(bolspy, ldt_timestamps)
print "Finding Events"
# Creating an empty dataframe
df_events = copy.deepcopy(boldf)
df_events = df_events * np.NAN
filename = open('orders_hw6.csv', 'wb')
writer = csv.writer(filename, delimiter=',')
for s_sym in ls_symbols:
for i in range(1, len(ldt_timestamps)-5):
# Calculating the returns for this timestamp
bol_today = boldf[s_sym].ix[ldt_timestamps[i]]
bol_yest = boldf[s_sym].ix[ldt_timestamps[i - 1]]
bolspy_today = bolspydf.ix[ldt_timestamps[i]]
# Event is found if the symbol drops from above $5 to below $5
if bol_today <= -2.0 and bol_yest >= -2.0 and bolspy_today >= 1.4:
writer.writerow([ldt_timestamps[i].year, ldt_timestamps[i].month, ldt_timestamps[i].day, s_sym, 'Buy', 100])
# Jump ahead to 5 days later
future = ldt_timestamps[i+5]
writer.writerow([future.year, future.month, future.day, s_sym, 'Sell', 100 ])
filename.close()
return df_events
def series_bollinger(series, window=20, sigma=1., plot=False):
mean = pd.rolling_mean(series, window=window)
std = pd.rolling_std(series, window=window)
df = pd.DataFrame({'value': series, 'mean': mean, 'upper': mean + sigma * std, 'lower': mean - sigma * std})
bollinger_values = (series - pd.rolling_mean(series, window=window)) / (pd.rolling_std(series, window=window))
if plot:
df.plot()
pd.DataFrame({'bollinger': bollinger_values}).plot()
plt.show()
return bollinger_values
def indicators(df, window=5):
rm = pd.rolling_mean(df, window=window)
rstd = pd.rolling_std(df, window=window)
bb = ((df-rm) / (2*rstd)).rename(columns={'IBM':'Bollinger band'})
momentum = (df / df.shift(window) - 1).rename(columns={'IBM':'Momentum'})
sma = (df / rm - 1).rename(columns={'IBM':'SMA'})
drets = df / df.shift(1) - 1
volatility = (pd.rolling_std(drets, window=window)).rename(columns={'IBM':'Volatility'})
ind_df = pd.concat([bb, momentum, sma, volatility], axis=1)
return ind_df
def enhance_data(df):
rolling_days = int(len(df) / 20)
df['Bol_upper'] = pd.rolling_mean(df['Adj Close'], window=rolling_days) + 2 * pd.rolling_std(df['Adj Close'], rolling_days, min_periods=rolling_days)
df['Bol_lower'] = pd.rolling_mean(df['Adj Close'], window=rolling_days) - 2 * pd.rolling_std(df['Adj Close'], rolling_days, min_periods=rolling_days)
rm_adjustedClose = pd.rolling_mean(df['Adj Close'].shift(), window=rolling_days)
df['Rolling'] = rm_adjustedClose #rm_adjustedClose.fillna(df['Adj Close'])
df = df[rolling_days:]
return df
def get_calc_data1(df): df = df df['15Std'] = pd.rolling_std(df['Return'],12) df['30Std'] = pd.rolling_std(df['Return'],22) df['45Std'] = pd.rolling_std(df['Return'],33) df['60Std'] = pd.rolling_std(df['Return'],44) df['15StdAnnual'] = df['15Std']*np.sqrt(261) df['30StdAnnual'] = df['30Std']*np.sqrt(261) df['45StdAnnual'] = df['45Std']*np.sqrt(261) df['60StdAnnual'] = df['60Std']*np.sqrt(261) return df
def plot_rolling_functions(series, window_size=128):
pd.rolling_median(series,window_size).plot(label='median')
pd.rolling_mean(series,window_size).plot(label='mean')
pd.rolling_std(series,window_size).plot(label='std')
pd.rolling_skew(series,window_size).plot(label='skew')
pd.rolling_kurt(series,window_size).plot(label='kurt')
pd.rolling_min(series,window_size).plot(label='min')
pd.rolling_max(series,window_size).plot(label='max')
plt.title('Various rolling window functions, window size %s' % (window_size))
plt.legend()
plt.show()
def get_calc_data(ticker): df = get_history_data(ticker) df['Return'] = df['Adj Close'].pct_change() df['15Std'] = pd.rolling_std(df['Return'],12) df['30Std'] = pd.rolling_std(df['Return'],22) df['45Std'] = pd.rolling_std(df['Return'],33) df['60Std'] = pd.rolling_std(df['Return'],44) df['15StdAnnual'] = df['15Std']*np.sqrt(261) df['30StdAnnual'] = df['30Std']*np.sqrt(261) df['45StdAnnual'] = df['45Std']*np.sqrt(261) df['60StdAnnual'] = df['60Std']*np.sqrt(261) return df
def _estMag(self, trigIndex, corSeries, MPcon, mags, events,
WFU, UtU, ewf, coef, times, name, sta):
"""
Estimate magnitudes by applying projected subspace mag estimates
and standard deviation mag estimates as outlined in Chambers et al.
2015.
"""
WFlen = np.shape(WFU)[1] # event waveform length
nc = corSeries.Nc # number of chans
# continuous data chunk that triggered subspace
ConDat = MPcon[trigIndex * nc:trigIndex * nc + WFlen]
if self.issubspace:
# continuous data chunk projected into subspace
ssCon = np.dot(UtU, ConDat)
# projected energy
proEn = np.var(ssCon) / np.var(WFU, axis=1)
# Try and estimate pre-event noise level (for estimating SNR)
if trigIndex * nc > 5 * WFlen: # take 5x waveform length before event
pe = MPcon[trigIndex * nc - 5 * WFlen: trigIndex * nc]
rollingstd = pd.rolling_std(pe, WFlen)[WFlen - 1:]
else: # if not enough data take 6 times after event
pe = MPcon[trigIndex * nc: trigIndex * nc + WFlen + 6 * WFlen]
rollingstd = pd.rolling_std(pe, WFlen)[WFlen - 1:]
baseNoise = np.median(rollingstd) # take median of std for noise level
SNR = np.std(ConDat) / baseNoise # estiamte SNR
# ensure mags are greater than -15, else assume no mag value for event
touse = mags > -15
if self.issubspace: # if subspace
if not any(touse): # if no defined magnitudes avaliable
msg = (('No magnitudes above -15 usable for detection at %s on'
' station %s and %s') % (times, sta, name))
detex.log(__name__, msg, level='warn')
return np.NaN, np.Nan, SNR
else:
# correlation coefs between each event and data block
ecor = [fast_normcorr(x, ConDat)[0] for x in ewf]
eventCors = np.array(ecor)
projectedEnergyMags = _estPEMag(mags, proEn, eventCors, touse)
stdMags = _estSTDMag(mags, ConDat, ewf, eventCors, touse)
else: # if singleton
assert len(mags) == 1
if np.isnan(mags[0]) or mags[0] < -15:
projectedEnergyMags = np.NaN
stdMags = np.NaN
else:
# use simple waveform scaling if single
d1 = np.dot(ConDat, WFU[0])
d2 = np.dot(WFU[0], WFU[0])
projectedEnergyMags = mags[0] + d1 / d2
stdMags = mags[0] + np.log10(np.std(ConDat) / np.std(WFU[0]))
return projectedEnergyMags, stdMags, SNR
def min_anal(name):
df = pd.DataFrame.from_csv(name)
df['time'] = pd.to_datetime(df['time'])
df['price'] = (df.low + df.high)/2
df = df.dropna()
maxtime = df['time'].max()
mintime = df['time'].min()
time_delta = maxtime-mintime
if time_delta.days > 0:
days = time_delta.days
else:
days = 1
time_range = [1,120,480,1000,2000]
fee = .0025
test_range =[.1,2,5,10,20]
print ('Check text file for ouptut')
f = open('cb_analysis.txt', 'w')
for j in test_range:
for i in time_range:
n1 = 'p_delta_'+ str(i)
n2 = 'ma_'+str(i)
n3='ma_diff_'+str(i)
n4='buy_vol_'+str(i)
n8='sell_vol_'+str(i)
n5 = 'volatility'+str(i)
n6 = 'u_thresh'+str(i)+str(j)
n7 = 'l_thresh'+str(i)+str(j)
df[n1] = df.price - df.price.shift(i)
df[n2] = pd.rolling_mean(df.price,i)
df[n3] = df.price -df[n2]
df['squared']=df[n1]*df[n1]
df[n5]=pd.rolling_mean(df['squared'],i)
df[n6]=df[n2] + pd.rolling_std(df.price,i)*j
df[n7]=df[n5] - pd.rolling_std(df.price,i)*j
df['fee']=df['price']*fee
df[n4] = df.apply(lambda x : x['p_delta_1'] if x['price'] > x[n6] else -1*x['p_delta_1'] if x['price'] < x[n7] else 0,axis = 1)
df[n8] = df.apply(lambda x : x['p_delta_1'] if x['price'] < x[n6] else -1*x['p_delta_1'] if x['price'] > x[n7] else 0,axis = 1)
df['dummy_n4'] = df.apply(lambda x : 1 if x['price'] > x[n6] or x['price'] < x[n7] else 0,axis = 1)
df['dummy2'] =abs(df['dummy_n4']-df['dummy_n4'].shift(1))
df['cost'] = df.apply(lambda x : x['fee'] if x['dummy2'] > 0 else 0,axis = 1)
prof_buyvol = df[n4].sum(axis=0)
prof_sellvol = df[n8].sum(axis=0)
scount = df['dummy_n4'].sum(axis=0)
cost = df['cost'].sum(axis=0)
Adj_prof_b = prof_buyvol - cost
Adj_prof_s = prof_sellvol - cost
output= ' '.join(['Test Range ', str(j),' Profit: ',str(i),': buy:',str(prof_buyvol),'sell:',str(prof_sellvol),' | Days:',str(days), ' Adj_prof: buy: ',str(Adj_prof_b),'sell: ',str(Adj_prof_s) , 'tradecount ',str(scount)])
f.write(output)
print (output)
print ('Obs:',len(df))
print (df.head())
f.close()
def find_events(ls_symbols, d_data):
''' Finding the event dataframe '''
df_close = d_data['close']
ts_market = df_close['SPY']
print "Finding Events"
# Creating an empty dataframe
df_events = copy.deepcopy(df_close)
df_events = df_events * np.NAN
# Time stamps for the event range
ldt_timestamps = df_close.index
spy_bollinger = (df_close['SPY'] - pd.rolling_mean(df_close['SPY'],20)) / pd.rolling_std(df_close['SPY'],20)
for s_sym in ls_symbols:
#if( s_sym == 'GOOG' ):
# print pd.rolling_mean(df_close[s_sym], 20)
#if( s_sym == 'SPY' ):
# spy_bollinger = (df_close[s_sym] - pd.rolling_mean(df_close[s_sym],20)) / pd.rolling_std(df_close[s_sym],20)
#else:
# equity_bollinger = (df_close[s_sym] - pd.rolling_mean(df_close[s_sym],20)) / pd.rolling_std(df_close[s_sym],20)
equity_bollinger = (df_close[s_sym] - pd.rolling_mean(df_close[s_sym],20)) / pd.rolling_std(df_close[s_sym],20)
for i in range(1, len(ldt_timestamps)):
# Calculating the returns for this timestamp
# f_symprice_today = df_close[s_sym].ix[ldt_timestamps[i]]
# f_symprice_yest = df_close[s_sym].ix[ldt_timestamps[i - 1]]
# f_marketprice_today = ts_market.ix[ldt_timestamps[i]]
# f_marketprice_yest = ts_market.ix[ldt_timestamps[i - 1]]
# f_symreturn_today = (f_symprice_today / f_symprice_yest) - 1
# f_marketreturn_today = (f_marketprice_today / f_marketprice_yest) - 1
# Event is found if the symbol is down more then 3% while the
# market is up more then 2%
#f_equity_today = (d_data['close'] - pd.rolling_mean(d_data['close'], int(sys.argv[4]))) / pd.rolling_std(d_data['close'], int(sys.argv[4]))
equity_bollinger_today = equity_bollinger.loc[ldt_timestamps[i]]
equity_bollinger_yest = equity_bollinger.loc[ldt_timestamps[i-1]]
spy_bollinger_today = spy_bollinger.loc[ldt_timestamps[i]]
if equity_bollinger_today < -2.0 and equity_bollinger_yest >= -2.0 and spy_bollinger_today >= 1.5:
buy_data = [ldt_timestamps[i].year, ldt_timestamps[i].month, ldt_timestamps[i].day, s_sym, 'Buy', 100]
#print buy_data
write_csv(buy_data)
sell_date = du.getNYSEoffset(ldt_timestamps[i],5)
sell_data = [sell_date.year, sell_date.month, sell_date.day, s_sym, 'Sell', 100]
#print sell_data
write_csv(sell_data)
#df_events[s_sym].ix[ldt_timestamps[i]] = 1
return df_events
def get_calc_data(self): df = self.df df['Return'] = df['Close'].pct_change() df = df[df['Return']>(-0.25)] df['15Std'] = pd.rolling_std(df['Return'],12,min_periods=10) df['30Std'] = pd.rolling_std(df['Return'],22,min_periods=20) df['45Std'] = pd.rolling_std(df['Return'],33,min_periods = 30) df['60Std'] = pd.rolling_std(df['Return'],44,min_periods = 40) df['15StdAnnual'] = df['15Std']*np.sqrt(261) df['30StdAnnual'] = df['30Std']*np.sqrt(261) df['45StdAnnual'] = df['45Std']*np.sqrt(261) df['60StdAnnual'] = df['60Std']*np.sqrt(261) return df
def bollinger_chart(timestamps, data, lookback=20):
sma = pd.rolling_mean(data, lookback)
upper = sma + pd.rolling_std(data, lookback)
lower = sma - pd.rolling_std(data, lookback)
plt.plot(ldt_timestamps, data, label=ls_symbols[0])
plt.plot(ldt_timestamps, sma, label="Moving AVG")
plt.plot(ldt_timestamps, upper, label="upper band")
plt.plot(ldt_timestamps, lower, label="lower band")
plt.legend(loc=3)
plt.ylabel('Returns')
plt.xlabel('Date')
plt.savefig('homework5_bollinger.pdf', format='pdf')
def main():
start = datetime.datetime(1940, 1, 1)
end = datetime.datetime.now()
tickers = sys.argv[1:] # command line arguments
for ticker in tickers:
tick_df = util.get_returns(ticker, start, end)
tick_df['Standard Deviation (60d)'] = pd.rolling_std(tick_df['Returns'], window=60)
tick_df['Standard Deviation (200d)'] = pd.rolling_std(tick_df['Returns'], window=200)
print(ticker + " Standard Deviation")
print(np.std(tick_df['Returns']))
tick_df.to_csv("%s.csv" % ticker)
def bollinger_band(self, tick, window=20, k=2, mi_only=False):
"""
Return four arrays for Bollinger Band.
The first one is the moving average.
The second one is the upper band.
The thrid one is the lower band.
The fourth one is the Bollinger value.
If mi_only, then return the moving average only.
"""
ldt_timestamps = self.ldt_timestamps()
pre_timestamps = ut.pre_timestamps(ldt_timestamps, window)
# ldf_data has the data prior to our current interest.
# This is used to calculate moving average for the first window.
ldf_data = ut.get_tickdata([tick], pre_timestamps)
merged_data = pd.concat([ldf_data[tick]['close'], self['close']])
bo = dict()
bo['mi'] = pd.rolling_mean(merged_data, window=window)[ldt_timestamps]
if mi_only:
return bo['mi']
else:
sigma = pd.rolling_std(merged_data, window=window)
bo['hi'] = bo['mi'] + k * sigma[ldt_timestamps]
bo['lo'] = bo['mi'] - k * sigma[ldt_timestamps]
bo['ba'] = (merged_data[ldt_timestamps] - bo['mi']) / (k * sigma[ldt_timestamps])
return bo
def test_pairs_md():
#prepare data
import DataHandler.DBReader as dbr
dbpath = "/home/phcostello/Documents/Data/FinanceData.sqlite"
dbreader = dbr.DBReader(dbpath)
SP500 = dbreader.readSeries("SP500")
BA = dbreader.readSeries("BA")
dim = 'Adj_Close'
SP500AdCl = SP500[dim]
BAAdCl = BA[dim]
dataObj = pd.merge(pd.DataFrame(BAAdCl), pd.DataFrame(SP500AdCl), how='inner',left_index = True, right_index = True)
dataObj.columns = ['y','x']
pmd = pairs_md(dataOb=dataObj,xInd='x',yInd='y')
pmd.fitOLS()
print pmd.results.params
resid = pmd.results.resid
resid.plot()
rllstd = pd.rolling_std(resid,100,min_periods=10)
rllstd.plot()
#plt.show()
#pmd.printSummary()
#print pmd.adfResids()
scale = pmd.results.params['x']
intercept = pmd.results.params['const']
reg_params = [intercept, scale]
pmd.generateTradeSigs(50,
entryScale=1.5,
exitScale=0,
ewma_par = 200,
reg_params=reg_params)
pmd.plot_spreadAndSignals()
plt.show()
def plot_rolling_auto_home(df_attack=None,df_defence=None, window=5, nstd=1,
detected_events_home=None,
detected_events_away=None, sky_events=None):
sns.set_context("notebook", font_scale=1.8 ,rc={"lines.linewidth": 3.5, "figure.figsize":(18,12) })
plt.subplots_adjust(bottom=0.85)
mean = pd.rolling_mean(df_attack, center=True, window=window)
std = pd.rolling_std(df_attack, center=True, window=window)
detected_plot_extrema = df_attack.ix[argrelextrema(df_attack.values, np.greater)]
df_filt_noise = df_attack[(df_attack > mean-std) & (df_attack < mean+std)]
df_filt_noise = df_filt_noise.ix[detected_plot_extrema.index].dropna()
df_filt_keep = df_attack[~((df_attack > mean-std) & (df_attack < mean+std))]
df_filt_keep = df_filt_keep.ix[detected_plot_extrema.index].dropna()
plt.plot(df_attack, color='#4CA64C', label='{} Attack'.format(all_matches[0]['home_team'].title()))
plt.fill_between(df_attack.index, (mean-nstd*std), (mean+nstd*std), interpolate=False, alpha=0.4, color='#B2B2B2', label='$\mu + {} \\times \sigma$'.format(nstd))
plt.scatter(df_filt_keep.index, df_filt_keep.values, marker='*', s=120, color='#000000', zorder=10, label='Selected maxima post-filtering')
plt.scatter(df_filt_noise.index, df_filt_noise.values, marker='x', s=120, color='#000000', zorder=10, label='Unselected maxima post-filtering')
df_defence.apply(lambda x: -1*x).plot(color='#000000', label='{} Defence'.format(all_matches[0]['home_team'].title()))
if(len(detected_events_home) > 0):
classifier_events_df_home= pd.DataFrame(detected_events_home)
classifier_events_df_home[classifier_events_df_home.category == 'GOAL']
if(len(detected_events_away) > 0):
classifier_events_df_away= pd.DataFrame(detected_events_away)
classifier_events_df_away[classifier_events_df_away.category == 'GOAL']
font0 = FontProperties(family='arial', weight='bold',style='italic', size=16)
for i, row in classifier_events_df_home.iterrows():
if row.category == 'OTHER':
continue
plt.text(row.event, df_attack.max(), "{} {} {}".format(all_matches[0]['home_team'].upper(), row.category, row.event), rotation='vertical', color='black', bbox=dict(facecolor='green', alpha=0.2))#, transform=transform)
for i, row in classifier_events_df_away.iterrows():
if row.category == 'OTHER':
continue
plt.text(row.event, (df_attack.max()), "{} {} {}".format(all_matches[0]['away_team'].upper(), row.category, row.event), rotation='vertical', color='black', bbox=dict(facecolor='red', alpha=0.2))
high_peak_position = 0;
if(df_attack.max() > df_defence.max()): high_peak_position = -(df_defence.max() * 2.0)
else: high_peak_position = -(df_defence.max() * 1.25)
# Functionality to include Sky Sports text commentary updates on plot for goal events.
# for i, row in pd.DataFrame(sky_events).iterrows():
# dedented_text = textwrap.dedent(row.text).strip()
# plt.text(row.event, high_peak_position, "@SkySports {} AT {}:\n{}:\n{}".format(row.category, row.event.time(), row.title, textwrap.fill(dedented_text, width=40)), color='black', bbox=dict(facecolor='blue', alpha=0.2))
plt.legend(loc=4)
ax = plt.gca()
label = ax.set_xlabel('time')
plt.ylabel('Tweet frequency')
plt.title('{} vs. {} (WK {}) - rolling averages window={} mins'.format(all_matches[0]['home_team'].title(), all_matches[0]['away_team'].title(), all_matches[0]['dbname'], window))
plt.savefig('{}attack_{}_plain.pdf'.format(all_matches[0]['home_team'].upper(), all_matches[0]['away_team'].upper()))
return detected_plot_extrema
def bollinger_band(self, tick, window=20, k=2, nml=False, mi_only=False):
"""
Return four arrays for Bollinger Band.
The first one is the moving average.
The second one is the upper band.
The thrid one is the lower band.
The fourth one is the Bollinger value.
If mi_only, then return the moving average only.
"""
ldt_timestamps = self.index
dt_timeofday = dt.timedelta(hours=16)
days_delta = dt.timedelta(days=(np.ceil(window*7/5)+5))
dt_start = ldt_timestamps[0] - days_delta
dt_end = ldt_timestamps[0] - dt.timedelta(days=1)
pre_timestamps = du.getNYSEdays(dt_start, dt_end, dt_timeofday)
# ldf_data has the data prior to our current interest.
# This is used to calculate moving average for the first window.
ldf_data = ut.get_tickdata([tick], pre_timestamps)
if nml:
ma_data = pd.concat([ldf_data[tick]['nml_close'], self['nml_close']])
else:
ma_data = pd.concat([ldf_data[tick]['close'], self['close']])
bo = dict()
bo['mi'] = pd.rolling_mean(ma_data, window=window)[ldt_timestamps]
if mi_only:
return bo['mi']
else:
sigma = pd.rolling_std(ma_data, window=window)
bo['up'] = bo['mi'] + k * sigma[ldt_timestamps]
bo['lo'] = bo['mi'] - k * sigma[ldt_timestamps]
bo['ba'] = (ma_data[ldt_timestamps] - bo['mi']) / (k * sigma[ldt_timestamps])
return bo
print('expire_date',expire_date )
delta = expire_date - today_date
print(delta.days)
daysbackfar = delta.days
import builddataframeofrefdateminusd2tod1stockpricechanges
df_0 = builddataframeofrefdateminusd2tod1stockpricechanges.perform(symbol,numberofweekstolookback,daysbackmid,daysbackfar,showresults).DataFrameResult
# ==========
print(df_0)
# ==========
## ################################################################################################################
import pandas as pd
df_std = pd.rolling_std(df_0[['DrawDownPctChange', 'DrawUpPctChange']], RollingNumberOfPeriods)
df_mean = pd.rolling_mean(df_0[['DrawDownPctChange', 'DrawUpPctChange']], RollingNumberOfPeriods)
#print('rolling std xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx')
#print(df_std)
#print('rolling mean xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx')
#print(df_mean)
#print('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx')
#print('testing test_builddataframeofrefdateminusd2tod1stockpricechanges.py')
#print('df_0 Column Names',df_0)
print('df_0 Column Names',df_0.columns)
print('Number Of Observations',len(df_0.index))
#df_prices = df_0['pRef']
#print(df_prices)
def fancy_this(data):
return mean(data) + mean(data) + 5
style.use('fivethirtyeight')
start = datetime.datetime(2005, 1, 1)
end = datetime.datetime(2015, 1, 1)
att = web.DataReader("T", 'yahoo', start, end)
describe = att.describe()
#print(describe['Open']['std'])
att['50MA'] = pd.rolling_mean(att['Close'], 50)
att['10MA'] = pd.rolling_mean(att['Close'], 10)
att['50STD'] = pd.rolling_std(att['Close'], 50)
att['MA_with_apply'] = pd.rolling_apply(att['Close'], 50, moving_average)
att['apply'] = pd.rolling_apply(att['Close'], 50, fancy_this)
print(att.head())
#att.dropna(inplace=True)
#att.dropna(how='all', inplace=True)
#att.fillna(method='bfill', inplace=True)
#att.fillna(value=-99999, inplace=True)
one_pct = len(att) * 0.01
att.fillna(value=-99999, inplace=True, limit=one_pct)
def get_data(self, df):
fd = self.form_data
df = df.fillna(0)
if fd.get("granularity") == "all":
raise Exception("Pick a time granularity for your time series")
df = df.pivot_table(index=DTTM_ALIAS,
columns=fd.get('groupby'),
values=fd.get('metrics'))
fm = fd.get("resample_fillmethod")
if not fm:
fm = None
how = fd.get("resample_how")
rule = fd.get("resample_rule")
if how and rule:
df = df.resample(rule, how=how, fill_method=fm)
if not fm:
df = df.fillna(0)
if self.sort_series:
dfs = df.sum()
dfs.sort_values(ascending=False, inplace=True)
df = df[dfs.index]
if fd.get("contribution"):
dft = df.T
df = (dft / dft.sum()).T
rolling_periods = fd.get("rolling_periods")
rolling_type = fd.get("rolling_type")
if rolling_type in ('mean', 'std', 'sum') and rolling_periods:
if rolling_type == 'mean':
df = pd.rolling_mean(df, int(rolling_periods), min_periods=0)
elif rolling_type == 'std':
df = pd.rolling_std(df, int(rolling_periods), min_periods=0)
elif rolling_type == 'sum':
df = pd.rolling_sum(df, int(rolling_periods), min_periods=0)
elif rolling_type == 'cumsum':
df = df.cumsum()
num_period_compare = fd.get("num_period_compare")
if num_period_compare:
num_period_compare = int(num_period_compare)
prt = fd.get('period_ratio_type')
if prt and prt == 'growth':
df = (df / df.shift(num_period_compare)) - 1
elif prt and prt == 'value':
df = df - df.shift(num_period_compare)
else:
df = df / df.shift(num_period_compare)
df = df[num_period_compare:]
chart_data = self.to_series(df)
time_compare = fd.get('time_compare')
if time_compare:
query_object = self.query_obj()
delta = utils.parse_human_timedelta(time_compare)
query_object['inner_from_dttm'] = query_object['from_dttm']
query_object['inner_to_dttm'] = query_object['to_dttm']
query_object['from_dttm'] -= delta
query_object['to_dttm'] -= delta
df2 = self.get_df(query_object)
df2[DTTM_ALIAS] += delta
df2 = df2.pivot_table(index=DTTM_ALIAS,
columns=fd.get('groupby'),
values=fd.get('metrics'))
chart_data += self.to_series(df2,
classed='superset',
title_suffix="---")
chart_data = sorted(chart_data, key=lambda x: x['key'])
return chart_data
def ma(data,column,type,name='MA_'):
for i in type:
data[name+str(i)]=pd.rolling_mean(data[column],i)
data[name+'_std_'+str(i)]=pd.rolling_std(data[column],i)
def addEvidence(self, symbol = "IBM", \
sd=dt.datetime(2008,1,1), \
ed=dt.datetime(2009,1,1), \
sv = 10000):
#self.learner = ql.QLearner()
syms = [symbol]
dates = pd.date_range(sd, ed)
prices_all = ut.get_data(syms, dates) # automatically adds SPY
prices = prices_all[syms] # only portfolio symbols
prices_SPY = prices_all['SPY'] # only SPY, for comparison later
start_val = sv
if self.verbose: print prices
# Calculate indicators
# Momentum indicator
window = int(21)
momentum = []
price_array = pd.DataFrame.as_matrix(prices)
for i in range(prices.shape[0]):
if i > window:
momentum_value = (price_array[i] /
price_array[i - window]) - 1.0
momentum.append(momentum_value)
pmomentum_value = prices.copy()
pmomentum_value[:] = 0.0
norm_momentum = (momentum - np.mean(momentum)) / np.std(momentum)
pmomentum_value.ix[window + 1:, :] = norm_momentum[:] #[0]
pmomentum_value.ix[:window, :] = norm_momentum[20][0]
# Bollinger Bands
rolling_std = pd.rolling_std(prices, window=21)
rolling_avg = pd.rolling_mean(prices, window=21)
bb_upper = rolling_avg + 2.0 * rolling_std
bb_lower = rolling_avg - 2.0 * rolling_std
bbp_band = (prices - bb_lower) / (bb_upper - bb_lower)
normed_bbp = (bbp_band - np.mean(bbp_band)) / np.std(bbp_band)
normed_bbp.ix[:21, :] = pd.DataFrame.as_matrix(normed_bbp)[20]
# SMA/Price indicator
sma = rolling_avg / prices
normed_sma = (sma - np.mean(sma)) / np.std(sma)
normed_sma.ix[:21, :] = pd.DataFrame.as_matrix(normed_sma)[20]
# Discritize indicators
nsteps = 4
momen_values = pmomentum_value.values #.ix[21:,:].values
sma_values = normed_sma.values #.ix[21:,:].values
bbp_values = normed_bbp.values # ix[21:,:].values
momen_array = np.zeros(len(momen_values))
sma_array = np.zeros(len(sma_values))
bbp_array = np.zeros(len(sma_values))
for i in range(len(momen_values)):
momen_array[i] = momen_values[i]
sma_array[i] = sma_values[i]
bbp_array[i] = bbp_values[i]
mout, self.mbins = pd.qcut(momen_array, nsteps, retbins=True)
sout, self.sbins = pd.qcut(sma_array, nsteps, retbins=True)
bout, self.bbins = pd.qcut(bbp_array, nsteps, retbins=True)
momen_states = pd.cut(momen_array,
bins=self.mbins,
labels=False,
include_lowest=True)
sma_states = pd.cut(sma_array,
bins=self.sbins,
labels=False,
include_lowest=True)
bbp_states = pd.cut(bbp_array,
bins=self.bbins,
labels=False,
include_lowest=True)
converged = False
total_states = nsteps**3
count = 0
check_conv = 0.0
#print "Total number of states is:", total_states
# Initialize QLearner, SET rar to zero
self.learner = ql.QLearner(num_states=total_states,\
num_actions = 3, \
alpha = 0.5, \
gamma = 0.9, \
rar = 0.0, \
radr = 0.0, \
dyna = 0, \
verbose=False) #initialize the learner
while (not converged) and (count < 20):
# Set first state to the first data point (first day)
result = 0
indicators = [momen_states[0], sma_states[0], bbp_states[0]]
for kl in range(len(indicators)):
result = result + indicators[kl] * (nsteps**kl)
first_state = result
action = self.learner.querysetstate(first_state)
total_reward = 0
df_prices = prices.copy()
df_prices['Cash'] = pd.Series(1.0, index=prices.index)
df_trades = df_prices.copy()
df_trades[:] = 0.0
indices = df_prices.index
holdings = 0
# Cycle through dates
for j in range(1, pmomentum_value.shape[0]):
daily_ret = 0.0
result = 0
indicators = [momen_states[j], sma_states[j], bbp_states[j]]
for k in range(len(indicators)):
result = result + indicators[k] * (nsteps**k)
new_combined_state = result
# Calculate reward for previous action
if j == 0:
reward = 0
else:
daily_ret = ((price_array[j] - price_array[j - 1]) /
price_array[j]) * 100.0
if holdings == 0:
reward = 0
else:
if action == 0:
reward = daily_ret
elif action == 1:
reward = -1.0 * daily_ret
else:
reward = -10
# Query learner with current state and reward to get action
action = self.learner.query(new_combined_state, reward)
# Implement action returned by learner and update portfolio
rDates = indices[j]
if action == 0:
if holdings == -200:
# Buy if holdings permit
df_trades['Cash'].ix[rDates] = df_trades['Cash'].ix[
rDates] + 400.0 * df_prices[
syms[0]].ix[rDates] * -1.0
df_trades[syms[0]].ix[rDates] = df_trades[
syms[0]].ix[rDates] + 400.0
holdings = holdings + 400.0
if holdings == 0:
df_trades['Cash'].ix[rDates] = df_trades['Cash'].ix[
rDates] + 200.0 * df_prices[
syms[0]].ix[rDates] * -1.0
df_trades[syms[0]].ix[rDates] = df_trades[
syms[0]].ix[rDates] + 200.0
holdings = holdings + 200.0
if action == 1:
if holdings == 200:
df_trades['Cash'].ix[rDates] = df_trades['Cash'].ix[
rDates] + 400.0 * df_prices[syms[0]].ix[rDates]
df_trades[syms[0]].ix[rDates] = df_trades[
syms[0]].ix[rDates] - 400.0
holdings = holdings - 400.0
if holdings == 0:
df_trades['Cash'].ix[rDates] = df_trades['Cash'].ix[
rDates] + 200.0 * df_prices[syms[0]].ix[rDates]
df_trades[syms[0]].ix[rDates] = df_trades[
syms[0]].ix[rDates] - 200.0
holdings = holdings - 200.0
total_reward += reward
df_holdings = df_trades.copy()
df_holdings[:] = 0.0
df_values = df_holdings.copy()
# Populate holdings dataframe, calculate value of holdings and portfolio value
df_holdings['Cash'].ix[indices[0]] = start_val
df_holdings.ix[indices[0]] = df_holdings.ix[
indices[0]] + df_trades.ix[indices[0]]
for i in range(1, len(indices)):
hDates = indices[i - 1]
rDates = indices[i]
df_holdings.ix[
rDates] = df_holdings.ix[hDates] + df_trades.ix[rDates]
df_values = df_holdings * df_prices
df_port_val = df_values.sum(axis=1)
period_end = df_port_val.ix[-1]
commul = df_port_val.pct_change()
cum_ret = (period_end - start_val) / start_val
count += 1
if abs((check_conv - cum_ret) * 100.0) < 0.00001:
converged = True
else:
check_conv = cum_ret
return df_trades
def get_rolling_std(df, window=20):
df["Std Dev"] = pd.rolling_std(df["Adj Close"], window=window)
return df
def stddev(df, n):
df = df.join(
pd.Series(pd.rolling_std(df['Close'], n), name='STD_' + str(n)))
return df
def cci(df, n):
pp = (df['High'] + df['Low'] + df['Close']) / 3
cci = pd.Series((pp - pd.rolling_mean(pp, n)) / pd.rolling_std(pp, n),
name='CCI_' + str(n))
df = df.join(cci)
return df
weight_tot = [
] #list of the total weighted trajectories for each gps point
for t in range(0, len(right_weight_tot)):
weight_tot.append(
left_weight_tot[t] +
right_weight_tot[t]) #just combine left and right trajectories
full_trajectory = [
] #weight each trajectory, with highest confidence for the points near the gps point
for t in range(0, gps.size):
for x in range(0, len(weight_tot)):
if x <= round(
n *
t): # this is weighting for points to right of gps point
w1 = (len(weight_tot) - 1 - x) / (len(weight_tot) - round(
n * t) - 1) #stuff to the left of gps point
full_trajectory.append(w1 * weight_tot[t][x])
else:
w2 = x / (round(n * t) - 1)
full_trajectory.append(w2 * weight_tot[t][x])
return full_trajectory
#%%
VZ = boundary(AZ)
SZ = fullboundary((VZ + [66.19] * VZ.size), H)
plt.plot(SZ)
error = pd.rolling_std(full_trajectory, n)
def get_bollinger(df_col, w = 20, s = 2): bandwidth = pd.rolling_std(df_col, window = w) upper = get_sma(df_col, w) + bandwidth*s lower = get_sma(df_col, w) - bandwidth*s return upper, lower
density._compute_covariance()
plt.plot(xs, density(xs))
plt.show()
### DELETE OUTLIERS
delete = np.concatenate([
np.where(resid9 < np.mean(resid9) - 2 * np.std(resid9))[0],
np.where(resid9 > np.mean(resid9) + 2 * np.std(resid9))[0]
])
train0 = np.delete(np.array(dataframe.ix[:, 4]), delete)
train = np.sqrt(train0)
rollmean = pd.rolling_mean(train, window=20)
rollstd = pd.rolling_std(train, window=20)
ts_log0 = np.log(train)
ts_log = pd.DataFrame(ts_log0).dropna()
decomposition = seasonal_decompose(np.array(ts_log).reshape(len(ts_log), ),
freq=100)
trend = decomposition.trend
seasonal = decomposition.seasonal
residual = decomposition.resid
z = np.where(seasonal == min(seasonal))[0]
period = z[2] - z[1]
look_back = period
def rolling_hist_vol(self, days, end_date=None):
if end_date:
data = self.returns[:end_date]
else:
data = self.returns
return pd.rolling_std(data, window=days) * math.sqrt(TRADING_DAYS)
# If more buys, then label the var as 'buy'
for j in range(0, len(d_price_diff)):
if buy_sell.buy[j] - buy_sell.sell[j] > 0:
buy_sell.loc[j, 'label'] = 'buy'
else:
buy_sell.loc[j, 'label'] = 'sell'
# caculate VPIN
#buy_sell_vspread=buy_sell.sell-buy_sell.buy
buy_sell_vspread = abs(buy_sell.sell - buy_sell.buy)
#vpin=np.nansum(abs(buy_sell.buy-buy_sell.sell))/np.nansum(buy_sell.total)
rolling_size = 1 # rolling wondow size in terms of how many volume bars
vpin = list(
pd.rolling_sum((buy_sell_vspread), rolling_size) /
pd.rolling_sum(buy_sell.total, rolling_size))
rstd = pd.rolling_std(last_price, rolling_size)
## calculate the cdf vpin
vpin_selected = vpin[0:vpin.index(max(vpin)) + 1]
def cdf(x):
miu = np.mean(vpin_selected)
sigma = np.std(vpin_selected)
cdf = 0.5 * (1 + sp.erf((x - miu) / (2**0.5) / sigma))
return (cdf)
cdf_vpin = [cdf(i) for i in vpin_selected]
#sorted_vpin=np.sort(vpin_selected)
DAX[['Close', 'Ret_loop', 'Return']].tail()
del DAX['Ret_loop']
DAX[['Close', 'Return']].plot(subplots=True, style='b', figsize=(8, 5))
import pandas as pd
DAX['42d'] = pd.rolling_mean(DAX['Close'], window=43)
DAX['252d'] = pd.rolling_mean(DAX['Close'], window=252)
DAX[['Close', '42d', '252d']].tail()
DAX[['Close', '42d', '252d']].plot(figsize=(8, 5))
import math
DAX['Mov_Vol'] = pd.rolling_std(DAX['Return'], window=252) * math.sqrt(252)
DAX[['Close', 'Mov_Vol', 'Return']].plot(subplots=True, style='b', figsize=(8, 7))
import pandas as pd
from urllib import urlretrieve
es_url = 'http://www.stoxx.com/download/historical_values/hbrbcpe.txt'
vs_url = 'http://www.stoxx.com/download/historical_values/h_vstoxx.txt'
urlretrieve(es_url, './data/es.txt')
urlretrieve(vs_url, './data/vs.txt')
!ls -o ./data/*.txt # 创建新文件夹
lines = open('./data/es.txt', 'r').readlines()
lines = [line.replace(' ', '') for line in lines]
# ###########################
# Get VIX or comparable stock
import pullprices
d_comparestockpricehistory = pullprices.stockhistorybackfilledtodatframeofstockhistoryinstances(comparesym, startdatecalculated_string, str(today_date))
compare_stock_price = pullprices.stock(mycomparesym)
print('runtime_delta',datetime.datetime.today() - today_datetime)
#print(f.loc[f.index == '2015-06-18'])
#f1 = f.loc[f.index.isin(['2015-06-18'])][['Adj Close']]
# ################################################################################################################
# performs some general statistics
import pandas as pd
d_std = pd.rolling_std(d_stockpricechanges[['DrawDownPctChange', 'DrawUpPctChange']], RollingNumberOfPeriods)
d_mean = pd.rolling_mean(d_stockpricechanges[['DrawDownPctChange', 'DrawUpPctChange']], RollingNumberOfPeriods)
# #########################################################################
# Adds a column to dataframe to Compare data to something (VIX for example)
d_stockpricechanges['comppratfar'] = float('NaN')
d_stockpricechanges['breachedaboveorbelow'] = int(0)
#print(d_stockpricechanges)
# #######################################################################################
# Counts number of observations that hit above and below threshold during trading period
idrawbeyond_upabove = 0
idrawbeyond_downbelow = 0
#icountfartomidbeyond_above = 0
#icountfartomidbeyond_below = 0
std = plt.plot(rolstd, color='black', label='Rolling Std')
plt.legend(loc='best')
plt.title('Rolling Mean & Standard Deviation')
plt.show(block=False)
# Perform Dickey-Fuller test:
'Results of Dickey-Fuller Test:'
dftest = adfuller(timeseries, autolag='AIC')
dfoutput = pd.Series(dftest[0:4], index=['Test Statistic', 'p-value', '#Lags Used', 'Number of Observations Used'])
for key, value in dftest[4].items():
dfoutput['Critical Value (%s)' % key] = value
print(dfoutput)
rolmean = pd.rolling_mean(subset['value'], window=12)
rolstd = pd.rolling_std(subset['value'], window=12)
# Plot rolling statistics:
orig = plt.plot(subset['value'], color='blue', label='Original')
mean = plt.plot(rolmean, color='red', label='Rolling Mean')
std = plt.plot(rolstd, color='black', label='Rolling Std')
plt.legend(loc='best')
plt.title('Rolling Mean & Standard Deviation')
plt.show(block=False)
subset_log = np.log(subset['value'])
moving_avg = pd.rolling_mean(subset_log,2)
plt.plot(subset_log,color = 'blue')
plt.plot(moving_avg, color='red')
plt.show()
def testPolicy(self, symbol = "IBM", \
sd=dt.datetime(2009,1,1), \
ed=dt.datetime(2010,1,1), \
sv = 10000):
# here we build a fake set of trades
# your code should return the same sort of data
dates = pd.date_range(sd, ed)
syms = [symbol]
prices_all = ut.get_data([symbol], dates) # automatically adds SPY
prices = prices_all[syms] # only portfolio symbols
prices_SPY = prices_all['SPY'] # only SPY, for comparison later
start_val = sv
# Calculate indicators
# Momentum indicator
window = int(21)
momentum = []
price_array = pd.DataFrame.as_matrix(prices)
for i in range(prices.shape[0]):
if i > window:
momentum_value = (price_array[i] /
price_array[i - window]) - 1.0
momentum.append(momentum_value)
pmomentum_value = prices.copy()
pmomentum_value[:] = 0.0
norm_momentum = (momentum - np.mean(momentum)) / np.std(momentum)
pmomentum_value.ix[window + 1:, :] = norm_momentum
pmomentum_value.ix[:window, :] = norm_momentum[20][0]
# Bollinger Bands
rolling_std = pd.rolling_std(prices, window=21)
rolling_avg = pd.rolling_mean(prices, window=21)
bb_upper = rolling_avg + 2.0 * rolling_std
bb_lower = rolling_avg - 2.0 * rolling_std
bbp_band = (prices - bb_lower) / (bb_upper - bb_lower)
normed_bbp = (bbp_band - np.mean(bbp_band)) / np.std(bbp_band)
normed_bbp.ix[:21, :] = pd.DataFrame.as_matrix(normed_bbp)[20]
# SMA/Price indicator
sma = rolling_avg / prices
normed_sma = (sma - np.mean(sma)) / np.std(sma)
normed_sma.ix[:21, :] = pd.DataFrame.as_matrix(normed_sma)[20]
nsteps = 4
momen_values = pmomentum_value.values #.ix[21:,:].values
sma_values = normed_sma.values #.ix[21:,:].values
bbp_values = normed_bbp.values # ix[21:,:].values
momen_array = np.zeros(len(momen_values))
sma_array = np.zeros(len(sma_values))
bbp_array = np.zeros(len(sma_values))
for i in range(len(momen_values)):
momen_array[i] = momen_values[i]
sma_array[i] = sma_values[i]
bbp_array[i] = bbp_values[i]
momen_states = pd.cut(momen_array,
bins=self.mbins,
labels=False,
include_lowest=True)
sma_states = pd.cut(sma_array,
bins=self.sbins,
labels=False,
include_lowest=True)
bbp_states = pd.cut(bbp_array,
bins=self.bbins,
labels=False,
include_lowest=True)
total_states = nsteps**3
df_prices = prices.copy()
df_prices['Cash'] = pd.Series(1.0, index=prices.index)
df_trades = df_prices.copy()
df_trades[:] = 0.0
indices = df_prices.index
holdings = 0
# Cycle through dates
for j in range(pmomentum_value.shape[0]):
daily_ret = 0.0
result = 0
indicators = [momen_states[j], sma_states[j], bbp_states[j]]
for k in range(len(indicators)):
if indicators[k] not in range(total_states): #== np.nan:
indicates = 0
else:
indicates = indicators[k]
result = result + indicates * (nsteps**k)
#result = result + indicators[k]*(nsteps**k)
new_combined_state = result
# Query learner with current state and reward to get action
action = self.learner.querysetstate(new_combined_state)
# Implement action returned by learner and update portfolio
rDates = indices[j]
if action == 0:
if holdings == -200:
# Buy if holdings permit
df_trades['Cash'].ix[rDates] = df_trades['Cash'].ix[
rDates] + 400.0 * df_prices[syms[0]].ix[rDates] * -1.0
df_trades[syms[0]].ix[rDates] = df_trades[
syms[0]].ix[rDates] + 400.0
holdings = holdings + 400.0
if holdings == 0:
df_trades['Cash'].ix[rDates] = df_trades['Cash'].ix[
rDates] + 200.0 * df_prices[syms[0]].ix[rDates] * -1.0
df_trades[syms[0]].ix[rDates] = df_trades[
syms[0]].ix[rDates] + 200.0
holdings = holdings + 200.0
if action == 1:
if holdings == 200:
df_trades['Cash'].ix[rDates] = df_trades['Cash'].ix[
rDates] + 400.0 * df_prices[syms[0]].ix[rDates]
df_trades[syms[0]].ix[rDates] = df_trades[
syms[0]].ix[rDates] - 400.0
holdings = holdings - 400.0
if holdings == 0:
df_trades['Cash'].ix[rDates] = df_trades['Cash'].ix[
rDates] + 200.0 * df_prices[syms[0]].ix[rDates]
df_trades[syms[0]].ix[rDates] = df_trades[
syms[0]].ix[rDates] - 200.0
holdings = holdings - 200.0
df_holdings = df_trades.copy()
df_holdings[:] = 0.0
df_values = df_holdings.copy()
# Populate holdings dataframe, calculate value of holdings and portfolio value
df_holdings['Cash'].ix[indices[0]] = start_val
df_holdings.ix[
indices[0]] = df_holdings.ix[indices[0]] + df_trades.ix[indices[0]]
for i in range(1, len(indices)):
hDates = indices[i - 1]
rDates = indices[i]
df_holdings.ix[
rDates] = df_holdings.ix[hDates] + df_trades.ix[rDates]
df_values = df_holdings * df_prices
df_port_val = df_values.sum(axis=1)
period_end = df_port_val.ix[-1]
commul = df_port_val.pct_change()
cum_ret = (period_end - start_val) / start_val
print cum_ret * 100
return df_trades
def generate_features(df):
df_new = df.copy()
df_new['value_1'] = df_new['value'].shift(1)
df_new['avg_price_5'] = pd.rolling_mean(df_new['value'],
window=5,
min_periods=1).shift(1)
df_new['avg_price_30'] = pd.rolling_mean(df_new['value'],
window=21,
min_periods=1).shift(1)
df_new['avg_price_365'] = pd.rolling_mean(df_new['value'],
window=252,
min_periods=1).shift(1)
df_new['ratio_avg_price_5_30'] = df_new['avg_price_5'] / df_new[
'avg_price_30']
df_new['ratio_avg_price_5_365'] = df_new['avg_price_5'] / df_new[
'avg_price_365']
df_new['ratio_avg_price_30_365'] = df_new['avg_price_30'] / df_new[
'avg_price_365']
# standard deviation of prices
df_new['std_price_5'] = pd.rolling_std(df_new['value'],
window=5,
min_periods=1).shift(1)
# rolling_mean calculates the moving standard deviation given a window
df_new['std_price_30'] = pd.rolling_std(df_new['value'],
window=21,
min_periods=1).shift(1)
df_new['std_price_365'] = pd.rolling_std(df_new['value'],
window=252,
min_periods=1).shift(1)
df_new['ratio_std_price_5_30'] = df_new['std_price_5'] / df_new[
'std_price_30']
df_new['ratio_std_price_5_365'] = df_new['std_price_5'] / df_new[
'std_price_365']
df_new['ratio_std_price_30_365'] = df_new['std_price_30'] / df_new[
'std_price_365']
# return
df_new['return_1'] = ((df_new['value'] - df_new['value'].shift(1)) /
df_new['value'].shift(1)).shift(1)
df_new['return_5'] = ((df_new['value'] - df_new['value'].shift(5)) /
df_new['value'].shift(5)).shift(1)
df_new['return_30'] = ((df_new['value'] - df_new['value'].shift(21)) /
df_new['value'].shift(21)).shift(1)
df_new['return_365'] = ((df_new['value'] - df_new['value'].shift(252)) /
df_new['value'].shift(252)).shift(1)
df_new['moving_avg_5'] = pd.rolling_mean(df_new['return_1'],
window=5,
min_periods=1)
df_new['moving_avg_30'] = pd.rolling_mean(df_new['return_1'],
window=21,
min_periods=1)
df_new['moving_avg_365'] = pd.rolling_mean(df_new['return_1'],
window=252,
min_periods=1)
# the target & clean
# df_new['value'] = df['value']
df_new = df_new.fillna(0)
return df_new
# -*- coding: utf-8 -*-
"""
Created on Thu Jun 08 11:19:04 2017
@author: dugj2403
"""
import pandas as pd
df1 = pd.read_csv("10_treated.csv")
df2 = pd.read_csv("10_treated.csv")
ax = df1.plot.scatter(x='Image frame',
y='x',
color='DarkBlue',
label='Group 1')
df2.plot.scatter(x='Image frame',
y='y',
color='Yellow',
label='Group 2',
ax=ax)
df['std'] = pd.rolling_std(df['u'], window=13, min_periods=2)
df['average_u'] = pd.rolling_mean(df['u'], window=5)
df['average_u']
sql = "SELECT * from myfruit"
row_list=[]
cursor.execute(sql)
for row in cursor.fetchall():
row_tmp=list(row)
row_list.append(row_tmp)
data=pd.DataFrame(row_list,columns=['date','close'])
#计算涨跌
data['change']=data['close'].astype(float)-data['close'].astype(float).shift(1)
#计算BOLL(26,2)
data['bl_mid']=pd.rolling_mean(data['close'].shift(1),26)
data['std']=pd.rolling_std(data['close'].shift(1),26)
data['bl_top']=data['bl_mid']+2*data['std']
data['bl_bottom']=data['bl_mid']-2*data['std']
#计算长短期RSI
up, bottom = data['change'].copy(), data['change'].copy()
up[up < 0] = 0
bottom[bottom > 0] = 0
data['up'] = up
data['bottom'] = bottom
data['shRSI'] = caculate_RSI(data['up'], data['bottom'], short_day)
data['lgRSI'] = caculate_RSI(data['up'], data['bottom'], long_day)
#计算买入卖出信号
data=calculate_sellOrbuy(data)
# In[ ]:
minute_lag=np.zeros((len(df_last),7))
for i in range(0,7):
minute_lag[:,i]=df_last['log_returns_min'].shift(i+1)
dd=pd.DataFrame(minute_lag)
dd.columns=['min_lg1','min_lg2','min_lg3','min_lg4','min_lg5','min_lg6','min_lg7']
dd=dd.set_index(df_last.index)
dd_minute=pd.merge(df_last,dd, left_index=True, right_index=True)
f=dd_minute.corr()[[1]]
f[2:]
# In[ ]:
dd_minute['Rolling_Std_5'] = pd.rolling_std(dd_minute[['price']],window=5)
dd_minute['Rolling_Std_25'] = pd.rolling_std(dd_minute[['price']],window=25)
dd_minute['Rolling_Std_50'] = pd.rolling_std(dd_minute[['price']],window=50)
dd_minute['Rolling_Std_150'] = pd.rolling_std(dd_minute[['price']],window=150)
dd_minute.corr()
# In[ ]:
#total sample
#X_1=dd_minute[151:].drop(['price', 'log_returns_min'], axis=1).as_matrix()
X_1=dd_minute[151:][['min_lg1','Rolling_Std_5']].as_matrix()
y_1=dd_minute[151:]['log_returns_min'].as_matrix()
y_1.shape
def get_rolling_std(values, window):
"""Return rolling standard deviation of given values, using specified window size."""
# TODO: Compute and return rolling standard deviation
return pd.rolling_std(values, window=window)
def create_events(currencies, date_from, date_to, db, plot=False):
events = dict()
time_start = int(time.mktime(date_from.timetuple()) +
3600) # +1h for time zone adjustment
time_end = int(time.mktime(date_to.timetuple()) +
3600) # +1h for time zone adjustment
avg_window_size = 11 * 72 # 11 samples takes one hour
std_window_size = 11 * 72 # 72h = 3d
for currency in currencies:
print(currency)
prices = []
price_values = []
for price in db[currency].find({
"$and": [{
"_id": {
"$gte": time_start
}
}, {
"_id": {
"$lte": time_end
}
}]
}).sort("_id", 1):
prices.append(price)
price_values.append(price["weightedAverage"])
moving_average = pandas.rolling_mean(np.array(price_values),
window=avg_window_size)
moving_std = pandas.rolling_std(np.array(price_values),
window=std_window_size)
events_min_max = []
curr_event_i = np.nan
curr_event_price = np.nan
prev_event_type = np.nan
in_event = False
in_wait = False
for i in range(len(moving_average)):
# end event or continue
if in_event and not in_wait:
in_wait = True
in_event = False
if in_wait and ((prev_event_type == "up"
and moving_average[i] > price_values[i]) or
(prev_event_type == "down"
and moving_average[i] < price_values[i])):
in_wait = False
events_min_max.append({
"i": curr_event_i,
"price": curr_event_price,
"event_type": prev_event_type
})
curr_event_i = np.nan
curr_event_price = np.nan
prev_event_type = np.nan
if np.isnan(moving_average[i]) or np.isnan(moving_std[i]):
continue
elif price_values[i] > moving_average[i] + moving_std[i]:
# positive event
in_event = True
in_wait = False
if np.isnan(curr_event_price
) or curr_event_price < price_values[i]:
curr_event_i = i
curr_event_price = price_values[i]
prev_event_type = "up"
elif price_values[i] < moving_average[i] - moving_std[i]:
# negative event
in_event = True
in_wait = False
if np.isnan(curr_event_price
) or curr_event_price > price_values[i]:
curr_event_i = i
curr_event_price = price_values[i]
prev_event_type = "down"
if len(events_min_max) > 0:
events_min_max.pop(0)
p_values = []
for event in events_min_max:
end = event["i"]
event_type = event["event_type"]
event_price = event["price"]
last_relevant_price = None
i = end - 1
while True:
if ((event_type == "up"
and price_values[i] <= event_price - moving_std[i]) or
(event_type == "down" and price_values[i] >= event_price +
moving_std[i])) and last_relevant_price is None:
last_relevant_price = prices[i]
last_relevant_price["time_x"] = i
if (event_type == "up" and moving_average[i] > price_values[i]
) or (event_type == "down"
and moving_average[i] < price_values[i]):
if i != end:
# create event
first_price = prices[i]
last_price = prices[end]
event_id = currency + ":" + str(first_price)
_event = Event(event_id=event_id, currency=currency)
_event.first_price_info = first_price
_event.last_relevant_price_info = last_relevant_price if last_relevant_price is not None else first_price
_event.last_price_info = last_price
_event.p = (last_price["weightedAverage"] -
first_price["weightedAverage"]
) / first_price["weightedAverage"]
_event.t = last_price["_id"] - first_price["_id"]
_event.magnitude = (
last_price["weightedAverage"] -
moving_average[end]) / moving_std[end]
_event.time_x1 = i
_event.time_x2 = i if last_relevant_price is None else last_relevant_price[
"time_x"]
_event.time_x3 = end
events[event_id] = _event
p_values.append(_event.p)
break
i -= 1
p_mean = np.mean(p_values)
p_std = np.std(p_values)
for event in events.values():
event.magnitude_p = event.p / p_std
if plot:
plt.plot(range(len(price_values)), price_values, "r-")
plt.plot(range(len(moving_average)), moving_average, "b-")
plt.fill_between(range(len(price_values)),
np.array(moving_average) + moving_std,
np.array(moving_average) - moving_std,
alpha=0.7)
for event in events.values():
plt.axvspan(
event.time_x1,
event.time_x2,
color="green",
alpha=abs(event.magnitude_p /
2) if abs(event.magnitude_p / 2) <= 1 else 1)
# plt.scatter([x["i"] for x in events_min_max], [y["price"] for y in events_min_max])
plt.show()
sorted_keys = sorted(events.keys())
return events, sorted_keys
def pd_std(df_in, periods):
if abs(periods) < 2:
return df_in
else:
return pd.rolling_std(df_in, abs(periods), min_periods=abs(periods))
time_averaged.append(ta)
coh = np.array(coh)
coh = np.convolve(coh, np.ones(averaging_window_size), mode='same')
coh = np.expand_dims(coh, axis=1) / coh.max()
time_averaged = np.array(time_averaged)
time_averaged = np.array([
np.convolve(ta, np.ones(averaging_window_size), mode='same')
for ta in time_averaged.T
]).T
time_averaged /= np.max(time_averaged, axis=0)
all_coherence = np.concatenate((coh, time_averaged), axis=1)
rolling_stds = np.array([
pd.rolling_std(coh, averaging_window_size)
for coh in all_coherence.T
]).T
plt.plot(ks[:-1 * averaging_window_size - 1],
all_coherence[:-1 * averaging_window_size - 1, :],
alpha=.5)
for i in range(num_d + 1):
plt.fill_between(
ks[:-1 * averaging_window_size - 1],
all_coherence[:-1 * averaging_window_size - 1, i] -
rolling_stds[:-1 * averaging_window_size - 1, i],
all_coherence[:-1 * averaging_window_size - 1, i] +
rolling_stds[:-1 * averaging_window_size - 1, i],
alpha=.2)
plt.legend(('Convergent', 'Average', 'Monic', 'Quadratic', 'Cubic'))
plt.xlabel('K')
for i in range(len(ts_ret2) / 2 - 1, len(ts_ret2) - 1):
# Fit Model again and make prediction
model2 = pf.GARCH(ts_ret2[:i], p=1, q=1)
results_GR2 = model2.fit()
#print model2.predict(h=1)
Next_GARCH_Forecast = math.sqrt(
model2.predict(h=1)["Returns"][ts_ret2.index[i]])
print str(i) + " " + str(Next_GARCH_Forecast)
#print Next_GARCH_Forecast["Series"][i]
GARCH_Pred.append(Next_GARCH_Forecast)
Instrument_Data["Forecast_Volatility"] = [0] + GARCH_Pred
Instrument_Data['hvol21'] = np.sqrt(
pd.rolling_std(np.abs(Instrument_Data["Returns"]), 4) * (52**0.5)) / 2
Instrument_Data["GARCH_PL"] = 0
std_vol = Instrument_Data['Forecast_Volatility'][0:274].mean()
Instrument_Data['GARCH_PL'] = Instrument_Data.apply(
lambda x: x["PUT_" + file] if x[player + "_Diff"] > 0 and x['hvol21'] > x[
'Forecast_Volatility'] else x["CALL_" + file]
if x[player + "_Diff"] < 0 and x['hvol21'] > x['Forecast_Volatility'
] else 0,
axis=1)
Instrument_Data['Total_GARCH__PL'] = list(
accumu(Instrument_Data['GARCH_PL'].tolist()))
#########################################################################
print "Finished Equities Calc"
Instrument_Data.to_csv(OutputDirectory + file + "_" + player + '_Output.csv')
pickle_out = open('fiftyStates3.pickle', 'wb')
pickle.dump(main_df, pickle_out)
pickle_out.close()
def HPI_Benchmark():
df = quandl.get("FMAC/HPI_USA", authtoken=api_key)
df.rename(columns={'Value': "United States"}, inplace=True)
df["United States"] = (df["United States"] - df["United States"][0]
) / df["United States"][0] * 100.0
return df
# grab_initial_state_data()
fig = plt.figure()
ax1 = plt.subplot2grid((2, 1), (0, 0))
ax2 = plt.subplot2grid((2, 1), (1, 0), sharex=ax1)
HPI_data = pd.read_pickle('fiftyStates3.pickle')
HPI_data['TX12MA'] = pd.rolling_mean(HPI_data['TX'], 12)
HPI_data['TX12STD'] = pd.rolling_std(HPI_data['TX'], 12)
print(HPI_data[['TX', 'TX12MA']].head())
HPI_data[['TX', 'TX12MA']].plot(ax=ax1)
HPI_data['TX12STD'].plot(ax=ax2)
plt.legend(loc=4)
plt.show()
def get_rolling_std(values, window):
return pd.rolling_std(values, window=window)
d_data[s_key] = d_data[s_key].fillna(method='bfill')
d_data[s_key] = d_data[s_key].fillna(1.0)
df_close = d_data['actual_close']
# Create empty dataframe
df_columns = np.concatenate((['Date'], ls_symbols))
df = pd.DataFrame(index=range(len(ldt_timestamps)), columns=df_columns)
df['Date'] = ldt_timestamps
for s_sym in ls_symbols:
# Calculate rolling mean and rolling std
close_price = df_close[s_sym]
rolling_mean = pd.rolling_mean(close_price, window=ROLLING_WINDOW)
rolling_std = pd.rolling_std(close_price, window=ROLLING_WINDOW)
# Calculate upper and lower Bollinger bands
upper_bollinger = rolling_mean + rolling_std * N_STD_FACTOR
lower_bollinger = rolling_mean - rolling_std * N_STD_FACTOR
# Calculate normalized Bollinger values
normalized_bollinger_values = (close_price - rolling_mean) / (rolling_std *
N_STD_FACTOR)
normalized_upper_values = normalized_bollinger_values > 1
normalized_lower_values = normalized_bollinger_values < -1
# Get dates where Bollinger values are not in the interval [-1, 1]
normalized_upper_dates = [
val for val, valmask in zip(ldt_timestamps, normalized_upper_values)
if valmask
ForecastScaleCapFixed(),
Rules()
], csvFuturesData(), config)
system.config.instrument_div_mult_estimate['dm_max'] = 100.0
system.set_logging_level("on")
idm.append(system.portfolio.get_instrument_diversification_multiplier())
acc = system.accounts.portfolio(roundpositions=False)
acc_curve.append(acc)
roll_acc.append(acc.weekly.rolling_ann_std(22))
mktcount = acc.to_frame().shape[1] - np.isnan(acc.to_frame()).sum(axis=1)
mkt_counters.append(mktcount)
import pandas as pd
roll_acc = []
mkt_counters = []
for acc in acc_curve:
y = pd.rolling_std(acc.weekly.as_df(), 20, min_periods=4,
center=True) * acc.weekly._vol_scalar
roll_acc.append(y)
mktcount = acc.to_frame().shape[1] - np.isnan(acc.to_frame()).sum(axis=1)
mkt_counters.append(mktcount)
ans = [roll_acc, idm, acc_curve, mkt_counters]
with open("/home/rob/data.pck", "wb") as f:
dump(ans, f)