def getTrainData_1(symbol, startDate, endDate, periods=14, remove_head=19):
dates = pd.date_range(startDate, endDate)
df = util.loadPriceData([symbol], dates)
util.fill_missing_values(df)
# skip periods day latest (14 days) that predict a price is new high or not
df_sliced = df.ix[0:len(df) - periods]
price_close = pd.DataFrame(df_sliced[symbol])
set = pd.DataFrame(df_sliced['SET'])
rsi = ind.rsi(price_close) / 100 # normalize
sr = ind.create_dataframe_SR(df_sliced, [symbol])
myRatio = ind.get_myRatio(price_close)
daily = ind.daily_returns(price_close) * 100 # normalize
_, _, macd = ind.average_convergence(price_close)
ohlcv = util.load_OHLCV(symbol, dates)
percent_KD = ind.percent_KD(ohlcv) / 100 # normalize
c2o = ind.daily_returns_2(ohlcv) * 100 # normalize
volume = util.loadVolumeData([symbol], dates)
#skip periods day latest
volume_sliced = volume.ix[0:len(volume) - periods]
assert len(volume_sliced) == len(df_sliced)
obv = ind.OBV(volume_sliced, df_sliced)
obv_rsi = ind.rsi(obv) / 100 # calcuate momentum with rsi and normalize
set_rsi = ind.rsi(set) / 100 # calcuate momentum with rsi and normalize
# Join data frame
# rename column
rsi.rename(columns={symbol: 'RSI'}, inplace=True)
#sr.rename(columns={symbol:'SR'},inplace=True)
myRatio.rename(columns={symbol: 'MY'}, inplace=True)
daily.rename(columns={symbol: 'DAILY'}, inplace=True)
macd.rename(columns={symbol: 'MACD'}, inplace=True)
obv_rsi.rename(columns={symbol: 'OBV_RSI'}, inplace=True)
set_rsi.rename(columns={'SET': 'SET_RSI'}, inplace=True)
Xtrain = pd.DataFrame(index=df_sliced.index)
Xtrain = Xtrain.join(rsi['RSI'])
Xtrain = Xtrain.join(percent_KD['%K'])
#Xtrain = Xtrain.join(sr['SR'])
Xtrain = Xtrain.join(myRatio['MY'])
Xtrain = Xtrain.join(daily['DAILY'])
Xtrain = Xtrain.join(macd['MACD'])
Xtrain = Xtrain.join(c2o)
Xtrain = Xtrain.join(obv_rsi['OBV_RSI'])
Xtrain = Xtrain.join(set_rsi)
upTrend = ind.isUpTrend(df, symbol, periods=periods)
Ydigit = 1 * upTrend[
symbol] # multiply 1 : True is converted to 1 (up trend) , False becomes 0
# skip at head row, avoid NaN values
Xtrain = Xtrain.ix[remove_head:]
Ydigit = Ydigit.ix[remove_head:]
price_close = price_close.ix[remove_head:]
return Xtrain, Ydigit, price_close
def find_validETF(filename, buy_signal, sell_signal):
ls_etf = []
ls_rsi = []
ls_bbd = []
for etf in get_ETFSymbols('nasdaq'):
try:
price = read_yahooData(etf)
except IOError:
continue
relative_strength = rsi(price)
bollinger_band, bollinger_band2 = bbands(price)
# ''' RSI is considered overbought when above 75 and oversold when below 25 '''
if (relative_strength.values[-1] < buy_signal and bollinger_band < 0.
) or (relative_strength.values[-1] > sell_signal
and bollinger_band > 0.):
ls_etf.append(etf)
ls_rsi.append(round(relative_strength, 3))
ls_bbd.append(round(bollinger_band, 3))
new = signal_generation(price, buy_signal, sell_signal, n=14)
plot_rsi(new, etf, buy_signal, sell_signal)
newbie = signal_generation2(price)
plot_bb(newbie, etf)
df = pandas.DataFrame({
'ETF': ls_etf,
'Relative Strength': ls_rsi,
'Bollinger Band': ls_bbd
})
df.set_index('ETF').to_csv(filename)
def getTrainData_2(symbol, startDate, endDate, remove_head=15):
dates = pd.date_range(startDate, endDate)
df = util.loadPriceData([symbol], dates) #1
close = df.ix[0:len(df) - 1] # skip tail
assert len(df) == len(close) + 1
ema5 = ind.ema(close, periods=5) #2
sma5 = ind.ema(close, periods=5) #3
ema15 = ind.ema(close, periods=5) #4
sma15 = ind.ema(close, periods=5) #5
bb1p = ind.get_BBANDS(close, symbol, periods=14, mul=1) #6, #7
bb2p = ind.get_BBANDS(close, symbol, periods=14, mul=2) #8, #9
_, _, macd = ind.average_convergence(close) #10
signal_macd = ind.signal_MACD(macd) #11
rsi = ind.rsi(close) #12
ohlcv = util.load_OHLCV(symbol, dates)
percent_KD = ind.percent_KD(ohlcv) #14, 15
# rename columns
close.rename(columns={symbol: 'CLOSE'}, inplace=True)
ema5.rename(columns={symbol: 'EMA5'}, inplace=True)
sma5.rename(columns={symbol: 'SMA5'}, inplace=True)
ema15.rename(columns={symbol: 'EMA15'}, inplace=True)
sma15.rename(columns={symbol: 'SMA15'}, inplace=True)
bb1p.rename(columns={'LOWER': 'LOWER_BB1P'}, inplace=True)
bb1p.rename(columns={'UPPER': 'UPPER_BB1P'}, inplace=True)
bb2p.rename(columns={'LOWER': 'LOWER_BB2P'}, inplace=True)
bb2p.rename(columns={'UPPER': 'UPPER_BB2P'}, inplace=True)
macd.rename(columns={symbol: 'MACD'}, inplace=True)
signal_macd.rename(columns={symbol: 'SG_MACD'}, inplace=True)
rsi.rename(columns={symbol: 'RSI'}, inplace=True)
Xtrain = pd.DataFrame(index=close.index)
Xtrain = Xtrain.join(close['CLOSE']) # 1
Xtrain = Xtrain.join(ema5['EMA5']) # 2
Xtrain = Xtrain.join(sma5['SMA5']) # 3
Xtrain = Xtrain.join(ema15['EMA15']) # 4
Xtrain = Xtrain.join(sma15['SMA15']) # 5
Xtrain = Xtrain.join(bb1p['LOWER_BB1P']) # 6
Xtrain = Xtrain.join(bb1p['UPPER_BB1P']) # 7
Xtrain = Xtrain.join(bb2p['LOWER_BB2P']) # 8
Xtrain = Xtrain.join(bb2p['UPPER_BB2P']) # 9
Xtrain = Xtrain.join(macd['MACD']) # 10
Xtrain = Xtrain.join(signal_macd['SG_MACD']) # 11
Xtrain = Xtrain.join(rsi['RSI']) # 12
Xtrain = Xtrain.join(percent_KD) # 13
Ytrain = df[symbol].shift(-1) # skip SET and shift
Ytrain.dropna(0, inplace=True)
assert len(Ytrain) == len(Xtrain)
# skip at head row, avoid NaN values
Xtrain = Xtrain[remove_head:]
Ytrain = Ytrain[remove_head:]
return Xtrain, Ytrain
def getTrainData_4(symbol, startDate, endDate, periods=14, remove_head=19):
dates = pd.date_range(startDate, endDate)
# day periods that predict a price is new high or not
df = util.loadPriceData([symbol], dates)
# skip periods day latest
df_sliced = df.ix[0:len(df) - periods]
price_close = pd.DataFrame(df_sliced[symbol])
set = pd.DataFrame(df_sliced['SET'])
bbands = ind.get_BBANDS(price_close, symbol)
ema26, ema12, macd = ind.average_convergence(price_close)
rsi = ind.rsi(price_close)
daily = ind.daily_returns(price_close)
ohlcv = util.load_OHLCV(symbol, dates)
percent_KD = ind.percent_KD(ohlcv)
volume = util.loadVolumeData(symbol, dates)
#skip periods day latest
volume_sliced = volume.ix[0:len(volume) - periods]
assert len(volume_sliced) == len(price_close)
volume_sliced = pd.DataFrame(volume_sliced[symbol])
obv = ind.OBV(volume_sliced, price_close)
# Join data frame
# rename column
price_close.rename(columns={symbol: 'CLOSE'}, inplace=True)
ema26.rename(columns={symbol: 'EMA26'}, inplace=True)
ema12.rename(columns={symbol: 'EMA12'}, inplace=True)
daily.rename(columns={symbol: 'DAILY'}, inplace=True)
rsi.rename(columns={symbol: 'RSI'}, inplace=True)
obv.rename(columns={symbol: 'OBV'}, inplace=True)
Xtrain = price_close
Xtrain = Xtrain.join(bbands['UPPER'])
Xtrain = Xtrain.join(bbands['LOWER'])
Xtrain = Xtrain.join(ema26)
Xtrain = Xtrain.join(ema12)
Xtrain = Xtrain.join(rsi)
Xtrain = Xtrain.join(percent_KD['%K'])
Xtrain = Xtrain.join(obv)
Xtrain = Xtrain.join(set)
upTrend = ind.isUpTrend(df, symbol, periods=periods)
Ydigit = 1 * upTrend[
symbol] # multiply 1 : True is converted to 1 (up trend) , False becomes 0
assert len(Xtrain) == len(Ydigit)
# skip at head row, avoid NaN values
Xtrain = Xtrain.ix[remove_head:]
Ydigit = Ydigit.ix[remove_head:]
Xtrain.fillna(0, inplace=True) # protected NaN value
return Xtrain, Ydigit
def signal_generation(df, buy_signal, sell_signal, n=14):
# indi df de rsi
df_f = pd.DataFrame()
df_f['Close'] = df
df_f['rsi'] = rsi(df) # [n:]
df_f['positions'] = np.where(df_f['rsi'] < buy_signal, 1,
np.where(df_f['rsi'] > sell_signal, -1, 0))
df_f['signals'] = df_f['positions'].diff()
# print(df_f)
return df_f[n:]
def generate_onesecurity_onefreq(filepath):
##########################################################################
# Input filepath = './ts/Future/IF/IF_15s.ts'
##########################################################################
# datestimes, volrmb, open, high, low,
# 2014-12-09 09:15:00, 880059660.000, 3306.800, 3307.200, 3306.800,
# close, avgprice, bid, ask
# 3307.200, 3307.331, 3306.900, 3308.100
#
##########################################################################
# Output s_outpath ='./ics/Future/IF/IF_15s.ics'
# datestimes, typ, ema50, ema27, rsi14, cci10, tr, kpi
# 2005-01-04, 0.2367, 0.2360, 0.2360, 0.5000, 0.0000, 0.0420, 0.2360
#
s_prod = filepath.split('/')[-1].split('_')[0]
s_freq = filepath.split('_')[-1].split('.')[0]
s_outdir = './ics/'+s_prodtype + '/' +s_prod+'/'
s_outpath = s_outdir + s_prod + '_' + s_freq + '.ics'
logger.info("generate_onesecurity_onefreq(%s,%s) start",s_prod,s_freq)
df_prices = pd.read_csv(filepath, parse_dates=[0], index_col=0)
df_c = df_prices['close']
df_h = df_prices['high']
df_l = df_prices['low']
typ = ic.typ(df_h , df_l , df_c)
ema50 = ic.ema(df_c, 50)
ema27 = ic.ema(df_c, 27)
rsi = ic.rsi(df_c)
cci10 = ic.cci(typ ,df_c, 10)
tr = ic.tr(df_h,df_l,df_c)
kpi = ic.kpi(df_c)
ics = pd.concat([typ, ema50, ema27 ,rsi, cci10, tr, kpi], axis=1)
ics.to_csv(s_outpath, float_format='%.4f')
logger.info("generate_onesecurity_onefreq(%s,%s) done",s_prod,s_freq)
def find_validETF(filename):
ls_etf = []
ls_rsi = []
ls_bbd = []
for etf in get_ETFSymbols('Nasdaq'):
try:
price = read_yahooData(etf)
except IOError:
continue
relative_strength = rsi(price)
bollinger_band = bbands(price)
if relative_strength < 30. and bollinger_band < 0.:
ls_etf.append(etf)
ls_rsi.append(round(relative_strength, 3))
ls_bbd.append(round(bollinger_band, 3))
df = pandas.DataFrame({
'ETF': ls_etf,
'Relative Strength': ls_rsi,
'Bollinger Band': ls_bbd
})
df.set_index('ETF').to_csv(filename)
def find_validETF(filename):
ls_etf = []
ls_rsi = []
ls_bbd = []
for etf in get_ETFSymbols('Nasdaq'):
try:
price = read_yahooData(etf)
except IOError:
continue
relative_strength = rsi(price)
bollinger_band = bbands(price)
if relative_strength < 30. and bollinger_band < 0.:
ls_etf.append(etf)
ls_rsi.append(round(relative_strength,3))
ls_bbd.append(round(bollinger_band,3))
df = pandas.DataFrame({
'ETF': ls_etf,
'Relative Strength': ls_rsi,
'Bollinger Band': ls_bbd
})
df.set_index('ETF').to_csv(filename)
def getTrainData_3(symbol, startDate, endDate, remove_head=15):
dates = pd.date_range(startDate, endDate)
df = util.loadPriceData([symbol], dates) #1
close = df.ix[0:len(df) - 1] # skip tail
assert len(df) == len(close) + 1
ema5 = ind.ema(close, periods=5) #2
sma5 = ind.ema(close, periods=5) #3
ema15 = ind.ema(close, periods=5) #4
sma15 = ind.ema(close, periods=5) #5
bb1p = ind.get_BBANDS(close, symbol, periods=14, mul=1) #6, #7
bb2p = ind.get_BBANDS(close, symbol, periods=14, mul=2) #8, #9
_, _, macd = ind.average_convergence(close) #10
signal_macd = ind.signal_MACD(macd) #11
rsi = ind.rsi(close) #12
ohlcv = util.load_OHLCV(symbol, dates)
percent_KD = ind.percent_KD(ohlcv) #14, 15
# rename columns
close.rename(columns={symbol: 'CLOSE'}, inplace=True)
ema5.rename(columns={symbol: 'EMA5'}, inplace=True)
sma5.rename(columns={symbol: 'SMA5'}, inplace=True)
#ema15.rename(columns={symbol:'EMA15'},inplace=True)
#sma15.rename(columns={symbol:'SMA15'},inplace=True)
bb1p.rename(columns={'LOWER': 'LOWER_BB1P'}, inplace=True)
bb1p.rename(columns={'UPPER': 'UPPER_BB1P'}, inplace=True)
#bb2p.rename(columns={'LOWER':'LOWER_BB2P'},inplace=True)
#bb2p.rename(columns={'UPPER':'UPPER_BB2P'},inplace=True)
macd.rename(columns={symbol: 'MACD'}, inplace=True)
#signal_macd.rename(columns={symbol:'SG_MACD'},inplace=True)
rsi.rename(columns={symbol: 'RSI'}, inplace=True)
Xtrain = pd.DataFrame(index=close.index)
Xtrain = Xtrain.join(close['CLOSE']) # 1
Xtrain = Xtrain.join(ema5['EMA5']) # 2
Xtrain = Xtrain.join(sma5['SMA5']) # 3
#Xtrain = Xtrain.join(ema15['EMA15']) # 4
#Xtrain = Xtrain.join(sma15['SMA15']) # 5
Xtrain = Xtrain.join(bb1p['LOWER_BB1P']) # 6
Xtrain = Xtrain.join(bb1p['UPPER_BB1P']) # 7
#Xtrain = Xtrain.join(bb2p['LOWER_BB2P']) # 8
#Xtrain = Xtrain.join(bb2p['UPPER_BB2P']) # 9
Xtrain = Xtrain.join(macd['MACD']) # 10
#Xtrain = Xtrain.join(signal_macd['SG_MACD']) # 11
Xtrain = Xtrain.join(rsi['RSI']) # 12
Xtrain = Xtrain.join(percent_KD) # 13
df = df[symbol] # skip SET
daily = ind.daily_returns(df)
daily = daily.shift(-1) # predict tommorow
daily.dropna(0, inplace=True) # drop NaN in last row
Ytrain = 1 * (daily > 0.0) # if positive it converted 1, or 0 in negative
assert len(Ytrain) == len(Xtrain)
# skip at head row, avoid NaN values
Xtrain = Xtrain[remove_head:]
Ytrain = Ytrain[remove_head:]
# normalize
normalizer = preprocessing.Normalizer().fit(Xtrain)
Xnorm = normalizer.transform(Xtrain)
Xnorm = pd.DataFrame(Xnorm, columns=Xtrain.columns)
return Xnorm, Ytrain
def rsi_check(candles):
inputs = indicator.gen_inputs(candles, 'close')
return indicator.rsi(inputs, 'close')
