def populate_indicators(self, dataframe: DataFrame,
metadata: dict) -> DataFrame:
dataframe['adx'] = ta.ADX(dataframe, timeperiod=14)
dataframe['plus_di'] = ta.PLUS_DI(dataframe, timeperiod=25)
dataframe['minus_di'] = ta.MINUS_DI(dataframe, timeperiod=25)
dataframe['sar'] = ta.SAR(dataframe)
dataframe['mom'] = ta.MOM(dataframe, timeperiod=14)
dataframe['sell-adx'] = ta.ADX(dataframe, timeperiod=14)
dataframe['sell-plus_di'] = ta.PLUS_DI(dataframe, timeperiod=25)
dataframe['sell-minus_di'] = ta.MINUS_DI(dataframe, timeperiod=25)
dataframe['sell-sar'] = ta.SAR(dataframe)
dataframe['sell-mom'] = ta.MOM(dataframe, timeperiod=14)
return dataframe
def analyze(self, historical_data, period_count=10,
hot_thresh=None, cold_thresh=None, all_data=False):
"""Performs momentum analysis on the historical data
Args:
historical_data (list): A matrix of historical OHCLV data.
period_count (int, optional): Defaults to 10. The number of data points to consider for
our simple moving average.
hot_thresh (float, optional): Defaults to None. The threshold at which this might be
good to purchase.
cold_thresh (float, optional): Defaults to None. The threshold at which this might be
good to sell.
all_data (bool, optional): Defaults to False. If True, we return the momentum
associated with each data point in our historical dataset. Otherwise just return
the last one.
Returns:
dict: A dictionary containing a tuple of indicator values and booleans for buy / sell
indication.
"""
dataframe = self.convert_to_dataframe(historical_data)
mom_values = abstract.MOM(dataframe, period_count)
analyzed_data = [(value,) for value in mom_values]
return self.analyze_results(
analyzed_data,
is_hot=lambda v: v > hot_thresh if hot_thresh else False,
is_cold=lambda v: v < cold_thresh if cold_thresh else False,
all_data=all_data
)
def populate_indicators(self, dataframe: DataFrame,
metadata: dict) -> DataFrame:
# SMA - Simple Moving Average
dataframe['fastMA'] = ta.SMA(dataframe, timeperiod=13)
dataframe['slowMA'] = ta.SMA(dataframe, timeperiod=25)
dataframe['mom'] = ta.MOM(dataframe, timeperiod=21)
return dataframe
def populate_indicators(dataframe: DataFrame) -> DataFrame:
"""
Adds several different TA indicators to the given DataFrame
"""
dataframe['sar'] = ta.SAR(dataframe)
dataframe['adx'] = ta.ADX(dataframe)
stoch = ta.STOCHF(dataframe)
dataframe['fastd'] = stoch['fastd']
dataframe['fastk'] = stoch['fastk']
dataframe['blower'] = ta.BBANDS(dataframe, nbdevup=2,
nbdevdn=2)['lowerband']
dataframe['sma'] = ta.SMA(dataframe, timeperiod=40)
dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9)
dataframe['mfi'] = ta.MFI(dataframe)
dataframe['cci'] = ta.CCI(dataframe)
dataframe['rsi'] = ta.RSI(dataframe)
dataframe['mom'] = ta.MOM(dataframe)
dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5)
dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10)
dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50)
dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100)
dataframe['ao'] = awesome_oscillator(dataframe)
macd = ta.MACD(dataframe)
dataframe['macd'] = macd['macd']
dataframe['macdsignal'] = macd['macdsignal']
dataframe['macdhist'] = macd['macdhist']
return dataframe
def analyze(self,
historical_data,
period_count=10,
signal=['momentum'],
hot_thresh=None,
cold_thresh=None):
"""Performs momentum analysis on the historical data
Args:
historical_data (list): A matrix of historical OHCLV data.
period_count (int, optional): Defaults to 10. The number of data points to consider for
our momentum.
signal (list, optional): Defaults to momentum. The indicator line to check hot/cold
against.
hot_thresh (float, optional): Defaults to None. The threshold at which this might be
good to purchase.
cold_thresh (float, optional): Defaults to None. The threshold at which this might be
good to sell.
Returns:
pandas.DataFrame: A dataframe containing the indicators and hot/cold values.
"""
dataframe = self.convert_to_dataframe(historical_data)
mom_values = abstract.MOM(dataframe, period_count).to_frame()
mom_values.dropna(how='all', inplace=True)
mom_values.rename(columns={mom_values.columns[0]: 'momentum'},
inplace=True)
if mom_values[signal[0]].shape[0]:
mom_values['is_hot'] = mom_values[signal[0]] > hot_thresh
mom_values['is_cold'] = mom_values[signal[0]] < cold_thresh
return mom_values
def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
dataframe['adx'] = ta.ADX(dataframe, timeperiod=14)
dataframe['plus_di'] = ta.PLUS_DI(dataframe, timeperiod=25)
dataframe['minus_di'] = ta.MINUS_DI(dataframe, timeperiod=25)
dataframe['sar'] = ta.SAR(dataframe)
dataframe['mom'] = ta.MOM(dataframe, timeperiod=14)
#print(f"\"{metadata['pair']}\"")
return dataframe
def mom(self):
mom = tb.MOM(self.dataframe, timeperiod=10)
value = mom[len(mom) - 1]
old_value = mom[len(mom) - 2]
if value > old_value:
return "buy"
else:
return "sell"
def populate_indicators(dataframe: DataFrame) -> DataFrame:
"""
Adds several different TA indicators to the given DataFrame
"""
# Log the dataframe
dataframe.to_csv("dataframe.csv", sep='\t')
#Determine Trend
dataframe = indicators.TREND(dataframe)
#Exponential Moving Average
dataframe['ema'] = ta.EMA(dataframe, timeperiod=33)
#Parabolic SAR
dataframe['sar'] = ta.SAR(dataframe, 0.02, 0.22)
#Average Directional Movement Index
dataframe['adx'] = ta.ADX(dataframe)
#MACD
macd = ta.MACD(dataframe)
dataframe['macd'] = macd['macd']
dataframe['macds'] = macd['macdsignal']
dataframe['macdh'] = macd['macdhist']
#Relative Strength Index
dataframe['rsi'] = ta.RSI(dataframe, 14)
dataframe = indicators.RSI(dataframe)
#Absolute Price Oscillator
dataframe['apo'] = ta.APO(dataframe,
fastperiod=12,
slowperiod=26,
matype=0)
#Momentum
dataframe['mom'] = ta.MOM(dataframe, 10)
#Bollinger Bands
dataframe = indicators.BBANDS(20, dataframe, 2)
# Stochcastic
dataframe['K'] = indicators.STOK(dataframe, 14)
dataframe['D'] = indicators.STOD(dataframe, 14)
dataframe = indicators.STOCH(dataframe)
# if Counter.counter < 30:
# print(dataframe)
# print (macd)
# print (dataframe ['adx'])
# Counter.counter = Counter.counter + 1
# else:
# exit()
return dataframe
def populate_indicators(self, dataframe: DataFrame,
metadata: dict) -> DataFrame:
dataframe["adx"] = ta.ADX(dataframe, timeperiod=14)
dataframe["plus_di"] = ta.PLUS_DI(dataframe, timeperiod=25)
dataframe["minus_di"] = ta.MINUS_DI(dataframe, timeperiod=25)
dataframe["sar"] = ta.SAR(dataframe)
dataframe["mom"] = ta.MOM(dataframe, timeperiod=14)
return dataframe
def generate_indicators(dataset,
timeperiod=5,
generate_target=True,
reset_index=False):
# To avoid changes from original dataset
df = pd.DataFrame(dataset).copy()
# To prevent index from being Date
if (reset_index):
df = df.reset_index()
df.columns = df.columns.str.lower()
# check the given dataset has necessary_columns
__check_columns(df)
df = df[necessary_columns]
# df_indicators has all columns except date, this is for talib to produces other indicators
df_indicators = df.iloc[:, 1:]
# Produce other indicators
RSI = abstract.RSI(df_indicators.close, timeperiod)
RSI = pd.DataFrame({'RSI': RSI})
MOM = abstract.MOM(df_indicators.close, timeperiod)
MOM = pd.DataFrame({'MOM': MOM})
KD = abstract.STOCH(df_indicators)
KD = pd.DataFrame(KD) # KD has slowd and slowk
MACD = abstract.MACD(df_indicators)
MACD = pd.DataFrame(MACD) # MACD has its own column names
ADX = abstract.ADX(df_indicators)
ADX = pd.DataFrame({'ADX': ADX})
SMA = abstract.SMA(df_indicators.close)
SMA = pd.DataFrame({'SMA': SMA})
upper, middle, lower = talib.BBANDS(df_indicators.close, matype=MA_Type.T3)
bb_df = pd.DataFrame({ \
'upper_bb' : upper,
'middel_bb' : middle,
'lower_bb' : lower})
# Combine all metrix
frames = [df, RSI, MOM, KD, MACD, ADX, SMA, bb_df]
combined = pd.concat(frames, axis=1)
if (generate_target):
target_name = 'next_' + str(timeperiod) + 'day_trend'
combined[target_name] = np.where(
combined.close.shift(-timeperiod) > combined.close, 1, 0)
return combined
def analyze_mom(self,
historial_data,
period_count=10,
hot_thresh=None,
cold_thresh=None,
all_data=False):
"""Performs momentum analysis on the historical data
Args:
historial_data (list): A matrix of historical OHCLV data.
period_count (int, optional): Defaults to 10. The number of data points to consider for
our simple moving average.
hot_thresh (float, optional): Defaults to None. The threshold at which this might be
good to purchase.
cold_thresh (float, optional): Defaults to None. The threshold at which this might be
good to sell.
all_data (bool, optional): Defaults to False. If True, we return the momentum
associated with each data point in our historical dataset. Otherwise just return
the last one.
Returns:
dict: A dictionary containing a tuple of indicator values and booleans for buy / sell
indication.
"""
dataframe = self.__convert_to_dataframe(historial_data)
mom_values = abstract.MOM(dataframe, period_count)
mom_result_data = []
for mom_value in mom_values:
if math.isnan(mom_value):
continue
is_hot = False
if hot_thresh is not None:
is_hot = mom_value > hot_thresh
is_cold = False
if cold_thresh is not None:
is_cold = mom_value < cold_thresh
data_point_result = {
'values': (mom_value, ),
'is_cold': is_cold,
'is_hot': is_hot
}
mom_result_data.append(data_point_result)
if all_data:
return mom_result_data
else:
try:
return mom_result_data[-1]
except IndexError:
return mom_result_data
def MOM(self):
MOM = tb.MOM(self.dataframe, timeperiod=10)
value = MOM[len(MOM) - 1]
oldValue = MOM[len(MOM) - 2]
if (value > oldValue):
print("MOM: " + str(value) + " buy")
return "buy"
else:
print("MOM: " + str(value) + " buy")
return "sell"
def TA_processing(dataframe):
bias(dataframe, days=[3, 6, 10, 25])
moving_average(dataframe, days=[5, 10, 20])
dataframe['ROC'] = abstract.ROC(dataframe, timeperiod=10)
dataframe['MACD'] = abstract.MACD(dataframe, fastperiod=12, slowperiod=26, signalperiod=9)['macd']
dataframe['MACD_signal'] = abstract.MACD(dataframe, fastperiod=12, slowperiod=26, signalperiod=9)['macdsignal']
dataframe['UBBANDS'] = abstract.BBANDS(dataframe, timeperiod=20, nbdevup=2, nbdevdn=2, matype=0)['upperband']
dataframe['MBBANDS'] = abstract.BBANDS(dataframe, timeperiod=20, nbdevup=2, nbdevdn=2, matype=0)['middleband']
dataframe['LBBANDS'] = abstract.BBANDS(dataframe, timeperiod=20, nbdevup=2, nbdevdn=2, matype=0)['lowerband']
dataframe['%K'] = abstract.STOCH(dataframe, fastk_period=9)['slowk']/100
dataframe['%D'] = abstract.STOCH(dataframe, fastk_period=9)['slowd']/100
dataframe['W%R'] = abstract.WILLR(dataframe, timeperiod=14)/100
dataframe['RSI9'] = abstract.RSI(dataframe, timeperiod = 9)/100
dataframe['RSI14'] = abstract.RSI(dataframe, timeperiod = 14)/100
dataframe['CCI'] = abstract.CCI(dataframe, timeperiod=14)/100
counter_daily_potential(dataframe)
dataframe['MOM'] = abstract.MOM(dataframe, timeperiod=10)
dataframe['DX'] = abstract.DX(dataframe, timeperiod=14)/100
psy_line(dataframe)
volumn_ratio(dataframe, d=26)
on_balance_volume(dataframe)
def evaluate_momentum(self,
period=20,
prefix="momentum",
impact_buy=1,
impact_sell=1):
"""
evaluates the momentum
:param dataframe:
:param period:
:param prefix:
:return:
"""
self._weights(impact_buy, impact_sell)
dataframe = self.dataframe
name = f"{prefix}_{period}"
dataframe[name] = ta.MOM(dataframe, timeperiod=period)
dataframe.loc[((dataframe[name] > 100)),
f"buy_{name}"] = 1 * impact_buy
dataframe.loc[((dataframe[name] < 100)),
f"sell_{name}"] = 1 * impact_sell
def evaluate_momentum(self,
period=20,
prefix="momentum",
impact_buy=1,
impact_sell=1):
"""
evaluates the osc
:param dataframe:
:param period:
:param prefix:
:return:
"""
self._weights(impact_buy, impact_sell)
dataframe = self.dataframe
name = '{}_{}'.format(prefix, period)
dataframe[name] = ta.MOM(dataframe, timeperiod=period)
dataframe.loc[((dataframe[name] > 100)),
'buy_{}'.format(name)] = (1 * impact_buy)
dataframe.loc[((dataframe[name] < 100)),
'sell_{}'.format(name)] = (1 * impact_sell)
def populate_indicators(dataframe: DataFrame) -> DataFrame:
"""
Adds several different TA indicators to the given DataFrame
"""
dataframe['sar'] = ta.SAR(dataframe)
dataframe['adx'] = ta.ADX(dataframe)
stoch = ta.STOCHF(dataframe)
dataframe['fastd'] = stoch['fastd']
dataframe['fastk'] = stoch['fastk']
dataframe['blower'] = ta.BBANDS(dataframe, nbdevup=2,
nbdevdn=2)['lowerband']
dataframe['sma'] = ta.SMA(dataframe, timeperiod=40)
dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9)
dataframe['mfi'] = ta.MFI(dataframe)
dataframe['cci'] = ta.CCI(dataframe)
dataframe['rsi'] = ta.RSI(dataframe)
dataframe['mom'] = ta.MOM(dataframe)
dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5)
dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10)
dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50)
dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100)
dataframe['ao'] = awesome_oscillator(dataframe)
macd = ta.MACD(dataframe)
dataframe['macd'] = macd['macd']
dataframe['macdsignal'] = macd['macdsignal']
dataframe['macdhist'] = macd['macdhist']
# add volatility indicators
dataframe['natr'] = ta.NATR(dataframe)
# add volume indicators
dataframe['obv'] = ta.OBV(dataframe)
# add more momentum indicators
dataframe['rocp'] = ta.ROCP(dataframe)
# add some pattern recognition
dataframe['CDL2CROWS'] = ta.CDL2CROWS(dataframe)
dataframe['CDL3BLACKCROWS'] = ta.CDL3BLACKCROWS(dataframe)
dataframe['CDL3INSIDE'] = ta.CDL3INSIDE(dataframe)
dataframe['CDL3LINESTRIKE'] = ta.CDL3LINESTRIKE(dataframe)
dataframe['CDL3OUTSIDE'] = ta.CDL3OUTSIDE(dataframe)
dataframe['CDL3STARSINSOUTH'] = ta.CDL3STARSINSOUTH(dataframe)
dataframe['CDL3WHITESOLDIERS'] = ta.CDL3WHITESOLDIERS(dataframe)
dataframe['CDLADVANCEBLOCK'] = ta.CDLADVANCEBLOCK(dataframe)
dataframe['CDLBELTHOLD'] = ta.CDLBELTHOLD(dataframe)
dataframe['CDLBREAKAWAY'] = ta.CDLBREAKAWAY(dataframe)
dataframe['CDLDOJI'] = ta.CDLDOJI(dataframe)
dataframe['CDLDOJISTAR'] = ta.CDLDOJISTAR(dataframe)
dataframe['CDLDRAGONFLYDOJI'] = ta.CDLDRAGONFLYDOJI(dataframe)
dataframe['CDLENGULFING'] = ta.CDLENGULFING(dataframe)
dataframe['CDLHAMMER'] = ta.CDLHAMMER(dataframe)
dataframe['CDLBREAKAWAY'] = ta.CDLBREAKAWAY(dataframe)
dataframe['CDLBREAKAWAY'] = ta.CDLBREAKAWAY(dataframe)
# enter categorical time
hour = datetime.strptime(str(dataframe['date'][len(dataframe) - 1]),
"%Y-%m-%d %H:%M:%S").hour
for h in range(24):
dataframe['hour_{0:02}'.format(h)] = int(h == hour)
return dataframe
def _build_indicators(self, df):
if not self.realtime:
inputs = df.to_dict(orient="list")
for col in inputs:
inputs[col] = np.array(inputs[col])
c = df["close"]
for n in range(2, 40):
inputs["bband_u_" +
str(n)], inputs["bband_m_" +
str(n)], inputs["bband_l_" +
str(n)] = ta.BBANDS(
inputs, n)
inputs["sma_" + str(n)] = ta.SMA(inputs, timeperiod=n)
inputs["adx_" + str(n)] = ta.ADX(inputs, timeperiod=n)
# fast_ema = c.ewm(span = n, adjust = False).mean()
# slow_ema = c.ewm(span = n*2, adjust = False).mean()
# macd1 = fast_ema - slow_ema
# macd2 = macd1.ewm(span = int(n*2/3), adjust = False).mean()
# macd3 = macd1 - macd2
# inputs["macd_"+str(n)] = macd1.values
# inputs["macdsignal_"+str(n)] = macd2.values
# inputs["macdhist_"+str(n)] = macd3.values
if n != 2:
inputs["macd_" +
str(n)], inputs["macdsignal_" +
str(n)], inputs["macdhist_" +
str(n)] = ta.MACD(
inputs, n,
n * 2,
int(n * 2 / 3))
else:
inputs["macd_" +
str(n)], inputs["macdsignal_" +
str(n)], inputs["macdhist_" +
str(n)] = ta.MACD(
inputs, n,
n * 2, 1)
# macd = [macd1.values, macd2.values, macd3.values]
# for idx, i in enumerate(["macd_"+str(n), "macdsignal_"+str(n), "macdhist_"+str(n)]):
# for day in zip(inputs[i], macd[idx]):
# print("Type: %s N: %d PD: %.3f TA: %.3f, " % (i, n, day[1], day[0]))
inputs["mfi_" + str(n)] = ta.MFI(inputs, n)
inputs["ult_" + str(n)] = ta.ULTOSC(inputs, n, n * 2, n * 4)
inputs["willr_" + str(n)] = ta.WILLR(inputs, n)
inputs["slowk"], inputs["slowd"] = ta.STOCH(inputs)
inputs["mom_" + str(n)] = ta.MOM(inputs, n)
inputs["volume"] = list(map(lambda x: x / 10000, inputs["volume"]))
df = pd.DataFrame().from_dict(inputs)
# df = df.ix[100:]
# print(df.tail(5)["macd_3"], df.tail(5)["macdsignal_3"], df.tail(5)["macdhist_3"])
return df
else:
# Build data one-by-one, as if it's coming in one at a time
output = pd.DataFrame()
sliding_window = pd.DataFrame()
for idx, day in df.iterrows():
print("\rNow building day", str(idx), end="", flush=True)
day = copy.deepcopy(day) # Avoid reference vs copy bullshit
sliding_window = sliding_window.append(day, ignore_index=True)
# print(day, type(day))
day_out = {}
# print(sliding_window)
o = sliding_window["open"].values
h = sliding_window["high"].values
l = sliding_window["low"].values
c_series = sliding_window["close"]
c = sliding_window["close"].values
# print("----")
# print(c)
v = sliding_window["volume"].values
for t in ["open", "high", "low", "close"]:
day_out[t] = sliding_window[t].values[-1]
for n in range(2, 40):
# time.sleep(0.1)
day_out["bband_u_" +
str(n)], day_out["bband_m_" + str(n)], day_out[
"bband_l_" + str(n)] = stream.BBANDS(c, n)
day_out["sma_" + str(n)] = stream.SMA(c, timeperiod=n)
day_out["adx_" + str(n)] = stream.ADX(h,
l,
c,
timeperiod=n)
fast_ema = c_series.ewm(span=n, adjust=False).mean()
slow_ema = c_series.ewm(span=n * 2, adjust=False).mean()
macd1 = fast_ema - slow_ema
macd2 = macd1.ewm(span=int(n * 2 / 3), adjust=False).mean()
macd3 = macd1 - macd2
day_out["macd_" + str(n)] = macd1.values[-1]
day_out["macdsignal_" + str(n)] = macd2.values[-1]
day_out["macdhist_" + str(n)] = macd3.values[-1]
# if n != 2:
# day_out["macd_"+str(n)], day_out["macdsignal_"+str(n)], day_out["macdhist_"+str(n)] = stream.MACD(c, n, n*2, int(n*2/3))
# elif idx > 100:
# macd = ta.MACD({"close":c}, n, n*2, 1)
# day_out["macd_2"], day_out["macdsignal_2"], day_out["macdhist_2"] = (x[-1] for x in macd)
# else:
# day_out["macd_2"], day_out["macdsignal_2"], day_out["macdhist_2"] = None, None, None
# macd = [macd1.values, macd2.values, macd3.values]
# for idx, i in enumerate(["macd_"+str(n), "macdsignal_"+str(n), "macdhist_"+str(n)]):
# for day in zip(inputs[i], macd[idx]):
# print("Type: %s N: %d PD: %.3f TA: %.3f, " % (i, n, day[1], day[0]))
day_out["mfi_" + str(n)] = stream.MFI(h, l, c, v, n)
day_out["ult_" + str(n)] = stream.ULTOSC(
h, l, c, n, n * 2, n * 4)
day_out["willr_" + str(n)] = stream.WILLR(h, l, c, n)
day_out["slowk"], day_out["slowd"] = stream.STOCH(h, l, c)
day_out["mom_" + str(n)] = stream.MOM(c, n)
day_out["volume"] = v[-1] / 10000
# print(day_out["macd_2"], day_out["macdsignal_2"], day_out["macdhist_2"])
output = output.append(day_out, ignore_index=True)
# print(output.tail(5)["macd_3"], output.tail(5)["macdsignal_3"], output.tail(5)["macdhist_3"])
return output
end_date_time = datetime.datetime.now() # mysqldb.get_maxdate()[0]
start_date_time = end_date_time - relativedelta(
days=constants.DAYS_FOR_TECHNICALS)
df_mkt = mongodao.getSymbolDataWithSymbol("SPY", start_date_time,
end_date_time)
symbol = "MSFT"
df_symbol = mongodao.getSymbolDataWithSymbol(symbol, start_date_time,
end_date_time)
df_symbol_close = df_symbol[['close']]
df_mkt_close = df_mkt[['close']]
mom = abstract.MOM(df_symbol_close, timeperiod=5)
df_merged = abstract.MACD(df_symbol_close,
fastperiod=12,
slowperiod=26,
signalperiod=9)
#df_merged=macd.apply(np.round)
df_std = abstract.STDDEV(df_symbol_close.pct_change(), timeperiod=100)
df_merged['stddev'] = df_std
df_merged['volatility'] = df_std * 100 * math.sqrt(252)
rsi = abstract.RSI(df_symbol_close).round(2)
rsi = abstract.RSI(df_symbol_close).round(2)
sma20 = abstract.SMA(df_symbol_close, timeperiod=20).round(2)
def populate_indicators(self, dataframe: DataFrame,
metadata: dict) -> DataFrame:
"""
Adds several different TA indicators to the given DataFrame
Performance Note: For the best performance be frugal on the number of indicators
you are using. Let uncomment only the indicator you are using in your strategies
or your hyperopt configuration, otherwise you will waste your memory and CPU usage.
:param dataframe: Dataframe with data from the exchange
:param metadata: Additional information, like the currently traded pair
:return: a Dataframe with all mandatory indicators for the strategies
"""
# Momentum Indicators
# ------------------------------------
# momentum
dataframe['mom'] = ta.MOM(dataframe['close'].values, 10)
dataframe['signal'] = 0
dataframe.loc[((dataframe['mom'] > 0) &
(dataframe['volume'] > 0) # Make sure Volume is not 0
), 'signal'] = 1
dataframe['signal'] = dataframe['signal'].diff()
# ADX
dataframe['adx'] = ta.ADX(dataframe)
# # Plus Directional Indicator / Movement
# dataframe['plus_dm'] = ta.PLUS_DM(dataframe)
# dataframe['plus_di'] = ta.PLUS_DI(dataframe)
# # Minus Directional Indicator / Movement
# dataframe['minus_dm'] = ta.MINUS_DM(dataframe)
# dataframe['minus_di'] = ta.MINUS_DI(dataframe)
# # Aroon, Aroon Oscillator
# aroon = ta.AROON(dataframe)
# dataframe['aroonup'] = aroon['aroonup']
# dataframe['aroondown'] = aroon['aroondown']
# dataframe['aroonosc'] = ta.AROONOSC(dataframe)
# # Awesome Oscillator
# dataframe['ao'] = qtpylib.awesome_oscillator(dataframe)
# # Keltner Channel
# keltner = qtpylib.keltner_channel(dataframe)
# dataframe["kc_upperband"] = keltner["upper"]
# dataframe["kc_lowerband"] = keltner["lower"]
# dataframe["kc_middleband"] = keltner["mid"]
# dataframe["kc_percent"] = (
# (dataframe["close"] - dataframe["kc_lowerband"]) /
# (dataframe["kc_upperband"] - dataframe["kc_lowerband"])
# )
# dataframe["kc_width"] = (
# (dataframe["kc_upperband"] - dataframe["kc_lowerband"]) / dataframe["kc_middleband"]
# )
# # Ultimate Oscillator
# dataframe['uo'] = ta.ULTOSC(dataframe)
# # Commodity Channel Index: values [Oversold:-100, Overbought:100]
# dataframe['cci'] = ta.CCI(dataframe)
# RSI
#dataframe['rsi'] = ta.RSI(dataframe)
# # Inverse Fisher transform on RSI: values [-1.0, 1.0] (https://goo.gl/2JGGoy)
# rsi = 0.1 * (dataframe['rsi'] - 50)
# dataframe['fisher_rsi'] = (np.exp(2 * rsi) - 1) / (np.exp(2 * rsi) + 1)
# # Inverse Fisher transform on RSI normalized: values [0.0, 100.0] (https://goo.gl/2JGGoy)
# dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1)
# # Stochastic Slow
# stoch = ta.STOCH(dataframe)
# dataframe['slowd'] = stoch['slowd']
# dataframe['slowk'] = stoch['slowk']
# Stochastic Fast
#stoch_fast = ta.STOCHF(dataframe)
#dataframe['fastd'] = stoch_fast['fastd']
#dataframe['fastk'] = stoch_fast['fastk']
# # Stochastic RSI
# stoch_rsi = ta.STOCHRSI(dataframe)
# dataframe['fastd_rsi'] = stoch_rsi['fastd']
# dataframe['fastk_rsi'] = stoch_rsi['fastk']
# MACD
#macd = ta.MACD(dataframe)
#dataframe['macd'] = macd['macd']
#dataframe['macdsignal'] = macd['macdsignal']
#dataframe['macdhist'] = macd['macdhist']
# MFI
#dataframe['mfi'] = ta.MFI(dataframe)
# # ROC
# dataframe['roc'] = ta.ROC(dataframe)
# Overlap Studies
# ------------------------------------
# Bollinger Bands
#bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2)
#dataframe['bb_lowerband'] = bollinger['lower']
#dataframe['bb_middleband'] = bollinger['mid']
#dataframe['bb_upperband'] = bollinger['upper']
#dataframe["bb_percent"] = (
# (dataframe["close"] - dataframe["bb_lowerband"]) /
# (dataframe["bb_upperband"] - dataframe["bb_lowerband"])
#)
#dataframe["bb_width"] = (
# (dataframe["bb_upperband"] - dataframe["bb_lowerband"]) / dataframe["bb_middleband"]
#)
# Bollinger Bands - Weighted (EMA based instead of SMA)
# weighted_bollinger = qtpylib.weighted_bollinger_bands(
# qtpylib.typical_price(dataframe), window=20, stds=2
# )
# dataframe["wbb_upperband"] = weighted_bollinger["upper"]
# dataframe["wbb_lowerband"] = weighted_bollinger["lower"]
# dataframe["wbb_middleband"] = weighted_bollinger["mid"]
# dataframe["wbb_percent"] = (
# (dataframe["close"] - dataframe["wbb_lowerband"]) /
# (dataframe["wbb_upperband"] - dataframe["wbb_lowerband"])
# )
# dataframe["wbb_width"] = (
# (dataframe["wbb_upperband"] - dataframe["wbb_lowerband"]) / dataframe["wbb_middleband"]
# )
# # EMA - Exponential Moving Average
# dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3)
# dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5)
# dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10)
# dataframe['ema21'] = ta.EMA(dataframe, timeperiod=21)
# dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50)
# dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100)
# # SMA - Simple Moving Average
# dataframe['sma3'] = ta.SMA(dataframe, timeperiod=3)
# dataframe['sma5'] = ta.SMA(dataframe, timeperiod=5)
# dataframe['sma10'] = ta.SMA(dataframe, timeperiod=10)
# dataframe['sma21'] = ta.SMA(dataframe, timeperiod=21)
# dataframe['sma50'] = ta.SMA(dataframe, timeperiod=50)
# dataframe['sma100'] = ta.SMA(dataframe, timeperiod=100)
# Parabolic SAR
#dataframe['sar'] = ta.SAR(dataframe)
# TEMA - Triple Exponential Moving Average
#dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9)
# Cycle Indicator
# ------------------------------------
# Hilbert Transform Indicator - SineWave
#hilbert = ta.HT_SINE(dataframe)
#dataframe['htsine'] = hilbert['sine']
#dataframe['htleadsine'] = hilbert['leadsine']
# Pattern Recognition - Bullish candlestick patterns
# ------------------------------------
# # Hammer: values [0, 100]
# dataframe['CDLHAMMER'] = ta.CDLHAMMER(dataframe)
# # Inverted Hammer: values [0, 100]
# dataframe['CDLINVERTEDHAMMER'] = ta.CDLINVERTEDHAMMER(dataframe)
# # Dragonfly Doji: values [0, 100]
# dataframe['CDLDRAGONFLYDOJI'] = ta.CDLDRAGONFLYDOJI(dataframe)
# # Piercing Line: values [0, 100]
# dataframe['CDLPIERCING'] = ta.CDLPIERCING(dataframe) # values [0, 100]
# # Morningstar: values [0, 100]
# dataframe['CDLMORNINGSTAR'] = ta.CDLMORNINGSTAR(dataframe) # values [0, 100]
# # Three White Soldiers: values [0, 100]
# dataframe['CDL3WHITESOLDIERS'] = ta.CDL3WHITESOLDIERS(dataframe) # values [0, 100]
# Pattern Recognition - Bearish candlestick patterns
# ------------------------------------
# # Hanging Man: values [0, 100]
# dataframe['CDLHANGINGMAN'] = ta.CDLHANGINGMAN(dataframe)
# # Shooting Star: values [0, 100]
# dataframe['CDLSHOOTINGSTAR'] = ta.CDLSHOOTINGSTAR(dataframe)
# # Gravestone Doji: values [0, 100]
# dataframe['CDLGRAVESTONEDOJI'] = ta.CDLGRAVESTONEDOJI(dataframe)
# # Dark Cloud Cover: values [0, 100]
# dataframe['CDLDARKCLOUDCOVER'] = ta.CDLDARKCLOUDCOVER(dataframe)
# # Evening Doji Star: values [0, 100]
# dataframe['CDLEVENINGDOJISTAR'] = ta.CDLEVENINGDOJISTAR(dataframe)
# # Evening Star: values [0, 100]
# dataframe['CDLEVENINGSTAR'] = ta.CDLEVENINGSTAR(dataframe)
# Pattern Recognition - Bullish/Bearish candlestick patterns
# ------------------------------------
# # Three Line Strike: values [0, -100, 100]
# dataframe['CDL3LINESTRIKE'] = ta.CDL3LINESTRIKE(dataframe)
# # Spinning Top: values [0, -100, 100]
# dataframe['CDLSPINNINGTOP'] = ta.CDLSPINNINGTOP(dataframe) # values [0, -100, 100]
# # Engulfing: values [0, -100, 100]
# dataframe['CDLENGULFING'] = ta.CDLENGULFING(dataframe) # values [0, -100, 100]
# # Harami: values [0, -100, 100]
# dataframe['CDLHARAMI'] = ta.CDLHARAMI(dataframe) # values [0, -100, 100]
# # Three Outside Up/Down: values [0, -100, 100]
# dataframe['CDL3OUTSIDE'] = ta.CDL3OUTSIDE(dataframe) # values [0, -100, 100]
# # Three Inside Up/Down: values [0, -100, 100]
# dataframe['CDL3INSIDE'] = ta.CDL3INSIDE(dataframe) # values [0, -100, 100]
# # Chart type
# # ------------------------------------
# # Heikin Ashi Strategy
# heikinashi = qtpylib.heikinashi(dataframe)
# dataframe['ha_open'] = heikinashi['open']
# dataframe['ha_close'] = heikinashi['close']
# dataframe['ha_high'] = heikinashi['high']
# dataframe['ha_low'] = heikinashi['low']
# Retrieve best bid and best ask from the orderbook
# ------------------------------------
"""
# first check if dataprovider is available
if self.dp:
if self.dp.runmode in ('live', 'dry_run'):
ob = self.dp.orderbook(metadata['pair'], 1)
dataframe['best_bid'] = ob['bids'][0][0]
dataframe['best_ask'] = ob['asks'][0][0]
"""
return dataframe
def build_data(raw = False, random_split = True, start_date = None, end_date = None, test_proportion = 0.1):
# if len(sec) == 1 and os.path.isfile(secs[0]): #it's a file
# with open(secs[0]) as f:
# secs = ["WIKI/" + line.strip() for line in f]
# print("SECURITIES: ", s[5:] for s in secs)
with open("stock_data/invalid_stocks.txt", "r+") as f:
invalid_stock_codes = [line.strip() for line in f]
f = open("stock_data/invalid_stocks.txt", "a")
stock_code = yield
while True and stock_code is not None:
valid_stock = False
while not valid_stock:
if "." in stock_code:
stock_code = yield None
continue
if stock_code in invalid_stock_codes:
# print("Skipping security", sec)
stock_code = yield None
continue
valid_stock = True
sec = stock_code.split("/")[1] # Just the ticker, not the database code
pickle_name = sec
if raw:
pickle_name += "_raw"
if not random_split:
pickle_name += "_notrand"
if start_date and end_date:
pickle_name += start_date + "to" + end_date
elif start_date:
pickle_name += start_date
elif end_date:
pickle_name += "to" + end_date
if not os.path.isfile("./stock_data/" + pickle_name + "_data.pickle"):
# print("No pickle found, getting data for", sec)
try:
# print("Getting data for", stock_code)
df = quandl.get(stock_code, start_date = start_date, end_date = end_date)
except quandl.errors.quandl_error.NotFoundError:
invalid_stock_codes += [stock_code]
f.write(stock_code + "\n")
stock_code = yield None
continue
if "Adj. Close" in df.columns:
df = df[["Adj. Open", "Adj. High", "Adj. Low", "Adj. Close", "Adj. Volume"]]
df.rename(columns=lambda x: x[5:].lower(), inplace=True) # Remove the "Adj. " and make lowercase
elif "Close" in df.columns:
df = df[["Open", "High", "Low", "Close", "Volume"]]
df.rename(columns=lambda x: x.lower(), inplace=True) # make lowercase
price = df['close'].values
minIdxs = argrelextrema(price, np.less)
maxIdxs = argrelextrema(price, np.greater)
Y = pd.Series(name="signal", dtype=np.ndarray, index=range(0, len(price)))
n=0
for _, idx in np.ndenumerate(minIdxs):
# if idx < MIN_MAX_PERIOD: continue
max_price = max(price[idx: idx + MIN_MAX_PERIOD])
if ((max_price - price[idx]) / price[idx]) > HI_LO_DIFF: #if the difference between max and min is > X%
Y.set_value(idx, np.array([1., 0.], np.float32))
n+=1
# print("MINS:", n)
n=0
for _, idx in np.ndenumerate(maxIdxs):
# if idx < MIN_MAX_PERIOD: continue
min_price = min(price[idx: idx + MIN_MAX_PERIOD])
if ((price[idx] - min_price)/ min_price) > HI_LO_DIFF: #if the difference between max and min is > X%
Y.set_value(idx, np.array([0., 1.], np.float32))
n+=1
# print("MAXS:", n)
_min_idx, _max_idx = 0, 0
for i, y in np.ndenumerate(Y.values):
if np.array_equal(y, [1., 0.]):
_min_idx = i[0]
elif np.array_equal(y, [0., 1.]):
_max_idx = i[0]
else:
if _min_idx > _max_idx:
s = np.array([1., 0.])
elif _max_idx > _min_idx:
s = np.array([0., 1.])
else:
s = np.array([0., 0.]) # no action taken, only occurs at the beginnings of datasets, afaik
Y.set_value(i, s, np.float32)
# x = list(zip(price[0:50], Y.values[0:50]))
# for i in x:
# print("{0:.2f} -- {1}".format(i[0], "sell" if np.array_equal(i[1], [0, 1]) else "buy" if np.array_equal(i[1], [1, 0]) else "nothing"))
df.reset_index(drop=True, inplace = True)
if isinstance(price, np.ndarray):
price = price.tolist()
''' INDICATORS '''
# print(len(df), len(Y))
# print("Building indicators...")
inputs = df.to_dict(orient="list")
for col in inputs:
inputs[col] = np.array(inputs[col])
for n in range(2, 40):
inputs["bband_u_"+str(n)], inputs["bband_m_"+str(n)], inputs["bband_l_"+str(n)] = ta.BBANDS(inputs, n)
inputs["sma_"+str(n)] = ta.SMA(inputs, timeperiod = n)
inputs["adx_"+str(n)] = ta.ADX(inputs, timeperiod = n)
inputs["macd_"+str(n)], inputs["macdsignal_"+str(n)], inputs["macdhist_"+str(n)] = ta.MACD(inputs, n, n*2, n*2/3)
inputs["mfi_"+str(n)] = ta.MFI(inputs, n)
inputs["ult_"+str(n)] = ta.ULTOSC(inputs, n, n*2, n*4)
inputs["willr_"+str(n)] = ta.WILLR(inputs, n)
inputs["slowk"], inputs["slowd"] = ta.STOCH(inputs)
inputs["mom_"+str(n)] = ta.MOM(inputs, n)
inputs["mom_"+str(n)] = ta.MOM(inputs, n)
inputs["volume"] = list(map(lambda x: x/10000, inputs["volume"]))
df = pd.DataFrame().from_dict(inputs)
broken = False
for idx, val in reversed(list(df.isnull().any(axis=1).iteritems())):
if val == True:
# print(actual_idx, val)
df.drop(idx, inplace = True)
Y.drop(idx, inplace = True)
try:
# price[actual_idx] = None
price.pop(idx)
except IndexError: #drop the security
# print("Error, dropping security", sec)
broken = True
break
# print(list(df.isnull().any(axis=1).iteritems()))
# print("PRICES", price)
# print(len(price), len(df.values))
# for i, p in reversed(list(enumerate(price))):
# actual_idx = len(price) - 1 - i
# if p is None:
# print(actual_idx)
# price.pop(actual_idx)
''' BUILD NEURAL NET INPUTS '''
if not broken:
Y = np.vstack(Y.values)
print(df["adx_10"])
X = df.values
# print(X[0:2])
if not raw:
rand = "_notrand" if not random_split else ""
if not os.path.isfile("./stock_data/" + sec + rand + ".scaler"):
scaler = prep.StandardScaler().fit(X)
X_norm = scaler.transform(X)
joblib.dump(scaler, "./stock_data/" + sec + rand + ".scaler")
else:
scaler = joblib.load("./stock_data/" + sec + rand + ".scaler")
X_norm = scaler.transform(X)
else:
X_norm = X
if random_split:
trX, testX, trY, testY = train_test_split(X_norm, Y, test_size = test_proportion, random_state=0)
else: # just clips the test data off the end
l = len(X_norm)
trX, testX = X_norm[:int(-test_proportion*l)], X_norm[int(-test_proportion*l):]
trY, testY = Y[:int(-test_proportion*l)], Y[int(-test_proportion*l):]
# print("Pickling...")
output = {"X_norm": X_norm, "Y": Y, "trX": trX, "trY": trY, "testX": testX, "testY": testY, "price": price}
pickle.dump(output, open("./stock_data/" + pickle_name + "_data.pickle", "wb"))
stock_code = yield output
else:
invalid_stock_codes += [stock_code]
f.write(stock_code + "\n")
stock_code = yield None
else:
# print("Pickle found, loading...")
_data = pickle.load(open("./stock_data/" + pickle_name + "_data.pickle", "rb"))
trX, trY, testX, testY, price, X_norm, Y = _data["trX"], _data["trY"], _data["testX"], _data["testY"], _data["price"], _data["X_norm"], _data["Y"]
stock_code = yield {"X_norm": X_norm, "Y": Y, "trX": trX, "trY": trY, "testX": testX, "testY": testY, "price": price}
def calculate_technical(df_symbol, symbol, df_mkt, start_date_time,
end_date_time, hist_dates, days_back):
#list_drop_cloumns = [ 'open', 'high','low','volume']
df_symbol_close = df_symbol[['close']]
df_mkt_close = df_mkt[['close']]
mom = abstract.MOM(df_symbol_close, timeperiod=5)
df_merged = abstract.MACD(df_symbol_close,
fastperiod=12,
slowperiod=26,
signalperiod=9)
#df_merged=macd.apply(np.round)
df_std = abstract.STDDEV(df_symbol_close.pct_change(), timeperiod=100)
df_merged['stddev'] = df_std
df_merged['volatility'] = df_std * 100 * math.sqrt(252)
rsi = abstract.RSI(df_symbol_close).round(2)
sma20 = abstract.SMA(df_symbol_close, timeperiod=20).round(2)
sma100 = abstract.SMA(df_symbol_close, timeperiod=100).round(2)
sma200 = abstract.SMA(df_symbol_close, timeperiod=200).round(2)
sma3 = abstract.SMA(df_symbol_close, timeperiod=3).round(2)
sma5 = abstract.SMA(df_symbol_close, timeperiod=5).round(2)
sma9 = abstract.SMA(df_symbol_close, timeperiod=9).round(2)
sma13 = abstract.SMA(df_symbol_close, timeperiod=13).round(2)
sma25 = abstract.SMA(df_symbol_close, timeperiod=25).round(2)
sma50 = abstract.SMA(df_symbol_close, timeperiod=50).round(2)
sma90 = abstract.SMA(df_symbol_close, timeperiod=90).round(2)
sma36 = abstract.SMA(df_symbol_close, timeperiod=36).round(2)
sma150 = abstract.SMA(df_symbol_close, timeperiod=150).round(2)
df_merged['mom'] = mom
df_merged['sma20'] = sma20
df_merged['sma50'] = sma50
df_merged['sma100'] = sma100
df_merged['sma200'] = sma200
df_merged['rsi'] = rsi
df_merged['close'] = df_symbol['close']
df_merged['open'] = df_symbol['open']
df_merged['low'] = df_symbol['low']
df_merged['high'] = df_symbol['high']
df_merged['volume'] = df_symbol['volume']
df_merged['sma_volume_6month'] = pd.rolling_mean(df_merged['volume'],
window=120).round(2)
df_merged = df_merged.dropna()
df_merged['symbol'] = symbol
df_merged['rsi_value'] = df_merged['rsi'].apply(
rsi_manager.calculate_rsi_values)
df_merged['sma3'] = sma3
df_merged['sma5'] = sma5
df_merged['sma9'] = sma9
df_merged['sma13'] = sma13
df_merged['sma20'] = sma20
df_merged['sma25'] = sma25
df_merged['sma50'] = sma50
df_merged['sma90'] = sma90
df_merged['sma36'] = sma36
df_merged['sma150'] = sma150
df_merged['Relative_strength'] = relative_strength(
df_symbol_close, df_mkt_close, symbol)['Relative_strength']
df_merged['stdabove'] = df_merged.apply(calculate_stdabove, axis=1)
df_merged['date'] = df_merged.index
df_res = df_symbol.apply(calc_res, axis=1)
df_merged = pd.concat([df_merged, df_res], axis=1)
df_rating = rating_manager.calc_rating_history(df_merged, days_back,
symbol)
df_trends = df_merged.apply(
lambda row: trend_manager.trend_calculation(row), axis=1)
df_merged = pd.concat([df_merged, df_trends], axis=1)
crossover_manager.TrendChangePositive(df_merged, "short_trend",
alert_constants.TREND_SHORT)
crossover_manager.TrendChangePositive(df_merged, "inter_trend",
alert_constants.TREND_INTERMEDIATE)
crossover_manager.TrendChangePositive(df_merged, "long_trend",
alert_constants.TREND_LONG)
crossover_manager.TrendChangeNegative(df_merged, "short_trend",
alert_constants.TREND_SHORT)
crossover_manager.TrendChangeNegative(df_merged, "inter_trend",
alert_constants.TREND_INTERMEDIATE)
crossover_manager.TrendChangeNegative(df_merged, "long_trend",
alert_constants.TREND_LONG)
df_merged['rating'] = df_rating['rating']
df_merged = df_merged.replace([np.inf, -np.inf], np.nan)
df_merged = df_merged.dropna()
print "********************************************************************************"
print "********************************************************************************"
print "********************************************************************************"
#print df_merged
if (df_merged is None or df_merged.symbol.count() == 0):
return
logger.info("Saving history for Symbol " + symbol + " length = " +
str(len(df_merged)))
df_merged.set_index('date', inplace=True)
alert_manager.relative_strength(df_merged)
alert_manager.fullGapPositive(df_merged)
alert_manager.fullGapNegative(df_merged)
alert_manager.partialGapPositive(df_merged)
alert_manager.partialGapNegative(df_merged)
alert_manager.keyReversalPositive(df_merged)
alert_manager.keyReversalNegative(df_merged)
alert_manager.volumePositive(df_merged)
alert_manager.volumeNegative(df_merged)
crossover_manager.smacrossovers(df_merged)
crossover_manager.macd_crossovers(df_merged)
crossover_manager.obos_alerts(df_merged)
dbdao.save_dataframe(df_merged, "df_history")
#alert_manager.relative_strength(df_merged, df_spy, symbol)
df_merged['stdabove_prev'] = df_merged['stdabove'].shift(1)
#latest data calculations
return_data = {}
return_data.update(calculate_beta(df_symbol_close, df_mkt_close, symbol))
return_data.update(calculate_prices_at_dates(df_symbol_close, hist_dates))
monthly_date = hist_dates['Monthly']
weekly_date = hist_dates['Weekly']
df_latest = df_merged.tail(1)
df_latest_sign = df_latest.apply(calc_signs, axis=1)
df_latest = pd.concat([df_latest, df_latest_sign], axis=1)
#df_latest[['sma_20day_sign']]=df_latest.apply(calc_signs,axis=1)
df_latest['volatility_monthly'] = price_manager.get_specific_date_value(
df_merged, monthly_date, 'volatility')
df_latest['volatility_weekly'] = price_manager.get_specific_date_value(
df_merged, weekly_date, 'volatility')
df_latest['std50days'] = df_latest['stdabove']
df_latest['date'] = df_latest.index
df_latest.set_index('date', inplace=True)
for key, value in return_data.iteritems():
df_latest[key] = value
return df_latest
def MOM(self): #10
mom = abstract.MOM(self.company_stock, timeperiod=10)
self.company_stock['MOM'] = mom
def _build_indicators(data):
# sliding_window = []
while True:
data_with_ind = []
for df in data:
df = copy.deepcopy(df)
if "Adj. Close" in df.columns:
df = df[["Adj. Open", "Adj. High", "Adj. Low", "Adj. Close", "Adj. Volume"]]
df.rename(columns=lambda x: x[5:].lower(), inplace=True) # Remove the "Adj. " and make lowercase
elif "Close" in df.columns:
df = df[["Open", "High", "Low", "Close", "Volume"]]
df.rename(columns=lambda x: x.lower(), inplace=True) # make lowercase
df.reset_index(drop=True, inplace = True)
inputs = df.to_dict(orient="list")
for col in inputs:
inputs[col] = np.array(inputs[col])
for n in range(2, 40):
inputs["bband_u_"+str(n)], inputs["bband_m_"+str(n)], inputs["bband_l_"+str(n)] = ta.BBANDS(inputs, n)
inputs["sma_"+str(n)] = ta.SMA(inputs, timeperiod = n)
inputs["adx_"+str(n)] = ta.ADX(inputs, timeperiod = n)
inputs["macd_"+str(n)], inputs["macdsignal_"+str(n)], inputs["macdhist_"+str(n)] = ta.MACD(inputs, n, n*2, n*2/3)
inputs["mfi_"+str(n)] = ta.MFI(inputs, n)
inputs["ult_"+str(n)] = ta.ULTOSC(inputs, n, n*2, n*4)
inputs["willr_"+str(n)] = ta.WILLR(inputs, n)
inputs["slowk"], inputs["slowd"] = ta.STOCH(inputs)
inputs["mom_"+str(n)] = ta.MOM(inputs, n)
inputs["mom_"+str(n)] = ta.MOM(inputs, n)
inputs["volume"] = list(map(lambda x: x/10000, inputs["volume"]))
df = pd.DataFrame().from_dict(inputs)
price = df["close"].values
if isinstance(price, np.ndarray):
price = price.tolist()
for idx, val in df.isnull().any(axis=1).iteritems():
if val == True:
df.drop(idx, inplace = True)
try:
price[idx] = None
except IndexError: #drop the security
print("Error, failed to drop price on index", idx)
sys.exit(1)
# print("Dropped index:", idx)
for i, p in reversed(list(enumerate(price))):
actual_idx = len(price) - 1 - i
if p == None:
price.pop(actual_idx)
print(df["adx_10"])
X = df.values
data_with_ind += [{"data": X, "price": price}]
return data_with_ind
def _build_indicators(num_secs): # accepts a list of one-day Series
sec_idx_range = range(num_secs)
sliding_window = [] # list of pd.DataFrames
data = yield
for datum in data:
sliding_window += [_rename_columns(datum)]
current_day = 0
while True:
passes_validity_check, num_validation_iterations = False, 0
# time.sleep(1)
while not passes_validity_check:
for i in sec_idx_range: # for each security
# print("Current day:", current_day)
if current_day != 0:
if current_day > 170 and num_validation_iterations == 0:
sliding_window[i] = sliding_window[i].iloc[
1:] # pop the first
for datum in data:
if num_validation_iterations == 0:
sliding_window[i] = sliding_window[i].append(
_rename_columns(datum))
data_with_ind = []
series = sliding_window[i]
series = series.reset_index(drop=True)
inputs = series.to_dict(orient="list")
for col in inputs:
inputs[col] = np.array(inputs[col])
c = series.close
for n in range(2, 40):
inputs["bband_u_" +
str(n)], inputs["bband_m_" +
str(n)], inputs["bband_l_" +
str(n)] = ta.BBANDS(
inputs, n)
inputs["sma_" + str(n)] = ta.SMA(inputs, timeperiod=n)
inputs["adx_" + str(n)] = ta.ADX(inputs, timeperiod=n)
# print("\nINPUTS:", inputs)
# if current_day > n*2:
fast_ema = c.ewm(span=n).mean()
slow_ema = c.ewm(span=n * 2).mean()
# print(fast_ema, slow_ema)
macd1 = fast_ema - slow_ema
macd2 = macd1.ewm(span=n * 2 / 3).mean()
macd3 = macd1 - macd2
inputs["macd_" +
str(n)], inputs["macdsignal_" + str(n)], inputs[
"macdhist_" + str(n)] = macd1.iloc[
-1], macd2.iloc[-1], macd3.iloc[-1]
if current_day == 160:
print(n)
print(macd1, macd2, macd3)
sys.exit(69)
# else:
# inputs["macd_"+str(n)], inputs["macdsignal_"+str(n)], inputs["macdhist_"+str(n)] = [np.NaN]*3
inputs["mfi_" + str(n)] = ta.MFI(inputs, n)
inputs["ult_" + str(n)] = ta.ULTOSC(
inputs, n, n * 2, n * 4)
inputs["willr_" + str(n)] = ta.WILLR(inputs, n)
inputs["slowk"], inputs["slowd"] = ta.STOCH(inputs)
inputs["mom_" + str(n)] = ta.MOM(inputs, n)
inputs["mom_" + str(n)] = ta.MOM(inputs, n)
inputs["volume"] = list(
map(lambda x: x / 10000, inputs["volume"]))
series = pd.DataFrame().from_dict(inputs)
price = series["close"].iloc[-1]
if isinstance(price, np.ndarray):
price = price.tolist()
# for idx, val in series.isnull().any(axis=1).iteritems():
# if val == True:
# series.drop(idx, inplace = True)
# try:
# price[idx] = None
# except IndexError: #drop the security
# print("Error, failed to drop price on index", idx)
# sys.exit(1)
# # print("Dropped index:", idx)
# for i, p in reversed(list(enumerate(price))):
# actual_idx = len(price) - 1 - i
# if p == None:
# price.pop(actual_idx)
# print(series["adx_10"])
X = series.iloc[-1].values
if current_day < 170:
passes_validity_check = True
elif not np.isnan(X).any():
passes_validity_check = True
# if num_validation_iterations != 0:
# with pd.option_context('display.max_rows', None, 'display.max_columns', None):
# print(series.iloc[-1])
# sys.exit(1)
else:
num_validation_iterations += 1
print("Reevaluating, iteration", num_validation_iterations,
"day:", current_day)
# with pd.option_context('display.max_rows', None, 'display.max_columns', None):
# print(series.iloc[-1])
# sys.exit(1)
# if current_day > 170:
# print(series.iloc[-1].values)
# if np.isnan(X).any() and current_day > 170:
# # with pd.option_context('display.max_rows', None, 'display.max_columns', None):
# # print(series)
# print(sliding_window[0])
# break
# print("ADX_10:\n", series["adx_10"].tail(3))
# if current_day == 900:
# print(series)
# print(X)
data_with_ind += [{"data": X, "price": round(price, 2)}]
data = yield data_with_ind
current_day += 1
def build_data_to_dict(secs, raw = False):
PICKLE_NAME = "_".join(s[5:] for s in secs)
print("SECURITIES: ", PICKLE_NAME.split("_"))
if not os.path.isfile("./stock_data/" + PICKLE_NAME + "_data.pickle"):
print("No pickle found, getting data...")
# df = pd.concat([quandl.get("WIKI/AAPL"), quandl.get("WIKI/F"), quandl.get("WIKI/XOM")])
df = pd.DataFrame()
Y = pd.Series()
prices = []
for sec in secs:
sec_df = quandl.get(sec)
if "Adj. Close" in sec_df.columns:
sec_df = sec_df[["Adj. Open", "Adj. High", "Adj. Low", "Adj. Close", "Adj. Volume"]]
sec_df.rename(columns=lambda x: x[5:].lower(), inplace=True) # Remove the "Adj. " and make lowercase
elif "Close" in sec_df.columns:
sec_df = sec_df[["Open", "High", "Low", "Close", "Volume"]]
sec_df.rename(columns=lambda x: x.lower(), inplace=True) # make lowercase
print("Calculating output for", sec)
price = sec_df['close'].values
minIdxs = argrelextrema(price, np.less)
maxIdxs = argrelextrema(price, np.greater)
sec_Y = pd.Series(name="signal", dtype=np.ndarray, index=range(0, len(price)))
n=0
for _, idx in np.ndenumerate(minIdxs):
if idx < MIN_MAX_PERIOD: continue
max_price = max(price[idx: idx + MIN_MAX_PERIOD])
if ((max_price - price[idx]) / price[idx]) > HI_LO_DIFF: #if the difference between max and min is > 2%
sec_Y.set_value(idx, np.array([1, 0, 0], np.int32))
n+=1
print("MINS:", n)
n=0
for _, idx in np.ndenumerate(maxIdxs):
if idx < MIN_MAX_PERIOD: continue
min_price = min(price[idx: idx + MIN_MAX_PERIOD])
if ((price[idx] - min_price)/ min_price) > HI_LO_DIFF: #if the difference between max and min is > 2%
sec_Y.set_value(idx, np.array([0, 0, 1], np.int32))
n+=1
print("MAXS:", n)
for idx in pd.isnull(sec_Y).nonzero()[0]:
sec_Y.set_value(idx, np.array([0, 1, 0], np.int32))
sec_df.reset_index(drop=True, inplace = True)
if isinstance(price, np.ndarray):
price = price.tolist()
''' INDICATORS '''
# print(len(sec_df), len(sec_Y))
print("Building indicators...")
inputs = sec_df.to_dict(orient="list")
for col in inputs:
inputs[col] = np.array(inputs[col])
for n in range(2, 40):
inputs["bband_u_"+str(n)], inputs["bband_m_"+str(n)], inputs["bband_l_"+str(n)] = ta.BBANDS(inputs, n)
inputs["sma_"+str(n)] = ta.SMA(inputs, timeperiod = n)
inputs["adx_"+str(n)] = ta.ADX(inputs, timeperiod = n)
inputs["macd_"+str(n)], inputs["macdsignal_"+str(n)], inputs["macdhist_"+str(n)] = ta.MACD(inputs, n, n*2, n*2/3)
inputs["mfi_"+str(n)] = ta.MFI(inputs, n)
inputs["ult_"+str(n)] = ta.ULTOSC(inputs, n, n*2, n*4)
inputs["willr_"+str(n)] = ta.WILLR(inputs, n)
inputs["slowk"], inputs["slowd"] = ta.STOCH(inputs)
inputs["mom_"+str(n)] = ta.MOM(inputs, n)
inputs["mom_"+str(n)] = ta.MOM(inputs, n)
inputs["volume"] = list(map(lambda x: x/10000, inputs["volume"]))
sec_df = pd.DataFrame().from_dict(inputs)
# print(sec_df.isnull().any(axis=1))
for idx, val in sec_df.isnull().any(axis=1).iteritems():
if val == True:
# print(idx, val)
sec_df.drop(idx, inplace = True)
sec_Y.drop(idx, inplace = True)
price.pop(idx)
prices.append(price)
df = pd.concat([df, sec_df])
Y = pd.concat([Y, sec_Y])
prices = [j for i in prices for j in i] # spooky magic
''' BUILD NEURAL NET INPUTS '''
Y = np.vstack(Y.values)
X = df.values
if not raw:
scaler = prep.StandardScaler().fit(X)
X_norm = scaler.transform(X)
from sklearn.externals import joblib
joblib.dump(scaler, "./stock_data/" + sec + ".scaler")
else:
X_norm = X
trX, testX, trY, testY= train_test_split(X_norm, Y, test_size = 0.1, random_state=0)
# print("Pickling...")
output = {"X_norm": X_norm, "Y": Y, "trX": trX, "trY": trY, "testX": testX, "testY": testY, "price": price}
pickle.dump(output, open("./stock_data/" + (PICKLE_NAME if not raw else PICKLE_NAME + "_raw") + "_data.pickle", "wb"))
return output
else:
print("Pickle found, loading...")
_data = pickle.load(open("./stock_data/" + PICKLE_NAME + "_data.pickle", "rb"))
trX, trY, testX, testY, price, X_norm, Y = _data["trX"], _data["trY"], _data["testX"], _data["testY"], _data["price"], _data["X_norm"], _data["Y"]
return {"X_norm": X_norm, "Y": Y, "trX": trX, "trY": trY, "testX": testX, "testY": testY, "price": price}
