def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: dataframe['adx'] = ta.ADX(dataframe, timeperiod=14) dataframe['plus_di'] = ta.PLUS_DI(dataframe, timeperiod=25) dataframe['minus_di'] = ta.MINUS_DI(dataframe, timeperiod=25) dataframe['sar'] = ta.SAR(dataframe) dataframe['mom'] = ta.MOM(dataframe, timeperiod=14) dataframe['sell-adx'] = ta.ADX(dataframe, timeperiod=14) dataframe['sell-plus_di'] = ta.PLUS_DI(dataframe, timeperiod=25) dataframe['sell-minus_di'] = ta.MINUS_DI(dataframe, timeperiod=25) dataframe['sell-sar'] = ta.SAR(dataframe) dataframe['sell-mom'] = ta.MOM(dataframe, timeperiod=14) return dataframe
def analyze(self, historical_data, period_count=10, hot_thresh=None, cold_thresh=None, all_data=False): """Performs momentum analysis on the historical data Args: historical_data (list): A matrix of historical OHCLV data. period_count (int, optional): Defaults to 10. The number of data points to consider for our simple moving average. hot_thresh (float, optional): Defaults to None. The threshold at which this might be good to purchase. cold_thresh (float, optional): Defaults to None. The threshold at which this might be good to sell. all_data (bool, optional): Defaults to False. If True, we return the momentum associated with each data point in our historical dataset. Otherwise just return the last one. Returns: dict: A dictionary containing a tuple of indicator values and booleans for buy / sell indication. """ dataframe = self.convert_to_dataframe(historical_data) mom_values = abstract.MOM(dataframe, period_count) analyzed_data = [(value,) for value in mom_values] return self.analyze_results( analyzed_data, is_hot=lambda v: v > hot_thresh if hot_thresh else False, is_cold=lambda v: v < cold_thresh if cold_thresh else False, all_data=all_data )
def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: # SMA - Simple Moving Average dataframe['fastMA'] = ta.SMA(dataframe, timeperiod=13) dataframe['slowMA'] = ta.SMA(dataframe, timeperiod=25) dataframe['mom'] = ta.MOM(dataframe, timeperiod=21) return dataframe
def populate_indicators(dataframe: DataFrame) -> DataFrame: """ Adds several different TA indicators to the given DataFrame """ dataframe['sar'] = ta.SAR(dataframe) dataframe['adx'] = ta.ADX(dataframe) stoch = ta.STOCHF(dataframe) dataframe['fastd'] = stoch['fastd'] dataframe['fastk'] = stoch['fastk'] dataframe['blower'] = ta.BBANDS(dataframe, nbdevup=2, nbdevdn=2)['lowerband'] dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) dataframe['mfi'] = ta.MFI(dataframe) dataframe['cci'] = ta.CCI(dataframe) dataframe['rsi'] = ta.RSI(dataframe) dataframe['mom'] = ta.MOM(dataframe) dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) dataframe['ao'] = awesome_oscillator(dataframe) macd = ta.MACD(dataframe) dataframe['macd'] = macd['macd'] dataframe['macdsignal'] = macd['macdsignal'] dataframe['macdhist'] = macd['macdhist'] return dataframe
def analyze(self, historical_data, period_count=10, signal=['momentum'], hot_thresh=None, cold_thresh=None): """Performs momentum analysis on the historical data Args: historical_data (list): A matrix of historical OHCLV data. period_count (int, optional): Defaults to 10. The number of data points to consider for our momentum. signal (list, optional): Defaults to momentum. The indicator line to check hot/cold against. hot_thresh (float, optional): Defaults to None. The threshold at which this might be good to purchase. cold_thresh (float, optional): Defaults to None. The threshold at which this might be good to sell. Returns: pandas.DataFrame: A dataframe containing the indicators and hot/cold values. """ dataframe = self.convert_to_dataframe(historical_data) mom_values = abstract.MOM(dataframe, period_count).to_frame() mom_values.dropna(how='all', inplace=True) mom_values.rename(columns={mom_values.columns[0]: 'momentum'}, inplace=True) if mom_values[signal[0]].shape[0]: mom_values['is_hot'] = mom_values[signal[0]] > hot_thresh mom_values['is_cold'] = mom_values[signal[0]] < cold_thresh return mom_values
def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: dataframe['adx'] = ta.ADX(dataframe, timeperiod=14) dataframe['plus_di'] = ta.PLUS_DI(dataframe, timeperiod=25) dataframe['minus_di'] = ta.MINUS_DI(dataframe, timeperiod=25) dataframe['sar'] = ta.SAR(dataframe) dataframe['mom'] = ta.MOM(dataframe, timeperiod=14) #print(f"\"{metadata['pair']}\"") return dataframe
def mom(self): mom = tb.MOM(self.dataframe, timeperiod=10) value = mom[len(mom) - 1] old_value = mom[len(mom) - 2] if value > old_value: return "buy" else: return "sell"
def populate_indicators(dataframe: DataFrame) -> DataFrame: """ Adds several different TA indicators to the given DataFrame """ # Log the dataframe dataframe.to_csv("dataframe.csv", sep='\t') #Determine Trend dataframe = indicators.TREND(dataframe) #Exponential Moving Average dataframe['ema'] = ta.EMA(dataframe, timeperiod=33) #Parabolic SAR dataframe['sar'] = ta.SAR(dataframe, 0.02, 0.22) #Average Directional Movement Index dataframe['adx'] = ta.ADX(dataframe) #MACD macd = ta.MACD(dataframe) dataframe['macd'] = macd['macd'] dataframe['macds'] = macd['macdsignal'] dataframe['macdh'] = macd['macdhist'] #Relative Strength Index dataframe['rsi'] = ta.RSI(dataframe, 14) dataframe = indicators.RSI(dataframe) #Absolute Price Oscillator dataframe['apo'] = ta.APO(dataframe, fastperiod=12, slowperiod=26, matype=0) #Momentum dataframe['mom'] = ta.MOM(dataframe, 10) #Bollinger Bands dataframe = indicators.BBANDS(20, dataframe, 2) # Stochcastic dataframe['K'] = indicators.STOK(dataframe, 14) dataframe['D'] = indicators.STOD(dataframe, 14) dataframe = indicators.STOCH(dataframe) # if Counter.counter < 30: # print(dataframe) # print (macd) # print (dataframe ['adx']) # Counter.counter = Counter.counter + 1 # else: # exit() return dataframe
def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: dataframe["adx"] = ta.ADX(dataframe, timeperiod=14) dataframe["plus_di"] = ta.PLUS_DI(dataframe, timeperiod=25) dataframe["minus_di"] = ta.MINUS_DI(dataframe, timeperiod=25) dataframe["sar"] = ta.SAR(dataframe) dataframe["mom"] = ta.MOM(dataframe, timeperiod=14) return dataframe
def generate_indicators(dataset, timeperiod=5, generate_target=True, reset_index=False): # To avoid changes from original dataset df = pd.DataFrame(dataset).copy() # To prevent index from being Date if (reset_index): df = df.reset_index() df.columns = df.columns.str.lower() # check the given dataset has necessary_columns __check_columns(df) df = df[necessary_columns] # df_indicators has all columns except date, this is for talib to produces other indicators df_indicators = df.iloc[:, 1:] # Produce other indicators RSI = abstract.RSI(df_indicators.close, timeperiod) RSI = pd.DataFrame({'RSI': RSI}) MOM = abstract.MOM(df_indicators.close, timeperiod) MOM = pd.DataFrame({'MOM': MOM}) KD = abstract.STOCH(df_indicators) KD = pd.DataFrame(KD) # KD has slowd and slowk MACD = abstract.MACD(df_indicators) MACD = pd.DataFrame(MACD) # MACD has its own column names ADX = abstract.ADX(df_indicators) ADX = pd.DataFrame({'ADX': ADX}) SMA = abstract.SMA(df_indicators.close) SMA = pd.DataFrame({'SMA': SMA}) upper, middle, lower = talib.BBANDS(df_indicators.close, matype=MA_Type.T3) bb_df = pd.DataFrame({ \ 'upper_bb' : upper, 'middel_bb' : middle, 'lower_bb' : lower}) # Combine all metrix frames = [df, RSI, MOM, KD, MACD, ADX, SMA, bb_df] combined = pd.concat(frames, axis=1) if (generate_target): target_name = 'next_' + str(timeperiod) + 'day_trend' combined[target_name] = np.where( combined.close.shift(-timeperiod) > combined.close, 1, 0) return combined
def analyze_mom(self, historial_data, period_count=10, hot_thresh=None, cold_thresh=None, all_data=False): """Performs momentum analysis on the historical data Args: historial_data (list): A matrix of historical OHCLV data. period_count (int, optional): Defaults to 10. The number of data points to consider for our simple moving average. hot_thresh (float, optional): Defaults to None. The threshold at which this might be good to purchase. cold_thresh (float, optional): Defaults to None. The threshold at which this might be good to sell. all_data (bool, optional): Defaults to False. If True, we return the momentum associated with each data point in our historical dataset. Otherwise just return the last one. Returns: dict: A dictionary containing a tuple of indicator values and booleans for buy / sell indication. """ dataframe = self.__convert_to_dataframe(historial_data) mom_values = abstract.MOM(dataframe, period_count) mom_result_data = [] for mom_value in mom_values: if math.isnan(mom_value): continue is_hot = False if hot_thresh is not None: is_hot = mom_value > hot_thresh is_cold = False if cold_thresh is not None: is_cold = mom_value < cold_thresh data_point_result = { 'values': (mom_value, ), 'is_cold': is_cold, 'is_hot': is_hot } mom_result_data.append(data_point_result) if all_data: return mom_result_data else: try: return mom_result_data[-1] except IndexError: return mom_result_data
def MOM(self): MOM = tb.MOM(self.dataframe, timeperiod=10) value = MOM[len(MOM) - 1] oldValue = MOM[len(MOM) - 2] if (value > oldValue): print("MOM: " + str(value) + " buy") return "buy" else: print("MOM: " + str(value) + " buy") return "sell"
def TA_processing(dataframe): bias(dataframe, days=[3, 6, 10, 25]) moving_average(dataframe, days=[5, 10, 20]) dataframe['ROC'] = abstract.ROC(dataframe, timeperiod=10) dataframe['MACD'] = abstract.MACD(dataframe, fastperiod=12, slowperiod=26, signalperiod=9)['macd'] dataframe['MACD_signal'] = abstract.MACD(dataframe, fastperiod=12, slowperiod=26, signalperiod=9)['macdsignal'] dataframe['UBBANDS'] = abstract.BBANDS(dataframe, timeperiod=20, nbdevup=2, nbdevdn=2, matype=0)['upperband'] dataframe['MBBANDS'] = abstract.BBANDS(dataframe, timeperiod=20, nbdevup=2, nbdevdn=2, matype=0)['middleband'] dataframe['LBBANDS'] = abstract.BBANDS(dataframe, timeperiod=20, nbdevup=2, nbdevdn=2, matype=0)['lowerband'] dataframe['%K'] = abstract.STOCH(dataframe, fastk_period=9)['slowk']/100 dataframe['%D'] = abstract.STOCH(dataframe, fastk_period=9)['slowd']/100 dataframe['W%R'] = abstract.WILLR(dataframe, timeperiod=14)/100 dataframe['RSI9'] = abstract.RSI(dataframe, timeperiod = 9)/100 dataframe['RSI14'] = abstract.RSI(dataframe, timeperiod = 14)/100 dataframe['CCI'] = abstract.CCI(dataframe, timeperiod=14)/100 counter_daily_potential(dataframe) dataframe['MOM'] = abstract.MOM(dataframe, timeperiod=10) dataframe['DX'] = abstract.DX(dataframe, timeperiod=14)/100 psy_line(dataframe) volumn_ratio(dataframe, d=26) on_balance_volume(dataframe)
def evaluate_momentum(self, period=20, prefix="momentum", impact_buy=1, impact_sell=1): """ evaluates the momentum :param dataframe: :param period: :param prefix: :return: """ self._weights(impact_buy, impact_sell) dataframe = self.dataframe name = f"{prefix}_{period}" dataframe[name] = ta.MOM(dataframe, timeperiod=period) dataframe.loc[((dataframe[name] > 100)), f"buy_{name}"] = 1 * impact_buy dataframe.loc[((dataframe[name] < 100)), f"sell_{name}"] = 1 * impact_sell
def evaluate_momentum(self, period=20, prefix="momentum", impact_buy=1, impact_sell=1): """ evaluates the osc :param dataframe: :param period: :param prefix: :return: """ self._weights(impact_buy, impact_sell) dataframe = self.dataframe name = '{}_{}'.format(prefix, period) dataframe[name] = ta.MOM(dataframe, timeperiod=period) dataframe.loc[((dataframe[name] > 100)), 'buy_{}'.format(name)] = (1 * impact_buy) dataframe.loc[((dataframe[name] < 100)), 'sell_{}'.format(name)] = (1 * impact_sell)
def populate_indicators(dataframe: DataFrame) -> DataFrame: """ Adds several different TA indicators to the given DataFrame """ dataframe['sar'] = ta.SAR(dataframe) dataframe['adx'] = ta.ADX(dataframe) stoch = ta.STOCHF(dataframe) dataframe['fastd'] = stoch['fastd'] dataframe['fastk'] = stoch['fastk'] dataframe['blower'] = ta.BBANDS(dataframe, nbdevup=2, nbdevdn=2)['lowerband'] dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) dataframe['mfi'] = ta.MFI(dataframe) dataframe['cci'] = ta.CCI(dataframe) dataframe['rsi'] = ta.RSI(dataframe) dataframe['mom'] = ta.MOM(dataframe) dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) dataframe['ao'] = awesome_oscillator(dataframe) macd = ta.MACD(dataframe) dataframe['macd'] = macd['macd'] dataframe['macdsignal'] = macd['macdsignal'] dataframe['macdhist'] = macd['macdhist'] # add volatility indicators dataframe['natr'] = ta.NATR(dataframe) # add volume indicators dataframe['obv'] = ta.OBV(dataframe) # add more momentum indicators dataframe['rocp'] = ta.ROCP(dataframe) # add some pattern recognition dataframe['CDL2CROWS'] = ta.CDL2CROWS(dataframe) dataframe['CDL3BLACKCROWS'] = ta.CDL3BLACKCROWS(dataframe) dataframe['CDL3INSIDE'] = ta.CDL3INSIDE(dataframe) dataframe['CDL3LINESTRIKE'] = ta.CDL3LINESTRIKE(dataframe) dataframe['CDL3OUTSIDE'] = ta.CDL3OUTSIDE(dataframe) dataframe['CDL3STARSINSOUTH'] = ta.CDL3STARSINSOUTH(dataframe) dataframe['CDL3WHITESOLDIERS'] = ta.CDL3WHITESOLDIERS(dataframe) dataframe['CDLADVANCEBLOCK'] = ta.CDLADVANCEBLOCK(dataframe) dataframe['CDLBELTHOLD'] = ta.CDLBELTHOLD(dataframe) dataframe['CDLBREAKAWAY'] = ta.CDLBREAKAWAY(dataframe) dataframe['CDLDOJI'] = ta.CDLDOJI(dataframe) dataframe['CDLDOJISTAR'] = ta.CDLDOJISTAR(dataframe) dataframe['CDLDRAGONFLYDOJI'] = ta.CDLDRAGONFLYDOJI(dataframe) dataframe['CDLENGULFING'] = ta.CDLENGULFING(dataframe) dataframe['CDLHAMMER'] = ta.CDLHAMMER(dataframe) dataframe['CDLBREAKAWAY'] = ta.CDLBREAKAWAY(dataframe) dataframe['CDLBREAKAWAY'] = ta.CDLBREAKAWAY(dataframe) # enter categorical time hour = datetime.strptime(str(dataframe['date'][len(dataframe) - 1]), "%Y-%m-%d %H:%M:%S").hour for h in range(24): dataframe['hour_{0:02}'.format(h)] = int(h == hour) return dataframe
def _build_indicators(self, df): if not self.realtime: inputs = df.to_dict(orient="list") for col in inputs: inputs[col] = np.array(inputs[col]) c = df["close"] for n in range(2, 40): inputs["bband_u_" + str(n)], inputs["bband_m_" + str(n)], inputs["bband_l_" + str(n)] = ta.BBANDS( inputs, n) inputs["sma_" + str(n)] = ta.SMA(inputs, timeperiod=n) inputs["adx_" + str(n)] = ta.ADX(inputs, timeperiod=n) # fast_ema = c.ewm(span = n, adjust = False).mean() # slow_ema = c.ewm(span = n*2, adjust = False).mean() # macd1 = fast_ema - slow_ema # macd2 = macd1.ewm(span = int(n*2/3), adjust = False).mean() # macd3 = macd1 - macd2 # inputs["macd_"+str(n)] = macd1.values # inputs["macdsignal_"+str(n)] = macd2.values # inputs["macdhist_"+str(n)] = macd3.values if n != 2: inputs["macd_" + str(n)], inputs["macdsignal_" + str(n)], inputs["macdhist_" + str(n)] = ta.MACD( inputs, n, n * 2, int(n * 2 / 3)) else: inputs["macd_" + str(n)], inputs["macdsignal_" + str(n)], inputs["macdhist_" + str(n)] = ta.MACD( inputs, n, n * 2, 1) # macd = [macd1.values, macd2.values, macd3.values] # for idx, i in enumerate(["macd_"+str(n), "macdsignal_"+str(n), "macdhist_"+str(n)]): # for day in zip(inputs[i], macd[idx]): # print("Type: %s N: %d PD: %.3f TA: %.3f, " % (i, n, day[1], day[0])) inputs["mfi_" + str(n)] = ta.MFI(inputs, n) inputs["ult_" + str(n)] = ta.ULTOSC(inputs, n, n * 2, n * 4) inputs["willr_" + str(n)] = ta.WILLR(inputs, n) inputs["slowk"], inputs["slowd"] = ta.STOCH(inputs) inputs["mom_" + str(n)] = ta.MOM(inputs, n) inputs["volume"] = list(map(lambda x: x / 10000, inputs["volume"])) df = pd.DataFrame().from_dict(inputs) # df = df.ix[100:] # print(df.tail(5)["macd_3"], df.tail(5)["macdsignal_3"], df.tail(5)["macdhist_3"]) return df else: # Build data one-by-one, as if it's coming in one at a time output = pd.DataFrame() sliding_window = pd.DataFrame() for idx, day in df.iterrows(): print("\rNow building day", str(idx), end="", flush=True) day = copy.deepcopy(day) # Avoid reference vs copy bullshit sliding_window = sliding_window.append(day, ignore_index=True) # print(day, type(day)) day_out = {} # print(sliding_window) o = sliding_window["open"].values h = sliding_window["high"].values l = sliding_window["low"].values c_series = sliding_window["close"] c = sliding_window["close"].values # print("----") # print(c) v = sliding_window["volume"].values for t in ["open", "high", "low", "close"]: day_out[t] = sliding_window[t].values[-1] for n in range(2, 40): # time.sleep(0.1) day_out["bband_u_" + str(n)], day_out["bband_m_" + str(n)], day_out[ "bband_l_" + str(n)] = stream.BBANDS(c, n) day_out["sma_" + str(n)] = stream.SMA(c, timeperiod=n) day_out["adx_" + str(n)] = stream.ADX(h, l, c, timeperiod=n) fast_ema = c_series.ewm(span=n, adjust=False).mean() slow_ema = c_series.ewm(span=n * 2, adjust=False).mean() macd1 = fast_ema - slow_ema macd2 = macd1.ewm(span=int(n * 2 / 3), adjust=False).mean() macd3 = macd1 - macd2 day_out["macd_" + str(n)] = macd1.values[-1] day_out["macdsignal_" + str(n)] = macd2.values[-1] day_out["macdhist_" + str(n)] = macd3.values[-1] # if n != 2: # day_out["macd_"+str(n)], day_out["macdsignal_"+str(n)], day_out["macdhist_"+str(n)] = stream.MACD(c, n, n*2, int(n*2/3)) # elif idx > 100: # macd = ta.MACD({"close":c}, n, n*2, 1) # day_out["macd_2"], day_out["macdsignal_2"], day_out["macdhist_2"] = (x[-1] for x in macd) # else: # day_out["macd_2"], day_out["macdsignal_2"], day_out["macdhist_2"] = None, None, None # macd = [macd1.values, macd2.values, macd3.values] # for idx, i in enumerate(["macd_"+str(n), "macdsignal_"+str(n), "macdhist_"+str(n)]): # for day in zip(inputs[i], macd[idx]): # print("Type: %s N: %d PD: %.3f TA: %.3f, " % (i, n, day[1], day[0])) day_out["mfi_" + str(n)] = stream.MFI(h, l, c, v, n) day_out["ult_" + str(n)] = stream.ULTOSC( h, l, c, n, n * 2, n * 4) day_out["willr_" + str(n)] = stream.WILLR(h, l, c, n) day_out["slowk"], day_out["slowd"] = stream.STOCH(h, l, c) day_out["mom_" + str(n)] = stream.MOM(c, n) day_out["volume"] = v[-1] / 10000 # print(day_out["macd_2"], day_out["macdsignal_2"], day_out["macdhist_2"]) output = output.append(day_out, ignore_index=True) # print(output.tail(5)["macd_3"], output.tail(5)["macdsignal_3"], output.tail(5)["macdhist_3"]) return output
end_date_time = datetime.datetime.now() # mysqldb.get_maxdate()[0] start_date_time = end_date_time - relativedelta( days=constants.DAYS_FOR_TECHNICALS) df_mkt = mongodao.getSymbolDataWithSymbol("SPY", start_date_time, end_date_time) symbol = "MSFT" df_symbol = mongodao.getSymbolDataWithSymbol(symbol, start_date_time, end_date_time) df_symbol_close = df_symbol[['close']] df_mkt_close = df_mkt[['close']] mom = abstract.MOM(df_symbol_close, timeperiod=5) df_merged = abstract.MACD(df_symbol_close, fastperiod=12, slowperiod=26, signalperiod=9) #df_merged=macd.apply(np.round) df_std = abstract.STDDEV(df_symbol_close.pct_change(), timeperiod=100) df_merged['stddev'] = df_std df_merged['volatility'] = df_std * 100 * math.sqrt(252) rsi = abstract.RSI(df_symbol_close).round(2) rsi = abstract.RSI(df_symbol_close).round(2) sma20 = abstract.SMA(df_symbol_close, timeperiod=20).round(2)
def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Adds several different TA indicators to the given DataFrame Performance Note: For the best performance be frugal on the number of indicators you are using. Let uncomment only the indicator you are using in your strategies or your hyperopt configuration, otherwise you will waste your memory and CPU usage. :param dataframe: Dataframe with data from the exchange :param metadata: Additional information, like the currently traded pair :return: a Dataframe with all mandatory indicators for the strategies """ # Momentum Indicators # ------------------------------------ # momentum dataframe['mom'] = ta.MOM(dataframe['close'].values, 10) dataframe['signal'] = 0 dataframe.loc[((dataframe['mom'] > 0) & (dataframe['volume'] > 0) # Make sure Volume is not 0 ), 'signal'] = 1 dataframe['signal'] = dataframe['signal'].diff() # ADX dataframe['adx'] = ta.ADX(dataframe) # # Plus Directional Indicator / Movement # dataframe['plus_dm'] = ta.PLUS_DM(dataframe) # dataframe['plus_di'] = ta.PLUS_DI(dataframe) # # Minus Directional Indicator / Movement # dataframe['minus_dm'] = ta.MINUS_DM(dataframe) # dataframe['minus_di'] = ta.MINUS_DI(dataframe) # # Aroon, Aroon Oscillator # aroon = ta.AROON(dataframe) # dataframe['aroonup'] = aroon['aroonup'] # dataframe['aroondown'] = aroon['aroondown'] # dataframe['aroonosc'] = ta.AROONOSC(dataframe) # # Awesome Oscillator # dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) # # Keltner Channel # keltner = qtpylib.keltner_channel(dataframe) # dataframe["kc_upperband"] = keltner["upper"] # dataframe["kc_lowerband"] = keltner["lower"] # dataframe["kc_middleband"] = keltner["mid"] # dataframe["kc_percent"] = ( # (dataframe["close"] - dataframe["kc_lowerband"]) / # (dataframe["kc_upperband"] - dataframe["kc_lowerband"]) # ) # dataframe["kc_width"] = ( # (dataframe["kc_upperband"] - dataframe["kc_lowerband"]) / dataframe["kc_middleband"] # ) # # Ultimate Oscillator # dataframe['uo'] = ta.ULTOSC(dataframe) # # Commodity Channel Index: values [Oversold:-100, Overbought:100] # dataframe['cci'] = ta.CCI(dataframe) # RSI #dataframe['rsi'] = ta.RSI(dataframe) # # Inverse Fisher transform on RSI: values [-1.0, 1.0] (https://goo.gl/2JGGoy) # rsi = 0.1 * (dataframe['rsi'] - 50) # dataframe['fisher_rsi'] = (np.exp(2 * rsi) - 1) / (np.exp(2 * rsi) + 1) # # Inverse Fisher transform on RSI normalized: values [0.0, 100.0] (https://goo.gl/2JGGoy) # dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) # # Stochastic Slow # stoch = ta.STOCH(dataframe) # dataframe['slowd'] = stoch['slowd'] # dataframe['slowk'] = stoch['slowk'] # Stochastic Fast #stoch_fast = ta.STOCHF(dataframe) #dataframe['fastd'] = stoch_fast['fastd'] #dataframe['fastk'] = stoch_fast['fastk'] # # Stochastic RSI # stoch_rsi = ta.STOCHRSI(dataframe) # dataframe['fastd_rsi'] = stoch_rsi['fastd'] # dataframe['fastk_rsi'] = stoch_rsi['fastk'] # MACD #macd = ta.MACD(dataframe) #dataframe['macd'] = macd['macd'] #dataframe['macdsignal'] = macd['macdsignal'] #dataframe['macdhist'] = macd['macdhist'] # MFI #dataframe['mfi'] = ta.MFI(dataframe) # # ROC # dataframe['roc'] = ta.ROC(dataframe) # Overlap Studies # ------------------------------------ # Bollinger Bands #bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) #dataframe['bb_lowerband'] = bollinger['lower'] #dataframe['bb_middleband'] = bollinger['mid'] #dataframe['bb_upperband'] = bollinger['upper'] #dataframe["bb_percent"] = ( # (dataframe["close"] - dataframe["bb_lowerband"]) / # (dataframe["bb_upperband"] - dataframe["bb_lowerband"]) #) #dataframe["bb_width"] = ( # (dataframe["bb_upperband"] - dataframe["bb_lowerband"]) / dataframe["bb_middleband"] #) # Bollinger Bands - Weighted (EMA based instead of SMA) # weighted_bollinger = qtpylib.weighted_bollinger_bands( # qtpylib.typical_price(dataframe), window=20, stds=2 # ) # dataframe["wbb_upperband"] = weighted_bollinger["upper"] # dataframe["wbb_lowerband"] = weighted_bollinger["lower"] # dataframe["wbb_middleband"] = weighted_bollinger["mid"] # dataframe["wbb_percent"] = ( # (dataframe["close"] - dataframe["wbb_lowerband"]) / # (dataframe["wbb_upperband"] - dataframe["wbb_lowerband"]) # ) # dataframe["wbb_width"] = ( # (dataframe["wbb_upperband"] - dataframe["wbb_lowerband"]) / dataframe["wbb_middleband"] # ) # # EMA - Exponential Moving Average # dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) # dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) # dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) # dataframe['ema21'] = ta.EMA(dataframe, timeperiod=21) # dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) # dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) # # SMA - Simple Moving Average # dataframe['sma3'] = ta.SMA(dataframe, timeperiod=3) # dataframe['sma5'] = ta.SMA(dataframe, timeperiod=5) # dataframe['sma10'] = ta.SMA(dataframe, timeperiod=10) # dataframe['sma21'] = ta.SMA(dataframe, timeperiod=21) # dataframe['sma50'] = ta.SMA(dataframe, timeperiod=50) # dataframe['sma100'] = ta.SMA(dataframe, timeperiod=100) # Parabolic SAR #dataframe['sar'] = ta.SAR(dataframe) # TEMA - Triple Exponential Moving Average #dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) # Cycle Indicator # ------------------------------------ # Hilbert Transform Indicator - SineWave #hilbert = ta.HT_SINE(dataframe) #dataframe['htsine'] = hilbert['sine'] #dataframe['htleadsine'] = hilbert['leadsine'] # Pattern Recognition - Bullish candlestick patterns # ------------------------------------ # # Hammer: values [0, 100] # dataframe['CDLHAMMER'] = ta.CDLHAMMER(dataframe) # # Inverted Hammer: values [0, 100] # dataframe['CDLINVERTEDHAMMER'] = ta.CDLINVERTEDHAMMER(dataframe) # # Dragonfly Doji: values [0, 100] # dataframe['CDLDRAGONFLYDOJI'] = ta.CDLDRAGONFLYDOJI(dataframe) # # Piercing Line: values [0, 100] # dataframe['CDLPIERCING'] = ta.CDLPIERCING(dataframe) # values [0, 100] # # Morningstar: values [0, 100] # dataframe['CDLMORNINGSTAR'] = ta.CDLMORNINGSTAR(dataframe) # values [0, 100] # # Three White Soldiers: values [0, 100] # dataframe['CDL3WHITESOLDIERS'] = ta.CDL3WHITESOLDIERS(dataframe) # values [0, 100] # Pattern Recognition - Bearish candlestick patterns # ------------------------------------ # # Hanging Man: values [0, 100] # dataframe['CDLHANGINGMAN'] = ta.CDLHANGINGMAN(dataframe) # # Shooting Star: values [0, 100] # dataframe['CDLSHOOTINGSTAR'] = ta.CDLSHOOTINGSTAR(dataframe) # # Gravestone Doji: values [0, 100] # dataframe['CDLGRAVESTONEDOJI'] = ta.CDLGRAVESTONEDOJI(dataframe) # # Dark Cloud Cover: values [0, 100] # dataframe['CDLDARKCLOUDCOVER'] = ta.CDLDARKCLOUDCOVER(dataframe) # # Evening Doji Star: values [0, 100] # dataframe['CDLEVENINGDOJISTAR'] = ta.CDLEVENINGDOJISTAR(dataframe) # # Evening Star: values [0, 100] # dataframe['CDLEVENINGSTAR'] = ta.CDLEVENINGSTAR(dataframe) # Pattern Recognition - Bullish/Bearish candlestick patterns # ------------------------------------ # # Three Line Strike: values [0, -100, 100] # dataframe['CDL3LINESTRIKE'] = ta.CDL3LINESTRIKE(dataframe) # # Spinning Top: values [0, -100, 100] # dataframe['CDLSPINNINGTOP'] = ta.CDLSPINNINGTOP(dataframe) # values [0, -100, 100] # # Engulfing: values [0, -100, 100] # dataframe['CDLENGULFING'] = ta.CDLENGULFING(dataframe) # values [0, -100, 100] # # Harami: values [0, -100, 100] # dataframe['CDLHARAMI'] = ta.CDLHARAMI(dataframe) # values [0, -100, 100] # # Three Outside Up/Down: values [0, -100, 100] # dataframe['CDL3OUTSIDE'] = ta.CDL3OUTSIDE(dataframe) # values [0, -100, 100] # # Three Inside Up/Down: values [0, -100, 100] # dataframe['CDL3INSIDE'] = ta.CDL3INSIDE(dataframe) # values [0, -100, 100] # # Chart type # # ------------------------------------ # # Heikin Ashi Strategy # heikinashi = qtpylib.heikinashi(dataframe) # dataframe['ha_open'] = heikinashi['open'] # dataframe['ha_close'] = heikinashi['close'] # dataframe['ha_high'] = heikinashi['high'] # dataframe['ha_low'] = heikinashi['low'] # Retrieve best bid and best ask from the orderbook # ------------------------------------ """ # first check if dataprovider is available if self.dp: if self.dp.runmode in ('live', 'dry_run'): ob = self.dp.orderbook(metadata['pair'], 1) dataframe['best_bid'] = ob['bids'][0][0] dataframe['best_ask'] = ob['asks'][0][0] """ return dataframe
def build_data(raw = False, random_split = True, start_date = None, end_date = None, test_proportion = 0.1): # if len(sec) == 1 and os.path.isfile(secs[0]): #it's a file # with open(secs[0]) as f: # secs = ["WIKI/" + line.strip() for line in f] # print("SECURITIES: ", s[5:] for s in secs) with open("stock_data/invalid_stocks.txt", "r+") as f: invalid_stock_codes = [line.strip() for line in f] f = open("stock_data/invalid_stocks.txt", "a") stock_code = yield while True and stock_code is not None: valid_stock = False while not valid_stock: if "." in stock_code: stock_code = yield None continue if stock_code in invalid_stock_codes: # print("Skipping security", sec) stock_code = yield None continue valid_stock = True sec = stock_code.split("/")[1] # Just the ticker, not the database code pickle_name = sec if raw: pickle_name += "_raw" if not random_split: pickle_name += "_notrand" if start_date and end_date: pickle_name += start_date + "to" + end_date elif start_date: pickle_name += start_date elif end_date: pickle_name += "to" + end_date if not os.path.isfile("./stock_data/" + pickle_name + "_data.pickle"): # print("No pickle found, getting data for", sec) try: # print("Getting data for", stock_code) df = quandl.get(stock_code, start_date = start_date, end_date = end_date) except quandl.errors.quandl_error.NotFoundError: invalid_stock_codes += [stock_code] f.write(stock_code + "\n") stock_code = yield None continue if "Adj. Close" in df.columns: df = df[["Adj. Open", "Adj. High", "Adj. Low", "Adj. Close", "Adj. Volume"]] df.rename(columns=lambda x: x[5:].lower(), inplace=True) # Remove the "Adj. " and make lowercase elif "Close" in df.columns: df = df[["Open", "High", "Low", "Close", "Volume"]] df.rename(columns=lambda x: x.lower(), inplace=True) # make lowercase price = df['close'].values minIdxs = argrelextrema(price, np.less) maxIdxs = argrelextrema(price, np.greater) Y = pd.Series(name="signal", dtype=np.ndarray, index=range(0, len(price))) n=0 for _, idx in np.ndenumerate(minIdxs): # if idx < MIN_MAX_PERIOD: continue max_price = max(price[idx: idx + MIN_MAX_PERIOD]) if ((max_price - price[idx]) / price[idx]) > HI_LO_DIFF: #if the difference between max and min is > X% Y.set_value(idx, np.array([1., 0.], np.float32)) n+=1 # print("MINS:", n) n=0 for _, idx in np.ndenumerate(maxIdxs): # if idx < MIN_MAX_PERIOD: continue min_price = min(price[idx: idx + MIN_MAX_PERIOD]) if ((price[idx] - min_price)/ min_price) > HI_LO_DIFF: #if the difference between max and min is > X% Y.set_value(idx, np.array([0., 1.], np.float32)) n+=1 # print("MAXS:", n) _min_idx, _max_idx = 0, 0 for i, y in np.ndenumerate(Y.values): if np.array_equal(y, [1., 0.]): _min_idx = i[0] elif np.array_equal(y, [0., 1.]): _max_idx = i[0] else: if _min_idx > _max_idx: s = np.array([1., 0.]) elif _max_idx > _min_idx: s = np.array([0., 1.]) else: s = np.array([0., 0.]) # no action taken, only occurs at the beginnings of datasets, afaik Y.set_value(i, s, np.float32) # x = list(zip(price[0:50], Y.values[0:50])) # for i in x: # print("{0:.2f} -- {1}".format(i[0], "sell" if np.array_equal(i[1], [0, 1]) else "buy" if np.array_equal(i[1], [1, 0]) else "nothing")) df.reset_index(drop=True, inplace = True) if isinstance(price, np.ndarray): price = price.tolist() ''' INDICATORS ''' # print(len(df), len(Y)) # print("Building indicators...") inputs = df.to_dict(orient="list") for col in inputs: inputs[col] = np.array(inputs[col]) for n in range(2, 40): inputs["bband_u_"+str(n)], inputs["bband_m_"+str(n)], inputs["bband_l_"+str(n)] = ta.BBANDS(inputs, n) inputs["sma_"+str(n)] = ta.SMA(inputs, timeperiod = n) inputs["adx_"+str(n)] = ta.ADX(inputs, timeperiod = n) inputs["macd_"+str(n)], inputs["macdsignal_"+str(n)], inputs["macdhist_"+str(n)] = ta.MACD(inputs, n, n*2, n*2/3) inputs["mfi_"+str(n)] = ta.MFI(inputs, n) inputs["ult_"+str(n)] = ta.ULTOSC(inputs, n, n*2, n*4) inputs["willr_"+str(n)] = ta.WILLR(inputs, n) inputs["slowk"], inputs["slowd"] = ta.STOCH(inputs) inputs["mom_"+str(n)] = ta.MOM(inputs, n) inputs["mom_"+str(n)] = ta.MOM(inputs, n) inputs["volume"] = list(map(lambda x: x/10000, inputs["volume"])) df = pd.DataFrame().from_dict(inputs) broken = False for idx, val in reversed(list(df.isnull().any(axis=1).iteritems())): if val == True: # print(actual_idx, val) df.drop(idx, inplace = True) Y.drop(idx, inplace = True) try: # price[actual_idx] = None price.pop(idx) except IndexError: #drop the security # print("Error, dropping security", sec) broken = True break # print(list(df.isnull().any(axis=1).iteritems())) # print("PRICES", price) # print(len(price), len(df.values)) # for i, p in reversed(list(enumerate(price))): # actual_idx = len(price) - 1 - i # if p is None: # print(actual_idx) # price.pop(actual_idx) ''' BUILD NEURAL NET INPUTS ''' if not broken: Y = np.vstack(Y.values) print(df["adx_10"]) X = df.values # print(X[0:2]) if not raw: rand = "_notrand" if not random_split else "" if not os.path.isfile("./stock_data/" + sec + rand + ".scaler"): scaler = prep.StandardScaler().fit(X) X_norm = scaler.transform(X) joblib.dump(scaler, "./stock_data/" + sec + rand + ".scaler") else: scaler = joblib.load("./stock_data/" + sec + rand + ".scaler") X_norm = scaler.transform(X) else: X_norm = X if random_split: trX, testX, trY, testY = train_test_split(X_norm, Y, test_size = test_proportion, random_state=0) else: # just clips the test data off the end l = len(X_norm) trX, testX = X_norm[:int(-test_proportion*l)], X_norm[int(-test_proportion*l):] trY, testY = Y[:int(-test_proportion*l)], Y[int(-test_proportion*l):] # print("Pickling...") output = {"X_norm": X_norm, "Y": Y, "trX": trX, "trY": trY, "testX": testX, "testY": testY, "price": price} pickle.dump(output, open("./stock_data/" + pickle_name + "_data.pickle", "wb")) stock_code = yield output else: invalid_stock_codes += [stock_code] f.write(stock_code + "\n") stock_code = yield None else: # print("Pickle found, loading...") _data = pickle.load(open("./stock_data/" + pickle_name + "_data.pickle", "rb")) trX, trY, testX, testY, price, X_norm, Y = _data["trX"], _data["trY"], _data["testX"], _data["testY"], _data["price"], _data["X_norm"], _data["Y"] stock_code = yield {"X_norm": X_norm, "Y": Y, "trX": trX, "trY": trY, "testX": testX, "testY": testY, "price": price}
def calculate_technical(df_symbol, symbol, df_mkt, start_date_time, end_date_time, hist_dates, days_back): #list_drop_cloumns = [ 'open', 'high','low','volume'] df_symbol_close = df_symbol[['close']] df_mkt_close = df_mkt[['close']] mom = abstract.MOM(df_symbol_close, timeperiod=5) df_merged = abstract.MACD(df_symbol_close, fastperiod=12, slowperiod=26, signalperiod=9) #df_merged=macd.apply(np.round) df_std = abstract.STDDEV(df_symbol_close.pct_change(), timeperiod=100) df_merged['stddev'] = df_std df_merged['volatility'] = df_std * 100 * math.sqrt(252) rsi = abstract.RSI(df_symbol_close).round(2) sma20 = abstract.SMA(df_symbol_close, timeperiod=20).round(2) sma100 = abstract.SMA(df_symbol_close, timeperiod=100).round(2) sma200 = abstract.SMA(df_symbol_close, timeperiod=200).round(2) sma3 = abstract.SMA(df_symbol_close, timeperiod=3).round(2) sma5 = abstract.SMA(df_symbol_close, timeperiod=5).round(2) sma9 = abstract.SMA(df_symbol_close, timeperiod=9).round(2) sma13 = abstract.SMA(df_symbol_close, timeperiod=13).round(2) sma25 = abstract.SMA(df_symbol_close, timeperiod=25).round(2) sma50 = abstract.SMA(df_symbol_close, timeperiod=50).round(2) sma90 = abstract.SMA(df_symbol_close, timeperiod=90).round(2) sma36 = abstract.SMA(df_symbol_close, timeperiod=36).round(2) sma150 = abstract.SMA(df_symbol_close, timeperiod=150).round(2) df_merged['mom'] = mom df_merged['sma20'] = sma20 df_merged['sma50'] = sma50 df_merged['sma100'] = sma100 df_merged['sma200'] = sma200 df_merged['rsi'] = rsi df_merged['close'] = df_symbol['close'] df_merged['open'] = df_symbol['open'] df_merged['low'] = df_symbol['low'] df_merged['high'] = df_symbol['high'] df_merged['volume'] = df_symbol['volume'] df_merged['sma_volume_6month'] = pd.rolling_mean(df_merged['volume'], window=120).round(2) df_merged = df_merged.dropna() df_merged['symbol'] = symbol df_merged['rsi_value'] = df_merged['rsi'].apply( rsi_manager.calculate_rsi_values) df_merged['sma3'] = sma3 df_merged['sma5'] = sma5 df_merged['sma9'] = sma9 df_merged['sma13'] = sma13 df_merged['sma20'] = sma20 df_merged['sma25'] = sma25 df_merged['sma50'] = sma50 df_merged['sma90'] = sma90 df_merged['sma36'] = sma36 df_merged['sma150'] = sma150 df_merged['Relative_strength'] = relative_strength( df_symbol_close, df_mkt_close, symbol)['Relative_strength'] df_merged['stdabove'] = df_merged.apply(calculate_stdabove, axis=1) df_merged['date'] = df_merged.index df_res = df_symbol.apply(calc_res, axis=1) df_merged = pd.concat([df_merged, df_res], axis=1) df_rating = rating_manager.calc_rating_history(df_merged, days_back, symbol) df_trends = df_merged.apply( lambda row: trend_manager.trend_calculation(row), axis=1) df_merged = pd.concat([df_merged, df_trends], axis=1) crossover_manager.TrendChangePositive(df_merged, "short_trend", alert_constants.TREND_SHORT) crossover_manager.TrendChangePositive(df_merged, "inter_trend", alert_constants.TREND_INTERMEDIATE) crossover_manager.TrendChangePositive(df_merged, "long_trend", alert_constants.TREND_LONG) crossover_manager.TrendChangeNegative(df_merged, "short_trend", alert_constants.TREND_SHORT) crossover_manager.TrendChangeNegative(df_merged, "inter_trend", alert_constants.TREND_INTERMEDIATE) crossover_manager.TrendChangeNegative(df_merged, "long_trend", alert_constants.TREND_LONG) df_merged['rating'] = df_rating['rating'] df_merged = df_merged.replace([np.inf, -np.inf], np.nan) df_merged = df_merged.dropna() print "********************************************************************************" print "********************************************************************************" print "********************************************************************************" #print df_merged if (df_merged is None or df_merged.symbol.count() == 0): return logger.info("Saving history for Symbol " + symbol + " length = " + str(len(df_merged))) df_merged.set_index('date', inplace=True) alert_manager.relative_strength(df_merged) alert_manager.fullGapPositive(df_merged) alert_manager.fullGapNegative(df_merged) alert_manager.partialGapPositive(df_merged) alert_manager.partialGapNegative(df_merged) alert_manager.keyReversalPositive(df_merged) alert_manager.keyReversalNegative(df_merged) alert_manager.volumePositive(df_merged) alert_manager.volumeNegative(df_merged) crossover_manager.smacrossovers(df_merged) crossover_manager.macd_crossovers(df_merged) crossover_manager.obos_alerts(df_merged) dbdao.save_dataframe(df_merged, "df_history") #alert_manager.relative_strength(df_merged, df_spy, symbol) df_merged['stdabove_prev'] = df_merged['stdabove'].shift(1) #latest data calculations return_data = {} return_data.update(calculate_beta(df_symbol_close, df_mkt_close, symbol)) return_data.update(calculate_prices_at_dates(df_symbol_close, hist_dates)) monthly_date = hist_dates['Monthly'] weekly_date = hist_dates['Weekly'] df_latest = df_merged.tail(1) df_latest_sign = df_latest.apply(calc_signs, axis=1) df_latest = pd.concat([df_latest, df_latest_sign], axis=1) #df_latest[['sma_20day_sign']]=df_latest.apply(calc_signs,axis=1) df_latest['volatility_monthly'] = price_manager.get_specific_date_value( df_merged, monthly_date, 'volatility') df_latest['volatility_weekly'] = price_manager.get_specific_date_value( df_merged, weekly_date, 'volatility') df_latest['std50days'] = df_latest['stdabove'] df_latest['date'] = df_latest.index df_latest.set_index('date', inplace=True) for key, value in return_data.iteritems(): df_latest[key] = value return df_latest
def MOM(self): #10 mom = abstract.MOM(self.company_stock, timeperiod=10) self.company_stock['MOM'] = mom
def _build_indicators(data): # sliding_window = [] while True: data_with_ind = [] for df in data: df = copy.deepcopy(df) if "Adj. Close" in df.columns: df = df[["Adj. Open", "Adj. High", "Adj. Low", "Adj. Close", "Adj. Volume"]] df.rename(columns=lambda x: x[5:].lower(), inplace=True) # Remove the "Adj. " and make lowercase elif "Close" in df.columns: df = df[["Open", "High", "Low", "Close", "Volume"]] df.rename(columns=lambda x: x.lower(), inplace=True) # make lowercase df.reset_index(drop=True, inplace = True) inputs = df.to_dict(orient="list") for col in inputs: inputs[col] = np.array(inputs[col]) for n in range(2, 40): inputs["bband_u_"+str(n)], inputs["bband_m_"+str(n)], inputs["bband_l_"+str(n)] = ta.BBANDS(inputs, n) inputs["sma_"+str(n)] = ta.SMA(inputs, timeperiod = n) inputs["adx_"+str(n)] = ta.ADX(inputs, timeperiod = n) inputs["macd_"+str(n)], inputs["macdsignal_"+str(n)], inputs["macdhist_"+str(n)] = ta.MACD(inputs, n, n*2, n*2/3) inputs["mfi_"+str(n)] = ta.MFI(inputs, n) inputs["ult_"+str(n)] = ta.ULTOSC(inputs, n, n*2, n*4) inputs["willr_"+str(n)] = ta.WILLR(inputs, n) inputs["slowk"], inputs["slowd"] = ta.STOCH(inputs) inputs["mom_"+str(n)] = ta.MOM(inputs, n) inputs["mom_"+str(n)] = ta.MOM(inputs, n) inputs["volume"] = list(map(lambda x: x/10000, inputs["volume"])) df = pd.DataFrame().from_dict(inputs) price = df["close"].values if isinstance(price, np.ndarray): price = price.tolist() for idx, val in df.isnull().any(axis=1).iteritems(): if val == True: df.drop(idx, inplace = True) try: price[idx] = None except IndexError: #drop the security print("Error, failed to drop price on index", idx) sys.exit(1) # print("Dropped index:", idx) for i, p in reversed(list(enumerate(price))): actual_idx = len(price) - 1 - i if p == None: price.pop(actual_idx) print(df["adx_10"]) X = df.values data_with_ind += [{"data": X, "price": price}] return data_with_ind
def _build_indicators(num_secs): # accepts a list of one-day Series sec_idx_range = range(num_secs) sliding_window = [] # list of pd.DataFrames data = yield for datum in data: sliding_window += [_rename_columns(datum)] current_day = 0 while True: passes_validity_check, num_validation_iterations = False, 0 # time.sleep(1) while not passes_validity_check: for i in sec_idx_range: # for each security # print("Current day:", current_day) if current_day != 0: if current_day > 170 and num_validation_iterations == 0: sliding_window[i] = sliding_window[i].iloc[ 1:] # pop the first for datum in data: if num_validation_iterations == 0: sliding_window[i] = sliding_window[i].append( _rename_columns(datum)) data_with_ind = [] series = sliding_window[i] series = series.reset_index(drop=True) inputs = series.to_dict(orient="list") for col in inputs: inputs[col] = np.array(inputs[col]) c = series.close for n in range(2, 40): inputs["bband_u_" + str(n)], inputs["bband_m_" + str(n)], inputs["bband_l_" + str(n)] = ta.BBANDS( inputs, n) inputs["sma_" + str(n)] = ta.SMA(inputs, timeperiod=n) inputs["adx_" + str(n)] = ta.ADX(inputs, timeperiod=n) # print("\nINPUTS:", inputs) # if current_day > n*2: fast_ema = c.ewm(span=n).mean() slow_ema = c.ewm(span=n * 2).mean() # print(fast_ema, slow_ema) macd1 = fast_ema - slow_ema macd2 = macd1.ewm(span=n * 2 / 3).mean() macd3 = macd1 - macd2 inputs["macd_" + str(n)], inputs["macdsignal_" + str(n)], inputs[ "macdhist_" + str(n)] = macd1.iloc[ -1], macd2.iloc[-1], macd3.iloc[-1] if current_day == 160: print(n) print(macd1, macd2, macd3) sys.exit(69) # else: # inputs["macd_"+str(n)], inputs["macdsignal_"+str(n)], inputs["macdhist_"+str(n)] = [np.NaN]*3 inputs["mfi_" + str(n)] = ta.MFI(inputs, n) inputs["ult_" + str(n)] = ta.ULTOSC( inputs, n, n * 2, n * 4) inputs["willr_" + str(n)] = ta.WILLR(inputs, n) inputs["slowk"], inputs["slowd"] = ta.STOCH(inputs) inputs["mom_" + str(n)] = ta.MOM(inputs, n) inputs["mom_" + str(n)] = ta.MOM(inputs, n) inputs["volume"] = list( map(lambda x: x / 10000, inputs["volume"])) series = pd.DataFrame().from_dict(inputs) price = series["close"].iloc[-1] if isinstance(price, np.ndarray): price = price.tolist() # for idx, val in series.isnull().any(axis=1).iteritems(): # if val == True: # series.drop(idx, inplace = True) # try: # price[idx] = None # except IndexError: #drop the security # print("Error, failed to drop price on index", idx) # sys.exit(1) # # print("Dropped index:", idx) # for i, p in reversed(list(enumerate(price))): # actual_idx = len(price) - 1 - i # if p == None: # price.pop(actual_idx) # print(series["adx_10"]) X = series.iloc[-1].values if current_day < 170: passes_validity_check = True elif not np.isnan(X).any(): passes_validity_check = True # if num_validation_iterations != 0: # with pd.option_context('display.max_rows', None, 'display.max_columns', None): # print(series.iloc[-1]) # sys.exit(1) else: num_validation_iterations += 1 print("Reevaluating, iteration", num_validation_iterations, "day:", current_day) # with pd.option_context('display.max_rows', None, 'display.max_columns', None): # print(series.iloc[-1]) # sys.exit(1) # if current_day > 170: # print(series.iloc[-1].values) # if np.isnan(X).any() and current_day > 170: # # with pd.option_context('display.max_rows', None, 'display.max_columns', None): # # print(series) # print(sliding_window[0]) # break # print("ADX_10:\n", series["adx_10"].tail(3)) # if current_day == 900: # print(series) # print(X) data_with_ind += [{"data": X, "price": round(price, 2)}] data = yield data_with_ind current_day += 1
def build_data_to_dict(secs, raw = False): PICKLE_NAME = "_".join(s[5:] for s in secs) print("SECURITIES: ", PICKLE_NAME.split("_")) if not os.path.isfile("./stock_data/" + PICKLE_NAME + "_data.pickle"): print("No pickle found, getting data...") # df = pd.concat([quandl.get("WIKI/AAPL"), quandl.get("WIKI/F"), quandl.get("WIKI/XOM")]) df = pd.DataFrame() Y = pd.Series() prices = [] for sec in secs: sec_df = quandl.get(sec) if "Adj. Close" in sec_df.columns: sec_df = sec_df[["Adj. Open", "Adj. High", "Adj. Low", "Adj. Close", "Adj. Volume"]] sec_df.rename(columns=lambda x: x[5:].lower(), inplace=True) # Remove the "Adj. " and make lowercase elif "Close" in sec_df.columns: sec_df = sec_df[["Open", "High", "Low", "Close", "Volume"]] sec_df.rename(columns=lambda x: x.lower(), inplace=True) # make lowercase print("Calculating output for", sec) price = sec_df['close'].values minIdxs = argrelextrema(price, np.less) maxIdxs = argrelextrema(price, np.greater) sec_Y = pd.Series(name="signal", dtype=np.ndarray, index=range(0, len(price))) n=0 for _, idx in np.ndenumerate(minIdxs): if idx < MIN_MAX_PERIOD: continue max_price = max(price[idx: idx + MIN_MAX_PERIOD]) if ((max_price - price[idx]) / price[idx]) > HI_LO_DIFF: #if the difference between max and min is > 2% sec_Y.set_value(idx, np.array([1, 0, 0], np.int32)) n+=1 print("MINS:", n) n=0 for _, idx in np.ndenumerate(maxIdxs): if idx < MIN_MAX_PERIOD: continue min_price = min(price[idx: idx + MIN_MAX_PERIOD]) if ((price[idx] - min_price)/ min_price) > HI_LO_DIFF: #if the difference between max and min is > 2% sec_Y.set_value(idx, np.array([0, 0, 1], np.int32)) n+=1 print("MAXS:", n) for idx in pd.isnull(sec_Y).nonzero()[0]: sec_Y.set_value(idx, np.array([0, 1, 0], np.int32)) sec_df.reset_index(drop=True, inplace = True) if isinstance(price, np.ndarray): price = price.tolist() ''' INDICATORS ''' # print(len(sec_df), len(sec_Y)) print("Building indicators...") inputs = sec_df.to_dict(orient="list") for col in inputs: inputs[col] = np.array(inputs[col]) for n in range(2, 40): inputs["bband_u_"+str(n)], inputs["bband_m_"+str(n)], inputs["bband_l_"+str(n)] = ta.BBANDS(inputs, n) inputs["sma_"+str(n)] = ta.SMA(inputs, timeperiod = n) inputs["adx_"+str(n)] = ta.ADX(inputs, timeperiod = n) inputs["macd_"+str(n)], inputs["macdsignal_"+str(n)], inputs["macdhist_"+str(n)] = ta.MACD(inputs, n, n*2, n*2/3) inputs["mfi_"+str(n)] = ta.MFI(inputs, n) inputs["ult_"+str(n)] = ta.ULTOSC(inputs, n, n*2, n*4) inputs["willr_"+str(n)] = ta.WILLR(inputs, n) inputs["slowk"], inputs["slowd"] = ta.STOCH(inputs) inputs["mom_"+str(n)] = ta.MOM(inputs, n) inputs["mom_"+str(n)] = ta.MOM(inputs, n) inputs["volume"] = list(map(lambda x: x/10000, inputs["volume"])) sec_df = pd.DataFrame().from_dict(inputs) # print(sec_df.isnull().any(axis=1)) for idx, val in sec_df.isnull().any(axis=1).iteritems(): if val == True: # print(idx, val) sec_df.drop(idx, inplace = True) sec_Y.drop(idx, inplace = True) price.pop(idx) prices.append(price) df = pd.concat([df, sec_df]) Y = pd.concat([Y, sec_Y]) prices = [j for i in prices for j in i] # spooky magic ''' BUILD NEURAL NET INPUTS ''' Y = np.vstack(Y.values) X = df.values if not raw: scaler = prep.StandardScaler().fit(X) X_norm = scaler.transform(X) from sklearn.externals import joblib joblib.dump(scaler, "./stock_data/" + sec + ".scaler") else: X_norm = X trX, testX, trY, testY= train_test_split(X_norm, Y, test_size = 0.1, random_state=0) # print("Pickling...") output = {"X_norm": X_norm, "Y": Y, "trX": trX, "trY": trY, "testX": testX, "testY": testY, "price": price} pickle.dump(output, open("./stock_data/" + (PICKLE_NAME if not raw else PICKLE_NAME + "_raw") + "_data.pickle", "wb")) return output else: print("Pickle found, loading...") _data = pickle.load(open("./stock_data/" + PICKLE_NAME + "_data.pickle", "rb")) trX, trY, testX, testY, price, X_norm, Y = _data["trX"], _data["trY"], _data["testX"], _data["testY"], _data["price"], _data["X_norm"], _data["Y"] return {"X_norm": X_norm, "Y": Y, "trX": trX, "trY": trY, "testX": testX, "testY": testY, "price": price}