def doPredict(self, data: StockData) -> float: """ Use the loaded trained neural network to predict the next stock value. Args: data: The historical stock values of a company Returns: The predicted next stock value for that company """ # Assumptions about data: at least INPUT_SIZE pairs of type (_, float) assert data is not None and data.get_row_count() >= INPUT_SIZE # Extract last INPUT_SIZE floats (here: stock values) as input for neural network # (format: numpy array of arrays) input_values = np.array([[x[1] for x in data.get_from_offset(-INPUT_SIZE)]]) normalized_prices = [] vector_min = np.min(input_values) vector_max = np.max(input_values) for price in input_values: normalized_prices.append((price - vector_min) / (vector_max - vector_min)) input_values = np.asarray(normalized_prices) try: # Let network predict the next stock value based on last 100 stock values prediction = self.model.predict(input_values)[0][0] return data.get_last()[1] + calculate_delta(prediction) except: logger.error("Error in predicting next stock value.") assert False
def doPredict(self, data: StockData) -> float: """ Use the loaded trained neural network to predict the next stock value. Args: data: historical stock values of a company Returns: predicted next stock value for that company """ #self.model.compile(loss='mean_squared_error', optimizer='sgd') tmp_data = data.get_from_offset(data.get_row_count() - 5) tmp = np.array([[ tmp_data[0][1], tmp_data[1][1], tmp_data[2][1], tmp_data[3][1], tmp_data[4][1] ]]) pred = self.model.predict(tmp) res = data.get_last()[1] if (pred <= 0.5): res = res * 0.9 else: res = res * 1.1 # TODO: extract needed data for neural network and predict result return res
def learn_nn_and_save(data: StockData, filename_to_save: str): """ Starts the training of the neural network and saves it to the file system Args: data: The data to train on filename_to_save: The filename to save the trained NN to """ dates = data.get_dates() prices = data.get_values() # Generate training data # Build chunks of prices from 100 consecutive days (input_prices) and 101th day (current_prices_for_plot) current_prices_for_plot, input_prices, wanted_results = get_data(prices) # Shape and configuration of network is optimized for binary classification problems # see: https://keras.io/getting-started/sequential-model-guide/ network = create_model() network.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy']) # Train the neural network reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.9, patience=5, min_lr=0.000001, verbose=1) history = network.fit(input_prices, wanted_results, epochs=500, batch_size=128, verbose=1, validation_data=(input_prices, wanted_results), shuffle=True, callbacks=[reduce_lr]) # Evaluate the trained neural network and plot results score = network.evaluate(input_prices, wanted_results, batch_size=128, verbose=0) logger.debug(f"Test score: {score}") # Draw plt.figure() plt.plot(history.history['loss']) plt.plot(history.history['val_loss']) plt.plot(history.history['acc']) plt.title('training loss / testing loss by epoch') plt.ylabel('loss/acc') plt.xlabel('epoch') plt.legend(['loss', 'val_loss', 'acc'], loc='best') plt.figure() current_price_prediction = network.predict(input_prices, batch_size=128) logger.debug(f"current_price_prediction:") iteration = 0 for x in current_price_prediction: logger.debug(f"iteration {iteration} - output: {x}") iteration = iteration + 1 plt.plot(dates[INPUT_SIZE:], current_prices_for_plot, color="black") # current prices in reality plt.plot(dates[INPUT_SIZE:], [calculate_delta(x) for x in current_price_prediction], color="green") # predicted prices by neural network plt.title('current prices / predicted prices by date') plt.ylabel('price') plt.xlabel('date') plt.legend(['current', 'predicted'], loc='best') plt.show() # Save trained model: separate network structure (stored as JSON) and trained weights (stored as HDF5) save_keras_sequential(network, RELATIVE_PATH, filename_to_save)
def read_stock_market_data(stocks: StockList, periods: PeriodList) -> StockMarketData: """ Reads the "cross product" from `stocks` and `periods` from CSV files and creates a `StockMarketData` object from this. For each defined stock in `stocks` the corresponding value from `CompanyEnum` is used as logical name. If there are `periods` provided those are each read. Args: stocks: The company names for which to read the stock data. *Important:* These values need to be stated in `CompanyEnum` periods: The periods to read. If not empty each period is appended to the filename like this: `[stock_name]_[period].csv` Returns: The created `StockMarketData` object Examples: * Preface: Provided stock names are supposed to be part to `CompanyEnum`. They are stated plaintext-ish here to show the point: * `(['stock_a', 'stock_b'], ['1962-2011', '2012-2017'])` reads: * 'stock_a_1962-2011.csv' * 'stock_a_2012-2015.csv' * 'stock_b_1962-2011.csv' * 'stock_b_2012-2015.csv' into a dict with keys `CompanyEnum.COMPANY_A` and `CompanyEnum.COMPANY_B` respectively * `(['stock_a'], ['1962-2011', '2012-2017'])` reads: * 'stock_a_1962-2011.csv' * 'stock_a_2012-2015.csv' into a dict with a key `CompanyEnum.COMPANY_A` * `(['stock_a', 'stock_b'], ['1962-2011'])` reads: * 'stock_a_1962-2011.csv' * 'stock_b_1962-2011.csv' into a dict with keys `CompanyEnum.COMPANY_A` and `CompanyEnum.COMPANY_B` respectively * `(['stock_a', 'stock_b'], [])` reads: * 'stock_a.csv' * 'stock_b.csv' into a dict with keys `CompanyEnum.COMPANY_A` and `CompanyEnum.COMPANY_B` respectively """ data = dict() # Read *all* available data for stock in stocks: filename = stock.value if len(periods) is 0: data[stock] = StockData( __read_stock_market_data([[stock, filename]])[stock]) else: period_data = list() for period in periods: period_data.append( __read_stock_market_data( [[stock, ('%s_%s' % (filename, period))]])) data[stock] = StockData([ item for period_dict in period_data if period_dict is not None for item in period_dict[stock] ]) return StockMarketData(data)
def learn_nn_and_save(data: StockData, filename_to_save: str): """ Starts the training of the neural network and saves it to the file system Args: data: The data to train on filename_to_save: The filename to save the trained NN to """ dates = data.get_dates() prices = data.get_values() # Generate training data # Build chunks of prices from 100 consecutive days (last_prices) and 101th day (current_price) last_prices, current_price = [], [] for i in range(0, len(prices) - 100): last_prices.append(prices[i:100 + i]) current_price.append(float(prices[100 + i])) network = create_model() network.compile(loss='mean_squared_error', optimizer='adam') # Train the neural network history = network.fit(last_prices, current_price, epochs=10, batch_size=128, verbose=1) # Evaluate the trained neural network and plot results score = network.evaluate(np.array(last_prices), current_price, batch_size=128, verbose=0) logger.debug(f"Test score: {score}") plt.figure() plt.plot(history.history['loss']) plt.title('training loss / testing loss by epoch') plt.ylabel('loss') plt.xlabel('epoch') plt.legend(['training', 'testing'], loc='best') plt.figure() current_price_prediction = network.predict(last_prices, batch_size=128) plt.plot(dates[100:], current_price, color="black") # current prices in reality plt.plot(dates[100:], current_price_prediction, color="green") # predicted prices by neural network plt.title('current prices / predicted prices by date') plt.ylabel('price') plt.xlabel('date') plt.legend(['current', 'predicted'], loc='best') plt.show() # Save trained model: separate network structure (stored as JSON) and trained weights (stored as HDF5) save_keras_sequential(network, RELATIVE_PATH, filename_to_save)
def testDoPredictStockB(self): predictor = PerfectPredictor(CompanyEnum.COMPANY_B) # only one possible value: take 30.12.2011 and predict 03.01.2012 current_value = StockData([(dt.date(2011, 12, 30), 157.077850)]) future_value = 159.145142 self.assertEqual(predictor.doPredict(current_value), future_value) # only one possible value: take 03.01.2012 and predict 04.01.2012 current_value = StockData([(dt.date(2012, 1, 3), 159.145142)]) future_value = 158.495911 self.assertEqual(predictor.doPredict(current_value), future_value)
def doPredict(self, data: StockData) -> float: """ Use the loaded trained neural network to predict the next stock value. Args: data: historical stock values of a company Returns: predicted next stock value for that company """ input = numpy.array([data.get_values()[-WINDOW_SIZE:]]) output = self.model.predict(input) print("predicted %f for price %f" % (output[0], data.get_last()[1])) return output[0] + data.get_last()[1]
def __trade_for_company(self, company_enum: CompanyEnum, company_data: StockData, predictor: IPredictor, portfolio: Portfolio, result_order_list: OrderList): last_value = company_data.get_last()[-1] # This determines the trade action to apply order = self.__determine_action(company_data, predictor, last_value) if order == OrderType.BUY: if portfolio.cash > last_value: # We can buy something amount_of_units_to_buy = int(portfolio.cash // last_value) result_order_list.buy(company_enum, amount_of_units_to_buy) # Update Cash in portfolio portfolio.cash = portfolio.cash - (amount_of_units_to_buy * last_value) elif order == OrderType.SELL: # Check if something can be selled shares_in_portfolio = self.__find_shares_of_company( company_enum, portfolio.shares) if shares_in_portfolio is not None: # Sell everything result_order_list.sell(company_enum, shares_in_portfolio.amount)
def test_update__do_not_drop_below_cash_0(self): """ Tests: Portfolio#update Flavour: When receiving two BUY orders the `#update` method should regard the available cash and NEVER drop below 0 Creates a portfolio, a stock market data object and a arbitrary `OrderList` and executes these orders on the portfolio. Checks if those are applied correctly """ cash_reserve = 16000.0 data = StockData([(date(2017, 1, 1), 150.0)]) stock_market_data = StockMarketData({CompanyEnum.COMPANY_A: data}) portfolio = Portfolio(cash_reserve, []) # Create a order list whose individual actions are within the limit but in sum are over the limit # Stock price: 150.0, quantity: 100 -> trade volume: 15000.0; cash: 16000.0 order_list = OrderList() order_list.buy(CompanyEnum.COMPANY_A, 100) order_list.buy(CompanyEnum.COMPANY_A, 100) updated_portfolio = portfolio.update(stock_market_data, order_list) assert updated_portfolio.cash >= 0
def test_update__sufficient_cash_reserve(self): """ Tests: Portfolio#update Flavour: Enough cash in the portfolio, so the trades should be applied Creates a portfolio, a stock market data object and a arbitrary `OrderList` and executes these orders on the portfolio. Checks if those are applied correctly """ cash_reserve = 20000.0 data = StockData([(date(2017, 1, 1), 150.0)]) stock_market_data = StockMarketData({CompanyEnum.COMPANY_A: data}) portfolio = Portfolio(cash_reserve, [SharesOfCompany(CompanyEnum.COMPANY_A, 200)]) order_list = OrderList() order_list.buy(CompanyEnum.COMPANY_A, 100) updated_portfolio = portfolio.update(stock_market_data, order_list) # Current cash reserve is sufficient for trade volume. Trade should happen assert updated_portfolio.cash < cash_reserve assert updated_portfolio.cash < portfolio.cash assert updated_portfolio.shares[ 0].company_enum == CompanyEnum.COMPANY_A assert updated_portfolio.shares[0].amount == 300
def learn_nn_and_save(training_data: StockData, test_data: StockData, filename_to_save: str): network = create_model() network.compile(loss='mean_squared_error', optimizer='sgd') values = training_data.get_values() setCount = len(values) - (WINDOW_SIZE + 1) xtrain = [] for element in range(0, setCount): xtrain.append(numpy.array(values[element:WINDOW_SIZE + element])) X_TRAIN = numpy.array(xtrain) Y_TRAIN = numpy.empty(setCount, dtype=numpy.float) offset = WINDOW_SIZE - 1 for element in range(0, setCount): current = values[element + offset] next = values[element + offset + 1] Y_TRAIN[element] = 1.0 if next > current else -1.0 history = network.fit(X_TRAIN, Y_TRAIN, epochs=EPOCHS, batch_size=BATCH_SIZE) draw_history(history) # Save trained model: separate network structure (stored as JSON) and trained weights (stored as HDF5) save_keras_sequential(network, RELATIVE_PATH, filename_to_save)
def testRandomPredictor(self): sp = Predictors.RandomPredictor() input_data = StockData([]) today = date(2017, 11, 8) yesterday = date(2017, 11, 8) tuple1 = (yesterday, 2.0) tuple2 = (today, 3.0) input_data.append(tuple1) input_data.append(tuple2) result = sp.doPredict(input_data) self.assertNotEqual(result, 3.0)
def testDoPredictStockA(self): predictor = PerfectPredictor(CompanyEnum.COMPANY_A) # only one possible value: take 30.12.2011 and predict 03.01.2012 current_value = StockData([(dt.date(2011, 12, 30), 34.802376)]) future_value = 35.554108 self.assertEqual(predictor.doPredict(current_value), future_value) # only one possible value: take 03.01.2012 and predict 04.01.2012 current_value = StockData([(dt.date(2012, 1, 3), 35.554108)]) future_value = 36.055264 self.assertEqual(predictor.doPredict(current_value), future_value) # more than one possible values: take 08.01.1962 and predict 09.01.1962 # because values from 03.01.1962 till 08.01.1962 are identical current_value = StockData([(dt.date(1962, 1, 8), 0.060421)]) future_value = 0.061621 self.assertEqual(predictor.doPredict(current_value), future_value)
def doPredict(self, data: StockData) -> float: """ Always returns last value form given input vector. Args: data : historical stock values of a company Returns: last value from input +/- Random value between -1 and +1 """ # return last value from input +- random value between -1 and +1 return data.get_last()[-1] + random.uniform(-1.0, 1.0)
def as_trend(self, stock_data: StockData): trends = [] values = stock_data.get_values() for i in range(1, len(values)): if values[i - 1] < values[i]: trends.append(1) elif values[i - 1] > values[i]: trends.append(-1) else: trends.append(0) return trends
def doPredict(self, data: StockData) -> float: """ Use the loaded trained neural network to predict the next stock value. Args: data: historical stock values of a company Returns: predicted next stock value for that company """ # self.model.predict_classes stocks = data.get_values() length = len(stocks) last = stocks[length - (MODEL_LENGTH + 1):] xtrain = [] tuple = [] for j in range(MODEL_LENGTH): increase = (last[j + 1] - last[j]) / last[j] if increase > 1: increase = 1 if increase < -1: increase = -1 tuple.append(increase) xtrain.append(tuple) np_xtrain = np.array(xtrain) result = self.model.predict(np_xtrain) value = result[0][0] absolute_last = data.get_values()[-1] return absolute_last + absolute_last * value
def doPredict(self, data: StockData) -> float: """ Use the loaded trained neural network to predict the next stock value. Args: data: The historical stock values of a company Returns: The predicted next stock value for that company """ # Assumptions about data: at least 100 pairs of type (_, float) assert data is not None and data.get_row_count() >= 100 # Extract last 100 floats (here: stock values) as input for neural network (format: numpy array of arrays) input_values = np.array([[x[1] for x in data.get_from_offset(-100)]]) try: # Let network predict the next stock value based on last 100 stock values return self.model.predict(input_values) except: logger.error("Error in predicting next stock value.") assert False
def load(self, args={}): symbol = args["symbol"] frequency = args["frequency"] last_price_result = self.get_last_price_date(symbol=symbol, frequency=frequency) function = "TIME_SERIES_{}".format(frequency.upper()) key = app.config["ALPHAVANTAGE_KEY"] endpoint = "https://www.alphavantage.co/query" params = { "function":function, "symbol":symbol, "outputsize":"full", "apikey":key } #series key series_key = { "daily":"Time Series (Daily)", "weekly":"Weekly Time Series" } response = requests.get(endpoint, params=params) data = response.json() symbol = data["Meta Data"]["2. Symbol"] series = data[series_key[frequency]] for key, element in series.items(): price_date = datetime.strptime(key, '%Y-%m-%d').date() if price_date > last_price_result["max_price_date"]: stock_data_row = StockData( symbol=symbol, price_date=key, frequency=frequency, open=element["1. open"], high=element["2. high"], low=element["3. low"], close=element["4. close"], volume=element["5. volume"] ) db.session.add(stock_data_row) else: break db.session.commit() msg = "Cargado correctamente hasta: {}".format(data["Meta Data"]["3. Last Refreshed"]) return Response(msg=msg).get()
def doPredict(self, data: StockData) -> float: """ Use the loaded stock values to predict the next stock value. Args: data: historical stock values Returns: predicted next stock value for that company """ # Assumptions about data: at least one pair of type (_, float) assert data is not None and data.get_row_count() > 0 (current_date, current_value) = data.get_last() index = self.stock_data.index((current_date, current_value)) if index is not None and index < self.stock_data.get_row_count() - 1: (_, next_value) = self.stock_data.get(index + 1) return next_value else: logger.error( f"Couldn't make a perfect prediction for the day after {current_date}" ) assert False
def doPredict(self, data: StockData) -> float: """ Use the loaded trained neural network to predict the next stock value. Args: data: historical stock values of a company Returns: predicted next stock value for that company """ price_history = self.as_trend(data)[-INPUT_SIZE:] result = self.model.predict(np.array([price_history])) predirect_trend = result[0][0] return data.get_last()[1] + predirect_trend
def test_update__action_order_does_not_matter(self): """ Tests: Portfolio#update Flavour: It shouldn't matter which order the orders are in, the result should always look the same. In this case the portfolio's cash reserve is too low to execute a BUY action. However, it shouldn't matter if we execute a SELL action first, because the updated cash reserve after a SELL action shouldn't affect the available cash reserve for a subsequent BUY action Creates a portfolio, a stock market data object and a arbitrary `OrderList` and executes these orders on the portfolio. Checks if those are applied correctly """ cash_reserve = 10.0 data = StockData([(date(2017, 1, 1), 150.0)]) stock_market_data = StockMarketData({CompanyEnum.COMPANY_A: data}) # Create two equal designed portfolios portfolio1 = Portfolio(cash_reserve, [SharesOfCompany(CompanyEnum.COMPANY_A, 200)]) portfolio2 = Portfolio(cash_reserve, [SharesOfCompany(CompanyEnum.COMPANY_A, 200)]) assert portfolio1 == portfolio2 # Create two order lists with the same entries, however in different order order_list_1 = OrderList() order_list_1.buy(CompanyEnum.COMPANY_A, 100) order_list_1.sell(CompanyEnum.COMPANY_A, 100) order_list_2 = OrderList() order_list_2.sell(CompanyEnum.COMPANY_A, 100) order_list_2.buy(CompanyEnum.COMPANY_A, 100) # Execute the trade action lists on the two portfolios updated_portfolio_order1 = portfolio1.update(stock_market_data, order_list_1) updated_portfolio_order2 = portfolio2.update(stock_market_data, order_list_2) # The portfolios should still be equal after applying the actions assert updated_portfolio_order1 == updated_portfolio_order2
def learn_nn_and_save(training_data: StockData, test_data: StockData, filename_to_save: str): network = create_model() # TODO: learn network and draw results #np.append([[1, 2, 3], [4, 5, 6]], [7, 8, 9], axis=0) x_train = np.array([]) y_train = np.array([]) i = 0 batch = 5 while i < training_data.get_row_count() - batch: tmp = np.array([]) for j in range(0, batch): i = i + 1 diff = training_data.get(i)[1] - training_data.get(i + 1)[1] tmp = np.insert(tmp, j, diff) diff_y = 0 if (i < training_data.get_row_count() - 1): diff_y = training_data.get(i - 1)[1] - training_data.get(i)[1] if (diff_y > 0): y_train = np.append(y_train, 1) else: y_train = np.append(y_train, 0) x_train = np.append(x_train, tmp, axis=0) x_train = x_train.reshape((int(x_train.shape[0] / 5), 5)) y_train = y_train.reshape((-1, 1)) #for i in range(0, training_data.get_row_count() - 1): # diff = training_data.get(i)[1] - training_data.get(i + 1)[1] # if (diff < 0): # y_train.append([1]) # else: # y_train.append([0]) network.compile(loss='mean_squared_error', optimizer='sgd') network.fit(x_train, y_train, epochs=10, batch_size=5) # Save trained model: separate network structure (stored as JSON) and trained weights (stored as HDF5) save_keras_sequential(network, RELATIVE_PATH, filename_to_save)
def load_daily_data(self, symbol, since): last_loaded_date = self.get_max_loaded_daily_price(symbol) last_quote, quotes = MarketAPI().get_daily_data_since(symbol=symbol, since=since) if last_quote is None: return None if date.fromisoformat(last_quote.price_date) > last_loaded_date: for item in quotes: sd = StockData( symbol=item.symbol, price_date=item.price_date, frequency="daily", open=item.open, high=item.high, low=item.low, close=item.close, volume=item.volume ) db.session.add(sd) db.session.commit() return last_quote
def test_inspect__date_offset(self): """ Tests: Evaluator#inspect_over_time Flavour: Test with an date offset """ data = StockData([(date(2017, 1, 1), 150.0), (date(2017, 1, 2), 200.0), (date(2017, 1, 3), 250.0)]) stock_market_data = StockMarketData({CompanyEnum.COMPANY_A: data}) portfolio = Portfolio(20000, [SharesOfCompany(CompanyEnum.COMPANY_A, 200)]) evaluator = PortfolioEvaluator( [SimpleTrader(RandomPredictor(), RandomPredictor())]) portfolio_over_time: dict = \ evaluator.inspect_over_time(stock_market_data, [portfolio], date_offset=date(2017, 1, 2))['nameless'] assert date(2016, 12, 31) not in portfolio_over_time.keys() assert date(2017, 1, 1) in portfolio_over_time.keys() assert date(2017, 1, 2) in portfolio_over_time.keys() assert date(2017, 1, 3) not in portfolio_over_time.keys()
def learn_nn_and_save(training_data: StockData, test_data: StockData, filename_to_save: str): network = create_model() stocks = training_data.get_values() xtrain = [] ytrain = [] for i in range(len(stocks) - (MODEL_LENGTH + 2)): tuple = [] for j in range(MODEL_LENGTH): increase = (stocks[i + j + 1] - stocks[i + j]) / stocks[i + j] if increase > 1: increase = 1 if increase < -1: increase = -1 tuple.append(increase) xtrain.append(tuple) j = MODEL_LENGTH increase = (stocks[i + j + 1] - stocks[i + j]) / stocks[i + j] if increase > 1: increase = 1 if increase < -1: increase = -1 tuple2 = [] tuple2.append(increase) ytrain.append(tuple2) np_xtrain = np.array(xtrain) np_ytrain = np.array(ytrain) BATCH_SIZE = 100 EPOCHS = 50 network.fit(np_xtrain, np_ytrain, epochs=EPOCHS, batch_size=BATCH_SIZE) stocks = test_data.get_values() xtrain = [] ytrain = [] for i in range(len(stocks) - (MODEL_LENGTH + 2)): tuple = [] for j in range(MODEL_LENGTH): increase = (stocks[i + j + 1] - stocks[i + j]) / stocks[i + j] if increase > 1: increase = 1 if increase < -1: increase = -1 tuple.append(increase) xtrain.append(tuple) j = MODEL_LENGTH increase = (stocks[i + j + 1] - stocks[i + j]) / stocks[i + j] if increase > 1: increase = 1 if increase < -1: increase = -1 tuple2 = [] tuple2.append(increase) ytrain.append(tuple2) np_xtrain = np.array(xtrain) np_ytrain = np.array(ytrain) score = network.evaluate(np_xtrain, np_ytrain, batch_size=BATCH_SIZE) # Save trained model: separate network structure (stored as JSON) and trained weights (stored as HDF5) save_keras_sequential(network, RELATIVE_PATH, filename_to_save)
def get_test_data(): return StockData([(date(2017, 1, 1), 150.0), (date(2017, 1, 2), 200.0)])