def train_pass(self): start_time = t() for step, batch in enumerate(self.trn_set): elapsed_time = t() - start_time time_per_step = elapsed_time / (step + 1) exp_rem_time = (len(self.trn_set) - (step + 1)) * time_per_step print("\r" + str(step + 1) + "/" + str(len(self.trn_set)) + " processed batches (elapsed time: " + utils.format_time(elapsed_time) + " exp. rem. time: " + utils.format_time(exp_rem_time) + ")", end="") # Get next input input_enc = batch["encoded_image"] if P.GPU: input_enc = input_enc.cuda() # Run the network on the input self.network.train(True) self.network.run(inputs={"X": input_enc}, time=self.config.TIME) self.network.train(False) # Reset network state self.network.reset_state_variables() # Evaluate performance at fixed intervals if (((step + 1) * self.config.BATCH_SIZE) % self.config.EVAL_INTERVAL == 0) or (step == len(self.trn_set) - 1): print("\nEvaluating...") print("Computing train accuracy...") self.eval_pass(self.trn_set4eval, train=True) print("Computing validation accuracy...") self.eval_pass(self.val_set, train=False) # Print results print("Current evaluation step: " + str(len(self.stats_manager.eval_accuracy))) print("Current trn. accuracy: " + str(100 * self.stats_manager.trn_accuracy[-1]) + "%") print("Current val. accuracy: " + str(100 * self.stats_manager.eval_accuracy[-1]) + "%") print("Top accuracy so far: " + str(100 * self.stats_manager.best_acc) + "%" + " at evaluation step: " + str(self.stats_manager.best_step)) # Plot results utils.plot_performance( self.stats_manager.trn_accuracy, self.stats_manager.eval_accuracy, self.config.RESULT_FOLDER + "/accuracy.png") # Check if accuracy improved if self.stats_manager.check_improvement(): # Save model print("Top accuracy improved! Saving new best model...") self.network.save(self.config.RESULT_FOLDER + "/model.pt") print("Model saved!") print("Evaluation complete!") print("Continuing training...")
def train_lazy(): # Load the dataset X, y = load_data() # Split the data X_train, X_val, y_train, y_val = split_dataset(X, y) # # Normalize X_train = normalize(X_train) X_val = normalize(X_val) # uncomment to check the performance of the 25 models # clf = LazyClassifier(verbose=0,ignore_warnings=True, custom_metric=None) # # fit # scores,_ = clf.fit(X_train, X_val, y_train, y_val) # # print # print(scores) # Final model # check if model exist if os.path.isfile(config.MODEL_PATH): model = XGBClassifier() model.load_model(config.MODEL_PATH) else: model = XGBClassifier() model.fit(X_train, y_train, eval_metric="error", eval_set=[(X_train, y_train), (X_val, y_val)], verbose=True) # save model model.save_model(config.MODEL_PATH) # performance on train set y_pred = model.predict(X_train) # evaluate predictions print_performance(y_train, y_pred, 'train') # performance on val set y_pred = model.predict(X_val) # evaluate predictions print_performance(y_val, y_pred, 'val') # Load the test dataset X_test, y_test = load_test_data() # # Normalize X_test = normalize(X_test) # get prediction y_pred = model.predict(X_test) # evaluate predictions print_performance(y_test, y_pred, 'test') # print plot_performance(model)
def main(): epochs = 20 batch_size = 128 x_list, y_list = generate_training_data_lists() steps = len(x_list) / batch_size train_sequence = TrainSequence(x_list, y_list, batch_size) model_name = 'Dual-stream 3D Convolution Neural Network' model = get_model(model_name)(protein_data_shape=(None, None, None, 2), ligand_data_shape=(None, None, None, 2)) plot_model(model, to_file='./{}.png'.format(model_name)) print('Model Summary:') model.summary() model.compile(loss='binary_crossentropy', optimizer='sgd', metrics=['acc']) history = model.fit_generator(train_sequence, epochs=epochs, steps_per_epoch=steps, verbose=1).history dump_pickle('./history.pkl', history) model.save_weights('./{}_weights.h5'.format(model_name)) plot_performance(history, model_name, epochs, batch_size)
def main(): # Prepare data x_list, y_list = generate_training_data_lists() steps = len(x_list) / batch_size train_sequence = TrainSequence(x_list, y_list, batch_size) # Prepare model model = mlp(10000) model.compile(loss='binary_crossentropy', optimizer='sgd', metrics=['acc']) # Plot model plot_model(model, to_file='./{}.png'.format(model_name)) # Fit model history = model.fit_generator(train_sequence, epochs=epochs, steps_per_epoch=steps, verbose=1).history # Plot loss vs accuracy plot_performance(history, model_name, epochs, batch_size) # Write loss and acc to file dump_pickle('./history.pkl', history) # Save model model.save_weights('./{}_weights.h5'.format(model_name))
] = nn.trainNN(sess, loss, train_step, num_epochs, x, y, y_, weights, biases, trainDataCombined, trainTargetsCombined, validationDataCombined, validationTargetsCombined) # Output - Final weights for index, weight in enumerate(weights): sess.run(weight.assign(final_weights[index])) for index, bias in enumerate(biases): sess.run(bias.assign(final_biases[index])) trainOutputFinal = sess.run(y, feed_dict={x: trainDataCombined}) validationOutputFinal = sess.run(y, feed_dict={x: validationDataCombined}) testOutputFinal = sess.run(y, feed_dict={x: testDataCombined}) # Evaluate Model name = "noisy+symbols-noisy_symbols - " + str(datetime.datetime.now()) utils.plot_performance(train_losses, validation_losses, num_epochs, "Noisy with Symbols to Noisy with Symbols", name) #utils.render_output(validationOutputFinal, testOutputFinal, validationDataCombined, testDataCombined, validationTargetsCombined, name + "-final") [testScoreFinal, testResultsFinal] = utils.validate(testOutputFinal, testTargets) #[trainScoreFinal, trainResultsFinal] = utils.validate(trainOutputFinal, trainTargetClasses) [validationScoreFinal, validationResultsFinal] = utils.validate(validationOutputFinal, validationTargetClasses) testSymbolsScore = utils.validateSymbols(testOutputFinal, testTargetsSymbols) f = open("summaries/" + name + ".txt", mode="w") f.write( "Noisy with Symbols to Noisy with Symbols - 2 hidden layers - 100 units each - 1000 epochs - AdamOptimizer - Learning Rate: 0.01 \n" ) f.write("Minimum Training Loss: " + str(min_training_loss) + "\n") f.write("Minimum Validation Loss: " + str(min_validation_loss) + "\n") f.write("Test Score: " + str(testScoreFinal) + "\n")
trainOutput = sess.run(y, feed_dict={x: trainData}) validationOutput = sess.run(y, feed_dict={x: validationData}) testOutput = sess.run(y, feed_dict={x: testData}) # Output - Final weights for index, weight in enumerate(weights): sess.run(weight.assign(final_weights[index])) for index, bias in enumerate(biases): sess.run(bias.assign(final_biases[index])) trainOutputFinal = sess.run(y, feed_dict={x: trainData}) validationOutputFinal = sess.run(y, feed_dict={x: validationData}) testOutputFinal = sess.run(y, feed_dict={x: testData}) # Evaluate Model name = "noisy-noisy - " + str(datetime.datetime.now()) utils.plot_performance(train_losses, validation_losses, num_epochs, "Noisy to Noisy", name) utils.render_output(validationOutput, testOutput, validationData, testData, validationTargets, name + "-optimum") utils.render_output(validationOutputFinal, testOutputFinal, validationData, testData, validationTargets, name + "-final") [testScore, testResults] = utils.validate(testOutput, testTargets) [testScoreFinal, testResultsFinal] = utils.validate(testOutputFinal, testTargets) [trainScore, trainResults] = utils.validate(trainOutput, trainTargetClasses) [trainScoreFinal, trainResultsFinal] = utils.validate(trainOutputFinal, trainTargetClasses) [validationScore, validationResults] = utils.validate(validationOutput, validationTargetClasses) [validationScoreFinal, validationResultsFinal] = utils.validate(validationOutputFinal, validationTargetClasses)
running_loss += (loss_t - running_loss) / (batch_index + 1) # compute the accuracy acc_t = compute_accuracy(y_pred, batch_dict['y_target']) running_acc += (acc_t - running_acc) / (batch_index + 1) val_bar.set_postfix(loss=running_loss, acc=running_acc, epoch=epoch_index) val_bar.update() train_state['val_loss'].append(running_loss) train_state['val_acc'].append(running_acc) train_state = update_train_state(args=args, model=classifier, train_state=train_state) scheduler.step(train_state['val_loss'][-1]) if train_state['stop_early']: break train_bar.n = 0 val_bar.n = 0 epoch_bar.update() except KeyboardInterrupt: print("Exiting loop") plot_performance(train_state)
from sklearn.ensemble import RandomForestRegressor from sklearn.datasets.base import load_boston from sklearn.model_selection import train_test_split from skpro.parametric import ParametricEstimator from skpro.parametric.estimators import Constant from skpro.metrics import log_loss # Define the parametric model model = ParametricEstimator(point=RandomForestRegressor(), std=Constant('std(y)'), shape='norm') # Train and predict on boston housing data X, y = load_boston(return_X_y=True) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3) y_pred = model.fit(X_train, y_train).predict(X_test) # Obtain the loss loss = log_loss(y_test, y_pred, sample=True, return_std=True) print('Loss: %f+-%f' % loss) # Plot the performance import sys sys.path.append('../') import utils utils.plot_performance(y_test, y_pred)
def backtest(start_date, end_date, start_asset, mode="lstm"): if mode == "uniform": rebalance = uniform_portfolio elif mode == "random": rebalance = random_portfolio else: rebalance = rebalance_porfolio # ================= Initialize Configurations ==================== # 1. Set Start Asset # 2. Set Back Test Dates # 3. Initialize Back Test Records # 4. Load Companies List # 5. Load Companies Data # 1. Set Start Asset current_asset = start_asset current_portfolio = dict() # 2. Set Back Test Dates day_out = 5 path = os.path.join(DIR_CONFIG["DATA_DIR"], 'AAPL_data.csv') df = pd.read_csv(path) df = df[(df["Date"] > start_date) & (df["Date"] <= end_date)] backtest_dates = df["Date"].values print("Back Test Dates: ", backtest_dates) # 3. Initialize Back Test Records benchmark_records = [] backtest_records = [] portfolio_records = [] # 4. Load Companies List companies_list = [] with open(FILE_CONFIG["COMPANY_LIST"], 'r') as f: for line in f: companies_list.append(line.rstrip("\n")) # 5. Load All Companies Data companies_data = dict() for company in companies_list: path = os.path.join(DIR_CONFIG["DATA_DIR"], '{}_data.csv'.format(company)) companies_data[company] = pd.read_csv(path) # ================================================================ print("======================Start Backtest==========================") print("Start Date: ", start_date) print("End Date: ", end_date) print("Start Asset: ", start_asset) print("Mode: ", mode) # Start Back Test for i in range(len(backtest_dates)): # Rebalance Portfolio Every 5 Days if (i % 5) == 0 and (i + day_out) < len(backtest_dates): current_portfolio, percentages_portfolio = rebalance( date=backtest_dates[i], asset=current_asset, companies_data=companies_data) portfolio_records.append(percentages_portfolio) # Evaluate Porfolio Every Day current_asset = evaluate_portfolio(date=backtest_dates[i], portfolio=current_portfolio, companies_data=companies_data) backtest_records.append(current_asset) print("Current Asset Value: ", current_asset) print("Records: ", backtest_records) # Plot Portfolio Performance Graph plot_performance(filename='portfolio_graph_{}_{}'.format(mode, start_date), backtest_dates=backtest_dates, backtest_records=backtest_records) # Save Performance and Portfolio Records save_json(filename='portfolio_performance_{}_{}'.format(mode, start_date), records=backtest_records) save_json(filename='portfolio_records_{}_{}'.format(mode, start_date), records=portfolio_records)
from sklearn.datasets.base import load_boston from sklearn.model_selection import train_test_split from skpro.baselines import DensityBaseline from skpro.metrics import log_loss # Load boston housing data X, y = load_boston(return_X_y=True) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3) # Train and predict on boston housing data using a baseline model y_pred = DensityBaseline().fit(X_train, y_train)\ .predict(X_test) # Obtain the loss loss = log_loss(y_test, y_pred, sample=True, return_std=True) print('Loss: %f+-%f' % loss) # Plot performance from utils import plot_performance plot_performance(y_test, y_pred)
def test(model, saver, sess, exp_string, data_generator, test_num_updates=None): num_classes = data_generator.num_classes # for classification, 1 otherwise np.random.seed(1) random.seed(1) steps = range(1000, 61000, 1000) accs = [] inner_loops = 5 for step in steps: print(f"Load model {step}") load_model(FLAGS.logdir, exp_string, saver, sess, step) metaval_accuracies = [] for _ in range(NUM_TEST_POINTS): if 'generate' not in dir(data_generator): feed_dict = {} feed_dict = {model.meta_lr: 0.0} else: batch_x, batch_y, amp, phase = data_generator.generate( train=False) if FLAGS.baseline == 'oracle': # NOTE - this flag is specific to sinusoid batch_x = np.concatenate([ batch_x, np.zeros([batch_x.shape[0], batch_x.shape[1], 2]) ], 2) batch_x[0, :, 1] = amp[0] batch_x[0, :, 2] = phase[0] inputa = batch_x[:, :num_classes * FLAGS.update_batch_size, :] inputb = batch_x[:, num_classes * FLAGS.update_batch_size:, :] labela = batch_y[:, :num_classes * FLAGS.update_batch_size, :] labelb = batch_y[:, num_classes * FLAGS.update_batch_size:, :] feed_dict = { model.inputa: inputa, model.inputb: inputb, model.labela: labela, model.labelb: labelb, model.meta_lr: 0.0 } if model.classification: result = sess.run([model.metaval_total_accuracy1] + model.metaval_total_accuracies2, feed_dict) else: # this is for sinusoid result = sess.run([model.total_loss1] + model.total_losses2, feed_dict) metaval_accuracies.append(result[inner_loops]) accs.append(np.array(metaval_accuracies)) plot_performance(steps, accs, NUM_TEST_POINTS)
with model: # Priors alpha = pm.Normal('alpha', mu=y.mean(), sd=10) betas = pm.Normal('beta', mu=0, sd=10, shape=X.get_value(borrow=True).shape[1]) sigma = pm.HalfNormal('sigma', sd=1) # Model (defines y_pred) mu = alpha + pm.math.dot(betas, X.T) y_pred = pm.Normal("y_pred", mu=mu, sd=sigma, observed=y) # Plug the model definition into the PyMC interface model = BayesianVendorEstimator( model=PymcInterface(model_definition=pymc_linear_regression) ) # Run prediction, print and plot the results data = DataManager('boston') y_pred = model.fit(data.X_train, data.y_train).predict(data.X_test) print('Log loss: ', log_loss(data.y_test, y_pred, return_std=True)) # Plot the performance import sys sys.path.append('../') import utils utils.plot_performance(data.y_test, y_pred)