def test_random(): # Sanity test to make sure that feature number positively impacts least squares error. num_points = 100 num_data_per_point = 55 learning_rate = 0.5 x_in = np.random.normal(size=(num_data_per_point, num_points)) for num_features in [1, 5, 10, 15, 20, 40, 70]: ae = AutoEncoder(x_in, num_features, random_seed=1234) w_in = np.random.normal(size=(num_data_per_point, num_features)) z_out, least_squares_test = ae.psi(w_in) print( f"(# features : Least squares error = ({num_features} : {least_squares_test})" ) print("Starting gradient decent...") loss_values = [] # Keep track of loss values over epochs for epoch in range(1000): z_grd, ls_grd, grd = ae.calc_g( w_in) # Calculate Z, Error, and Gradient Matrix w_in = w_in - (learning_rate * grd ) # Update W using Gradient Matrix loss_values.append(ls_grd) # Log loss print(f"Epoch: {epoch}\t----------\tLoss: {ls_grd}") # print(loss_values) plotter.plot_loss( loss_values, f"Gradient Loss Over Epochs (test) (num_features: {num_features})")
def test_mnist(): # Gradient check using MNIST (train_x, _), (_, _) = mnist.load_data() train_x = train_x / 255 # Normalizing images # plotter.plot_mnist(train_x, "original") # Show original mnist images num_img, img_dim, _ = train_x.shape # Get number of images and # pixels per square img num_features = 500 mnist_in = np.reshape( train_x, (img_dim * img_dim, num_img)) # Reshape images to match autoencoder input ga = Algorithm(x=mnist_in, num_features=num_features, debug=1, pop_size=20) w_out, best_cost, logs = ga.run() print( f"Average time/generation (sec): {sum(logs['times']) / len(logs['times'])}" ) print(f"Total time to run GA (sec): {logs['times']}") ae = AutoEncoder(mnist_in, num_features, random_seed=1234, use_gpu=True) z, _ = ae.psi(w_out) phi_w_img = ae.phi(w_out) # Calculate phi(W) new_mnist = z @ phi_w_img # Recreate original images using Z and phi(W) new_imgs = np.reshape( new_mnist, train_x.shape) # Reshape new images have original shape plotter.plot_mnist(new_imgs, f"{num_features}_features_ga") # Show new images # print(loss_values) plotter.plot_loss(logs['min'], "MNIST_Gradient_Loss_Over_Generations")
def test_mnist(num_epochs=None): # Gradient check using MNIST (train_x, _), (_, _) = mnist.load_data() train_x = train_x / 255 # Normalizing images # plotter.plot_mnist(train_x, "original") # Show original mnist images num_img, img_dim, _ = train_x.shape # Get number of images and # pixels per square img learning_rate = 0.5 num_features = 200 loss_values = [] # Keep track of loss values over epochs loss_values_less = [] loss_diffs = [] w_in = np.random.normal( size=(img_dim * img_dim, num_features)) # Generate random W matrix to test mnist_in = np.reshape( train_x, (img_dim * img_dim, num_img)) # Reshape images to match autoencoder input ae = AutoEncoder(mnist_in, num_features, random_seed=1234, use_gpu=True) start_time = time.time() times = [] if num_epochs: for epoch in range(num_epochs): w_in, z_grd = do_epoch(ae, w_in, learning_rate, loss_values, times, loss_values_less, loss_diffs, epoch, start_time) else: epoch_history_check = 5 epoch = 0 loss_avg = 1000 tol = 0.03 while loss_avg > tol: w_in, z_grd = do_epoch(ae, w_in, learning_rate, loss_values, times, loss_values_less, loss_diffs, epoch, start_time) loss_check = loss_diffs[-epoch_history_check:] loss_avg = sum(loss_check) / len(loss_check) epoch += 1 print( f"Total time to run gradient decent (sec): {time.time() - start_time}") phi_w_img = ae.phi(w_in) # Calculate phi(W) new_mnist = z_grd @ phi_w_img # Recreate original images using Z and phi(W) new_imgs = np.reshape( new_mnist, train_x.shape) # Reshape new images have original shape plotter.plot_mnist(new_imgs, f"{num_features}_features_gradient") # Show new images # print(loss_values) plotter.plot_loss(loss_values, "MNIST_Gradient_Loss_Over_Epochs") plotter.plot_loss( loss_values_less, "MNIST_Gradient_Loss_Over_Epochs_all_epochs_except_zero")
def test_cifar10(num_epochs=None): # (train_x, _), (_, _) = cifar10.load_data() (_, _), (train_x, _) = cifar10.load_data() print(train_x.shape) plotter.plot_mnist(train_x, "original") train_x = rgb2gray(train_x) train_x = train_x / 255 plotter.plot_mnist(train_x, "grayscale") num_img, img_h, img_w = train_x.shape print(train_x.shape) learning_rate = 0.5 num_features = 768; loss_values = [] loss_values_less = [] loss_diffs = [] w_in = np.random.normal(size=(img_h * img_w, num_features)) cifar_in = np.reshape(train_x, (img_h * img_w, num_img)) # cifar_in = np.reshape(train_x, (img_h, img_w, num_img*img_ch)) print(cifar_in.shape) ae = AutoEncoder(cifar_in, num_features, random_seed=1234, use_gpu=True) start_time = time.time() times = [] if num_epochs: for epoch in range(num_epochs): w_in, z_grd = do_epoch(ae, w_in, learning_rate, loss_values, times, loss_values_less, loss_diffs, epoch, start_time) else: epoch_history_check = 5 epoch = 0 loss_avg = 1000 tol = 0.03 while loss_avg > tol: w_in, z_grd = do_epoch(ae, w_in, learning_rate, loss_values, times, loss_values_less, loss_diffs, epoch, start_time) loss_check = loss_diffs[-epoch_history_check:] loss_avg = sum(loss_check) / len(loss_check) epoch += 1 print(f"Total time to run gradient decent (sec): {time.time() - start_time}") phi_w_img = ae.phi(w_in) # Calculate phi(W) new_cifar = z_grd @ phi_w_img # Recreate original images using Z and phi(W) print(new_cifar.shape) new_imgs = np.reshape(new_cifar, train_x.shape) # Reshape new images have original shape plotter.plot_mnist(new_imgs, f"{num_features}_features_gradient") # Show new images # print(loss_values) plotter.plot_loss(loss_values, "CIFAR10_Gradient_Loss_Over_Epochs") plotter.plot_loss(loss_values_less, "CIFAR10_Gradient_Loss_Over_Epochs_all_epochs_except_zero") # return train_x return new_imgs
def test_random(): # Sanity test to make sure that feature number positively impacts least squares error. num_points = 100 num_data_per_point = 55 x_in = np.random.normal(size=(num_data_per_point, num_points)) loss_values = [] for num_features in [1, 5, 10, 15, 20, 40, 70]: ga = Algorithm(x=x_in, num_features=num_features, debug=1) # w_in = np.random.normal(size=(num_data_per_point, num_features)) w_out, best_cost, logs = ga.run() loss_values.append(best_cost) plotter.plot_loss(loss_values, "Random_Test_with_Features", "Num features from list [1, 5, 10, 15, 20, 40, 70]")
def test_random(): # Sanity test to make sure that feature number positively impacts least squares error. num_points = 100 num_data_per_point = 55 x_in = np.random.normal(size=(num_data_per_point, num_points)) loss_values = [] for num_features in [1, 5, 10, 15, 20, 40, 70]: ae = AutoEncoder(x_in, num_features, random_seed=1234) w_in = np.random.normal(size=(num_data_per_point, num_features)) z_out, least_squares_test = ae.psi(w_in) loss_values.append(least_squares_test) print(f"(# features : Least squares error = ({num_features} : {least_squares_test})") plotter.plot_loss(loss_values, "Random_Test_with_Features", "Num features from list [1, 5, 10, 15, 20, 40, 70]")
def main(is_train, prediction, plotting, scaling, selected_model): if len(sys.argv) < 3: print(LINESPLIT) print("Usage: python3 {} <path_to_ssn_datafile> <path_to_aa_datafile>". format(os.path.basename(__file__))) data_file = "data/SILSO/TSN/SN_m_tot_V2.0.txt" aa_file = "data/ISGI/aa_1869-01-01_2020-12-19_D.dat" else: data_file = sys.argv[1] aa_file = sys.argv[2] print(LINESPLIT) print("Code running on device: {}".format(device)) ssn_data = datasets.SSN(data_file) aa_data = datasets.AA(aa_file) print(LINESPLIT) print('''Data loaded from file locations : SSN - {} AA - {}'''.format(os.path.abspath(data_file), os.path.abspath(aa_file))) if plotting: plotter.plot_all("combined_data1.jpg") cycle_data = ut.get_cycles(ssn_data) print(LINESPLIT) print("Solar cycle data loaded/saved as: cycle_data.pickle") print(LINESPLIT) ut.print_cycles(cycle_data) train_samples = datasets.Features(ssn_data, aa_data, cycle_data, normalize=scaling,\ start_cycle=13, end_cycle=22) valid_samples = datasets.Features(ssn_data, aa_data, cycle_data, normalize=scaling,\ start_cycle=23, end_cycle=23) valid_timestamps, _ = ut.gen_samples(ssn_data, aa_data, cycle_data,\ cycle=23, normalize=scaling, tf=cycle_data["length"][23]) predn_timestamps, predn_samples = ut.gen_samples(ssn_data, aa_data, cycle_data,\ cycle=24, normalize=scaling) print(LINESPLIT) print('''Selected data: Training: SC 13 to 22 Validation: SC 23 Prediction: SC 24''') ############ FFNN/RNN/LSTM (model chosen by user) ############ model = getattr(models, selected_model)(inp_dim=6).to(device) optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate) scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode="min", factor=0.9, verbose=True) print(LINESPLIT) print('''Selected model: {}\ Training mode: {}\ Prediction mode: {}'''.format(model, is_train, prediction)) print(LINESPLIT) print("Selected optimizer: {}".format(optimizer)) print(LINESPLIT) print('''Selected scheduler: {}( {})'''.format(scheduler.__class__.__name__, scheduler.state_dict())) pre_trained = load_model(model) if not pre_trained: if not is_train and prediction: print(LINESPLIT) print( "Warning: Prediction is ON with training OFF and no pretrained models available" ) train_loader = DataLoader(dataset=train_samples, batch_size=BATCH_SIZE, shuffle=True) valid_loader = DataLoader(dataset=valid_samples, batch_size=1, shuffle=False) ### Training ### if is_train: if not pre_trained: model.train() print(LINESPLIT) print("Training model with solar cycle {} to {} data with: num_epochs={}".\ format(datasets.START_CYCLE, datasets.END_CYCLE - 2, epochs)) loss = train(model, train_loader, optimizer, scheduler, epochs) torch.save( model.state_dict(), "{}_{}_{}.pth".format(modelfolder, model.__class__.__name__, MAX_EPOCHS)) plotter.plot_loss("Average Training Loss", range(len(loss)), loss, "tr_{}.png".\ format(model.__class__.__name__)) print(LINESPLIT) print('''Training finished successfully. Saved model checkpoints can be found in: {} Saved data/loss graphs can be found in: {}'''.format( modelfolder, graphfolder)) else: print(LINESPLIT) print( "Skipping training, using pre-trained model for validation and prediction" ) ### Validating ### model.eval() print(LINESPLIT) print("Validating model for solar cycle {} data".format( datasets.END_CYCLE - 1)) valid_predictions, valid_loss = validate(model, valid_loader, valid_timestamps) plotter.plot_predictions("SC{} Prediction".format(datasets.END_CYCLE - 1),\ valid_timestamps, valid_predictions, "SC 23 Validation.png", compare=True) plotter.plot_loss("Validation Loss", range(len(valid_loss)), valid_loss, "val_{}.png".\ format(model.__class__.__name__)) print(LINESPLIT) print('''Validation finished successfully.\n Saved prediction/loss graphs can be found in: {}'''.format( graphfolder)) ### Predicting ### if prediction: model.eval() print(LINESPLIT) print("Predicting SC {} using the above trained model".format( datasets.END_CYCLE)) predn_predictions = predict(model, predn_samples, predn_timestamps) plotter.plot_predictions("SC{} Prediction".format(datasets.END_CYCLE),\ predn_timestamps, predn_predictions, "SC 24 Prediction.png", compare=True)
json.dump(validation_list, f, indent=4) train_data_loader = DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True) test_data_loader = DataLoader(test_dataset, batch_size=args.batch_size, shuffle=True) predictor = HiddenLabelPredictorModel(bert, 768, args.num_predictors) trainer = UI2VecTrainer(predictor, train_data_loader, test_data_loader, vocab, len(vocab_list), args.rate, args.num_predictors, 768, args.loss) test_loss_data = [] train_loss_data = [] for epoch in tqdm.tqdm(range(args.epochs)): print(epoch) train_loss = trainer.train(epoch) print(train_loss) train_loss_data.append(train_loss) if test_data_loader is not None: test_loss = trainer.test(epoch) print(test_loss) test_loss_data.append(test_loss) if (epoch % 20) == 0: trainer.save(epoch, args.output_path) trainer.save(args.epochs, args.output_path) plot_loss(train_loss_data, test_loss_data)
2: hyper_params[2]['name'], 3: hyper_params[3]['name'], 4: hyper_params[4]['name']}) print(df) print() # Loop through features and train with given hyperparameters for params in hyper_params: column_name = params['name'] if column_name == 'Cancer' or column_name == 'Heart Disease' or column_name == 'Alsheimers' or column_name == 'Total': continue weight, bias, error, epoch_data = trainer.train_model( feature=df[column_name], label=df[label_name], learning_rate=params['learning_rate'], number_epochs=params['epochs'], batch_size=params['batch_size']) print(f'bias={bias}, weight={weight}') print() plotter.plot_model(title='Causes of Death', feature_title=column_name, label_title=label_name, weight=weight, bias=bias, feature_data=df[column_name], label_data=df[label_name]) plotter.plot_loss(epoch_data=epoch_data, root_mean_squared_error=error)
nn.add(Dense(512, activation='relu')) nn.add(Dropout(0.2)) nn.add(Dense(256, activation='relu')) nn.add(Dropout(0.2)) nn.add(Dense(256, activation='relu')) nn.add(Dropout(0.2)) nn.add(Dense(nb_breeds, activation='softmax')) # Compile the NN nn.compile(optimizer='sgd', loss='categorical_crossentropy', metrics=['accuracy']) # Train the NN history = nn.fit(x_train, y_train, batch_size=batch_size, epochs=nb_epochs, validation_split=0.2) # Evaluate the model with test set score = nn.evaluate(x_test, y_test, verbose=0) print('Test loss:', score[0]) print('Test accuracy:', score[1]) # Plots plt.plot_accuracy(history) plt.plot_loss(history) # Confusion matrix and classification report plt.get_statistics(nn, x_test, y_test)