def plot_and_print(t, rng, log, viz, ds, model, elapsed_time): """Utility function for plotting images and printing logs. """ # Generate model predictions. # --------------------------- Y_pred, F_pred, K_pred = model.predict(model.X, return_latent=True) # Plot visualizations. # -------------------- viz.plot_iteration(t, Y_pred, F_pred, K_pred, model.X) log.log_hline() log.log(t) # Log metrics. # ------------ mse_Y = mean_squared_error(Y_pred, ds.Y) log.log_pair('MSE Y', mse_Y) if ds.has_true_F: mse_F = mean_squared_error(F_pred, ds.F) log.log_pair('MSE F', mse_F) if ds.has_true_K: mse_K = mean_squared_error(K_pred, ds.K) log.log_pair('MSE K', mse_K) if ds.has_true_X: r2_X = r_squared(model.X, ds.X) log.log_pair('R2 X', r2_X) if ds.is_categorical: knn_acc = knn_classify(model.X, ds.labels, rng) log.log_pair('KNN acc', knn_acc) # Log parameters. # --------------- log.log_pair('DPMM LL', model.calc_dpgmm_ll()) log.log_pair('K', model.Z_count.tolist()) log.log_pair('alpha', model.alpha) n_mh_iters = (model.t + 1) * model.M log.log_pair('W MH acc', model.mh_accept / n_mh_iters) if hasattr(model, 'R'): log.log_pair('R median', np.median(model.R)) # Record time. # ------------ log.log_pair('time', elapsed_time) # Flush and save state. # --------------------- params = model.get_params() fpath = f'{args.directory}/{args.model}_rflvm.pickle' pickle.dump(params, open(fpath, 'wb'))
def compute_metrics(self, y_true, y_pred, num_classes): # Calculate metric qwk = np_quadratic_weighted_kappa(np.argmax(y_true, axis=1), np.argmax(y_pred, axis=1), 0, num_classes - 1) ms = minimum_sensitivity(y_true, y_pred) mae = mean_absolute_error(y_true, y_pred) omae = overall_mean_squared_error(y_true, y_pred) mse = mean_squared_error(y_true, y_pred) acc = categorical_accuracy(y_true, y_pred) top2 = top_2_accuracy(y_true, y_pred) top3 = top_3_accuracy(y_true, y_pred) off1 = accuracy_off1(y_true, y_pred) conf_mat = confusion_matrix(np.argmax(y_true, axis=1), np.argmax(y_pred, axis=1)) metrics = { 'QWK': qwk, 'MS': ms, 'MAE': mae, 'OMAE': omae, 'MSE': mse, 'CCR': acc, 'Top-2': top2, 'Top-3': top3, '1-off': off1, 'Confusion matrix': conf_mat } return metrics
def generate_reconstruction_perf(truths, preds): """Given truths and probs, generate appropriate perf object""" with warnings.catch_warnings(): warnings.simplefilter("ignore") retval = ReconstructionModelPerf(mse_loss=metrics.mean_squared_error( truths, preds), ) return retval
def gbdt(kind=1): if kind == 1: # Generate a random binary classification problem. X, y = make_classification(n_samples=350, n_features=15, n_informative=10, random_state=1111, n_classes=2, class_sep=1., n_redundant=0) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.15, random_state=1111) model = GBDT(n_estimators=50, max_tree_depth=4,loss=LogisticLoss(), max_features=8, lr=0.1) model.fit(X_train, y_train) predictions = model.predict(X_test) print(predictions) print(predictions.min()) print(predictions.max()) print('classification, roc auc score: %s' % roc_auc_score(y_test, predictions)) else: # Generate a random regression problem X, y = make_regression(n_samples=500, n_features=5, n_informative=5, n_targets=1, noise=0.05, random_state=1111, bias=0.5) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=1111) model = GBDT(n_estimators=50, max_tree_depth=5, max_features=3, min_samples_split=10, lr=0.1) model.fit(X_train, y_train) predictions = model.predict(X_test) print(y_test[:10], predictions[:10]) print('regression, mse: %s' % mean_squared_error(y_test.flatten(), predictions.flatten()))
def calculate_metrics_of_model(x_test, y_test, w, b): # calculate y using coefficients w and b y_calc = np.dot(x_test, w) + b # calculate mse, mae and r2 mse = mean_squared_error(y_test, y_calc) mae = mean_absolute_error(y_test, y_calc) r2 = r_square(y_test, y_calc) return mse, mae, r2
def regression(): # Generate a random regression problem X, y = make_regression(n_samples=10000, n_features=100, n_informative=75, n_targets=1, noise=0.01, bias=0.5) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25) model = LinearRegression(lr=0.001, max_iters=3000, alpha=0.03, descent_grad=None) model.fit(X_train, y_train) predictions = model.predict(X_test) # print(np.sort(np.abs(predictions.ravel() - y_test))[0:50]) print('regression mse', mean_squared_error(y_test, predictions))
def compare_sklearn(self, training_features, training_labels, test_features, test_labels, my_mse, my_r2): # do linear regression with sklearn from sklearn.linear_model import LinearRegression skmodel = LinearRegression() skmodel.fit(training_features, training_labels) skpreds = skmodel.predict(test_features) # get sklearn mse and r2 score skmse = metrics.mean_squared_error(test_labels, skpreds) skr2 = metrics.r2_score(test_labels, skpreds) print("Your model's MSE: {}\nsklearn's MSE: {}".format(my_mse, skmse)) print() print("Your model's R2 score: {}\nsklearn's R2 score: {}".format( my_r2, skr2))
rd_cv.fit(X_train, y_train) rd_cv.alpha_ 805.0291812295973 ------------------------------------------------------------------------------------------------- rd = Ridge(alpha=805.0291812295973) #, fit_intercept=Falserd.fit(X_train, y_train)print(rd.coef_)print(rd.intercept_) [ 0.00074612 -0.00382265 0.00532093 0.01100823 0.03375475 -0.01582157 0.0584206 0.09708992 0.02639369 0.0604242 -0.00116086] 2.7977274604845856 ------------------------------------------------------------------------------------------------- from sklearn import metrics from math import sqrt #分别预测训练数据和测试数据 y_train_pred = rd.predict(X_train) y_test_pred = rd.predict(X_test) #分别计算其均方根误差和拟合优度 y_train_rmse = sqrt(metrics.mean_squared_error(y_train, y_train_pred)) y_train_score = rd.score(X_train, y_train) y_test_rmse = sqrt(metrics.mean_squared_error(y_test, y_test_pred)) y_test_score = rd.score(X_test, y_test) print('训练集RMSE: {0}, 评分: {1}'.format(y_train_rmse, y_train_score)) print('测试集RMSE: {0}, 评分: {1}'.format(y_test_rmse, y_test_score)) ------------------------------------------------------------------------------------------------- from sklearn.linear_model import Lasso alphas = 10**np.linspace(-5, 10, 500) betas = [] for alpha in alphas: Las = Lasso(alpha = alpha) Las.fit(X_train, y_train) betas.append(Las.coef_) plt.figure(figsize=(8,6)) plt.plot(alphas, betas)
for m in range(number_of_models): # Polynomial order p = m + 2 # Design matrix X = design_matrix(p, x, y) Xm, Xn = np.shape(X) # Least squares normal_equation = X.T @ X B = np.linalg.solve(normal_equation, X.T @ z) # Regression statistics calulations zhat = X @ B MSE_train[m] = metrics.mean_squared_error(z, zhat) R2_train[m] = metrics.r2_score(z, zhat) Beta_conf_interval[:Xn, m] = metrics.confidance_interval( z, zhat, p, normal_equation, B) Bias2[m] = metrics.bias2(z, zhat) Variance_error[m] = metrics.variance_error(zhat) # Cross validation 2-fold CV_pred = [] for X_train, X_test, z_train, z_test in k_fold_CV(k, X, z): # Least squares B = np.linalg.solve(X_train.T @ X_train, X_train.T @ z_train) # Cross validation predictions
def main(_run, stock_file, days_back, days_forward, max_epochs, early_stopping_threshold, num_neurons, num_hidden_layers, seed, learning_rate, batch_size, activation, optimizer, kernel_init, regularization, loss, timesteps, use_sent_and_trends=False): # Read the stocks csv into a dataframe stock = data.Stocks(stock_file) stock.calc_patel_TI(days_back) if use_sent_and_trends: # If we have a sentiment file add it to the stock df sentiments = pd.read_csv( '../data/nytarticles/microsoft.2013-12-31.2018-12-31.imputed.sent.csv', index_col='date') trends = pd.read_csv( '../data/trends/msft.2013-12-31.2018-12-31.fixed.dates.csv', index_col='date') sent_trends = pd.merge(sentiments, trends, how='left', left_index=True, right_index=True) sent_trends[ 'sent_trends'] = sent_trends['sentiment'] * sent_trends['msft'] import numpy as np sent_trends['randNumCol'] = np.random.randint(1, 100, sent_trends.shape[0]) stock.df = pd.merge(stock.df, sent_trends, how='left', left_index=True, right_index=True) stock.df.drop(['sentiment', 'msft', 'sent_trends'], axis='columns', inplace=True) stock.shift(days_forward) # Create the model model = K.Sequential() # Create the kernel initializer with the seed if kernel_init == 'glorot_uniform': kernel_initializer = K.initializers.glorot_uniform(seed) else: raise NotImplementedError # Add the layers return_sequences = True if num_hidden_layers == 1: return_sequences = False data_dim = stock.raw_values()['X'].shape[1] model.add( K.layers.LSTM(num_neurons, input_shape=(timesteps, data_dim), activation=activation, return_sequences=return_sequences, kernel_initializer=kernel_initializer)) for i in range(num_hidden_layers - 1): # If not in the last layer return sequences if i != num_hidden_layers - 2: model.add( K.layers.LSTM(num_neurons, activation=activation, return_sequences=True, kernel_initializer=kernel_initializer)) else: model.add( K.layers.LSTM(num_neurons, activation=activation, kernel_initializer=kernel_initializer)) # Add output layer model.add( K.layers.Dense(1, activation='linear', kernel_initializer=kernel_initializer)) # Define Root Mean Squared Relative Error metric def root_mean_squared_relative_error(y_true, y_pred): squared_relative_error = K.backend.square( (y_true - y_pred) / K.backend.clip(K.backend.abs(y_true), K.backend.epsilon(), None)) mean_squared_relative_error = K.backend.mean(squared_relative_error, axis=-1) return K.backend.sqrt(mean_squared_relative_error) # Define Direction Accuracy metric def direction_accuracy(y_true, y_pred): # sign returns either -1 (if <0), 0 (if ==0), or 1 (if >0) true_signs = K.backend.sign(y_true[days_forward:] - y_true[:-days_forward]) pred_signs = K.backend.sign(y_pred[days_forward:] - y_true[:-days_forward]) equal_signs = K.backend.equal(true_signs, pred_signs) return K.backend.mean(equal_signs, axis=-1) # Create the optimizer if optimizer == 'adagrad': optimizer = K.optimizers.Adagrad(learning_rate) elif optimizer == 'adam': optimizer = K.optimizers.Adam(learning_rate) else: raise NotImplementedError model.compile(optimizer=optimizer, loss=loss, metrics=[ 'mean_absolute_percentage_error', 'mean_absolute_error', root_mean_squared_relative_error, 'mean_squared_error', direction_accuracy ]) # Create the logging callback # The metrics are logged in the run's metrics and at heartbeat events # every 10 secs they get written to mongodb def on_epoch_end_metrics_log(epoch, logs): for metric_name, metric_value in logs.items(): # The validation set keys have val_ prepended to the metric, # add train_ to the training set keys if 'val' not in metric_name: metric_name = 'train_' + metric_name _run.log_scalar(metric_name, metric_value, epoch) metrics_log_callback = K.callbacks.LambdaCallback( on_epoch_end=on_epoch_end_metrics_log) callbacks_list = [ K.callbacks.EarlyStopping(monitor='val_loss', patience=early_stopping_threshold), K.callbacks.ModelCheckpoint(filepath='../models/best_model.h5', monitor='val_loss', save_best_only=True), metrics_log_callback ] model.fit(stock.raw_values_lstm_wrapper(dataset='train', norm=True, timesteps=timesteps)['X'], stock.raw_values_lstm_wrapper(dataset='train', norm=True, timesteps=timesteps)['y'], epochs=max_epochs, batch_size=batch_size, verbose=0, callbacks=callbacks_list, validation_data=(stock.raw_values_lstm_wrapper( dataset='val', norm=True, timesteps=timesteps)['X'], stock.raw_values_lstm_wrapper( dataset='val', norm=True, timesteps=timesteps)['y'])) # Calculate metrics for normalized values test_norm_metrics = model.evaluate( stock.raw_values_lstm_wrapper(dataset='test', norm=True, timesteps=timesteps)['X'], stock.raw_values_lstm_wrapper(dataset='test', norm=True, timesteps=timesteps)['y'], verbose=0) # Log the metrics from the normalized values for metric in zip(model.metrics_names, test_norm_metrics): _run.log_scalar('test_norm_' + metric[0], metric[1]) # Now calculate and save the unnormalised metrics # Predict returns normalised values y_pred_norm = model.predict( stock.raw_values_lstm_wrapper(dataset='test', norm=True, timesteps=timesteps)['X']) # Scale the output back to the actual stock price y_pred = stock.denorm_predictions(y_pred_norm) # Calculate the unnormalized metrics y_true = stock.raw_values_lstm_wrapper(dataset='test', timesteps=timesteps)['y'] # df1 = pd.DataFrame({'date': stock.df.index.values[-y_pred.shape[0]:], 'y_pred': y_pred.flatten(), 'y_true': y_true.flatten()}) # df1.set_index('date', inplace=True) # df1.to_csv('plot_data_lstm.csv') test_metrics = { 'test_loss': metrics.mean_squared_error(y_true, y_pred), 'test_mean_absolute_percentage_error': metrics.mean_absolute_percentage_error(y_true, y_pred), 'test_mean_absolute_error': metrics.mean_absolute_error(y_true, y_pred), 'test_root_mean_squared_relative_error': metrics.root_mean_squared_relative_error(y_true, y_pred), 'test_mean_squared_error': metrics.mean_squared_error(y_true, y_pred), 'test_direction_accuracy': metrics.direction_accuracy(y_true, y_pred, days_forward) } # Save the metrics for metric_name, metric_value in test_metrics.items(): _run.log_scalar(metric_name, metric_value)
boston.head() from sklearn.preprocessing import StandardScaler X = boston.drop("MEDV", axis=1).values Y = boston["MEDV"].values X_train, X_test, Y_train, Y_test = train_test_split(X,Y, test_size=0.3, random_state=0) ss = StandardScaler() X_train_std = ss.fit_transform(X_train) X_test_std = ss.transform(X_test) ll = LinearRegressionGD() ll.fit(X_train_std, Y_train) Y_pred = ll.predict(X_test_std) print(mean_squared_error(Y_test, Y_pred)) ll = LinearRegression() ll.fit(X_train_std, Y_train) Y_pred = ll.predict(X_test_std) print(mean_squared_error(Y_test, Y_pred)) from sklearn.linear_model import LinearRegression as lr ll = lr() ll.fit(X_train_std, Y_train) Y_pred = ll.predict(X_test_std) print(mean_squared_error(Y_test, Y_pred))
from utils import datasets import metrics from linear_regression import LinearRegression import numpy as np X_train, y_train, X_test, y_test = datasets.boston_split(0.87) solve_by = 'gdesc' # the other option is 'ols' from sklearn.preprocessing import StandardScaler scaler = StandardScaler() X_train = scaler.fit_transform(X_train) X_test = scaler.transform(X_test) model = LinearRegression(solve_by=solve_by) model.train(X_train, y_train) predictions = model.predict(X_test) mse = metrics.mean_squared_error(y_test, predictions) r2_score = metrics.r2_score(y_test, predictions) model.compare_sklearn(X_train, y_train, X_test, y_test, mse, r2_score)
def polynomial_regression(): mse_train = [] mse_test = [] N = 100 max_degree = 10 random_list_size = 10 d = 4 x, y = generate_regression_data(d, N, amount_of_noise=0.1) x_train, y_train = np.zeros( (random_list_size, 1)), np.zeros((random_list_size, 1)) x_test, y_test = np.zeros( (N - random_list_size, 1)), np.zeros((N - random_list_size, 1)) random_list = [] for i in range(0, random_list_size): n = random.randint(0, N - 1) while n in random_list: n = random.randint(0, N - 1) random_list.append(n) counter_train = 0 counter_test = 0 for i in range(N): if i in random_list: x_train[counter_train] = x[i] y_train[counter_train] = y[i] counter_train += 1 else: x_test[counter_test] = x[i] y_test[counter_test] = y[i] counter_test += 1 for degree in range(max_degree): p = PolynomialRegression(degree) p.fit(x_train, y_train) y_hat_train = p.predict(x_train) y_hat_test = p.predict(x_test) if len(mse_train) == 0 or min(mse_train) > mean_squared_error(y_train, y_hat_train): min_train_y_predict = y_hat_train if len(mse_test) == 0 or min(mse_test) > mean_squared_error(y_test, y_hat_test): min_test_y_predict = y_hat_test mse_train.append(mean_squared_error(y_train, y_hat_train)) mse_test.append(mean_squared_error(y_test, y_hat_test)) # p.visualize(x_test, y_test) # Q1A plt.figure() plt.plot(range(max_degree), mse_train, color='orange', label='The train error') plt.plot(range(max_degree), mse_test, color='blue', label='The test error') plt.title('error vs degree') plt.xlabel('degree') plt.ylabel('error') plt.yscale('log') plt.legend(loc="best") plt.savefig("Q1A.png") # Q1B features_sorted = np.zeros(x_train.shape) targets_sorted = np.zeros(min_train_y_predict.shape) sort_indexes = x_train.argsort(axis=0) for i in range(len(x_train.argsort(axis=0))): features_sorted[i] = x_train[sort_indexes[i]] targets_sorted[i] = min_train_y_predict[sort_indexes[i]] features2_sorted = np.zeros(x_test.shape) targets2_sorted = np.zeros(min_test_y_predict.shape) sort_indexes = x_test.argsort(axis=0) for i in range(len(x_test.argsort(axis=0))): features2_sorted[i] = x_test[sort_indexes[i]] targets2_sorted[i] = min_test_y_predict[sort_indexes[i]] plt.figure() plt.scatter(x_train, y_train, color='blue') plt.plot(features_sorted, targets_sorted, color='orange', label='The lowest training error') plt.plot(features2_sorted, targets2_sorted, color='green', label='The lowest testing error') plt.title('X vs Y') plt.xlabel('X') plt.ylabel('Y') plt.legend(loc="best") plt.savefig("Q1B.png") # Q5 # we create 50 separable points X, Y = make_blobs(n_samples=50, centers=2, random_state=0, cluster_std=0.60) # fit the model clf = SGDClassifier(loss="hinge", alpha=0.01, max_iter=200) clf.fit(X, Y) # plot the line, the points, and the nearest vectors to the plane xx = np.linspace(-1, 5, 10) yy = np.linspace(-1, 5, 10) X1, X2 = np.meshgrid(xx, yy) Z = np.empty(X1.shape) for (i, j), val in np.ndenumerate(X1): x1 = val x2 = X2[i, j] p = clf.decision_function([[x1, x2]]) Z[i, j] = p[0] levels = [-1.0, 0.0, 1.0] linestyles = ['dashed', 'solid', 'dashed'] colors = 'k' cs = plt.contour(X1, X2, Z, levels, colors=colors, linestyles=linestyles) plt.scatter(X[:, 0], X[:, 1], c=Y, cmap=plt.cm.Paired, edgecolor='black', s=20, label='data points') cs.collections[0].set_label('h(x)=0') plt.axis('tight') plt.title('Linear Classification Example') plt.xlabel('x1') plt.ylabel('x2') plt.legend(loc="best") plt.savefig("Q5.png")
def score(self, X: np.array, y: np.array) -> np.float: y_pred = self.raw_predict(X) score = mean_squared_error(y, y_pred) return score
def main(): # Load args args = get_args() # Load Dataset # data_transform = getTransforms() # Consider adding additional transforms image_datasets = { 'train': MatteDataset(root_dir=PATH), 'val': MatteDataset(root_dir=VAL) } dataloaders = { x: torch.utils.data.DataLoader(image_datasets[x], batch_size=args.batch_size, shuffle=True, num_workers=args.threads) for x in ['train', 'val'] } # Setup Model epoch = 0 best_loss = 50000 early_stopping = args.early_cutoff # encdec = LinkNet34(1) encdec = DeepMattingVGG() if (args.checkpoint != 'fresh'): save_name = os.listdir('models/' + args.checkpoint + '/encdec/')[np.argmax( np.array([ int(s[-8:-4]) for s in os.listdir('models/' + args.checkpoint + '/encdec/') ]))] print("Loading encoder-decoder:", save_name) ckpt = torch.load('models/' + args.checkpoint + '/encdec/' + save_name) epoch = ckpt['epoch'] best_loss = ckpt['loss'] encdec.load_state_dict(ckpt['state_dict']) if (args.stage != 0): refinement = MatteRefinementLayer() if (os.listdir('models/' + args.checkpoint + '/refinement/')): save_name = os.listdir( 'models/' + args.checkpoint + '/refinement/')[np.argmax( np.array([ int(s[-8:-4]) for s in os.listdir('models/' + args.checkpoint + '/refinement/') ]))] print("Loading refinement:", save_name) ckpt = torch.load('models/' + args.checkpoint + '/refinement/' + save_name) refinement.load_state_dict(ckpt['state_dict']) if (args.stage == 1): best_loss = ckpt['loss'] refinement = refinement.to(device) encdec = encdec.to(device) # _ed suffix refers to encoder-decoder part of the model, # _r suffix refers to refinement part if (args.weighted_loss): crit_ed = metrics.AlphaCompLoss_u() else: crit_ed = metrics.AlphaCompLoss() optim_ed = optim.Adam(encdec.parameters(), lr=1e-5) sched_ed = CyclicLR(optim_ed, 5e-6, 1e-4, 200) if (args.stage != 0): if (args.weighted_loss): crit_r = metrics.AlphaLoss_u() else: crit_r = metrics.AlphaLoss() optim_r = optim.Adam(refinement.parameters(), lr=1e-5) sched_r = CyclicLR(optim_r, 5e-6, 5e-5, 200) # TRAIN # Writers for TensorBoard train_writer = SummaryWriter('logs/train' + args.save_dir) val_writer = SummaryWriter('logs/val' + args.save_dir) if (args.epochs == -1): num_epochs = 10000 else: num_epochs = args.epochs for e in tqdm(range(num_epochs)): if (not early_stopping): break # Each epoch has a training and validation phase for phase in ['train', 'val']: if phase == 'train': if (args.stage == 0): encdec.train() elif (args.stage == 1): encdec.eval() refinement.train() elif (args.stage == 2): encdec.train() refinement.train() else: encdec.eval() # Set model to evaluate mode if (args.stage != 0): refinement.eval() if (args.stage != 1): running_loss_ed = [] if (args.stage != 0): running_loss_r = [] running_sad = [] running_mse = [] # Iterate over dataset. for i, sample in tqdm(enumerate(dataloaders[phase])): # Get inputs and labels, put them on GPU inputs_ed, labels, fg, bg = sample['inputs'].to( device), sample['mask'].to(device), sample['fg'].to( device), sample['bg'].to(device) trimap = inputs_ed[:, 3, :, :] # zero the gradients optim_ed.zero_grad() if (args.stage != 0): optim_r.zero_grad() # Calculate loss with torch.set_grad_enabled( phase == 'train'): # Only track grads if in train mode outputs_ed = encdec( inputs_ed) # Output is single channel matte if (args.stage != 0): inputs_r = torch.cat( (inputs_ed[:, :3, :, :], outputs_ed), 1) outputs_r = refinement(inputs_r) # _, preds = torch.max(outputs, 1) if (args.stage != 1): if (args.weighted_loss): loss_ed = crit_ed(outputs_ed, labels, fg, bg, trimap) else: loss_ed = crit_ed(outputs_ed, labels, fg, bg) running_loss_ed.append(loss_ed.item()) if (args.stage != 0): if (args.weighted_loss): loss_r = crit_r(outputs_r, labels, trimap) else: loss_r = crit_r(outputs_r, labels) running_loss_r.append(loss_r.item()) # backward + optimize only if in training phase if (phase == 'train'): if (args.stage == 0): loss_ed.backward() optim_ed.step() sched_ed.batch_step() if (args.stage == 1): loss_r.backward() optim_r.step() sched_r.batch_step() if (args.stage == 2): loss_ed.backward() optim_ed.step() optim_r.step() sched_ed.batch_step() sched_r.batch_step() if (phase == 'val'): if (args.stage == 0): if (args.weighted_loss): running_sad.append( metrics.sum_absolute_differences_u( outputs_ed, labels, trimap).item()) running_mse.append( metrics.mean_squared_error_u( outputs_ed, labels, trimap).item()) else: running_sad.append( metrics.sum_absolute_differences( outputs_ed, labels).item()) running_mse.append( metrics.mean_squared_error( outputs_ed, labels).item()) else: if (args.weighted_loss): running_sad.append( metrics.sum_absolute_differences_u( outputs_r, labels, trimap).item()) running_mse.append( metrics.mean_squared_error_u( outputs_r, labels, trimap).item()) else: running_sad.append( metrics.sum_absolute_differences( outputs_r, labels).item()) running_mse.append( metrics.mean_squared_error( outputs_r, labels).item()) # Record average epoch loss for TensorBoard if (args.stage != 1): epoch_loss_ed = np.array(running_loss_ed).mean() if (args.stage != 0): epoch_loss_r = np.array(running_loss_r).mean() if (phase == 'train'): if (args.stage != 1): train_writer.add_scalar("Encoder-Decoder Loss", epoch_loss_ed, epoch + e) if (args.stage != 0): train_writer.add_scalar("Refinement Loss", epoch_loss_r, epoch + e) if (phase == 'val'): early_stopping -= 1 val_writer.add_scalar("Mean Squared Error", np.array(running_mse).mean(), epoch + e) val_writer.add_scalar("Sum of Absolute Differences", np.array(running_sad).mean(), epoch + e) if (args.stage != 1): if (epoch_loss_ed < best_loss): early_stopping = args.early_cutoff best_loss = epoch_loss_ed checkpoint(epoch + e, args.save_dir, encdec, best_loss, encdec=True) if (args.stage != 0): checkpoint(epoch + e, args.save_dir, refinement, best_loss, encdec=False) val_writer.add_scalar("Encoder-Decoder Loss", epoch_loss_ed, epoch + e) if (args.stage != 0): if (args.stage == 1 and epoch_loss_r < best_loss): early_stopping = args.early_cutoff best_loss = epoch_loss_r checkpoint(epoch + e, args.save_dir, refinement, best_loss, encdec=False) checkpoint(epoch + e, args.save_dir, encdec, best_loss, encdec=True) val_writer.add_scalar("Refinement Loss", epoch_loss_r, epoch + e) train_writer.close() val_writer.close()
for m in range(number_of_models): # Polynomial degree p = m + 2 # Design matrix X = design_matrix(p, x, y) Xm, Xn = np.shape(X) # Lasso regression model = linear_model.LassoCV(alphas=lambdas, fit_intercept=False, cv=k) model.fit(X, z) print('p =', p, ', lambda = ', model.alpha_) # Regression statistics calculations zhat = model.predict(X) MSE[m] = metrics.mean_squared_error(z, zhat) R2[m] = metrics.r2_score(z, zhat) Variance_model = metrics.variance_model(z, zhat, p) Beta_variance[:Xn, m] = np.diag(metrics.covariance_matrix(X, Variance_model)) Bias[m] = metrics.bias(z, zhat) Variance_error[m] = metrics.variance_error(zhat) ########################################################################### # Plot model image = np.reshape(zhat, (a, b)).astype(int) terrain_plot(image)
def plot_single_number_metric_helper(dataset, dsmetric, models, rs, true_result, metric, norm, ds_kernel, thresh_pos, thresh_neg, thresh_pos_sim, thresh_neg_sim, plot_results, extra_dir): # dsmetric: distance/similarity metric, e.g. ged, mcs, etc. # metric: eval metric. print_ids = [] rtn = {} val_list = [] for model in models: if metric == 'mrr': val = mean_reciprocal_rank( true_result, rs[model], norm, print_ids) elif metric == 'mse': val = mean_squared_error( true_result, rs[model], ds_kernel, norm) elif metric == 'dev': val = mean_deviation( true_result, rs[model], ds_kernel, norm) elif metric == 'time': val = average_time(rs[model]) elif 'acc' in metric: val = accuracy( true_result, rs[model], thresh_pos, thresh_neg, thresh_pos_sim, thresh_neg_sim, norm) pos_acc, neg_acc, acc = val if metric == 'pos_acc': val = pos_acc elif metric == 'neg_acc': val = neg_acc elif metric == 'acc': val = acc # only the overall acc else: assert (metric == 'accall') elif metric == 'kendalls_tau': val = kendalls_tau(true_result, rs[model], norm) elif metric == 'spearmans_rho': val = spearmans_rho(true_result, rs[model], norm) else: raise RuntimeError('Unknown {}'.format(metric)) # print('{} {}: {}'.format(metric, model, mrr_mse_time)) rtn[model] = val val_list.append(val) rtn = {'{}{}'.format(metric, get_norm_str(norm)): rtn} if not plot_results: return rtn plt = plot_multiple_bars(val_list, models, metric) if metric == 'time': ylabel = 'time (msec)' norm = None elif metric == 'pos_acc': ylabel = 'pos_recall' elif metric == 'neg_acc': ylabel = 'neg_recall' elif metric == 'kendalls_tau': ylabel = 'Kendall\'s $\\tau$' elif metric == 'spearmans_rho': ylabel = 'Spearman\'s $\\rho$' else: ylabel = metric plt.ylabel(ylabel) if metric == 'time': plt.yscale('log') metric_addi_info = '' bfn = '{}_{}{}_{}_{}{}'.format( dsmetric, metric, metric_addi_info, dataset, '_'.join(models), get_norm_str(norm)) sp = get_result_path() + '/{}/{}/'.format(dataset, metric) save_fig(plt, sp, bfn) if extra_dir: save_fig(plt, extra_dir, bfn) print(metric, 'plotted') return rtn
def score(self, x_test, y_test): y_predict = self.predict(x_test) return 1 - mean_squared_error(y_test, y_predict) / np.var(y_test)