def test(model_name, plot_file=None, test_directory=DEFAULT_TEST_DIRECTORY, model_directory=DEFAULT_MODEL_DIRECTORY):
model = deepconvnet(model_name, LR, (IMG_SIZE, IMG_SIZE), model_directory=model_directory)
X_orig, y_test = load_images_with_labels(test_directory)
X = preprocess(X_orig, [conv_gray_scale, lambda x: resize_image(x, (IMG_SIZE, IMG_SIZE))])
X = np.array(X).reshape(-1, IMG_SIZE, IMG_SIZE, 1)
start = time.time()
predictions = model.predict(X)
print("\nTime taken to predict outcomes: {} secs \n".format(time.time() - start))
if plot_file:
plot_predictions(X_orig, predictions, plot_file)
def fit(self, X, y):
show_plots = False
self.num_experts_ = len(self.experts)
self.num_classes_ = y.shape[1]
self.__initialize(X, y)
obj_vals = []
while len(obj_vals) <= 1 or (abs(obj_vals[-2] - obj_vals[-1]) > 1e-4
and len(obj_vals) < self.max_iter):
expert_weights = self.__E_step(X, y)
obj_val = self.__M_step(X, y, expert_weights)
obj_vals += [obj_val]
print obj_val
if show_plots:
plot_predictions(X, y, self.gate, "Gate predictions")
for i in range(self.num_experts_):
plot_predictions(X, y, self.experts[i],
"Expert #" + str(i) + " predictions")
plot_predictions(X, y, self)
# print obj_vals
return self
# Invert the transformation process on X, Y_pred, Y_gt to get the final history, and horizon predictions:
X, Y_pred, Y_gt = tsfake_task.standard_scale_inverse(
X, Y_pred, Y_gt, transform_params)
#**The [batch x T x M x Q][0] indices are the point estimates (0th index along last axis)
# so use Y_gt[:,:,MM,0] and Y_pred[:,:,MM,0]
multivar_inds = [mm for mm in range(logger.n_multivariate)]
for nn, MM in enumerate(multivar_inds):
plot_regression_scatterplot(Y_pred[:, :, MM, 0].view(-1),
Y_gt[:, :, MM, 0].view(-1),
logger.output_dir,
logger.n_epochs_completed, nn)
plot_predictions(X[INDEX, :, MM], Y_gt[INDEX, :, MM,
0], Y_pred[INDEX, :,
MM],
logger.output_dir, logger.n_epochs_completed,
nn, QUANTILES_LIST, QUANTILES_INDS)
# print(Y_pred[:10])
# print(Y[:10])
print()
t_val_end = time.perf_counter()
elapsed_val_time = t_val_end - t_val_start
print(f'elapsed_val_time = {elapsed_val_time}')
# Save model / optimizer checkpoints:
#...
# Plot training/validation loss
loss = loss_func(pred, labels).item()
predictions.append(pred.item())
ground_truth.append(labels.item())
# print out useful logging information for user
avg_loss += loss
if batch_id % 50 == 0:
print("(test) Batch {}/{} -- Loss: {} -- Pred: {}, True: {}".format(
batch_id + 1, len(loader), loss, pred.item(), labels.item()))
avg_loss /= len(loader)
print("(test) avg loss: {}".format(avg_loss))
if len(ood_stock_fns) == 1:
fluctuation_correct = 0
for i in range(1, len(predictions)):
if ground_truth[i] > ground_truth[
i - 1] and predictions[i] > predictions[i - 1]:
fluctuation_correct += 1
elif ground_truth[i] < ground_truth[
i - 1] and predictions[i] < predictions[i - 1]:
fluctuation_correct += 1
fluctuation_accuracy = fluctuation_correct / (len(predictions) - 1)
print("Fluctuation accuracy: {}%".format(
round(fluctuation_accuracy * 100.0, 2)))
if len(ood_stock_fns) == 1: # only have to plot one stock's predictions
stock_ticker = ood_stock_fns[0].split(".")[0]
pred_graph_filename = "{}_price_prediction_smoothed".format(stock_ticker)
plot_predictions(stock_ticker, pred_graph_filename, ground_truth,
predictions)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
# Scale inputs
scaler = MinMaxScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)
# MLP regressor fit
regressor = MLPRegressor(hidden_layer_sizes=[256, 128, 64], max_iter=1000)
regressor.fit(X_train_scaled, y_train)
y_hat_train = regressor.predict(X_train_scaled)
y_hat_test = regressor.predict(X_test_scaled)
# Plot predictions
utils.plot_predictions(y=[y_train, y_test],
y_hat=[y_hat_train, y_hat_test],
labels=['Train', 'Test'],
save_path='./nn_fit.pdf')
# ############################ #
# Illustration of over-fitting #
# ############################ #
# Data split with validation
X_train, X_valid, y_train, y_valid = train_test_split(X_train,
y_train,
test_size=0.1)
scaler = MinMaxScaler()
scaler.fit(X_train)
# MLP regressor fit
# For demo purposes, we are going to fit the NN on a smaller dataset
def train(config):
Dataset = get_dataloader(config['data']['dataset'])
dataset_path = config['data']['path']
if 'augmentations' in config['training']:
augmentation_dict = config['training']['augmentations']
data_aug = get_augmentations(augmentation_dict)
else:
data_aug = None
data_train = Dataset(root=dataset_path, train=True, augmentations=data_aug)
data_test = Dataset(root=dataset_path, train=False)
os.makedirs("samples", exist_ok=True)
batch_size = config['training']['batch_size']
training_loader = DataLoader(data_train,
batch_size=batch_size,
shuffle=True)
testing_loader = DataLoader(data_test,
batch_size=batch_size,
shuffle=False)
# Setup device
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
if device == 'cpu':
warnings.warn(
"You don't have cuda available. The training takes long time.")
n_classes = config['model']['n_classes']
model = UNet(num_classes=n_classes,
in_channels=config['model']['in_channels'],
depth=config['model']['depth'],
start_filt_num=config['model']['n_start_filters'],
filt_num_factor=config['model']['filter_num_scale'],
merge_mode=config['model']['merge_mode']).to(device)
if torch.cuda.device_count() > 1:
model = nn.DataParallel(model)
loss_weight = torch.ones(n_classes)
loss_weight[0] = 0.6
loss_weight[2] = 0.8
loss_param = (config['training']['loss']['name']
) #{'weight_ce': loss_weight.to(device)}
loss_func = get_loss_function(loss_param).to(device)
Optimizer = get_optimizer(config['training']['optimizer']['name'])
optimizer_params = {
k: v
for k, v in config['training']['optimizer'].items() if k != 'name'
}
optimizer = Optimizer(model.parameters(), **optimizer_params)
if 'lr_scheduler' in config['training']:
Scheduler = get_lr_scheduler(
config['training']['lr_scheduler']['name'])
scheduler_params = {
k: v
for k, v in config['training']['lr_scheduler'].items()
if k != 'name'
}
lr_scheduler = Scheduler(optimizer, **scheduler_params)
else:
lr_scheduler = None
epochs = config['training']['n_epochs']
logger = Logger("./logs")
iter = 0
for epoch in range(1, epochs + 1):
epoch_loss = []
epoch_dice_loss = []
epoch_ce_loss = []
model.train()
if lr_scheduler is not None:
lr_scheduler.step()
for images, labels, weights in training_loader:
images = Variable(images.to(device))
labels = Variable(labels.to(device))
weights = Variable(weights.to(device))
optimizer.zero_grad()
outputs = model(images)
loss, dice_loss, ce_loss = loss_func(outputs, labels, weights)
loss.backward()
optimizer.step()
iter += 1
# 1. Log scalar values (scalar summary)
info = {
'loss': loss.item(),
'ce_loss': ce_loss.item(),
'dice_loss': dice_loss.item()
}
for tag, value in info.items():
logger.scalar_summary(tag, value, iter)
epoch_ce_loss.append(ce_loss.item())
epoch_dice_loss.append(dice_loss.item())
epoch_loss.append(loss.item())
epoch_avg_loss = sum(epoch_loss) / len(epoch_loss)
epoch_avg_dice = sum(epoch_dice_loss) / len(epoch_dice_loss)
epoch_avg_ce = sum(epoch_ce_loss) / len(epoch_ce_loss)
model.eval()
cnf_matrix_validation = np.zeros((n_classes, n_classes))
for idx, (img, lbls, _) in enumerate(testing_loader):
img = Variable(img.to(device))
lbls = Variable(lbls.to(device))
with torch.no_grad():
outs = model(img)
preds = outs.data.max(1)[1]
cnf_matrix_validation += confusion_matrix(
preds.cpu().view(-1).numpy(),
lbls.cpu().view(-1).numpy())
if idx == 2:
plt_title = 'Train Results Epoch ' + str(epoch)
file_save_name = os.path.join(
"samples",
'Epoch_' + str(epoch) + '_Train_Predictions.pdf')
classes = torch.unique(lbls).cpu().numpy()
plot_predictions(img, lbls, preds, plt_title, file_save_name)
nTotal = len(testing_loader)
cnf_matrix = cnf_matrix_validation / nTotal
save_name = os.path.join("samples",
'Epoch_' + str(epoch) + '_Validation_CM.pdf')
plot_confusion_matrix(cnf_matrix, classes, file_save_name=save_name)
dice_score = 2 * np.diag(cnf_matrix) / (cnf_matrix.sum(axis=1) +
cnf_matrix.sum(axis=0))
dice_score = np.mean(dice_score)
out_msg = '[ Epoch {}/{} ] [ Total Loss: {:.4f} ] [Dice Loss: {:.4f}] [CE Loss: {:.4f}] [Avg Dice Score: {:.4f}]'.format(
epoch, epochs, epoch_avg_loss, epoch_avg_dice, epoch_avg_ce,
dice_score)
print(out_msg)
def gradient_boosting(doplot=False):
from sklearn.ensemble import GradientBoostingRegressor
from utils import plot_predictions, save_fig
np.random.seed(42)
X = np.random.rand(100, 1) - 0.5
y = 3 * X[:, 0]**2 + 0.05 * np.random.randn(100)
gbrt_fast = GradientBoostingRegressor(max_depth=2,
n_estimators=3,
learning_rate=1.0)
gbrt_fast.fit(X, y)
gbrt_slow = GradientBoostingRegressor(max_depth=2,
n_estimators=200,
learning_rate=0.1)
gbrt_slow.fit(X, y)
localX_train, localX_val, localy_train, localy_val = train_test_split(X, y)
gbrt = GradientBoostingRegressor(max_depth=2, n_estimators=120)
gbrt.fit(localX_train, localy_train)
errors = [
mean_squared_error(localy_val, localy_pred)
for localy_pred in gbrt.staged_predict(localX_val)
]
best_n_estimators = np.argmin(errors)
print("Best n_estimators for GBRT = %d trees\n" % best_n_estimators)
gbrt_best = GradientBoostingRegressor(max_depth=2,
n_estimators=best_n_estimators)
gbrt_best.fit(localX_train, localy_train)
if doplot:
plt.figure(figsize=(11, 4))
plt.subplot(131)
plot_predictions([gbrt_fast],
X,
y,
axes=[-0.5, 0.5, -0.1, 0.8],
label="Ensemble predictions")
plt.title("learning_rate = {}, n_estimators = {}".format(
gbrt_fast.learning_rate, gbrt_fast.n_estimators))
plt.subplot(132)
plot_predictions([gbrt_slow],
X,
y,
axes=[-0.5, 0.5, -0.1, 0.8],
label="Ensemble predictions")
plt.title("learning_rate = {}, n_estimators = {}".format(
gbrt_slow.learning_rate, gbrt_slow.n_estimators))
plt.subplot(133)
plot_predictions([gbrt_best],
X,
y,
axes=[-0.5, 0.5, -0.1, 0.8],
label="Ensemble predictions")
plt.title("Best model({} trees)".format(gbrt_best.n_estimators))
save_fig("gbrt_learning_rate_plot", CHAPTER_ID)
plt.show()
def gb_intuition(doplot=False):
np.random.seed(42)
X = np.random.rand(100, 1) - 0.5
y = 3 * X[:, 0]**2 + 0.05 * np.random.randn(100)
from sklearn.tree import DecisionTreeRegressor
from utils import plot_predictions, save_fig
tree_reg1 = DecisionTreeRegressor(max_depth=2, random_state=42)
tree_reg1.fit(X, y)
y2 = y - tree_reg1.predict(X)
tree_reg2 = DecisionTreeRegressor(max_depth=2, random_state=42)
tree_reg2.fit(X, y2)
y3 = y2 - tree_reg2.predict(X)
tree_reg3 = DecisionTreeRegressor(max_depth=2, random_state=42)
tree_reg3.fit(X, y3)
X_new = np.array([[0.8]])
y_pred = sum(
tree.predict(X_new) for tree in (tree_reg1, tree_reg2, tree_reg3))
print("Prediction of 3 trees ", y_pred)
if doplot:
plt.figure(figsize=(11, 11))
plt.subplot(321)
plot_predictions([tree_reg1],
X,
y,
axes=[-0.5, 0.5, -0.1, 0.8],
label="$h_1(x_1)$",
style='g-',
data_label='Training set')
plt.ylabel("$y$", fontsize=16, rotation=90)
plt.title("Residuals and tree predictions", fontsize=16)
plt.subplot(322)
plot_predictions([tree_reg1],
X,
y,
axes=[-0.5, 0.5, -0.1, 0.8],
label="$h(x_1) = h_1(x_1)$",
data_label='Training set')
plt.ylabel("$y$", fontsize=16, rotation=90)
plt.title("Ensemble predictions", fontsize=16)
plt.subplot(323)
plot_predictions([tree_reg2],
X,
y2,
axes=[-0.5, 0.5, -0.1, 0.8],
label="$h_2(x_1)$",
style='g-',
data_style="k+",
data_label='Residuals')
plt.ylabel("$y - h_1(x_1)$", fontsize=16, rotation=90)
plt.subplot(324)
plot_predictions([tree_reg1, tree_reg2],
X,
y,
axes=[-0.5, 0.5, -0.1, 0.8],
label="$h(x_1) = h_1(x_1) + h_2(x_1)$")
plt.ylabel("$y$", fontsize=16, rotation=90)
plt.subplot(325)
plot_predictions([tree_reg3],
X,
y3,
axes=[-0.5, 0.5, -0.1, 0.8],
label="$h_3(x_1)$",
style='g-',
data_style="k+")
plt.ylabel("$y - h_1(x_1) - h_2(x_1)$", fontsize=16, rotation=90)
plt.xlabel("$x_1", fontsize=16)
plt.subplot(326)
plot_predictions([tree_reg1, tree_reg2, tree_reg3],
X,
y,
axes=[-0.5, 0.5, -0.1, 0.8],
label="$h(x_1) = h_1(x_1) + h_2(x_1) + h_3(x_1)$")
plt.ylabel("$y$", fontsize=16, rotation=90)
plt.xlabel("$x_1", fontsize=16)
save_fig("gradient_boosting_plot", CHAPTER_ID)
plt.show()
np.random.seed(0)
# Load data and do train-test Split
df = pd.read_excel('./data/Concrete_Data.xls', sheet_name='Sheet1')
X, y = df[df.columns[:-1]], df[df.columns[-1]]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
# Gradient boosting (decision tree) fit
regressor = GradientBoostingRegressor()
regressor.fit(X_train, y_train)
y_hat_train = regressor.predict(X_train)
y_hat_test = regressor.predict(X_test)
# Plot predictions and feature importances
utils.plot_predictions(y=[y_train, y_test],
y_hat=[y_hat_train, y_hat_test],
labels=['Train', 'Test'],
save_path='./boosting_fit.pdf')
utils.plot_feature_importances(importances=regressor.feature_importances_,
columns=df.columns[:-1],
save_path='./boosting_feat.pdf')
# ####################################### #
# Cross-Validation Hyper-parameter Tuning #
# ####################################### #
# Define random search space
param_distributions = {
'n_estimators': stats.randint(low=10, high=1000),
'max_depth': stats.randint(low=2, high=6),
'min_samples_split': stats.randint(low=2, high=5),
'learning_rate': [1, 0.5, 0.25, 0.1, 0.05, 0.01]
model.load_weights(MODEL_WEIGHTS)
test_gen = ImageDataGenerator(rescale=1. / 255.0)
test_gen = test_gen.flow_from_directory(directory=TEST_DIR,
target_size=(64, 64),
batch_size=1,
class_mode=None,
color_mode='rgb',
shuffle=False,
seed=69)
class_indices = train_gen.class_indices
class_indices = dict((v, k) for k, v in class_indices.items())
test_gen.reset()
predictions = model.predict_generator(test_gen,
steps=len(test_gen.filenames))
predicted_classes = np.argmax(np.rint(predictions), axis=1)
true_classes = test_gen.classes
prf, conf_mat = display_results(true_classes, predicted_classes,
class_indices.values())
print(prf)
plot_predictions(true_classes, predictions, test_gen, class_indices)
print("saving model..")
model.save(SAVE_PATH)
X_test = ds_test[0]
Y_test = ds_test[1]
i_c0 = (Y == 0)
i_c1 = (Y == 1)
#TRAIN: true and false predictions
i_c0_t = (Y[i_c0] == 0)
i_c0_f = (Y[i_c0] == 1)
i_c1_t = (Y[i_c1] == 1)
i_c1_f = (Y[i_c1] == 0)
plot_predictions(i + 1,
X,
Y,
X_test,
Y_test,
pred_train=Y,
pred_test=Y_test)
# As we can see, we are given 3 distinct datasets, each of which contains a training set and a test set. Each of the datasets was generated by a distinct data generative process, resulting in distinct class separations.
# ## 1. Linear Classification
# ### 1.1. Linear least squares for classification
# In the previous practical session, we explored how to tune a linear regression using a least squares loss function. We saw that minimization of the least square loss function led to a simple closed-form solution (see also Week #2 Slide 31).
#
# As discussed in the capsules, similar idea can also be applied to the classification task, with the difference that now we additionally require a decision rule for classification.
# As discussed in class, let the decision rule be simply $sign(y(x))$, where $y(x) = W^Tx$ is our discriminant function. Thus, such a classifier will return class '+' for all the points lying on one side of the decision boundary and '-' for the ones lying on the other side. Let's implement this simple classifier. We will encode the two classes as '-1' and '+1'.
for epoch in range(epochs):
gen = get_batches(data, data_path + 'images_small/', batch_size)
#if epoch % 5 == 0:
# learning_rate /= 2
print('Start training epoch {} with learning rate {}'.format(
epoch, learning_rate))
idx = 0
for X, y, msk in gen:
_, batch_loss = sess.run(
[train_op, loss],
feed_dict={
input_tensor: X,
train_flag: True,
labels: y,
mask: msk,
l_rate: learning_rate
})
message = "Epoch {}/{}: Training batch {}/{} Current loss is: {}\r".format(
epoch, epochs, idx, batches_per_epoch, batch_loss)
idx += 1
sys.stdout.write(message)
saver.save(sess, 'model_weights/new_model_{}.ckpt'.format(epoch))
y_hat = sess.run(pred, feed_dict={input_tensor: X, train_flag: False})
# plot predictions after training
threshold = 0.6
plot_predictions(y_hat, X, anchors, threshold)
optimizer = tf.train.AdamOptimizer(learning_rate=1e-4)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_op = optimizer.minimize(loss)
saver = tf.train.Saver()
# overfit to one image
gen = get_batches(data, batch_size=1)
x, y, msk = next(gen)
with tf.Session() as sess:
saver.restore(sess, 'model/new_model.ckpt')
for i in tqdm.tqdm(range(100)):
sess.run(train_op,
feed_dict={
input_tensor: x,
train_flag: True,
labels: y,
mask: msk
})
saver.save(sess, 'model/overfit.ckpt')
y_hat = sess.run(pred, feed_dict={input_tensor: x, train_flag: False})
threshold = 0.5
for i in range(5):
plot_predictions(y_hat, x.copy(), anchors, threshold + i * 0.1)
(train_images, dataset_augmentation(train_images))) # 3744 images now
train_masks = np.concatenate((train_masks, dataset_augmentation(train_masks)))
# ------- MODEL TRAINING ----------
model = get_model()
model.compile(optimizer='adam', loss=[LOSS], metrics=['accuracy'])
fit = model.fit(train_images,
train_masks,
batch_size=BATCH_SIZE,
epochs=EPOCHS_NUMBER)
test_loss, test_acc = model.evaluate(test_images, test_masks, verbose=2)
summary = model.summary()
# output into array with shape (100,100,1)
predictions = model.predict(test_images).reshape(
(len(test_images), 100, 100, 1))
# save model, loss values, accuracy values and predicted masks on computer
model_path = os.path.join(PATH, 'models', '')
model.save(model_path + SAVING_NAME)
results_file = open(
'models/{}test_loss{}test_accuracy.txt'.format(test_loss, test_acc), 'w')
results_file.write(SAVING_NAME)
results_file.close()
plot_acc_loss(fit.history['accuracy'], fit.history['loss'], SAVING_NAME)
plot_predictions(predictions, test_masks, SAVING_NAME)