def main():
# convert_to_numpy()
print 'Loading data...'
data = load_data_set('Data/trainData.npy')
obj = DimReductionMissingData(data, reduced_dim=100)
obj.optimize(num_epochs=5, batch_size=1000)
def load_checkpoint(checkpoint_path, use_gpu):
''' Loads a checkpoint and rebuilds the model '''
if use_gpu and torch.cuda.is_available():
checkpoint = torch.load(checkpoint_path)
else:
checkpoint = torch.load(checkpoint_path,
map_location=lambda storage, loc: storage)
# Retrieve variables for classifier
data_dir_name = checkpoint['data_dir']
arch_name = checkpoint['arch']
out_size = checkpoint['output_size']
hidden_layer_sizes = checkpoint['hidden_layers']
dropout_p = checkpoint['dropout_p']
# Retrieve variables for model
epochs = checkpoint['epochs']
learning_rate = checkpoint['learning_rate']
_, _image_datasets = utils.load_data_set(data_dir_name)
# Re-build model
model = DerivedModel()
model.load_base_model(arch_name)
model.assign_new_classifier(out_size, hidden_layer_sizes,
_image_datasets['train'], dropout_p)
model.assign_optimizer(learning_rate)
model.base_model.load_state_dict(checkpoint['state_dict'])
model.optimizer.load_state_dict(checkpoint['optimizer'])
print("\nModel from '{}' has been successfully loaded".format(
checkpoint_path))
return model
def test_with_model(model):
print(" - Loading test data")
testing_files_path = "./dataset/test/raw"
testing_num_files = count_files_in_folder(testing_files_path)
TESTING_BATCH_SIZE = testing_num_files
raw_dataset = load_data_set()
test_raw_dataset = raw_dataset['test']
test = test_raw_dataset.map(load_image_test)
test_dataset = test.batch(TESTING_BATCH_SIZE)
for images, true_masks in test_dataset:
pass # Hack needed to be able to extrac images and true masks from map datasets
images = images.numpy()
true_masks = true_masks.numpy()
print(f" - Test data loaded for {testing_num_files} images")
print(" - Prediction started")
predictions = model.predict(test_dataset)
predicted_images_number = len(predictions)
print(f" - Prediction finished for {predicted_images_number} images")
print(f" - Let's transform predictions into labeled values.")
labeled_predictions = []
for image_index in range(predicted_images_number):
prediction = predictions[image_index]
predicted_mask = map_prediction_to_mask(prediction)
labeled_predictions.append(predicted_mask)
print(f" - Saving labeled images into ./output folder")
predicted_index = 0
output_path = "./output/"
if not os.path.exists(output_path):
os.makedirs(output_path)
for predicted_result in labeled_predictions:
prediction_as_image = labeled_prediction_to_image(predicted_result)
prediction_as_image.save(f"{output_path}{predicted_index:03d}.jpg")
prediction_as_image.close()
print(f" - Image with index {predicted_index} saved.")
predicted_index += 1
print(f" - Generating sample output page")
generate_output_template()
print(f" - Images saved. Time to check accuracy metrics per label:")
background_acc, border_acc, content_acc = compare_accuracy(true_masks, labeled_predictions)
print(" - Accuracy measures per label:")
print(f" - Border label = {border_acc}")
print(f" - Content label = {content_acc}")
print(f" - Background label = {background_acc}")
print(f" - Saving predicted masks as images")
def main():
''' Pre-process inputs '''
# Define base argument(s)
_output_size = 102 # Model's output size
data_dir = os.path.abspath('flowers') # Main data directory
# Parse CLI input argument(s)
in_args, extras = get_input_args()
data_dir = utils.check_file_folder(extras[0], True) # Extract data set path
# Parameters with default argparse values
_checkpoint_path = in_args.save_dir # checkpoint file path
checkpoint_path = os.path.abspath(_checkpoint_path)
arch_name = in_args.arch # model architecture
_learn_rate = in_args.learning_rate # learning rate
_epochs = in_args.epochs # training epochs
hidden_units = in_args.hidden_units # hidden layer sizes
_use_gpu = in_args.gpu # gpu flag
_dropout_p = in_args.dropout # dropout probability
_print_every = in_args.print_every # Print stats every batch instance count
# Load data set
data_dir_name = str(data_dir.split('/')[-1])
_dataloaders, _image_datasets = utils.load_data_set(data_dir_name)
''' Build and train the network '''
# Instantiate Derived Model
dmodel = DerivedModel()
dmodel.load_base_model(arch_name)
dmodel.assign_new_classifier(_output_size, hidden_units, _image_datasets['train'], _dropout_p)
dmodel.assign_optimizer(_learn_rate)
# Train model: Do Deep Learning
dmodel.deep_learn(_dataloaders, epochs=_epochs, print_every=_print_every, use_gpu=_use_gpu)
''' Save the model state '''
# Save checkpoint
if save_checkpoint(dmodel, checkpoint_path, data_dir_name):
print("\nCheckpoint saved in {}".format(checkpoint_path))
def train(learning_rate=0.01, max_step=5):
train_x, train_y, test_x, test_y = utils.load_data_set(one_hot=True)
ws, bs, hyper_parameters = init_variable()
for step in range(max_step):
z1 = conv_forward(train_x, ws[0], bs[0], hyper_parameters[0])
a1 = pool_forward(z1, hyper_parameters[1], mode="max")
a2 = fully_connection_nn(a1, ws[1], bs[1])
y_pred = soft_max(a2.T)
activations = [zero_pad(train_x, hyper_parameters[0]["pad"]), z1, a1]
dws, dbs = back_propagation(y_pred, train_y, ws, bs, activations,
hyper_parameters)
ws = [w - learning_rate * dw for w, dw in zip(ws, dws)]
bs = [b - learning_rate * db for b, db in zip(bs, dbs)]
print("step=%s loss is %s " %
(step, cross_entropy_loss(y_pred, train_y)))
print(" - Loading saved model")
tf.random.set_seed(11)
custom_objects = {
"border_acc": border_acc,
"background_acc": background_acc,
"content_acc": content_acc,
"iou_coef": iou_coef,
"dice_coef": dice_coef
}
model = keras.models.load_model("./model", custom_objects=custom_objects)
print(" - Loading test data")
testing_files_path = "./dataset/test/raw"
testing_num_files = count_files_in_folder(testing_files_path)
TESTING_BATCH_SIZE = testing_num_files
raw_dataset = load_data_set()
test_raw_dataset = raw_dataset['test']
test = test_raw_dataset.map(load_image_test)
test_dataset = test.batch(TESTING_BATCH_SIZE)
for images, true_masks in test_dataset:
pass # Hack needed to be able to extrac images and true masks from map datasets
images = images.numpy()
true_masks = true_masks.numpy()
print(f" - Test data loaded for {testing_num_files} images")
print(" - Prediction started")
predictions = model.predict(test_dataset)
predicted_images_number = len(predictions)
print(f" - Prediction finished for {predicted_images_number} images")
print(f" - Let's transform predictions into labeled values.")
def run_experiment(arguments):
# Load data set
X, Y, log_tf = load_data_set(arguments['dataset'], path_to_source)
estim = load_estimator(arguments['estimator_kwargs'], path_to_source)
# Prepare for experiments
n_test_sets = arguments['n_test_sets']
test_size = arguments['test_size']
param_grid = arguments[
'param_grid'] # Parameter grid for estimator to CV over
cv_folds = arguments['cv_folds']
n_jobs = arguments['n_jobs']
kf = ShuffleSplit(n_splits=arguments['n_test_sets'],
test_size=arguments['test_size'])
test_error = np.zeros(n_test_sets)
best_parameters = {}
test_iter = 0
computational_time = np.zeros(n_test_sets)
# Extra array to store dot products if estimator is nsim
almost_linearity_param = np.zeros(n_test_sets)
for idx_train, idx_test in kf.split(X):
start = time.time()
reg = GSCV(estimator=estim,
param_grid=param_grid,
scoring='neg_mean_squared_error',
iid=False,
cv=cv_folds,
verbose=0,
pre_dispatch=n_jobs,
error_score=np.nan,
refit=True) # If estimator fitting raises an exception
X_train, Y_train = X[idx_train, :], Y[idx_train]
X_test, Y_test = X[idx_test, :], Y[idx_test]
reg = reg.fit(X_train, Y_train)
Y_predict = reg.best_estimator_.predict(X_test)
end = time.time()
best_parameters[test_iter] = reg.best_params_
if arguments['estimator_kwargs']['estimator'] in [
'isotron', 'slisotron'
]:
best_parameters[test_iter] = reg.best_estimator_.n_iter_cv()
if log_tf:
test_error[test_iter] = np.sqrt(
mean_squared_error(np.exp(Y_test), np.exp(Y_predict)))
else:
test_error[test_iter] = np.sqrt(
mean_squared_error(Y_test, Y_predict))
computational_time[test_iter] = end - start
if arguments['estimator_kwargs']['estimator'] == 'nsim':
almost_linearity_param[
test_iter] = reg.best_estimator_.measure_almost_linearity()
test_iter += 1
print best_parameters
print test_error
# Save results
mean_error = np.mean(test_error)
std_error = np.std(test_error)
mean_computational_time = np.mean(computational_time)
mean_almost_linearity_param = np.mean(almost_linearity_param)
filename = arguments['filename']
filename_mod = filename
save_itr = 0
while os.path.exists('../results/' + filename_mod + '/'):
save_itr += 1
filename_mod = filename + '_' + str(save_itr)
else:
os.makedirs('../results/' + filename_mod + '/')
np.save('../results/' + filename_mod + '/test_errors.npy', test_error)
np.savetxt('../results/' + filename_mod + '/test_errors.txt',
test_error)
np.savetxt('../results/' + filename_mod + '/computational_time.txt',
computational_time)
np.savetxt(
'../results/' + filename_mod + '/computational_time_summary.txt',
[mean_computational_time])
np.savetxt('../results/' + filename_mod + '/test_errors_summary.txt',
np.array([mean_error, std_error]))
np.save('../results/' + filename_mod + '/best_params.npy',
best_parameters)
if arguments['estimator_kwargs']['estimator'] == 'nsim':
np.savetxt(
'../results/' + filename_mod + '/almost_linearity_param.txt',
almost_linearity_param)
np.savetxt(
'../results/' + filename_mod + '/almost_linearity_summary.txt',
[mean_almost_linearity_param])
with open('../results/' + filename_mod + '/best_params_json.txt',
'w') as file:
file.write(json.dumps(best_parameters, indent=4))
with open('../results/' + filename_mod + '/log.txt', 'w') as file:
file.write(json.dumps(arguments, indent=4))
transforms_image = transforms.Compose([
transforms.Resize((256, 256)),
transforms.ToTensor(),
transforms.Normalize((0., 0., 0.), (1., 1., 1.))
])
transforms_mask = transforms.Compose([
transforms.Resize((256, 256)),
transforms.ToTensor(),
transforms.Normalize((0., ), (1., ))
])
train_dataset, val_dataset = load_data_set(
config['image_paths'],
config['image_dir'],
config['segmentation_dir'],
transforms=[transforms_image, transforms_mask],
batch_size=config['batch_size'])
print("loaded", len(train_dataset), "batches")
model = UNet(3).to(config['device'])
optimiser = torch.optim.Adam(params=model.parameters(), lr=config['lr'])
if config['continue_train']:
state_dict = torch.load(config['checkpoint'])
optimiser_state = torch.load(config['optimiser'])
model.load_state_dict(state_dict)
optimiser.load_state_dict(optimiser_state)
loss_fn = torch.nn.BCEWithLogitsLoss()
self.n_h_neurons +
[self.n_o_neurons]) + "; Eta: " + str(self.h_eta +
[self.eta]) + ""
n_rounds = 20
epochs = 30
learning_rate = 0.257181
mom = 0.75
sucess_rate_sum = 0
highest_sucess_rate = 0
lowest_sucess_rate = 1
x, y = utils.load_data_set()
n_training_samples = int(np.floor(0.8 * len(x)))
n_tests = int(np.floor(0.2 * len(x)))
x_training = x[0:n_training_samples]
y_training = y[0:n_training_samples]
tests = [x[-n_tests:], y[-n_tests:]]
net = mlp(x_training, y_training)
for i in range(n_rounds):
net.config(epochs, learning_rate, mom, False, len(y_training[0]), 1, [10],
[0.15214523])
net.train(n_training_samples)
net.test_regression([tests[0][i] for i in range(0, n_tests)],
def load_tflite():
# Load the TFLite model and allocate tensors.
interpreter = tf.lite.Interpreter(model_path="pretrained.model.tflite")
interpreter.allocate_tensors()
# Get input and output tensors.
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
# Test the model on random input data.
input_shape = input_details[0]['shape']
print(" - Loading test data")
testing_files_path = "./dataset/test/raw"
testing_num_files = count_files_in_folder(testing_files_path)
TESTING_BATCH_SIZE = testing_num_files
raw_dataset = load_data_set()
test_raw_dataset = raw_dataset['test']
test = test_raw_dataset.map(load_image_test)
test_dataset = test.batch(TESTING_BATCH_SIZE)
for images, true_masks in test_dataset:
pass # Hack needed to be able to extrac images and true masks from map datasets
images = images.numpy()
true_masks = true_masks.numpy()
# We need to manipulate here
random_input = np.array(np.random.random_sample(input_shape),
dtype=np.float32)
input_data = images
num_images = images.shape[0]
predictions = []
for image_idx in range(0, num_images):
img = np.expand_dims(input_data[image_idx, :, :, :], axis=0)
interpreter.set_tensor(input_details[0]['index'], img)
interpreter.invoke()
# The function `get_tensor()` returns a copy of the tensor data.
# Use `tensor()` in order to get a pointer to the tensor.
output_data = interpreter.get_tensor(output_details[0]['index'])
predictions.append(output_data)
print(f" - Let's transform predictions into labeled values.")
labeled_predictions = []
for image_index in range(len(predictions)):
prediction = predictions[image_index]
prediction = np.squeeze(prediction)
predicted_mask = map_prediction_to_mask(prediction)
labeled_predictions.append(predicted_mask)
print(f" - Saving labeled images into ./output folder")
predicted_index = 0
output_path = "./output/"
if not os.path.exists(output_path):
os.makedirs(output_path)
for predicted_result in labeled_predictions:
prediction_as_image = labeled_prediction_to_image(predicted_result)
prediction_as_image.save(f"{output_path}{predicted_index:03d}.jpg")
prediction_as_image.close()
print(f" - Image with index {predicted_index} saved.")
predicted_index += 1
print(f" - Generating sample output page")
generate_output_template()