def autoencoder():
log = Log(experiment_name = 'AE',
description = 'This experiment is about autoencoder',
save_outputs = True,
save_hyperparams = True,
save_model = True,
send_to_database = 'Database_Name.db')
learning_rule = LearningRule(max_col_norm = None,
learning_rate = 0.01,
momentum = 0.1,
momentum_type = 'normal',
L1_lambda = None,
L2_lambda = None,
cost = Cost(type='mse'),
stopping_criteria = {'max_epoch' : 100,
'cost' : Cost(type='mse'),
'epoch_look_back' : 10,
'percent_decrease' : 0.001}
)
# building dataset, change to P276 to train on P276 dataset
data = Mnist(train_valid_test_ratio=[5,1,1])
# for AutoEncoder, the inputs and outputs must be the same
train = data.get_train()
data.set_train(train.X, train.X)
valid = data.get_valid()
data.set_valid(valid.X, valid.X)
test = data.get_test()
data.set_test(test.X, test.X)
# building autoencoder
ae = AutoEncoder(input_dim = data.feature_size(), rand_seed=None)
h1_layer = RELU(dim=100, name='h1_layer', W=None, b=None)
# adding encoding layer
ae.add_encode_layer(h1_layer)
# adding decoding mirror layer
ae.add_decode_layer(Sigmoid(dim=data.target_size(), name='output_layer', W=h1_layer.W.T, b=None))
train_object = TrainObject(model = ae,
dataset = data,
learning_rule = learning_rule,
log = log)
train_object.run()
def build_dataset(self, part=None):
dataset = None
preprocessor = None if self.state.dataset.preprocessor is None else \
getattr(preproc, self.state.dataset.preprocessor)()
if self.state.dataset.type == 'Mnist':
dataset = Mnist(train_valid_test_ratio = self.state.dataset.train_valid_test_ratio,
preprocessor = preprocessor,
batch_size = self.state.dataset.batch_size,
num_batches = self.state.dataset.num_batches,
iter_class = self.state.dataset.iter_class,
rng = self.state.dataset.rng)
elif self.state.dataset.type[:4] == 'P276':
dataset = getattr(spec, self.state.dataset.type)(
train_valid_test_ratio = self.state.dataset.train_valid_test_ratio,
preprocessor = preprocessor,
batch_size = self.state.dataset.batch_size,
num_batches = self.state.dataset.num_batches,
iter_class = self.state.dataset.iter_class,
rng = self.state.dataset.rng)
elif self.state.dataset.type[:5] == 'Laura':
assert part is not None, 'split name is required for Laura dataset'
dataset = getattr(spec, self.state.dataset.type)(
part = part,
train_valid_test_ratio = self.state.dataset.train_valid_test_ratio,
preprocessor = preprocessor,
batch_size = self.state.dataset.batch_size,
num_batches = self.state.dataset.num_batches,
iter_class = self.state.dataset.iter_class,
rng = self.state.dataset.rng)
train = dataset.get_train()
dataset.set_train(train.X, train.X)
valid = dataset.get_valid()
dataset.set_valid(valid.X, valid.X)
test = dataset.get_test()
dataset.set_test(test.X, test.X)
return dataset
def stacked_autoencoder():
name = 'Stacked_AE'
#=====[ Train First layer of stack autoencoder ]=====#
print('Start training First Layer of AutoEncoder')
log = Log(experiment_name = name + '_layer1',
description = 'This experiment is to test the model',
save_outputs = True,
save_hyperparams = True,
save_model = True,
send_to_database = 'Database_Name.db')
learning_rule = LearningRule(max_col_norm = None,
learning_rate = 0.01,
momentum = 0.1,
momentum_type = 'normal',
L1_lambda = None,
L2_lambda = None,
cost = Cost(type='mse'),
stopping_criteria = {'max_epoch' : 3,
'epoch_look_back' : 1,
'cost' : Cost(type='mse'),
'percent_decrease' : 0.001}
)
data = Mnist()
train = data.get_train()
data.set_train(train.X, train.X)
valid = data.get_valid()
data.set_valid(valid.X, valid.X)
test = data.get_test()
data.set_test(test.X, test.X)
ae = AutoEncoder(input_dim = data.feature_size(), rand_seed=None)
h1_layer = RELU(dim=500, name='h1_layer', W=None, b=None)
ae.add_encode_layer(h1_layer)
h1_mirror = RELU(dim = data.target_size(), name='h1_mirror', W=h1_layer.W.T, b=None)
ae.add_decode_layer(h1_mirror)
train_object = TrainObject(model = ae,
dataset = data,
learning_rule = learning_rule,
log = log)
train_object.run()
#=====[ Train Second Layer of autoencoder ]=====#
print('Start training Second Layer of AutoEncoder')
log2 = Log(experiment_name = name + '_layer2',
description = 'This experiment is to test the model',
save_outputs = True,
save_hyperparams = True,
save_model = True,
send_to_database = 'Database_Name.db')
learning_rule = LearningRule(max_col_norm = None,
learning_rate = 0.01,
momentum = 0.1,
momentum_type = 'normal',
L1_lambda = None,
L2_lambda = None,
cost = Cost(type='mse'),
stopping_criteria = {'max_epoch' : 3,
'epoch_look_back' : 1,
'cost' : Cost(type='mse'),
'percent_decrease' : 0.001}
)
# fprop == forward propagation
reduced_train_X = ae.encode(train.X)
reduced_valid_X = ae.encode(valid.X)
reduced_test_X = ae.encode(test.X)
data.set_train(X=reduced_train_X, y=reduced_train_X)
data.set_valid(X=reduced_valid_X, y=reduced_valid_X)
data.set_test(X=reduced_test_X, y=reduced_test_X)
# create a new mlp taking inputs from the encoded outputs of first autoencoder
ae2 = AutoEncoder(input_dim = data.feature_size(), rand_seed=None)
h2_layer = RELU(dim=100, name='h2_layer', W=None, b=None)
ae2.add_encode_layer(h2_layer)
h2_mirror = RELU(dim=h1_layer.dim, name='h2_mirror', W=h2_layer.W.T, b=None)
ae2.add_decode_layer(h2_mirror)
train_object = TrainObject(model = ae2,
dataset = data,
learning_rule = learning_rule,
log = log2)
train_object.run()
#=====[ Fine Tuning ]=====#
print('Fine Tuning')
log3 = Log(experiment_name = name + '_full',
description = 'This experiment is to test the model',
save_outputs = True,
save_hyperparams = True,
save_model = True,
send_to_database = 'Database_Name.db')
data = Mnist()
train = data.get_train()
data.set_train(train.X, train.X)
valid = data.get_valid()
data.set_valid(valid.X, valid.X)
test = data.get_test()
data.set_test(test.X, test.X)
ae3 = AutoEncoder(input_dim = data.feature_size(), rand_seed=None)
ae3.add_encode_layer(h1_layer)
ae3.add_encode_layer(h2_layer)
ae3.add_decode_layer(h2_mirror)
ae3.add_decode_layer(h1_mirror)
train_object = TrainObject(model = ae3,
dataset = data,
learning_rule = learning_rule,
log = log3)
train_object.run()
print('Training Done')