def autoencoder():
# building dataset, batch_size and preprocessor
data = Mnist(train_valid_test_ratio=[8, 1, 1], batch_size=100, preprocessor=GCN())
# for AutoEncoder, the inputs and outputs must be the same
train = data.get_train()
data.set_train(train.X, train.X)
valid = data.get_valid()
data.set_valid(valid.X, valid.X)
test = data.get_test()
data.set_test(test.X, test.X)
# building autoencoder
ae = AutoEncoder(input_dim=data.feature_size(), rand_seed=123)
h1_layer = Tanh(dim=500, name="h1_layer", W=None, b=None)
# adding encoding layer
ae.add_encode_layer(h1_layer)
# mirror layer has W = h1_layer.W.T
h1_mirror = Tanh(dim=ae.input_dim, name="h1_mirror", W=h1_layer.W.T, b=None)
# adding decoding mirror layer
ae.add_decode_layer(h1_mirror)
# build learning method
learning_method = AdaGrad(learning_rate=0.1, momentum=0.9)
# set the learning rules
learning_rule = LearningRule(
max_col_norm=10,
L1_lambda=None,
L2_lambda=None,
training_cost=Cost(type="mse"),
learning_rate_decay_factor=None,
stopping_criteria={
"max_epoch": 300,
"epoch_look_back": 10,
"cost": Cost(type="error"),
"percent_decrease": 0.01,
},
)
# put all the components into a TrainObject
train_object = TrainObject(model=ae, dataset=data, learning_rule=learning_rule, learning_method=learning_method)
# finally run the training
train_object.run()
import theano
from pynet.model import AutoEncoder
import pynet.layer as layers
floatX = theano.config.floatX
parser = argparse.ArgumentParser(description='''Convert gpu pickle pynet model to cpu pickle pynet model''')
parser.add_argument('--gpu_model', metavar='Path', required=True, help='the path to the gpu model pickle file')
parser.add_argument('--cpu_model', metavar='Path', required=True, help='''path to save the cpu model pickle file''')
args = parser.parse_args()
print ('loading gpu autoencoder..')
fin = open(args.gpu_model)
gpu_model = cPickle.load(fin)
ae = AutoEncoder(input_dim=gpu_model.input_dim)
for layer in gpu_model.encode_layers:
layerW = T._shared(np.array(layer.W.get_value(), floatX),
name=layer.W.name, borrow=False)
layerb = T._shared(np.array(layer.b.get_value(), floatX),
name=layer.b.name, borrow=False)
encode_layer = getattr(layers, layer.__class__.__name__)(dim=layer.dim, name=layer.name,
W=layerW, b=layerb)
ae.add_encode_layer(encode_layer)
print 'encode layer', encode_layer.name, encode_layer.dim
print 'encode layers', ae.encode_layers
for ae_layer, gpu_layer in zip(reversed(ae.encode_layers), gpu_model.decode_layers):
gpu_decode_layer_b = T._shared(np.array(gpu_layer.b.get_value(), floatX),
name=gpu_layer.b.name, borrow=False)
decode_layer = getattr(layers, gpu_layer.__class__.__name__)(name=gpu_layer.name, dim=gpu_layer.dim,
description='''Convert gpu pickle pynet model to cpu pickle pynet model''')
parser.add_argument('--gpu_model',
metavar='Path',
required=True,
help='the path to the gpu model pickle file')
parser.add_argument('--cpu_model',
metavar='Path',
required=True,
help='''path to save the cpu model pickle file''')
args = parser.parse_args()
print('loading gpu autoencoder..')
fin = open(args.gpu_model)
gpu_model = cPickle.load(fin)
ae = AutoEncoder(input_dim=gpu_model.input_dim)
for layer in gpu_model.encode_layers:
layerW = T._shared(np.array(layer.W.get_value(), floatX),
name=layer.W.name,
borrow=False)
layerb = T._shared(np.array(layer.b.get_value(), floatX),
name=layer.b.name,
borrow=False)
encode_layer = getattr(layers, layer.__class__.__name__)(dim=layer.dim,
name=layer.name,
W=layerW,
b=layerb)
ae.add_encode_layer(encode_layer)
print 'encode layer', encode_layer.name, encode_layer.dim
print 'encode layers', ae.encode_layers
def stacked_autoencoder():
name = "Stacked_AE"
# =====[ Train First layer of stack autoencoder ]=====#
print("Start training First Layer of AutoEncoder")
# building dataset, batch_size and preprocessor
data = Mnist(train_valid_test_ratio=[8, 1, 1], batch_size=100)
# for AutoEncoder, the inputs and outputs must be the same
train = data.get_train()
data.set_train(train.X, train.X)
valid = data.get_valid()
data.set_valid(valid.X, valid.X)
test = data.get_test()
data.set_test(test.X, test.X)
# building autoencoder
ae = AutoEncoder(input_dim=data.feature_size(), rand_seed=123)
h1_layer = RELU(dim=500, name="h1_layer", W=None, b=None)
# adding encoding layer
ae.add_encode_layer(h1_layer)
# mirror layer has W = h1_layer.W.T
h1_mirror = RELU(dim=ae.input_dim, name="h1_mirror", W=h1_layer.W.T, b=None)
# adding decoding mirror layer
ae.add_decode_layer(h1_mirror)
# build learning method
learning_method = SGD(learning_rate=0.001, momentum=0.9)
# set the learning rules
learning_rule = LearningRule(
max_col_norm=10,
L1_lambda=None,
L2_lambda=None,
training_cost=Cost(type="mse"),
learning_rate_decay_factor=None,
stopping_criteria={"max_epoch": 3, "epoch_look_back": 1, "cost": Cost(type="error"), "percent_decrease": 0.01},
)
# put all the components into a TrainObject
train_object = TrainObject(model=ae, dataset=data, learning_rule=learning_rule, learning_method=learning_method)
# finally run the training
train_object.run()
# =====[ Train Second Layer of autoencoder ]=====#
print("Start training Second Layer of AutoEncoder")
# fprop == forward propagation
reduced_train_X = ae.encode(train.X)
reduced_valid_X = ae.encode(valid.X)
reduced_test_X = ae.encode(test.X)
data.set_train(X=reduced_train_X, y=reduced_train_X)
data.set_valid(X=reduced_valid_X, y=reduced_valid_X)
data.set_test(X=reduced_test_X, y=reduced_test_X)
# create a new mlp taking inputs from the encoded outputs of first autoencoder
ae2 = AutoEncoder(input_dim=data.feature_size(), rand_seed=None)
h2_layer = RELU(dim=100, name="h2_layer", W=None, b=None)
ae2.add_encode_layer(h2_layer)
h2_mirror = RELU(dim=h1_layer.dim, name="h2_mirror", W=h2_layer.W.T, b=None)
ae2.add_decode_layer(h2_mirror)
train_object = TrainObject(model=ae2, dataset=data, learning_rule=learning_rule, learning_method=learning_method)
train_object.run()
# =====[ Fine Tuning ]=====#
print("Fine Tuning")
data = Mnist()
train = data.get_train()
data.set_train(train.X, train.X)
valid = data.get_valid()
data.set_valid(valid.X, valid.X)
test = data.get_test()
data.set_test(test.X, test.X)
ae3 = AutoEncoder(input_dim=data.feature_size(), rand_seed=None)
ae3.add_encode_layer(h1_layer)
ae3.add_encode_layer(h2_layer)
ae3.add_decode_layer(h2_mirror)
ae3.add_decode_layer(h1_mirror)
train_object = TrainObject(model=ae3, dataset=data, learning_rule=learning_rule, learning_method=learning_method)
train_object.run()
print("Training Done")