def init_pretrained(cp, dataset):
prefix = cp.get('Experiment', 'prefix')
num = cp.get('Experiment', 'num')
# Initialize theano tensors
log(dataset.shape)
input_var = theano.shared(name='input_var', value=np.asarray(dataset,
dtype=theano.config.floatX),
borrow=True)
index = T.lscalar()
# Reconstruct pretrained neural network
# Get activation functions from config file
enc_act = cp.get('NeuralNetwork', 'encoderactivation')
dec_act = cp.get('NeuralNetwork', 'decoderactivation')
relu = lasagne.nonlinearities.rectify
linear = lasagne.nonlinearities.linear
batch_size = int(cp.get('NeuralNetwork', 'batchsize'))
# Get learning decay from config file
lr_decay = int(cp.get('NeuralNetwork', 'lrepochdecay'))
# Stacking layers into network
# Input layer
input_layer = network = lasagne.layers.InputLayer(shape=(None, int(cp.get('NeuralNetwork', 'inputlayer'))),
input_var=input_var)
# Introduce noise with DropoutLayer
network = lasagne.layers.DropoutLayer(incoming=network,
p=float(cp.get('NeuralNetwork', 'corruptionfactor')))
# Encoding/Hidden layer
encoder_layer = network = lasagne.layers.DenseLayer(incoming=network,
num_units=int(
cp.get('NeuralNetwork', 'hiddenlayer')),
W=np.load(
prefix + '_' + num + '_W1.npy'),
b=np.load(
prefix + '_' + num + '_b1.npy'),
nonlinearity=relu if enc_act == 'ReLU' else linear)
# Decoding/Output layer
network = lasagne.layers.DenseLayer(incoming=network,
num_units=int(
cp.get('NeuralNetwork', 'outputlayer')),
W=np.load(prefix + '_' +
num + '_W2.npy'),
b=np.load(prefix + '_' +
num + '_b2.npy'),
nonlinearity=relu if dec_act == 'ReLU' else linear)
# Neural network has been reconstructed, yet hyperparameters are randomly
# initialized, load pretrained hyperparameters
lasagne.layers.set_all_param_values(
network, np.load(prefix + '_' + num + '_model.npy'))
# Initalize model template object that is used for saving/archiving purposes.
# model template is a custom class, see modeltemplate.py for more info
model = Model(input_layer=input_layer, encoder_layer=encoder_layer,
decoder_layer=network, network=network)
# Compressed pickle
model.save(prefix + '_model.zip')
# If the function was called for testing purposes, save hidden and output layer
# results.
input_layer.input_var = input_var
hidden = lasagne.layers.get_output(encoder_layer).eval()
np.save(prefix + '_' + num + '_pretrained_hidden.npy', hidden)
output = lasagne.layers.get_output(network).eval()
np.save(prefix + '_' + num + '_pretrained_output.npy', output)
def init_pretrained(cp, dataset, length):
relu = lasagne.nonlinearities.rectify
linear = lasagne.nonlinearities.linear
batch_size = int(cp[length].get('NeuralNetwork', 'batchsize'))
lr_decay = int(cp[length].get('NeuralNetwork', 'lrepochdecay'))
prefix = cp[length].get('Experiment', 'prefix')
log(dataset.shape)
# Reconstruct deep network
# Input layer
input_var = theano.shared(name='input_var',
value=np.asarray(dataset,
dtype=theano.config.floatX),
borrow=True)
index = T.lscalar()
input_layer = network = lasagne.layers.InputLayer(
shape=(None, int(cp[0].get('NeuralNetwork', 'inputlayer'))),
input_var=input_var)
# Layerwise encoders
for i in xrange(length):
enc_act = cp[i].get('NeuralNetwork', 'encoderactivation')
network = lasagne.layers.DenseLayer(
incoming=input_layer if i == 0 else network,
num_units=int(cp[i].get('NeuralNetwork', 'hiddenlayer')),
W=np.load(prefix + '_' + str(i) + '_W1.npy'),
b=np.load(prefix + '_' + str(i) + '_b1.npy'),
nonlinearity=relu if enc_act == 'ReLU' else linear)
encoder_layer = network
# Layerwise decoders
for i in reversed(xrange(length)):
dec_act = cp[i].get('NeuralNetwork', 'decoderactivation')
network = lasagne.layers.DenseLayer(
incoming=network,
num_units=int(cp[i].get('NeuralNetwork', 'outputlayer')),
W=np.load(prefix + '_' + str(i) + '_W2.npy'),
b=np.load(prefix + '_' + str(i) + '_b2.npy'),
nonlinearity=relu if dec_act == 'ReLU' else linear)
# Neural network has been reconstructed, yet hyperparameters are initialized
# with the layerwise hyperparameters, load pretrained deep hyperparameters
lasagne.layers.set_all_param_values(network,
np.load(prefix + '_sda_model.npy'))
# Initalize model template object that is used for saving/archiving purposes.
# model template is a custom class, see modeltemplate.py for more info
model = Model(input_layer=input_layer,
encoder_layer=encoder_layer,
decoder_layer=network,
network=network)
model.save(prefix + '_model.zip')
# If the function was called for testing purposes, save hidden and output layer
# results.
input_layer.input_var = input_var
hidden = lasagne.layers.get_output(encoder_layer).eval()
np.save(prefix + '_sda_pretrained_hidden.npy', hidden)
output = lasagne.layers.get_output(network).eval()
np.save(prefix + '_sda_pretrained_output.npy', output)
def init_pretrained(cp, dataset_test, batch):
output = cp.get('Experiment', 'output')
prefix = cp.get('Experiment', 'prefix')
# Create "Weather" neural network
[win_layer, win, weather_net] = init_weather_conv(cp, dataset_test)
# Create "Dispersion" neural network
[din_layer, din, disp_net] = init_disp_conv(cp, dataset_test)
batch_size = int(cp.get('NeuralNetwork', 'batchsize'))
index = T.lscalar()
lw_epochs = int(cp.get('NeuralNetwork', 'maxepochs'))
base_lr = float(cp.get('NeuralNetwork', 'learningrate'))
# ConcatLayer is used in order to connect the outputs of each network
concat = network = lasagne.layers.ConcatLayer(incomings=(weather_net,
disp_net),
axis=1)
# Introduce noise
network = lasagne.layers.DropoutLayer(incoming=network,
p=float(
cp.get('NeuralNetwork',
'corruptionfactor')))
# Fully connected #1, Shape: close to original
network = lasagne.layers.DenseLayer(
incoming=network,
num_units=int(cp.get('NeuralNetwork', 'hidden0')),
)
# Fully connected #2, Shape: compressing
network = lasagne.layers.DenseLayer(
incoming=network,
num_units=int(cp.get('NeuralNetwork', 'hidden1')),
)
# Fully connected #3, Shape: convert to original
network = lasagne.layers.DenseLayer(
incoming=network,
num_units=int(cp.get('NeuralNetwork', 'hidden2')),
)
# Softmax
network = lasagne.layers.DenseLayer(
incoming=network,
num_units=int(cp.get('NeuralNetwork', 'stationnum')),
nonlinearity=lasagne.nonlinearities.softmax)
log(lasagne.layers.get_output_shape(
lasagne.layers.get_all_layers(network)))
params = np.load(output + prefix + '_model.npy')
print params.shape
# Load pretrained parameters
lasagne.layers.set_all_param_values(network, params)
# Save model
model = Model(input_layer=[win_layer, din_layer],
encoder_layer=None,
decoder_layer=network,
network=network)
model.save(output + prefix + '_model.zip')
def make_template(layer_dict, aae):
return Model(layer_dict=layer_dict, aae=aae)
def init_pretrained(cp, dataset):
prefix = cp.get('Experiment', 'prefix')
print dataset.shape
input_var = theano.shared(name='input_var', value=np.asarray(dataset,
dtype=theano.config.floatX),
borrow=True)
index = T.lscalar()
conv_filters = int(cp.get('NeuralNetwork', 'convfilters'))
deconv_filters = conv_filters
filter_sizes = int(cp.get('NeuralNetwork', 'filtersize'))
featurex = int(cp.get('NeuralNetwork', 'feature_x'))
featurey = int(cp.get('NeuralNetwork', 'feature_y'))
channels = int(cp.get('NeuralNetwork', 'channels'))
pool_size = int(cp.get('NeuralNetwork', 'pool'))
# Reconstructing the convolutional autoencoder structure, more info above
input_layer = network = lasagne.layers.InputLayer(shape=(None, channels * featurex * featurey),
input_var=input_var)
network = lasagne.layers.ReshapeLayer(
incoming=network, shape=([0], channels, featurex, featurey))
network = lasagne.layers.Conv2DLayer(incoming=network,
num_filters=conv_filters, filter_size=(
filter_sizes, filter_sizes),
stride=int(cp.get('NeuralNetwork', 'stride')))
try:
dual_conv = int(cp.get('NeuralNetwork', 'dualconv'))
network = lasagne.layers.Conv2DLayer(incoming=network,
num_filters=conv_filters, filter_size=(
dual_conv, dual_conv),
stride=int(cp.get('NeuralNetwork', 'dualstride')))
except:
pass
network = lasagne.layers.MaxPool2DLayer(
incoming=network, pool_size=(pool_size, pool_size))
pool_shape = lasagne.layers.get_output_shape(network)
flatten = network = lasagne.layers.ReshapeLayer(
incoming=network, shape=([0], -1))
network = lasagne.layers.DropoutLayer(incoming=network,
p=float(cp.get('NeuralNetwork', 'corruptionfactor')))
network = lasagne.layers.DenseLayer(incoming=network,
num_units=int(
cp.get('NeuralNetwork', 'hidden0')),
)
network = lasagne.layers.DenseLayer(incoming=network,
num_units=int(
cp.get('NeuralNetwork', 'hidden1')),
)
network = lasagne.layers.DenseLayer(incoming=network,
num_units=int(
cp.get('NeuralNetwork', 'hidden2')),
)
encoder_layer = network = lasagne.layers.DenseLayer(incoming=network,
num_units=int(
cp.get('NeuralNetwork', 'hidden3')),
)
network = lasagne.layers.DenseLayer(incoming=network,
num_units=int(
cp.get('NeuralNetwork', 'dec3')),
)
network = lasagne.layers.DenseLayer(incoming=network,
num_units=int(
cp.get('NeuralNetwork', 'dec2')),
)
network = lasagne.layers.DenseLayer(incoming=network,
num_units=int(
cp.get('NeuralNetwork', 'dec1')),
)
network = lasagne.layers.DenseLayer(incoming=network,
num_units=lasagne.layers.get_output_shape(flatten)[
1],
)
network = lasagne.layers.ReshapeLayer(
incoming=network, shape=([0], deconv_filters, pool_shape[2], pool_shape[3]))
network = lasagne.layers.Upscale2DLayer(incoming=network, scale_factor=(pool_size, pool_size))
try:
dual_conv = int(cp.get('NeuralNetwork', 'dualconv'))
network = lasagne.layers.TransposedConv2DLayer(incoming=network,
num_filters=channels, filter_size=(dual_conv, dual_conv), stride=int(cp.get('NeuralNetwork', 'dualstride')), nonlinearity=None)
except:
pass
network = lasagne.layers.TransposedConv2DLayer(incoming=network,
num_filters=channels, filter_size=(filter_sizes, filter_sizes), stride=int(cp.get('NeuralNetwork', 'stride')), nonlinearity=None)
network = lasagne.layers.ReshapeLayer(
incoming=network, shape=([0], -1))
# Neural network has been reconstructed, yet hyperparameters are randomly
# initialized, load pretrained hyperparameters
lasagne.layers.set_all_param_values(
network, np.load(prefix + '_model.npy'))
# Initalize model template object that is used for saving/archiving purposes.
# model template is a custom class, see modeltemplate.py for more info
model = Model(input_layer=input_layer, encoder_layer=encoder_layer,
decoder_layer=network, network=network)
model.save(prefix+'_model.zip')
# If the function was called for testing purposes, save hidden and output layer
# results.
input_layer.input_var = input_var
hidden = lasagne.layers.get_output(encoder_layer).eval()
np.save(prefix + '_pretrained_hidden.npy', hidden)
output = lasagne.layers.get_output(network).eval()
np.save(prefix + '_pretrained_output.npy', output)