def train_SdA(train_names, valid_names,
output_folder, base_folder,
window_size,
corruption_levels,
pretraining_epochs,
start_base,
rank,
pretrain_lr):
"""
Demonstrates how to train and test a stochastic denoising autoencoder.
This is demonstrated on ICHI.
:type pretraining_epochs: int
:param pretraining_epochs: number of epoch to do pretraining
:type n_iter: int
:param n_iter: maximal number of iterations ot run the optimizer
:type datasets: array
:param datasets: [train_set, valid_set, test_set]
:type output_folder: string
:param output_folder: folder for costand error graphics with results
"""
# compute number of examples given in training set
n_in = window_size*3 # number of input units
n_out = 7 # number of output units
# numpy random generator
# start-snippet-3
numpy_rng = numpy.random.RandomState(89677)
print '... building the model'
# construct the stacked denoising autoencoder class
sda = SdA(
numpy_rng=numpy_rng,
n_ins=n_in,
hidden_layers_sizes=[window_size*2, window_size],
n_outs=n_out
)
# end-snippet-3 start-snippet-4
#########################
# PRETRAINING THE MODEL #
#########################
start_time = timeit.default_timer()
'''
pretrained_sda = pretrain_sda_sgd(sda=sda,
train_names=train_names,
window_size=window_size,
pretraining_epochs=pretraining_epochs,
pretrain_lr=pretrain_lr,
corruption_levels=corruption_levels)
'''
pretrained_sda = pretrain_sda_cg(sda=sda,
train_names=train_names,
window_size=window_size,
pretraining_epochs=pretraining_epochs,
corruption_levels=corruption_levels)
end_time = timeit.default_timer()
for i in xrange(sda.n_layers):
print(i, 'i pretrained')
visualize_pretraining(train_cost=pretrained_sda.dA_layers[i].train_cost_array,
window_size=window_size,
learning_rate=0,
corruption_level=corruption_levels[i],
n_hidden=sda.dA_layers[i].n_hidden,
da_layer=i,
datasets_folder=output_folder,
base_folder=base_folder)
print >> sys.stderr, ('The pretraining code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))
# end-snippet-4
########################
# FINETUNING THE MODEL #
########################
#create matrices for params of HMM layer
n_hiddens=[5]*n_out
#create hmm container
hmmLayer = GeneralHMM(
n_hiddens = n_hiddens,
n_hmms = n_out
)
#train_hmm
train_reader = ICHISeqDataReader(train_names)
n_train_patients = len(train_names)
#train hmms on data of each pattient
for train_patient in xrange(n_train_patients):
#get data divided on sequences with respect to labels
train_set = train_reader.read_one_with_window(
window_size=window_size,
divide=True
)
for i in xrange(hmmLayer.n_hmms):
cur_train_set = train_set[i].eval()
if cur_train_set.shape[0] <= 0:
continue
print('train_set[i].eval(): ', train_set[i].eval().shape)
#get (avg, disp) labels for x-values
train_visible_after_sda = numpy.array([sda.get_da_output(
numpy.array(cur_train_set[time]).reshape(1, -1))
for time in xrange(cur_train_set.shape[0])])
x_labels = create_labels_after_das(
da_output_matrix = train_visible_after_sda,
rank=rank,
start_base=start_base
)
hmmLayer.hmm_models[i].fit([numpy.array(x_labels).reshape(-1, 1)])
error_cur_epoch = hmmLayer.validate_model(
valid_names = valid_names,
window_size = window_size,
rank = rank,
start_base = start_base
)
hmmLayer.valid_error_array.append([])
hmmLayer.valid_error_array[-1].append(train_patient)
hmmLayer.valid_error_array[-1].append(error_cur_epoch)
gc.collect()
gc.collect()
print('MultinomialHMM created')
sda.set_hmm_layer(
hmm_model=hmmLayer
)
return sda
def test_SdA(datasets,
output_folder, base_folder,
window_size,
pretraining_epochs=15,
training_epochs=1000):
"""
Demonstrates how to train and test a stochastic denoising autoencoder.
This is demonstrated on ICHI.
:type pretraining_epochs: int
:param pretraining_epochs: number of epoch to do pretraining
:type n_iter: int
:param n_iter: maximal number of iterations ot run the optimizer
:type datasets: array
:param datasets: [train_set, valid_set, test_set]
:type output_folder: string
:param output_folder: folder for costand error graphics with results
"""
# split the datasets
(train_set_x, train_set_y) = datasets[0]
(valid_set_x, valid_set_y) = datasets[1]
(test_set_x, test_set_y) = datasets[2]
# compute number of examples given in training set
n_in = window_size*3 # number of input units
n_out = 7 # number of output units
x = T.matrix('x')
# numpy random generator
# start-snippet-3
numpy_rng = numpy.random.RandomState(89677)
print '... building the model'
# construct the stacked denoising autoencoder class
sda = SdA(
numpy_rng=numpy_rng,
n_ins=n_in,
hidden_layers_sizes=[window_size*2, window_size*2],
n_outs=n_out
)
# end-snippet-3 start-snippet-4
#########################
# PRETRAINING THE MODEL #
#########################
'''
print '... getting the pretraining functions'
pretraining_fns, pretraining_updates = sda.pretraining_functions(train_set_x=train_set_x,
window_size=window_size)
'''
print '... pre-training the model'
# using scipy conjugate gradient optimizer
import scipy.optimize
print ("Optimizing using scipy.optimize.fmin_cg...")
start_time = timeit.default_timer()
index = T.lscalar('index')
n_train_samples = train_set_x.get_value(borrow=True).shape[0] - window_size + 1
## Pre-train layer-wise
corruption_levels = [.1, .2]
for da_index in xrange(sda.n_layers):
cur_dA=sda.dA_layers[da_index]
# get the cost and the updates list
cost = cur_dA.get_cost(corruption_levels[da_index])
# compile a theano function that returns the cost
sample_cost = theano.function(
inputs=[index],
outputs=cost,
givens={
sda.x: train_set_x[index: index + window_size]
},
on_unused_input='warn'
)
# compile a theano function that returns the gradient with respect to theta
sample_grad = theano.function(
inputs=[index],
outputs=T.grad(cost, cur_dA.theta),
givens={
sda.x: train_set_x[index: index + window_size]
},
on_unused_input='warn'
)
def train_fn(theta_value):
sda.dA_layers[da_index].theta.set_value(theta_value, borrow=True)
train_losses = [sample_cost(i)
for i in xrange(n_train_samples)]
this_train_loss = float(numpy.mean(train_losses))
sda.dA_layers[da_index].train_cost_array.append([])
sda.dA_layers[da_index].train_cost_array[-1].append(sda.dA_layers[da_index].epoch)
sda.dA_layers[da_index].train_cost_array[-1].append(this_train_loss)
sda.dA_layers[da_index].epoch += 1
return numpy.mean(train_losses)
def train_fn_grad(theta_value):
sda.dA_layers[da_index].theta.set_value(theta_value, borrow=True)
grad = sample_grad(0)
for i in xrange(1, n_train_samples):
grad += sample_grad(i)
return grad / n_train_samples
best_w_b = scipy.optimize.fmin_cg(
f=train_fn,
x0=numpy.zeros((sda.dA_layers[da_index].n_visible + 1) * sda.dA_layers[da_index].n_hidden,
dtype=x.dtype),
fprime=train_fn_grad,
disp=0,
maxiter=pretraining_epochs
)
visualize_pretraining(train_cost=sda.dA_layers[da_index].train_cost_array,
window_size=window_size,
learning_rate=0,
corruption_level=corruption_levels[da_index],
n_hidden=sda.dA_layers[da_index].n_hidden,
da_layer=da_index,
datasets_folder=output_folder,
base_folder=base_folder)
end_time = timeit.default_timer()
print >> sys.stderr, ('The pretraining code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))
# end-snippet-4
########################
# FINETUNING THE MODEL #
########################
# get the training, validation and testing functions for the model
print '... getting the finetuning functions'
train_fn, train_fn_grad, callback = sda.build_finetune_functions(
datasets=datasets,
window_size=window_size
)
print '... finetunning the model'
import scipy.optimize
print ("Optimizing using scipy.optimize.fmin_cg...")
start_time = timeit.default_timer()
best_w_b = scipy.optimize.fmin_cg(
f=train_fn,
x0=numpy.zeros((n_in + 1) * n_out, dtype=sda.x.dtype),
fprime=train_fn_grad,
callback=callback,
disp=0,
maxiter=training_epochs
)
print(
(
'Optimization complete with best validation score of %f %%, with '
'test performance %f %%'
)
% (sda.validation_scores[0] * 100., sda.validation_scores[1] * 100.)
)
visualize_finetuning(train_cost=sda.logLayer.train_cost_array,
train_error=sda.logLayer.train_error_array,
valid_error=sda.logLayer.valid_error_array,
test_error=sda.logLayer.test_error_array,
window_size=window_size,
learning_rate=0,
datasets_folder=output_folder,
base_folder=base_folder)
print >> sys.stderr, ('The code for file ' +
os.path.split(__file__)[1] +
' ran for %.1fs' % ((end_time - start_time)))
def train_SdA(datasets, train_names,
output_folder, base_folder,
window_size,
corruption_levels,
pretraining_epochs,
base,
pretrain_lr=0):
"""
Demonstrates how to train and test a stochastic denoising autoencoder.
This is demonstrated on ICHI.
:type pretraining_epochs: int
:param pretraining_epochs: number of epoch to do pretraining
:type n_iter: int
:param n_iter: maximal number of iterations ot run the optimizer
:type datasets: array
:param datasets: [train_set, valid_set, test_set]
:type output_folder: string
:param output_folder: folder for costand error graphics with results
"""
# split the datasets
(train_set_x, train_set_y) = datasets[0]
(valid_set_x, valid_set_y) = datasets[1]
(test_set_x, test_set_y) = datasets[2]
# compute number of examples given in training set
n_in = window_size*3 # number of input units
n_out = 7 # number of output units
# numpy random generator
# start-snippet-3
numpy_rng = numpy.random.RandomState(89677)
print '... building the model'
# construct the stacked denoising autoencoder class
sda = SdA(
numpy_rng=numpy_rng,
n_ins=n_in,
hidden_layers_sizes=[window_size*2, window_size],
n_outs=n_out
)
# end-snippet-3 start-snippet-4
#########################
# PRETRAINING THE MODEL #
#########################
start_time = timeit.default_timer()
pretrained_sda = pretrain_sda_sgd(sda=sda,
train_names=train_names,
window_size=window_size,
pretraining_epochs=pretraining_epochs,
pretrain_lr=pretrain_lr,
corruption_levels=corruption_levels)
'''
pretrained_sda = pretrain_sda_cg(sda=sda,
train_set_x=train_set_x,
window_size=window_size,
pretraining_epochs=pretraining_epochs,
corruption_levels=corruption_levels)
'''
end_time = timeit.default_timer()
for i in xrange(sda.n_layers):
print(i, 'i pretrained')
visualize_pretraining(train_cost=pretrained_sda.dA_layers[i].train_cost_array,
window_size=window_size,
learning_rate=0,
corruption_level=corruption_levels[i],
n_hidden=sda.dA_layers[i].n_hidden,
da_layer=i,
datasets_folder=output_folder,
base_folder=base_folder)
print >> sys.stderr, ('The pretraining code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))
# end-snippet-4
########################
# FINETUNING THE MODEL #
########################
#create matrices for params of HMM layer
train_data_names = ['p10a','p011','p013','p014','p020','p022','p040',
'p045','p048','p09b','p023','p035','p038', 'p09a','p033']
n_train_patients=len(train_data_names)
n_visible=pow(base, sda.da_layers_output_size)
n_hidden=n_out
train_reader = ICHISeqDataReader(train_data_names)
pi_values = numpy.zeros((n_hidden,))
a_values = numpy.zeros((n_hidden, n_hidden))
b_values = numpy.zeros((n_hidden, n_visible))
array_from_hidden = numpy.zeros((n_hidden,))
for train_patient in xrange(n_train_patients):
#get data divided on sequences with respect to labels
train_set_x, train_set_y = train_reader.read_next_doc()
train_x_array = train_set_x.get_value()
n_train_times = train_x_array.shape[0] - window_size + 1
train_visible_after_sda = numpy.array([sda.get_da_output(
train_x_array[time: time+window_size]).ravel()
for time in xrange(n_train_times)]).ravel()
new_train_visible, new_train_hidden = change_data_for_one_patient(
hiddens_patient=train_set_y.eval(),
visibles_patient=train_visible_after_sda,
window_size=sda.da_layers_output_size,
base_for_labels=base
)
pi_values, a_values, b_values, array_from_hidden = update_params_on_patient(
pi_values=pi_values,
a_values=a_values,
b_values=b_values,
array_from_hidden=array_from_hidden,
hiddens_patient=new_train_hidden,
visibles_patient=new_train_visible,
n_hidden=n_hidden
)
gc.collect()
pi_values, a_values, b_values = finish_training(
pi_values=pi_values,
a_values=a_values,
b_values=b_values,
array_from_hidden=array_from_hidden,
n_hidden=n_hidden,
n_patients=n_train_patients
)
hmm_model = hmm.MultinomialHMM(
n_components=n_hidden,
startprob=pi_values,
transmat=a_values
)
hmm_model.n_symbols=n_visible
hmm_model.emissionprob_=b_values
gc.collect()
print('MultinomialHMM created')
sda.set_hmm_layer(
hmm_model=hmm_model
)
return sda
def test_SdA(
datasets,
output_folder,
base_folder,
window_size,
corruption_levels,
pretraining_epochs,
training_epochs,
pretrain_lr=0,
finetune_lr=0,
):
"""
Demonstrates how to train and test a stochastic denoising autoencoder.
This is demonstrated on ICHI.
:type pretraining_epochs: int
:param pretraining_epochs: number of epoch to do pretraining
:type n_iter: int
:param n_iter: maximal number of iterations ot run the optimizer
:type datasets: array
:param datasets: [train_set, valid_set, test_set]
:type output_folder: string
:param output_folder: folder for costand error graphics with results
"""
# split the datasets
(train_set_x, train_set_y) = datasets[0]
(valid_set_x, valid_set_y) = datasets[1]
(test_set_x, test_set_y) = datasets[2]
# compute number of examples given in training set
n_in = window_size * 3 # number of input units
n_out = 7 # number of output units
# numpy random generator
# start-snippet-3
numpy_rng = numpy.random.RandomState(89677)
print "... building the model"
# construct the stacked denoising autoencoder class
sda = SdA(numpy_rng=numpy_rng, n_ins=n_in, hidden_layers_sizes=[window_size * 2, window_size], n_outs=n_out)
# end-snippet-3 start-snippet-4
#########################
# PRETRAINING THE MODEL #
#########################
start_time = timeit.default_timer()
pretrained_sda = pretrain_sda_sgd(
sda=sda,
train_set_x=train_set_x,
window_size=window_size,
pretraining_epochs=pretraining_epochs,
pretrain_lr=pretrain_lr,
corruption_levels=corruption_levels,
)
"""
pretrained_sda = pretrain_sda_cg(sda=sda,
train_set_x=train_set_x,
window_size=window_size,
pretraining_epochs=pretraining_epochs,
corruption_levels=corruption_levels)
"""
end_time = timeit.default_timer()
for i in xrange(sda.n_layers):
print (i, "i pretrained")
visualize_pretraining(
train_cost=pretrained_sda.dA_layers[i].train_cost_array,
window_size=window_size,
learning_rate=0,
corruption_level=corruption_levels[i],
n_hidden=sda.dA_layers[i].n_hidden,
da_layer=i,
datasets_folder=output_folder,
base_folder=base_folder,
)
print >> sys.stderr, (
"The pretraining code for file "
+ os.path.split(__file__)[1]
+ " ran for %.2fm" % ((end_time - start_time) / 60.0)
)
# end-snippet-4
########################
# FINETUNING THE MODEL #
########################
start_time = timeit.default_timer()
"""
finetuned_sda = finetune_sda_sgd(sda=pretrained_sda,
datasets=datasets,
window_size=window_size,
finetune_lr=finetune_lr,
training_epochs=training_epochs)
"""
finetuned_sda = finetune_sda_cg(
sda=pretrained_sda, datasets=datasets, window_size=window_size, training_epochs=training_epochs
)
end_time = timeit.default_timer()
visualize_finetuning(
train_cost=finetuned_sda.logLayer.train_cost_array,
train_error=finetuned_sda.logLayer.train_error_array,
valid_error=finetuned_sda.logLayer.valid_error_array,
test_error=finetuned_sda.logLayer.test_error_array,
window_size=window_size,
learning_rate=0,
datasets_folder=output_folder,
base_folder=base_folder,
)
