def pretrain_sda_sgd(sda, train_names, window_size, pretraining_epochs,
pretrain_lr, corruption_levels):
# compute number of examples given in training set
n_train_patients = len(train_names)
print '... getting the pretraining functions'
pretraining_fns = pretraining_functions_sda_sgd(sda=sda,
window_size=window_size)
print '... pre-training the model'
## Pre-train layer-wise
for i in xrange(sda.n_layers):
cur_dA = sda.dA_layers[i]
cur_dA.train_cost_array = []
cur_train_cost = []
train_reader = ICHISeqDataReader(train_names)
for patients in xrange(n_train_patients):
# go through the training set
train_set_x, train_set_y = train_reader.read_next_doc()
n_train_samples = train_set_x.get_value(borrow=True).shape[0] - window_size + 1
cur_train_cost.append([])
# go through pretraining epochs
for epoch in xrange(pretraining_epochs):
cur_epoch_cost=[]
for index in xrange(n_train_samples):
cur_epoch_cost.append(pretraining_fns[i](index=index,
train_set = train_set_x.get_value(borrow=True),
corruption=corruption_levels[i],
lr=pretrain_lr))
cur_train_cost[-1].append(numpy.mean(cur_epoch_cost))
gc.collect()
cur_dA.train_cost_array = [[epoch, cost] for epoch, cost in zip(xrange(pretraining_epochs), numpy.mean(cur_train_cost, axis=0))]
return sda
def pretrain_sda_cg(sda, train_names, window_size, pretraining_epochs, corruption_levels):
## Pre-train layer-wise
print "... getting the pretraining functions"
import scipy.optimize
train_reader = ICHISeqDataReader(train_names)
n_train_patients = len(train_names)
for patients in xrange(n_train_patients):
train_set_x, train_set_y = train_reader.read_next_doc()
pretraining_fn, pretraining_update = pretraining_functions_sda_cg(
sda=sda, train_set_x=train_set_x, window_size=window_size, corruption_levels=corruption_levels
)
print "... pre-training the model"
# using scipy conjugate gradient optimizer
print ("Optimizing using scipy.optimize.fmin_cg...")
for i in xrange(sda.n_layers):
best_w_b = scipy.optimize.fmin_cg(
f=partial(pretraining_fn, da_index=i),
x0=numpy.zeros(
(sda.dA_layers[i].n_visible + 1) * sda.dA_layers[i].n_hidden, dtype=sda.dA_layers[i].input.dtype
),
fprime=partial(pretraining_update, da_index=i),
maxiter=pretraining_epochs,
)
return sda
def test_sda(sda, test_names, base, window_size=1, algo='viterbi'):
test_reader = ICHISeqDataReader(test_names)
test_set_x, test_set_y = test_reader.read_all()
n_test_patients = len(test_names)
for test_patient in xrange(n_test_patients):
#get data divided on sequences with respect to labels
test_set_x, test_set_y = test_reader.read_next_doc()
test_x_array = test_set_x.get_value()
n_test_times = test_x_array.shape[0] - window_size + 1
test_visible_after_sda = numpy.array([sda.get_da_output(
test_x_array[time: time+window_size]).ravel()
for time in xrange(n_test_times)]).ravel()
new_test_visible, new_test_hidden = change_data_for_one_patient(
hiddens_patient=test_set_y.eval(),
visibles_patient=test_visible_after_sda,
window_size=sda.da_layers_output_size,
base_for_labels=base
)
patient_error = get_error_on_patient(
model=sda.hmmLayer,
visible_set=new_test_visible,
hidden_set=new_test_hidden,
algo=algo
)
print(patient_error, ' error for patient ' + str(test_patient))
gc.collect()
def test_sda(sda, test_names, rank, start_base, window_size=1, algo='viterbi'):
test_reader = ICHISeqDataReader(test_names)
test_set_x, test_set_y = test_reader.read_all()
n_test_patients = len(test_names)
for test_patient in xrange(n_test_patients):
#get data divided on sequences with respect to labels
test_set_x, test_set_y = test_reader.read_next_doc()
test_set_x = test_set_x.get_value()
test_set_y = test_set_y.eval()
n_test_times = test_set_x.shape[0] - window_size
test_visible_after_sda = numpy.array([sda.get_da_output(
test_set_x[time: time+window_size]).ravel()
for time in xrange(n_test_times)])
new_test_visible = create_labels_after_das(
da_output_matrix=test_visible_after_sda,
rank=rank,
start_base=start_base,
window_size=window_size
)
n_patient_samples = len(test_set_y)
half_window_size = int(window_size/2)
new_test_hidden=test_set_y[half_window_size:n_patient_samples-half_window_size]
patient_error = get_error_on_patient(
model=sda.hmmLayer,
visible_set=new_test_visible,
hidden_set=new_test_hidden,
algo=algo
)
print(patient_error, ' error for patient ' + str(test_patient))
gc.collect()
def train_SdA(datasets, train_names,
output_folder, base_folder,
window_size,
corruption_levels,
pretraining_epochs,
base,
pretrain_lr=0):
"""
Demonstrates how to train and test a stochastic denoising autoencoder.
This is demonstrated on ICHI.
:type pretraining_epochs: int
:param pretraining_epochs: number of epoch to do pretraining
:type n_iter: int
:param n_iter: maximal number of iterations ot run the optimizer
:type datasets: array
:param datasets: [train_set, valid_set, test_set]
:type output_folder: string
:param output_folder: folder for costand error graphics with results
"""
# split the datasets
(train_set_x, train_set_y) = datasets[0]
(valid_set_x, valid_set_y) = datasets[1]
(test_set_x, test_set_y) = datasets[2]
# compute number of examples given in training set
n_in = window_size*3 # number of input units
n_out = 7 # number of output units
# numpy random generator
# start-snippet-3
numpy_rng = numpy.random.RandomState(89677)
print '... building the model'
# construct the stacked denoising autoencoder class
sda = SdA(
numpy_rng=numpy_rng,
n_ins=n_in,
hidden_layers_sizes=[window_size*2, window_size],
n_outs=n_out
)
# end-snippet-3 start-snippet-4
#########################
# PRETRAINING THE MODEL #
#########################
start_time = timeit.default_timer()
pretrained_sda = pretrain_sda_sgd(sda=sda,
train_names=train_names,
window_size=window_size,
pretraining_epochs=pretraining_epochs,
pretrain_lr=pretrain_lr,
corruption_levels=corruption_levels)
'''
pretrained_sda = pretrain_sda_cg(sda=sda,
train_set_x=train_set_x,
window_size=window_size,
pretraining_epochs=pretraining_epochs,
corruption_levels=corruption_levels)
'''
end_time = timeit.default_timer()
for i in xrange(sda.n_layers):
print(i, 'i pretrained')
visualize_pretraining(train_cost=pretrained_sda.dA_layers[i].train_cost_array,
window_size=window_size,
learning_rate=0,
corruption_level=corruption_levels[i],
n_hidden=sda.dA_layers[i].n_hidden,
da_layer=i,
datasets_folder=output_folder,
base_folder=base_folder)
print >> sys.stderr, ('The pretraining code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))
# end-snippet-4
########################
# FINETUNING THE MODEL #
########################
#create matrices for params of HMM layer
train_data_names = ['p10a','p011','p013','p014','p020','p022','p040',
'p045','p048','p09b','p023','p035','p038', 'p09a','p033']
n_train_patients=len(train_data_names)
n_visible=pow(base, sda.da_layers_output_size)
n_hidden=n_out
train_reader = ICHISeqDataReader(train_data_names)
pi_values = numpy.zeros((n_hidden,))
a_values = numpy.zeros((n_hidden, n_hidden))
b_values = numpy.zeros((n_hidden, n_visible))
array_from_hidden = numpy.zeros((n_hidden,))
for train_patient in xrange(n_train_patients):
#get data divided on sequences with respect to labels
train_set_x, train_set_y = train_reader.read_next_doc()
train_x_array = train_set_x.get_value()
n_train_times = train_x_array.shape[0] - window_size + 1
train_visible_after_sda = numpy.array([sda.get_da_output(
train_x_array[time: time+window_size]).ravel()
for time in xrange(n_train_times)]).ravel()
new_train_visible, new_train_hidden = change_data_for_one_patient(
hiddens_patient=train_set_y.eval(),
visibles_patient=train_visible_after_sda,
window_size=sda.da_layers_output_size,
base_for_labels=base
)
pi_values, a_values, b_values, array_from_hidden = update_params_on_patient(
pi_values=pi_values,
a_values=a_values,
b_values=b_values,
array_from_hidden=array_from_hidden,
hiddens_patient=new_train_hidden,
visibles_patient=new_train_visible,
n_hidden=n_hidden
)
gc.collect()
pi_values, a_values, b_values = finish_training(
pi_values=pi_values,
a_values=a_values,
b_values=b_values,
array_from_hidden=array_from_hidden,
n_hidden=n_hidden,
n_patients=n_train_patients
)
hmm_model = hmm.MultinomialHMM(
n_components=n_hidden,
startprob=pi_values,
transmat=a_values
)
hmm_model.n_symbols=n_visible
hmm_model.emissionprob_=b_values
gc.collect()
print('MultinomialHMM created')
sda.set_hmm_layer(
hmm_model=hmm_model
)
return sda
