def VFAE_training(source_data,
target_data,
n_train_batches,
n_epochs,
struct,
coef,
description,
process_display=True):
#########################################################
### Data ###
#########################################################
# must input data like [
# [[trainimg, label], [img, label], [img, label]],
# [[validationimg, label], [img, label], [img, label]],
# [[testimg, label], [img, label], [img, label]]
# ]
train_ftd_source, train_labeld_source = source_data[0]
valid_ftd_source, valid_labeld_source = source_data[1]
test_ftd_source, test_labeld_source = source_data[2]
train_ftd_target, train_labeld_target = target_data[0]
valid_ftd_target, valid_labeld_target = target_data[1]
test_ftd_target, test_labeld_target = target_data[2]
train_ftd_source, train_labeld_source = util.shared_dataset(
(train_ftd_source, train_labeld_source))
valid_ftd_source, valid_labeld_source = util.shared_dataset(
(valid_ftd_source, valid_labeld_source))
test_ftd_source, test_labeld_source = util.shared_dataset(
(test_ftd_source, test_labeld_source))
train_ftd_target, train_labeld_target = util.shared_dataset(
(train_ftd_target, train_labeld_target))
valid_ftd_target, valid_labeld_target = util.shared_dataset(
(valid_ftd_target, valid_labeld_target))
test_ftd_target, test_labeld_target = util.shared_dataset(
(test_ftd_target, test_labeld_target))
batch_size_S = train_ftd_source.get_value(
borrow=True).shape[0] // n_train_batches
batch_size_T = train_ftd_target.get_value(
borrow=True).shape[0] // n_train_batches
validate_S_size = valid_ftd_source.get_value(borrow=True).shape[0]
validate_T_size = valid_ftd_target.get_value(borrow=True).shape[0]
test_S_size = test_ftd_source.get_value(borrow=True).shape[0]
test_T_size = test_ftd_target.get_value(borrow=True).shape[0]
print(
'number of minibatch at one epoch: %i, batch size source : %i, target : %i \n validation size, S:%i, T:%i, test size, S:%i, T:%i'
% (n_train_batches, batch_size_S, batch_size_T, validate_S_size,
validate_T_size, test_S_size, test_T_size))
#######################################################################
### BUILD ACTUAL MODEL ###
#######################################################################
print('... building the model')
# allocate symbolic variables for the data
#index_source = T.lscalar() # index to a [mini]batch
#index_target = T.lscalar() # index to a [mini]batch
index = T.lscalar() # index to a [mini]batch
x_source = T.matrix(
'x_source') # the data is presented as rasterized images
y_source = T.matrix(
'y_source') # the labels are presented as signal vector
x_target = T.matrix(
'x_target') # the data is presented as rasterized images
y_target = T.matrix(
'y_target') # the labels are presented as signal vector
rng = np.random.RandomState(1234)
# construct the VFAE class
classifier = VFAE(rng=rng,
input_source=x_source,
input_target=x_target,
label_source=y_source,
batch_size=[batch_size_S, batch_size_T],
struct=struct,
coef=coef,
train=True)
validate_classifier = VFAE(rng=rng,
input_source=x_source,
input_target=x_target,
label_source=y_source,
batch_size=[validate_S_size, validate_T_size],
struct=struct,
coef=coef,
init_params=classifier.params_symbol())
test_classifier = VFAE(rng=rng,
input_source=x_source,
input_target=x_target,
label_source=y_source,
batch_size=[test_S_size, test_T_size],
struct=struct,
coef=coef,
init_params=classifier.params_symbol())
#update function
updates = classifier.updates
test_model = theano.function(inputs=[],
outputs=[
test_classifier.cost,
test_classifier.source_errors(y_source),
test_classifier.target_errors(y_target),
test_classifier.source_predict(),
test_classifier.target_predict()
],
givens={
x_source: test_ftd_source,
y_source: test_labeld_source,
x_target: test_ftd_target,
y_target: test_labeld_target
})
validate_model = theano.function(
inputs=[],
outputs=[
validate_classifier.cost,
validate_classifier.source_errors(y_source),
validate_classifier.target_errors(y_target),
validate_classifier.source_predict_raw(),
validate_classifier.target_predict_raw()
],
givens={
x_source: valid_ftd_source,
y_source: valid_labeld_source,
x_target: valid_ftd_target,
y_target: valid_labeld_target
})
validate_bytraindata_model = theano.function(
inputs=[index],
outputs=[
classifier.cost,
classifier.source_errors(y_source),
classifier.target_errors(y_target),
classifier.source_predict(),
classifier.target_predict()
],
givens={
x_source:
train_ftd_source[index * batch_size_S:(index + 1) *
batch_size_S, :],
y_source:
train_labeld_source[index * batch_size_S:(index + 1) *
batch_size_S, :],
x_target:
train_ftd_target[index * batch_size_T:(index + 1) *
batch_size_T, :],
y_target:
train_labeld_target[index * batch_size_T:(index + 1) *
batch_size_T, :]
})
train_model = theano.function(
inputs=[index],
outputs=[
classifier.cost,
classifier.source_errors(y_source),
classifier.target_errors(y_target),
classifier.source_predict(),
classifier.target_predict()
],
updates=updates,
givens={
x_source:
train_ftd_source[index * batch_size_S:(index + 1) *
batch_size_S, :],
y_source:
train_labeld_source[index * batch_size_S:(index + 1) *
batch_size_S, :],
x_target:
train_ftd_target[index * batch_size_T:(index + 1) *
batch_size_T, :],
y_target:
train_labeld_target[index * batch_size_T:(index + 1) *
batch_size_T, :]
})
################################################################
### TRAIN MODEL ###
################################################################
'''
Define :
xx_loss : Cost function value
xx_score : Classification accuracy rate
'''
print('... training')
# early-stopping parameters
patience = 10000 # look as this many examples regardless
patience_increase = 2 # wait this much longer when a new best is
# found
improvement_threshold = 0.995 # a relative improvement of this much is
# considered significant
validation_frequency = min(1, patience // 2)
# go through this many
# minibatche before checking the network
# on the validation set; in this case we
# check every epoch
validation_frequency = n_train_batches
best_iter = 0
best_train_loss = np.inf
best_validation_loss = np.inf
test_loss = np.inf
train_score = 0.
validation_score = 0.
test_score = 0.
start_time = timeit.default_timer()
epoch = 0
iter = 0
done_looping = False
train_losses_record = []
validate_losses_record = []
test_losses = test_model()[1]
test_score_S = 1 - np.mean(test_losses)
test_losses = test_model()[2]
test_score_T = 1 - np.mean(test_losses)
print(
('Initial, test accuracy: source domain :%f %%, target domain %f %%') %
(test_score_S * 100., test_score_T * 100.))
while (epoch < n_epochs) and (not done_looping):
epoch = epoch + 1
for minibatch_index in range(n_train_batches):
minibatch_avg_cost = train_model(minibatch_index)[0]
# iteration number
iter = (epoch - 1) * n_train_batches + minibatch_index
if (iter + 1) % validation_frequency == 0:
# compute loss on all training set
train_losses = [
validate_bytraindata_model(i)[0]
for i in range(n_train_batches)
]
this_train_loss = np.mean(train_losses)
# compute loss on validation set
this_validation_loss = validate_model()[0]
if (iter + 1) % 5 == 0 and process_display:
print(
'epoch %i, minibatch %i/%i, training loss %f, validation loss %f '
% (epoch, minibatch_index + 1, n_train_batches,
this_train_loss, this_validation_loss))
total_train_losses = [
validate_bytraindata_model(i)[0]
for i in range(n_train_batches)
]
total_train_losses = np.mean(total_train_losses)
train_losses_record.append(total_train_losses)
validate_losses_record.append(this_validation_loss)
# if we got the best validation score until now
if this_validation_loss < best_validation_loss:
#improve patience if loss improvement is good enough
if (this_validation_loss <
best_validation_loss * improvement_threshold):
patience = max(patience, iter * patience_increase)
train_loss = this_train_loss
best_validation_loss = this_validation_loss
best_iter = iter
#Get Accuracy
train_losses = [
validate_bytraindata_model(i)[1]
for i in range(n_train_batches)
]
train_score_S = 1 - np.mean(train_losses)
train_losses = [
validate_bytraindata_model(i)[2]
for i in range(n_train_batches)
]
train_score_T = 1 - np.mean(train_losses)
validation_losses = validate_model()[1]
validation_score_S = 1 - np.mean(validation_losses)
validation_losses = validate_model()[2]
validation_score_T = 1 - np.mean(validation_losses)
# test it on the test set
test_losses = test_model()[1]
test_score_S = 1 - np.mean(test_losses)
test_losses = test_model()[2]
test_score_T = 1 - np.mean(test_losses)
trained_params_name = classifier.params_name()
trained_params_value = classifier.params_value()
if process_display:
print((
' epoch %i, minibatch %i/%i, test accuracy of '
'best model: source domain :%f %%, target domain %f %%'
) % (epoch, minibatch_index + 1, n_train_batches,
test_score_S * 100., test_score_T * 100.))
if patience <= iter:
done_looping = True
break
end_time = timeit.default_timer()
if process_display:
print((
'Optimization complete. Best validation loss of %f '
'obtained at iteration %i, with train loss %f \n'
'train accuracy : source domain %f %%, target domain %f %%\n'
'validation accuracy : source domain %f %%, target domain %f %%\n'
'test accuracy : source domain %f %%, target domain %f %%') %
(best_validation_loss, best_iter + 1, train_loss,
train_score_S * 100., train_score_T * 100.,
validation_score_S * 100., validation_score_T * 100.,
test_score_S * 100., test_score_T * 100.))
print(
'-------------------------------------------------------------------------'
)
#Converge curve
index = range(len(train_losses_record))
title = 'Converge_Curve_%s' % (description)
fts = (index, train_losses_record, index, validate_losses_record)
label = ('train loss', 'validation loss')
color = [1, 2]
marker = [0, 0]
line = True
legend = True
util.data2plot(title=title,
fts=fts,
label=label,
color=color,
marker=marker,
line=line,
legend=legend,
plot_enable=process_display)
print(
'-------------------------------------------------------------------------'
)
trained_param = VFAE_params()
trained_param.update_value(trained_params_name, trained_params_value,
struct)
num_S = train_ftd_source.get_value(borrow=True).shape[0]
num_T = train_ftd_target.get_value(borrow=True).shape[0]
feature_classifier = VFAE(rng=rng,
input_source=x_source,
input_target=x_target,
label_source=y_source,
batch_size=[num_S, num_T],
struct=struct,
coef=coef,
init_params=trained_param)
features_model = theano.function(
inputs=[],
outputs=feature_classifier.feature_outputs() + [
feature_classifier.source_predict(),
feature_classifier.target_predict()
] + [
feature_classifier.source_reconstruct(),
feature_classifier.target_reconstruct()
],
givens={
x_source: train_ftd_source,
x_target: train_ftd_target
})
return features_model, test_model, trained_param
def main():
# setup output directory
d = datetime.datetime.today()
output_folder = "out/{}-{}-{}_{}:{}:{}".format(d.year, d.month, d.day,
d.hour, d.minute, d.second)
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
os.chdir(output_folder)
# load dataset
datasets = load_data()
train_set_x, train_set_y = util.shared_dataset(datasets[0])
valid_set_x, valid_set_y = util.shared_dataset(datasets[1])
test_set_x, test_set_y = util.shared_dataset(datasets[2])
train_set = (train_set_x, train_set_y)
valid_set = (valid_set_x, valid_set_y)
test_set = (test_set_x, test_set_y)
n_input = train_set_x.get_value(borrow=True).shape[1]
n_output = train_set_y.get_value(borrow=True).shape[1]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0]
n_train_batches /= batch_size
# numpy random generator
# start-snippet-3
numpy_rng = numpy.random.RandomState(89677)
print '... building the model'
# construct the stacked denoising autoencoder class
sda = SdA(numpy_rng=numpy_rng,
n_ins=n_input,
hidden_layers_sizes=[1000, 1000, 1000],
n_outs=n_output)
predict_fn = sda.build_predict_function()
#########################
# PRETRAINING THE MODEL #
#########################
print '... getting the pretraining functions'
pretraining_fns = sda.pretraining_functions(train_set_x=train_set_x,
batch_size=batch_size)
print '... pre-training the model'
start_time = time.clock()
## Pre-train layer-wise
corruption_levels = [.1, .2, .3]
for i in xrange(sda.n_layers):
# go through pretraining epochs
for epoch in xrange(pretraining_epochs):
# go through the training set
c = []
for batch_index in xrange(n_train_batches):
c.append(pretraining_fns[i](index=batch_index,
corruption=corruption_levels[i],
lr=pretrain_lr))
print("Pre-training layer {}, epoch {}, cost ".format(i, epoch)),
print("{}".format(numpy.mean(c)))
end_time = time.clock()
print >> sys.stderr, ('The pretraining code for file ' +
os.path.split(__file__)[1] + ' ran for %.2fm' %
((end_time - start_time) / 60.))
########################
# FINETUNING THE MODEL #
########################
# get the training, validation and testing function for the model
print '... getting the finetuning functions'
train_fn, validate_model, test_model = sda.build_finetune_functions(
datasets=(train_set, valid_set, test_set),
batch_size=batch_size,
learning_rate=finetune_lr)
print '... finetunning the model'
# early-stopping parameters
patience = 10 * n_train_batches # look as this many examples regardless
patience_increase = 2. # wait this much longer when a new best is
# found
improvement_threshold = 0.995 # a relative improvement of this much is
# considered significant
validation_frequency = min(n_train_batches, patience / 2)
# go through this many
# minibatche before checking the network
# on the validation set; in this case we
# check every epoch
best_validation_loss = numpy.inf
test_score = 0.
start_time = time.clock()
done_looping = False
epoch = 0
while (epoch < training_epochs) and (not done_looping):
epoch = epoch + 1
for minibatch_index in xrange(n_train_batches):
minibatch_avg_cost = train_fn(minibatch_index)
iter = (epoch - 1) * n_train_batches + minibatch_index
if (iter + 1) % validation_frequency == 0:
validation_losses = validate_model()
this_validation_loss = numpy.mean(validation_losses)
print('epoch %i, minibatch %i/%i, validation error %f %%' %
(epoch, minibatch_index + 1, n_train_batches,
this_validation_loss * 100.))
# if we got the best validation score until now
if this_validation_loss < best_validation_loss:
#improve patience if loss improvement is good enough
if (this_validation_loss <
best_validation_loss * improvement_threshold):
patience = max(patience, iter * patience_increase)
# save best validation score and iteration number
best_validation_loss = this_validation_loss
best_iter = iter
# test it on the test set
test_losses = test_model()
test_score = numpy.mean(test_losses)
print((' epoch %i, minibatch %i/%i, test error of '
'best model %f %%') %
(epoch, minibatch_index + 1, n_train_batches,
test_score * 100.))
if patience <= iter:
done_looping = True
break
end_time = time.clock()
print(('Optimization complete with best validation score of %f %%, '
'on iteration %i, '
'with test performance %f %%') %
(best_validation_loss * 100., best_iter + 1, test_score * 100.))
print >> sys.stderr, ('The training code for file ' +
os.path.split(__file__)[1] + ' ran for %.2fm' %
((end_time - start_time) / 60.))
###########
# PREDICT #
###########
y_pred = predict_fn(test_set_x.get_value(borrow=True))
mae, mre = util.calculate_error_indexes(test_set_y, y_pred)
print("-*-*RESULT*-*-")
print("mae={}".format(mae))
print("mre={}".format(mre))
# plot
for i in xrange(n_output):
filename = "{}.png".format(str(i))
plot.savefig(filename, test_set_x, y_pred, indexes=[i])
def main():
# setup output directory
d = datetime.datetime.today()
output_folder = "out/{}-{}-{}_{}:{}:{}".format(d.year, d.month, d.day, d.hour, d.minute, d.second)
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
os.chdir(output_folder)
# load dataset
datasets = load_data()
train_set_x, train_set_y = util.shared_dataset(datasets[0])
valid_set_x, valid_set_y = util.shared_dataset(datasets[1])
test_set_x, test_set_y = util.shared_dataset(datasets[2])
train_set = (train_set_x, train_set_y)
valid_set = (valid_set_x, valid_set_y)
test_set = (test_set_x, test_set_y)
n_input = train_set_x.get_value(borrow=True).shape[1]
n_output = train_set_y.get_value(borrow=True).shape[1]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0]
n_train_batches /= batch_size
# numpy random generator
# start-snippet-3
numpy_rng = numpy.random.RandomState(89677)
print '... building the model'
# construct the stacked denoising autoencoder class
sda = SdA(
numpy_rng=numpy_rng,
n_ins=n_input,
hidden_layers_sizes=[1000, 1000, 1000],
n_outs=n_output
)
predict_fn = sda.build_predict_function()
#########################
# PRETRAINING THE MODEL #
#########################
print '... getting the pretraining functions'
pretraining_fns = sda.pretraining_functions(train_set_x=train_set_x, batch_size=batch_size)
print '... pre-training the model'
start_time = time.clock()
## Pre-train layer-wise
corruption_levels = [.1, .2, .3]
for i in xrange(sda.n_layers):
# go through pretraining epochs
for epoch in xrange(pretraining_epochs):
# go through the training set
c = []
for batch_index in xrange(n_train_batches):
c.append(pretraining_fns[i](index=batch_index,
corruption=corruption_levels[i],
lr=pretrain_lr))
print("Pre-training layer {}, epoch {}, cost ".format(i, epoch)),
print("{}".format(numpy.mean(c)))
end_time = time.clock()
print >> sys.stderr, ('The pretraining code for file ' + os.path.split(__file__)[1] + ' ran for %.2fm' % ((end_time - start_time) / 60.))
########################
# FINETUNING THE MODEL #
########################
# get the training, validation and testing function for the model
print '... getting the finetuning functions'
train_fn, validate_model, test_model = sda.build_finetune_functions(
datasets=(train_set, valid_set, test_set),
batch_size=batch_size,
learning_rate=finetune_lr
)
print '... finetunning the model'
# early-stopping parameters
patience = 10 * n_train_batches # look as this many examples regardless
patience_increase = 2. # wait this much longer when a new best is
# found
improvement_threshold = 0.995 # a relative improvement of this much is
# considered significant
validation_frequency = min(n_train_batches, patience / 2)
# go through this many
# minibatche before checking the network
# on the validation set; in this case we
# check every epoch
best_validation_loss = numpy.inf
test_score = 0.
start_time = time.clock()
done_looping = False
epoch = 0
while (epoch < training_epochs) and (not done_looping):
epoch = epoch + 1
for minibatch_index in xrange(n_train_batches):
minibatch_avg_cost = train_fn(minibatch_index)
iter = (epoch - 1) * n_train_batches + minibatch_index
if (iter + 1) % validation_frequency == 0:
validation_losses = validate_model()
this_validation_loss = numpy.mean(validation_losses)
print('epoch %i, minibatch %i/%i, validation error %f %%' %
(epoch, minibatch_index + 1, n_train_batches,
this_validation_loss * 100.))
# if we got the best validation score until now
if this_validation_loss < best_validation_loss:
#improve patience if loss improvement is good enough
if (
this_validation_loss < best_validation_loss *
improvement_threshold
):
patience = max(patience, iter * patience_increase)
# save best validation score and iteration number
best_validation_loss = this_validation_loss
best_iter = iter
# test it on the test set
test_losses = test_model()
test_score = numpy.mean(test_losses)
print((' epoch %i, minibatch %i/%i, test error of '
'best model %f %%') %
(epoch, minibatch_index + 1, n_train_batches,
test_score * 100.))
if patience <= iter:
done_looping = True
break
end_time = time.clock()
print(
(
'Optimization complete with best validation score of %f %%, '
'on iteration %i, '
'with test performance %f %%'
)
% (best_validation_loss * 100., best_iter + 1, test_score * 100.)
)
print >> sys.stderr, ('The training code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))
###########
# PREDICT #
###########
y_pred = predict_fn(test_set_x.get_value(borrow=True))
mae, mre = util.calculate_error_indexes(test_set_y, y_pred)
print("-*-*RESULT*-*-")
print("mae={}".format(mae))
print("mre={}".format(mre))
# plot
for i in xrange(n_output):
filename = "{}.png".format(str(i))
plot.savefig(filename, test_set_x, y_pred, indexes=[i])
def object_reconition_test(s):
"""
Demonstrate stochastic gradient descent optimization for a multilayer
perceptron
This is demonstrated on MNIST.
:type learning_rate: float
:param learning_rate: learning rate used (factor for the stochastic
gradient
:type L1_reg: float
:param L1_reg: L1-norm's weight when added to the cost (see
regularization)
:type L2_reg: float
:param L2_reg: L2-norm's weight when added to the cost (see
regularization)
:type n_epochs: int
:param n_epochs: maximal number of epochs to run the optimizer
:type dataset: string
:param dataset: the path of the MNIST dataset file from
http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz
"""
if s == 0 :
print('Semi-Supervised Learning')
else :
print('Supervised Learning')
'''Load Data'''
source_file = '/home/cwhuang/Dataset/Office_Object/dslr_SURF_L10.npy'
target_file = '/home/cwhuang/Dataset/Office_Object/webcam_SURF_L10.npy'
source_data = np.load(source_file)
target_data = np.load(target_file)
train_ftd_source, train_labeld_source = source_data[0]
valid_ftd_source, valid_labeld_source = source_data[1]
test_ftd_source, test_labeld_source = source_data[2]
train_ftd_target, train_labeld_target = target_data[0]
valid_ftd_target, valid_labeld_target = target_data[1]
test_ftd_target, test_labeld_target = target_data[2]
#Make Source & Target size same by discard part data
if train_ftd_source.shape[0] > train_ftd_target.shape[0]:
train_ftd_source = train_ftd_source[0:train_ftd_target.shape[0], :]
train_labeld_source = train_labeld_source[0:train_labeld_target.shape[0], :]
valid_ftd_source = valid_ftd_source[0:valid_ftd_target.shape[0], :]
valid_labeld_source = valid_labeld_source[0:valid_labeld_target.shape[0], :]
test_ftd_source = test_ftd_source[0:test_ftd_target.shape[0], :]
test_labeld_source = test_labeld_source[0:test_labeld_target.shape[0], :]
elif train_ftd_source.shape[0] < train_ftd_target.shape[0]:
train_ftd_target = train_ftd_target[0:train_ftd_source.shape[0], :]
train_labeld_target = train_labeld_target[0:train_labeld_source.shape[0], :]
valid_ftd_target = valid_ftd_target[0:valid_ftd_source.shape[0], :]
valid_labeld_target = valid_labeld_target[0:valid_labeld_source.shape[0], :]
test_ftd_target = test_ftd_target[0:test_ftd_source.shape[0], :]
test_labeld_target = test_labeld_target[0:test_labeld_source.shape[0], :]
train_ftd_source, train_labeld_source = util.shared_dataset((train_ftd_source, train_labeld_source))
valid_ftd_source, valid_labeld_source = util.shared_dataset((valid_ftd_source, valid_labeld_source))
test_ftd_source, test_labeld_source = util.shared_dataset((test_ftd_source, test_labeld_source))
train_ftd_target, train_labeld_target = util.shared_dataset((train_ftd_target, train_labeld_target))
valid_ftd_target, valid_labeld_target = util.shared_dataset((valid_ftd_target, valid_labeld_target))
test_ftd_target, test_labeld_target = util.shared_dataset((test_ftd_target, test_labeld_target))
'''
n_train_source_batches = train_ftd_source.shape[0] // batch_size
n_valid_source_batches = train_ftd_source.shape[0] // batch_size
n_test_source_batches = train_ftd_source.shape[0] // batch_size
n_train_target_batches = train_ftd_target.shape[0] // batch_size
n_valid_target_batches = train_ftd_target.shape[0] // batch_size
n_test_target_batches = train_ftd_target.shape[0] // batch_size
'''
'''
print(train_ftd_source.get_value(borrow=True).shape[0])
print(train_ftd_target.get_value(borrow=True).shape[0])
print(valid_ftd_source.get_value(borrow=True).shape[0])
print(valid_ftd_target.get_value(borrow=True).shape[0])
print(test_ftd_source.get_value(borrow=True).shape[0])
print(test_ftd_target.get_value(borrow=True).shape[0])
print(train_labeld_source.get_value(borrow=True).shape[0])
print(train_labeld_target.get_value(borrow=True).shape[0])
print(valid_labeld_source.get_value(borrow=True).shape[0])
print(valid_labeld_target.get_value(borrow=True).shape[0])
print(test_labeld_source.get_value(borrow=True).shape[0])
print(test_labeld_target.get_value(borrow=True).shape[0])
print(train_ftd_source.get_value(borrow=True).shape[1])
print(train_ftd_target.get_value(borrow=True).shape[1])
print(valid_ftd_source.get_value(borrow=True).shape[1])
print(valid_ftd_target.get_value(borrow=True).shape[1])
print(test_ftd_source.get_value(borrow=True).shape[1])
print(test_ftd_target.get_value(borrow=True).shape[1])
print(train_labeld_source.get_value(borrow=True).shape[1])
print(train_labeld_target.get_value(borrow=True).shape[1])
print(valid_labeld_source.get_value(borrow=True).shape[1])
print(valid_labeld_target.get_value(borrow=True).shape[1])
print(test_labeld_source.get_value(borrow=True).shape[1])
print(test_labeld_target.get_value(borrow=True).shape[1])
'''
'''Coefficient Initial'''
batch_size = 14
epsilon_std = 0.01
n_epochs = 50
learning_rate = 0.0001
D = 800
alpha = 10 # Weight of classification error
beta = 100 # Weight of MMD penalty
n_train_batches = train_ftd_source.get_value(borrow=True).shape[0] // batch_size
n_valid_batches = valid_ftd_source.get_value(borrow=True).shape[0] // batch_size
n_test_batches = test_ftd_source.get_value(borrow=True).shape[0] // batch_size
print(
'number of minibatch at one epoch: train %i, validation %i, test %i' %
(n_train_batches, n_valid_batches, n_test_batches)
)
z_dim = 100 #dimension of latent feature
a_dim = 50 #dimension of prior of latent feature
x_dim = train_ftd_source.get_value(borrow=True).shape[1]
y_dim = train_labeld_target.get_value(borrow=True).shape[1]
d_dim = 2
activation = None
encoder1_struct=nn.NN_struct()
encoder1_struct.layer_dim = [x_dim+d_dim, z_dim]
encoder1_struct.activation = [activation]
encoder2_struct=nn.NN_struct()
encoder2_struct.layer_dim = [z_dim+y_dim, a_dim]
encoder2_struct.activation = [activation]
encoder3_struct=nn.NN_struct()
encoder3_struct.layer_dim = [z_dim, y_dim]
encoder3_struct.activation = [T.nnet.softmax]
decoder1_struct=nn.NN_struct()
decoder1_struct.layer_dim = [z_dim+d_dim, x_dim]
decoder1_struct.activation = [activation]
decoder2_struct=nn.NN_struct()
decoder2_struct.layer_dim = [a_dim+y_dim, z_dim]
decoder2_struct.activation = [activation]
######################
# BUILD ACTUAL MODEL #
######################
print('... building the model')
# allocate symbolic variables for the data
#index_source = T.lscalar() # index to a [mini]batch
#index_target = T.lscalar() # index to a [mini]batch
index = T.lscalar() # index to a [mini]batch
x_source = T.matrix('x_source') # the data is presented as rasterized images
y_source = T.matrix('y_source') # the labels are presented as signal vector
x_target = T.matrix('x_target') # the data is presented as rasterized images
y_target = T.matrix('y_target') # the labels are presented as signal vector
rng = np.random.RandomState(1234)
# construct the DAVAE class
if s == 0 :
classifier = VFAE.VFAE(
rng=rng,
input_source = x_source,
input_target = x_target,
label_source = y_source,
batch_size = batch_size,
encoder1_struct = encoder1_struct,
encoder2_struct = encoder2_struct,
encoder3_struct = encoder3_struct,
decoder1_struct = decoder1_struct,
decoder2_struct = decoder2_struct,
alpha = alpha,
beta = beta,
D = D
)
else :
classifier = VFAE.Supervised_VFAE(
rng=rng,
input_source = x_source,
input_target = x_target,
label_source = y_source,
label_target = y_target,
batch_size = batch_size,
encoder1_struct = encoder1_struct,
encoder2_struct = encoder2_struct,
encoder3_struct = encoder3_struct,
decoder1_struct = decoder1_struct,
decoder2_struct = decoder2_struct,
alpha = alpha,
beta = beta,
D = D
)
cost = (classifier.cost)
gparams = [T.grad(cost, param) for param in classifier.params]
updates = [
(param, param - learning_rate * gparam)
for param, gparam in zip(classifier.params, gparams)
]
Output_test_model = theano.function(
inputs=[index],
outputs=classifier.params+classifier.outputs+gparams,
givens={
x_source: train_ftd_source[index * batch_size : (index + 1) * batch_size, :],
y_source: train_labeld_source[index * batch_size : (index + 1) * batch_size, :],
x_target: train_ftd_target[index * batch_size : (index + 1) * batch_size, :]
#y_target: train_labeld_target[index * batch_size : (index + 1) * batch_size, :]
}
)
test_model = theano.function(
inputs=[index],
outputs=[classifier.cost, classifier.source_errors(y_source), classifier.target_errors(y_target),
classifier.source_predict_raw(), classifier.target_predict_raw()],
givens={
x_source: test_ftd_source[index * batch_size : (index + 1) * batch_size, :],
y_source: test_labeld_source[index * batch_size : (index + 1) * batch_size, :],
x_target: test_ftd_target[index * batch_size : (index + 1) * batch_size, :],
y_target: test_labeld_target[index * batch_size : (index + 1) * batch_size, :]
}
)
validate_model = theano.function(
inputs=[index],
outputs=[classifier.cost, classifier.source_errors(y_source), classifier.target_errors(y_target),
classifier.source_predict_raw(), classifier.target_predict_raw()],
givens={
x_source: valid_ftd_source[index * batch_size : (index + 1) * batch_size, :],
y_source: valid_labeld_source[index * batch_size : (index + 1) * batch_size, :],
x_target: valid_ftd_target[index * batch_size : (index + 1) * batch_size, :],
y_target: valid_labeld_target[index * batch_size : (index + 1) * batch_size, :]
}
)
validate_bytraindata_model = theano.function(
inputs=[index],
outputs=[classifier.cost, classifier.source_errors(y_source), classifier.target_errors(y_target),
classifier.source_predict_raw(), classifier.target_predict_raw()],
givens={
x_source: train_ftd_source[index * batch_size : (index + 1) * batch_size, :],
y_source: train_labeld_source[index * batch_size : (index + 1) * batch_size, :],
x_target: train_ftd_target[index * batch_size : (index + 1) * batch_size, :],
y_target: train_labeld_target[index * batch_size : (index + 1) * batch_size, :]
}
)
train_model = theano.function(
inputs=[index],
outputs=[classifier.cost, classifier.source_errors(y_source), classifier.target_errors(y_target),
classifier.source_predict_raw(), classifier.target_predict_raw()],
updates=updates,
givens={
x_source: train_ftd_source[index * batch_size : (index + 1) * batch_size, :],
y_source: train_labeld_source[index * batch_size : (index + 1) * batch_size, :],
x_target: train_ftd_target[index * batch_size : (index + 1) * batch_size, :],
y_target: train_labeld_target[index * batch_size : (index + 1) * batch_size, :]
}
)
###############
# TRAIN MODEL #
###############
'''
Define :
xx_loss : Cost function value
xx_score : Classification accuracy rate
'''
print('... training')
# early-stopping parameters
patience = 10000 # look as this many examples regardless
patience_increase = 2 # wait this much longer when a new best is
# found
improvement_threshold = 0.995 # a relative improvement of this much is
# considered significant
validation_frequency = min(n_valid_batches, patience // 2)
# go through this many
# minibatche before checking the network
# on the validation set; in this case we
# check every epoch
validation_frequency = n_train_batches
best_iter = 0
best_train_loss = np.inf
best_validation_loss = np.inf
test_loss = np.inf
train_score = 0.
validation_score = 0.
test_score = 0.
start_time = timeit.default_timer()
epoch = 0
done_looping = False
while (epoch < n_epochs) and (not done_looping):
epoch = epoch + 1
for minibatch_index in range(n_train_batches):
minibatch_avg_cost = train_model(minibatch_index)[0]
# iteration number
iter = (epoch - 1) * n_train_batches + minibatch_index
if (iter + 1) % validation_frequency == 0:
# compute loss on all training set
train_losses = [validate_bytraindata_model(i)[0] for i in range(n_train_batches)]
this_train_loss = np.mean(train_losses)
# compute loss on validation set
validation_losses = [validate_model(i)[0] for i in range(n_valid_batches)]
this_validation_loss = np.mean(validation_losses)
print(
'epoch %i, minibatch %i/%i, training loss %f, validation loss %f ' %
(
epoch,
minibatch_index + 1,
n_train_batches,
this_train_loss,
this_validation_loss
)
)
# if we got the best validation score until now
if this_validation_loss < best_validation_loss:
#improve patience if loss improvement is good enough
if (
this_validation_loss < best_validation_loss *
improvement_threshold
):
patience = max(patience, iter * patience_increase)
train_loss = this_train_loss
best_validation_loss = this_validation_loss
best_iter = iter
#Get Accuracy
train_losses = [validate_bytraindata_model(i)[1]for i in range(n_train_batches)]
train_score_S = 1 - np.mean(train_losses)
train_losses = [validate_bytraindata_model(i)[2]for i in range(n_train_batches)]
train_score_T = 1 - np.mean(train_losses)
validation_losses = [validate_model(i)[1] for i in range(n_valid_batches)]
validation_score_S = 1 - np.mean(validation_losses)
validation_losses = [validate_model(i)[2] for i in range(n_valid_batches)]
validation_score_T = 1 - np.mean(validation_losses)
# test it on the test set
test_losses = [test_model(i)[1]for i in range(n_test_batches)]
test_score_S = 1 - np.mean(test_losses)
test_losses = [test_model(i)[2]for i in range(n_test_batches)]
test_score_T = 1 - np.mean(test_losses)
print((' epoch %i, minibatch %i/%i, test accuracy of '
'best model: source domain :%f %%, target domain %f %%') %
(epoch, minibatch_index + 1, n_train_batches,
test_score_S * 100., test_score_T * 100.))
if patience <= iter:
done_looping = True
break
end_time = timeit.default_timer()
print(('Optimization complete. Best validation loss of %f '
'obtained at iteration %i, with train loss %f \n'
'train accuracy : source domain %f %%, target domain %f %%\n'
'validation accuracy : source domain %f %%, target domain %f %%\n'
'test accuracy : source domain %f %%, target domain %f %%') %
(best_validation_loss, best_iter + 1, train_loss, train_score_S * 100., train_score_T * 100.,
validation_score_S * 100., validation_score_T * 100., test_score_S * 100., test_score_T * 100.))
'''
print(('The code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.)), file=sys.stderr)
'''
#Return Trained Parameter
'''Model Construct'''
def test_SdA(state_file=None, output_folder=None):
# load data
datasets = load_data(r=2, d=1)
train_set_x, train_set_y = util.shared_dataset(datasets[0])
valid_set_x, valid_set_y = util.shared_dataset(datasets[1])
test_set_x, test_set_y = util.shared_dataset(datasets[2])
train_set = (train_set_x, train_set_y)
valid_set = (valid_set_x, valid_set_y)
test_set = (test_set_x, test_set_y)
n_input = train_set_x.get_value(borrow=True).shape[1]
n_output = train_set_y.get_value(borrow=True).shape[1]
# prepare output folder
if output_folder is None:
d = datetime.datetime.today()
output_folder = "out/{0:04d}{1:02d}{2:02d}_{3:02d}{4:02d}{5:02d}".format(
d.year, d.month, d.day, d.hour, d.minute, d.second)
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
# instantiate TestBed
if state_file is None:
bed = TestBed.new(n_input, [400, 400, 400], n_output, output_folder)
else:
bed = TestBed.load(state_file)
######################
# PRETRAIN THE MODEL #
######################
bed.pretrain(test_set_x, epochs=1, learning_rate=0.1, batch_size=1)
########################
# FINETUNING THE MODEL #
########################
bed.finetune(train_set,
valid_set,
test_set,
epochs=1000,
learning_rate=0.1,
batch_size=1)
bed.finetune(train_set,
valid_set,
test_set,
epochs=1000,
learning_rate=0.01,
batch_size=1)
bed.finetune(train_set,
valid_set,
test_set,
epochs=1000,
learning_rate=0.001,
batch_size=1)
bed.finetune(train_set,
valid_set,
test_set,
epochs=1000,
learning_rate=0.0001,
batch_size=1)
bed.finetune(train_set,
valid_set,
test_set,
epochs=1000,
learning_rate=0.00001,
batch_size=1)
###########
# PREDICT #
###########
y_pred = bed.predict(test_set_x.get_value(borrow=True))
mae, mre, rmse = util.calculate_error_indexes(
test_set_y.get_value(borrow=True), y_pred)
print("-*-*RESULT*-*-")
print("mae={}".format(mae))
print("mre={}".format(mre))
print("rmse={}".format(rmse))
# plot
os.chdir(output_folder)
cut = min(10 * 144, test_set_x.get_value(borrow=True).shape[0])
plot_y = test_set_x.get_value(borrow=True)[:cut]
plot_y_pred = y_pred[:cut]
for i in xrange(n_output):
filename = "{}.png".format(str(i))
plot.savefig(filename, plot_y, plot_y_pred, indexes=[i])