def evaluate_model(sda, testdata, options):
train_set_x, train_set_y = testdata
if options['retrain'] == 0:
test_model = sda.build_test_function(
dataset=testdata,
batch_size=1, # options['batchsize']
)
else:
test_model = sda.build_test_function_reuse(
dataset=testdata,
batch_size=1, #options['batchsize'],
)
# print sda.params[1].get_value()[-1]
# print sda.params_b[1].get_value()[-1]
# kkk
(ytest, ypred, ypred_prob) = test_model()
print >> sys.stderr, "Test GT Differences: "
print >> sys.stderr, sum(ytest != train_set_y.eval())
if options['oneclass'] == True:
options['nclasses'] = 2
# ypred = numpy.array( ypred_prob[:,0] < options['threshold'], dtype=numpy.uint8)
if options['threshold'] != None:
ypred = numpy.array(ypred_prob[:, 0] < options['threshold'],
dtype=numpy.uint8)
else:
ypred = numpy.argmax(ypred_prob, axis=1)
test_score = evaluate_error(ytest, ypred, options)
cm = confusion_matrix(ytest, ypred, options['nclasses'])
print >> sys.stderr, "Test CM"
print >> sys.stderr, cm
if options['verbose'] > 0:
print >> sys.stderr, ((' test error of best model %03f %%') %
(test_score * 100.))
return (test_score, ytest, ypred)
def evaluate_model(sda,testdata,options):
train_set_x, train_set_y = testdata
if options['retrain'] == 0:
test_model = sda.build_test_function(
dataset = testdata,
batch_size = 1, # options['batchsize']
)
else:
test_model = sda.build_test_function_reuse(
dataset = testdata,
batch_size = 1, #options['batchsize'],
)
# print sda.params[1].get_value()[-1]
# print sda.params_b[1].get_value()[-1]
# kkk
(ytest,ypred,ypred_prob) = test_model()
print >> sys.stderr, "Test GT Differences: "
print >> sys.stderr, sum(ytest != train_set_y.eval())
if options['oneclass'] == True:
options['nclasses'] = 2
# ypred = numpy.array( ypred_prob[:,0] < options['threshold'], dtype=numpy.uint8)
if options['threshold'] != None:
ypred = numpy.array( ypred_prob[:,0] < options['threshold'], dtype=numpy.uint8)
else:
ypred = numpy.argmax( ypred_prob, axis = 1 )
test_score = evaluate_error( ytest, ypred, options )
cm = confusion_matrix( ytest, ypred, options['nclasses'] )
print >> sys.stderr, "Test CM"
print >> sys.stderr, cm
if options['verbose'] > 0:
print >> sys.stderr, ((' test error of best model %03f %%') %
(test_score * 100.))
return (test_score, ytest, ypred)
def pretrain_finetune_model(sda, pretraining_fns, train_set, test_set,
options):
train_set_x, train_set_y = train_set
n_train_batches = train_set_x.get_value(borrow=True).shape[0]
n_train_batches /= options['batchsize']
if options['retrain'] == 0:
bestmodelsda = copy.copy(sda)
# -----------------------------------------------
# PRETRAINING
# -----------------------------------------------
if options['verbose'] > 5:
print >> sys.stderr, ('... pre-training the model')
start_time = time.clock()
## Pre-train layer-wise
corruption_levels = options['corruptlevels']
for i in xrange(sda.n_layers):
# go through pretraining epochs
for epoch in xrange(options['pretraining_epochs']):
# go through the training set
c = []
for batch_index in xrange(n_train_batches):
c.append(pretraining_fns[i](
index=batch_index,
corruption=corruption_levels[i],
lr=options['pretrain_lr']))
if epoch % 100 == 0 and options['verbose'] > 5:
print >> sys.stderr, (
'Pre-training layer %02i, epoch %04d, cost ' %
(i, epoch)),
print >> sys.stderr, (numpy.mean(c))
end_time = time.clock()
if options['savetimes']:
filename = '{0:s}/times_pr_{1:03d}_{2:03d}.pkl.gz'.format(
options['outputfolderres'], options['nrun'],
string.atoi(options['resolution']))
save_gzdata(filename, end_time - start_time)
if options['verbose'] > 4:
print >> sys.stderr, ('The pretraining code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' %
((end_time - start_time) / 60.))
# get the training, validation and testing function for the model
#dataset = [folds[0][0], folds[1][0], folds[2]]
dataset = [train_set, test_set]
if options['verbose'] > 5:
print >> sys.stderr, ('... getting the finetuning functions')
train_fn, validate_model = sda.build_finetune_functions(
datasets=dataset,
batch_size=options['batchsize'],
learning_rate=options['finetune_lr'])
else:
dataset = [train_set, test_set]
train_fn, validate_model = sda.build_finetune_functions_reuse(
datasets=dataset,
batch_size=options['batchsize'],
learning_rate=options['finetune_lr'],
update_layerwise=options['retrain_ft_layers'])
# ------------------------------------------------------------------------------------------------
# -----------------------------------------------
# FINETUNE
# -----------------------------------------------
if options['verbose'] > 5:
print >> sys.stderr, ('... finetunning the model')
# early-stopping parameters
patience = 10 * n_train_batches # look as this many examples regardless
patience_increase = 2. #2. # wait this much longer when a new best is found
improvement_threshold = 0.995 # 0.995 # a relative improvement of this much is
# considered significant
validation_frequency = min(n_train_batches, patience / 2)
# go through this many
# minibatche before checking the network
# on the validation set; in this case we
# check every epoch
best_validation_loss = numpy.inf
test_score = 0.
start_time = time.clock()
done_looping = False
epoch = 0
while (epoch < options['training_epochs']) and (not done_looping):
epoch = epoch + 1
for minibatch_index in xrange(n_train_batches):
minibatch_avg_cost = train_fn(minibatch_index)
iter = (epoch - 1) * n_train_batches + minibatch_index
if (iter + 1) % validation_frequency == 0:
#this_validation_loss = numpy.mean( validate_model() )
(y_valid, y_pred, y_pred_prob) = validate_model()
# pos = numpy.random.randint(len(y_pred),size=(100,))
# print options
# print pos
# print y_pred_prob[pos,:].T
# raw_input()
# alll
# we are going to control the predictions according to their prob
if options['threshold'] != None:
y_pred = numpy.array(
y_pred_prob[:, 0] < options['threshold'],
dtype=numpy.uint8)
else:
y_pred = numpy.argmax(y_pred_prob, axis=1)
this_validation_loss = evaluate_error(y_valid, y_pred, options)
# if epoch % 10 == 0:
# cm = confusion_matrix(y_valid, y_pred, options['nclasses'])
# print >> sys.stderr, cm, this_validation_loss
# print >> sys.stderr, this_validation_loss
# this_validation_loss = numpy.mean(validation_losses)
if epoch % 30 == 0 and options['verbose'] > 5:
# print >> sys.stderr, y_valid
# print >> sys.stderr, y_pred_prob
# print >> sys.stderr, y_pred
# print >> sys.stderr, y_valid.shape
# print >> sys.stderr, y_pred.shape
# print >> sys.stderr, y_pred_prob[1:10,:], y_pred[1:10], y_valid[1:10]
# print >> sys.stderr, test_set[1].eval()
print >> sys.stderr, (
'epoch %04i, minibatch %04i/%04i, validation error %03f %%'
% (epoch, minibatch_index + 1, n_train_batches,
this_validation_loss * 100.))
# if we got the best validation score until now
if this_validation_loss < best_validation_loss:
bestmodelsda = copy.copy(sda)
# % ------------------------------------------------------------
if options['oneclass'] == True:
options['nclasses'] = 2
#print >> sys.stderr, sda.params[-2].get_value().T, sda.params[-1].get_value()
pos = numpy.random.randint(len(y_pred), size=(10, ))
# print options
# print pos
# print y_pred_prob.shape
#print >> sys.stderr, options['threshold']
# print >> sys.stderr, numpy.array( y_pred_prob[:,0] < options['threshold'], dtype=numpy.uint8)
#print >> sys.stderr, y_pred_prob[pos,:].T
#print >> sys.stderr, y_pred[pos]
cm = confusion_matrix(y_valid, y_pred, options['nclasses'])
#print >> sys.stderr, ("Fine tune...epoch %04i" % epoch)
#print >> sys.stderr, this_validation_loss
#print >> sys.stderr, cm
# options['nclasses'] = 1
# % ------------------------------------------------------------
# improve patience if loss improvement is good enough
if (this_validation_loss <
best_validation_loss * improvement_threshold):
patience = max(patience, iter * patience_increase)
# save best validation score and iteration number
best_validation_loss = this_validation_loss
best_iter = iter
if patience <= iter:
done_looping = True
break
end_time = time.clock()
if options['savetimes']:
filename = '{0:s}/times_fn_{1:03d}_{2:03d}.pkl.gz'.format(
options['outputfolderres'], options['nrun'],
string.atoi(options['resolution']))
save_gzdata(filename, end_time - start_time)
print >> sys.stderr, ("Stopped at epoch %04i" % epoch)
return (best_validation_loss, bestmodelsda)
def pretrain_finetune_model(sda,pretraining_fns,train_set,test_set,options):
train_set_x, train_set_y = train_set
n_train_batches = train_set_x.get_value(borrow=True).shape[0]
n_train_batches /= options['batchsize']
if options['retrain'] == 0:
bestmodelsda = copy.copy( sda )
# -----------------------------------------------
# PRETRAINING
# -----------------------------------------------
if options['verbose'] > 5:
print >> sys.stderr, ('... pre-training the model')
start_time = time.clock()
## Pre-train layer-wise
corruption_levels = options['corruptlevels']
for i in xrange(sda.n_layers):
# go through pretraining epochs
for epoch in xrange(options['pretraining_epochs']):
# go through the training set
c = []
for batch_index in xrange(n_train_batches):
c.append(pretraining_fns[i](index=batch_index,
corruption=corruption_levels[i],
lr=options['pretrain_lr']))
if epoch % 100 == 0 and options['verbose'] > 5:
print >> sys.stderr, ('Pre-training layer %02i, epoch %04d, cost ' % (i, epoch)),
print >> sys.stderr, (numpy.mean(c))
end_time = time.clock()
if options['savetimes']:
filename = '{0:s}/times_pr_{1:03d}_{2:03d}.pkl.gz'.format(options['outputfolderres'],options['nrun'],string.atoi(options['resolution']))
save_gzdata(filename, end_time - start_time)
if options['verbose'] > 4:
print >> sys.stderr, ('The pretraining code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))
# get the training, validation and testing function for the model
#dataset = [folds[0][0], folds[1][0], folds[2]]
dataset = [train_set, test_set]
if options['verbose'] > 5:
print >> sys.stderr,('... getting the finetuning functions')
train_fn, validate_model = sda.build_finetune_functions(
datasets=dataset,
batch_size=options['batchsize'],
learning_rate=options['finetune_lr']
)
else:
dataset = [train_set, test_set]
train_fn, validate_model = sda.build_finetune_functions_reuse(
datasets=dataset, batch_size=options['batchsize'],
learning_rate=options['finetune_lr'], update_layerwise=options['retrain_ft_layers'])
# ------------------------------------------------------------------------------------------------
# -----------------------------------------------
# FINETUNE
# -----------------------------------------------
if options['verbose'] > 5:
print >> sys.stderr, ('... finetunning the model')
# early-stopping parameters
patience = 10 * n_train_batches # look as this many examples regardless
patience_increase = 2. #2. # wait this much longer when a new best is found
improvement_threshold = 0.995 # 0.995 # a relative improvement of this much is
# considered significant
validation_frequency = min(n_train_batches, patience / 2)
# go through this many
# minibatche before checking the network
# on the validation set; in this case we
# check every epoch
best_validation_loss = numpy.inf
test_score = 0.
start_time = time.clock()
done_looping = False
epoch = 0
while (epoch < options['training_epochs']) and (not done_looping):
epoch = epoch + 1
for minibatch_index in xrange(n_train_batches):
minibatch_avg_cost = train_fn(minibatch_index)
iter = (epoch - 1) * n_train_batches + minibatch_index
if (iter + 1) % validation_frequency == 0:
#this_validation_loss = numpy.mean( validate_model() )
(y_valid, y_pred, y_pred_prob) = validate_model()
# pos = numpy.random.randint(len(y_pred),size=(100,))
# print options
# print pos
# print y_pred_prob[pos,:].T
# raw_input()
# alll
# we are going to control the predictions according to their prob
if options['threshold'] != None:
y_pred = numpy.array( y_pred_prob[:,0] < options['threshold'], dtype=numpy.uint8)
else:
y_pred = numpy.argmax( y_pred_prob, axis = 1 )
this_validation_loss = evaluate_error( y_valid, y_pred, options )
# if epoch % 10 == 0:
# cm = confusion_matrix(y_valid, y_pred, options['nclasses'])
# print >> sys.stderr, cm, this_validation_loss
# print >> sys.stderr, this_validation_loss
# this_validation_loss = numpy.mean(validation_losses)
if epoch % 30 == 0 and options['verbose'] > 5:
# print >> sys.stderr, y_valid
# print >> sys.stderr, y_pred_prob
# print >> sys.stderr, y_pred
# print >> sys.stderr, y_valid.shape
# print >> sys.stderr, y_pred.shape
# print >> sys.stderr, y_pred_prob[1:10,:], y_pred[1:10], y_valid[1:10]
# print >> sys.stderr, test_set[1].eval()
print >> sys.stderr,('epoch %04i, minibatch %04i/%04i, validation error %03f %%' %
(epoch, minibatch_index + 1, n_train_batches,
this_validation_loss * 100.))
# if we got the best validation score until now
if this_validation_loss < best_validation_loss:
bestmodelsda = copy.copy(sda)
# % ------------------------------------------------------------
if options['oneclass'] == True:
options['nclasses'] = 2
#print >> sys.stderr, sda.params[-2].get_value().T, sda.params[-1].get_value()
pos = numpy.random.randint(len(y_pred),size=(10,))
# print options
# print pos
# print y_pred_prob.shape
#print >> sys.stderr, options['threshold']
# print >> sys.stderr, numpy.array( y_pred_prob[:,0] < options['threshold'], dtype=numpy.uint8)
#print >> sys.stderr, y_pred_prob[pos,:].T
#print >> sys.stderr, y_pred[pos]
cm = confusion_matrix(y_valid, y_pred, options['nclasses'])
#print >> sys.stderr, ("Fine tune...epoch %04i" % epoch)
#print >> sys.stderr, this_validation_loss
#print >> sys.stderr, cm
# options['nclasses'] = 1
# % ------------------------------------------------------------
# improve patience if loss improvement is good enough
if (
this_validation_loss < best_validation_loss *
improvement_threshold
):
patience = max(patience, iter * patience_increase)
# save best validation score and iteration number
best_validation_loss = this_validation_loss
best_iter = iter
if patience <= iter:
done_looping = True
break
end_time = time.clock()
if options['savetimes']:
filename = '{0:s}/times_fn_{1:03d}_{2:03d}.pkl.gz'.format(options['outputfolderres'],options['nrun'],string.atoi(options['resolution']))
save_gzdata(filename, end_time - start_time)
print >> sys.stderr, ("Stopped at epoch %04i" % epoch )
return (best_validation_loss,bestmodelsda)
