def run(self,burn_in=None,thinning=None,minimum_TN=None):
folds_test = mask.compute_folds(self.I,self.J,self.folds,self.M)
folds_training = mask.compute_Ms(folds_test)
for i,(train,test) in enumerate(zip(folds_training,folds_test)):
print "Fold %s." % (i+1)
# Run the line search
line_search = LineSearch(
classifier=self.classifier,
values_K=self.values_K,
R=self.R,
M=self.M,
priors=self.priors,
initUV=self.init_UV,
iterations=self.iterations,
restarts=self.restarts)
line_search.search(burn_in=burn_in,thinning=thinning,minimum_TN=minimum_TN)
# Store the model fits, and find the best one according to the metric
all_performances = line_search.all_values(metric=self.quality_metric)
self.fout.write("All model fits for fold %s, metric %s: %s.\n" % (i+1,self.quality_metric,all_performances))
self.fout.flush()
best_K = line_search.best_value(metric=self.quality_metric)
self.fout.write("Best K for fold %s: %s.\n" % (i+1,best_K))
# Train a model with this K and measure performance on the test set
performance = self.run_model(train,test,best_K,burn_in=burn_in,thinning=thinning,minimum_TN=minimum_TN)
self.fout.write("Performance: %s.\n\n" % performance)
self.fout.flush()
def test_best_value():
I,J = 10,9
values_K = [1,2,4,5]
R = 2*numpy.ones((I,J))
M = numpy.ones((I,J))
priors = { 'alpha':3, 'beta':4, 'lambdaU':5, 'lambdaV':6 }
initUV = 'exp'
iterations = 11
linesearch = LineSearch(classifier,values_K,R,M,priors,initUV,iterations)
linesearch.all_performances = {
'BIC' : [10,9,8,7],
'AIC' : [11,13,12,14],
'loglikelihood' : [16,15,18,17],
'MSE' : [16,20,18,17]
}
assert linesearch.best_value('BIC') == 5
assert linesearch.best_value('AIC') == 1
assert linesearch.best_value('loglikelihood') == 2
assert linesearch.best_value('MSE') == 1
with pytest.raises(AssertionError) as error:
linesearch.all_values('FAIL')
assert str(error.value) == "Unrecognised metric name: FAIL."
def test_best_value():
I, J = 10, 9
values_K = [1, 2, 4, 5]
R = 2 * numpy.ones((I, J))
M = numpy.ones((I, J))
priors = {'alpha': 3, 'beta': 4, 'lambdaU': 5, 'lambdaV': 6}
initUV = 'exp'
iterations = 11
linesearch = LineSearch(classifier, values_K, R, M, priors, initUV,
iterations)
linesearch.all_performances = {
'BIC': [10, 9, 8, 7],
'AIC': [11, 13, 12, 14],
'loglikelihood': [16, 15, 18, 17],
'MSE': [16, 20, 18, 17]
}
assert linesearch.best_value('BIC') == 5
assert linesearch.best_value('AIC') == 1
assert linesearch.best_value('loglikelihood') == 2
assert linesearch.best_value('MSE') == 1
with pytest.raises(AssertionError) as error:
linesearch.all_values('FAIL')
assert str(error.value) == "Unrecognised metric name: FAIL."