def test_conservative_predict_cancel(self):
"""
The termination criterion expects the learning_curve in a file
called learning_curve.txt as well as the current best value in
ybest.txt. We create both files and see if the termination criterion
correctly predicts to cancel or continue running under various artificial
ybest.
"""
for prob_x_greater_type in [
"posterior_mean_prob_x_greater_than",
"posterior_prob_x_greater_than"
]:
np.random.seed(13)
#generate some data:
for model_name in ["pow3", "log_power"]:
function = all_models[model_name]
params = model_defaults[model_name]
xlim = 500
x = np.arange(1, xlim, 1)
y = function(x, **params)
noise = 0.0005 * np.random.randn(len(y))
y_noisy = y + noise
y_final = y_noisy[-1]
num_train = 200
np.savetxt("learning_curve.txt", y_noisy[:200])
write_xlim(xlim)
print(("Actual ybest: %f" % y_noisy[-1]))
#we set ybest to be higher than the final value of this curve
#hence we DO want the evaluation to stop!
open("ybest.txt", "w").write(str(y_final + 0.05))
open("termination_criterion_running", "w").write("running")
ret = main(mode="conservative",
prob_x_greater_type=prob_x_greater_type,
nthreads=4)
self.assertEqual(ret, 1)
self.assertTrue(os.path.exists("y_predict.txt"))
y_predict = float(open("y_predict.txt").read())
abserr = np.abs(y_predict - y_noisy[-1])
print(("abs error %f" % abserr))
self.assertTrue(abserr < 0.03)
#we set ybest to be lower than the final value of this curve
#hence we DON'T want the evaluation to stop!
open("ybest.txt", "w").write(str(y_final - 0.05))
open("termination_criterion_running", "w").write("running")
ret = main(mode="conservative",
prob_x_greater_type=prob_x_greater_type,
nthreads=4)
self.assertEqual(ret, 0)
self.assertFalse(os.path.exists("y_predict.txt"))
self.assertFalse(
os.path.exists("termination_criterion_running"))
self.assertFalse(
os.path.exists("termination_criterion_running_pid"))
self.cleanup()
def test_conservative_predict_cancel(self):
"""
The termination criterion expects the learning_curve in a file
called learning_curve.txt as well as the current best value in
ybest.txt. We create both files and see if the termination criterion
correctly predicts to cancel or continue running under various artificial
ybest.
"""
for prob_x_greater_type in ["posterior_mean_prob_x_greater_than", "posterior_prob_x_greater_than"]:
np.random.seed(13)
#generate some data:
for model_name in ["pow3", "log_power"]:
function = all_models[model_name]
params = model_defaults[model_name]
xlim = 500
x = np.arange(1, xlim, 1)
y = function(x, **params)
noise = 0.0005 * np.random.randn(len(y))
y_noisy = y + noise
y_final = y_noisy[-1]
num_train = 200
np.savetxt("learning_curve.txt", y_noisy[:200])
write_xlim(xlim)
print "Actual ybest: %f" % y_noisy[-1]
#we set ybest to be higher than the final value of this curve
#hence we DO want the evaluation to stop!
open("ybest.txt", "w").write(str(y_final + 0.05))
open("termination_criterion_running", "w").write("running")
ret = main(mode="conservative",
prob_x_greater_type=prob_x_greater_type,
nthreads=4)
self.assertEqual(ret, 1)
self.assertTrue(os.path.exists("y_predict.txt"))
y_predict = float(open("y_predict.txt").read())
abserr = np.abs(y_predict - y_noisy[-1])
print "abs error %f" % abserr
self.assertTrue(abserr < 0.03)
#we set ybest to be lower than the final value of this curve
#hence we DON'T want the evaluation to stop!
open("ybest.txt", "w").write(str(y_final - 0.05))
open("termination_criterion_running", "w").write("running")
ret = main(mode="conservative",
prob_x_greater_type=prob_x_greater_type,
nthreads=4)
self.assertEqual(ret, 0)
self.assertFalse(os.path.exists("y_predict.txt"))
self.assertFalse(os.path.exists("termination_criterion_running"))
self.assertFalse(os.path.exists("termination_criterion_running_pid"))
self.cleanup()
def test_conservative_predictive_std_predict_cancel(self):
for prob_x_greater_type in [
"posterior_mean_prob_x_greater_than",
"posterior_prob_x_greater_than"
]:
np.random.seed(13)
#generate some data:
model_name = "pow3"
function = all_models[model_name]
params = {'a': 0.52, 'alpha': 0.2, 'c': 0.84}
xlim = 500
x = np.arange(1, xlim, 1)
y = function(x, **params)
noise = 0.01 * np.random.randn(len(y))
y_noisy = y + noise
y_final = y_noisy[-1]
num_train = 30
np.savetxt("learning_curve.txt", y_noisy[:num_train])
write_xlim(xlim)
#first check:
#if there's no ybest and the predictive_std is high
#then we want the evaluation to continue
if os.path.exists("ybest.txt"):
os.remove("ybest.txt")
ret = main(mode="conservative",
prob_x_greater_type=prob_x_greater_type,
predictive_std_threshold=0.00001,
nthreads=4)
self.assertEqual(ret, 0)
print(("Actual ybest: %f" % y_noisy[-1]))
#we set ybest to be higher than the final value of this curve
#BUT because the predictive std is still high we don't want to stop
open("ybest.txt", "w").write(str(y_final + 0.05))
open("termination_criterion_running", "w").write("running")
ret = main(mode="conservative",
prob_x_greater_type=prob_x_greater_type,
predictive_std_threshold=0.00001,
nthreads=4)
self.assertEqual(ret, 0)
self.assertFalse(os.path.exists("y_predict.txt"))
self.cleanup()
def test_conservative_real_example(self):
"""
The termination criterion expects the learning_curve in a file
called learning_curve.txt as well as the current best value in
ybest.txt. We create both files and see if the termination criterion
correctly predicts to cancel or continue running under various artificial
ybest.
"""
for prob_x_greater_type in ["posterior_mean_prob_x_greater_than", "posterior_prob_x_greater_than"]:
np.savetxt("learning_curve.txt", real_abort_learning_curve)
write_xlim(real_abort_xlim)
open("ybest.txt", "w").write(str(real_abort_ybest))
open("termination_criterion_running", "w").write("running")
ret = main(mode="conservative",
prob_x_greater_type=prob_x_greater_type,
nthreads=4)
#ybest is higher than what the curve will ever reach
#hence we expect to cancel the run:
self.assertEqual(ret, 1)
self.assertTrue(os.path.exists("y_predict.txt"))
self.assertFalse(os.path.exists("termination_criterion_running"))
self.assertFalse(os.path.exists("termination_criterion_running_pid"))
self.cleanup()
def test_data2_optimistic_prob_example(self):
prob_types = ["posterior_prob_x_greater_than"]
for mode in ["optimistic"]:
for prob_x_greater_type in prob_types:
np.savetxt("learning_curve.txt", lr_p)
write_xlim(xlim)
open("ybest.txt", "w").write(str(ybest))
open("termination_criterion_running", "w").write("running")
ret = main(mode=mode,
prob_x_greater_type=prob_x_greater_type,
nthreads=4)
self.assertTrue(os.path.exists("y_predict.txt"))
y_predict = float(open("y_predict.txt").read())
print("{} predicted accuracy: {}".format(
prob_x_greater_type, y_predict))
self.update_result(mode, prob_x_greater_type, lr,
num_checkpoint, y_predict)
self.assertFalse(
os.path.exists("termination_criterion_running"))
self.assertFalse(
os.path.exists("termination_criterion_running_pid"))
self.cleanup()
def test_conservative_predictive_std_predict_cancel(self):
for prob_x_greater_type in ["posterior_mean_prob_x_greater_than", "posterior_prob_x_greater_than"]:
np.random.seed(13)
#generate some data:
model_name = "pow3"
function = all_models[model_name]
params = {'a': 0.52, 'alpha': 0.2, 'c': 0.84}
xlim = 500
x = np.arange(1, xlim, 1)
y = function(x, **params)
noise = 0.01 * np.random.randn(len(y))
y_noisy = y + noise
y_final = y_noisy[-1]
num_train = 30
np.savetxt("learning_curve.txt", y_noisy[:num_train])
write_xlim(xlim)
#first check:
#if there's no ybest and the predictive_std is high
#then we want the evaluation to continue
if os.path.exists("ybest.txt"):
os.remove("ybest.txt")
ret = main(mode="conservative",
prob_x_greater_type=prob_x_greater_type,
predictive_std_threshold=0.00001,
nthreads=4)
self.assertEqual(ret, 0)
print "Actual ybest: %f" % y_noisy[-1]
#we set ybest to be higher than the final value of this curve
#BUT because the predictive std is still high we don't want to stop
open("ybest.txt", "w").write(str(y_final + 0.05))
open("termination_criterion_running", "w").write("running")
ret = main(mode="conservative",
prob_x_greater_type=prob_x_greater_type,
predictive_std_threshold=0.00001,
nthreads=4)
self.assertEqual(ret, 0)
self.assertFalse(os.path.exists("y_predict.txt"))
self.cleanup()
def test_error_logging(self):
"""
Test in case of an error, the error will be logged.
"""
open("ybest.txt", "w").write(str(0.5))
#Let's e.g. run main without creating any files
if os.path.exists("learning_curve.txt"):
os.remove("learning_curve.txt")
ret = main()
self.assertTrue(os.path.exists("term_crit_error.txt"))
os.remove("ybest.txt")
def test_optimistic_predict_cancel(self):
"""
Optimisitic mode
The termination criterion expects the learning_curve in a file
called learning_curve.txt as well as the current best value in
ybest.txt. We create both files and see if the termination criterion
correctly predicts to cancel or continue running under various artificial
ybest.
"""
for prob_x_greater_type in ["posterior_mean_prob_x_greater_than", "posterior_prob_x_greater_than"]:
np.random.seed(13)
#generate some data:
model_name = "pow3"
function = all_models[model_name]
params = {'a': 0.52, 'alpha': 0.2, 'c': 0.84}
xlim = 500
x = np.arange(1, xlim, 1)
y = function(x, **params)
noise = 0.01 * np.random.randn(len(y))
y_noisy = y + noise
y_final = y_noisy[-1]
num_train = 30
np.savetxt("learning_curve.txt", y_noisy[:num_train])
write_xlim(xlim)
#first check:
#if there's no ybest and the predictive_std is high
#then we want the evaluation to continue
if os.path.exists("ybest.txt"):
os.remove("ybest.txt")
ret = main(mode="optimistic",
prob_x_greater_type=prob_x_greater_type,
nthreads=4)
self.assertEqual(ret, 0)
print "Actual ybest: %f" % y_noisy[-1]
#we set ybest to be higher than the final value of this curve
#hence we DO want the evaluation to stop!
open("ybest.txt", "w").write(str(y_final + 0.05))
open("termination_criterion_running", "w").write("running")
ret = main(mode="optimistic",
prob_x_greater_type=prob_x_greater_type,
nthreads=4)
self.assertEqual(ret, 1)
self.assertTrue(os.path.exists("y_predict.txt"))
y_predict = float(open("y_predict.txt").read())
abserr = np.abs(y_predict-y_noisy[-1])
self.assertTrue(abserr < 0.05)
print "abs error %f" % abserr
#we set ybest to be lower than the final value of this curve
#hence we DON'T want the evaluation to stop!
#we assume here that because the model was set up like this
#the predictive_std is above (it should actually be around 0.019)
open("ybest.txt", "w").write(str(y_final - 0.05))
open("termination_criterion_running", "w").write("running")
ret = main(mode="optimistic", nthreads=4)
self.assertEqual(ret, 0)
self.assertFalse(os.path.exists("y_predict.txt"))
self.assertFalse(os.path.exists("termination_criterion_running"))
self.assertFalse(os.path.exists("termination_criterion_running_pid"))
num_train = 300
np.savetxt("learning_curve.txt", y_noisy[:num_train])
#we set ybest to be lower than the final value of this curve
#HOWEVER we except the predictive std to be around .0027
#so the the run should be cancelled nevertheless
open("ybest.txt", "w").write(str(y_final - 0.05))
open("termination_criterion_running", "w").write("running")
ret = main(mode="optimistic",
prob_x_greater_type=prob_x_greater_type,
nthreads=4)
self.assertEqual(ret, 1)
self.assertTrue(os.path.exists("y_predict.txt"))
y_predict = float(open("y_predict.txt").read())
abserr = np.abs(y_predict-y_noisy[-1])
self.assertTrue(abserr < 0.05)
print "abs error %f" % abserr
self.assertFalse(os.path.exists("termination_criterion_running"))
self.assertFalse(os.path.exists("termination_criterion_running_pid"))
self.cleanup()
def run(self,
index,
modes=None,
prob_types=None,
checkpoints=None,
as_process=True,
restore=True):
if modes == None:
modes = self.modes
if prob_types == None:
prob_types = self.prob_types
lr = self.lcr.get_lr(index)
if checkpoints == None:
checkpoints = [self.num_checkpoint]
r = {"max_acc": max(lr)}
if str(index) in self.results:
r = self.results[str(index)]
start_time = None
eval_time = None
for num_checkpoint in checkpoints:
for mode in modes:
for prob_type in prob_types:
key = "{}-{}-{}".format(num_checkpoint, mode, prob_type)
ybest = max(
lr
) + 0.5 # Unrealistic fatasy to finding stopping prediction value.
ret = 0
y_predict = None
if key in r and restore == True:
y_predict = r[key]['y_predict']
if 'y_best' in r[key]:
ybest = r[key]['y_best']
if y_predict != None:
print("Restore [{}] {}: {}".format(
index, key, y_predict))
if y_predict == None:
print("Run [{}] {}".format(index, key))
while ybest <= max(lr) + 0.5:
start_time = time.time()
self.prepare(lr, num_checkpoint, ybest)
if as_process == False:
ret = main(mode=mode,
prob_x_greater_type=prob_type,
nthreads=4)
else:
ret = run_program([
"python", "-m",
"pylrpredictor.terminationcriterion",
"--nthreads", "5", "--mode", mode,
"--prob-x-greater-type", prob_type
])
print("{}:{}-{}-{}:{} returns {}".format(
self.lcr.name, index, num_checkpoint, mode,
prob_type, ret))
if ret == 1:
break
else:
#ybest += 0.5
# here means no termination.
break
if os.path.exists("y_predict.txt"):
y_predict = float(open("y_predict.txt").read())
else:
y_predict = 0.0
if start_time != None:
eval_time = time.time() - start_time
r[key] = {
"num_checkpoint": num_checkpoint,
"y_predict": y_predict,
"y_best": ybest,
"est_time": eval_time
}
self.cleanup()
self.add_result(index, r)
