def test(self):
rb = RosenbrockVanilla()
func_name = rb.getFuncName()
pathToPickles = 'picklesTest'
### Test load_data_from_a_min_problem()
name_testfile = 'load_and_store_Test'
samples = numpy.array([[[1, 1], [1, 2]]])
#print samples
values = [[1.0, 2.0]]
data = {"points": samples, "vals": values}
with open("{0}/{1}.pickle".format(pathToPickles, name_testfile),
"wb") as output_file:
pickle.dump(data, output_file)
loaded_pts, loaded_vals = load_data_from_a_min_problem(
pathToPickles, name_testfile)
for index in range(len(samples)):
self.assertTrue((samples[index] == loaded_pts[index]).all)
for index in range(len(values)):
self.assertTrue((values[index] == loaded_vals[index]))
# test overwriting
samples = numpy.array([[[1, 4], [1, 2]]])
with open("{0}/{1}.pickle".format(pathToPickles, name_testfile),
"wb") as output_file:
pickle.dump(data, output_file)
loaded_pts, loaded_vals = load_data_from_a_min_problem(
pathToPickles, name_testfile)
for index in range(len(samples)):
self.assertTrue((samples[index] == loaded_pts[index]).all)
for index in range(len(values)):
self.assertTrue((values[index] == loaded_vals[index]))
### Test obtainHistoricalDataForEGO()
#TODO come up with tests for these functions
list_IS_to_query = [0]
num_init_pts_each_IS = 10
name_testfile = rb.getFuncName() + '_' + 'IS_' + '_'.join(
str(element) for element in list_IS_to_query) + '_' + str(
num_init_pts_each_IS) + "_points_each"
with open("{0}/{1}.pickle".format(pathToPickles, name_testfile),
"wb") as output_file:
pickle.dump(data, output_file)
# testHistoricalData = obtainHistoricalDataForEGO(True, rb, pathToPickles, list_IS_to_query, num_init_pts_each_IS)
# print testHistoricalData
#
# testHistoricalDataRandom = obtainHistoricalDataForEGO(False, rb, pathToPickles, list_IS_to_query, num_init_pts_each_IS)
# print testHistoricalDataRandom
### Test createHistoricalDataForKG()
listPrevData = []
samples = [[1, 1], [1, 2]]
values = [1.0, 2.0]
list_noise_variance_at_sample = [0.1, 0.3]
listPrevData.append((samples, values, list_noise_variance_at_sample))
hist_kg = createHistoricalDataForKG(rb._dim, listPrevData)
#print hist_kg
IS_samples = [[0, 1, 1], [0, 1, 2]]
for index in range(len(hist_kg.points_sampled)):
self.assertTrue(
(IS_samples[index] == hist_kg.points_sampled[index]).all)
for index in range(len(hist_kg.points_sampled_value)):
self.assertTrue(
(values[index] == hist_kg.points_sampled_value[index]).all)
samples = [[0, 0], [4, 3]]
for index in range(len(hist_kg.points_sampled)):
self.assertTrue(
(IS_samples[index] == hist_kg.points_sampled[index]).all)
listPrevData = [(samples, values, list_noise_variance_at_sample)]
bestpt, bestval, best_truth = findBestSampledValue(rb, listPrevData, 0)
# print findBestSampledValue(rb, listPrevData, 0)
self.assertAlmostEqual(bestval, 1.0, delta=.0001)
self.assertAlmostEqual(bestval, 1.0, delta=0.0001)
# self.assertAlmostEqual(bestval, 1.0, delta=0.0001)
self.assertAlmostEqual(best_truth, numpy.float64(-9.0), delta=1.0)
self.assertTrue((bestpt == [0.0, 0.0]))
list_sampled_IS = [0, 0]
gathered_data_from_all_replications = []
gathered_data_from_all_replications.append({
"points": samples,
"vals": values,
"noise_variance": list_noise_variance_at_sample,
"sampledIS": list_sampled_IS
})
for indexList in range(len(gathered_data_from_all_replications)):
for indexElem in range(
len(gathered_data_from_all_replications[indexList]
['vals'])):
self.assertAlmostEqual(
values[indexElem],
gathered_data_from_all_replications[indexList]['vals']
[indexElem],
delta=0.0001)
for indexElem in range(
len(gathered_data_from_all_replications[indexList]
['points'])):
self.assertTrue(samples[indexElem] ==
gathered_data_from_all_replications[indexList]
['points'][indexElem])
for indexElem in range(
len(gathered_data_from_all_replications[indexList]
['sampledIS'])):
self.assertTrue(list_sampled_IS[indexElem] ==
gathered_data_from_all_replications[indexList]
['sampledIS'][indexElem])
gathered_data_from_all_replications.append({
"points": samples,
"vals": values,
"noise_variance": list_noise_variance_at_sample,
"sampledIS": list_sampled_IS
})
for indexList in range(len(gathered_data_from_all_replications)):
for indexElem in range(
len(gathered_data_from_all_replications[indexList]
['vals'])):
self.assertAlmostEqual(
values[indexElem],
gathered_data_from_all_replications[indexList]['vals']
[indexElem],
delta=0.0001)
for indexElem in range(
len(gathered_data_from_all_replications[indexList]
['points'])):
self.assertTrue(samples[indexElem] ==
gathered_data_from_all_replications[indexList]
['points'][indexElem])
for indexElem in range(
len(gathered_data_from_all_replications[indexList]
['sampledIS'])):
self.assertTrue(list_sampled_IS[indexElem] ==
gathered_data_from_all_replications[indexList]
['sampledIS'][indexElem])
samples = [[-1., 0], [0.1, -2.0]]
values = [0.2, 1.5]
list_sampled_IS = [3, 3]
gathered_data_from_all_replications.append({
"points": samples,
"vals": values,
"noise_variance": list_noise_variance_at_sample,
"sampledIS": list_sampled_IS
})
for indexElem in range(
len(gathered_data_from_all_replications[2]['vals'])):
self.assertAlmostEqual(
values[indexElem],
gathered_data_from_all_replications[2]['vals'][indexElem],
delta=0.0001)
for indexElem in range(
len(gathered_data_from_all_replications[2]['points'])):
self.assertTrue(
samples[indexElem] == gathered_data_from_all_replications[2]
['points'][indexElem])
for indexElem in range(
len(gathered_data_from_all_replications[2]['sampledIS'])):
self.assertTrue(
list_sampled_IS[indexElem] ==
gathered_data_from_all_replications[2]['sampledIS'][indexElem])
listPrevData.append(
(gathered_data_from_all_replications[2]['points'],
gathered_data_from_all_replications[2]['vals'],
gathered_data_from_all_replications[2]['noise_variance']))
hist_kg = createHistoricalDataForKG(rb._dim, listPrevData)
#print hist_kg
self.assertTrue((hist_kg.points_sampled[0] == [0, 0, 0]).all)
self.assertTrue((hist_kg.points_sampled[1] == [0, 4, 3]).all)
self.assertTrue((hist_kg.points_sampled[2] == [1, -1.0, 0]).all)
self.assertTrue((hist_kg.points_sampled[3] == [1, .1, -2]).all)
self.assertAlmostEqual(values[0],
-1.0 * hist_kg.points_sampled_value[2],
delta=0.0001)
self.assertAlmostEqual(values[1],
-1.0 * hist_kg.points_sampled_value[3],
delta=0.0001)
self.assertAlmostEqual(list_noise_variance_at_sample[0],
hist_kg.points_sampled_noise_variance[2],
delta=0.0001)
self.assertAlmostEqual(list_noise_variance_at_sample[1],
hist_kg.points_sampled_noise_variance[3],
delta=0.0001)
### Test for findBestSampledValueFromHistoricalData()
atoext = AssembleToOrderExtended(mult=-1.0)
hd = HistoricalData(atoext.getDim())
pts = numpy.array([[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0],
[1.0, 0.2, 0.3, 0.4, 0.5, 0.2, 0.3, 0.4]])
self.assertTrue(len(pts) == 2)
self.assertTrue(len(pts[0]) == atoext.getDim())
self.assertTrue(len(pts[1]) == atoext.getDim())
vals = [-1.0, 0.2]
noises = [0.1, 0.2]
hd.append_historical_data(pts, vals, noises)
# print hd.to_list_of_sample_points()
bestpt, best_val, best_truth = findBestSampledValueFromHistoricalData(
atoext, hd)
# print bestpt
# print best_val
# print best_truth
self.assertTrue((pts[0] == bestpt).all)
self.assertTrue(best_val == -1.0)
self.assertAlmostEqual(best_truth,
atoext.evaluate(2, bestpt),
delta=10.0)
pts = numpy.array([[1.3, 1.4, 10.0, 11.0, 19.0, 1.0, 1.0, 1.0],
[13.0, 10.2, 10.3, 10.4, 10.5, 0.2, 10.3, 0.4]])
vals = [-11.0, 10.2]
noises = [10.1, 1000.2]
hd.append_historical_data(pts, vals, noises)
bestpt, best_val, best_truth = findBestSampledValueFromHistoricalData(
atoext, hd)
self.assertTrue((pts[0] == bestpt).all)
self.assertTrue(best_val == -11.0)
pts2 = numpy.array([[10.3, 10.4, 10.0, 11.0, 19.0, 1.0, 1.0, 1.0],
[13.0, 10.2, 10.3, 10.4, 10.5, 0.2, 10.3, 0.4]])
vals = [11.0, 10.2]
hd.append_historical_data(pts, vals, noises)
bestpt, best_val, best_truth = findBestSampledValueFromHistoricalData(
atoext, hd)
self.assertTrue((pts[0] == bestpt).all)
self.assertTrue(best_val == -11.0)
def test(self):
ato = AssembleToOrderExtended()
self.assertEqual(ato.getRunlength(0), 500)
self.assertEqual(ato.getRunlength(1), 10)
self.assertEqual(ato.getRunlength(2), 100)
self.assertEqual(ato.getBestInitialObjValue(), -numpy.inf)
listIS = [0, 1, 2]
self.assertTrue((ato.getList_IS_to_query() == listIS))
self.assertEqual('atoext', ato.getFuncName())
# for IS in ato.getList_IS_to_query():
# self.assertEqual(ato.getMeanValue(IS), 17.049)
self.assertEqual(8, ato.getDim())
x = numpy.array([19.0, 17.0, 14.0, 20.0, 16.0, 13.0, 17.0, 15.0])
self.assertEqual(ato.noise_and_cost_func(0, x),
(0.056, (24.633 - 7.5)))
self.assertEqual(ato.noise_and_cost_func(1, x), (2.944, (8.064 - 7.5)))
self.assertEqual(ato.noise_and_cost_func(2, x),
(0.332, (11.422 - 7.5)))
xstring = "[19.0 17.0 14.0 20.0 16.0 13.0 17.0 15.0 ]"
self.assertEqual(ato.convertXtoString(x), xstring)
# print 'ato.evaluate(1,x)='+str(ato.evaluate(1,x))
# print 'ato.evaluate(1,x)='+str(ato.evaluate(1,x))
# print 'ato.evaluate(1,x)='+str(ato.evaluate(1,x))
# print 'ato.evaluate(1,x)='+str(ato.evaluate(1,x))
# print 'ato.evaluate(1,x)='+str(ato.evaluate(1,x))
# exit(0)
self.assertAlmostEqual(ato.evaluate(2, x),
13.0 - ato.getMeanValue(2),
delta=3.0)
self.assertAlmostEqual(ato.evaluate(1, x),
-37.0 - ato.getMeanValue(1),
delta=3.0)
self.assertAlmostEqual(ato.evaluate(0, x),
13.0 - ato.getMeanValue(0),
delta=3.0)
x3 = numpy.array([20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0])
# print '(20,...,20) = ' + str(ato.evaluate(1,x3))
self.assertAlmostEqual(-94.0,
ato.evaluate(1, x3) + ato.getMeanValue(1),
delta=10.0)
self.assertAlmostEqual(-36.0,
ato.evaluate(2, x3) + ato.getMeanValue(2),
delta=10.0)
x4 = numpy.array([10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0])
self.assertAlmostEqual(65.0,
ato.evaluate(1, x4) + ato.getMeanValue(1),
delta=10.0)
self.assertAlmostEqual(78.0,
ato.evaluate(2, x4) + ato.getMeanValue(2),
delta=10.0)
atoneg = AssembleToOrderExtended(mult=-1.0)
self.assertAlmostEqual(-78.0 + ato.getMeanValue(2),
atoneg.evaluate(2, x4),
delta=10.0)
for i in xrange(ato.getDim()):
self.assertAlmostEqual(ato.getNumInitPtsAllIS()[i], 26., delta=0.1)
x = ato.drawRandomPoint()
for i in xrange(ato.getDim()):
self.assertGreaterEqual(x[i], 0.0)
self.assertLessEqual(x[i], 20.0)
xa = numpy.zeros(8)
xb = numpy.zeros((8, 5))
for i in xrange(8):
xa[i] = i
xb[i][1] = i
self.assertAlmostEqual(numpy.mean(xa), 3.5, delta=0.1)
self.assertAlmostEqual(numpy.mean(xb, axis=0)[1], 3.5, delta=0.1)
self.assertAlmostEqual(numpy.mean(xb, axis=0)[0], 0., delta=0.1)
### Test of ensureBoundaries()
x2 = numpy.array(
[-0.000010, 20.0000000001, 14.0, 20.0, 16.0, 13.0, 17.0, 15.0])
ato.ensureBoundaries(x2)
for i in xrange(ato.getDim()):
self.assertGreaterEqual(x2[i], 0.0)
self.assertLessEqual(x2[i], 20.0)
from moe.optimal_learning.python.python_version.domain import TensorProductDomain as pythonTensorProductDomain
from moe.optimal_learning.python.python_version.covariance import SquareExponential
from assembleToOrderExtended.assembleToOrderExtended import AssembleToOrderExtended
from multifidelity_KG.model.hyperparameter_optimization import NormalPrior, hyper_opt
from load_and_store_data import load_data_from_a_min_problem, obtainHistoricalDataForEGO, \
create_listPrevData, createHistoricalDataForMisoEI, match_pickle_filename
import sql_util
__author__ = 'jialeiwang'
num_hyper_multistart = 5
##############################################
### ATO
obj_func_min = AssembleToOrderExtended(mult=-1.0)
func_name = obj_func_min.getFuncName()
# how many prev datasets should be incorporated? This determines the dim of the GP and the num of hypers
list_previous_datasets_to_load = []
indexFirstIS = 0 # id of the first IS, used to label data accordingly
separateIS0 = 0 # 1 if IS0 is not observable and hence the data supplied for IS1, IS2, ...
### in coldstart this should be 0
### Load the training data
pathToPickles = "/fs/europa/g_pf/pickles/miso"
bias_filename = "atoext_IS_1_2_1000_points_eachdiffToIS0"
init_data_pickle_filename_prefix = func_name + '_IS_0_1_2_200_points_each_repl_' # the dataset for IS 0
replication_no = 2 # random.randint(0,2) # There are 3 pickle sets with 200 points. Pick one at random
# complete_list_prev_datasets_to_load = ["{0}_repl_{1}".format(dataset_name, replication_no) for dataset_name in list_previous_datasets_to_load]
listPrevData = [] # create_listPrevData(obj_func_min, complete_list_prev_datasets_to_load, replication_no, pathToPickles, init_data_pickle_filename_prefix)
'''
Evaluate points add a Latin Hypercube design and store them in a pickle file
Note that the pickle file has one array giving the list of points for each IS,
and one array giving the list of objective values for each IS.
The objective value must be for the MINIMIZATION VERSION of the problem, so choose mult accordingly when
instantiating the problem object.
This version creates several files, with the intention that there will be one for each replication.
Thus, the output of the algorithms will be reproducable.
'''
if __name__ == "__main__":
### Need to set the following parameters!
#obj_func_min = RosenbrockShifted( )
obj_func_min = AssembleToOrderExtended(mult=-1.0)
# list of IS that are to be queried
list_IS_to_query = obj_func_min.getList_IS_to_query(
) #[1,2,3] # [0]# for coldstart # range(obj_func_min._num_IS)
#string_list_IS_to_query = 'IS_' + '_'.join(str(element) for element in list_IS_to_query)
# print string_list_IS_to_query
# exit(0)
# create initial data for runs
num_init_pts_each_IS = 20 ###5 # for Rosenbrock # 20 # for ATO
num_replications = 100
# # create data for hyper opt.
# num_init_pts_each_IS = 200
# num_replications = 3
'''
Evaluate points add a Latin Hypercube design and store them in a pickle file
Note that the pickle file has one array giving the list of points for each IS,
and one array giving the list of objective values for each IS.
The objective value must be for the MINIMIZATION VERSION of the problem, so choose mult accordingly when
instantiating the problem object.
This version creates several files, with the intention that there will be one for each replication.
Thus, the output of the algorithms will be reproducable.
'''
if __name__ == "__main__":
### Need to set the following parameters!
#obj_func_min = RosenbrockShifted( )
obj_func_min = AssembleToOrderExtended(numIS=3, mult=-1.0)
# list of IS that are to be queried
list_IS_to_query = [1, 2,
3] # [0]# for coldstart # range(obj_func_min._num_IS)
#string_list_IS_to_query = 'IS_' + '_'.join(str(element) for element in list_IS_to_query)
# print string_list_IS_to_query
# exit(0)
# # create initial data for runs
num_init_pts_each_IS = 20 ###5 # for Rosenbrock # 20 # for ATO
num_replications = 50
# # create data for hyper opt.
# num_init_pts_each_IS = 200
# num_replications = 3
# replication_no = int(argv[0])
# additional_naming = "_diff"
additional_naming = "_diff"
additional_2 = "150"
# additional_2 = ""
for replication_no in range(rep_begin, rep_end):
### The following parameters must be adapted for each simulator
### Rosenbrock family
# obj_func_min = RosenbrockVanilla( mult=1.0 )
# obj_func_max = RosenbrockVanilla( mult=-1.0 ) # used by KG
# num_iterations = 25 # iterations per replication
# num_init_pts = 5 # points training data from the current IS
### AssembleToOrderExtended
obj_func_min = AssembleToOrderExtended( mult=-1.0 )
obj_func_max = AssembleToOrderExtended( ) # used by KG
num_iterations = 150 # iterations per replication
num_init_pts = 20
list_IS_to_query = obj_func_min.getList_IS_to_query()
func_name = obj_func_min.getFuncName()
truth_IS = obj_func_min.getTruthIS()
exploitation_IS = 1
# list of strings that give the filenames of the datasets to be loaded as additional IS
#TODO choose correct previous data for instance
list_previous_datasets_to_load = [] #['data_vKG_rb_sinN'] #['data_vKG_rb_van'] # []
print "previous datasets: {0}".format(list_previous_datasets_to_load)
path_to_pickle_data = "/fs/europa/g_pf/pickles/miso"
def obj_func_min(self):
return AssembleToOrderExtended(mult=-1.0)
def __init__(self, replication_no, method_name, bucket):
self.replication_no = replication_no
self.method_name = method_name
self._hist_data = None
self._bucket = bucket
self._obj_func_min = AssembleToOrderExtended(mult=-1.0)