def test_parameter_descriptions(self):
domain = OptimizationParameterDomain(component_mapping)
assert domain.get_parameter_descriptions() == {
"abc": {
"testDouble": {
"type": "double",
"default": 0.53,
"min": 0.05,
"max": 1.01,
"construction_key": 1,
},
"testInt": {
"type": "int",
"default": 6,
"min": 1,
"max": 11,
"construction_key": 0,
},
"testCat": {
"default": "a",
"type": "cat",
"values": ["a", "b", "c"],
"construction_key": "key_c",
},
},
"def": {
"testCat": {
"default": "a",
"type": "cat",
"values": ["a", "b", "c"],
}
},
"ghi": {},
}
def test_leaf_node_pipeline_creation(self):
data_x, data_y, _, _ = read_arff("res/datasets/blood_transfusion.arff",
4)
for leaf in leaf_nodes:
logger.info("Component mapping: ")
cm = leaf.get_rest_problem().get_component_mapping()
param_domain = OptimizationParameterDomain(cm)
default_params = param_domain.config_from_vector(
param_domain.get_default_config())
for id, component in cm.items():
logger.info(f"\t{id}:{component.get_name()}")
logger.info(f"\tParams: {default_params}")
evaluator = pipeline_evaluator.PipelineEvaluator(
numpy.copy(data_x),
numpy.copy(data_y),
"sklearn.pipeline.make_pipeline",
leaf.get_rest_problem(),
123,
)
pipeline = pipeline_constructor.construct_pipeline(
"sklearn.pipeline.make_pipeline",
leaf.get_rest_problem(),
default_params,
)
assert pipeline is not None
assert isinstance(pipeline, Pipeline)
try:
score = evaluator.evaluate_pipeline(default_params, timeout=30)
logger.info(f"\tScore: {score}")
assert score > 0.5
logger.info("################################")
except Exception as e:
logger.exception(f"Error while scoring pipeline: {e}")
def test_random_configuration_completeness(self):
domain = OptimizationParameterDomain(component_mapping)
random_config = domain.draw_random_config()
assert random_config is not None
assert len(random_config) == 4
assert numpy.all(random_config >= domain.get_min_vector())
assert numpy.all(random_config <= domain.get_max_vector())
def test_optimizer_uses_warmstart(self):
existing_score = -2.4200000000000004
for optimizer_class in optimizer_classes:
domain = OptimizationParameterDomain(component_mapping)
domain.add_result(numpy.array([1.1, -1.1]), existing_score)
optimizer = optimizer_class(
domain,
TestEvaluator(),
2.0,
1,
numpy.random.RandomState(seed=1),
)
_, score = optimizer.perform_optimization(0.1, Event())
assert existing_score <= score
def test_duplicates_not_inserted(self):
domain = OptimizationParameterDomain(component_mapping)
r1 = numpy.array([123.0])
r2 = numpy.array([456.0])
domain.add_result(r1, 12)
domain.add_result(r2, 3)
domain.add_result(r2, 12)
top_results = domain.get_top_results(3)
expected = [(12, r1), (3, r2)]
for i in range(1):
assert top_results[i][0] == expected[i][0]
assert numpy.array_equal(top_results[i][1], expected[i][1])
def test_config_vector_transformation(self):
domain = OptimizationParameterDomain(component_mapping)
config = {
"abc": {
"testDouble": 0.53,
"testInt": 6,
"testCat": "a"
},
"def": {
"testCat": "a"
},
"ghi": {},
}
vector = [0.53, 6.0, 0.0, 0.0]
assert numpy.array_equal(domain.config_to_vector(config), vector)
assert domain.config_from_vector(vector) == config
def test_optimization_result_improvements(self):
for optimizer_class in optimizer_classes:
domain = OptimizationParameterDomain(component_mapping)
optimizer = optimizer_class(
domain,
TestEvaluator(),
2.0,
1,
numpy.random.RandomState(seed=1),
)
_, score = optimizer.perform_optimization(10, Event())
assert score > -8.0
def __init__(self, predecessor, rest_problem, specified_interface,
optimizer):
super().__init__(predecessor, rest_problem, specified_interface)
self.optimizer = optimizer
self.parameter_domain = None
if (self.rest_problem.is_satisfied()
and super().is_leaf_node()) and self.optimizer is None:
self.parameter_domain = OptimizationParameterDomain(
self.rest_problem.get_component_mapping())
self.node_value = 1.0
self.simulation_visits = 0
self.score_avg = 0.0
self.best_optimization_value = 0.0
def test_optimization_timeout(self):
timeout_in_seconds = 5
for optimizer_class in optimizer_classes:
domain = OptimizationParameterDomain(component_mapping)
optimizer = optimizer_class(
domain,
TestEvaluator(),
2.0,
1,
numpy.random.RandomState(seed=1),
)
start_time = perf_counter()
optimizer.perform_optimization(timeout_in_seconds, Event())
stop_time = perf_counter()
assert (stop_time - start_time) < (timeout_in_seconds + 1)
assert (stop_time - start_time) > (timeout_in_seconds - 1)
def test_default_configuration(self):
domain = OptimizationParameterDomain(component_mapping)
assert numpy.array_equal(domain.get_default_config(),
[0.53, 6.0, 0, 0])
def test_max_vector(self):
domain = OptimizationParameterDomain(component_mapping)
assert numpy.array_equal(domain.get_max_vector(),
[1.01, 11.0, 2.0, 2.0])
def test_queuing_order(self):
domain = OptimizationParameterDomain(component_mapping)
assert not domain.has_results()
r1 = numpy.array([123.0])
r2 = numpy.array([456.0])
r3 = numpy.array([789.0])
domain.add_result(r1, 12)
domain.add_result(r2, 3)
domain.add_result(r3, 42)
assert domain.has_results()
top_score, top_candidate = domain.get_top_results(1)[0]
assert top_score == 42
assert numpy.array_equal(r3, top_candidate)
top_results = domain.get_top_results(3)
expected = [(42, r3), (12, r1), (3, r2)]
for i in range(3):
assert top_results[i][0] == expected[i][0]
assert numpy.array_equal(top_results[i][1], expected[i][1])