def test_hyperband_config(self): config_dict = { "kind": "hyperband", "maxIterations": 10, "eta": 3, "resource": {"name": "steps", "type": "int"}, "resume": False, "metric": V1OptimizationMetric( name="loss", optimization=Optimization.MINIMIZE ).to_dict(), "params": {"lr": {"kind": "choice", "value": [[0.1], [0.9]]}}, } config = V1Hyperband.from_dict(config_dict) assert_equal_dict(config.to_dict(), config_dict) # Raises for negative values config_dict["maxIterations"] = 0 with self.assertRaises(ValidationError): V1Hyperband.from_dict(config_dict) config_dict["maxIterations"] = -0.5 with self.assertRaises(ValidationError): V1Hyperband.from_dict(config_dict) config_dict["maxIterations"] = 3 # Add numRuns percent config_dict["eta"] = -0.5 with self.assertRaises(ValidationError): V1Hyperband.from_dict(config_dict) config_dict["eta"] = 2.9 config = V1Hyperband.from_dict(config_dict) assert_equal_dict(config.to_dict(), config_dict)
def test_bayes_config(self): config_dict = { "kind": "bayes", "metric": V1OptimizationMetric( name="loss", optimization=V1Optimization.MINIMIZE).to_dict(), "numInitialRuns": 2, "numIterations": 19, "utilityFunction": { "acquisitionFunction": AcquisitionFunctions.UCB, "kappa": 1.2, "gaussianProcess": { "kernel": GaussianProcessesKernels.MATERN, "lengthScale": 1.0, "nu": 1.9, "numRestartsOptimizer": 2, }, }, "params": { "lr": { "kind": "choice", "value": [[0.1], [0.9]] } }, } config = V1Bayes.from_dict(config_dict) assert_equal_dict(config.to_dict(), config_dict)
def test_workflow_config_raise_conditions(self): config_dict = { "kind": "compiled_operation", "matrix": { "kind": "mapping", "concurrency": 2, "values": [{ "foo": 1 }, { "foo": 2 }, { "foo": 3 }], }, "run": { "kind": V1RunKind.JOB, "container": { "image": "test" } }, } config = V1CompiledOperation.from_dict(config_dict) assert config.to_dict() == config_dict # Add random_search without matrix should raise config_dict["matrix"] = {"kind": "random", "numRuns": 10} with self.assertRaises(ValidationError): V1CompiledOperation.from_dict(config_dict) # Add a matrix definition with 2 methods config_dict["matrix"]["params"] = { "lr": { "kind": "choice", "value": [1, 2, 3], "pvalues": [(1, 0.3), (2, 0.3), (3, 0.3)], } } with self.assertRaises(ValidationError): V1CompiledOperation.from_dict(config_dict) # Using a distribution with random search should pass config_dict["matrix"]["params"] = { "lr": { "kind": "pchoice", "value": [(1, 0.3), (2, 0.3), (3, 0.3)] } } config = V1CompiledOperation.from_dict(config_dict) assert config.to_dict() == config_dict # Add matrix definition should pass config_dict["matrix"]["params"] = { "lr": { "kind": "choice", "value": [1, 2, 3] } } config = V1CompiledOperation.from_dict(config_dict) assert config.to_dict() == config_dict # Add grid_search should raise config_dict["matrix"] = {"kind": "grid", "numRuns": 10} config_dict["matrix"]["params"] = { "lr": { "kind": "choice", "value": [1, 2, 3] } } config = V1CompiledOperation.from_dict(config_dict) assert config.to_dict() == config_dict # Adding a distribution should raise config_dict["matrix"]["params"] = { "lr": { "kind": "pchoice", "value": [(1, 0.3), (2, 0.3), (3, 0.3)] } } with self.assertRaises(ValidationError): V1CompiledOperation.from_dict(config_dict) # Updating the matrix should pass config_dict["matrix"]["params"] = { "lr": { "kind": "choice", "value": [1, 2, 3] } } config = V1CompiledOperation.from_dict(config_dict) assert config.to_dict() == config_dict # Add hyperband should raise config_dict["matrix"] = { "kind": "hyperband", "maxIterations": 10, "eta": 3, "resource": { "name": "steps", "type": "int" }, "resume": False, "metric": V1OptimizationMetric( name="loss", optimization=V1Optimization.MINIMIZE).to_dict(), "params": { "lr": { "kind": "pchoice", "value": [(1, 0.3), (2, 0.3), (3, 0.3)] } }, "seed": 1, } config = V1CompiledOperation.from_dict(config_dict) assert config.to_dict() == config_dict # Add early stopping config_dict["matrix"]["earlyStopping"] = [ { "kind": "metric_early_stopping", "metric": "loss", "value": 0.1, "optimization": V1Optimization.MINIMIZE, "policy": { "kind": "median", "evaluationInterval": 1 }, }, { "kind": "metric_early_stopping", "metric": "accuracy", "value": 0.9, "optimization": V1Optimization.MAXIMIZE, "policy": { "kind": "truncation", "percent": 50, "evaluationInterval": 1, }, }, ] config = V1CompiledOperation.from_dict(config_dict) assert config.to_dict() == config_dict # Add bayes should raise config_dict["matrix"] = { "kind": "bayes", "metric": V1OptimizationMetric( name="loss", optimization=V1Optimization.MINIMIZE).to_dict(), "numInitialRuns": 2, "numIterations": 10, "utilityFunction": { "acquisitionFunction": AcquisitionFunctions.UCB, "kappa": 1.2, "gaussianProcess": { "kernel": GaussianProcessesKernels.MATERN, "lengthScale": 1.0, "nu": 1.9, "numRestartsOptimizer": 2, }, }, "params": { "lr": { "kind": "pchoice", "value": [(1, 0.3), (2, 0.3), (3, 0.3)] } }, "seed": 1, } with self.assertRaises(ValidationError): V1CompiledOperation.from_dict(config_dict) # Using non uniform distribution should raise # Updating the matrix should pass config_dict["matrix"]["params"] = { "lr": { "kind": "pchoice", "value": [[0.1, 0.1], [0.2, 0.9]] } } with self.assertRaises(ValidationError): V1CompiledOperation.from_dict(config_dict) config_dict["matrix"]["params"] = { "lr": { "kind": "normal", "value": [0.1, 0.2] } } with self.assertRaises(ValidationError): V1CompiledOperation.from_dict(config_dict) # Using uniform distribution should not raise config_dict["matrix"]["params"] = { "lr": { "kind": "uniform", "value": { "low": 0.1, "high": 0.2 } } } config = V1CompiledOperation.from_dict(config_dict) assert_equal_dict(config.to_dict(), config_dict)