def test_save():
j = JSONBackend('results.json')
searchspace = SearchSpace([ConstantDomain('A', 0)])
j.save([searchspace])
with open('results.json', 'r') as f:
objs = json.load(f)
assert os.path.exists(j.path) and os.path.isfile(j.path)
assert SearchSpace.from_json(objs[0]) == searchspace
searchspace2 = SearchSpace([ConstantDomain('B', 12)])
searchspace2()
print(len(searchspace2.trials))
j.save([searchspace2])
assert os.path.exists(j.path) and os.path.isfile(j.path)
backup = j.path + '.bak.1'
assert os.path.exists(backup) and os.path.isfile(backup)
with open(backup, 'r') as f:
objs = json.load(f)
assert SearchSpace.from_json(objs[0]) == searchspace
with open(j.path, 'r') as f:
objs = json.load(f)
assert SearchSpace.from_json(objs[0]) == searchspace2
def test_parameter_dict():
s = SearchSpace([ConstantDomain('A', 8)])
t = Trial(s, hyperparameters=[8])
assert t.parameter_dict == {'A': 8}
s = SearchSpace([ConstantDomain('/A', 8), ConstantDomain('/B/a/b', 2)])
t = Trial(s, hyperparameters=[8, 2])
assert t.parameter_dict == {'A': 8, 'B': {'a': {'b': 2}}}
s = SearchSpace([
ConstantDomain('/A', 8),
ConstantDomain('/B/a/b', 2),
ConstantDomain('/B/a/c', 4)
])
t = Trial(s, hyperparameters=[8, 2, 4])
assert t.parameter_dict == {'A': 8, 'B': {'a': {'b': 2, 'c': 4}}}
def test_to_json():
s = SearchSpace([ConstantDomain('A', 8)])
t = Trial(s)
assert t.to_json() == {
'searchspace': s.id,
'status': TrialStatus.INIT.value,
'hyperparameters': None,
'results': None,
'objective': None,
'errmsg': None
}
s = SearchSpace([ConstantDomain('A', 8)])
t = Trial(s, hyperparameters=[8])
assert t.to_json() == {
'searchspace': s.id,
'status': TrialStatus.READY.value,
'hyperparameters': [8],
'results': None,
'objective': None,
'errmsg': None
}
s = SearchSpace([ConstantDomain('A', 8)])
t = Trial(s, hyperparameters=[8], objective=0.374)
assert t.to_json() == {
'searchspace': s.id,
'status': TrialStatus.READY.value,
'hyperparameters': [8],
'results': None,
'objective': 0.374,
'errmsg': None
}
s = SearchSpace([ConstantDomain('A', 8)])
t = Trial(s, hyperparameters=[8], objective=0.374, errmsg='hi')
assert t.to_json() == {
'searchspace': s.id,
'status': TrialStatus.ERROR.value,
'hyperparameters': [8],
'results': None,
'objective': 0.374,
'errmsg': 'hi'
}
def test_init():
s = SearchSpace([ConstantDomain('A', 8)])
t = Trial(s)
assert t.searchspace() is s
assert t.hyperparameters is None
assert t.results is None
assert t.objective is None
assert t.errmsg is None
assert t.dirty
assert t.status == TrialStatus.INIT
s = SearchSpace([ConstantDomain('A', 8)])
t = Trial(s, hyperparameters=[8])
assert t.searchspace() is s
assert t.hyperparameters == [8]
assert t.results is None
assert t.objective is None
assert t.errmsg is None
assert t.dirty
assert t.status == TrialStatus.READY
s = SearchSpace([ConstantDomain('A', 8)])
t = Trial(s, hyperparameters=[8], results={'loss': 10}, objective=10)
assert t.searchspace() is s
assert t.hyperparameters == [8]
assert t.results == {'loss': 10}
assert t.objective == 10
assert t.errmsg is None
assert t.dirty
assert t.status == TrialStatus.DONE
s = SearchSpace([ConstantDomain('A', 8)])
t = Trial(s,
hyperparameters=[8],
results={'loss': 10},
objective=10,
errmsg='HI!')
assert t.searchspace() is s
assert t.hyperparameters == [8]
assert t.results == {'loss': 10}
assert t.objective == 10
assert t.errmsg == 'HI!'
assert t.dirty
assert t.status == TrialStatus.ERROR
def load(self):
"""Load a hyperparameter search state.
Returns
-------
searchspaces : list of pyrameter.domains.SearchSpace
The experiment state in the JSON file.
"""
with open(self.path, 'r') as f:
objs = json.load(f, cls=PyrameterDecoder)
return [SearchSpace.from_json(obj) for obj in objs]
def __init__(self, exp_key, spec, method, backend, max_evals=None):
self.exp_key = exp_key
if not isinstance(spec, Specification):
if isinstance(spec, JointDomain):
spec = Specification('', **spec.domain)
elif isinstance(spec, dict):
spec = Specification('', **spec)
elif isinstance(spec, Domain):
spec = Specification('', **{spec.name: spec})
else:
spec = Specification('', domain=spec)
self.spec = spec
self.searchspaces = [SearchSpace(ss, exp_key=exp_key)
for ss in self.spec.split()]
self.trials = {}
self.max_evals = max_evals
if isinstance(backend, str):
if '.json' in backend:
self.backend = JSONBackend(backend)
else:
self.backend = backend
if not isinstance(self.backend, BaseBackend) and self.backend is not None:
raise ValueError(
'Provided backend {} is not a valid backend.'.format(self.backend))
self._did_sort = False
try:
self.method = getattr(pyrameter.methods, method)
except AttributeError:
self.method = pyrameter.methods.random
def __init__(self, exp_key, spec, method, backend, max_evals=None):
self.exp_key = exp_key
if not isinstance(spec, Specification):
if isinstance(spec, JointDomain):
spec = Specification('', **spec.domain)
elif isinstance(spec, dict):
spec = Specification('', **spec)
elif isinstance(spec, Domain):
spec = Specification('', **{spec.name: spec})
else:
spec = Specification('', domain=spec)
self.spec = spec
domainsets = self.spec.split()
if callable(method):
self.method = method
else:
try:
self.method = getattr(pyrameter.methods, method)()
except AttributeError:
self.method = pyrameter.methods.random()
raise UserWarning(
f'Unknown optimization method {method}. Falling back to random search.'
)
if isinstance(
self.method, PopulationMethod
) or self.method == 'surrogate_p1' or self.method == 'surrogate_p2':
if any(
map(lambda x: isinstance(x, ExhaustiveDomain),
itertools.chain.from_iterable(domainsets))):
raise ValueError(
'ExhaustiveDomain provided to a population-based optimizer.'
+ 'Please reformat this as a DiscreteDomain and re-run.')
self.searchspaces = [
PopulationSearchSpace(d, exp_key=self.exp_key)
for d in domainsets
]
else:
self.searchspaces = [
SearchSpace(d, exp_key=self.exp_key)
if not any(map(lambda x: isinstance(x, ExhaustiveDomain), d))
else GridSearchSpace(d, exp_key=self.exp_key)
for d in domainsets
]
self.trials = {}
self.active = [ss for ss in self.searchspaces]
self.max_evals = max_evals if max_evals is not None else np.inf
if isinstance(backend, str):
if '.json' in backend:
self.backend = JSONBackend(backend)
else:
self.backend = backend
if not isinstance(self.backend,
BaseBackend) and self.backend is not None:
raise ValueError(
'Provided backend {} is not a valid backend.'.format(
self.backend))
self._did_sort = False