def test_vectorize_simple():
N = as_apply(15)
p0 = hp_uniform('p0', 0, 1)
loss = p0**2
print loss
expr_idxs = scope.range(N)
vh = VectorizeHelper(loss, expr_idxs, build=True)
vloss = vh.v_expr
full_output = as_apply([vloss, vh.idxs_by_label(), vh.vals_by_label()])
fo2 = replace_repeat_stochastic(full_output)
new_vc = recursive_set_rng_kwarg(
fo2,
as_apply(np.random.RandomState(1)),
)
#print new_vc
losses, idxs, vals = rec_eval(new_vc)
print 'losses', losses
print 'idxs p0', idxs['p0']
print 'vals p0', vals['p0']
p0dct = dict(zip(idxs['p0'], vals['p0']))
for ii, li in enumerate(losses):
assert p0dct[ii]**2 == li
def test_vectorize_multipath():
N = as_apply(15)
p0 = hp_uniform('p0', 0, 1)
loss = hp_choice('p1', [1, p0, -p0])**2
expr_idxs = scope.range(N)
vh = VectorizeHelper(loss, expr_idxs, build=True)
vloss = vh.v_expr
print vloss
full_output = as_apply([vloss, vh.idxs_by_label(), vh.vals_by_label()])
new_vc = recursive_set_rng_kwarg(
full_output,
as_apply(np.random.RandomState(1)),
)
losses, idxs, vals = rec_eval(new_vc)
print 'losses', losses
print 'idxs p0', idxs['p0']
print 'vals p0', vals['p0']
print 'idxs p1', idxs['p1']
print 'vals p1', vals['p1']
p0dct = dict(zip(idxs['p0'], vals['p0']))
p1dct = dict(zip(idxs['p1'], vals['p1']))
for ii, li in enumerate(losses):
print ii, li
if p1dct[ii] != 0:
assert li == p0dct[ii]**2
else:
assert li == 1
def __init__(self, expr, name=None, rseed=None, loss_target=None, exceptions=None, do_checks=True):
if do_checks:
if isinstance(expr, pyll.Apply):
self.expr = expr
# XXX: verify that expr is a dictionary with the right keys,
# then refactor the code below
elif isinstance(expr, dict):
if "loss" not in expr:
raise ValueError("expr must define a loss")
if "status" not in expr:
expr["status"] = STATUS_OK
self.expr = pyll.as_apply(expr)
else:
raise TypeError("expr must be a dictionary")
else:
self.expr = pyll.as_apply(expr)
self.params = {}
for node in pyll.dfs(self.expr):
if node.name == "hyperopt_param":
self.params[node.arg["label"].obj] = node.arg["obj"]
if exceptions is not None:
self.exceptions = exceptions
self.loss_target = loss_target
self.installed_rng = False
if rseed is None:
self.rng = None
else:
self.rng = np.random.RandomState(rseed)
self.name = name
def test_vectorize_multipath():
N = as_apply(15)
p0 = hp_uniform('p0', 0, 1)
loss = hp_choice('p1', [1, p0, -p0]) ** 2
expr_idxs = scope.range(N)
vh = VectorizeHelper(loss, expr_idxs, build=True)
vloss = vh.v_expr
print vloss
full_output = as_apply([vloss,
vh.idxs_by_label(),
vh.vals_by_label()])
new_vc = recursive_set_rng_kwarg(
full_output,
as_apply(np.random.RandomState(1)),
)
losses, idxs, vals = rec_eval(new_vc)
print 'losses', losses
print 'idxs p0', idxs['p0']
print 'vals p0', vals['p0']
print 'idxs p1', idxs['p1']
print 'vals p1', vals['p1']
p0dct = dict(zip(idxs['p0'], vals['p0']))
p1dct = dict(zip(idxs['p1'], vals['p1']))
for ii, li in enumerate(losses):
print ii, li
if p1dct[ii] != 0:
assert li == p0dct[ii] ** 2
else:
assert li == 1
def test_vectorize_simple():
N = as_apply(15)
p0 = hp_uniform('p0', 0, 1)
loss = p0 ** 2
print loss
expr_idxs = scope.range(N)
vh = VectorizeHelper(loss, expr_idxs, build=True)
vloss = vh.v_expr
full_output = as_apply([vloss,
vh.idxs_by_label(),
vh.vals_by_label()])
fo2 = replace_repeat_stochastic(full_output)
new_vc = recursive_set_rng_kwarg(
fo2,
as_apply(np.random.RandomState(1)),
)
#print new_vc
losses, idxs, vals = rec_eval(new_vc)
print 'losses', losses
print 'idxs p0', idxs['p0']
print 'vals p0', vals['p0']
p0dct = dict(zip(idxs['p0'], vals['p0']))
for ii, li in enumerate(losses):
assert p0dct[ii] ** 2 == li
def __init__(self, expr,
name=None,
loss_target=None,
exceptions=None,
do_checks=True,
):
if do_checks:
if isinstance(expr, pyll.Apply):
self.expr = expr
# XXX: verify that expr is a dictionary with the right keys,
# then refactor the code below
elif isinstance(expr, dict):
if 'loss' not in expr:
raise ValueError('expr must define a loss')
if 'status' not in expr:
expr['status'] = STATUS_OK
self.expr = pyll.as_apply(expr)
else:
raise TypeError('expr must be a dictionary')
else:
self.expr = pyll.as_apply(expr)
self.params = {}
for node in pyll.dfs(self.expr):
if node.name == 'hyperopt_param':
label = node.arg['label'].obj
if label in self.params:
raise DuplicateLabel(label)
self.params[label] = node.arg['obj']
if exceptions is not None:
self.exceptions = exceptions
self.loss_target = loss_target
self.name = name
def n_arms(N=2):
"""
Each arm yields a reward from a different Gaussian.
The correct arm is arm 0.
"""
x = hp_choice('x', [0, 1])
reward_mus = as_apply([-1] + [0] * (N - 1))
reward_sigmas = as_apply([1] * N)
return {'loss': scope.normal(reward_mus[x], reward_sigmas[x]),
'loss_variance': 1.0}
def n_arms(N=2):
"""
Each arm yields a reward from a different Gaussian.
The correct arm is arm 0.
"""
x = hp_choice('x', [0, 1])
reward_mus = as_apply([-1] + [0] * (N - 1))
reward_sigmas = as_apply([1] * N)
return {
'loss': scope.normal(reward_mus[x], reward_sigmas[x]),
'loss_variance': 1.0
}
def test2(self):
self.expr = as_apply(dict(p0=one_of(0, 1)))
self.wanted = [
[('p0.randint', [0], [0])], [('p0.randint', [1], [1])],
[('p0.randint', [2], [0])], [('p0.randint', [3], [0])],
[('p0.randint', [4], [0])]]
self.foo()
def evaluate(self, config, ctrl):
memo = self.memo_from_config(config)
memo[self.pyll_ctrl] = ctrl
if self.init_pyll_memo:
memo = self.init_pyll_memo(memo, config=config, ctrl=ctrl)
if self.rng is not None and not self.installed_rng:
# -- N.B. this modifies the expr graph in-place
# XXX this feels wrong
self.expr = recursive_set_rng_kwarg(self.expr,
pyll.as_apply(self.rng))
self.installed_rng = True
try:
# -- the "work" of evaluating `config` can be written
# either into the pyll part (self.expr)
# or the normal Python part (self.fn)
pyll_rval = pyll.rec_eval(self.expr, memo=memo)
rval = self.fn(pyll_rval)
except Exception, e:
n_match = 0
for match, match_pair in self.exceptions:
if match(e):
rval = match_pair(e)
logger.info('Caught fn exception %s' % str(rval))
n_match += 1
break
if n_match == 0:
raise
def n_arms(N=2):
"""
Each arm yields a reward from a different Gaussian.
The correct arm is arm 0.
"""
rng = np.random.RandomState(123)
x = hp_choice('x', [0, 1])
reward_mus = as_apply([-1] + [0] * (N - 1))
reward_sigmas = as_apply([1] * N)
return {
'loss': scope.normal(reward_mus[x], reward_sigmas[x], rng=rng),
'loss_variance': 1.0,
'status': base.STATUS_OK
}
def __init__(self, bandit, seed=seed, cmd=None, workdir=None):
self.bandit = bandit
self.seed = seed
self.rng = np.random.RandomState(self.seed)
self.cmd = cmd
self.workdir = workdir
self.s_new_ids = pyll.Literal('new_ids') # -- list at eval-time
before = pyll.dfs(self.bandit.expr)
# -- raises exception if expr contains cycles
pyll.toposort(self.bandit.expr)
vh = self.vh = VectorizeHelper(self.bandit.expr, self.s_new_ids)
# -- raises exception if v_expr contains cycles
pyll.toposort(vh.v_expr)
idxs_by_label = vh.idxs_by_label()
vals_by_label = vh.vals_by_label()
after = pyll.dfs(self.bandit.expr)
# -- try to detect if VectorizeHelper screwed up anything inplace
assert before == after
assert set(idxs_by_label.keys()) == set(vals_by_label.keys())
assert set(idxs_by_label.keys()) == set(self.bandit.params.keys())
# -- make the graph runnable and SON-encodable
# N.B. operates inplace
self.s_idxs_vals = recursive_set_rng_kwarg(
scope.pos_args(idxs_by_label, vals_by_label),
pyll.as_apply(self.rng))
# -- raises an exception if no topological ordering exists
pyll.toposort(self.s_idxs_vals)
def __init__(self, bandit, seed=seed, cmd=None, workdir=None):
self.bandit = bandit
self.seed = seed
self.rng = np.random.RandomState(self.seed)
self.cmd = cmd
self.workdir = workdir
self.s_new_ids = pyll.Literal('new_ids') # -- list at eval-time
before = pyll.dfs(self.bandit.expr)
# -- raises exception if expr contains cycles
pyll.toposort(self.bandit.expr)
vh = self.vh = VectorizeHelper(self.bandit.expr, self.s_new_ids)
# -- raises exception if v_expr contains cycles
pyll.toposort(vh.v_expr)
idxs_by_label = vh.idxs_by_label()
vals_by_label = vh.vals_by_label()
after = pyll.dfs(self.bandit.expr)
# -- try to detect if VectorizeHelper screwed up anything inplace
assert before == after
assert set(idxs_by_label.keys()) == set(vals_by_label.keys())
assert set(idxs_by_label.keys()) == set(self.bandit.params.keys())
# -- make the graph runnable and SON-encodable
# N.B. operates inplace
self.s_idxs_vals = recursive_set_rng_kwarg(
scope.pos_args(idxs_by_label, vals_by_label),
pyll.as_apply(self.rng))
# -- raises an exception if no topological ordering exists
pyll.toposort(self.s_idxs_vals)
def test1(self):
self.expr = as_apply(dict(p0=normal(0, 1)))
self.wanted = [[('p0', [0], [-1.0856306033005612])],
[('p0', [1], [0.99734544658358582])],
[('p0', [2], [0.28297849805199204])],
[('p0', [3], [-1.506294713918092])],
[('p0', [4], [-0.57860025196853637])]]
self.foo()
def test_repeatable():
u = scope.uniform(0, 1)
aa = as_apply(dict(u=u, n=scope.normal(5, 0.1), l=[0, 1, scope.one_of(2, 3), u]))
dd1 = sample(aa, np.random.RandomState(3))
dd2 = sample(aa, np.random.RandomState(3))
dd3 = sample(aa, np.random.RandomState(4))
assert dd1 == dd2
assert dd1 != dd3
def __init__(self):
d = dict(
# -- alphabetical order
lu = scope.loguniform(-2, 2),
qlu = scope.qloguniform(np.log(0.01), np.log(20), 2),
qu = scope.quniform(-4.999, 5, 1),
u = scope.uniform(0, 10),
)
base.Bandit.__init__(self, as_apply(d))
def test5(self):
p0 = uniform(0, 1)
self.expr = as_apply(dict(p0=p0, p1=p0))
self.wanted = [[('p0', [0], [0.69646918559786164])], [('p0', [1],
[0.28613933495037946])], [('p0', [2],
[0.22685145356420311])], [('p0', [3],
[0.55131476908289123])], [('p0', [4],
[0.71946896978556307])]]
self.foo()
def test_repeatable():
u = scope.uniform(0, 1)
aa = as_apply(
dict(u=u, n=scope.normal(5, 0.1), l=[0, 1, scope.one_of(2, 3), u]))
dd1 = sample(aa, np.random.RandomState(3))
dd2 = sample(aa, np.random.RandomState(3))
dd3 = sample(aa, np.random.RandomState(4))
assert dd1 == dd2
assert dd1 != dd3
def test5(self):
p0 = uniform(0, 1)
self.expr = as_apply(dict(p0=p0, p1=p0))
self.wanted = [[('p0', [0], [0.69646918559786164])],
[('p0', [1], [0.28613933495037946])],
[('p0', [2], [0.22685145356420311])],
[('p0', [3], [0.55131476908289123])],
[('p0', [4], [0.71946896978556307])]]
self.foo()
def test_recursive_set_rng_kwarg():
uniform = scope.uniform
a = as_apply([uniform(0, 1), uniform(2, 3)])
rng = np.random.RandomState(234)
recursive_set_rng_kwarg(a, rng)
print a
val_a = rec_eval(a)
assert 0 < val_a[0] < 1
assert 2 < val_a[1] < 3
def test_recursive_set_rng_kwarg():
uniform = scope.uniform
a = as_apply([uniform(0, 1), uniform(2, 3)])
rng = np.random.RandomState(234)
recursive_set_rng_kwarg(a, rng)
print a
val_a = rec_eval(a)
assert 0 < val_a[0] < 1
assert 2 < val_a[1] < 3
def test1(self):
self.expr = as_apply(dict(p0=normal(0, 1)))
self.wanted = [
[('p0', [0], [-1.0856306033005612])],
[('p0', [1], [0.99734544658358582])],
[('p0', [2], [0.28297849805199204])],
[('p0', [3], [-1.506294713918092])],
[('p0', [4], [-0.57860025196853637])]]
self.foo()
def test_sample():
u = scope.uniform(0, 1)
aa = as_apply(dict(u=u, n=scope.normal(5, 0.1), l=[0, 1, scope.one_of(2, 3), u]))
print aa
dd = sample(aa, np.random.RandomState(3))
assert 0 < dd["u"] < 1
assert 4 < dd["n"] < 6
assert dd["u"] == dd["l"][3]
assert dd["l"][:2] == (0, 1)
assert dd["l"][2] in (2, 3)
def test6(self):
p0 = uniform(0, 1)
self.expr = as_apply(dict(p0=p0, p1=normal(p0, 1)))
self.wanted = [
[('p0', [0], [0.69646918559786164]), ('p1', [0], [-0.25562802126346051])],
[('p0', [1], [0.55131476908289123]), ('p1', [1], [-0.19412629039976703])],
[('p0', [2], [0.71946896978556307]), ('p1', [2], [1.0415750381251847])],
[('p0', [3], [0.68482973858486329]), ('p1', [3], [0.63331201764547818])],
[('p0', [4], [0.48093190148436094]), ('p1', [4], [-1.1383681635523848])]]
self.foo()
def test_fg11_top_bandit():
L = lfw.FG11Bandit()
config = stochastic.sample(L.template, np.random.RandomState(0))
config['decisions'] = None
config['slm'] = stochastic.sample(pyll.as_apply(params.fg11_top),
np.random.RandomState(0))
config['comparison'] = 'sqrtabsdiff'
rec = L.evaluate(config, hyperopt.base.Ctrl(None))
assert np.abs(rec['loss'] - .194) < 1e-2
return rec
def config0():
p0 = scope.uniform(0, 1)
p1 = scope.uniform(2, 3)
p2 = scope.one_of(-1, p0)
p3 = scope.one_of(-2, p1)
p4 = 1
p5 = [3, 4, p0]
d = locals()
del d['p1'] # -- don't sample p1 all the time
s = as_apply(d)
return s
def test_sample():
u = scope.uniform(0, 1)
aa = as_apply(
dict(u=u, n=scope.normal(5, 0.1), l=[0, 1, scope.one_of(2, 3), u]))
print aa
dd = sample(aa, np.random.RandomState(3))
assert 0 < dd['u'] < 1
assert 4 < dd['n'] < 6
assert dd['u'] == dd['l'][3]
assert dd['l'][:2] == (0, 1)
assert dd['l'][2] in (2, 3)
def config0():
p0 = scope.uniform(0, 1)
p1 = scope.uniform(2, 3)
p2 = scope.one_of(-1, p0)
p3 = scope.one_of(-2, p1)
p4 = 1
p5 = [3, 4, p0]
p6 = scope.one_of(-3, p1)
d = locals()
d['p1'] = None # -- don't sample p1 all the time, only if p3 says so
s = as_apply(d)
return s
def config0():
p0 = scope.uniform(0, 1)
p1 = scope.uniform(2, 3)
p2 = scope.one_of(-1, p0)
p3 = scope.one_of(-2, p1)
p4 = 1
p5 = [3, 4, p0]
p6 = scope.one_of(-3, p1)
d = locals()
d['p1'] = None # -- don't sample p1 all the time, only if p3 says so
s = as_apply(d)
return s
def build_vals(self):
for node in self.dfs_nodes:
if node.name == "literal":
n_times = scope.len(self.idxs_memo[node])
vnode = scope.asarray(scope.repeat(n_times, node))
elif node in self.choice_memo:
# -- choices are natively vectorized
choices = self.choice_memo[node]
self.vals_memo[choices] = choices
# -- this stitches together the various sub-graphs
# to define the original node
vnode = scope.vchoice_merge(self.idxs_memo[node], self.choice_memo[node])
vnode.pos_args.extend(
[as_apply([self.idxs_memo[inode], self.vals_memo[inode]]) for inode in node.pos_args]
)
else:
vnode = scope.idxs_map(self.idxs_memo[node], node.name)
vnode.pos_args.extend(node.pos_args)
vnode.named_args.extend(node.named_args)
for arg in node.inputs():
vnode.replace_input(arg, as_apply([self.idxs_memo[arg], self.vals_memo[arg]]))
self.vals_memo[node] = vnode
def test4(self):
self.expr = as_apply(dict(p0=uniform(0, 1) + normal(0, 1)))
self.wanted = [[('p0.arg:0', [0], [0.69646918559786164]),
('p0.arg:1', [0], [-0.95209720686132215])],
[('p0.arg:0', [1], [0.55131476908289123]),
('p0.arg:1', [1], [-0.74544105948265826])],
[('p0.arg:0', [2], [0.71946896978556307]),
('p0.arg:1', [2], [0.32210606833962163])],
[('p0.arg:0', [3], [0.68482973858486329]),
('p0.arg:1', [3], [-0.0515177209393851])],
[('p0.arg:0', [4], [0.48093190148436094]),
('p0.arg:1', [4], [-1.6193000650367457])]]
self.foo()
def evaluate(self, config, ctrl, attach_attachments=True):
memo = self.memo_from_config(config)
self.use_obj_for_literal_in_memo(ctrl, base.Ctrl, memo)
if self.rng is not None and not self.installed_rng:
# -- N.B. this modifies the expr graph in-place
# XXX this feels wrong
self.expr = recursive_set_rng_kwarg(self.expr,
pyll.as_apply(self.rng))
self.installed_rng = True
if self.pass_expr_memo_ctrl:
rval = self.fn(
expr=self.expr,
memo=memo,
ctrl=ctrl,
*self.args)
else:
# -- the "work" of evaluating `config` can be written
# either into the pyll part (self.expr)
# or the normal Python part (self.fn)
pyll_rval = pyll.rec_eval(self.expr, memo=memo,
print_node_on_error=self.rec_eval_print_node_on_error)
rval = self.fn(pyll_rval, *self.args)
if isinstance(rval, (float, int, np.number)):
dict_rval = {'loss': rval}
elif isinstance(rval, (dict,)):
dict_rval = rval
if 'loss' not in dict_rval:
raise ValueError('dictionary must have "loss" key',
dict_rval.keys())
else:
raise TypeError('invalid return type (neither number nor dict)', rval)
if dict_rval['loss'] is not None:
# -- fail if cannot be cast to float
dict_rval['loss'] = float(dict_rval['loss'])
dict_rval.setdefault('status', base.STATUS_OK)
if dict_rval['status'] not in base.STATUS_STRINGS:
raise ValueError('invalid status string', dict_rval['status'])
if attach_attachments:
attachments = dict_rval.pop('attachments', {})
for key, val in attachments.items():
ctrl.attachments[key] = val
# -- don't do this here because SON-compatibility is only a requirement
# for trials destined for a mongodb. In-memory rvals can contain
# anything.
#return base.SONify(dict_rval)
return dict_rval
def test4(self):
self.expr = as_apply(dict(p0=uniform(0, 1) + normal(0, 1)))
self.wanted = [
[('p0.arg:0', [0], [0.69646918559786164]),
('p0.arg:1', [0], [-0.95209720686132215])],
[('p0.arg:0', [1], [0.55131476908289123]),
('p0.arg:1', [1], [-0.74544105948265826])],
[('p0.arg:0', [2], [0.71946896978556307]),
('p0.arg:1', [2], [0.32210606833962163])],
[('p0.arg:0', [3], [0.68482973858486329]),
('p0.arg:1', [3], [-0.0515177209393851])],
[('p0.arg:0', [4], [0.48093190148436094]),
('p0.arg:1', [4], [-1.6193000650367457])]]
self.foo()
def test6(self):
p0 = uniform(0, 1)
self.expr = as_apply(dict(p0=p0, p1=normal(p0, 1)))
self.wanted = [[('p0', [0], [0.69646918559786164]),
('p1', [0], [-0.25562802126346051])],
[('p0', [1], [0.55131476908289123]),
('p1', [1], [-0.19412629039976703])],
[('p0', [2], [0.71946896978556307]),
('p1', [2], [1.0415750381251847])],
[('p0', [3], [0.68482973858486329]),
('p1', [3], [0.63331201764547818])],
[('p0', [4], [0.48093190148436094]),
('p1', [4], [-1.1383681635523848])]]
self.foo()
def evaluate(self, config, ctrl, attach_attachments=True):
memo = self.memo_from_config(config)
self.use_obj_for_literal_in_memo(ctrl, base.Ctrl, memo)
if self.rng is not None and not self.installed_rng:
# -- N.B. this modifies the expr graph in-place
# XXX this feels wrong
self.expr = recursive_set_rng_kwarg(self.expr,
pyll.as_apply(self.rng))
self.installed_rng = True
if self.pass_expr_memo_ctrl:
rval = self.fn(expr=self.expr, memo=memo, ctrl=ctrl)
else:
# -- the "work" of evaluating `config` can be written
# either into the pyll part (self.expr)
# or the normal Python part (self.fn)
pyll_rval = pyll.rec_eval(
self.expr,
memo=memo,
print_node_on_error=self.rec_eval_print_node_on_error)
rval = self.fn(pyll_rval)
if isinstance(rval, (float, int, np.number)):
dict_rval = {'loss': rval}
elif isinstance(rval, (dict, )):
dict_rval = rval
if 'loss' not in dict_rval:
raise ValueError('dictionary must have "loss" key',
dict_rval.keys())
else:
raise TypeError('invalid return type (neither number nor dict)',
rval)
if dict_rval['loss'] is not None:
# -- fail if cannot be cast to float
dict_rval['loss'] = float(dict_rval['loss'])
dict_rval.setdefault('status', base.STATUS_OK)
if dict_rval['status'] not in base.STATUS_STRINGS:
raise ValueError('invalid status string', dict_rval['status'])
if attach_attachments:
attachments = dict_rval.pop('attachments', {})
for key, val in attachments.items():
ctrl.attachments[key] = val
# -- don't do this here because SON-compatibility is only a requirement
# for trials destined for a mongodb. In-memory rvals can contain
# anything.
#return base.SONify(dict_rval)
return dict_rval
def expr_to_config(expr, conditions, hps):
"""
Populate dictionary `hps` with the hyperparameters in pyll graph `expr`
and conditions for participation in the evaluation of `expr`.
Arguments:
expr - a pyll expression root.
conditions - a tuple of conditions (`Cond`) that must be True for
`expr` to be evaluated.
hps - dictionary to populate
Creates `hps` dictionary:
label -> { 'node': apply node of hyperparameter distribution,
'conditions': `conditions` + tuple,
'label': label
}
"""
expr = as_apply(expr)
if conditions is None:
conditions = ()
assert isinstance(expr, Apply)
if expr.name == 'switch':
idx = expr.inputs()[0]
options = expr.inputs()[1:]
assert idx.name == 'hyperopt_param'
assert idx.arg['obj'].name in (
'randint', # -- in case of hp.choice
'categorical', # -- in case of hp.pchoice
)
expr_to_config(idx, conditions, hps)
for ii, opt in enumerate(options):
expr_to_config(opt,
conditions + (EQ(idx.arg['label'].obj, ii),),
hps)
elif expr.name == 'hyperopt_param':
label = expr.arg['label'].obj
if label in hps:
if hps[label]['node'] != expr.arg['obj']:
raise DuplicateLabel(label)
hps[label]['conditions'].add(conditions)
else:
hps[label] = {'node': expr.arg['obj'],
'conditions': set((conditions,)),
'label': label,
}
else:
for ii in expr.inputs():
expr_to_config(ii, conditions, hps)
def expr_to_config(expr, conditions, hps):
"""
Populate dictionary `hps` with the hyperparameters in pyll graph `expr`
and conditions for participation in the evaluation of `expr`.
Arguments:
expr - a pyll expression root.
conditions - a tuple of conditions (`Cond`) that must be True for
`expr` to be evaluated.
hps - dictionary to populate
Creates `hps` dictionary:
label -> { 'node': apply node of hyperparameter distribution,
'conditions': `conditions` + tuple,
'label': label
}
"""
expr = as_apply(expr)
if conditions is None:
conditions = ()
assert isinstance(expr, Apply)
if expr.name == 'switch':
idx = expr.inputs()[0]
options = expr.inputs()[1:]
assert idx.name == 'hyperopt_param'
assert idx.arg['obj'].name in (
'randint', # -- in case of hp.choice
'categorical', # -- in case of hp.pchoice
)
expr_to_config(idx, conditions, hps)
for ii, opt in enumerate(options):
expr_to_config(opt, conditions + (EQ(idx.arg['label'].obj, ii), ),
hps)
elif expr.name == 'hyperopt_param':
label = expr.arg['label'].obj
if label in hps:
if hps[label]['node'] != expr.arg['obj']:
raise DuplicateLabel(label)
hps[label]['conditions'].add(conditions)
else:
hps[label] = {
'node': expr.arg['obj'],
'conditions': set((conditions, )),
'label': label,
}
else:
for ii in expr.inputs():
expr_to_config(ii, conditions, hps)
def __init__(self,
fn,
expr,
args=[],
workdir=None,
pass_expr_memo_ctrl=None,
**bandit_kwargs):
self.cmd = ('domain_attachment', 'FMinIter_Domain')
self.fn = fn
self.expr = expr
self.args = args
if pass_expr_memo_ctrl is None:
self.pass_expr_memo_ctrl = getattr(fn, 'fmin_pass_expr_memo_ctrl',
False)
else:
self.pass_expr_memo_ctrl = pass_expr_memo_ctrl
base.Bandit.__init__(self, expr, do_checks=False, **bandit_kwargs)
# -- This code was stolen from base.BanditAlgo, a class which may soon
# be gone
self.workdir = workdir
self.s_new_ids = pyll.Literal('new_ids') # -- list at eval-time
before = pyll.dfs(self.expr)
# -- raises exception if expr contains cycles
pyll.toposort(self.expr)
vh = self.vh = VectorizeHelper(self.expr, self.s_new_ids)
# -- raises exception if v_expr contains cycles
pyll.toposort(vh.v_expr)
idxs_by_label = vh.idxs_by_label()
vals_by_label = vh.vals_by_label()
after = pyll.dfs(self.expr)
# -- try to detect if VectorizeHelper screwed up anything inplace
assert before == after
assert set(idxs_by_label.keys()) == set(vals_by_label.keys())
assert set(idxs_by_label.keys()) == set(self.params.keys())
# -- make the graph runnable and SON-encodable
# N.B. operates inplace
self.s_idxs_vals = recursive_set_rng_kwarg(
pyll.scope.pos_args(idxs_by_label, vals_by_label),
pyll.as_apply(self.rng))
# -- raises an exception if no topological ordering exists
pyll.toposort(self.s_idxs_vals)
def test_lnorm():
G = scope
choice = G.choice
uniform = G.uniform
quantized_uniform = G.quniform
inker_size = quantized_uniform(low=0, high=7.99, q=2) + 3
# -- test that it runs
lnorm = as_apply({'kwargs': {'inker_shape' : (inker_size, inker_size),
'outker_shape' : (inker_size, inker_size),
'remove_mean' : choice([0, 1]),
'stretch' : uniform(low=0, high=10),
'threshold' : uniform(
low=.1 / np.sqrt(10.),
high=10 * np.sqrt(10))
}})
print lnorm
print 'len', len(str(lnorm))
def work(self):
"""Test that all prior samplers are gone"""
tpe_algo = TreeParzenEstimator(self.bandit)
foo = pyll.as_apply(
[tpe_algo.post_below['idxs'], tpe_algo.post_below['vals']])
prior_names = [
'uniform',
'quniform',
'loguniform',
'qloguniform',
'normal',
'qnormal',
'lognormal',
'qlognormal',
'randint',
]
for node in pyll.dfs(foo):
assert node.name not in prior_names
def space_eval(space, hp_assignment):
"""Compute a point in a search space from a hyperparameter assignment.
Parameters:
-----------
space - a pyll graph involving hp nodes (see `pyll_utils`).
hp_assignment - a dictionary mapping hp node labels to values.
"""
space = pyll.as_apply(space)
nodes = pyll.toposort(space)
memo = {}
for node in nodes:
if node.name == 'hyperopt_param':
label = node.arg['label'].eval()
if label in hp_assignment:
memo[node] = hp_assignment[label]
rval = pyll.rec_eval(space, memo=memo)
return rval
def space_eval(space, hp_assignment):
"""Compute a point in a search space from a hyperparameter assignment.
Parameters:
-----------
space - a pyll graph involving hp nodes (see `pyll_utils`).
hp_assignment - a dictionary mapping hp node labels to values.
"""
space = pyll.as_apply(space)
nodes = pyll.toposort(space)
memo = {}
for node in nodes:
if node.name == 'hyperopt_param':
label = node.arg['label'].eval()
if label in hp_assignment:
memo[node] = hp_assignment[label]
rval = pyll.rec_eval(space, memo=memo)
return rval
def work(self):
"""Test that all prior samplers are gone"""
tpe_algo = TreeParzenEstimator(self.bandit)
foo = pyll.as_apply([
tpe_algo.post_below['idxs'],
tpe_algo.post_below['vals']])
prior_names = [
'uniform',
'quniform',
'loguniform',
'qloguniform',
'normal',
'qnormal',
'lognormal',
'qlognormal',
'randint',
]
for node in pyll.dfs(foo):
assert node.name not in prior_names
def test_lnorm():
G = scope
choice = G.choice
uniform = G.uniform
quantized_uniform = G.quniform
inker_size = quantized_uniform(low=0, high=7.99, q=2) + 3
# -- test that it runs
lnorm = as_apply({
'kwargs': {
'inker_shape': (inker_size, inker_size),
'outker_shape': (inker_size, inker_size),
'remove_mean': choice([0, 1]),
'stretch': uniform(low=0, high=10),
'threshold': uniform(low=.1 / np.sqrt(10.), high=10 * np.sqrt(10))
}
})
print lnorm
print 'len', len(str(lnorm))
def __init__(self, fn, expr, args=[],
workdir=None,
pass_expr_memo_ctrl=None,
**bandit_kwargs):
self.cmd = ('domain_attachment', 'FMinIter_Domain')
self.fn = fn
self.expr = expr
self.args = args
if pass_expr_memo_ctrl is None:
self.pass_expr_memo_ctrl = getattr(fn,
'fmin_pass_expr_memo_ctrl', False)
else:
self.pass_expr_memo_ctrl = pass_expr_memo_ctrl
base.Bandit.__init__(self, expr, do_checks=False, **bandit_kwargs)
# -- This code was stolen from base.BanditAlgo, a class which may soon
# be gone
self.workdir = workdir
self.s_new_ids = pyll.Literal('new_ids') # -- list at eval-time
before = pyll.dfs(self.expr)
# -- raises exception if expr contains cycles
pyll.toposort(self.expr)
vh = self.vh = VectorizeHelper(self.expr, self.s_new_ids)
# -- raises exception if v_expr contains cycles
pyll.toposort(vh.v_expr)
idxs_by_label = vh.idxs_by_label()
vals_by_label = vh.vals_by_label()
after = pyll.dfs(self.expr)
# -- try to detect if VectorizeHelper screwed up anything inplace
assert before == after
assert set(idxs_by_label.keys()) == set(vals_by_label.keys())
assert set(idxs_by_label.keys()) == set(self.params.keys())
# -- make the graph runnable and SON-encodable
# N.B. operates inplace
self.s_idxs_vals = recursive_set_rng_kwarg(
pyll.scope.pos_args(idxs_by_label, vals_by_label),
pyll.as_apply(self.rng))
# -- raises an exception if no topological ordering exists
pyll.toposort(self.s_idxs_vals)
def test7(self):
p0 = uniform(0, 1)
p1 = normal(0, 1)
self.expr = as_apply(dict(
p0=p0,
p1=p1,
p2=one_of(1, p0),
p3=one_of(2, p1, uniform(2, 3))))
self.n_randints = 2
self.wanted = [
[
('p0', [0], [0.71295532052322719]),
('p1', [0], [0.28297849805199204]),
('p2.randint', [0], [0]),
('p3.arg:2', [0], [2.719468969785563]),
('p3.randint', [0], [2])],
[
('p0', [1], [0.78002776191207912]),
('p1', [1], [-1.506294713918092]),
('p2.randint', [1], [1]),
('p3.arg:2', [], []),
('p3.randint', [1], [1])],
[
('p0', [2], [0.57969429702261011]),
('p1', [2], [1.6796003743035337]),
('p2.randint', [2], [0]),
('p3.arg:2', [], []),
('p3.randint', [2], [1])],
[
('p0', [3], [0.43857224467962441]),
('p1', [3], [-1.3058031267484451]),
('p2.randint', [3], [1]),
('p3.arg:2', [], []),
('p3.randint', [3], [1])],
[
('p0', [4], [0.39804425533043142]),
('p1', [4], [-0.91948540682140967]),
('p2.randint', [4], [0]),
('p3.arg:2', [], []),
('p3.randint', [4], [0])]]
self.foo()
def test_lnorm():
G = scope
choice = G.choice
uniform = G.uniform
quantized_uniform = G.quniform
inker_size = quantized_uniform(low=0, high=7.99, q=2) + 3
# -- test that it runs
lnorm = as_apply(
{
"kwargs": {
"inker_shape": (inker_size, inker_size),
"outker_shape": (inker_size, inker_size),
"remove_mean": choice([0, 1]),
"stretch": uniform(low=0, high=10),
"threshold": uniform(low=0.1 / np.sqrt(10.0), high=10 * np.sqrt(10)),
}
}
)
print lnorm
print "len", len(str(lnorm))
def evaluate(self, config, ctrl):
"""Return a result document
"""
memo = self.memo_from_config(config)
self.use_obj_for_literal_in_memo(ctrl, Ctrl, memo)
if self.rng is not None and not self.installed_rng:
# -- N.B. this modifies the expr graph in-place
# XXX this feels wrong
self.expr = recursive_set_rng_kwarg(self.expr, pyll.as_apply(self.rng))
self.installed_rng = True
try:
r_dct = pyll.rec_eval(self.expr, memo=memo)
except Exception, e:
n_match = 0
for match, match_pair in self.exceptions:
if match(e):
r_dct = match_pair(e)
n_match += 1
break
if n_match == 0:
raise
def evaluate(self, config, ctrl):
"""Return a result document
"""
memo = self.memo_from_config(config)
self.use_obj_for_literal_in_memo(ctrl, Ctrl, memo)
if self.rng is not None and not self.installed_rng:
# -- N.B. this modifies the expr graph in-place
# XXX this feels wrong
self.expr = recursive_set_rng_kwarg(self.expr,
pyll.as_apply(self.rng))
self.installed_rng = True
try:
r_dct = pyll.rec_eval(self.expr, memo=memo)
except Exception, e:
n_match = 0
for match, match_pair in self.exceptions:
if match(e):
r_dct = match_pair(e)
n_match += 1
break
if n_match == 0:
raise
def expr_to_config(expr, conditions, hps):
"""
Populate dictionary `hps` with the hyperparameters in pyll graph `expr`
and conditions for participation in the evaluation of `expr`.
Arguments:
expr - a pyll expression root.
conditions - a tuple of conditions (`Cond`) that must be True for
`expr` to be evaluated.
hps - dictionary to populate
Creates `hps` dictionary:
label -> { 'node': apply node of hyperparameter distribution,
'conditions': `conditions` + tuple,
'label': label
}
"""
expr = as_apply(expr)
if conditions is None:
conditions = ()
assert isinstance(expr, Apply)
_expr_to_config(expr, conditions, hps)
_remove_allpaths(hps, conditions)
def expr_to_config(expr, conditions, hps):
"""
Populate dictionary `hps` with the hyperparameters in pyll graph `expr`
and conditions for participation in the evaluation of `expr`.
Arguments:
expr - a pyll expression root.
conditions - a tuple of conditions (`Cond`) that must be True for
`expr` to be evaluated.
hps - dictionary to populate
Creates `hps` dictionary:
label -> { 'node': apply node of hyperparameter distribution,
'conditions': `conditions` + tuple,
'label': label
}
"""
expr = as_apply(expr)
if conditions is None:
conditions = ()
assert isinstance(expr, Apply)
_expr_to_config(expr, conditions, hps)
_remove_allpaths(hps, conditions)
def test_vectorize_config0():
p0 = hp_uniform('p0', 0, 1)
p1 = hp_loguniform('p1', 2, 3)
p2 = hp_choice('p2', [-1, p0])
p3 = hp_choice('p3', [-2, p1])
p4 = 1
p5 = [3, 4, p0]
p6 = hp_choice('p6', [-3, p1])
d = locals()
d['p1'] = None # -- don't sample p1 all the time, only if p3 says so
config = as_apply(d)
N = as_apply('N:TBA')
expr = config
expr_idxs = scope.range(N)
vh = VectorizeHelper(expr, expr_idxs, build=True)
vconfig = vh.v_expr
full_output = as_apply([vconfig, vh.idxs_by_label(), vh.vals_by_label()])
if 1:
print '=' * 80
print 'VECTORIZED'
print full_output
print '\n' * 1
fo2 = replace_repeat_stochastic(full_output)
if 0:
print '=' * 80
print 'VECTORIZED STOCHASTIC'
print fo2
print '\n' * 1
new_vc = recursive_set_rng_kwarg(fo2, as_apply(np.random.RandomState(1)))
if 0:
print '=' * 80
print 'VECTORIZED STOCHASTIC WITH RNGS'
print new_vc
Nval = 10
foo, idxs, vals = rec_eval(new_vc, memo={N: Nval})
print 'foo[0]', foo[0]
print 'foo[1]', foo[1]
assert len(foo) == Nval
if 0: # XXX refresh these values to lock down sampler
assert foo[0] == {
'p0': 0.39676747423066994,
'p1': None,
'p2': 0.39676747423066994,
'p3': 2.1281244479293568,
'p4': 1,
'p5': (3, 4, 0.39676747423066994)
}
assert foo[1] != foo[2]
print idxs
print vals['p3']
print vals['p6']
print idxs['p1']
print vals['p1']
assert len(vals['p3']) == Nval
assert len(vals['p6']) == Nval
assert len(idxs['p1']) < Nval
p1d = dict(zip(idxs['p1'], vals['p1']))
for ii, (p3v, p6v) in enumerate(zip(vals['p3'], vals['p6'])):
if p3v == p6v == 0:
assert ii not in idxs['p1']
if p3v:
assert foo[ii]['p3'] == p1d[ii]
if p6v:
print 'p6', foo[ii]['p6'], p1d[ii]
assert foo[ii]['p6'] == p1d[ii]
def build_idxs_vals(self, node, wanted_idxs):
"""
This recursive procedure should be called on an output-node.
"""
checkpoint_asserts = False
def checkpoint():
if checkpoint_asserts:
self.assert_integrity_idxs_take()
if node in self.idxs_memo:
toposort(self.idxs_memo[node])
if node in self.take_memo:
for take in self.take_memo[node]:
toposort(take)
checkpoint()
# wanted_idxs are fixed, whereas idxs_memo
# is full of unions, that can grow in subsequent recursive
# calls to build_idxs_vals with node as argument.
assert wanted_idxs != self.idxs_memo.get(node)
# -- easy exit case
if node.name == 'hyperopt_param':
# -- ignore, not vectorizing
return self.build_idxs_vals(node.arg['obj'], wanted_idxs)
# -- easy exit case
elif node.name == 'hyperopt_result':
# -- ignore, not vectorizing
return self.build_idxs_vals(node.arg['obj'], wanted_idxs)
# -- literal case: always take from universal set
elif node.name == 'literal':
if node in self.idxs_memo:
all_idxs, all_vals = self.take_memo[node][0].pos_args[:2]
wanted_vals = scope.idxs_take(all_idxs, all_vals, wanted_idxs)
self.take_memo[node].append(wanted_vals)
checkpoint()
else:
# -- initialize idxs_memo to full set
all_idxs = self.expr_idxs
n_times = scope.len(all_idxs)
# -- put array_union into graph for consistency, though it is
# not necessary
all_idxs = scope.array_union(all_idxs)
self.idxs_memo[node] = all_idxs
all_vals = scope.asarray(scope.repeat(n_times, node))
wanted_vals = scope.idxs_take(all_idxs, all_vals, wanted_idxs)
assert node not in self.take_memo
self.take_memo[node] = [wanted_vals]
checkpoint()
return wanted_vals
# -- switch case: complicated
elif node.name == 'switch':
if (node in self.idxs_memo
and wanted_idxs in self.idxs_memo[node].pos_args):
# -- phew, easy case
all_idxs, all_vals = self.take_memo[node][0].pos_args[:2]
wanted_vals = scope.idxs_take(all_idxs, all_vals, wanted_idxs)
self.take_memo[node].append(wanted_vals)
checkpoint()
else:
# -- we need to add some indexes
if node in self.idxs_memo:
all_idxs = self.idxs_memo[node]
assert all_idxs.name == 'array_union'
all_idxs.pos_args.append(wanted_idxs)
else:
all_idxs = scope.array_union(wanted_idxs)
choice = node.pos_args[0]
all_choices = self.build_idxs_vals(choice, all_idxs)
options = node.pos_args[1:]
args_idxs = scope.vchoice_split(all_idxs, all_choices,
len(options))
all_vals = scope.vchoice_merge(all_idxs, all_choices)
for opt_ii, idxs_ii in zip(options, args_idxs):
all_vals.pos_args.append(
as_apply([
idxs_ii,
self.build_idxs_vals(opt_ii, idxs_ii),
]))
wanted_vals = scope.idxs_take(
all_idxs, # -- may grow in future
all_vals, # -- may be replaced in future
wanted_idxs) # -- fixed.
if node in self.idxs_memo:
assert self.idxs_memo[node].name == 'array_union'
self.idxs_memo[node].pos_args.append(wanted_idxs)
for take in self.take_memo[node]:
assert take.name == 'idxs_take'
take.pos_args[1] = all_vals
self.take_memo[node].append(wanted_vals)
else:
self.idxs_memo[node] = all_idxs
self.take_memo[node] = [wanted_vals]
checkpoint()
# -- general case
else:
# -- this is a general node.
# It is generally handled with idxs_memo,
# but vectorize_stochastic may immediately transform it into
# a more compact form.
if (node in self.idxs_memo
and wanted_idxs in self.idxs_memo[node].pos_args):
# -- phew, easy case
for take in self.take_memo[node]:
if take.pos_args[2] == wanted_idxs:
return take
raise NotImplementedError('how did this happen?')
#all_idxs, all_vals = self.take_memo[node][0].pos_args[:2]
#wanted_vals = scope.idxs_take(all_idxs, all_vals, wanted_idxs)
#self.take_memo[node].append(wanted_vals)
#checkpoint()
else:
# XXX
# -- determine if wanted_idxs is actually a subset of the idxs
# that we are already computing. This is not only an
# optimization, but prevents the creation of cycles, which
# would otherwise occur if we have a graph of the form
# switch(f(a), g(a), 0). If there are other switches inside f
# and g, does this get trickier?
# -- assume we need to add some indexes
checkpoint()
if node in self.idxs_memo:
all_idxs = self.idxs_memo[node]
else:
all_idxs = scope.array_union(wanted_idxs)
checkpoint()
all_vals = scope.idxs_map(all_idxs, node.name)
for ii, aa in enumerate(node.pos_args):
all_vals.pos_args.append(
as_apply(
[all_idxs,
self.build_idxs_vals(aa, all_idxs)]))
checkpoint()
for ii, (nn, aa) in enumerate(node.named_args):
all_vals.named_args.append([
nn,
as_apply(
[all_idxs,
self.build_idxs_vals(aa, all_idxs)])
])
checkpoint()
all_vals = vectorize_stochastic(all_vals)
checkpoint()
wanted_vals = scope.idxs_take(
all_idxs, # -- may grow in future
all_vals, # -- may be replaced in future
wanted_idxs) # -- fixed.
if node in self.idxs_memo:
assert self.idxs_memo[node].name == 'array_union'
self.idxs_memo[node].pos_args.append(wanted_idxs)
toposort(self.idxs_memo[node])
# -- this catches the cycle bug mentioned above
for take in self.take_memo[node]:
assert take.name == 'idxs_take'
take.pos_args[1] = all_vals
self.take_memo[node].append(wanted_vals)
else:
self.idxs_memo[node] = all_idxs
self.take_memo[node] = [wanted_vals]
checkpoint()
return wanted_vals
class Bandit(object):
"""Specification of bandit problem.
template - pyll specification of search domain
evaluate - interruptible/checkpt calling convention for evaluation routine
"""
# -- the Ctrl object is not used directly, but rather
# a live Ctrl instance is inserted for the pyll_ctrl
# in self.evaluate so that it can be accessed from within
# the pyll graph describing the search space.
pyll_ctrl = pyll.as_apply(Ctrl)
exceptions = []
def __init__(
self,
expr,
name=None,
rseed=None,
loss_target=None,
exceptions=None,
do_checks=True,
):
if do_checks:
if isinstance(expr, pyll.Apply):
self.expr = expr
# XXX: verify that expr is a dictionary with the right keys,
# then refactor the code below
elif isinstance(expr, dict):
if 'loss' not in expr:
raise ValueError('expr must define a loss')
if 'status' not in expr:
expr['status'] = STATUS_OK
self.expr = pyll.as_apply(expr)
else:
raise TypeError('expr must be a dictionary')
else:
self.expr = pyll.as_apply(expr)
self.params = {}
for node in pyll.dfs(self.expr):
if node.name == 'hyperopt_param':
label = node.arg['label'].obj
if label in self.params:
raise DuplicateLabel(label)
self.params[label] = node.arg['obj']
if exceptions is not None:
self.exceptions = exceptions
self.loss_target = loss_target
self.installed_rng = False
if rseed is None:
self.rng = None
else:
self.rng = np.random.RandomState(rseed)
self.name = name
def memo_from_config(self, config):
memo = {}
for node in pyll.dfs(self.expr):
if node.name == 'hyperopt_param':
label = node.arg['label'].obj
# -- hack because it's not really garbagecollected
# this does have the desired effect of crashing the
# function if rec_eval actually needs a value that
# the the optimization algorithm thought to be unnecessary
memo[node] = config.get(label, pyll.base.GarbageCollected)
return memo
def short_str(self):
return self.__class__.__name__
def use_obj_for_literal_in_memo(self, obj, lit, memo):
return use_obj_for_literal_in_memo(self.expr, obj, lit, memo)
def evaluate(self, config, ctrl):
"""Return a result document
"""
memo = self.memo_from_config(config)
self.use_obj_for_literal_in_memo(ctrl, Ctrl, memo)
if self.rng is not None and not self.installed_rng:
# -- N.B. this modifies the expr graph in-place
# XXX this feels wrong
self.expr = recursive_set_rng_kwarg(self.expr,
pyll.as_apply(self.rng))
self.installed_rng = True
try:
r_dct = pyll.rec_eval(self.expr, memo=memo)
except Exception, e:
n_match = 0
for match, match_pair in self.exceptions:
if match(e):
r_dct = match_pair(e)
n_match += 1
break
if n_match == 0:
raise
assert 'loss' in r_dct
if r_dct['loss'] is not None:
# -- assert that it can at least be cast to float
float(r_dct['loss'])
if r_dct['status'] not in STATUS_STRINGS:
raise ValueError('invalid status string', r_dct['status'])
return r_dct
def __init__(
self,
fn,
expr,
workdir=None,
pass_expr_memo_ctrl=None,
name=None,
loss_target=None,
):
"""
Paramaters
----------
fn : callable
This stores the `fn` argument to `fmin`. (See `hyperopt.fmin.fmin`)
expr : hyperopt.pyll.Apply
This is the `space` argument to `fmin`. (See `hyperopt.fmin.fmin`)
workdir : string (or None)
If non-None, the current working directory will be `workdir`while
`expr` and `fn` are evaluated. (XXX Currently only respected by
jobs run via MongoWorker)
pass_expr_memo_ctrl : bool
If True, `fn` will be called like this:
`fn(self.expr, memo, ctrl)`,
where `memo` is a dictionary mapping `Apply` nodes to their
computed values, and `ctrl` is a `Ctrl` instance for communicating
with a Trials database. This lower-level calling convention is
useful if you want to call e.g. `hyperopt.pyll.rec_eval` yourself
in some customized way.
name : string (or None)
Label, used for pretty-printing.
loss_target : float (or None)
The actual or estimated minimum of `fn`.
Some optimization algorithms may behave differently if their first
objective is to find an input that achieves a certain value,
rather than the more open-ended objective of pure minimization.
XXX: Move this from Domain to be an fmin arg.
"""
self.fn = fn
if pass_expr_memo_ctrl is None:
self.pass_expr_memo_ctrl = getattr(fn, 'fmin_pass_expr_memo_ctrl',
False)
else:
self.pass_expr_memo_ctrl = pass_expr_memo_ctrl
self.expr = pyll.as_apply(expr)
self.params = {}
for node in pyll.dfs(self.expr):
if node.name == 'hyperopt_param':
label = node.arg['label'].obj
if label in self.params:
raise DuplicateLabel(label)
self.params[label] = node.arg['obj']
self.loss_target = loss_target
self.name = name
self.workdir = workdir
self.s_new_ids = pyll.Literal('new_ids') # -- list at eval-time
before = pyll.dfs(self.expr)
# -- raises exception if expr contains cycles
pyll.toposort(self.expr)
vh = self.vh = VectorizeHelper(self.expr, self.s_new_ids)
# -- raises exception if v_expr contains cycles
pyll.toposort(vh.v_expr)
idxs_by_label = vh.idxs_by_label()
vals_by_label = vh.vals_by_label()
after = pyll.dfs(self.expr)
# -- try to detect if VectorizeHelper screwed up anything inplace
assert before == after
assert set(idxs_by_label.keys()) == set(vals_by_label.keys())
assert set(idxs_by_label.keys()) == set(self.params.keys())
self.s_rng = pyll.Literal('rng-placeholder')
# -- N.B. operates inplace:
self.s_idxs_vals = recursive_set_rng_kwarg(
pyll.scope.pos_args(idxs_by_label, vals_by_label), self.s_rng)
# -- raises an exception if no topological ordering exists
pyll.toposort(self.s_idxs_vals)
# -- Protocol for serialization.
# self.cmd indicates to e.g. MongoWorker how this domain
# should be [un]serialized.
# XXX This mechanism deserves review as support for ipython
# workers improves.
self.cmd = ('domain_attachment', 'FMinIter_Domain')
class Domain(object):
"""Picklable representation of search space and evaluation function.
"""
rec_eval_print_node_on_error = False
# -- the Ctrl object is not used directly, but rather
# a live Ctrl instance is inserted for the pyll_ctrl
# in self.evaluate so that it can be accessed from within
# the pyll graph describing the search space.
pyll_ctrl = pyll.as_apply(Ctrl)
def __init__(
self,
fn,
expr,
workdir=None,
pass_expr_memo_ctrl=None,
name=None,
loss_target=None,
):
"""
Paramaters
----------
fn : callable
This stores the `fn` argument to `fmin`. (See `hyperopt.fmin.fmin`)
expr : hyperopt.pyll.Apply
This is the `space` argument to `fmin`. (See `hyperopt.fmin.fmin`)
workdir : string (or None)
If non-None, the current working directory will be `workdir`while
`expr` and `fn` are evaluated. (XXX Currently only respected by
jobs run via MongoWorker)
pass_expr_memo_ctrl : bool
If True, `fn` will be called like this:
`fn(self.expr, memo, ctrl)`,
where `memo` is a dictionary mapping `Apply` nodes to their
computed values, and `ctrl` is a `Ctrl` instance for communicating
with a Trials database. This lower-level calling convention is
useful if you want to call e.g. `hyperopt.pyll.rec_eval` yourself
in some customized way.
name : string (or None)
Label, used for pretty-printing.
loss_target : float (or None)
The actual or estimated minimum of `fn`.
Some optimization algorithms may behave differently if their first
objective is to find an input that achieves a certain value,
rather than the more open-ended objective of pure minimization.
XXX: Move this from Domain to be an fmin arg.
"""
self.fn = fn
if pass_expr_memo_ctrl is None:
self.pass_expr_memo_ctrl = getattr(fn, 'fmin_pass_expr_memo_ctrl',
False)
else:
self.pass_expr_memo_ctrl = pass_expr_memo_ctrl
self.expr = pyll.as_apply(expr)
self.params = {}
for node in pyll.dfs(self.expr):
if node.name == 'hyperopt_param':
label = node.arg['label'].obj
if label in self.params:
raise DuplicateLabel(label)
self.params[label] = node.arg['obj']
self.loss_target = loss_target
self.name = name
self.workdir = workdir
self.s_new_ids = pyll.Literal('new_ids') # -- list at eval-time
before = pyll.dfs(self.expr)
# -- raises exception if expr contains cycles
pyll.toposort(self.expr)
vh = self.vh = VectorizeHelper(self.expr, self.s_new_ids)
# -- raises exception if v_expr contains cycles
pyll.toposort(vh.v_expr)
idxs_by_label = vh.idxs_by_label()
vals_by_label = vh.vals_by_label()
after = pyll.dfs(self.expr)
# -- try to detect if VectorizeHelper screwed up anything inplace
assert before == after
assert set(idxs_by_label.keys()) == set(vals_by_label.keys())
assert set(idxs_by_label.keys()) == set(self.params.keys())
self.s_rng = pyll.Literal('rng-placeholder')
# -- N.B. operates inplace:
self.s_idxs_vals = recursive_set_rng_kwarg(
pyll.scope.pos_args(idxs_by_label, vals_by_label), self.s_rng)
# -- raises an exception if no topological ordering exists
pyll.toposort(self.s_idxs_vals)
# -- Protocol for serialization.
# self.cmd indicates to e.g. MongoWorker how this domain
# should be [un]serialized.
# XXX This mechanism deserves review as support for ipython
# workers improves.
self.cmd = ('domain_attachment', 'FMinIter_Domain')
def memo_from_config(self, config):
memo = {}
for node in pyll.dfs(self.expr):
if node.name == 'hyperopt_param':
label = node.arg['label'].obj
# -- hack because it's not really garbagecollected
# this does have the desired effect of crashing the
# function if rec_eval actually needs a value that
# the the optimization algorithm thought to be unnecessary
memo[node] = config.get(label, pyll.base.GarbageCollected)
return memo
def evaluate(self, config, ctrl, attach_attachments=True):
memo = self.memo_from_config(config)
use_obj_for_literal_in_memo(self.expr, ctrl, Ctrl, memo)
if self.pass_expr_memo_ctrl:
rval = self.fn(expr=self.expr, memo=memo, ctrl=ctrl)
else:
# -- the "work" of evaluating `config` can be written
# either into the pyll part (self.expr)
# or the normal Python part (self.fn)
pyll_rval = pyll.rec_eval(
self.expr,
memo=memo,
print_node_on_error=self.rec_eval_print_node_on_error)
rval = self.fn(pyll_rval)
if isinstance(rval, (float, int, np.number)):
dict_rval = {'loss': float(rval), 'status': STATUS_OK}
else:
dict_rval = dict(rval)
status = dict_rval['status']
if status not in STATUS_STRINGS:
raise InvalidResultStatus(dict_rval)
if status == STATUS_OK:
# -- make sure that the loss is present and valid
try:
dict_rval['loss'] = float(dict_rval['loss'])
except (TypeError, KeyError):
raise InvalidLoss(dict_rval)
if attach_attachments:
attachments = dict_rval.pop('attachments', {})
for key, val in attachments.items():
ctrl.attachments[key] = val
# -- don't do this here because SON-compatibility is only a requirement
# for trials destined for a mongodb. In-memory rvals can contain
# anything.
#return base.SONify(dict_rval)
return dict_rval
def evaluate_async(
self,
config,
ctrl,
attach_attachments=True,
):
'''
this is the first part of async evaluation for ipython parallel engines (see ipy.py)
This breaks evaluate into two parts to allow for the apply_async call
to only pass the objective function and arguments.
'''
memo = self.memo_from_config(config)
use_obj_for_literal_in_memo(self.expr, ctrl, Ctrl, memo)
if self.pass_expr_memo_ctrl:
rval = self.fn(expr=self.expr, memo=memo, ctrl=ctrl)
else:
# -- the "work" of evaluating `config` can be written
# either into the pyll part (self.expr)
# or the normal Python part (self.fn)
pyll_rval = pyll.rec_eval(
self.expr,
memo=memo,
print_node_on_error=self.rec_eval_print_node_on_error)
return (self.fn, pyll_rval)
def evaluate_async2(self, rval, ctrl, attach_attachments=True):
'''
this is the second part of async evaluation for ipython parallel engines (see ipy.py)
'''
if isinstance(rval, (float, int, np.number)):
dict_rval = {'loss': float(rval), 'status': STATUS_OK}
else:
dict_rval = dict(rval)
status = dict_rval['status']
if status not in STATUS_STRINGS:
raise InvalidResultStatus(dict_rval)
if status == STATUS_OK:
# -- make sure that the loss is present and valid
try:
dict_rval['loss'] = float(dict_rval['loss'])
except (TypeError, KeyError):
raise InvalidLoss(dict_rval)
if attach_attachments:
attachments = dict_rval.pop('attachments', {})
for key, val in attachments.items():
ctrl.attachments[key] = val
# -- don't do this here because SON-compatibility is only a requirement
# for trials destined for a mongodb. In-memory rvals can contain
# anything.
#return base.SONify(dict_rval)
return dict_rval
def short_str(self):
return 'Domain{%s}' % str(self.fn)
def loss(self, result, config=None):
"""Extract the scalar-valued loss from a result document
"""
return result.get('loss', None)
def loss_variance(self, result, config=None):
"""Return the variance in the estimate of the loss"""
return result.get('loss_variance', 0.0)
def true_loss(self, result, config=None):
"""Return a true loss, in the case that the `loss` is a surrogate"""
# N.B. don't use get() here, it evaluates self.loss un-necessarily
try:
return result['true_loss']
except KeyError:
return self.loss(result, config=config)
def true_loss_variance(self, config=None):
"""Return the variance in true loss,
in the case that the `loss` is a surrogate.
"""
raise NotImplementedError()
def status(self, result, config=None):
"""Extract the job status from a result document
"""
return result['status']
def new_result(self):
"""Return a JSON-encodable object
to serve as the 'result' for new jobs.
"""
return {'status': STATUS_NEW}
def build_posterior(specs, prior_idxs, prior_vals, obs_idxs, obs_vals,
oloss_idxs, oloss_vals, oloss_gamma, prior_weight):
"""
This method clones a posterior inference graph by iterating forward in
topological order, and replacing prior random-variables (prior_vals) with
new posterior distributions that make use of observations (obs_vals).
"""
assert all(isinstance(arg, pyll.Apply)
for arg in [oloss_idxs, oloss_vals, oloss_gamma])
expr = pyll.as_apply([specs, prior_idxs, prior_vals])
nodes = pyll.dfs(expr)
# build the joint posterior distribution as the values in this memo
memo = {}
# map prior RVs to observations
obs_memo = {}
for nid in prior_vals:
# construct the leading args for each call to adaptive_parzen_sampler
# which will permit the "adaptive parzen samplers" to adapt to the
# correct samples.
obs_below, obs_above = scope.ap_filter_trials(
obs_idxs[nid], obs_vals[nid],
oloss_idxs, oloss_vals, oloss_gamma)
obs_memo[prior_vals[nid]] = [obs_below, obs_above]
for node in nodes:
if node not in memo:
new_inputs = [memo[arg] for arg in node.inputs()]
if node in obs_memo:
# -- this case corresponds to an observed Random Var
# node.name is a distribution like "normal", "randint", etc.
obs_below, obs_above = obs_memo[node]
aa = [memo[a] for a in node.pos_args]
fn = adaptive_parzen_samplers[node.name]
b_args = [obs_below, prior_weight] + aa
named_args = [[kw, memo[arg]]
for (kw, arg) in node.named_args]
b_post = fn(*b_args, **dict(named_args))
a_args = [obs_above, prior_weight] + aa
a_post = fn(*a_args, **dict(named_args))
assert a_post.name == b_post.name
fn_lpdf = getattr(scope, a_post.name + '_lpdf')
#print fn_lpdf
a_kwargs = dict([(n, a) for n, a in a_post.named_args
if n not in ('rng', 'size')])
b_kwargs = dict([(n, a) for n, a in b_post.named_args
if n not in ('rng', 'size')])
# calculate the llik of b_post under both distributions
below_llik = fn_lpdf(*([b_post] + b_post.pos_args), **b_kwargs)
above_llik = fn_lpdf(*([b_post] + a_post.pos_args), **a_kwargs)
#improvement = below_llik - above_llik
#new_node = scope.broadcast_best(b_post, improvement)
new_node = scope.broadcast_best(b_post, below_llik, above_llik)
elif hasattr(node, 'obj'):
# -- keep same literals in the graph
new_node = node
else:
# -- this case is for all the other stuff in the graph
new_node = node.clone_from_inputs(new_inputs)
memo[node] = new_node
post_specs = memo[specs]
post_idxs = dict([(nid, memo[idxs])
for nid, idxs in prior_idxs.items()])
post_vals = dict([(nid, memo[vals])
for nid, vals in prior_vals.items()])
assert set(post_idxs.keys()) == set(post_vals.keys())
assert set(post_idxs.keys()) == set(prior_idxs.keys())
return post_specs, post_idxs, post_vals
def test2(self):
self.expr = as_apply(dict(p0=one_of(0, 1)))
self.wanted = [[('p0.randint', [0], [0])], [('p0.randint', [1], [1])],
[('p0.randint', [2], [0])], [('p0.randint', [3], [0])],
[('p0.randint', [4], [0])]]
self.foo()
def test_sample_deterministic():
aa = as_apply([0, 1])
print aa
dd = sample(aa, np.random.RandomState(3))
assert dd == (0, 1)
