def test_vectorize_simple(): N = as_apply(15) p0 = hp_uniform('p0', 0, 1) loss = p0**2 print loss expr_idxs = scope.range(N) vh = VectorizeHelper(loss, expr_idxs, build=True) vloss = vh.v_expr full_output = as_apply([vloss, vh.idxs_by_label(), vh.vals_by_label()]) fo2 = replace_repeat_stochastic(full_output) new_vc = recursive_set_rng_kwarg( fo2, as_apply(np.random.RandomState(1)), ) #print new_vc losses, idxs, vals = rec_eval(new_vc) print 'losses', losses print 'idxs p0', idxs['p0'] print 'vals p0', vals['p0'] p0dct = dict(zip(idxs['p0'], vals['p0'])) for ii, li in enumerate(losses): assert p0dct[ii]**2 == li
def test_vectorize_multipath(): N = as_apply(15) p0 = hp_uniform('p0', 0, 1) loss = hp_choice('p1', [1, p0, -p0])**2 expr_idxs = scope.range(N) vh = VectorizeHelper(loss, expr_idxs, build=True) vloss = vh.v_expr print vloss full_output = as_apply([vloss, vh.idxs_by_label(), vh.vals_by_label()]) new_vc = recursive_set_rng_kwarg( full_output, as_apply(np.random.RandomState(1)), ) losses, idxs, vals = rec_eval(new_vc) print 'losses', losses print 'idxs p0', idxs['p0'] print 'vals p0', vals['p0'] print 'idxs p1', idxs['p1'] print 'vals p1', vals['p1'] p0dct = dict(zip(idxs['p0'], vals['p0'])) p1dct = dict(zip(idxs['p1'], vals['p1'])) for ii, li in enumerate(losses): print ii, li if p1dct[ii] != 0: assert li == p0dct[ii]**2 else: assert li == 1
def __init__(self, expr, name=None, rseed=None, loss_target=None, exceptions=None, do_checks=True): if do_checks: if isinstance(expr, pyll.Apply): self.expr = expr # XXX: verify that expr is a dictionary with the right keys, # then refactor the code below elif isinstance(expr, dict): if "loss" not in expr: raise ValueError("expr must define a loss") if "status" not in expr: expr["status"] = STATUS_OK self.expr = pyll.as_apply(expr) else: raise TypeError("expr must be a dictionary") else: self.expr = pyll.as_apply(expr) self.params = {} for node in pyll.dfs(self.expr): if node.name == "hyperopt_param": self.params[node.arg["label"].obj] = node.arg["obj"] if exceptions is not None: self.exceptions = exceptions self.loss_target = loss_target self.installed_rng = False if rseed is None: self.rng = None else: self.rng = np.random.RandomState(rseed) self.name = name
def test_vectorize_multipath(): N = as_apply(15) p0 = hp_uniform('p0', 0, 1) loss = hp_choice('p1', [1, p0, -p0]) ** 2 expr_idxs = scope.range(N) vh = VectorizeHelper(loss, expr_idxs, build=True) vloss = vh.v_expr print vloss full_output = as_apply([vloss, vh.idxs_by_label(), vh.vals_by_label()]) new_vc = recursive_set_rng_kwarg( full_output, as_apply(np.random.RandomState(1)), ) losses, idxs, vals = rec_eval(new_vc) print 'losses', losses print 'idxs p0', idxs['p0'] print 'vals p0', vals['p0'] print 'idxs p1', idxs['p1'] print 'vals p1', vals['p1'] p0dct = dict(zip(idxs['p0'], vals['p0'])) p1dct = dict(zip(idxs['p1'], vals['p1'])) for ii, li in enumerate(losses): print ii, li if p1dct[ii] != 0: assert li == p0dct[ii] ** 2 else: assert li == 1
def test_vectorize_simple(): N = as_apply(15) p0 = hp_uniform('p0', 0, 1) loss = p0 ** 2 print loss expr_idxs = scope.range(N) vh = VectorizeHelper(loss, expr_idxs, build=True) vloss = vh.v_expr full_output = as_apply([vloss, vh.idxs_by_label(), vh.vals_by_label()]) fo2 = replace_repeat_stochastic(full_output) new_vc = recursive_set_rng_kwarg( fo2, as_apply(np.random.RandomState(1)), ) #print new_vc losses, idxs, vals = rec_eval(new_vc) print 'losses', losses print 'idxs p0', idxs['p0'] print 'vals p0', vals['p0'] p0dct = dict(zip(idxs['p0'], vals['p0'])) for ii, li in enumerate(losses): assert p0dct[ii] ** 2 == li
def __init__(self, expr, name=None, loss_target=None, exceptions=None, do_checks=True, ): if do_checks: if isinstance(expr, pyll.Apply): self.expr = expr # XXX: verify that expr is a dictionary with the right keys, # then refactor the code below elif isinstance(expr, dict): if 'loss' not in expr: raise ValueError('expr must define a loss') if 'status' not in expr: expr['status'] = STATUS_OK self.expr = pyll.as_apply(expr) else: raise TypeError('expr must be a dictionary') else: self.expr = pyll.as_apply(expr) self.params = {} for node in pyll.dfs(self.expr): if node.name == 'hyperopt_param': label = node.arg['label'].obj if label in self.params: raise DuplicateLabel(label) self.params[label] = node.arg['obj'] if exceptions is not None: self.exceptions = exceptions self.loss_target = loss_target self.name = name
def n_arms(N=2): """ Each arm yields a reward from a different Gaussian. The correct arm is arm 0. """ x = hp_choice('x', [0, 1]) reward_mus = as_apply([-1] + [0] * (N - 1)) reward_sigmas = as_apply([1] * N) return {'loss': scope.normal(reward_mus[x], reward_sigmas[x]), 'loss_variance': 1.0}
def n_arms(N=2): """ Each arm yields a reward from a different Gaussian. The correct arm is arm 0. """ x = hp_choice('x', [0, 1]) reward_mus = as_apply([-1] + [0] * (N - 1)) reward_sigmas = as_apply([1] * N) return { 'loss': scope.normal(reward_mus[x], reward_sigmas[x]), 'loss_variance': 1.0 }
def test2(self): self.expr = as_apply(dict(p0=one_of(0, 1))) self.wanted = [ [('p0.randint', [0], [0])], [('p0.randint', [1], [1])], [('p0.randint', [2], [0])], [('p0.randint', [3], [0])], [('p0.randint', [4], [0])]] self.foo()
def evaluate(self, config, ctrl): memo = self.memo_from_config(config) memo[self.pyll_ctrl] = ctrl if self.init_pyll_memo: memo = self.init_pyll_memo(memo, config=config, ctrl=ctrl) if self.rng is not None and not self.installed_rng: # -- N.B. this modifies the expr graph in-place # XXX this feels wrong self.expr = recursive_set_rng_kwarg(self.expr, pyll.as_apply(self.rng)) self.installed_rng = True try: # -- the "work" of evaluating `config` can be written # either into the pyll part (self.expr) # or the normal Python part (self.fn) pyll_rval = pyll.rec_eval(self.expr, memo=memo) rval = self.fn(pyll_rval) except Exception, e: n_match = 0 for match, match_pair in self.exceptions: if match(e): rval = match_pair(e) logger.info('Caught fn exception %s' % str(rval)) n_match += 1 break if n_match == 0: raise
def n_arms(N=2): """ Each arm yields a reward from a different Gaussian. The correct arm is arm 0. """ rng = np.random.RandomState(123) x = hp_choice('x', [0, 1]) reward_mus = as_apply([-1] + [0] * (N - 1)) reward_sigmas = as_apply([1] * N) return { 'loss': scope.normal(reward_mus[x], reward_sigmas[x], rng=rng), 'loss_variance': 1.0, 'status': base.STATUS_OK }
def __init__(self, bandit, seed=seed, cmd=None, workdir=None): self.bandit = bandit self.seed = seed self.rng = np.random.RandomState(self.seed) self.cmd = cmd self.workdir = workdir self.s_new_ids = pyll.Literal('new_ids') # -- list at eval-time before = pyll.dfs(self.bandit.expr) # -- raises exception if expr contains cycles pyll.toposort(self.bandit.expr) vh = self.vh = VectorizeHelper(self.bandit.expr, self.s_new_ids) # -- raises exception if v_expr contains cycles pyll.toposort(vh.v_expr) idxs_by_label = vh.idxs_by_label() vals_by_label = vh.vals_by_label() after = pyll.dfs(self.bandit.expr) # -- try to detect if VectorizeHelper screwed up anything inplace assert before == after assert set(idxs_by_label.keys()) == set(vals_by_label.keys()) assert set(idxs_by_label.keys()) == set(self.bandit.params.keys()) # -- make the graph runnable and SON-encodable # N.B. operates inplace self.s_idxs_vals = recursive_set_rng_kwarg( scope.pos_args(idxs_by_label, vals_by_label), pyll.as_apply(self.rng)) # -- raises an exception if no topological ordering exists pyll.toposort(self.s_idxs_vals)
def test1(self): self.expr = as_apply(dict(p0=normal(0, 1))) self.wanted = [[('p0', [0], [-1.0856306033005612])], [('p0', [1], [0.99734544658358582])], [('p0', [2], [0.28297849805199204])], [('p0', [3], [-1.506294713918092])], [('p0', [4], [-0.57860025196853637])]] self.foo()
def test_repeatable(): u = scope.uniform(0, 1) aa = as_apply(dict(u=u, n=scope.normal(5, 0.1), l=[0, 1, scope.one_of(2, 3), u])) dd1 = sample(aa, np.random.RandomState(3)) dd2 = sample(aa, np.random.RandomState(3)) dd3 = sample(aa, np.random.RandomState(4)) assert dd1 == dd2 assert dd1 != dd3
def __init__(self): d = dict( # -- alphabetical order lu = scope.loguniform(-2, 2), qlu = scope.qloguniform(np.log(0.01), np.log(20), 2), qu = scope.quniform(-4.999, 5, 1), u = scope.uniform(0, 10), ) base.Bandit.__init__(self, as_apply(d))
def test5(self): p0 = uniform(0, 1) self.expr = as_apply(dict(p0=p0, p1=p0)) self.wanted = [[('p0', [0], [0.69646918559786164])], [('p0', [1], [0.28613933495037946])], [('p0', [2], [0.22685145356420311])], [('p0', [3], [0.55131476908289123])], [('p0', [4], [0.71946896978556307])]] self.foo()
def test_repeatable(): u = scope.uniform(0, 1) aa = as_apply( dict(u=u, n=scope.normal(5, 0.1), l=[0, 1, scope.one_of(2, 3), u])) dd1 = sample(aa, np.random.RandomState(3)) dd2 = sample(aa, np.random.RandomState(3)) dd3 = sample(aa, np.random.RandomState(4)) assert dd1 == dd2 assert dd1 != dd3
def test_recursive_set_rng_kwarg(): uniform = scope.uniform a = as_apply([uniform(0, 1), uniform(2, 3)]) rng = np.random.RandomState(234) recursive_set_rng_kwarg(a, rng) print a val_a = rec_eval(a) assert 0 < val_a[0] < 1 assert 2 < val_a[1] < 3
def test1(self): self.expr = as_apply(dict(p0=normal(0, 1))) self.wanted = [ [('p0', [0], [-1.0856306033005612])], [('p0', [1], [0.99734544658358582])], [('p0', [2], [0.28297849805199204])], [('p0', [3], [-1.506294713918092])], [('p0', [4], [-0.57860025196853637])]] self.foo()
def test_sample(): u = scope.uniform(0, 1) aa = as_apply(dict(u=u, n=scope.normal(5, 0.1), l=[0, 1, scope.one_of(2, 3), u])) print aa dd = sample(aa, np.random.RandomState(3)) assert 0 < dd["u"] < 1 assert 4 < dd["n"] < 6 assert dd["u"] == dd["l"][3] assert dd["l"][:2] == (0, 1) assert dd["l"][2] in (2, 3)
def test6(self): p0 = uniform(0, 1) self.expr = as_apply(dict(p0=p0, p1=normal(p0, 1))) self.wanted = [ [('p0', [0], [0.69646918559786164]), ('p1', [0], [-0.25562802126346051])], [('p0', [1], [0.55131476908289123]), ('p1', [1], [-0.19412629039976703])], [('p0', [2], [0.71946896978556307]), ('p1', [2], [1.0415750381251847])], [('p0', [3], [0.68482973858486329]), ('p1', [3], [0.63331201764547818])], [('p0', [4], [0.48093190148436094]), ('p1', [4], [-1.1383681635523848])]] self.foo()
def test_fg11_top_bandit(): L = lfw.FG11Bandit() config = stochastic.sample(L.template, np.random.RandomState(0)) config['decisions'] = None config['slm'] = stochastic.sample(pyll.as_apply(params.fg11_top), np.random.RandomState(0)) config['comparison'] = 'sqrtabsdiff' rec = L.evaluate(config, hyperopt.base.Ctrl(None)) assert np.abs(rec['loss'] - .194) < 1e-2 return rec
def config0(): p0 = scope.uniform(0, 1) p1 = scope.uniform(2, 3) p2 = scope.one_of(-1, p0) p3 = scope.one_of(-2, p1) p4 = 1 p5 = [3, 4, p0] d = locals() del d['p1'] # -- don't sample p1 all the time s = as_apply(d) return s
def test_sample(): u = scope.uniform(0, 1) aa = as_apply( dict(u=u, n=scope.normal(5, 0.1), l=[0, 1, scope.one_of(2, 3), u])) print aa dd = sample(aa, np.random.RandomState(3)) assert 0 < dd['u'] < 1 assert 4 < dd['n'] < 6 assert dd['u'] == dd['l'][3] assert dd['l'][:2] == (0, 1) assert dd['l'][2] in (2, 3)
def config0(): p0 = scope.uniform(0, 1) p1 = scope.uniform(2, 3) p2 = scope.one_of(-1, p0) p3 = scope.one_of(-2, p1) p4 = 1 p5 = [3, 4, p0] p6 = scope.one_of(-3, p1) d = locals() d['p1'] = None # -- don't sample p1 all the time, only if p3 says so s = as_apply(d) return s
def build_vals(self): for node in self.dfs_nodes: if node.name == "literal": n_times = scope.len(self.idxs_memo[node]) vnode = scope.asarray(scope.repeat(n_times, node)) elif node in self.choice_memo: # -- choices are natively vectorized choices = self.choice_memo[node] self.vals_memo[choices] = choices # -- this stitches together the various sub-graphs # to define the original node vnode = scope.vchoice_merge(self.idxs_memo[node], self.choice_memo[node]) vnode.pos_args.extend( [as_apply([self.idxs_memo[inode], self.vals_memo[inode]]) for inode in node.pos_args] ) else: vnode = scope.idxs_map(self.idxs_memo[node], node.name) vnode.pos_args.extend(node.pos_args) vnode.named_args.extend(node.named_args) for arg in node.inputs(): vnode.replace_input(arg, as_apply([self.idxs_memo[arg], self.vals_memo[arg]])) self.vals_memo[node] = vnode
def test4(self): self.expr = as_apply(dict(p0=uniform(0, 1) + normal(0, 1))) self.wanted = [[('p0.arg:0', [0], [0.69646918559786164]), ('p0.arg:1', [0], [-0.95209720686132215])], [('p0.arg:0', [1], [0.55131476908289123]), ('p0.arg:1', [1], [-0.74544105948265826])], [('p0.arg:0', [2], [0.71946896978556307]), ('p0.arg:1', [2], [0.32210606833962163])], [('p0.arg:0', [3], [0.68482973858486329]), ('p0.arg:1', [3], [-0.0515177209393851])], [('p0.arg:0', [4], [0.48093190148436094]), ('p0.arg:1', [4], [-1.6193000650367457])]] self.foo()
def evaluate(self, config, ctrl, attach_attachments=True): memo = self.memo_from_config(config) self.use_obj_for_literal_in_memo(ctrl, base.Ctrl, memo) if self.rng is not None and not self.installed_rng: # -- N.B. this modifies the expr graph in-place # XXX this feels wrong self.expr = recursive_set_rng_kwarg(self.expr, pyll.as_apply(self.rng)) self.installed_rng = True if self.pass_expr_memo_ctrl: rval = self.fn( expr=self.expr, memo=memo, ctrl=ctrl, *self.args) else: # -- the "work" of evaluating `config` can be written # either into the pyll part (self.expr) # or the normal Python part (self.fn) pyll_rval = pyll.rec_eval(self.expr, memo=memo, print_node_on_error=self.rec_eval_print_node_on_error) rval = self.fn(pyll_rval, *self.args) if isinstance(rval, (float, int, np.number)): dict_rval = {'loss': rval} elif isinstance(rval, (dict,)): dict_rval = rval if 'loss' not in dict_rval: raise ValueError('dictionary must have "loss" key', dict_rval.keys()) else: raise TypeError('invalid return type (neither number nor dict)', rval) if dict_rval['loss'] is not None: # -- fail if cannot be cast to float dict_rval['loss'] = float(dict_rval['loss']) dict_rval.setdefault('status', base.STATUS_OK) if dict_rval['status'] not in base.STATUS_STRINGS: raise ValueError('invalid status string', dict_rval['status']) if attach_attachments: attachments = dict_rval.pop('attachments', {}) for key, val in attachments.items(): ctrl.attachments[key] = val # -- don't do this here because SON-compatibility is only a requirement # for trials destined for a mongodb. In-memory rvals can contain # anything. #return base.SONify(dict_rval) return dict_rval
def test4(self): self.expr = as_apply(dict(p0=uniform(0, 1) + normal(0, 1))) self.wanted = [ [('p0.arg:0', [0], [0.69646918559786164]), ('p0.arg:1', [0], [-0.95209720686132215])], [('p0.arg:0', [1], [0.55131476908289123]), ('p0.arg:1', [1], [-0.74544105948265826])], [('p0.arg:0', [2], [0.71946896978556307]), ('p0.arg:1', [2], [0.32210606833962163])], [('p0.arg:0', [3], [0.68482973858486329]), ('p0.arg:1', [3], [-0.0515177209393851])], [('p0.arg:0', [4], [0.48093190148436094]), ('p0.arg:1', [4], [-1.6193000650367457])]] self.foo()
def test6(self): p0 = uniform(0, 1) self.expr = as_apply(dict(p0=p0, p1=normal(p0, 1))) self.wanted = [[('p0', [0], [0.69646918559786164]), ('p1', [0], [-0.25562802126346051])], [('p0', [1], [0.55131476908289123]), ('p1', [1], [-0.19412629039976703])], [('p0', [2], [0.71946896978556307]), ('p1', [2], [1.0415750381251847])], [('p0', [3], [0.68482973858486329]), ('p1', [3], [0.63331201764547818])], [('p0', [4], [0.48093190148436094]), ('p1', [4], [-1.1383681635523848])]] self.foo()
def evaluate(self, config, ctrl, attach_attachments=True): memo = self.memo_from_config(config) self.use_obj_for_literal_in_memo(ctrl, base.Ctrl, memo) if self.rng is not None and not self.installed_rng: # -- N.B. this modifies the expr graph in-place # XXX this feels wrong self.expr = recursive_set_rng_kwarg(self.expr, pyll.as_apply(self.rng)) self.installed_rng = True if self.pass_expr_memo_ctrl: rval = self.fn(expr=self.expr, memo=memo, ctrl=ctrl) else: # -- the "work" of evaluating `config` can be written # either into the pyll part (self.expr) # or the normal Python part (self.fn) pyll_rval = pyll.rec_eval( self.expr, memo=memo, print_node_on_error=self.rec_eval_print_node_on_error) rval = self.fn(pyll_rval) if isinstance(rval, (float, int, np.number)): dict_rval = {'loss': rval} elif isinstance(rval, (dict, )): dict_rval = rval if 'loss' not in dict_rval: raise ValueError('dictionary must have "loss" key', dict_rval.keys()) else: raise TypeError('invalid return type (neither number nor dict)', rval) if dict_rval['loss'] is not None: # -- fail if cannot be cast to float dict_rval['loss'] = float(dict_rval['loss']) dict_rval.setdefault('status', base.STATUS_OK) if dict_rval['status'] not in base.STATUS_STRINGS: raise ValueError('invalid status string', dict_rval['status']) if attach_attachments: attachments = dict_rval.pop('attachments', {}) for key, val in attachments.items(): ctrl.attachments[key] = val # -- don't do this here because SON-compatibility is only a requirement # for trials destined for a mongodb. In-memory rvals can contain # anything. #return base.SONify(dict_rval) return dict_rval
def expr_to_config(expr, conditions, hps): """ Populate dictionary `hps` with the hyperparameters in pyll graph `expr` and conditions for participation in the evaluation of `expr`. Arguments: expr - a pyll expression root. conditions - a tuple of conditions (`Cond`) that must be True for `expr` to be evaluated. hps - dictionary to populate Creates `hps` dictionary: label -> { 'node': apply node of hyperparameter distribution, 'conditions': `conditions` + tuple, 'label': label } """ expr = as_apply(expr) if conditions is None: conditions = () assert isinstance(expr, Apply) if expr.name == 'switch': idx = expr.inputs()[0] options = expr.inputs()[1:] assert idx.name == 'hyperopt_param' assert idx.arg['obj'].name in ( 'randint', # -- in case of hp.choice 'categorical', # -- in case of hp.pchoice ) expr_to_config(idx, conditions, hps) for ii, opt in enumerate(options): expr_to_config(opt, conditions + (EQ(idx.arg['label'].obj, ii),), hps) elif expr.name == 'hyperopt_param': label = expr.arg['label'].obj if label in hps: if hps[label]['node'] != expr.arg['obj']: raise DuplicateLabel(label) hps[label]['conditions'].add(conditions) else: hps[label] = {'node': expr.arg['obj'], 'conditions': set((conditions,)), 'label': label, } else: for ii in expr.inputs(): expr_to_config(ii, conditions, hps)
def expr_to_config(expr, conditions, hps): """ Populate dictionary `hps` with the hyperparameters in pyll graph `expr` and conditions for participation in the evaluation of `expr`. Arguments: expr - a pyll expression root. conditions - a tuple of conditions (`Cond`) that must be True for `expr` to be evaluated. hps - dictionary to populate Creates `hps` dictionary: label -> { 'node': apply node of hyperparameter distribution, 'conditions': `conditions` + tuple, 'label': label } """ expr = as_apply(expr) if conditions is None: conditions = () assert isinstance(expr, Apply) if expr.name == 'switch': idx = expr.inputs()[0] options = expr.inputs()[1:] assert idx.name == 'hyperopt_param' assert idx.arg['obj'].name in ( 'randint', # -- in case of hp.choice 'categorical', # -- in case of hp.pchoice ) expr_to_config(idx, conditions, hps) for ii, opt in enumerate(options): expr_to_config(opt, conditions + (EQ(idx.arg['label'].obj, ii), ), hps) elif expr.name == 'hyperopt_param': label = expr.arg['label'].obj if label in hps: if hps[label]['node'] != expr.arg['obj']: raise DuplicateLabel(label) hps[label]['conditions'].add(conditions) else: hps[label] = { 'node': expr.arg['obj'], 'conditions': set((conditions, )), 'label': label, } else: for ii in expr.inputs(): expr_to_config(ii, conditions, hps)
def __init__(self, fn, expr, args=[], workdir=None, pass_expr_memo_ctrl=None, **bandit_kwargs): self.cmd = ('domain_attachment', 'FMinIter_Domain') self.fn = fn self.expr = expr self.args = args if pass_expr_memo_ctrl is None: self.pass_expr_memo_ctrl = getattr(fn, 'fmin_pass_expr_memo_ctrl', False) else: self.pass_expr_memo_ctrl = pass_expr_memo_ctrl base.Bandit.__init__(self, expr, do_checks=False, **bandit_kwargs) # -- This code was stolen from base.BanditAlgo, a class which may soon # be gone self.workdir = workdir self.s_new_ids = pyll.Literal('new_ids') # -- list at eval-time before = pyll.dfs(self.expr) # -- raises exception if expr contains cycles pyll.toposort(self.expr) vh = self.vh = VectorizeHelper(self.expr, self.s_new_ids) # -- raises exception if v_expr contains cycles pyll.toposort(vh.v_expr) idxs_by_label = vh.idxs_by_label() vals_by_label = vh.vals_by_label() after = pyll.dfs(self.expr) # -- try to detect if VectorizeHelper screwed up anything inplace assert before == after assert set(idxs_by_label.keys()) == set(vals_by_label.keys()) assert set(idxs_by_label.keys()) == set(self.params.keys()) # -- make the graph runnable and SON-encodable # N.B. operates inplace self.s_idxs_vals = recursive_set_rng_kwarg( pyll.scope.pos_args(idxs_by_label, vals_by_label), pyll.as_apply(self.rng)) # -- raises an exception if no topological ordering exists pyll.toposort(self.s_idxs_vals)
def test_lnorm(): G = scope choice = G.choice uniform = G.uniform quantized_uniform = G.quniform inker_size = quantized_uniform(low=0, high=7.99, q=2) + 3 # -- test that it runs lnorm = as_apply({'kwargs': {'inker_shape' : (inker_size, inker_size), 'outker_shape' : (inker_size, inker_size), 'remove_mean' : choice([0, 1]), 'stretch' : uniform(low=0, high=10), 'threshold' : uniform( low=.1 / np.sqrt(10.), high=10 * np.sqrt(10)) }}) print lnorm print 'len', len(str(lnorm))
def work(self): """Test that all prior samplers are gone""" tpe_algo = TreeParzenEstimator(self.bandit) foo = pyll.as_apply( [tpe_algo.post_below['idxs'], tpe_algo.post_below['vals']]) prior_names = [ 'uniform', 'quniform', 'loguniform', 'qloguniform', 'normal', 'qnormal', 'lognormal', 'qlognormal', 'randint', ] for node in pyll.dfs(foo): assert node.name not in prior_names
def space_eval(space, hp_assignment): """Compute a point in a search space from a hyperparameter assignment. Parameters: ----------- space - a pyll graph involving hp nodes (see `pyll_utils`). hp_assignment - a dictionary mapping hp node labels to values. """ space = pyll.as_apply(space) nodes = pyll.toposort(space) memo = {} for node in nodes: if node.name == 'hyperopt_param': label = node.arg['label'].eval() if label in hp_assignment: memo[node] = hp_assignment[label] rval = pyll.rec_eval(space, memo=memo) return rval
def work(self): """Test that all prior samplers are gone""" tpe_algo = TreeParzenEstimator(self.bandit) foo = pyll.as_apply([ tpe_algo.post_below['idxs'], tpe_algo.post_below['vals']]) prior_names = [ 'uniform', 'quniform', 'loguniform', 'qloguniform', 'normal', 'qnormal', 'lognormal', 'qlognormal', 'randint', ] for node in pyll.dfs(foo): assert node.name not in prior_names
def test_lnorm(): G = scope choice = G.choice uniform = G.uniform quantized_uniform = G.quniform inker_size = quantized_uniform(low=0, high=7.99, q=2) + 3 # -- test that it runs lnorm = as_apply({ 'kwargs': { 'inker_shape': (inker_size, inker_size), 'outker_shape': (inker_size, inker_size), 'remove_mean': choice([0, 1]), 'stretch': uniform(low=0, high=10), 'threshold': uniform(low=.1 / np.sqrt(10.), high=10 * np.sqrt(10)) } }) print lnorm print 'len', len(str(lnorm))
def test7(self): p0 = uniform(0, 1) p1 = normal(0, 1) self.expr = as_apply(dict( p0=p0, p1=p1, p2=one_of(1, p0), p3=one_of(2, p1, uniform(2, 3)))) self.n_randints = 2 self.wanted = [ [ ('p0', [0], [0.71295532052322719]), ('p1', [0], [0.28297849805199204]), ('p2.randint', [0], [0]), ('p3.arg:2', [0], [2.719468969785563]), ('p3.randint', [0], [2])], [ ('p0', [1], [0.78002776191207912]), ('p1', [1], [-1.506294713918092]), ('p2.randint', [1], [1]), ('p3.arg:2', [], []), ('p3.randint', [1], [1])], [ ('p0', [2], [0.57969429702261011]), ('p1', [2], [1.6796003743035337]), ('p2.randint', [2], [0]), ('p3.arg:2', [], []), ('p3.randint', [2], [1])], [ ('p0', [3], [0.43857224467962441]), ('p1', [3], [-1.3058031267484451]), ('p2.randint', [3], [1]), ('p3.arg:2', [], []), ('p3.randint', [3], [1])], [ ('p0', [4], [0.39804425533043142]), ('p1', [4], [-0.91948540682140967]), ('p2.randint', [4], [0]), ('p3.arg:2', [], []), ('p3.randint', [4], [0])]] self.foo()
def test_lnorm(): G = scope choice = G.choice uniform = G.uniform quantized_uniform = G.quniform inker_size = quantized_uniform(low=0, high=7.99, q=2) + 3 # -- test that it runs lnorm = as_apply( { "kwargs": { "inker_shape": (inker_size, inker_size), "outker_shape": (inker_size, inker_size), "remove_mean": choice([0, 1]), "stretch": uniform(low=0, high=10), "threshold": uniform(low=0.1 / np.sqrt(10.0), high=10 * np.sqrt(10)), } } ) print lnorm print "len", len(str(lnorm))
def evaluate(self, config, ctrl): """Return a result document """ memo = self.memo_from_config(config) self.use_obj_for_literal_in_memo(ctrl, Ctrl, memo) if self.rng is not None and not self.installed_rng: # -- N.B. this modifies the expr graph in-place # XXX this feels wrong self.expr = recursive_set_rng_kwarg(self.expr, pyll.as_apply(self.rng)) self.installed_rng = True try: r_dct = pyll.rec_eval(self.expr, memo=memo) except Exception, e: n_match = 0 for match, match_pair in self.exceptions: if match(e): r_dct = match_pair(e) n_match += 1 break if n_match == 0: raise
def expr_to_config(expr, conditions, hps): """ Populate dictionary `hps` with the hyperparameters in pyll graph `expr` and conditions for participation in the evaluation of `expr`. Arguments: expr - a pyll expression root. conditions - a tuple of conditions (`Cond`) that must be True for `expr` to be evaluated. hps - dictionary to populate Creates `hps` dictionary: label -> { 'node': apply node of hyperparameter distribution, 'conditions': `conditions` + tuple, 'label': label } """ expr = as_apply(expr) if conditions is None: conditions = () assert isinstance(expr, Apply) _expr_to_config(expr, conditions, hps) _remove_allpaths(hps, conditions)
def test_vectorize_config0(): p0 = hp_uniform('p0', 0, 1) p1 = hp_loguniform('p1', 2, 3) p2 = hp_choice('p2', [-1, p0]) p3 = hp_choice('p3', [-2, p1]) p4 = 1 p5 = [3, 4, p0] p6 = hp_choice('p6', [-3, p1]) d = locals() d['p1'] = None # -- don't sample p1 all the time, only if p3 says so config = as_apply(d) N = as_apply('N:TBA') expr = config expr_idxs = scope.range(N) vh = VectorizeHelper(expr, expr_idxs, build=True) vconfig = vh.v_expr full_output = as_apply([vconfig, vh.idxs_by_label(), vh.vals_by_label()]) if 1: print '=' * 80 print 'VECTORIZED' print full_output print '\n' * 1 fo2 = replace_repeat_stochastic(full_output) if 0: print '=' * 80 print 'VECTORIZED STOCHASTIC' print fo2 print '\n' * 1 new_vc = recursive_set_rng_kwarg(fo2, as_apply(np.random.RandomState(1))) if 0: print '=' * 80 print 'VECTORIZED STOCHASTIC WITH RNGS' print new_vc Nval = 10 foo, idxs, vals = rec_eval(new_vc, memo={N: Nval}) print 'foo[0]', foo[0] print 'foo[1]', foo[1] assert len(foo) == Nval if 0: # XXX refresh these values to lock down sampler assert foo[0] == { 'p0': 0.39676747423066994, 'p1': None, 'p2': 0.39676747423066994, 'p3': 2.1281244479293568, 'p4': 1, 'p5': (3, 4, 0.39676747423066994) } assert foo[1] != foo[2] print idxs print vals['p3'] print vals['p6'] print idxs['p1'] print vals['p1'] assert len(vals['p3']) == Nval assert len(vals['p6']) == Nval assert len(idxs['p1']) < Nval p1d = dict(zip(idxs['p1'], vals['p1'])) for ii, (p3v, p6v) in enumerate(zip(vals['p3'], vals['p6'])): if p3v == p6v == 0: assert ii not in idxs['p1'] if p3v: assert foo[ii]['p3'] == p1d[ii] if p6v: print 'p6', foo[ii]['p6'], p1d[ii] assert foo[ii]['p6'] == p1d[ii]
def build_idxs_vals(self, node, wanted_idxs): """ This recursive procedure should be called on an output-node. """ checkpoint_asserts = False def checkpoint(): if checkpoint_asserts: self.assert_integrity_idxs_take() if node in self.idxs_memo: toposort(self.idxs_memo[node]) if node in self.take_memo: for take in self.take_memo[node]: toposort(take) checkpoint() # wanted_idxs are fixed, whereas idxs_memo # is full of unions, that can grow in subsequent recursive # calls to build_idxs_vals with node as argument. assert wanted_idxs != self.idxs_memo.get(node) # -- easy exit case if node.name == 'hyperopt_param': # -- ignore, not vectorizing return self.build_idxs_vals(node.arg['obj'], wanted_idxs) # -- easy exit case elif node.name == 'hyperopt_result': # -- ignore, not vectorizing return self.build_idxs_vals(node.arg['obj'], wanted_idxs) # -- literal case: always take from universal set elif node.name == 'literal': if node in self.idxs_memo: all_idxs, all_vals = self.take_memo[node][0].pos_args[:2] wanted_vals = scope.idxs_take(all_idxs, all_vals, wanted_idxs) self.take_memo[node].append(wanted_vals) checkpoint() else: # -- initialize idxs_memo to full set all_idxs = self.expr_idxs n_times = scope.len(all_idxs) # -- put array_union into graph for consistency, though it is # not necessary all_idxs = scope.array_union(all_idxs) self.idxs_memo[node] = all_idxs all_vals = scope.asarray(scope.repeat(n_times, node)) wanted_vals = scope.idxs_take(all_idxs, all_vals, wanted_idxs) assert node not in self.take_memo self.take_memo[node] = [wanted_vals] checkpoint() return wanted_vals # -- switch case: complicated elif node.name == 'switch': if (node in self.idxs_memo and wanted_idxs in self.idxs_memo[node].pos_args): # -- phew, easy case all_idxs, all_vals = self.take_memo[node][0].pos_args[:2] wanted_vals = scope.idxs_take(all_idxs, all_vals, wanted_idxs) self.take_memo[node].append(wanted_vals) checkpoint() else: # -- we need to add some indexes if node in self.idxs_memo: all_idxs = self.idxs_memo[node] assert all_idxs.name == 'array_union' all_idxs.pos_args.append(wanted_idxs) else: all_idxs = scope.array_union(wanted_idxs) choice = node.pos_args[0] all_choices = self.build_idxs_vals(choice, all_idxs) options = node.pos_args[1:] args_idxs = scope.vchoice_split(all_idxs, all_choices, len(options)) all_vals = scope.vchoice_merge(all_idxs, all_choices) for opt_ii, idxs_ii in zip(options, args_idxs): all_vals.pos_args.append( as_apply([ idxs_ii, self.build_idxs_vals(opt_ii, idxs_ii), ])) wanted_vals = scope.idxs_take( all_idxs, # -- may grow in future all_vals, # -- may be replaced in future wanted_idxs) # -- fixed. if node in self.idxs_memo: assert self.idxs_memo[node].name == 'array_union' self.idxs_memo[node].pos_args.append(wanted_idxs) for take in self.take_memo[node]: assert take.name == 'idxs_take' take.pos_args[1] = all_vals self.take_memo[node].append(wanted_vals) else: self.idxs_memo[node] = all_idxs self.take_memo[node] = [wanted_vals] checkpoint() # -- general case else: # -- this is a general node. # It is generally handled with idxs_memo, # but vectorize_stochastic may immediately transform it into # a more compact form. if (node in self.idxs_memo and wanted_idxs in self.idxs_memo[node].pos_args): # -- phew, easy case for take in self.take_memo[node]: if take.pos_args[2] == wanted_idxs: return take raise NotImplementedError('how did this happen?') #all_idxs, all_vals = self.take_memo[node][0].pos_args[:2] #wanted_vals = scope.idxs_take(all_idxs, all_vals, wanted_idxs) #self.take_memo[node].append(wanted_vals) #checkpoint() else: # XXX # -- determine if wanted_idxs is actually a subset of the idxs # that we are already computing. This is not only an # optimization, but prevents the creation of cycles, which # would otherwise occur if we have a graph of the form # switch(f(a), g(a), 0). If there are other switches inside f # and g, does this get trickier? # -- assume we need to add some indexes checkpoint() if node in self.idxs_memo: all_idxs = self.idxs_memo[node] else: all_idxs = scope.array_union(wanted_idxs) checkpoint() all_vals = scope.idxs_map(all_idxs, node.name) for ii, aa in enumerate(node.pos_args): all_vals.pos_args.append( as_apply( [all_idxs, self.build_idxs_vals(aa, all_idxs)])) checkpoint() for ii, (nn, aa) in enumerate(node.named_args): all_vals.named_args.append([ nn, as_apply( [all_idxs, self.build_idxs_vals(aa, all_idxs)]) ]) checkpoint() all_vals = vectorize_stochastic(all_vals) checkpoint() wanted_vals = scope.idxs_take( all_idxs, # -- may grow in future all_vals, # -- may be replaced in future wanted_idxs) # -- fixed. if node in self.idxs_memo: assert self.idxs_memo[node].name == 'array_union' self.idxs_memo[node].pos_args.append(wanted_idxs) toposort(self.idxs_memo[node]) # -- this catches the cycle bug mentioned above for take in self.take_memo[node]: assert take.name == 'idxs_take' take.pos_args[1] = all_vals self.take_memo[node].append(wanted_vals) else: self.idxs_memo[node] = all_idxs self.take_memo[node] = [wanted_vals] checkpoint() return wanted_vals
class Bandit(object): """Specification of bandit problem. template - pyll specification of search domain evaluate - interruptible/checkpt calling convention for evaluation routine """ # -- the Ctrl object is not used directly, but rather # a live Ctrl instance is inserted for the pyll_ctrl # in self.evaluate so that it can be accessed from within # the pyll graph describing the search space. pyll_ctrl = pyll.as_apply(Ctrl) exceptions = [] def __init__( self, expr, name=None, rseed=None, loss_target=None, exceptions=None, do_checks=True, ): if do_checks: if isinstance(expr, pyll.Apply): self.expr = expr # XXX: verify that expr is a dictionary with the right keys, # then refactor the code below elif isinstance(expr, dict): if 'loss' not in expr: raise ValueError('expr must define a loss') if 'status' not in expr: expr['status'] = STATUS_OK self.expr = pyll.as_apply(expr) else: raise TypeError('expr must be a dictionary') else: self.expr = pyll.as_apply(expr) self.params = {} for node in pyll.dfs(self.expr): if node.name == 'hyperopt_param': label = node.arg['label'].obj if label in self.params: raise DuplicateLabel(label) self.params[label] = node.arg['obj'] if exceptions is not None: self.exceptions = exceptions self.loss_target = loss_target self.installed_rng = False if rseed is None: self.rng = None else: self.rng = np.random.RandomState(rseed) self.name = name def memo_from_config(self, config): memo = {} for node in pyll.dfs(self.expr): if node.name == 'hyperopt_param': label = node.arg['label'].obj # -- hack because it's not really garbagecollected # this does have the desired effect of crashing the # function if rec_eval actually needs a value that # the the optimization algorithm thought to be unnecessary memo[node] = config.get(label, pyll.base.GarbageCollected) return memo def short_str(self): return self.__class__.__name__ def use_obj_for_literal_in_memo(self, obj, lit, memo): return use_obj_for_literal_in_memo(self.expr, obj, lit, memo) def evaluate(self, config, ctrl): """Return a result document """ memo = self.memo_from_config(config) self.use_obj_for_literal_in_memo(ctrl, Ctrl, memo) if self.rng is not None and not self.installed_rng: # -- N.B. this modifies the expr graph in-place # XXX this feels wrong self.expr = recursive_set_rng_kwarg(self.expr, pyll.as_apply(self.rng)) self.installed_rng = True try: r_dct = pyll.rec_eval(self.expr, memo=memo) except Exception, e: n_match = 0 for match, match_pair in self.exceptions: if match(e): r_dct = match_pair(e) n_match += 1 break if n_match == 0: raise assert 'loss' in r_dct if r_dct['loss'] is not None: # -- assert that it can at least be cast to float float(r_dct['loss']) if r_dct['status'] not in STATUS_STRINGS: raise ValueError('invalid status string', r_dct['status']) return r_dct
def __init__( self, fn, expr, workdir=None, pass_expr_memo_ctrl=None, name=None, loss_target=None, ): """ Paramaters ---------- fn : callable This stores the `fn` argument to `fmin`. (See `hyperopt.fmin.fmin`) expr : hyperopt.pyll.Apply This is the `space` argument to `fmin`. (See `hyperopt.fmin.fmin`) workdir : string (or None) If non-None, the current working directory will be `workdir`while `expr` and `fn` are evaluated. (XXX Currently only respected by jobs run via MongoWorker) pass_expr_memo_ctrl : bool If True, `fn` will be called like this: `fn(self.expr, memo, ctrl)`, where `memo` is a dictionary mapping `Apply` nodes to their computed values, and `ctrl` is a `Ctrl` instance for communicating with a Trials database. This lower-level calling convention is useful if you want to call e.g. `hyperopt.pyll.rec_eval` yourself in some customized way. name : string (or None) Label, used for pretty-printing. loss_target : float (or None) The actual or estimated minimum of `fn`. Some optimization algorithms may behave differently if their first objective is to find an input that achieves a certain value, rather than the more open-ended objective of pure minimization. XXX: Move this from Domain to be an fmin arg. """ self.fn = fn if pass_expr_memo_ctrl is None: self.pass_expr_memo_ctrl = getattr(fn, 'fmin_pass_expr_memo_ctrl', False) else: self.pass_expr_memo_ctrl = pass_expr_memo_ctrl self.expr = pyll.as_apply(expr) self.params = {} for node in pyll.dfs(self.expr): if node.name == 'hyperopt_param': label = node.arg['label'].obj if label in self.params: raise DuplicateLabel(label) self.params[label] = node.arg['obj'] self.loss_target = loss_target self.name = name self.workdir = workdir self.s_new_ids = pyll.Literal('new_ids') # -- list at eval-time before = pyll.dfs(self.expr) # -- raises exception if expr contains cycles pyll.toposort(self.expr) vh = self.vh = VectorizeHelper(self.expr, self.s_new_ids) # -- raises exception if v_expr contains cycles pyll.toposort(vh.v_expr) idxs_by_label = vh.idxs_by_label() vals_by_label = vh.vals_by_label() after = pyll.dfs(self.expr) # -- try to detect if VectorizeHelper screwed up anything inplace assert before == after assert set(idxs_by_label.keys()) == set(vals_by_label.keys()) assert set(idxs_by_label.keys()) == set(self.params.keys()) self.s_rng = pyll.Literal('rng-placeholder') # -- N.B. operates inplace: self.s_idxs_vals = recursive_set_rng_kwarg( pyll.scope.pos_args(idxs_by_label, vals_by_label), self.s_rng) # -- raises an exception if no topological ordering exists pyll.toposort(self.s_idxs_vals) # -- Protocol for serialization. # self.cmd indicates to e.g. MongoWorker how this domain # should be [un]serialized. # XXX This mechanism deserves review as support for ipython # workers improves. self.cmd = ('domain_attachment', 'FMinIter_Domain')
class Domain(object): """Picklable representation of search space and evaluation function. """ rec_eval_print_node_on_error = False # -- the Ctrl object is not used directly, but rather # a live Ctrl instance is inserted for the pyll_ctrl # in self.evaluate so that it can be accessed from within # the pyll graph describing the search space. pyll_ctrl = pyll.as_apply(Ctrl) def __init__( self, fn, expr, workdir=None, pass_expr_memo_ctrl=None, name=None, loss_target=None, ): """ Paramaters ---------- fn : callable This stores the `fn` argument to `fmin`. (See `hyperopt.fmin.fmin`) expr : hyperopt.pyll.Apply This is the `space` argument to `fmin`. (See `hyperopt.fmin.fmin`) workdir : string (or None) If non-None, the current working directory will be `workdir`while `expr` and `fn` are evaluated. (XXX Currently only respected by jobs run via MongoWorker) pass_expr_memo_ctrl : bool If True, `fn` will be called like this: `fn(self.expr, memo, ctrl)`, where `memo` is a dictionary mapping `Apply` nodes to their computed values, and `ctrl` is a `Ctrl` instance for communicating with a Trials database. This lower-level calling convention is useful if you want to call e.g. `hyperopt.pyll.rec_eval` yourself in some customized way. name : string (or None) Label, used for pretty-printing. loss_target : float (or None) The actual or estimated minimum of `fn`. Some optimization algorithms may behave differently if their first objective is to find an input that achieves a certain value, rather than the more open-ended objective of pure minimization. XXX: Move this from Domain to be an fmin arg. """ self.fn = fn if pass_expr_memo_ctrl is None: self.pass_expr_memo_ctrl = getattr(fn, 'fmin_pass_expr_memo_ctrl', False) else: self.pass_expr_memo_ctrl = pass_expr_memo_ctrl self.expr = pyll.as_apply(expr) self.params = {} for node in pyll.dfs(self.expr): if node.name == 'hyperopt_param': label = node.arg['label'].obj if label in self.params: raise DuplicateLabel(label) self.params[label] = node.arg['obj'] self.loss_target = loss_target self.name = name self.workdir = workdir self.s_new_ids = pyll.Literal('new_ids') # -- list at eval-time before = pyll.dfs(self.expr) # -- raises exception if expr contains cycles pyll.toposort(self.expr) vh = self.vh = VectorizeHelper(self.expr, self.s_new_ids) # -- raises exception if v_expr contains cycles pyll.toposort(vh.v_expr) idxs_by_label = vh.idxs_by_label() vals_by_label = vh.vals_by_label() after = pyll.dfs(self.expr) # -- try to detect if VectorizeHelper screwed up anything inplace assert before == after assert set(idxs_by_label.keys()) == set(vals_by_label.keys()) assert set(idxs_by_label.keys()) == set(self.params.keys()) self.s_rng = pyll.Literal('rng-placeholder') # -- N.B. operates inplace: self.s_idxs_vals = recursive_set_rng_kwarg( pyll.scope.pos_args(idxs_by_label, vals_by_label), self.s_rng) # -- raises an exception if no topological ordering exists pyll.toposort(self.s_idxs_vals) # -- Protocol for serialization. # self.cmd indicates to e.g. MongoWorker how this domain # should be [un]serialized. # XXX This mechanism deserves review as support for ipython # workers improves. self.cmd = ('domain_attachment', 'FMinIter_Domain') def memo_from_config(self, config): memo = {} for node in pyll.dfs(self.expr): if node.name == 'hyperopt_param': label = node.arg['label'].obj # -- hack because it's not really garbagecollected # this does have the desired effect of crashing the # function if rec_eval actually needs a value that # the the optimization algorithm thought to be unnecessary memo[node] = config.get(label, pyll.base.GarbageCollected) return memo def evaluate(self, config, ctrl, attach_attachments=True): memo = self.memo_from_config(config) use_obj_for_literal_in_memo(self.expr, ctrl, Ctrl, memo) if self.pass_expr_memo_ctrl: rval = self.fn(expr=self.expr, memo=memo, ctrl=ctrl) else: # -- the "work" of evaluating `config` can be written # either into the pyll part (self.expr) # or the normal Python part (self.fn) pyll_rval = pyll.rec_eval( self.expr, memo=memo, print_node_on_error=self.rec_eval_print_node_on_error) rval = self.fn(pyll_rval) if isinstance(rval, (float, int, np.number)): dict_rval = {'loss': float(rval), 'status': STATUS_OK} else: dict_rval = dict(rval) status = dict_rval['status'] if status not in STATUS_STRINGS: raise InvalidResultStatus(dict_rval) if status == STATUS_OK: # -- make sure that the loss is present and valid try: dict_rval['loss'] = float(dict_rval['loss']) except (TypeError, KeyError): raise InvalidLoss(dict_rval) if attach_attachments: attachments = dict_rval.pop('attachments', {}) for key, val in attachments.items(): ctrl.attachments[key] = val # -- don't do this here because SON-compatibility is only a requirement # for trials destined for a mongodb. In-memory rvals can contain # anything. #return base.SONify(dict_rval) return dict_rval def evaluate_async( self, config, ctrl, attach_attachments=True, ): ''' this is the first part of async evaluation for ipython parallel engines (see ipy.py) This breaks evaluate into two parts to allow for the apply_async call to only pass the objective function and arguments. ''' memo = self.memo_from_config(config) use_obj_for_literal_in_memo(self.expr, ctrl, Ctrl, memo) if self.pass_expr_memo_ctrl: rval = self.fn(expr=self.expr, memo=memo, ctrl=ctrl) else: # -- the "work" of evaluating `config` can be written # either into the pyll part (self.expr) # or the normal Python part (self.fn) pyll_rval = pyll.rec_eval( self.expr, memo=memo, print_node_on_error=self.rec_eval_print_node_on_error) return (self.fn, pyll_rval) def evaluate_async2(self, rval, ctrl, attach_attachments=True): ''' this is the second part of async evaluation for ipython parallel engines (see ipy.py) ''' if isinstance(rval, (float, int, np.number)): dict_rval = {'loss': float(rval), 'status': STATUS_OK} else: dict_rval = dict(rval) status = dict_rval['status'] if status not in STATUS_STRINGS: raise InvalidResultStatus(dict_rval) if status == STATUS_OK: # -- make sure that the loss is present and valid try: dict_rval['loss'] = float(dict_rval['loss']) except (TypeError, KeyError): raise InvalidLoss(dict_rval) if attach_attachments: attachments = dict_rval.pop('attachments', {}) for key, val in attachments.items(): ctrl.attachments[key] = val # -- don't do this here because SON-compatibility is only a requirement # for trials destined for a mongodb. In-memory rvals can contain # anything. #return base.SONify(dict_rval) return dict_rval def short_str(self): return 'Domain{%s}' % str(self.fn) def loss(self, result, config=None): """Extract the scalar-valued loss from a result document """ return result.get('loss', None) def loss_variance(self, result, config=None): """Return the variance in the estimate of the loss""" return result.get('loss_variance', 0.0) def true_loss(self, result, config=None): """Return a true loss, in the case that the `loss` is a surrogate""" # N.B. don't use get() here, it evaluates self.loss un-necessarily try: return result['true_loss'] except KeyError: return self.loss(result, config=config) def true_loss_variance(self, config=None): """Return the variance in true loss, in the case that the `loss` is a surrogate. """ raise NotImplementedError() def status(self, result, config=None): """Extract the job status from a result document """ return result['status'] def new_result(self): """Return a JSON-encodable object to serve as the 'result' for new jobs. """ return {'status': STATUS_NEW}
def build_posterior(specs, prior_idxs, prior_vals, obs_idxs, obs_vals, oloss_idxs, oloss_vals, oloss_gamma, prior_weight): """ This method clones a posterior inference graph by iterating forward in topological order, and replacing prior random-variables (prior_vals) with new posterior distributions that make use of observations (obs_vals). """ assert all(isinstance(arg, pyll.Apply) for arg in [oloss_idxs, oloss_vals, oloss_gamma]) expr = pyll.as_apply([specs, prior_idxs, prior_vals]) nodes = pyll.dfs(expr) # build the joint posterior distribution as the values in this memo memo = {} # map prior RVs to observations obs_memo = {} for nid in prior_vals: # construct the leading args for each call to adaptive_parzen_sampler # which will permit the "adaptive parzen samplers" to adapt to the # correct samples. obs_below, obs_above = scope.ap_filter_trials( obs_idxs[nid], obs_vals[nid], oloss_idxs, oloss_vals, oloss_gamma) obs_memo[prior_vals[nid]] = [obs_below, obs_above] for node in nodes: if node not in memo: new_inputs = [memo[arg] for arg in node.inputs()] if node in obs_memo: # -- this case corresponds to an observed Random Var # node.name is a distribution like "normal", "randint", etc. obs_below, obs_above = obs_memo[node] aa = [memo[a] for a in node.pos_args] fn = adaptive_parzen_samplers[node.name] b_args = [obs_below, prior_weight] + aa named_args = [[kw, memo[arg]] for (kw, arg) in node.named_args] b_post = fn(*b_args, **dict(named_args)) a_args = [obs_above, prior_weight] + aa a_post = fn(*a_args, **dict(named_args)) assert a_post.name == b_post.name fn_lpdf = getattr(scope, a_post.name + '_lpdf') #print fn_lpdf a_kwargs = dict([(n, a) for n, a in a_post.named_args if n not in ('rng', 'size')]) b_kwargs = dict([(n, a) for n, a in b_post.named_args if n not in ('rng', 'size')]) # calculate the llik of b_post under both distributions below_llik = fn_lpdf(*([b_post] + b_post.pos_args), **b_kwargs) above_llik = fn_lpdf(*([b_post] + a_post.pos_args), **a_kwargs) #improvement = below_llik - above_llik #new_node = scope.broadcast_best(b_post, improvement) new_node = scope.broadcast_best(b_post, below_llik, above_llik) elif hasattr(node, 'obj'): # -- keep same literals in the graph new_node = node else: # -- this case is for all the other stuff in the graph new_node = node.clone_from_inputs(new_inputs) memo[node] = new_node post_specs = memo[specs] post_idxs = dict([(nid, memo[idxs]) for nid, idxs in prior_idxs.items()]) post_vals = dict([(nid, memo[vals]) for nid, vals in prior_vals.items()]) assert set(post_idxs.keys()) == set(post_vals.keys()) assert set(post_idxs.keys()) == set(prior_idxs.keys()) return post_specs, post_idxs, post_vals
def test2(self): self.expr = as_apply(dict(p0=one_of(0, 1))) self.wanted = [[('p0.randint', [0], [0])], [('p0.randint', [1], [1])], [('p0.randint', [2], [0])], [('p0.randint', [3], [0])], [('p0.randint', [4], [0])]] self.foo()
def test_sample_deterministic(): aa = as_apply([0, 1]) print aa dd = sample(aa, np.random.RandomState(3)) assert dd == (0, 1)