def init_kernels(self): self.kernels = [] self.is_refinable = {} self.bounds = {} self.params = [] self.param_bounds = [] self.idxs_mulsets = {} self.post_refinement_cleanup = {} for iv in self.s_prior: dist_name = montetheano.rstreams.rv_dist_name(iv.vals) if dist_name == 'normal': k = SquaredExponentialKernel() self.is_refinable[k] = get_refinability(iv, dist_name) self.bounds[k] = (None, None) elif dist_name == 'uniform': k = SquaredExponentialKernel() self.is_refinable[k] = get_refinability(iv, dist_name) if self.is_refinable[k]: low = tensor.get_constant_value( mt_dist.uniform_get_low(iv.vals)) high = tensor.get_constant_value( mt_dist.uniform_get_high(iv.vals)) self.bounds[k] = (low, high) elif dist_name == 'lognormal': k = LogSquaredExponentialKernel() self.is_refinable[k] = get_refinability(iv, dist_name) self.bounds[k] = (1e-8, None) elif dist_name == 'quantized_lognormal': k = LogSquaredExponentialKernel() self.is_refinable[k] = get_refinability(iv, dist_name) if self.is_refinable: lbound = tensor.get_constant_value( mt_dist.quantized_lognormal_get_round( iv.vals)) self.bounds[k] = (lbound, None) ff = picklable_instancemethod(self, 'qln_cleanup') self.post_refinement_cleanup[k] = ff elif dist_name == 'categorical': # XXX: a better CategoryKernel would have different # similarities for different choices k = CategoryKernel() self.is_refinable[k] = False # refinable is false, so not setting bounds else: raise TypeError("unsupported distribution", dist_name) self.kernels.append(k) self.params.extend(k.params()) self.param_bounds.extend(k.param_bounds()) # XXX : to be more robust, it would be nice to build an Env with # the idxs as outputs, and then run the MergeOptimizer on it. self.idxs_mulsets.setdefault(iv.idxs, []).append(k)
def init_kernels(self): self.kernels = [] self.is_refinable = {} self.bounds = {} self.params = [] self.param_bounds = [] self.idxs_mulsets = {} self.post_refinement_cleanup = {} for iv in self.s_prior: dist_name = montetheano.rstreams.rv_dist_name(iv.vals) if dist_name == 'normal': k = SquaredExponentialKernel() self.is_refinable[k] = get_refinability(iv, dist_name) self.bounds[k] = (None, None) elif dist_name == 'uniform': k = SquaredExponentialKernel() self.is_refinable[k] = get_refinability(iv, dist_name) if self.is_refinable[k]: low = tensor.get_constant_value( mt_dist.uniform_get_low(iv.vals)) high = tensor.get_constant_value( mt_dist.uniform_get_high(iv.vals)) self.bounds[k] = (low, high) elif dist_name == 'lognormal': k = LogSquaredExponentialKernel() self.is_refinable[k] = get_refinability(iv, dist_name) self.bounds[k] = (1e-8, None) elif dist_name == 'quantized_lognormal': k = LogSquaredExponentialKernel() self.is_refinable[k] = get_refinability(iv, dist_name) if self.is_refinable: lbound = tensor.get_constant_value( mt_dist.quantized_lognormal_get_round(iv.vals)) self.bounds[k] = (lbound, None) ff = picklable_instancemethod(self, 'qln_cleanup') self.post_refinement_cleanup[k] = ff elif dist_name == 'categorical': # XXX: a better CategoryKernel would have different # similarities for different choices k = CategoryKernel() self.is_refinable[k] = False # refinable is false, so not setting bounds else: raise TypeError("unsupported distribution", dist_name) self.kernels.append(k) self.params.extend(k.params()) self.param_bounds.extend(k.param_bounds()) # XXX : to be more robust, it would be nice to build an Env with # the idxs as outputs, and then run the MergeOptimizer on it. self.idxs_mulsets.setdefault(iv.idxs, []).append(k)
def get_refinability(v, dist_name): v = v.vals if dist_name == 'uniform': params = [mt_dist.uniform_get_low(v), mt_dist.uniform_get_high(v)] elif dist_name == 'normal': params = [mt_dist.normal_get_mu(v), mt_dist.normal_get_sigma(v)] elif dist_name == 'lognormal': params = [mt_dist.lognormal_get_mu(v), mt_dist.lognormal_get_sigma(v)] elif dist_name == 'quantized_lognormal': params = [mt_dist.quantized_lognormal_get_mu(v), mt_dist.quantized_lognormal_get_sigma(v), mt_dist.quantized_lognormal_get_round(v)] for p in params: try: tensor.get_constant_value(p) except TypeError: return False return True
def belongs_to_set(self, node, set_nodes): """ This function checks if node `node` belongs to `set_nodes`, in the sense that it can be merged together with every other node in `set_nodes`. In order for two nodes to be mergeable, they have to go over the same number of steps, have the same condition (if any), have the same value for truncate_gradient, and have the same mode. Questionable, we should also consider profile ? """ rep = set_nodes[0] if not rep.op.as_while and node.op.as_while: return False nsteps = node.inputs[0] try: nsteps = int(get_constant_value(nsteps)) except TypeError: pass rep_nsteps = rep.inputs[0] try: rep_nsteps = int(get_constant_value(rep_nsteps)) except TypeError: pass # Check to see if it is an input of a different node can_add = True for nd in set_nodes: if find_up(node, nd) or find_up(nd, node): can_add = False can_add = can_add and (node.op.truncate_gradient == rep.op.truncate_gradient) can_add = can_add and (node.op.mode == rep.op.mode) if not node.op.as_while: return nsteps == rep_nsteps and can_add cond = node.op.outputs[-1] rep_cond = rep.op.outputs[-1] same_cond = scan_utils.equal_computations([cond], [rep_cond], node.op.inputs, rep.op.inputs) return same_cond and (nsteps == rep_nsteps) and can_add
def get_refinability(v, dist_name): v = v.vals if dist_name == 'uniform': params = [mt_dist.uniform_get_low(v), mt_dist.uniform_get_high(v)] elif dist_name == 'normal': params = [mt_dist.normal_get_mu(v), mt_dist.normal_get_sigma(v)] elif dist_name == 'lognormal': params = [mt_dist.lognormal_get_mu(v), mt_dist.lognormal_get_sigma(v)] elif dist_name == 'quantized_lognormal': params = [ mt_dist.quantized_lognormal_get_mu(v), mt_dist.quantized_lognormal_get_sigma(v), mt_dist.quantized_lognormal_get_round(v) ] for p in params: try: tensor.get_constant_value(p) except TypeError: return False return True
def is_positive(v): if hints(v).get('positive', False): return True #TODO: how to handle this - a registry? # infer_hints on Ops? logger.debug('is_positive: %s' % str(v)) if v.owner and v.owner.op == tensor.pow: try: exponent = tensor.get_constant_value(v.owner.inputs[1]) except TypeError: return False if 0 == exponent % 2: return True return False
def is_positive(v): if hints(v).get('positive', False): return True #TODO: how to handle this - a registry? # infer_hints on Ops? print 'is_positive', v if v.owner and v.owner.op == tensor.pow: print 'try for pow', v, v.owner.inputs try: exponent = tensor.get_constant_value(v.owner.inputs[1]) except TypeError: return False if 0 == exponent % 2: return True return False
def qln_cleanup(self, prior_vals, kern, candidate_vals): """ Undo the smooth relaxation applied to quantized log-normal variables """ round = tensor.get_constant_value( mt_dist.quantized_lognormal_get_round(prior_vals)) intlike = numpy.ceil(candidate_vals / float(round)) assert intlike.ndim >= 1 # in test problems, it seems possible to get stuck in a mode # where the EI optimum always gets rounded up to 3 # and so 2 is never tried, even though it is actually the best point. intlike = numpy.maximum( 1, intlike - self.numpy_rng.randint(2, size=len(intlike))) assert intlike.ndim >= 1 rval = intlike * float(round) rval = rval.astype(prior_vals.dtype) return rval
def qln_cleanup(self, prior_vals, kern, candidate_vals): """ Undo the smooth relaxation applied to quantized log-normal variables """ round = tensor.get_constant_value( mt_dist.quantized_lognormal_get_round( prior_vals)) intlike = numpy.ceil(candidate_vals / float(round)) assert intlike.ndim >= 1 # in test problems, it seems possible to get stuck in a mode # where the EI optimum always gets rounded up to 3 # and so 2 is never tried, even though it is actually the best point. intlike = numpy.maximum(1, intlike - self.numpy_rng.randint(2, size=len(intlike))) assert intlike.ndim >= 1 rval = intlike * float(round) rval = rval.astype(prior_vals.dtype) return rval
def remove_constants_and_unused_inputs_scan(node): """ Move constants into the inner graph, and remove unused inputs. Constants that are in the outer graph are represented by a free symbolic variable in the inner graph. If we move them into the inner graph, constant-folding can happen in the inner graph. This is applied only on sequences and non-sequences, not on initial states. """ if not isinstance(node.op, scan_op.Scan): return False op = node.op # We only need to take care of sequences and other arguments st = op.n_seqs st += int(numpy.sum([len(x) for x in op.tap_array[: (op.n_mit_mot + op.n_mit_sot)]])) st += op.n_sit_sot st += op.n_shared_outs op_ins, op_outs = scan_utils.reconstruct_graph(op.inputs, op.outputs) # Corresponds to the initial states, which should stay untouched. # We put those variables aside, and put them back at the end. out_stuff_inner = op_ins[op.n_seqs : st] non_seqs = op_ins[st:] st = op.n_seqs + op.n_mit_mot + op.n_mit_sot + op.n_sit_sot + op.n_nit_sot + op.n_shared_outs + 1 outer_non_seqs = node.inputs[st:] out_stuff_outer = node.inputs[1 + op.n_seqs : st] # To replace constants in the outer graph by clones in the inner graph givens = {} # All the inputs of the inner graph of the new scan nw_inner = [] # Same for the outer graph, initialized w/ number of steps nw_outer = [node.inputs[0]] all_ins = gof.graph.inputs(op_outs) for idx in xrange(op.n_seqs): if ( isinstance(node.inputs[idx + 1], tensor.TensorConstant) and node.inputs[idx + 1].tag.unique_value is not None ): try: # This works if input is a constant that has all entries # equal val = tensor.get_constant_value(node.inputs[idx + 1]) givens[op_ins[idx]] = node.inputs[idx + 1].clone()[0] except TypeError: pass elif op_ins[idx] in all_ins: nw_inner += [op_ins[idx]] nw_outer += [node.inputs[idx + 1]] nw_n_seqs = len(nw_inner) # Add outputs stuff nw_inner += out_stuff_inner nw_outer += out_stuff_outer # Look through non sequences for nw_in, nw_out in zip(non_seqs, outer_non_seqs): if isinstance(nw_out, tensor.Constant): givens[nw_in] = nw_out.clone() elif nw_in in all_ins: nw_inner += [nw_in] nw_outer += [nw_out] if len(nw_inner) != len(op_ins): op_outs = scan_utils.clone(op_outs, replace=givens) nw_info = op.info.copy() nw_info["n_seqs"] = nw_n_seqs # DEBUG CHECK nwScan = scan_op.Scan(nw_inner, op_outs, nw_info) nw_outs = nwScan.make_node(*nw_outer).outputs return nw_outs else: return False
def validate(self, image_shape, filter_shape, border_mode='valid', subsample=(1, 1), N_image_shape=None, N_filter_shape=None, input=None, filters=None, unroll_batch=None, unroll_kern=None, unroll_patch=None, verify_grad=True, should_raise=False): if N_image_shape is None: N_image_shape = [ T.get_constant_value(T.as_tensor_variable(x)) for x in image_shape ] if N_filter_shape is None: N_filter_shape = [ T.get_constant_value(T.as_tensor_variable(x)) for x in filter_shape ] if not input: input = self.input if not filters: filters = self.filters ############# THEANO IMPLEMENTATION ############ # we create a symbolic function so that verify_grad can work def sym_conv2d(input, filters): # define theano graph and function return conv.conv2d(input, filters, image_shape, filter_shape, border_mode, subsample, unroll_batch=unroll_batch, unroll_kern=unroll_kern, unroll_patch=unroll_patch) output = sym_conv2d(input, filters) theano_conv = theano.function([input, filters], output) # initialize input and compute result image_data = numpy.random.random(N_image_shape) filter_data = numpy.random.random(N_filter_shape) try: theano_output = theano_conv(image_data, filter_data) except ValueError: if not should_raise: raise return else: if should_raise: raise Exception("ConvOp should have generated an error") ############# REFERENCE IMPLEMENTATION ############ s = 1. orig_image_data = image_data if border_mode is not 'full': s = -1. out_shape2d = numpy.array(N_image_shape[-2:]) +\ s*numpy.array(N_filter_shape[-2:]) - s out_shape2d = numpy.ceil(out_shape2d / numpy.array(subsample)) out_shape = (N_image_shape[0], N_filter_shape[0]) + tuple(out_shape2d) ref_output = numpy.zeros(out_shape) # loop over output feature maps ref_output.fill(0) if border_mode == 'full': image_data2 = numpy.zeros( (N_image_shape[0], N_image_shape[1], N_image_shape[2] + 2 * N_filter_shape[2] - 2, N_image_shape[3] + 2 * N_filter_shape[3] - 2)) image_data2[:, :, N_filter_shape[2] - 1:N_filter_shape[2] - 1 + N_image_shape[2], N_filter_shape[3] - 1:N_filter_shape[3] - 1 + N_image_shape[3]] = image_data image_data = image_data2 N_image_shape = image_data.shape for bb in range(N_image_shape[0]): for nn in range(N_filter_shape[0]): for im0 in range(N_image_shape[1]): filter2d = filter_data[nn, im0, :, :] image2d = image_data[bb, im0, :, :] for row in range(ref_output.shape[2]): irow = row * subsample[0] #image row for col in range(ref_output.shape[3]): icol = col * subsample[1] #image col ref_output[bb, nn, row, col] += ( image2d[irow:irow + N_filter_shape[2], icol:icol + N_filter_shape[3]] * filter2d[::-1, ::-1]).sum() self.assertTrue(_allclose(theano_output, ref_output)) ############# TEST GRADIENT ############ if verify_grad: utt.verify_grad(sym_conv2d, [orig_image_data, filter_data])
def validate(self, image_shape, filter_shape, border_mode='valid', subsample=(1, 1), N_image_shape=None, N_filter_shape=None, input=None, filters=None, unroll_batch=None, unroll_kern=None, unroll_patch=None, verify_grad=True, should_raise=False): if N_image_shape is None: N_image_shape = [T.get_constant_value(T. as_tensor_variable(x)) for x in image_shape] if N_filter_shape is None: N_filter_shape = [T.get_constant_value(T. as_tensor_variable(x)) for x in filter_shape] if not input: input = self.input if not filters: filters = self.filters ############# THEANO IMPLEMENTATION ############ # we create a symbolic function so that verify_grad can work def sym_conv2d(input, filters): # define theano graph and function return conv.conv2d(input, filters, image_shape, filter_shape, border_mode, subsample, unroll_batch=unroll_batch, unroll_kern=unroll_kern, unroll_patch=unroll_patch) output = sym_conv2d(input, filters) theano_conv = theano.function([input, filters], output) # initialize input and compute result image_data = numpy.random.random(N_image_shape) filter_data = numpy.random.random(N_filter_shape) try: theano_output = theano_conv(image_data, filter_data) except ValueError: if not should_raise: raise return else: if should_raise: raise Exception( "ConvOp should have generated an error") ############# REFERENCE IMPLEMENTATION ############ s = 1. orig_image_data = image_data if border_mode is not 'full': s = -1. out_shape2d = numpy.array(N_image_shape[-2:]) +\ s * numpy.array(N_filter_shape[-2:]) - s out_shape2d = numpy.ceil(out_shape2d / numpy.array(subsample)) out_shape = (N_image_shape[0], N_filter_shape[0]) + tuple(out_shape2d) ref_output = numpy.zeros(out_shape) # loop over output feature maps ref_output.fill(0) if border_mode == 'full': image_data2 = numpy.zeros((N_image_shape[0], N_image_shape[1], N_image_shape[2] + 2 * N_filter_shape[2] - 2, N_image_shape[3] + 2 * N_filter_shape[3] - 2)) image_data2[:, :, N_filter_shape[2] - 1:N_filter_shape[2] - 1 + N_image_shape[2], N_filter_shape[3] - 1:N_filter_shape[3] - 1 + N_image_shape[3]] = image_data image_data = image_data2 N_image_shape = image_data.shape for bb in range(N_image_shape[0]): for nn in range(N_filter_shape[0]): for im0 in range(N_image_shape[1]): filter2d = filter_data[nn, im0, :, :] image2d = image_data[bb, im0, :, :] for row in range(ref_output.shape[2]): irow = row * subsample[0] # image row for col in range(ref_output.shape[3]): icol = col * subsample[1] # image col ref_output[bb, nn, row, col] += (image2d[ irow:irow + N_filter_shape[2], icol:icol + N_filter_shape[3]] * filter2d[::-1,::-1] ).sum() self.assertTrue(_allclose(theano_output, ref_output)) ############# TEST GRADIENT ############ if verify_grad: utt.verify_grad(sym_conv2d, [orig_image_data, filter_data])
def remove_constants_and_unused_inputs_scan(node): ''' Move constants into the inner graph, and remove unused inputs. Constants that are in the outer graph are represented by a free symbolic variable in the inner graph. If we move them into the inner graph, constant-folding can happen in the inner graph. This is applied only on sequences and non-sequences, not on initial states. ''' if not isinstance(node.op, scan_op.Scan): return False op = node.op # We only need to take care of sequences and other arguments st = op.n_seqs st += int(numpy.sum([len(x) for x in op.tap_array[:(op.n_mit_mot + op.n_mit_sot)]])) st += op.n_sit_sot st += op.n_shared_outs op_ins, op_outs = scan_utils.reconstruct_graph(op.inputs, op.outputs) # Corresponds to the initial states, which should stay untouched. # We put those variables aside, and put them back at the end. out_stuff_inner = op_ins[op.n_seqs:st] non_seqs = op_ins[st:] st = (op.n_seqs + op.n_mit_mot + op.n_mit_sot + op.n_sit_sot + op.n_nit_sot + op.n_shared_outs + 1) outer_non_seqs = node.inputs[st:] out_stuff_outer = node.inputs[1 + op.n_seqs:st] # To replace constants in the outer graph by clones in the inner graph givens = {} # All the inputs of the inner graph of the new scan nw_inner = [] # Same for the outer graph, initialized w/ number of steps nw_outer = [node.inputs[0]] all_ins = gof.graph.inputs(op_outs) for idx in xrange(op.n_seqs): if (isinstance(node.inputs[idx + 1], tensor.TensorConstant) and node.inputs[idx + 1].tag.unique_value is not None): try: # This works if input is a constant that has all entries # equal val = tensor.get_constant_value(node.inputs[idx + 1]) givens[op_ins[idx]] = node.inputs[idx + 1].clone()[0] except TypeError: pass elif op_ins[idx] in all_ins: nw_inner += [op_ins[idx]] nw_outer += [node.inputs[idx + 1]] nw_n_seqs = len(nw_inner) # Add outputs stuff nw_inner += out_stuff_inner nw_outer += out_stuff_outer # Look through non sequences for nw_in, nw_out in zip(non_seqs, outer_non_seqs): if isinstance(nw_out, tensor.Constant): givens[nw_in] = nw_out.clone() elif nw_in in all_ins: nw_inner += [nw_in] nw_outer += [nw_out] if len(nw_inner) != len(op_ins): op_outs = scan_utils.clone(op_outs, replace=givens) nw_info = op.info.copy() nw_info['n_seqs'] = nw_n_seqs # DEBUG CHECK nwScan = scan_op.Scan(nw_inner, op_outs, nw_info) nw_outs = nwScan.make_node(*nw_outer).outputs return nw_outs else: return False