def init_kernels(self):
self.kernels = []
self.is_refinable = {}
self.bounds = {}
self.params = []
self.param_bounds = []
self.idxs_mulsets = {}
self.post_refinement_cleanup = {}
for iv in self.s_prior:
dist_name = montetheano.rstreams.rv_dist_name(iv.vals)
if dist_name == 'normal':
k = SquaredExponentialKernel()
self.is_refinable[k] = get_refinability(iv, dist_name)
self.bounds[k] = (None, None)
elif dist_name == 'uniform':
k = SquaredExponentialKernel()
self.is_refinable[k] = get_refinability(iv, dist_name)
if self.is_refinable[k]:
low = tensor.get_constant_value(
mt_dist.uniform_get_low(iv.vals))
high = tensor.get_constant_value(
mt_dist.uniform_get_high(iv.vals))
self.bounds[k] = (low, high)
elif dist_name == 'lognormal':
k = LogSquaredExponentialKernel()
self.is_refinable[k] = get_refinability(iv, dist_name)
self.bounds[k] = (1e-8, None)
elif dist_name == 'quantized_lognormal':
k = LogSquaredExponentialKernel()
self.is_refinable[k] = get_refinability(iv, dist_name)
if self.is_refinable:
lbound = tensor.get_constant_value(
mt_dist.quantized_lognormal_get_round(
iv.vals))
self.bounds[k] = (lbound, None)
ff = picklable_instancemethod(self, 'qln_cleanup')
self.post_refinement_cleanup[k] = ff
elif dist_name == 'categorical':
# XXX: a better CategoryKernel would have different
# similarities for different choices
k = CategoryKernel()
self.is_refinable[k] = False
# refinable is false, so not setting bounds
else:
raise TypeError("unsupported distribution", dist_name)
self.kernels.append(k)
self.params.extend(k.params())
self.param_bounds.extend(k.param_bounds())
# XXX : to be more robust, it would be nice to build an Env with
# the idxs as outputs, and then run the MergeOptimizer on it.
self.idxs_mulsets.setdefault(iv.idxs, []).append(k)
def init_kernels(self):
self.kernels = []
self.is_refinable = {}
self.bounds = {}
self.params = []
self.param_bounds = []
self.idxs_mulsets = {}
self.post_refinement_cleanup = {}
for iv in self.s_prior:
dist_name = montetheano.rstreams.rv_dist_name(iv.vals)
if dist_name == 'normal':
k = SquaredExponentialKernel()
self.is_refinable[k] = get_refinability(iv, dist_name)
self.bounds[k] = (None, None)
elif dist_name == 'uniform':
k = SquaredExponentialKernel()
self.is_refinable[k] = get_refinability(iv, dist_name)
if self.is_refinable[k]:
low = tensor.get_constant_value(
mt_dist.uniform_get_low(iv.vals))
high = tensor.get_constant_value(
mt_dist.uniform_get_high(iv.vals))
self.bounds[k] = (low, high)
elif dist_name == 'lognormal':
k = LogSquaredExponentialKernel()
self.is_refinable[k] = get_refinability(iv, dist_name)
self.bounds[k] = (1e-8, None)
elif dist_name == 'quantized_lognormal':
k = LogSquaredExponentialKernel()
self.is_refinable[k] = get_refinability(iv, dist_name)
if self.is_refinable:
lbound = tensor.get_constant_value(
mt_dist.quantized_lognormal_get_round(iv.vals))
self.bounds[k] = (lbound, None)
ff = picklable_instancemethod(self, 'qln_cleanup')
self.post_refinement_cleanup[k] = ff
elif dist_name == 'categorical':
# XXX: a better CategoryKernel would have different
# similarities for different choices
k = CategoryKernel()
self.is_refinable[k] = False
# refinable is false, so not setting bounds
else:
raise TypeError("unsupported distribution", dist_name)
self.kernels.append(k)
self.params.extend(k.params())
self.param_bounds.extend(k.param_bounds())
# XXX : to be more robust, it would be nice to build an Env with
# the idxs as outputs, and then run the MergeOptimizer on it.
self.idxs_mulsets.setdefault(iv.idxs, []).append(k)
def get_refinability(v, dist_name):
v = v.vals
if dist_name == 'uniform':
params = [mt_dist.uniform_get_low(v), mt_dist.uniform_get_high(v)]
elif dist_name == 'normal':
params = [mt_dist.normal_get_mu(v), mt_dist.normal_get_sigma(v)]
elif dist_name == 'lognormal':
params = [mt_dist.lognormal_get_mu(v), mt_dist.lognormal_get_sigma(v)]
elif dist_name == 'quantized_lognormal':
params = [mt_dist.quantized_lognormal_get_mu(v),
mt_dist.quantized_lognormal_get_sigma(v),
mt_dist.quantized_lognormal_get_round(v)]
for p in params:
try:
tensor.get_constant_value(p)
except TypeError:
return False
return True
def belongs_to_set(self, node, set_nodes):
"""
This function checks if node `node` belongs to `set_nodes`, in the
sense that it can be merged together with every other node in
`set_nodes`. In order for two nodes to be mergeable, they have to go
over the same number of steps, have the same condition (if any),
have the same value for truncate_gradient, and have the same mode.
Questionable, we should also consider profile ?
"""
rep = set_nodes[0]
if not rep.op.as_while and node.op.as_while:
return False
nsteps = node.inputs[0]
try:
nsteps = int(get_constant_value(nsteps))
except TypeError:
pass
rep_nsteps = rep.inputs[0]
try:
rep_nsteps = int(get_constant_value(rep_nsteps))
except TypeError:
pass
# Check to see if it is an input of a different node
can_add = True
for nd in set_nodes:
if find_up(node, nd) or find_up(nd, node):
can_add = False
can_add = can_add and (node.op.truncate_gradient ==
rep.op.truncate_gradient)
can_add = can_add and (node.op.mode == rep.op.mode)
if not node.op.as_while:
return nsteps == rep_nsteps and can_add
cond = node.op.outputs[-1]
rep_cond = rep.op.outputs[-1]
same_cond = scan_utils.equal_computations([cond], [rep_cond],
node.op.inputs,
rep.op.inputs)
return same_cond and (nsteps == rep_nsteps) and can_add
def belongs_to_set(self, node, set_nodes):
"""
This function checks if node `node` belongs to `set_nodes`, in the
sense that it can be merged together with every other node in
`set_nodes`. In order for two nodes to be mergeable, they have to go
over the same number of steps, have the same condition (if any),
have the same value for truncate_gradient, and have the same mode.
Questionable, we should also consider profile ?
"""
rep = set_nodes[0]
if not rep.op.as_while and node.op.as_while:
return False
nsteps = node.inputs[0]
try:
nsteps = int(get_constant_value(nsteps))
except TypeError:
pass
rep_nsteps = rep.inputs[0]
try:
rep_nsteps = int(get_constant_value(rep_nsteps))
except TypeError:
pass
# Check to see if it is an input of a different node
can_add = True
for nd in set_nodes:
if find_up(node, nd) or find_up(nd, node):
can_add = False
can_add = can_add and (node.op.truncate_gradient ==
rep.op.truncate_gradient)
can_add = can_add and (node.op.mode == rep.op.mode)
if not node.op.as_while:
return nsteps == rep_nsteps and can_add
cond = node.op.outputs[-1]
rep_cond = rep.op.outputs[-1]
same_cond = scan_utils.equal_computations([cond], [rep_cond],
node.op.inputs,
rep.op.inputs)
return same_cond and (nsteps == rep_nsteps) and can_add
def get_refinability(v, dist_name):
v = v.vals
if dist_name == 'uniform':
params = [mt_dist.uniform_get_low(v), mt_dist.uniform_get_high(v)]
elif dist_name == 'normal':
params = [mt_dist.normal_get_mu(v), mt_dist.normal_get_sigma(v)]
elif dist_name == 'lognormal':
params = [mt_dist.lognormal_get_mu(v), mt_dist.lognormal_get_sigma(v)]
elif dist_name == 'quantized_lognormal':
params = [
mt_dist.quantized_lognormal_get_mu(v),
mt_dist.quantized_lognormal_get_sigma(v),
mt_dist.quantized_lognormal_get_round(v)
]
for p in params:
try:
tensor.get_constant_value(p)
except TypeError:
return False
return True
def is_positive(v):
if hints(v).get('positive', False):
return True
#TODO: how to handle this - a registry?
# infer_hints on Ops?
logger.debug('is_positive: %s' % str(v))
if v.owner and v.owner.op == tensor.pow:
try:
exponent = tensor.get_constant_value(v.owner.inputs[1])
except TypeError:
return False
if 0 == exponent % 2:
return True
return False
def is_positive(v):
if hints(v).get('positive', False):
return True
#TODO: how to handle this - a registry?
# infer_hints on Ops?
logger.debug('is_positive: %s' % str(v))
if v.owner and v.owner.op == tensor.pow:
try:
exponent = tensor.get_constant_value(v.owner.inputs[1])
except TypeError:
return False
if 0 == exponent % 2:
return True
return False
def is_positive(v):
if hints(v).get('positive', False):
return True
#TODO: how to handle this - a registry?
# infer_hints on Ops?
print 'is_positive', v
if v.owner and v.owner.op == tensor.pow:
print 'try for pow', v, v.owner.inputs
try:
exponent = tensor.get_constant_value(v.owner.inputs[1])
except TypeError:
return False
if 0 == exponent % 2:
return True
return False
def qln_cleanup(self, prior_vals, kern, candidate_vals):
"""
Undo the smooth relaxation applied to quantized log-normal variables
"""
round = tensor.get_constant_value(
mt_dist.quantized_lognormal_get_round(prior_vals))
intlike = numpy.ceil(candidate_vals / float(round))
assert intlike.ndim >= 1
# in test problems, it seems possible to get stuck in a mode
# where the EI optimum always gets rounded up to 3
# and so 2 is never tried, even though it is actually the best point.
intlike = numpy.maximum(
1, intlike - self.numpy_rng.randint(2, size=len(intlike)))
assert intlike.ndim >= 1
rval = intlike * float(round)
rval = rval.astype(prior_vals.dtype)
return rval
def qln_cleanup(self, prior_vals, kern, candidate_vals):
"""
Undo the smooth relaxation applied to quantized log-normal variables
"""
round = tensor.get_constant_value(
mt_dist.quantized_lognormal_get_round(
prior_vals))
intlike = numpy.ceil(candidate_vals / float(round))
assert intlike.ndim >= 1
# in test problems, it seems possible to get stuck in a mode
# where the EI optimum always gets rounded up to 3
# and so 2 is never tried, even though it is actually the best point.
intlike = numpy.maximum(1,
intlike - self.numpy_rng.randint(2, size=len(intlike)))
assert intlike.ndim >= 1
rval = intlike * float(round)
rval = rval.astype(prior_vals.dtype)
return rval
def remove_constants_and_unused_inputs_scan(node):
"""
Move constants into the inner graph, and remove unused inputs.
Constants that are in the outer graph are represented by a free symbolic
variable in the inner graph. If we move them into the inner graph,
constant-folding can happen in the inner graph.
This is applied only on sequences and non-sequences,
not on initial states.
"""
if not isinstance(node.op, scan_op.Scan):
return False
op = node.op
# We only need to take care of sequences and other arguments
st = op.n_seqs
st += int(numpy.sum([len(x) for x in op.tap_array[: (op.n_mit_mot + op.n_mit_sot)]]))
st += op.n_sit_sot
st += op.n_shared_outs
op_ins, op_outs = scan_utils.reconstruct_graph(op.inputs, op.outputs)
# Corresponds to the initial states, which should stay untouched.
# We put those variables aside, and put them back at the end.
out_stuff_inner = op_ins[op.n_seqs : st]
non_seqs = op_ins[st:]
st = op.n_seqs + op.n_mit_mot + op.n_mit_sot + op.n_sit_sot + op.n_nit_sot + op.n_shared_outs + 1
outer_non_seqs = node.inputs[st:]
out_stuff_outer = node.inputs[1 + op.n_seqs : st]
# To replace constants in the outer graph by clones in the inner graph
givens = {}
# All the inputs of the inner graph of the new scan
nw_inner = []
# Same for the outer graph, initialized w/ number of steps
nw_outer = [node.inputs[0]]
all_ins = gof.graph.inputs(op_outs)
for idx in xrange(op.n_seqs):
if (
isinstance(node.inputs[idx + 1], tensor.TensorConstant)
and node.inputs[idx + 1].tag.unique_value is not None
):
try:
# This works if input is a constant that has all entries
# equal
val = tensor.get_constant_value(node.inputs[idx + 1])
givens[op_ins[idx]] = node.inputs[idx + 1].clone()[0]
except TypeError:
pass
elif op_ins[idx] in all_ins:
nw_inner += [op_ins[idx]]
nw_outer += [node.inputs[idx + 1]]
nw_n_seqs = len(nw_inner)
# Add outputs stuff
nw_inner += out_stuff_inner
nw_outer += out_stuff_outer
# Look through non sequences
for nw_in, nw_out in zip(non_seqs, outer_non_seqs):
if isinstance(nw_out, tensor.Constant):
givens[nw_in] = nw_out.clone()
elif nw_in in all_ins:
nw_inner += [nw_in]
nw_outer += [nw_out]
if len(nw_inner) != len(op_ins):
op_outs = scan_utils.clone(op_outs, replace=givens)
nw_info = op.info.copy()
nw_info["n_seqs"] = nw_n_seqs
# DEBUG CHECK
nwScan = scan_op.Scan(nw_inner, op_outs, nw_info)
nw_outs = nwScan.make_node(*nw_outer).outputs
return nw_outs
else:
return False
def validate(self,
image_shape,
filter_shape,
border_mode='valid',
subsample=(1, 1),
N_image_shape=None,
N_filter_shape=None,
input=None,
filters=None,
unroll_batch=None,
unroll_kern=None,
unroll_patch=None,
verify_grad=True,
should_raise=False):
if N_image_shape is None:
N_image_shape = [
T.get_constant_value(T.as_tensor_variable(x))
for x in image_shape
]
if N_filter_shape is None:
N_filter_shape = [
T.get_constant_value(T.as_tensor_variable(x))
for x in filter_shape
]
if not input:
input = self.input
if not filters:
filters = self.filters
############# THEANO IMPLEMENTATION ############
# we create a symbolic function so that verify_grad can work
def sym_conv2d(input, filters):
# define theano graph and function
return conv.conv2d(input,
filters,
image_shape,
filter_shape,
border_mode,
subsample,
unroll_batch=unroll_batch,
unroll_kern=unroll_kern,
unroll_patch=unroll_patch)
output = sym_conv2d(input, filters)
theano_conv = theano.function([input, filters], output)
# initialize input and compute result
image_data = numpy.random.random(N_image_shape)
filter_data = numpy.random.random(N_filter_shape)
try:
theano_output = theano_conv(image_data, filter_data)
except ValueError:
if not should_raise: raise
return
else:
if should_raise:
raise Exception("ConvOp should have generated an error")
############# REFERENCE IMPLEMENTATION ############
s = 1.
orig_image_data = image_data
if border_mode is not 'full': s = -1.
out_shape2d = numpy.array(N_image_shape[-2:]) +\
s*numpy.array(N_filter_shape[-2:]) - s
out_shape2d = numpy.ceil(out_shape2d / numpy.array(subsample))
out_shape = (N_image_shape[0], N_filter_shape[0]) + tuple(out_shape2d)
ref_output = numpy.zeros(out_shape)
# loop over output feature maps
ref_output.fill(0)
if border_mode == 'full':
image_data2 = numpy.zeros(
(N_image_shape[0], N_image_shape[1],
N_image_shape[2] + 2 * N_filter_shape[2] - 2,
N_image_shape[3] + 2 * N_filter_shape[3] - 2))
image_data2[:, :, N_filter_shape[2] - 1:N_filter_shape[2] - 1 +
N_image_shape[2],
N_filter_shape[3] - 1:N_filter_shape[3] - 1 +
N_image_shape[3]] = image_data
image_data = image_data2
N_image_shape = image_data.shape
for bb in range(N_image_shape[0]):
for nn in range(N_filter_shape[0]):
for im0 in range(N_image_shape[1]):
filter2d = filter_data[nn, im0, :, :]
image2d = image_data[bb, im0, :, :]
for row in range(ref_output.shape[2]):
irow = row * subsample[0] #image row
for col in range(ref_output.shape[3]):
icol = col * subsample[1] #image col
ref_output[bb, nn, row, col] += (
image2d[irow:irow + N_filter_shape[2],
icol:icol + N_filter_shape[3]] *
filter2d[::-1, ::-1]).sum()
self.assertTrue(_allclose(theano_output, ref_output))
############# TEST GRADIENT ############
if verify_grad:
utt.verify_grad(sym_conv2d, [orig_image_data, filter_data])
def validate(self, image_shape, filter_shape,
border_mode='valid', subsample=(1, 1),
N_image_shape=None, N_filter_shape=None,
input=None, filters=None,
unroll_batch=None, unroll_kern=None, unroll_patch=None,
verify_grad=True, should_raise=False):
if N_image_shape is None:
N_image_shape = [T.get_constant_value(T.
as_tensor_variable(x)) for x in image_shape]
if N_filter_shape is None:
N_filter_shape = [T.get_constant_value(T.
as_tensor_variable(x)) for x in filter_shape]
if not input:
input = self.input
if not filters:
filters = self.filters
############# THEANO IMPLEMENTATION ############
# we create a symbolic function so that verify_grad can work
def sym_conv2d(input, filters):
# define theano graph and function
return conv.conv2d(input, filters, image_shape, filter_shape,
border_mode, subsample, unroll_batch=unroll_batch,
unroll_kern=unroll_kern, unroll_patch=unroll_patch)
output = sym_conv2d(input, filters)
theano_conv = theano.function([input, filters], output)
# initialize input and compute result
image_data = numpy.random.random(N_image_shape)
filter_data = numpy.random.random(N_filter_shape)
try:
theano_output = theano_conv(image_data, filter_data)
except ValueError:
if not should_raise:
raise
return
else:
if should_raise:
raise Exception(
"ConvOp should have generated an error")
############# REFERENCE IMPLEMENTATION ############
s = 1.
orig_image_data = image_data
if border_mode is not 'full':
s = -1.
out_shape2d = numpy.array(N_image_shape[-2:]) +\
s * numpy.array(N_filter_shape[-2:]) - s
out_shape2d = numpy.ceil(out_shape2d / numpy.array(subsample))
out_shape = (N_image_shape[0], N_filter_shape[0]) + tuple(out_shape2d)
ref_output = numpy.zeros(out_shape)
# loop over output feature maps
ref_output.fill(0)
if border_mode == 'full':
image_data2 = numpy.zeros((N_image_shape[0], N_image_shape[1],
N_image_shape[2] + 2 * N_filter_shape[2] - 2,
N_image_shape[3] + 2 * N_filter_shape[3] - 2))
image_data2[:, :, N_filter_shape[2] - 1:N_filter_shape[2] - 1 + N_image_shape[2],
N_filter_shape[3] - 1:N_filter_shape[3] - 1 + N_image_shape[3]] = image_data
image_data = image_data2
N_image_shape = image_data.shape
for bb in range(N_image_shape[0]):
for nn in range(N_filter_shape[0]):
for im0 in range(N_image_shape[1]):
filter2d = filter_data[nn, im0, :, :]
image2d = image_data[bb, im0, :, :]
for row in range(ref_output.shape[2]):
irow = row * subsample[0] # image row
for col in range(ref_output.shape[3]):
icol = col * subsample[1] # image col
ref_output[bb, nn, row, col] += (image2d[
irow:irow + N_filter_shape[2],
icol:icol + N_filter_shape[3]] * filter2d[::-1,::-1]
).sum()
self.assertTrue(_allclose(theano_output, ref_output))
############# TEST GRADIENT ############
if verify_grad:
utt.verify_grad(sym_conv2d, [orig_image_data, filter_data])
def remove_constants_and_unused_inputs_scan(node):
'''
Move constants into the inner graph, and remove unused inputs.
Constants that are in the outer graph are represented by a free symbolic
variable in the inner graph. If we move them into the inner graph,
constant-folding can happen in the inner graph.
This is applied only on sequences and non-sequences,
not on initial states.
'''
if not isinstance(node.op, scan_op.Scan):
return False
op = node.op
# We only need to take care of sequences and other arguments
st = op.n_seqs
st += int(numpy.sum([len(x) for x in
op.tap_array[:(op.n_mit_mot + op.n_mit_sot)]]))
st += op.n_sit_sot
st += op.n_shared_outs
op_ins, op_outs = scan_utils.reconstruct_graph(op.inputs, op.outputs)
# Corresponds to the initial states, which should stay untouched.
# We put those variables aside, and put them back at the end.
out_stuff_inner = op_ins[op.n_seqs:st]
non_seqs = op_ins[st:]
st = (op.n_seqs +
op.n_mit_mot +
op.n_mit_sot +
op.n_sit_sot +
op.n_nit_sot +
op.n_shared_outs + 1)
outer_non_seqs = node.inputs[st:]
out_stuff_outer = node.inputs[1 + op.n_seqs:st]
# To replace constants in the outer graph by clones in the inner graph
givens = {}
# All the inputs of the inner graph of the new scan
nw_inner = []
# Same for the outer graph, initialized w/ number of steps
nw_outer = [node.inputs[0]]
all_ins = gof.graph.inputs(op_outs)
for idx in xrange(op.n_seqs):
if (isinstance(node.inputs[idx + 1], tensor.TensorConstant) and
node.inputs[idx + 1].tag.unique_value is not None):
try:
# This works if input is a constant that has all entries
# equal
val = tensor.get_constant_value(node.inputs[idx + 1])
givens[op_ins[idx]] = node.inputs[idx + 1].clone()[0]
except TypeError:
pass
elif op_ins[idx] in all_ins:
nw_inner += [op_ins[idx]]
nw_outer += [node.inputs[idx + 1]]
nw_n_seqs = len(nw_inner)
# Add outputs stuff
nw_inner += out_stuff_inner
nw_outer += out_stuff_outer
# Look through non sequences
for nw_in, nw_out in zip(non_seqs, outer_non_seqs):
if isinstance(nw_out, tensor.Constant):
givens[nw_in] = nw_out.clone()
elif nw_in in all_ins:
nw_inner += [nw_in]
nw_outer += [nw_out]
if len(nw_inner) != len(op_ins):
op_outs = scan_utils.clone(op_outs, replace=givens)
nw_info = op.info.copy()
nw_info['n_seqs'] = nw_n_seqs
# DEBUG CHECK
nwScan = scan_op.Scan(nw_inner, op_outs, nw_info)
nw_outs = nwScan.make_node(*nw_outer).outputs
return nw_outs
else:
return False
