def test_local_sigm_times_exp(self): # Test the `local_sigm_times_exp` optimization. # exp(x) * sigm(-x) -> sigm(x) # exp(-x) * sigm(x) -> sigm(-x) def match(func, ops): # print [node.op.scalar_op for node in func.maker.fgraph.toposort()] assert [node.op for node in func.maker.fgraph.toposort()] == ops m = self.get_mode(excluding=["local_elemwise_fusion", "inplace"]) x, y = tt.vectors("x", "y") f = theano.function([x], sigmoid(-x) * tt.exp(x), mode=m) match(f, [sigmoid]) assert check_stack_trace(f, ops_to_check=sigmoid) f = theano.function([x], sigmoid(x) * tt.exp(-x), mode=m) match(f, [tt.neg, sigmoid]) assert check_stack_trace(f, ops_to_check=sigmoid) f = theano.function([x], -(-(-(sigmoid(x)))) * tt.exp(-x), mode=m) match(f, [tt.neg, sigmoid, tt.neg]) # assert check_stack_trace(f, ops_to_check=sigmoid) f = theano.function( [x, y], (sigmoid(x) * sigmoid(-y) * -tt.exp(-x) * tt.exp(x * y) * tt.exp(y)), mode=m, ) topo = f.maker.fgraph.toposort() for op, nb in [(sigmoid, 2), (tt.mul, 2), (tt.neg, 1), (tt.exp, 1)]: assert sum([n.op == op for n in topo]) == nb
def test_local_sigm_times_exp(self): """ Test the `local_sigm_times_exp` optimization. exp(x) * sigm(-x) -> sigm(x) exp(-x) * sigm(x) -> sigm(-x) """ def match(func, ops): # print [node.op.scalar_op for node in func.maker.fgraph.toposort()] assert [node.op for node in func.maker.fgraph.toposort()] == ops m = self.get_mode(excluding=['local_elemwise_fusion', 'inplace']) x, y = tensor.vectors('x', 'y') f = theano.function([x], sigmoid(-x) * tensor.exp(x), mode=m) match(f, [sigmoid]) assert check_stack_trace(f, ops_to_check=sigmoid) f = theano.function([x], sigmoid(x) * tensor.exp(-x), mode=m) match(f, [tensor.neg, sigmoid]) assert check_stack_trace(f, ops_to_check=sigmoid) f = theano.function([x], -(-(-(sigmoid(x)))) * tensor.exp(-x), mode=m) match(f, [tensor.neg, sigmoid, tensor.neg]) # assert check_stack_trace(f, ops_to_check=sigmoid) f = theano.function([x, y], (sigmoid(x) * sigmoid(-y) * -tensor.exp(-x) * tensor.exp(x * y) * tensor.exp(y)), mode=m) match( f, [sigmoid, tensor.mul, tensor.neg, tensor.exp, sigmoid, tensor.mul])
def test_local_sigm_times_exp(self): """ Test the `local_sigm_times_exp` optimization. exp(x) * sigm(-x) -> sigm(x) exp(-x) * sigm(x) -> sigm(-x) """ def match(func, ops): # print [node.op.scalar_op for node in func.maker.fgraph.toposort()] assert [node.op for node in func.maker.fgraph.toposort()] == ops m = self.get_mode(excluding=['local_elemwise_fusion', 'inplace']) x, y = tensor.vectors('x', 'y') f = theano.function([x], sigmoid(-x) * tensor.exp(x), mode=m) match(f, [sigmoid]) assert check_stack_trace(f, ops_to_check=sigmoid) f = theano.function([x], sigmoid(x) * tensor.exp(-x), mode=m) match(f, [tensor.neg, sigmoid]) assert check_stack_trace(f, ops_to_check=sigmoid) f = theano.function([x], -(-(-(sigmoid(x)))) * tensor.exp(-x), mode=m) match(f, [tensor.neg, sigmoid, tensor.neg]) # assert check_stack_trace(f, ops_to_check=sigmoid) f = theano.function( [x, y], (sigmoid(x) * sigmoid(-y) * -tensor.exp(-x) * tensor.exp(x * y) * tensor.exp(y)), mode=m) topo = f.maker.fgraph.toposort() for op, nb in [(sigmoid, 2), (tensor.mul, 2), (tensor.neg, 1), (tensor.exp, 1)]: assert sum([n.op == op for n in topo]) == nb
def test_blocksparse_inplace_gemv_opt(): b = tensor.fmatrix() W = tensor.ftensor4() h = tensor.ftensor3() iIdx = tensor.lmatrix() oIdx = tensor.lmatrix() o = sparse_block_dot(W, h, iIdx, b, oIdx) f = theano.function([W, h, iIdx, b, oIdx], o) if theano.config.mode == "FAST_COMPILE": assert not f.maker.fgraph.toposort()[-1].op.inplace assert check_stack_trace(f, ops_to_check=[sparse_block_gemv]) else: assert f.maker.fgraph.toposort()[-1].op.inplace assert check_stack_trace(f, ops_to_check=[sparse_block_gemv_inplace])
def test_blocksparse_inplace_outer_opt(): b = tensor.fmatrix() W = tensor.ftensor4() h = tensor.ftensor3() iIdx = tensor.lmatrix() oIdx = tensor.lmatrix() o = sparse_block_dot(W, h, iIdx, b, oIdx) f = theano.function([W, h, iIdx, b, oIdx], [o, tensor.grad(o.sum(), wrt=W)]) if theano.config.mode == "FAST_COMPILE": assert not f.maker.fgraph.toposort()[-1].op.inplace assert check_stack_trace(f, ops_to_check=sparse_block_outer) else: assert f.maker.fgraph.toposort()[-1].op.inplace assert check_stack_trace(f, ops_to_check=sparse_block_outer_inplace)
def run_gradinput(self, inputs_shape, filters_shape, output_shape, gradInputs_fn, ref, subsample=None, filter_flip=True, verify_grad=True, mode=None, border_mode='valid', provide_shape=False, target_op=None, check_trace=False, filter_dilation=None): if subsample is None: subsample = (1,) * (len(inputs_shape) - 2) if filter_dilation is None: filter_dilation = (1,) * (len(inputs_shape) - 2) output_val = numpy.random.random(output_shape).astype('float32') filters_val = numpy.random.random(filters_shape).astype('float32') output = self.shared(output_val) filters = self.shared(filters_val) if provide_shape: imshp = inputs_shape kshp = filters_shape else: imshp = None kshp = None if filter_flip: conv_mode = 'conv' else: conv_mode = 'cross' c = gradInputs_fn(border_mode=border_mode, subsample=subsample, filter_flip=filter_flip, imshp=imshp, kshp=kshp, filter_dilation=filter_dilation) c = c(filters, output, inputs_shape[2:]) c_ref = ref(filters, output, inputs_shape, border_mode=border_mode, subsample=subsample, conv_mode=conv_mode, filter_dilation=filter_dilation) f = theano.function([], c, mode=mode) f_ref = theano.function([], c_ref, mode='FAST_RUN') if target_op is not None: assert any([isinstance(n.op, target_op) for n in f.maker.fgraph.toposort()]) if check_trace: assert_true(check_stack_trace(f, ops_to_check=target_op)) res_ref = numpy.array(f_ref()) res = numpy.array(f()) utt.assert_allclose(res_ref, res) def abstract_conv_gradinputs(filters_val, output_val): conv_op = gradInputs_fn(border_mode=border_mode, subsample=subsample, filter_dilation=filter_dilation) return conv_op(filters_val, output_val, inputs_shape[2:]) if verify_grad: utt.verify_grad(abstract_conv_gradinputs, [filters_val, output_val], mode=mode, eps=1)
def run_gradweight(self, inputs_shape, filters_shape, output_shape, ref=conv_corr_gw, subsample=(1, 1), filter_flip=True, verify_grad=True, mode=None, border_mode='valid', provide_shape=False, target_op=None, check_trace=False, filter_dilation=(1, 1)): inputs_val = numpy.random.random(inputs_shape).astype('float32') output_val = numpy.random.random(output_shape).astype('float32') inputs = self.shared(inputs_val) output = self.shared(output_val) if provide_shape: imshp = inputs_shape kshp = filters_shape else: imshp = None kshp = None if filter_flip: conv_mode = 'conv' else: conv_mode = 'cross' c = conv.AbstractConv2d_gradWeights(border_mode=border_mode, filter_flip=filter_flip, subsample=subsample, imshp=imshp, kshp=kshp, filter_dilation=filter_dilation) c = c(inputs, output, filters_shape[-2:]) c_ref = ref(inputs, output, filters_shape, border_mode=border_mode, subsample=subsample, conv_mode=conv_mode, filter_dilation=filter_dilation) f = theano.function([], c, mode=mode) f_ref = theano.function([], c_ref, mode='FAST_RUN') if target_op is not None: assert any([isinstance(n.op, target_op) for n in f.maker.fgraph.toposort()]) if check_trace: self.assertTrue(check_stack_trace(f, ops_to_check=target_op)) res_ref = numpy.array(f_ref()) res = numpy.array(f()) utt.assert_allclose(res_ref, res) def abstract_conv2d_gradweight(inputs_val, output_val): conv_op = conv.AbstractConv2d_gradWeights(border_mode=border_mode, subsample=subsample, filter_dilation=filter_dilation) return conv_op(inputs_val, output_val, filters_shape[-2:]) if verify_grad: utt.verify_grad(abstract_conv2d_gradweight, [inputs_val, output_val], mode=mode, eps=1)
def test_1msigmoid(self): if not register_local_1msigmoid: return m = self.get_mode() x = T.fmatrix() # tests exp_over_1_plus_exp f = theano.function([x], 1 - T.exp(x) / (1 + T.exp(x)), mode=m) assert check_stack_trace(f, ops_to_check=[tensor.neg, sigmoid_inplace]) assert [node.op for node in f.maker.fgraph.toposort()] == [ tensor.neg, sigmoid_inplace] # tests inv_1_plus_exp f = theano.function([x], 1 - T.fill(x, 1.0) / (1 + T.exp(-x)), mode=m) assert check_stack_trace(f, ops_to_check=[tensor.neg, sigmoid_inplace]) assert ([node.op for node in f.maker.fgraph.toposort()] == [tensor.neg, sigmoid_inplace])
def test_1msigmoid(self): if not register_local_1msigmoid: return m = self.get_mode() x = T.fmatrix() # tests exp_over_1_plus_exp f = theano.function([x], 1 - T.exp(x) / (1 + T.exp(x)), mode=m) assert check_stack_trace(f, ops_to_check=[tensor.neg, sigmoid_inplace]) assert [node.op for node in f.maker.fgraph.toposort() ] == [tensor.neg, sigmoid_inplace] # tests inv_1_plus_exp f = theano.function([x], 1 - T.fill(x, 1.0) / (1 + T.exp(-x)), mode=m) assert check_stack_trace(f, ops_to_check=[tensor.neg, sigmoid_inplace]) assert ([node.op for node in f.maker.fgraph.toposort() ] == [tensor.neg, sigmoid_inplace])
def test_local_ultra_fast_sigmoid(self): x = tt.matrix("x") s = sigmoid(x) mode = self.get_mode("local_ultra_fast_sigmoid") f = theano.function([x], s, mode=mode) assert check_stack_trace(f, ops_to_check=sigmoid) topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert topo[0].op == sigmoid mode = self.get_mode().including("local_ultra_fast_sigmoid") f = theano.function([x], s, mode=mode) assert check_stack_trace(f, ops_to_check=ultra_fast_sigmoid) topo = f.maker.fgraph.toposort() assert topo[0].op == ultra_fast_sigmoid assert len(topo) == 1 f([[-50, -10, -4, -1, 0, 1, 4, 10, 50]])
def test_local_ultra_fast_sigmoid(self): x = tensor.matrix('x') s = sigmoid(x) mode = self.get_mode('local_ultra_fast_sigmoid') f = theano.function([x], s, mode=mode) assert check_stack_trace(f, ops_to_check=sigmoid) topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert topo[0].op == sigmoid mode = self.get_mode().including('local_ultra_fast_sigmoid') f = theano.function([x], s, mode=mode) assert check_stack_trace(f, ops_to_check=ultra_fast_sigmoid) topo = f.maker.fgraph.toposort() assert topo[0].op == ultra_fast_sigmoid assert len(topo) == 1 f([[-50, -10, -4, -1, 0, 1, 4, 10, 50]])
def test_local_hard_sigmoid(self): x = tt.matrix("x") s = sigmoid(x) mode = self.get_mode("local_hard_sigmoid") f = theano.function([x], s, mode=mode) assert check_stack_trace(f, ops_to_check=sigmoid) topo = f.maker.fgraph.toposort() assert topo[0].op == sigmoid assert len(topo) == 1 mode = self.get_mode().including("local_hard_sigmoid") f = theano.function([x], s, mode=mode) topo = f.maker.fgraph.toposort() assert not any([n.op == sigmoid for n in topo]) f([[-50, -10, -4, -1, 0, 1, 4, 10, 50]]) mode2 = mode.excluding("fusion").excluding("inplace") f2 = theano.function([x], s, mode=mode2) assert check_stack_trace(f2, ops_to_check=tt.clip)
def run_fwd(self, inputs_shape, filters_shape, ref=conv_corr, subsample=(1, 1), verify_grad=True, mode=None, border_mode='valid', filter_flip=True, provide_shape=False, target_op=None, check_trace=False, filter_dilation=(1, 1)): inputs_val = numpy.random.random(inputs_shape).astype('float32') filters_val = numpy.random.random(filters_shape).astype('float32') inputs = self.shared(inputs_val) filters = self.shared(filters_val) if provide_shape: imshp = inputs_shape kshp = filters_shape else: imshp = None kshp = None if filter_flip: conv_mode = 'conv' else: conv_mode = 'cross' c_ref = ref(inputs, filters, border_mode=border_mode, subsample=subsample, conv_mode=conv_mode, filter_dilation=filter_dilation) c = conv.conv2d(inputs, filters, border_mode=border_mode, subsample=subsample, filter_flip=filter_flip, input_shape=imshp, filter_shape=kshp, filter_dilation=filter_dilation) f_ref = theano.function([], c_ref, mode='FAST_RUN') f = theano.function([], c, mode=mode) if target_op is not None: assert any([isinstance(n.op, target_op) for n in f.maker.fgraph.toposort()]) if check_trace: assert_true(check_stack_trace(f, ops_to_check=target_op)) res_ref = numpy.array(f_ref()) res = numpy.array(f()) utt.assert_allclose(res_ref, res) if verify_grad: utt.verify_grad(conv.AbstractConv2d(border_mode=border_mode, imshp=imshp, kshp=kshp, subsample=subsample, filter_dilation=filter_dilation), [inputs_val, filters_val], mode=mode)
def test_local_hard_sigmoid(self): x = tensor.matrix('x') s = sigmoid(x) mode = self.get_mode('local_hard_sigmoid') f = theano.function([x], s, mode=mode) assert check_stack_trace(f, ops_to_check=sigmoid) topo = f.maker.fgraph.toposort() assert topo[0].op == sigmoid assert len(topo) == 1 mode = self.get_mode().including('local_hard_sigmoid') f = theano.function([x], s, mode=mode) topo = f.maker.fgraph.toposort() assert not any([n.op == sigmoid for n in topo]) f([[-50, -10, -4, -1, 0, 1, 4, 10, 50]]) mode2 = mode.excluding('fusion').excluding('inplace') f2 = theano.function([x], s, mode=mode2) self.assertTrue(check_stack_trace(f2, ops_to_check=theano.tensor.clip))
def run_gradweight(self, inputs_shape, filters_shape, output_shape, ref=conv_corr_gw, subsample=(1, 1), filter_flip=True, verify_grad=True, mode=None, border_mode='valid', provide_shape=False, target_op=None, check_trace=False): inputs_val = numpy.random.random(inputs_shape).astype('float32') output_val = numpy.random.random(output_shape).astype('float32') inputs = self.shared(inputs_val) output = self.shared(output_val) if provide_shape: imshp = inputs_shape kshp = filters_shape else: imshp = None kshp = None if filter_flip: conv_mode = 'conv' else: conv_mode = 'cross' c = conv.AbstractConv2d_gradWeights(border_mode=border_mode, filter_flip=filter_flip, subsample=subsample, imshp=imshp, kshp=kshp) c = c(inputs, output, filters_shape[-2:]) c_ref = ref(inputs, output, filters_shape, border_mode=border_mode, subsample=subsample, conv_mode=conv_mode) f = theano.function([], c, mode=mode) f_ref = theano.function([], c_ref, mode='FAST_RUN') if target_op is not None: assert any([isinstance(n.op, target_op) for n in f.maker.fgraph.toposort()]) if check_trace: self.assertTrue(check_stack_trace(f, ops_to_check=target_op)) res_ref = numpy.array(f_ref()) res = numpy.array(f()) utt.assert_allclose(res_ref, res) def abstract_conv2d_gradweight(inputs_val, output_val): conv_op = conv.AbstractConv2d_gradWeights(border_mode=border_mode, subsample=subsample) return conv_op(inputs_val, output_val, filters_shape[-2:]) if verify_grad: utt.verify_grad(abstract_conv2d_gradweight, [inputs_val, output_val], mode=mode, eps=1)
def run_fwd(self, inputs_shape, filters_shape, ref=conv_corr, subsample=(1, 1), verify_grad=True, mode=None, border_mode='valid', filter_flip=True, provide_shape=False, target_op=None, check_trace=False): inputs_val = numpy.random.random(inputs_shape).astype('float32') filters_val = numpy.random.random(filters_shape).astype('float32') inputs = self.shared(inputs_val) filters = self.shared(filters_val) if provide_shape: imshp = inputs_shape kshp = filters_shape else: imshp = None kshp = None if filter_flip: conv_mode = 'conv' else: conv_mode = 'cross' c_ref = ref(inputs, filters, border_mode=border_mode, subsample=subsample, conv_mode=conv_mode) c = conv.conv2d(inputs, filters, border_mode=border_mode, subsample=subsample, filter_flip=filter_flip, input_shape=imshp, filter_shape=kshp) f_ref = theano.function([], c_ref, mode='FAST_RUN') f = theano.function([], c, mode=mode) if target_op is not None: assert any([isinstance(n.op, target_op) for n in f.maker.fgraph.toposort()]) if check_trace: self.assertTrue(check_stack_trace(f, ops_to_check=target_op)) res_ref = numpy.array(f_ref()) res = numpy.array(f()) utt.assert_allclose(res_ref, res) if verify_grad: utt.verify_grad(conv.AbstractConv2d(border_mode=border_mode, imshp=imshp, kshp=kshp, subsample=subsample), [inputs_val, filters_val], mode=mode)
def check_diagonal_subtensor_view_traces(fn): assert check_stack_trace(fn, ops_to_check=(DiagonalSubtensor, IncDiagonalSubtensor))
def run_fwd(self, inputs_shape, filters_shape, conv_fn, conv_op, ref, subsample=None, verify_grad=True, mode=None, border_mode='valid', filter_flip=True, provide_shape=False, target_op=None, check_trace=False, filter_dilation=None): if subsample is None: subsample = (1,) * (len(inputs_shape) - 2) if filter_dilation is None: filter_dilation = (1,) * (len(inputs_shape) - 2) inputs_val = numpy.random.random(inputs_shape).astype('float32') filters_val = numpy.random.random(filters_shape).astype('float32') # scale down values to prevent rounding errors inputs_val /= 10 filters_val /= 10 inputs = self.shared(inputs_val) filters = self.shared(filters_val) if provide_shape: imshp = inputs_shape kshp = filters_shape else: imshp = None kshp = None if filter_flip: conv_mode = 'conv' else: conv_mode = 'cross' c_ref = ref(inputs, filters, border_mode=border_mode, subsample=subsample, conv_mode=conv_mode, filter_dilation=filter_dilation) c = conv_fn(inputs, filters, border_mode=border_mode, subsample=subsample, filter_flip=filter_flip, input_shape=imshp, filter_shape=kshp, filter_dilation=filter_dilation) f_ref = theano.function([], c_ref, mode='FAST_RUN') f = theano.function([], c, mode=mode) if target_op is not None: assert any([isinstance(n.op, target_op) for n in f.maker.fgraph.toposort()]) if check_trace: assert_true(check_stack_trace(f, ops_to_check=target_op)) res_ref = numpy.array(f_ref()) res = numpy.array(f()) utt.assert_allclose(res_ref, res) if verify_grad: utt.verify_grad(conv_op(border_mode=border_mode, imshp=imshp, kshp=kshp, subsample=subsample, filter_dilation=filter_dilation), [inputs_val, filters_val], mode=mode)