def test_grad(self): a = np.random.random((3, 5, 2)).astype(config.floatX) utt.verify_grad(self.op, [a]) # Test axis=None for axis in range(len(a.shape)): utt.verify_grad(self.op_class(axis=axis), [a], eps=4e-4)
def test_max_pool_2d_2D(self): rng = numpy.random.RandomState(utt.fetch_seed()) maxpoolshps = ((1, 1), (3, 2)) imval = rng.rand(4, 5) images = tensor.dmatrix() for maxpoolshp, ignore_border, mode in product(maxpoolshps, [True, False], ['max', 'sum', 'average_inc_pad', 'average_exc_pad']): # print 'maxpoolshp =', maxpoolshp # print 'ignore_border =', ignore_border numpy_output_val = self.numpy_max_pool_2d(imval, maxpoolshp, ignore_border, mode=mode) output = max_pool_2d(images, maxpoolshp, ignore_border, mode=mode) output_val = function([images], output)(imval) assert numpy.all(output_val == numpy_output_val), ( "output_val is %s, numpy_output_val is %s" % (output_val, numpy_output_val)) def mp(input): return max_pool_2d(input, maxpoolshp, ignore_border, mode=mode) utt.verify_grad(mp, [imval], rng=rng)
def test_DownsampleFactorMaxPaddingStride_grad_grad(self): rng = numpy.random.RandomState(utt.fetch_seed()) imgsizes = ((10, 10), (10, 5), (5, 5)) maxpoolsizes = ((5, 3), (3, 5), (3, 3)) stridesizes = ((3, 2), (2, 3), (3, 3)) paddingsizes = ((2, 2), (2, 1), (2, 2)) for i in range(len(imgsizes)): imgsize = imgsizes[i] imval = rng.rand(1, 1, imgsize[0], imgsize[1]) * 10.0 maxpoolsize = maxpoolsizes[i] stridesize = stridesizes[i] paddingsize = paddingsizes[i] grad_shape = DownsampleFactorMaxGradGrad.out_shape( imval.shape, maxpoolsize, st=stridesize, ignore_border=True, padding=paddingsize ) grad_val = rng.rand(*grad_shape) * 10.0 def mp(input, grad): out = DownsampleFactorMax(maxpoolsize, ignore_border=True, st=stridesize, padding=paddingsize)(input) grad_op = DownsampleFactorMaxGrad(maxpoolsize, ignore_border=True, st=stridesize, padding=paddingsize) return grad_op(input, out, grad) utt.verify_grad(mp, [imval, grad_val], rng=rng)
def test_DownsampleFactorMaxGrad_grad_st_extra(self): """checks the gradient of the gradient for the case that stride is used for extra examples""" rng = numpy.random.RandomState(utt.fetch_seed()) maxpoolshps = ((5, 3), (5, 3), (5, 3), (5, 5), (3, 2), (7, 7), (9, 9)) stridesizes = ((3, 2), (7, 5), (10, 6), (1, 1), (2, 3), (10, 10), (1, 1)) imvsizs = ((16, 16), (16, 16), (16, 16), (8, 5), (8, 5), (8, 5), (8, 5)) for indx in numpy.arange(len(maxpoolshps)): imvsize = imvsizs[indx] imval = rng.rand(1, 2, imvsize[0], imvsize[1]) stride = stridesizes[indx] maxpoolshp = maxpoolshps[indx] for ignore_border in [True, False]: grad_shape = DownsampleFactorMax.out_shape( imval.shape, maxpoolshp, ignore_border=ignore_border, st=stride ) grad_val = rng.rand(*grad_shape) def mp(input, grad): out = DownsampleFactorMax(maxpoolshp, ignore_border=ignore_border, st=stride)(input) grad_op = DownsampleFactorMaxGrad(maxpoolshp, ignore_border=ignore_border, st=stride) return grad_op(input, out, grad) # skip the grad verification when the output is empty if numpy.prod(grad_shape) == 0: continue utt.verify_grad(mp, [imval, grad_val], rng=rng)
def test_DownsampleFactorMaxGradGrad_grad(self): rng = numpy.random.RandomState(utt.fetch_seed()) # maxpool, stride, padding, input sizes examples = ( ((3,), (2,), (2,), (10,)), ((3,), (2,), (2,), (2, 10,)), ((3,), (2,), (2,), (2, 1, 10,)), ((5, 3), (3, 2), (2, 2), (1, 1, 10, 10)), ((5, 3), (3, 2), (2, 2), (1, 1, 10, 10)), ((3, 5), (2, 3), (2, 1), (1, 1, 10, 5)), ((3, 3), (3, 3), (2, 2), (1, 1, 5, 5)), ((5, 3, 3), (3, 2, 2), (2, 2, 2), (1, 1, 10, 5, 5)), ((3, 5, 3), (2, 3, 2), (2, 1, 2), (1, 1, 5, 10, 5)), ((3, 3, 5), (2, 2, 3), (2, 2, 1), (1, 1, 5, 5, 10)), ) for (maxpoolshp, stridesize, paddingsize, inputsize) in examples: imval1 = rng.rand(*inputsize) * 10.0 imval2 = rng.rand(*inputsize) * 10.0 def mp(input1, input2): op1 = Pool(ndim=len(maxpoolshp), ignore_border=True) pooled_out = op1(input1, maxpoolshp, stridesize, paddingsize) op2 = DownsampleFactorMaxGradGrad( ndim=len(maxpoolshp), ignore_border=True) out = op2(input1, pooled_out, input2, maxpoolshp, stridesize, paddingsize) return out utt.verify_grad(mp, [imval1, imval2], rng=rng)
def test_max_pool_3d_3D(self): rng = numpy.random.RandomState(utt.fetch_seed()) maxpoolshps = ((1, 1, 1), (3, 2, 1)) imval = rng.rand(4, 5, 6) images = tensor.dtensor3() for maxpoolshp, ignore_border, mode in product(maxpoolshps, [True, False], ['max', 'sum', 'average_inc_pad', 'average_exc_pad']): # print 'maxpoolshp =', maxpoolshp # print 'ignore_border =', ignore_border numpy_output_val = self.numpy_max_pool_nd(imval, maxpoolshp, ignore_border, mode=mode) output = pool_3d(images, maxpoolshp, ignore_border, mode=mode) output_val = function([images], output)(imval) utt.assert_allclose(output_val, numpy_output_val) def mp(input): return pool_3d(input, maxpoolshp, ignore_border, mode=mode) utt.verify_grad(mp, [imval], rng=rng)
def test_AveragePoolPaddingStride_grad_grad(self): rng = numpy.random.RandomState(utt.fetch_seed()) # avgpool, stride, padding, input sizes examples = ( ((3,), (2,), (2,), (10,)), ((3,), (2,), (2,), (2, 10,)), ((3,), (2,), (2,), (2, 1, 10,)), ((5, 3), (3, 2), (2, 2), (1, 1, 10, 10)), ((5, 3), (3, 2), (2, 2), (1, 1, 10, 10)), ((3, 5), (2, 3), (2, 1), (1, 1, 10, 5)), ((3, 3), (3, 3), (2, 2), (1, 1, 5, 5)), ((5, 3, 3), (3, 2, 2), (2, 2, 2), (1, 1, 10, 5, 5)), ((3, 5, 3), (2, 3, 2), (2, 1, 2), (1, 1, 5, 10, 5)), ((3, 3, 5), (2, 2, 3), (2, 2, 1), (1, 1, 5, 5, 10)), ) for (avgpoolshp, stridesize, paddingsize, inputsize) in examples: imval = rng.rand(*inputsize) * 10.0 # 'average_exc_pad' with non-zero padding is not implemented for mode in ['sum', 'average_inc_pad']: grad_shape = Pool.out_shape(imval.shape, avgpoolshp, ndim=len(avgpoolshp), st=stridesize, ignore_border=True, padding=paddingsize) grad_val = rng.rand(*grad_shape) * 10.0 def mp(input, grad): grad_op = AveragePoolGrad(ndim=len(avgpoolshp), ignore_border=True, mode=mode) return grad_op(input, grad, avgpoolshp, stridesize, paddingsize) utt.verify_grad(mp, [imval, grad_val], rng=rng)
def test_DownsampleFactorMax_grad(self): rng = numpy.random.RandomState(utt.fetch_seed()) # maxpool, input sizes examples = ( ((2,), (3,)), ((2,), (2, 3)), ((2,), (2, 3, 3)), ((1, 1), (2, 3, 3, 4)), ((3, 2), (2, 3, 3, 4)), ((2, 3), (2, 3, 3, 4)), ((1, 1, 1), (2, 3, 3)), ((3, 2, 2), (2, 3, 3, 4)), ((2, 3, 2), (2, 3, 3, 4, 4)), ((2, 2, 3), (2, 3, 3, 4, 4)), ) for example, ignore_border, mode in product(examples, [True, False], ['max', 'sum', 'average_inc_pad', 'average_exc_pad']): (maxpoolshp, inputsize) = example imval = rng.rand(*inputsize) * 10.0 # more variance means numeric gradient will be more accurate def mp(input): return Pool(ndim=len(maxpoolshp), ignore_border=ignore_border, mode=mode)(input, maxpoolshp) utt.verify_grad(mp, [imval], rng=rng)
def test_DownsampleFactorMaxPaddingStride_grad(self): rng = numpy.random.RandomState(utt.fetch_seed()) # maxpool, stride, padding, input sizes examples = ( ((10,), (5,), (3,), (2,)), ((10,), (5,), (3,), (2, 2)), ((10,), (5,), (3,), (1, 1, 2)), ((10, 10), (5, 3), (3, 2), (1, 1, 2, 2)), ((10, 5), (3, 5), (2, 3), (1, 1, 2, 1)), ((5, 5), (3, 3), (3, 3), (1, 1, 2, 2)), ((5, 5, 5), (3, 3, 3), (3, 3, 3), (1, 1, 2, 2, 2)), ) # average_inc_pad and average_exc_pad do not # support grad with padding for mode in ['max', 'sum']: for example in examples: (maxpoolshp, stridesize, paddingsize, inputsize) = example imval = rng.rand(*inputsize) * 10.0 def mp(input): return Pool( ndim=len(maxpoolshp), ignore_border=True, mode=mode, )(input, maxpoolshp, stridesize, paddingsize) utt.verify_grad(mp, [imval], rng=rng)
def test_AveragePoolPaddingStride_grad_grad(self): rng = numpy.random.RandomState(utt.fetch_seed()) imgsizes = ((10, 10), (10, 5), (5, 5)) avgpoolsizes = ((5, 3), (3, 5), (3, 3)) stridesizes = ((3, 2), (2, 3), (3, 3)) paddingsizes = ((2, 2), (2, 1), (2, 2)) for i in range(len(imgsizes)): imgsize = imgsizes[i] imval = rng.rand(1, 1, imgsize[0], imgsize[1]) * 10.0 avgpoolsize = avgpoolsizes[i] stridesize = stridesizes[i] paddingsize = paddingsizes[i] # 'average_exc_pad' with non-zero padding is not implemented for mode in ['sum', 'average_inc_pad']: grad_shape = DownsampleFactorMax.out_shape(imval.shape, avgpoolsize, st=stridesize, ignore_border=True, padding=paddingsize) grad_val = rng.rand(*grad_shape) * 10.0 def mp(input, grad): grad_op = AveragePoolGrad(avgpoolsize, ignore_border=True, st=stridesize, padding=paddingsize, mode=mode) return grad_op(input, grad) utt.verify_grad(mp, [imval, grad_val], rng=rng)
def test_AveragePoolGrad_grad_st_extra(self): """checks the gradient of the gradient for the case that stride is used for extra examples""" rng = numpy.random.RandomState(utt.fetch_seed()) avgpoolshps = ((5, 3), (5, 3), (5, 3), (5, 5), (3, 2), (7, 7), (9, 9)) stridesizes = ((3, 2), (7, 5), (10, 6), (1, 1), (2, 3), (10, 10), (1, 1)) imvsizs = ((16, 16), (16, 16), (16, 16), (8, 5), (8, 5), (8, 5), (8, 5)) for indx in numpy.arange(len(avgpoolshps)): imvsize = imvsizs[indx] imval = rng.rand(1, 2, imvsize[0], imvsize[1]) stride = stridesizes[indx] avgpoolshp = avgpoolshps[indx] for ignore_border in [True, False]: for mode in ['sum', 'average_inc_pad', 'average_exc_pad']: grad_shape = Pool.out_shape( imval.shape, avgpoolshp, ignore_border=ignore_border, st=stride) grad_val = rng.rand(*grad_shape) def mp(input, grad): grad_op = AveragePoolGrad( avgpoolshp, ignore_border=ignore_border, st=stride, mode=mode) return grad_op(input, grad) # skip the grad verification when the output is empty if numpy.prod(grad_shape) == 0: continue utt.verify_grad(mp, [imval, grad_val], rng=rng)
def test_gradient(self): utt.verify_grad(fill_diagonal, [numpy.random.rand(5, 8), numpy.random.rand()], n_tests=1, rng=TestFillDiagonal.rng) utt.verify_grad(fill_diagonal, [numpy.random.rand(8, 5), numpy.random.rand()], n_tests=1, rng=TestFillDiagonal.rng)
def test_light_curve_grad(): u_val = np.array([0.2, 0.3, 0.1, 0.5]) b_val = np.linspace(-1.5, 1.5, 20) r_val = 0.1 + np.zeros_like(b_val) lc = lambda u, b, r: StarryLightCurve(u)._compute_light_curve(b, r) # NOQA utt.verify_grad(lc, [u_val, b_val, r_val])
def test_verify_grad_gauntlet(self): maxpoolshps = ((1, 1), (3, 3), (5, 3),) stridesizes = ((1, 1), (3, 3), (5, 7),) # generate random images imval = self.rng.rand(4, 10, 16, 16) for index_type, index_scope, maxpoolshp, stride, ignore_border in product(['flattened', 'array'], ['local'], maxpoolshps, stridesizes, [True, False]): unpoolswitch_op = UnpoolSwitch(ds = maxpoolshp, st=stride, index_type=index_type, index_scope=index_scope) if(index_type == 'flattened'): def op_with_fixed_switchs(x): x_with_zero_switchs = T.concatenate((x, T.zeros_like(x)), 1) return unpoolswitch_op(x_with_zero_switchs) else: def op_with_fixed_switchs(x): x_with_zero_switchs = T.concatenate((x, T.zeros_like(x), T.zeros_like(x)), 1) return unpoolswitch_op(x_with_zero_switchs) utt.verify_grad(op_with_fixed_switchs, [imval], rng=self.rng)
def test_batched_dot_gradient(self): for threshold in [0, 100]: unittest_tools.verify_grad( GpuBatchedDot(stream_threshold=threshold), [numpy.random.randn(5,7,2).astype(numpy.float32), numpy.random.randn(5,2,6).astype(numpy.float32)], mode=mode_with_gpu)
def test_DownsampleFactorMaxGradGrad_grad(self): rng = numpy.random.RandomState(utt.fetch_seed()) imgsizes = ((10, 10), (10, 5), (5, 5)) maxpoolsizes = ((5, 3), (3, 5), (3, 3)) stridesizes = ((3, 2), (2, 3), (3, 3)) paddingsizes = ((2, 2), (2, 1), (2, 2)) for i in range(len(imgsizes)): imgsize = imgsizes[i] imval1 = rng.rand(1, 1, imgsize[0], imgsize[1]) * 10.0 imval2 = rng.rand(1, 1, imgsize[0], imgsize[1]) * 10.0 maxpoolsize = maxpoolsizes[i] stridesize = stridesizes[i] paddingsize = paddingsizes[i] def mp(input1, input2): pooled_out = Pool( maxpoolsize, ignore_border=True, st=stridesize, padding=paddingsize, )(input1) out = DownsampleFactorMaxGradGrad( ds=maxpoolsize, ignore_border=True, st=stridesize, padding=paddingsize)(input1, pooled_out, input2) return out utt.verify_grad(mp, [imval1, imval2], rng=rng)
def test0(self): y_idx = [0, 1, 3] def f(a, b): return crossentropy_softmax_1hot_with_bias(a, b, y_idx)[0] utt.verify_grad(f, [numpy.random.rand(3, 4), numpy.random.rand(4)])
def test_maxpool(self): # generate flatted images maxpoolshps = ((2, 2), (3, 3), (4, 4), (5, 5), (6, 6)) imval = numpy.random.rand(4, 5, 10, 10) images = tensor.dmatrix() for maxpoolshp in maxpoolshps: # symbolic stuff output, outshp = sp.max_pool(images, imval.shape[1:], maxpoolshp) f = function([images], [output]) output_val = f(imval.reshape(imval.shape[0], -1)) # numeric verification my_output_val = numpy.zeros( (imval.shape[0], imval.shape[1], imval.shape[2] / maxpoolshp[0], imval.shape[3] / maxpoolshp[1]) ) assert numpy.prod(my_output_val.shape[1:]) == numpy.prod(numpy.r_[imval.shape[1], outshp]) for n in range(imval.shape[0]): for k in range(imval.shape[1]): for i in range(imval.shape[2] / maxpoolshp[0]): for j in range(imval.shape[3] / maxpoolshp[1]): ii, jj = i * maxpoolshp[0], j * maxpoolshp[1] patch = imval[n, k, ii : ii + maxpoolshp[0], jj : jj + maxpoolshp[1]] my_output_val[n, k, i, j] = numpy.max(patch) my_output_val = my_output_val.reshape(imval.shape[0], -1) assert numpy.all(output_val == my_output_val) def mp(input): output, outshp = sp.max_pool(input, imval.shape[1:], maxpoolshp) return output utt.verify_grad(mp, [imval.reshape(imval.shape[0], -1)])
def test_verify_grad_with_zeros(self): # including zeros, as the case with zeros is important # (and special cases: 1 zero in the row, more than 1 zero in the row) x_val = numpy.asarray([[1., 2., 3.], [0., 5., 6.], [0., 0., 9.]], dtype='float32') x = theano.tensor.dmatrix() # sanity check x2 = theano.tensor.dmatrix() p = Prod(axis=1)(x) p2 = Prod(axis=1)(x2) fn = theano.function([x, x2], [p - p2], mode=self.mode) #print "hand computed diff for each row" x2_val = numpy.asarray([[1., 2., 3.003], [0.003, 5., 6], [ 0., 0., 9.01]]) #print fn(x_val, x2_val) fn2 = theano.function([x], [theano.tensor.grad(p.sum(), x)], mode=self.mode) #print "real grad" #print fn2(x_val) fn3 = theano.function([x], [p], mode=self.mode) assert numpy.allclose(fn3(x_val), [6., 0., 0.]) # now with verify_grad unittest_tools.verify_grad(Prod(axis=1), [x_val], mode=self.mode)
def run_conv_valid(self, inputs_shape, filters_shape, border_mode='valid', filter_dilation=(1, 1, 1), subsample=(1, 1, 1), verify_grad=False): inputs_shape = [inputs_shape[i] for i in (0, 4, 1, 2, 3)] filters_shape = [filters_shape[i] for i in (0, 4, 1, 2, 3)] inputs_val = np.random.random(inputs_shape).astype(config.floatX) filters_val = np.random.random(filters_shape).astype(config.floatX) inputs = gpuarray_shared_constructor(inputs_val) filters = gpuarray_shared_constructor(filters_val) conv_ref = Corr3dMM(border_mode=border_mode, filter_dilation=filter_dilation, subsample=subsample)(ref_cast(inputs), ref_cast(filters)) f_ref = theano.function([], conv_ref, mode=mode_without_gpu) conv = GpuCorr3dMM(border_mode=border_mode, filter_dilation=filter_dilation, subsample=subsample)(inputs, filters) f = theano.function([], conv, mode=mode_with_gpu) res_ref = f_ref() res = f() utt.assert_allclose(res_ref, res) if verify_grad: utt.verify_grad(GpuCorr3dMM(border_mode=border_mode, filter_dilation=filter_dilation, subsample=subsample), [inputs_val, filters_val])
def test_doubleop_grad(): utt.verify_grad( # Op instance DoubleOp(), # Numeric inputs [numpy.random.rand(5, 7, 2)] )
def test_1Dfft(self): inputs_val = np.random.random((1, N)).astype('float32') x = T.matrix('x', dtype='float32') rfft = theano.gpuarray.fft.curfft(x) f_rfft = theano.function([x], rfft, mode=mode_with_gpu) res_rfft = f_rfft(inputs_val) res_rfft_comp = (np.asarray(res_rfft[:, :, 0]) + 1j * np.asarray(res_rfft[:, :, 1])) rfft_ref = numpy.fft.rfft(inputs_val, axis=1) utt.assert_allclose(rfft_ref, res_rfft_comp) m = rfft.type() irfft = theano.gpuarray.fft.cuirfft(m) f_irfft = theano.function([m], irfft, mode=mode_with_gpu) res_irfft = f_irfft(res_rfft) utt.assert_allclose(inputs_val, np.asarray(res_irfft)) # The numerical gradient of the FFT is sensitive, must set large # enough epsilon to get good accuracy. eps = 1e-1 def f_rfft(inp): return theano.gpuarray.fft.curfft(inp) inputs_val = np.random.random((1, N)).astype('float32') utt.verify_grad(f_rfft, [inputs_val], eps=eps) def f_irfft(inp): return theano.gpuarray.fft.cuirfft(inp) inputs_val = np.random.random((1, N // 2 + 1, 2)).astype('float32') utt.verify_grad(f_irfft, [inputs_val], eps=eps)
def test_pseudoinverse_grad(): rng = np.random.RandomState(utt.fetch_seed()) d1 = rng.randint(4) + 2 d2 = rng.randint(4) + 2 r = rng.randn(d1, d2).astype(theano.config.floatX) utt.verify_grad(pinv, [r])
def test_CSMGrad(self): imshp = (3, 3) nkern = 1 # per output pixel kshp = (2, 2) # ssizes = ((1,1),(2,2)) ssizes = ((1, 1),) # convmodes = ('full','valid',) convmodes = ("full",) kerns = tensor.dvector() indices = tensor.ivector() indptr = tensor.ivector() spmat_shape = tensor.ivector() for mode in ["FAST_COMPILE", "FAST_RUN"]: for conv_mode in convmodes: for ss in ssizes: indvals, indptrvals, spshapevals, sptype, outshp, kmap = sp.convolution_indices.sparse_eval( imshp, kshp, nkern, ss, conv_mode ) kvals = numpy.random.random(nkern * numpy.prod(kshp) * numpy.prod(outshp)).flatten() def d(kerns): return theano.sparse.dense_from_sparse( theano.sparse.CSM(sptype, kmap)(kerns, indvals, indptrvals, spshapevals) ) # symbolic stuff utt.verify_grad(d, [kvals])
def test_DownsampleFactorMaxGrad_grad_st(self): """checks the gradient of the gradient for the case that stride is used""" rng = numpy.random.RandomState(utt.fetch_seed()) maxpoolshps = ((1, 1), (3, 3), (5, 3)) stridesizes = ((1, 1), (3, 3), (5, 7)) imval = rng.rand(1, 2, 16, 16) for maxpoolshp in maxpoolshps: for ignore_border in [True, False]: for stride in stridesizes: grad_shape = DownsampleFactorMax.out_shape( imval.shape, maxpoolshp, ignore_border=ignore_border, st=stride) grad_val = rng.rand(*grad_shape) def mp(input, grad): out = DownsampleFactorMax( maxpoolshp, ignore_border=ignore_border, st=stride)(input) grad_op = MaxPoolGrad( maxpoolshp, ignore_border=ignore_border, st=stride) return grad_op(input, out, grad) utt.verify_grad(mp, [imval, grad_val], rng=rng)
def run_fwd( self, inputs_shape, filters_shape, ref=dnn_conv, subsample=(1, 1), verify_grad=True, mode=mode_without_gpu, border_mode="valid", filter_flip=True, device="cpu", provide_shape=False, target_op=None, ): inputs_val = numpy.random.random(inputs_shape).astype("float32") filters_val = numpy.random.random(filters_shape).astype("float32") if device == "gpu": inputs = gpu_shared(inputs_val) filters = gpu_shared(filters_val) else: inputs = theano.tensor.as_tensor_variable(cpu_shared(inputs_val)) filters = theano.tensor.as_tensor_variable(cpu_shared(filters_val)) if provide_shape: imshp = inputs_shape kshp = filters_shape else: imshp = None kshp = None if filter_flip: conv_mode = "conv" else: conv_mode = "cross" c_ref = ref(inputs, filters, border_mode=border_mode, subsample=subsample, conv_mode=conv_mode) c = conv.conv2d( inputs, filters, border_mode=border_mode, subsample=subsample, filter_flip=filter_flip, input_shape=imshp, filter_shape=kshp, ) f_ref = theano.function([], c_ref, mode=mode) f = theano.function([], c, mode) if target_op is not None: assert any([isinstance(n.op, target_op) for n in f.maker.fgraph.toposort()]) self.assertTrue(hasattr(f.maker.fgraph.outputs[0].tag, "trace")) res_ref = numpy.array(f_ref()) res = numpy.array(f()) utt.assert_allclose(res_ref, res) if verify_grad: utt.verify_grad( conv.AbstractConv2d(border_mode="valid", imshp=imshp, kshp=kshp, subsample=subsample), [inputs_val, filters_val], mode=mode, )
def test_reshape(): a = tcn.CudaNdarrayType((False,))() b = tcn.CudaNdarrayType((False,False))() c = T.reshape(a, [2,3]) #basic f = theano.function([a], c, mode=mode_without_gpu) fv = f(cuda_ndarray.CudaNdarray(theano._asarray([0,1,2,3,4,5],dtype='float32'))) assert numpy.all(fv == numpy.asarray([[0,1,2], [3,4,5]])) #test that it works without inplace operations a_val = cuda_ndarray.CudaNdarray(theano._asarray([0,1,2,3,4,5],dtype='float32')) a_val_copy = cuda_ndarray.CudaNdarray(theano._asarray([0,1,2,3,4,5],dtype='float32')) b_val = cuda_ndarray.CudaNdarray(theano._asarray([[0,1,2],[3,4,5]],dtype='float32')) f_sub = theano.function([a,b], c-b, mode=mode_without_gpu) assert numpy.all(f_sub(a_val, b_val) == 0.0) assert numpy.all(numpy.asarray(a_val) == numpy.asarray(a_val_copy)) #test that it works with inplace operations a_val = theano._asarray([0,1,2,3,4,5], dtype='float32') a_val_copy = theano._asarray([0,1,2,3,4,5], dtype='float32') b_val = theano._asarray([[0,1,2],[3,4,5]], dtype='float32') f_sub = theano.function([a,b], c-b, mode=mode_without_gpu) assert numpy.all(f_sub(a_val, b_val) == 0.0) assert numpy.all(numpy.asarray(a_val) == numpy.asarray(a_val_copy)) # verify gradient def just_vals(v): return T.Reshape(2)(v, theano._asarray([2,3], dtype='int32')) utt.verify_grad(just_vals, [a_val])
def test_verify_grad_with_zeros(self): # including zeros, as the case with zeros is important # (and special cases: 1 zero in the row, more than 1 zero in the row) x_val = numpy.asarray([[1.0, 2.0, 3.0], [0.0, 5.0, 6.0], [0.0, 0.0, 9.0]], dtype="float32") x = theano.tensor.dmatrix() # sanity check p = Prod(axis=1)(x) # Uncomment this for debugging if needed # x2 = theano.tensor.dmatrix() # p2 = Prod(axis=1)(x2) # fn = theano.function([x, x2], [p - p2], mode=self.mode) # print("hand computed diff for each row") # x2_val = numpy.asarray([[1., 2., 3.003], [0.003, 5., 6], [ # 0., 0., 9.01]]) # print(fn(x_val, x2_val)) # fn2 = theano.function([x], [theano.tensor.grad(p.sum(), x)], # mode=self.mode) # print("real grad") # print(fn2(x_val)) fn3 = theano.function([x], [p], mode=self.mode) assert numpy.allclose(fn3(x_val), [6.0, 0.0, 0.0]) # now with verify_grad unittest_tools.verify_grad(Prod(axis=1), [x_val], mode=self.mode)
def test_1Drfft(self): inputs_val = np.random.random((1, N)).astype(theano.config.floatX) x = T.matrix('x') rfft = fft.rfft(x) f_rfft = theano.function([x], rfft) res_rfft = f_rfft(inputs_val) res_rfft_comp = (np.asarray(res_rfft[:, :, 0]) + 1j * np.asarray(res_rfft[:, :, 1])) rfft_ref = np.fft.rfft(inputs_val, axis=1) utt.assert_allclose(rfft_ref, res_rfft_comp) m = rfft.type() print(m.ndim) irfft = fft.irfft(m) f_irfft = theano.function([m], irfft) res_irfft = f_irfft(res_rfft) utt.assert_allclose(inputs_val, np.asarray(res_irfft)) # The numerical gradient of the FFT is sensitive, must set large # enough epsilon to get good accuracy. eps = 1e-1 def f_rfft(inp): return fft.rfft(inp) inputs_val = np.random.random((1, N)).astype(theano.config.floatX) utt.verify_grad(f_rfft, [inputs_val], eps=eps) def f_irfft(inp): return fft.irfft(inp) inputs_val = np.random.random((1, N // 2 + 1, 2)).astype(theano.config.floatX) utt.verify_grad(f_irfft, [inputs_val], eps=eps)
def test_complex_grads(self): def f(m): c = complex(m[0], m[1]) return .5 * real(c) + .9 * imag(c) rng = numpy.random.RandomState(9333) mval = numpy.asarray(rng.randn(2, 5)) utt.verify_grad(f, [mval])
def test_flux_quad_ld(abs_tol=1e-5, rel_tol=1e-5, eps=1e-7): with change_flags(compute_test_value="off"): map = starry.Map(udeg=2) xo = np.linspace(-1.5, 1.5, 10) yo = np.ones_like(xo) * 0.3 zo = 1.0 * np.ones_like(xo) ro = 0.1 np.random.seed(14) u = np.array([-1.0] + list(np.random.randn(2))) func = lambda *args: map.ops.flux(*args) verify_grad( func, (xo, yo, zo, ro, u), abs_tol=abs_tol, rel_tol=rel_tol, eps=eps, n_tests=1, )
def test_max_pool_2d_2D_same_size(self): rng = numpy.random.RandomState(utt.fetch_seed()) test_input_array = numpy.array([[[ [1., 2., 3., 4.], [5., 6., 7., 8.] ]]]).astype(theano.config.floatX) test_answer_array = numpy.array([[[ [0., 0., 0., 0.], [0., 6., 0., 8.] ]]]).astype(theano.config.floatX) input = tensor.tensor4(name='input') patch_size = (2, 2) op = max_pool_2d_same_size(input, patch_size) op_output = function([input], op)(test_input_array) utt.assert_allclose(op_output, test_answer_array) def mp(input): return max_pool_2d_same_size(input, patch_size) utt.verify_grad(mp, [test_input_array], rng=rng)
def test_DownsampleFactorMax_grad_st(self): """checks the gradient for the case that stride is used""" rng = numpy.random.RandomState(utt.fetch_seed()) maxpoolshps = ((1, 1), (3, 3), (5, 3)) stridesizes = ((1, 1), (3, 3), (5, 7)) imval = rng.rand(1, 2, 16, 16) for maxpoolshp, ignore_border, mode, stride in product(maxpoolshps, [True, False], ['max', 'sum', 'average_inc_pad', 'average_exc_pad'], stridesizes): def mp(input): return Pool(maxpoolshp, ignore_border=ignore_border, st=stride, mode=mode)(input) utt.verify_grad(mp, [imval], rng=rng)
def test_sparseblockgemv_grad_1(self): # Test that we correctly handle cases where dimensions are 1. h_val = randn(1, 1, 1).astype('float32') iIdx_val = numpy.random.permutation(1)[:1][None, :] oIdx_val = numpy.random.permutation(1)[:1][None, :] W_val = randn(1, 1, 1, 1).astype('float32') b_val = randn(1, 1).astype('float32') iIdx = theano.tensor.constant(iIdx_val) oIdx = theano.tensor.constant(oIdx_val) def metaop(b, h, W): return sparse_block_dot(W, h, iIdx, b, oIdx) def op(b, h, W): return self.gemv_op(b.take(oIdx, axis=0), W, h, iIdx, oIdx) utt.verify_grad(metaop, [b_val, h_val, W_val], mode=self.mode) utt.verify_grad(op, [b_val, h_val, W_val], mode=self.mode)
def test_reshape(): a = tcn.CudaNdarrayType((False,))() b = tcn.CudaNdarrayType((False, False))() c = T.reshape(a, [2, 3]) #basic f = theano.function([a], c, mode=mode_with_gpu) fv = f(cuda_ndarray.CudaNdarray(theano._asarray([0, 1, 2, 3, 4, 5], dtype='float32'))) topo = f.maker.fgraph.toposort() assert any([isinstance(node.op, B.GpuReshape) for node in topo]) assert numpy.all(fv == numpy.asarray([[0, 1, 2], [3, 4, 5]])) #test that it works without inplace operations a_val = cuda_ndarray.CudaNdarray(theano._asarray([0, 1, 2, 3, 4, 5], dtype='float32')) a_val_copy = cuda_ndarray.CudaNdarray(theano._asarray([0, 1, 2, 3, 4, 5], dtype='float32')) b_val = cuda_ndarray.CudaNdarray(theano._asarray([[0, 1, 2], [3, 4, 5]], dtype='float32')) f_sub = theano.function([a, b], c - b, mode=mode_with_gpu) topo = f_sub.maker.fgraph.toposort() assert any([isinstance(node.op, B.GpuReshape) for node in topo]) assert numpy.all(f_sub(a_val, b_val) == 0.0) assert numpy.all(numpy.asarray(a_val) == numpy.asarray(a_val_copy)) #test that it works with inplace operations a_val = theano._asarray([0, 1, 2, 3, 4, 5], dtype='float32') a_val_copy = theano._asarray([0, 1, 2, 3, 4, 5], dtype='float32') b_val = theano._asarray([[0, 1, 2], [3, 4, 5]], dtype='float32') f_sub = theano.function([a, b], c - b, mode=mode_with_gpu) topo = f_sub.maker.fgraph.toposort() assert any([isinstance(node.op, B.GpuReshape) for node in topo]) assert numpy.all(f_sub(a_val, b_val) == 0.0) assert numpy.all(numpy.asarray(a_val) == numpy.asarray(a_val_copy)) # verify gradient def just_vals(v): return T.Reshape(2)(v, theano._asarray([2, 3], dtype='int32')) utt.verify_grad(just_vals, [a_val])
def run_conv_valid(self, inputs_shape, filters_shape, border_mode='valid', filter_dilation=(1, 1, 1), subsample=(1, 1, 1), verify_grad=False): inputs_val = numpy.random.random(inputs_shape).astype('float32') filters_val = numpy.random.random(filters_shape).astype('float32') inputs = shared(inputs_val) filters = shared(filters_val) conv_ref = Corr3dMM(border_mode=border_mode, filter_dilation=filter_dilation, subsample=subsample)(inputs.dimshuffle( 0, 4, 1, 2, 3), filters.dimshuffle(0, 4, 1, 2, 3)) conv_ref = conv_ref.dimshuffle(0, 2, 3, 4, 1) f_ref = theano.function([], conv_ref, mode='FAST_RUN') conv = GpuCorr3dMM(border_mode=border_mode, filter_dilation=filter_dilation, subsample=subsample)(inputs.dimshuffle( 0, 4, 1, 2, 3), filters.dimshuffle(0, 4, 1, 2, 3)) conv = conv.dimshuffle(0, 2, 3, 4, 1) f = theano.function([], conv, mode=mode_with_gpu) res_ref = f_ref() res = f() utt.assert_allclose(res_ref, res) if verify_grad: utt.verify_grad(GpuCorr3dMM(border_mode=border_mode, filter_dilation=filter_dilation, subsample=subsample), [ inputs_val.transpose(0, 4, 1, 2, 3), filters_val.transpose(0, 4, 1, 2, 3) ], mode=mode_with_gpu)
def run_gradinput(self, inputs_shape, filters_shape, output_shape, ref=dnn_gradinput, subsample=(1, 1), filter_flip=True, verify_grad=True, mode=mode_without_gpu, border_mode='valid', device='cpu', provide_shape=False): output_val = numpy.random.random(output_shape).astype('float32') filters_val = numpy.random.random(filters_shape).astype('float32') if device == 'gpu': output = gpu_shared(output_val) filters = gpu_shared(filters_val) else: output = theano.tensor.as_tensor_variable(cpu_shared(output_val)) filters = theano.tensor.as_tensor_variable(cpu_shared(filters_val)) if provide_shape: imshp = inputs_shape kshp = filters_shape else: imshp = None kshp = None if filter_flip: conv_mode = 'conv' else: conv_mode = 'cross' c = conv.AbstractConv2d_gradInputs(border_mode=border_mode, subsample=subsample, filter_flip=filter_flip, imshp=imshp, kshp=kshp) c = c(filters, output, inputs_shape[-2:]) c_ref = ref(filters, output, inputs_shape, border_mode=border_mode, subsample=subsample, conv_mode=conv_mode) f = theano.function([], c, mode) f_ref = theano.function([], c_ref, mode) res_ref = numpy.array(f_ref()) res = numpy.array(f()) utt.assert_allclose(res_ref, res) def abstract_conv2d_gradinputs(filters_val, output_val): conv_op = conv.AbstractConv2d_gradInputs(border_mode=border_mode, subsample=subsample) return conv_op(filters_val, output_val, inputs_shape[-2:]) if verify_grad: utt.verify_grad(abstract_conv2d_gradinputs, [filters_val, output_val], mode=mode, eps=1)
def test_DownsampleFactorMaxPaddingStride_grad(self): rng = numpy.random.RandomState(utt.fetch_seed()) imgsizes = ((10, 10), (10, 5), (5, 5)) maxpoolsizes = ((5, 3), (3, 5), (3, 3)) stridesizes = ((3, 2), (2, 3), (3, 3)) paddingsizes = ((2, 2), (2, 1), (2, 2)) for i in range(len(imgsizes)): imgsize = imgsizes[i] imval = rng.rand(1, 1, imgsize[0], imgsize[1]) * 10.0 maxpoolsize = maxpoolsizes[i] stridesize = stridesizes[i] paddingsize = paddingsizes[i] def mp(input): return DownsampleFactorMax( maxpoolsize, ignore_border=True, st=stridesize, padding=paddingsize, )(input) utt.verify_grad(mp, [imval], rng=rng)
def test_grad(self): np.random.seed(1234) M_val = np.concatenate( ( np.linspace(-10, 10, 100), [ 0.0, -np.pi + 1e-3, np.pi - 1e-3, 0.5 * np.pi, -0.5 * np.pi, 1.5 * np.pi, 2 * np.pi + 1e-3, ], ) ) e_val = np.random.uniform(0, 0.9, len(M_val)) a = lambda *args: tt.arctan2(*self.op(*args)) # NOQA utt.verify_grad(a, [M_val, e_val], eps=1e-8)
def test_cross_map_norm_grad_simple(): rng = numpy.random.RandomState([2013, 2, 10]) op = CrossMapNorm(16, 15 / 16., 1, True) make_graph = lambda inp: op(gpu_from_host(inp))[0] verify = lambda array: verify_grad(make_graph, [array]) inputs = [ numpy.ones((16, 1, 1, 1), dtype='float32'), rng.normal(size=(32, 5, 5, 10)).astype('float32') ] for arr in inputs: yield verify, arr
def test_max_pool_2d_2D(self): rng = numpy.random.RandomState(utt.fetch_seed()) maxpoolshps = ((1, 1), (3, 2)) imval = rng.rand(4, 5) images = tensor.dmatrix() for maxpoolshp in maxpoolshps: for ignore_border in [True, False]: #print 'maxpoolshp =', maxpoolshp #print 'ignore_border =', ignore_border numpy_output_val = self.numpy_max_pool_2d( imval, maxpoolshp, ignore_border) output = max_pool_2d(images, maxpoolshp, ignore_border) output_val = function([images], output)(imval) assert numpy.all(output_val == numpy_output_val) def mp(input): return max_pool_2d(input, maxpoolshp, ignore_border) utt.verify_grad(mp, [imval], rng=rng)
def test_maxpool(self): # generate flatted images maxpoolshps = ((2, 2), (3, 3), (4, 4), (5, 5), (6, 6)) imval = numpy.random.rand(4, 5, 10, 10) images = tensor.dmatrix() for maxpoolshp in maxpoolshps: # symbolic stuff output, outshp = sp.max_pool(images, imval.shape[1:], maxpoolshp) f = function([ images, ], [ output, ]) output_val = f(imval.reshape(imval.shape[0], -1)) # numeric verification my_output_val = numpy.zeros((imval.shape[0], imval.shape[1], imval.shape[2] / maxpoolshp[0], imval.shape[3] / maxpoolshp[1])) assert numpy.prod(my_output_val.shape[1:]) == numpy.prod( numpy.r_[imval.shape[1], outshp]) for n in range(imval.shape[0]): for k in range(imval.shape[1]): for i in range(imval.shape[2] / maxpoolshp[0]): for j in range(imval.shape[3] / maxpoolshp[1]): ii, jj = i * maxpoolshp[0], j * maxpoolshp[1] patch = imval[n, k, ii:ii + maxpoolshp[0], jj:jj + maxpoolshp[1]] my_output_val[n, k, i, j] = numpy.max(patch) my_output_val = my_output_val.reshape(imval.shape[0], -1) assert numpy.all(output_val == my_output_val) def mp(input): output, outshp = sp.max_pool(input, imval.shape[1:], maxpoolshp) return output utt.verify_grad(mp, [imval.reshape(imval.shape[0], -1)])
def test_grad(self): np.random.seed(42) def func(chol_vec, delta): chol = tt.stack([ tt.stack([tt.exp(0.1 * chol_vec[0]), 0]), tt.stack([chol_vec[1], 2 * tt.exp(chol_vec[2])]), ]) cov = tt.dot(chol, chol.T) return MvNormalLogp()(cov, delta) chol_vec_val = np.array([0.5, 1., -0.1]) delta_val = np.random.randn(1, 2) try: utt.verify_grad(func, [chol_vec_val, delta_val]) except ValueError as e: print(e.args[0]) delta_val = np.random.randn(5, 2) utt.verify_grad(func, [chol_vec_val, delta_val])
def test_DownsampleFactorMax_grad_st_extra(self): """checks the gradient for the case that stride is used for extra examples""" rng = numpy.random.RandomState(utt.fetch_seed()) maxpoolshps = ((5, 3), (5, 3), (5, 3), (5, 5), (3, 2), (7, 7), (9, 9)) stridesizes = ((3, 2), (7, 5), (10, 6), (1, 1), (2, 3), (10, 10), (1, 1)) imvsizs = ((16, 16), (16, 16), (16, 16), (8, 5), (8, 5), (8, 5), (8, 5)) for indx in numpy.arange(len(maxpoolshps)): imvsize = imvsizs[indx] imval = rng.rand(1, 2, imvsize[0], imvsize[1]) stride = stridesizes[indx] maxpoolshp = maxpoolshps[indx] for ignore_border in [True, False]: def mp(input): return DownsampleFactorMax(maxpoolshp, ignore_border=ignore_border, st=stride)(input) utt.verify_grad(mp, [imval], rng=rng)
def test_dnn_conv_grad(): if not dnn.dnn_available(test_ctx_name): raise SkipTest(dnn.dnn_available.msg) b = 1 c = 4 f = 3 ih = 2 iw = 8 kh = 2 kw = 2 img_val = numpy.random.random((b, c, ih, iw)).astype('float32') kern_val = numpy.random.random((f, c, kh, kw)).astype('float32') out_val = numpy.random.random((b, f, ih - kw + 1, iw - kw + 1)).astype('float32') def dconv(img, kern, out): desc = dnn.GpuDnnConvDesc(border_mode='valid', subsample=(1, 1), conv_mode='conv')(kern.shape) return dnn.GpuDnnConv()(img, kern, out, desc, alpha=0.5, beta=0.75) def dconvi(img, kern, out): desc = dnn.GpuDnnConvDesc(border_mode='valid', subsample=(1, 1), conv_mode='conv')(kern.shape) return dnn.GpuDnnConvGradI()(kern, out, img, desc, alpha=-1.0, beta=0.0) def dconvw(img, kern, out): desc = dnn.GpuDnnConvDesc(border_mode='valid', subsample=(1, 1), conv_mode='conv')(kern.shape) return dnn.GpuDnnConvGradW()(img, out, kern, desc, alpha=0.75, beta=-1.0) utt.verify_grad(dconv, [img_val, kern_val, out_val]) utt.verify_grad(dconvi, [img_val, kern_val, out_val]) utt.verify_grad(dconvw, [img_val, kern_val, out_val])
def test_grad_nonnegative_axis_3d(self): data = np.random.rand(2, 3, 4).astype(theano.config.floatX) utt.verify_grad(lambda x: sort(x, 0), [data]) data = np.random.rand(2, 3, 4).astype(theano.config.floatX) utt.verify_grad(lambda x: sort(x, 1), [data]) data = np.random.rand(2, 3, 4).astype(theano.config.floatX) utt.verify_grad(lambda x: sort(x, 2), [data])
def test_grad_none_axis(self): data = np.random.rand(10).astype(theano.config.floatX) utt.verify_grad(lambda x: sort(x, None), [data]) utt.verify_grad(lambda x: sort(x, 0), [data]) data = np.random.rand(2, 3).astype(theano.config.floatX) utt.verify_grad(lambda x: sort(x, None), [data])
def test_prod_no_zeros_in_input(self): x = theano.tensor.dmatrix() x_val = numpy.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype='float32') pwz = Prod(axis=1, no_zeros_in_input=True)(x) fn = theano.function([x], pwz, mode=self.mode) assert numpy.allclose(fn(x_val), [6, 120, 504]) pwz = Prod(no_zeros_in_input=True)(x) g = theano.grad(pwz, x) gg = theano.grad(g.sum(), x) fn = theano.function([x], g, mode=self.mode) assert numpy.allclose( fn(x_val), [[362880., 181440., 120960.], [90720., 72576., 60480.], [51840., 45360., 40320.]]) fn = theano.function([x], gg, mode=self.mode) assert numpy.allclose( fn(x_val), [[663696., 422568., 301872.], [233964., 190800., 161016.], [139248., 122652., 109584.]]) unittest_tools.verify_grad(Prod(axis=1, no_zeros_in_input=True), [x_val], mode=self.mode) unittest_tools.verify_grad(Prod(no_zeros_in_input=True), [x_val], mode=self.mode) def second_deriv(x): return theano.grad(Prod(no_zeros_in_input=True)(x), x) unittest_tools.verify_grad(second_deriv, [x_val], mode=self.mode)
def test_grad_inc_set(self): def inc_slice(*s): def just_numeric_args(a, b): return tt.inc_subtensor(a[s], b) return just_numeric_args def set_slice(*s): def just_numeric_args(a, b): return tt.set_subtensor(a[s], b) return just_numeric_args for f_slice in [inc_slice, set_slice]: # vector utt.verify_grad( f_slice(slice(2, 4, None)), (numpy.asarray([0, 1, 2, 3, 4, 5.]), numpy.asarray([9, 9.]), )) # matrix utt.verify_grad( f_slice(slice(1, 2, None), slice(None, None, None)), (numpy.asarray([[0, 1], [2, 3], [4, 5.]]), numpy.asarray([[9, 9.]]), )) #single element utt.verify_grad( f_slice(2, 1), (numpy.asarray([[0, 1], [2, 3], [4, 5.]]), numpy.asarray(9.),))
def test_rv(abs_tol=1e-5, rel_tol=1e-5, eps=1e-7): with change_flags(compute_test_value="off"): map = starry.Map(ydeg=2, rv=True) theta = np.linspace(0, 30, 10) xo = np.linspace(-1.5, 1.5, len(theta)) yo = np.ones_like(xo) * 0.3 zo = 1.0 * np.ones_like(xo) ro = 0.1 inc = 85.0 * np.pi / 180.0 obl = 30.0 * np.pi / 180.0 veq = 0.5 alpha = 0.3 tau = 0.5 delta = 0.0 y = np.ones(9) u = [-1.0] # Just rotation verify_grad( map.ops.rv, (theta, xo, yo, zo, 0.0, inc, obl, y, u, veq, alpha, tau, delta), abs_tol=abs_tol, rel_tol=rel_tol, eps=eps, n_tests=1, ) # Just occultation verify_grad( map.ops.rv, ( theta, xo / 3, yo, zo, ro, inc, obl, y, u, veq, alpha, tau, delta, ), abs_tol=abs_tol, rel_tol=rel_tol, eps=eps, n_tests=1, ) # Rotation + occultation verify_grad( map.ops.rv, (theta, xo, yo, zo, ro, inc, obl, y, u, veq, alpha, tau, delta), abs_tol=abs_tol, rel_tol=rel_tol, eps=eps, n_tests=1, )
def test_batch_normalization_train_grad_grad(): utt.seed_rng() for axes in ('per-activation', 'spatial', (1, 2, 3, 4)): for vartype in (T.tensor5, T.tensor4, T.tensor3, T.matrix, T.vector): # run these experiments with float64 for sufficient numerical stability x, dy, scale, x_mean, x_invstd = (vartype(n, dtype='float64') for n in ('x', 'dy', 'scale', 'x_mean', 'x_invstd')) ndim = x.ndim # reference forward pass if axes == 'per-activation': axes = (0, ) elif axes == 'spatial': axes = (0, ) + tuple(range(2, ndim)) else: # remove non-existing axes axes = tuple(i for i in axes if i < ndim) if len(axes) == 0: continue def bn_grad_wrt_inputs_f(x, dy, scale, x_mean, x_invstd): g_inputs, g_scale, g_bias = bn.AbstractBatchNormTrainGrad( axes)(x, dy, scale, x_mean, x_invstd) return g_inputs def bn_grad_wrt_scale_f(x, dy, scale, x_mean, x_invstd): g_inputs, g_scale, g_bias = bn.AbstractBatchNormTrainGrad( axes)(x, dy, scale, x_mean, x_invstd) return g_scale def bn_grad_wrt_bias_f(x, dy, scale, x_mean, x_invstd): g_inputs, g_scale, g_bias = bn.AbstractBatchNormTrainGrad( axes)(x, dy, scale, x_mean, x_invstd) return g_bias # run for data_shape in ((4, 3, 3, 3, 3), (4, 3, 1, 1, 1), (2, 3, 5, 3, 2)): data_shape = data_shape[:ndim] param_shape = tuple(1 if d in axes else s for d, s in enumerate(data_shape)) # force float64 for sufficient numerical stability x_val = 4 + 3 * np.random.randn(*data_shape).astype('float64') dy_val = -1 + 2 * np.random.randn( *data_shape).astype('float64') scale_val = np.random.randn(*param_shape).astype('float64') x_mean_val = np.random.randn(*param_shape).astype('float64') x_invstd_val = np.random.randn(*param_shape).astype('float64') utt.verify_grad( bn_grad_wrt_inputs_f, [x_val, dy_val, scale_val, x_mean_val, x_invstd_val]) utt.verify_grad( bn_grad_wrt_scale_f, [x_val, dy_val, scale_val, x_mean_val, x_invstd_val]) utt.verify_grad( bn_grad_wrt_bias_f, [x_val, dy_val, scale_val, x_mean_val, x_invstd_val])
def verify_solve_grad(self, m, n, A_structure, lower, rng): # ensure diagonal elements of A relatively large to avoid numerical # precision issues A_val = (rng.normal(size=(m, m)) * 0.5 + np.eye(m)).astype( config.floatX) if A_structure == 'lower_triangular': A_val = np.tril(A_val) elif A_structure == 'upper_triangular': A_val = np.triu(A_val) if n is None: b_val = rng.normal(size=m).astype(config.floatX) else: b_val = rng.normal(size=(m, n)).astype(config.floatX) eps = None if config.floatX == "float64": eps = 2e-8 if A_structure in ('lower_triangular', 'upper_triangular'): solve_op = GpuCublasTriangularSolve(lower=lower) else: solve_op = GpuCusolverSolve(A_structure="general") utt.verify_grad(solve_op, [A_val, b_val], 3, rng, eps=eps)
def test_AveragePoolGrad_grad(self): rng = numpy.random.RandomState(utt.fetch_seed()) avgpoolshps = ((1, 1), (3, 2), (2, 3)) imval = rng.rand(2, 3, 3, 4) * 10.0 # more variance means numeric gradient will be more accurate for avgpoolshp in avgpoolshps: for ignore_border in [True, False]: for mode in ['sum', 'average_inc_pad', 'average_exc_pad']: # print 'maxpoolshp =', maxpoolshp # print 'ignore_border =', ignore_border # The shape of the gradient will be the shape of the output grad_shape = Pool.out_shape( imval.shape, avgpoolshp, ignore_border=ignore_border) grad_val = rng.rand(*grad_shape) * 10.0 def mp(input, grad): grad_op = AveragePoolGrad( avgpoolshp, ignore_border=ignore_border, mode=mode) return grad_op(input, grad) utt.verify_grad(mp, [imval, grad_val], rng=rng)
def test_max_pool_2d_2D_same_size(self): rng = numpy.random.RandomState(utt.fetch_seed()) test_input_array = numpy.array([[[ [1., 2., 3., 4.], [5., 6., 7., 8.] ]]]) test_answer_array = numpy.array([[[ [0., 0., 0., 0.], [0., 6., 0., 8.] ]]]) input = tensor.tensor4(name='input') patch_size = (2, 2) op = max_pool_2d_same_size(input, patch_size) op_output = function([input], op)(test_input_array) assert numpy.all(op_output == test_answer_array), ( "op_output is %s, test_answer_array is %s" % ( op_output, numpy_output_val ) ) def mp(input): return max_pool_2d_same_size(input, patch_size) utt.verify_grad(mp, [test_input_array], rng=rng)
def test_verify_grad_with_zeros(self): # including zeros, as the case with zeros is important # (and special cases: 1 zero in the row, more than 1 zero in the row) x_val = numpy.asarray([[1.,2.,3.],[0.,5.,6.],[0.,0.,9.]], dtype='float32') x = theano.tensor.dmatrix() # sanity check x2 = theano.tensor.dmatrix() p = Prod(axis=1)(x) p2 = Prod(axis=1)(x2) fn = theano.function([x,x2],[p-p2], mode=self.mode) #print "hand computed diff for each row" x2_val = numpy.asarray([[1., 2., 3.003], [0.003,5.,6], [0.,0.,9.01]]) #print fn(x_val, x2_val) fn2 = theano.function([x],[theano.tensor.grad(p.sum(),x)], mode=self.mode) #print "real grad" #print fn2(x_val) fn3 = theano.function([x],[p], mode=self.mode) assert numpy.allclose(fn3(x_val), [6.,0.,0.]) # now with verify_grad unittest_tools.verify_grad(Prod(axis=1), [x_val], mode=self.mode)
def test_intensity(abs_tol=1e-5, rel_tol=1e-5, eps=1e-7): with change_flags(compute_test_value="off"): map = starry.Map(ydeg=2, udeg=2) np.random.seed(11) lat = 180 * (np.random.random(10) - 0.5) lon = 360 * (np.random.random(10) - 0.5) y = [1.0] + list(np.random.randn(8)) u = [-1.0] + list(np.random.randn(2)) f = [np.pi] wta = 0.0 def intensity(lat, lon, y, u, f, wta): return map.ops.intensity(lat, lon, y, u, f, wta, np.array(True)) verify_grad( intensity, (lat, lon, y, u, f, wta), abs_tol=abs_tol, rel_tol=rel_tol, eps=eps, n_tests=1, )
def test_DownsampleFactorMaxPaddingStride_grad(self): rng = numpy.random.RandomState(utt.fetch_seed()) imgsizes = ((10, 10), (10, 5), (5, 5)) maxpoolsizes = ((5, 3), (3, 5), (3, 3)) stridesizes = ((3, 2), (2, 3), (3, 3)) paddingsizes = ((2, 2), (2, 1), (2, 2)) # average_inc_pad and average_exc_pad do not # support grad with padding for mode in ['max', 'sum']: for i in range(len(imgsizes)): imgsize = imgsizes[i] imval = rng.rand(1, 1, imgsize[0], imgsize[1]) * 10.0 maxpoolsize = maxpoolsizes[i] stridesize = stridesizes[i] paddingsize = paddingsizes[i] def mp(input): return Pool(ignore_border=True, mode=mode)(input, maxpoolsize, stridesize, paddingsize) utt.verify_grad(mp, [imval], rng=rng)
def test_DownsampleFactorMaxGrad_grad(self): rng = numpy.random.RandomState(utt.fetch_seed()) maxpoolshps = ((1, 1), (3, 2), (2, 3)) imval = rng.rand(2, 3, 3, 4) * 10.0 # more variance means numeric gradient will be more accurate for maxpoolshp in maxpoolshps: for ignore_border in [True, False]: # print 'maxpoolshp =', maxpoolshp # print 'ignore_border =', ignore_border # The shape of the gradient will be the shape of the output grad_shape = Pool.out_shape(imval.shape, maxpoolshp, ignore_border=ignore_border) grad_val = rng.rand(*grad_shape) * 10.0 def mp(input, grad): out = Pool(ignore_border=ignore_border)(input, maxpoolshp) grad_op = MaxPoolGrad(ignore_border=ignore_border) return grad_op(input, out, grad, maxpoolshp) utt.verify_grad(mp, [imval, grad_val], rng=rng)
def test_large_sparse_targets_grad(self): V_mat, U_mat, UinvT_mat, Q_mat, H_mat, Y_indexes_mat, Y_values_mat, Y = \ TestLargeSparseTargets.generate_data() V = shared(V_mat) U = shared(U_mat) UinvT = shared(UinvT_mat) Q = shared(Q_mat) H = T.matrix() Y_indexes = T.imatrix() Y_values = T.matrix() learning_rate = 0.1 def f(H): LST_grad = self.op(COST)(V, U, UinvT, Q, H, Y_indexes_mat, Y_values_mat, learning_rate) return LST_grad utt.verify_grad(f, [H_mat])
def test_sparseblockgemv_grad(self): W_val, h_val, iIdx_val, b_val, oIdx_val = \ BlockSparse_Gemv_and_Outer.gemv_data() h_val = randn(1, 1, 1).astype('float32') iIdx_val = numpy.random.permutation(1)[:1][None, :] oIdx_val = numpy.random.permutation(1)[:1][None, :] W_val = randn(1, 1, 1, 1).astype('float32') b_val = randn(1, 1).astype('float32') iIdx = theano.tensor.constant(iIdx_val) oIdx = theano.tensor.constant(oIdx_val) def metaop(b, h, W): return sparse_block_dot(W, h, iIdx, b, oIdx) def op(b, h, W): return self.gemv_op(b.take(oIdx, axis=0), W, h, iIdx, oIdx) utt.verify_grad(metaop, [b_val, h_val, W_val], mode=self.mode) utt.verify_grad(op, [b_val, h_val, W_val], mode=self.mode)