def test_grad(self):
a = np.random.random((3, 5, 2)).astype(config.floatX)
utt.verify_grad(self.op, [a]) # Test axis=None
for axis in range(len(a.shape)):
utt.verify_grad(self.op_class(axis=axis), [a], eps=4e-4)
def test_max_pool_2d_2D(self):
rng = numpy.random.RandomState(utt.fetch_seed())
maxpoolshps = ((1, 1), (3, 2))
imval = rng.rand(4, 5)
images = tensor.dmatrix()
for maxpoolshp, ignore_border, mode in product(maxpoolshps,
[True, False],
['max', 'sum',
'average_inc_pad',
'average_exc_pad']):
# print 'maxpoolshp =', maxpoolshp
# print 'ignore_border =', ignore_border
numpy_output_val = self.numpy_max_pool_2d(imval, maxpoolshp,
ignore_border,
mode=mode)
output = max_pool_2d(images, maxpoolshp, ignore_border,
mode=mode)
output_val = function([images], output)(imval)
assert numpy.all(output_val == numpy_output_val), (
"output_val is %s, numpy_output_val is %s"
% (output_val, numpy_output_val))
def mp(input):
return max_pool_2d(input, maxpoolshp, ignore_border,
mode=mode)
utt.verify_grad(mp, [imval], rng=rng)
def test_DownsampleFactorMaxPaddingStride_grad_grad(self):
rng = numpy.random.RandomState(utt.fetch_seed())
imgsizes = ((10, 10), (10, 5), (5, 5))
maxpoolsizes = ((5, 3), (3, 5), (3, 3))
stridesizes = ((3, 2), (2, 3), (3, 3))
paddingsizes = ((2, 2), (2, 1), (2, 2))
for i in range(len(imgsizes)):
imgsize = imgsizes[i]
imval = rng.rand(1, 1, imgsize[0], imgsize[1]) * 10.0
maxpoolsize = maxpoolsizes[i]
stridesize = stridesizes[i]
paddingsize = paddingsizes[i]
grad_shape = DownsampleFactorMaxGradGrad.out_shape(
imval.shape, maxpoolsize, st=stridesize, ignore_border=True, padding=paddingsize
)
grad_val = rng.rand(*grad_shape) * 10.0
def mp(input, grad):
out = DownsampleFactorMax(maxpoolsize, ignore_border=True, st=stridesize, padding=paddingsize)(input)
grad_op = DownsampleFactorMaxGrad(maxpoolsize, ignore_border=True, st=stridesize, padding=paddingsize)
return grad_op(input, out, grad)
utt.verify_grad(mp, [imval, grad_val], rng=rng)
def test_DownsampleFactorMaxGrad_grad_st_extra(self):
"""checks the gradient of the gradient for the case that
stride is used for extra examples"""
rng = numpy.random.RandomState(utt.fetch_seed())
maxpoolshps = ((5, 3), (5, 3), (5, 3), (5, 5), (3, 2), (7, 7), (9, 9))
stridesizes = ((3, 2), (7, 5), (10, 6), (1, 1), (2, 3), (10, 10), (1, 1))
imvsizs = ((16, 16), (16, 16), (16, 16), (8, 5), (8, 5), (8, 5), (8, 5))
for indx in numpy.arange(len(maxpoolshps)):
imvsize = imvsizs[indx]
imval = rng.rand(1, 2, imvsize[0], imvsize[1])
stride = stridesizes[indx]
maxpoolshp = maxpoolshps[indx]
for ignore_border in [True, False]:
grad_shape = DownsampleFactorMax.out_shape(
imval.shape, maxpoolshp, ignore_border=ignore_border, st=stride
)
grad_val = rng.rand(*grad_shape)
def mp(input, grad):
out = DownsampleFactorMax(maxpoolshp, ignore_border=ignore_border, st=stride)(input)
grad_op = DownsampleFactorMaxGrad(maxpoolshp, ignore_border=ignore_border, st=stride)
return grad_op(input, out, grad)
# skip the grad verification when the output is empty
if numpy.prod(grad_shape) == 0:
continue
utt.verify_grad(mp, [imval, grad_val], rng=rng)
def test_DownsampleFactorMaxGradGrad_grad(self):
rng = numpy.random.RandomState(utt.fetch_seed())
# maxpool, stride, padding, input sizes
examples = (
((3,), (2,), (2,), (10,)),
((3,), (2,), (2,), (2, 10,)),
((3,), (2,), (2,), (2, 1, 10,)),
((5, 3), (3, 2), (2, 2), (1, 1, 10, 10)),
((5, 3), (3, 2), (2, 2), (1, 1, 10, 10)),
((3, 5), (2, 3), (2, 1), (1, 1, 10, 5)),
((3, 3), (3, 3), (2, 2), (1, 1, 5, 5)),
((5, 3, 3), (3, 2, 2), (2, 2, 2), (1, 1, 10, 5, 5)),
((3, 5, 3), (2, 3, 2), (2, 1, 2), (1, 1, 5, 10, 5)),
((3, 3, 5), (2, 2, 3), (2, 2, 1), (1, 1, 5, 5, 10)),
)
for (maxpoolshp, stridesize, paddingsize, inputsize) in examples:
imval1 = rng.rand(*inputsize) * 10.0
imval2 = rng.rand(*inputsize) * 10.0
def mp(input1, input2):
op1 = Pool(ndim=len(maxpoolshp), ignore_border=True)
pooled_out = op1(input1, maxpoolshp, stridesize, paddingsize)
op2 = DownsampleFactorMaxGradGrad(
ndim=len(maxpoolshp),
ignore_border=True)
out = op2(input1, pooled_out, input2, maxpoolshp, stridesize, paddingsize)
return out
utt.verify_grad(mp, [imval1, imval2], rng=rng)
def test_max_pool_3d_3D(self):
rng = numpy.random.RandomState(utt.fetch_seed())
maxpoolshps = ((1, 1, 1), (3, 2, 1))
imval = rng.rand(4, 5, 6)
images = tensor.dtensor3()
for maxpoolshp, ignore_border, mode in product(maxpoolshps,
[True, False],
['max', 'sum',
'average_inc_pad',
'average_exc_pad']):
# print 'maxpoolshp =', maxpoolshp
# print 'ignore_border =', ignore_border
numpy_output_val = self.numpy_max_pool_nd(imval, maxpoolshp,
ignore_border,
mode=mode)
output = pool_3d(images, maxpoolshp, ignore_border,
mode=mode)
output_val = function([images], output)(imval)
utt.assert_allclose(output_val, numpy_output_val)
def mp(input):
return pool_3d(input, maxpoolshp, ignore_border,
mode=mode)
utt.verify_grad(mp, [imval], rng=rng)
def test_AveragePoolPaddingStride_grad_grad(self):
rng = numpy.random.RandomState(utt.fetch_seed())
# avgpool, stride, padding, input sizes
examples = (
((3,), (2,), (2,), (10,)),
((3,), (2,), (2,), (2, 10,)),
((3,), (2,), (2,), (2, 1, 10,)),
((5, 3), (3, 2), (2, 2), (1, 1, 10, 10)),
((5, 3), (3, 2), (2, 2), (1, 1, 10, 10)),
((3, 5), (2, 3), (2, 1), (1, 1, 10, 5)),
((3, 3), (3, 3), (2, 2), (1, 1, 5, 5)),
((5, 3, 3), (3, 2, 2), (2, 2, 2), (1, 1, 10, 5, 5)),
((3, 5, 3), (2, 3, 2), (2, 1, 2), (1, 1, 5, 10, 5)),
((3, 3, 5), (2, 2, 3), (2, 2, 1), (1, 1, 5, 5, 10)),
)
for (avgpoolshp, stridesize, paddingsize, inputsize) in examples:
imval = rng.rand(*inputsize) * 10.0
# 'average_exc_pad' with non-zero padding is not implemented
for mode in ['sum', 'average_inc_pad']:
grad_shape = Pool.out_shape(imval.shape,
avgpoolshp,
ndim=len(avgpoolshp),
st=stridesize,
ignore_border=True,
padding=paddingsize)
grad_val = rng.rand(*grad_shape) * 10.0
def mp(input, grad):
grad_op = AveragePoolGrad(ndim=len(avgpoolshp),
ignore_border=True,
mode=mode)
return grad_op(input, grad, avgpoolshp, stridesize, paddingsize)
utt.verify_grad(mp, [imval, grad_val], rng=rng)
def test_DownsampleFactorMax_grad(self):
rng = numpy.random.RandomState(utt.fetch_seed())
# maxpool, input sizes
examples = (
((2,), (3,)),
((2,), (2, 3)),
((2,), (2, 3, 3)),
((1, 1), (2, 3, 3, 4)),
((3, 2), (2, 3, 3, 4)),
((2, 3), (2, 3, 3, 4)),
((1, 1, 1), (2, 3, 3)),
((3, 2, 2), (2, 3, 3, 4)),
((2, 3, 2), (2, 3, 3, 4, 4)),
((2, 2, 3), (2, 3, 3, 4, 4)),
)
for example, ignore_border, mode in product(examples,
[True, False],
['max',
'sum',
'average_inc_pad',
'average_exc_pad']):
(maxpoolshp, inputsize) = example
imval = rng.rand(*inputsize) * 10.0
# more variance means numeric gradient will be more accurate
def mp(input):
return Pool(ndim=len(maxpoolshp),
ignore_border=ignore_border,
mode=mode)(input, maxpoolshp)
utt.verify_grad(mp, [imval], rng=rng)
def test_DownsampleFactorMaxPaddingStride_grad(self):
rng = numpy.random.RandomState(utt.fetch_seed())
# maxpool, stride, padding, input sizes
examples = (
((10,), (5,), (3,), (2,)),
((10,), (5,), (3,), (2, 2)),
((10,), (5,), (3,), (1, 1, 2)),
((10, 10), (5, 3), (3, 2), (1, 1, 2, 2)),
((10, 5), (3, 5), (2, 3), (1, 1, 2, 1)),
((5, 5), (3, 3), (3, 3), (1, 1, 2, 2)),
((5, 5, 5), (3, 3, 3), (3, 3, 3), (1, 1, 2, 2, 2)),
)
# average_inc_pad and average_exc_pad do not
# support grad with padding
for mode in ['max', 'sum']:
for example in examples:
(maxpoolshp, stridesize, paddingsize, inputsize) = example
imval = rng.rand(*inputsize) * 10.0
def mp(input):
return Pool(
ndim=len(maxpoolshp),
ignore_border=True,
mode=mode,
)(input, maxpoolshp, stridesize, paddingsize)
utt.verify_grad(mp, [imval], rng=rng)
def test_AveragePoolPaddingStride_grad_grad(self):
rng = numpy.random.RandomState(utt.fetch_seed())
imgsizes = ((10, 10), (10, 5), (5, 5))
avgpoolsizes = ((5, 3), (3, 5), (3, 3))
stridesizes = ((3, 2), (2, 3), (3, 3))
paddingsizes = ((2, 2), (2, 1), (2, 2))
for i in range(len(imgsizes)):
imgsize = imgsizes[i]
imval = rng.rand(1, 1, imgsize[0], imgsize[1]) * 10.0
avgpoolsize = avgpoolsizes[i]
stridesize = stridesizes[i]
paddingsize = paddingsizes[i]
# 'average_exc_pad' with non-zero padding is not implemented
for mode in ['sum', 'average_inc_pad']:
grad_shape = DownsampleFactorMax.out_shape(imval.shape,
avgpoolsize, st=stridesize,
ignore_border=True, padding=paddingsize)
grad_val = rng.rand(*grad_shape) * 10.0
def mp(input, grad):
grad_op = AveragePoolGrad(avgpoolsize, ignore_border=True,
st=stridesize, padding=paddingsize,
mode=mode)
return grad_op(input, grad)
utt.verify_grad(mp, [imval, grad_val], rng=rng)
def test_AveragePoolGrad_grad_st_extra(self):
"""checks the gradient of the gradient for the case that
stride is used for extra examples"""
rng = numpy.random.RandomState(utt.fetch_seed())
avgpoolshps = ((5, 3), (5, 3), (5, 3), (5, 5), (3, 2), (7, 7), (9, 9))
stridesizes = ((3, 2), (7, 5), (10, 6), (1, 1),
(2, 3), (10, 10), (1, 1))
imvsizs = ((16, 16), (16, 16), (16, 16), (8, 5),
(8, 5), (8, 5), (8, 5))
for indx in numpy.arange(len(avgpoolshps)):
imvsize = imvsizs[indx]
imval = rng.rand(1, 2, imvsize[0], imvsize[1])
stride = stridesizes[indx]
avgpoolshp = avgpoolshps[indx]
for ignore_border in [True, False]:
for mode in ['sum', 'average_inc_pad', 'average_exc_pad']:
grad_shape = Pool.out_shape(
imval.shape, avgpoolshp,
ignore_border=ignore_border, st=stride)
grad_val = rng.rand(*grad_shape)
def mp(input, grad):
grad_op = AveragePoolGrad(
avgpoolshp, ignore_border=ignore_border,
st=stride, mode=mode)
return grad_op(input, grad)
# skip the grad verification when the output is empty
if numpy.prod(grad_shape) == 0:
continue
utt.verify_grad(mp, [imval, grad_val], rng=rng)
def test_gradient(self):
utt.verify_grad(fill_diagonal, [numpy.random.rand(5, 8),
numpy.random.rand()],
n_tests=1, rng=TestFillDiagonal.rng)
utt.verify_grad(fill_diagonal, [numpy.random.rand(8, 5),
numpy.random.rand()],
n_tests=1, rng=TestFillDiagonal.rng)
def test_light_curve_grad():
u_val = np.array([0.2, 0.3, 0.1, 0.5])
b_val = np.linspace(-1.5, 1.5, 20)
r_val = 0.1 + np.zeros_like(b_val)
lc = lambda u, b, r: StarryLightCurve(u)._compute_light_curve(b, r) # NOQA
utt.verify_grad(lc, [u_val, b_val, r_val])
def test_verify_grad_gauntlet(self):
maxpoolshps = ((1, 1), (3, 3), (5, 3),)
stridesizes = ((1, 1), (3, 3), (5, 7),)
# generate random images
imval = self.rng.rand(4, 10, 16, 16)
for index_type, index_scope, maxpoolshp, stride, ignore_border in product(['flattened',
'array'],
['local'],
maxpoolshps,
stridesizes,
[True, False]):
unpoolswitch_op = UnpoolSwitch(ds = maxpoolshp,
st=stride, index_type=index_type,
index_scope=index_scope)
if(index_type == 'flattened'):
def op_with_fixed_switchs(x):
x_with_zero_switchs = T.concatenate((x, T.zeros_like(x)), 1)
return unpoolswitch_op(x_with_zero_switchs)
else:
def op_with_fixed_switchs(x):
x_with_zero_switchs = T.concatenate((x, T.zeros_like(x), T.zeros_like(x)), 1)
return unpoolswitch_op(x_with_zero_switchs)
utt.verify_grad(op_with_fixed_switchs, [imval], rng=self.rng)
def test_batched_dot_gradient(self):
for threshold in [0, 100]:
unittest_tools.verify_grad(
GpuBatchedDot(stream_threshold=threshold),
[numpy.random.randn(5,7,2).astype(numpy.float32),
numpy.random.randn(5,2,6).astype(numpy.float32)],
mode=mode_with_gpu)
def test_DownsampleFactorMaxGradGrad_grad(self):
rng = numpy.random.RandomState(utt.fetch_seed())
imgsizes = ((10, 10), (10, 5), (5, 5))
maxpoolsizes = ((5, 3), (3, 5), (3, 3))
stridesizes = ((3, 2), (2, 3), (3, 3))
paddingsizes = ((2, 2), (2, 1), (2, 2))
for i in range(len(imgsizes)):
imgsize = imgsizes[i]
imval1 = rng.rand(1, 1, imgsize[0], imgsize[1]) * 10.0
imval2 = rng.rand(1, 1, imgsize[0], imgsize[1]) * 10.0
maxpoolsize = maxpoolsizes[i]
stridesize = stridesizes[i]
paddingsize = paddingsizes[i]
def mp(input1, input2):
pooled_out = Pool(
maxpoolsize, ignore_border=True,
st=stridesize,
padding=paddingsize,
)(input1)
out = DownsampleFactorMaxGradGrad(
ds=maxpoolsize,
ignore_border=True,
st=stridesize,
padding=paddingsize)(input1, pooled_out, input2)
return out
utt.verify_grad(mp, [imval1, imval2], rng=rng)
def test0(self):
y_idx = [0, 1, 3]
def f(a, b):
return crossentropy_softmax_1hot_with_bias(a, b, y_idx)[0]
utt.verify_grad(f, [numpy.random.rand(3, 4),
numpy.random.rand(4)])
def test_maxpool(self):
# generate flatted images
maxpoolshps = ((2, 2), (3, 3), (4, 4), (5, 5), (6, 6))
imval = numpy.random.rand(4, 5, 10, 10)
images = tensor.dmatrix()
for maxpoolshp in maxpoolshps:
# symbolic stuff
output, outshp = sp.max_pool(images, imval.shape[1:], maxpoolshp)
f = function([images], [output])
output_val = f(imval.reshape(imval.shape[0], -1))
# numeric verification
my_output_val = numpy.zeros(
(imval.shape[0], imval.shape[1], imval.shape[2] / maxpoolshp[0], imval.shape[3] / maxpoolshp[1])
)
assert numpy.prod(my_output_val.shape[1:]) == numpy.prod(numpy.r_[imval.shape[1], outshp])
for n in range(imval.shape[0]):
for k in range(imval.shape[1]):
for i in range(imval.shape[2] / maxpoolshp[0]):
for j in range(imval.shape[3] / maxpoolshp[1]):
ii, jj = i * maxpoolshp[0], j * maxpoolshp[1]
patch = imval[n, k, ii : ii + maxpoolshp[0], jj : jj + maxpoolshp[1]]
my_output_val[n, k, i, j] = numpy.max(patch)
my_output_val = my_output_val.reshape(imval.shape[0], -1)
assert numpy.all(output_val == my_output_val)
def mp(input):
output, outshp = sp.max_pool(input, imval.shape[1:], maxpoolshp)
return output
utt.verify_grad(mp, [imval.reshape(imval.shape[0], -1)])
def test_verify_grad_with_zeros(self):
# including zeros, as the case with zeros is important
# (and special cases: 1 zero in the row, more than 1 zero in the row)
x_val = numpy.asarray([[1., 2., 3.], [0., 5., 6.], [0., 0., 9.]],
dtype='float32')
x = theano.tensor.dmatrix()
# sanity check
x2 = theano.tensor.dmatrix()
p = Prod(axis=1)(x)
p2 = Prod(axis=1)(x2)
fn = theano.function([x, x2], [p - p2], mode=self.mode)
#print "hand computed diff for each row"
x2_val = numpy.asarray([[1., 2., 3.003], [0.003, 5., 6], [
0., 0., 9.01]])
#print fn(x_val, x2_val)
fn2 = theano.function([x], [theano.tensor.grad(p.sum(), x)],
mode=self.mode)
#print "real grad"
#print fn2(x_val)
fn3 = theano.function([x], [p], mode=self.mode)
assert numpy.allclose(fn3(x_val), [6., 0., 0.])
# now with verify_grad
unittest_tools.verify_grad(Prod(axis=1), [x_val], mode=self.mode)
def run_conv_valid(self, inputs_shape, filters_shape,
border_mode='valid',
filter_dilation=(1, 1, 1),
subsample=(1, 1, 1),
verify_grad=False):
inputs_shape = [inputs_shape[i] for i in (0, 4, 1, 2, 3)]
filters_shape = [filters_shape[i] for i in (0, 4, 1, 2, 3)]
inputs_val = np.random.random(inputs_shape).astype(config.floatX)
filters_val = np.random.random(filters_shape).astype(config.floatX)
inputs = gpuarray_shared_constructor(inputs_val)
filters = gpuarray_shared_constructor(filters_val)
conv_ref = Corr3dMM(border_mode=border_mode,
filter_dilation=filter_dilation,
subsample=subsample)(ref_cast(inputs), ref_cast(filters))
f_ref = theano.function([], conv_ref, mode=mode_without_gpu)
conv = GpuCorr3dMM(border_mode=border_mode,
filter_dilation=filter_dilation,
subsample=subsample)(inputs, filters)
f = theano.function([], conv, mode=mode_with_gpu)
res_ref = f_ref()
res = f()
utt.assert_allclose(res_ref, res)
if verify_grad:
utt.verify_grad(GpuCorr3dMM(border_mode=border_mode,
filter_dilation=filter_dilation,
subsample=subsample),
[inputs_val, filters_val])
def test_doubleop_grad():
utt.verify_grad(
# Op instance
DoubleOp(),
# Numeric inputs
[numpy.random.rand(5, 7, 2)]
)
def test_1Dfft(self):
inputs_val = np.random.random((1, N)).astype('float32')
x = T.matrix('x', dtype='float32')
rfft = theano.gpuarray.fft.curfft(x)
f_rfft = theano.function([x], rfft, mode=mode_with_gpu)
res_rfft = f_rfft(inputs_val)
res_rfft_comp = (np.asarray(res_rfft[:, :, 0]) +
1j * np.asarray(res_rfft[:, :, 1]))
rfft_ref = numpy.fft.rfft(inputs_val, axis=1)
utt.assert_allclose(rfft_ref, res_rfft_comp)
m = rfft.type()
irfft = theano.gpuarray.fft.cuirfft(m)
f_irfft = theano.function([m], irfft, mode=mode_with_gpu)
res_irfft = f_irfft(res_rfft)
utt.assert_allclose(inputs_val, np.asarray(res_irfft))
# The numerical gradient of the FFT is sensitive, must set large
# enough epsilon to get good accuracy.
eps = 1e-1
def f_rfft(inp):
return theano.gpuarray.fft.curfft(inp)
inputs_val = np.random.random((1, N)).astype('float32')
utt.verify_grad(f_rfft, [inputs_val], eps=eps)
def f_irfft(inp):
return theano.gpuarray.fft.cuirfft(inp)
inputs_val = np.random.random((1, N // 2 + 1, 2)).astype('float32')
utt.verify_grad(f_irfft, [inputs_val], eps=eps)
def test_pseudoinverse_grad():
rng = np.random.RandomState(utt.fetch_seed())
d1 = rng.randint(4) + 2
d2 = rng.randint(4) + 2
r = rng.randn(d1, d2).astype(theano.config.floatX)
utt.verify_grad(pinv, [r])
def test_CSMGrad(self):
imshp = (3, 3)
nkern = 1 # per output pixel
kshp = (2, 2)
# ssizes = ((1,1),(2,2))
ssizes = ((1, 1),)
# convmodes = ('full','valid',)
convmodes = ("full",)
kerns = tensor.dvector()
indices = tensor.ivector()
indptr = tensor.ivector()
spmat_shape = tensor.ivector()
for mode in ["FAST_COMPILE", "FAST_RUN"]:
for conv_mode in convmodes:
for ss in ssizes:
indvals, indptrvals, spshapevals, sptype, outshp, kmap = sp.convolution_indices.sparse_eval(
imshp, kshp, nkern, ss, conv_mode
)
kvals = numpy.random.random(nkern * numpy.prod(kshp) * numpy.prod(outshp)).flatten()
def d(kerns):
return theano.sparse.dense_from_sparse(
theano.sparse.CSM(sptype, kmap)(kerns, indvals, indptrvals, spshapevals)
)
# symbolic stuff
utt.verify_grad(d, [kvals])
def test_DownsampleFactorMaxGrad_grad_st(self):
"""checks the gradient of the gradient for
the case that stride is used"""
rng = numpy.random.RandomState(utt.fetch_seed())
maxpoolshps = ((1, 1), (3, 3), (5, 3))
stridesizes = ((1, 1), (3, 3), (5, 7))
imval = rng.rand(1, 2, 16, 16)
for maxpoolshp in maxpoolshps:
for ignore_border in [True, False]:
for stride in stridesizes:
grad_shape = DownsampleFactorMax.out_shape(
imval.shape, maxpoolshp,
ignore_border=ignore_border, st=stride)
grad_val = rng.rand(*grad_shape)
def mp(input, grad):
out = DownsampleFactorMax(
maxpoolshp, ignore_border=ignore_border,
st=stride)(input)
grad_op = MaxPoolGrad(
maxpoolshp, ignore_border=ignore_border,
st=stride)
return grad_op(input, out, grad)
utt.verify_grad(mp, [imval, grad_val], rng=rng)
def run_fwd(
self,
inputs_shape,
filters_shape,
ref=dnn_conv,
subsample=(1, 1),
verify_grad=True,
mode=mode_without_gpu,
border_mode="valid",
filter_flip=True,
device="cpu",
provide_shape=False,
target_op=None,
):
inputs_val = numpy.random.random(inputs_shape).astype("float32")
filters_val = numpy.random.random(filters_shape).astype("float32")
if device == "gpu":
inputs = gpu_shared(inputs_val)
filters = gpu_shared(filters_val)
else:
inputs = theano.tensor.as_tensor_variable(cpu_shared(inputs_val))
filters = theano.tensor.as_tensor_variable(cpu_shared(filters_val))
if provide_shape:
imshp = inputs_shape
kshp = filters_shape
else:
imshp = None
kshp = None
if filter_flip:
conv_mode = "conv"
else:
conv_mode = "cross"
c_ref = ref(inputs, filters, border_mode=border_mode, subsample=subsample, conv_mode=conv_mode)
c = conv.conv2d(
inputs,
filters,
border_mode=border_mode,
subsample=subsample,
filter_flip=filter_flip,
input_shape=imshp,
filter_shape=kshp,
)
f_ref = theano.function([], c_ref, mode=mode)
f = theano.function([], c, mode)
if target_op is not None:
assert any([isinstance(n.op, target_op) for n in f.maker.fgraph.toposort()])
self.assertTrue(hasattr(f.maker.fgraph.outputs[0].tag, "trace"))
res_ref = numpy.array(f_ref())
res = numpy.array(f())
utt.assert_allclose(res_ref, res)
if verify_grad:
utt.verify_grad(
conv.AbstractConv2d(border_mode="valid", imshp=imshp, kshp=kshp, subsample=subsample),
[inputs_val, filters_val],
mode=mode,
)
def test_reshape():
a = tcn.CudaNdarrayType((False,))()
b = tcn.CudaNdarrayType((False,False))()
c = T.reshape(a, [2,3])
#basic
f = theano.function([a], c, mode=mode_without_gpu)
fv = f(cuda_ndarray.CudaNdarray(theano._asarray([0,1,2,3,4,5],dtype='float32')))
assert numpy.all(fv == numpy.asarray([[0,1,2], [3,4,5]]))
#test that it works without inplace operations
a_val = cuda_ndarray.CudaNdarray(theano._asarray([0,1,2,3,4,5],dtype='float32'))
a_val_copy = cuda_ndarray.CudaNdarray(theano._asarray([0,1,2,3,4,5],dtype='float32'))
b_val = cuda_ndarray.CudaNdarray(theano._asarray([[0,1,2],[3,4,5]],dtype='float32'))
f_sub = theano.function([a,b], c-b, mode=mode_without_gpu)
assert numpy.all(f_sub(a_val, b_val) == 0.0)
assert numpy.all(numpy.asarray(a_val) == numpy.asarray(a_val_copy))
#test that it works with inplace operations
a_val = theano._asarray([0,1,2,3,4,5], dtype='float32')
a_val_copy = theano._asarray([0,1,2,3,4,5], dtype='float32')
b_val = theano._asarray([[0,1,2],[3,4,5]], dtype='float32')
f_sub = theano.function([a,b], c-b, mode=mode_without_gpu)
assert numpy.all(f_sub(a_val, b_val) == 0.0)
assert numpy.all(numpy.asarray(a_val) == numpy.asarray(a_val_copy))
# verify gradient
def just_vals(v):
return T.Reshape(2)(v, theano._asarray([2,3], dtype='int32'))
utt.verify_grad(just_vals, [a_val])
def test_verify_grad_with_zeros(self):
# including zeros, as the case with zeros is important
# (and special cases: 1 zero in the row, more than 1 zero in the row)
x_val = numpy.asarray([[1.0, 2.0, 3.0], [0.0, 5.0, 6.0], [0.0, 0.0, 9.0]], dtype="float32")
x = theano.tensor.dmatrix()
# sanity check
p = Prod(axis=1)(x)
# Uncomment this for debugging if needed
# x2 = theano.tensor.dmatrix()
# p2 = Prod(axis=1)(x2)
# fn = theano.function([x, x2], [p - p2], mode=self.mode)
# print("hand computed diff for each row")
# x2_val = numpy.asarray([[1., 2., 3.003], [0.003, 5., 6], [
# 0., 0., 9.01]])
# print(fn(x_val, x2_val))
# fn2 = theano.function([x], [theano.tensor.grad(p.sum(), x)],
# mode=self.mode)
# print("real grad")
# print(fn2(x_val))
fn3 = theano.function([x], [p], mode=self.mode)
assert numpy.allclose(fn3(x_val), [6.0, 0.0, 0.0])
# now with verify_grad
unittest_tools.verify_grad(Prod(axis=1), [x_val], mode=self.mode)
def test_1Drfft(self):
inputs_val = np.random.random((1, N)).astype(theano.config.floatX)
x = T.matrix('x')
rfft = fft.rfft(x)
f_rfft = theano.function([x], rfft)
res_rfft = f_rfft(inputs_val)
res_rfft_comp = (np.asarray(res_rfft[:, :, 0]) +
1j * np.asarray(res_rfft[:, :, 1]))
rfft_ref = np.fft.rfft(inputs_val, axis=1)
utt.assert_allclose(rfft_ref, res_rfft_comp)
m = rfft.type()
print(m.ndim)
irfft = fft.irfft(m)
f_irfft = theano.function([m], irfft)
res_irfft = f_irfft(res_rfft)
utt.assert_allclose(inputs_val, np.asarray(res_irfft))
# The numerical gradient of the FFT is sensitive, must set large
# enough epsilon to get good accuracy.
eps = 1e-1
def f_rfft(inp):
return fft.rfft(inp)
inputs_val = np.random.random((1, N)).astype(theano.config.floatX)
utt.verify_grad(f_rfft, [inputs_val], eps=eps)
def f_irfft(inp):
return fft.irfft(inp)
inputs_val = np.random.random((1, N // 2 + 1, 2)).astype(theano.config.floatX)
utt.verify_grad(f_irfft, [inputs_val], eps=eps)
def test_complex_grads(self):
def f(m):
c = complex(m[0], m[1])
return .5 * real(c) + .9 * imag(c)
rng = numpy.random.RandomState(9333)
mval = numpy.asarray(rng.randn(2, 5))
utt.verify_grad(f, [mval])
def test_flux_quad_ld(abs_tol=1e-5, rel_tol=1e-5, eps=1e-7):
with change_flags(compute_test_value="off"):
map = starry.Map(udeg=2)
xo = np.linspace(-1.5, 1.5, 10)
yo = np.ones_like(xo) * 0.3
zo = 1.0 * np.ones_like(xo)
ro = 0.1
np.random.seed(14)
u = np.array([-1.0] + list(np.random.randn(2)))
func = lambda *args: map.ops.flux(*args)
verify_grad(
func,
(xo, yo, zo, ro, u),
abs_tol=abs_tol,
rel_tol=rel_tol,
eps=eps,
n_tests=1,
)
def test_max_pool_2d_2D_same_size(self):
rng = numpy.random.RandomState(utt.fetch_seed())
test_input_array = numpy.array([[[
[1., 2., 3., 4.],
[5., 6., 7., 8.]
]]]).astype(theano.config.floatX)
test_answer_array = numpy.array([[[
[0., 0., 0., 0.],
[0., 6., 0., 8.]
]]]).astype(theano.config.floatX)
input = tensor.tensor4(name='input')
patch_size = (2, 2)
op = max_pool_2d_same_size(input, patch_size)
op_output = function([input], op)(test_input_array)
utt.assert_allclose(op_output, test_answer_array)
def mp(input):
return max_pool_2d_same_size(input, patch_size)
utt.verify_grad(mp, [test_input_array], rng=rng)
def test_DownsampleFactorMax_grad_st(self):
"""checks the gradient for the case that stride is used"""
rng = numpy.random.RandomState(utt.fetch_seed())
maxpoolshps = ((1, 1), (3, 3), (5, 3))
stridesizes = ((1, 1), (3, 3), (5, 7))
imval = rng.rand(1, 2, 16, 16)
for maxpoolshp, ignore_border, mode, stride in product(maxpoolshps,
[True, False],
['max',
'sum',
'average_inc_pad',
'average_exc_pad'],
stridesizes):
def mp(input):
return Pool(maxpoolshp,
ignore_border=ignore_border,
st=stride, mode=mode)(input)
utt.verify_grad(mp, [imval], rng=rng)
def test_sparseblockgemv_grad_1(self):
# Test that we correctly handle cases where dimensions are 1.
h_val = randn(1, 1, 1).astype('float32')
iIdx_val = numpy.random.permutation(1)[:1][None, :]
oIdx_val = numpy.random.permutation(1)[:1][None, :]
W_val = randn(1, 1, 1, 1).astype('float32')
b_val = randn(1, 1).astype('float32')
iIdx = theano.tensor.constant(iIdx_val)
oIdx = theano.tensor.constant(oIdx_val)
def metaop(b, h, W):
return sparse_block_dot(W, h, iIdx, b, oIdx)
def op(b, h, W):
return self.gemv_op(b.take(oIdx, axis=0), W, h, iIdx, oIdx)
utt.verify_grad(metaop, [b_val, h_val, W_val], mode=self.mode)
utt.verify_grad(op, [b_val, h_val, W_val], mode=self.mode)
def test_reshape():
a = tcn.CudaNdarrayType((False,))()
b = tcn.CudaNdarrayType((False, False))()
c = T.reshape(a, [2, 3])
#basic
f = theano.function([a], c, mode=mode_with_gpu)
fv = f(cuda_ndarray.CudaNdarray(theano._asarray([0, 1, 2, 3, 4, 5],
dtype='float32')))
topo = f.maker.fgraph.toposort()
assert any([isinstance(node.op, B.GpuReshape) for node in topo])
assert numpy.all(fv == numpy.asarray([[0, 1, 2], [3, 4, 5]]))
#test that it works without inplace operations
a_val = cuda_ndarray.CudaNdarray(theano._asarray([0, 1, 2, 3, 4, 5],
dtype='float32'))
a_val_copy = cuda_ndarray.CudaNdarray(theano._asarray([0, 1, 2, 3, 4, 5],
dtype='float32'))
b_val = cuda_ndarray.CudaNdarray(theano._asarray([[0, 1, 2], [3, 4, 5]],
dtype='float32'))
f_sub = theano.function([a, b], c - b, mode=mode_with_gpu)
topo = f_sub.maker.fgraph.toposort()
assert any([isinstance(node.op, B.GpuReshape) for node in topo])
assert numpy.all(f_sub(a_val, b_val) == 0.0)
assert numpy.all(numpy.asarray(a_val) == numpy.asarray(a_val_copy))
#test that it works with inplace operations
a_val = theano._asarray([0, 1, 2, 3, 4, 5], dtype='float32')
a_val_copy = theano._asarray([0, 1, 2, 3, 4, 5], dtype='float32')
b_val = theano._asarray([[0, 1, 2], [3, 4, 5]], dtype='float32')
f_sub = theano.function([a, b], c - b, mode=mode_with_gpu)
topo = f_sub.maker.fgraph.toposort()
assert any([isinstance(node.op, B.GpuReshape) for node in topo])
assert numpy.all(f_sub(a_val, b_val) == 0.0)
assert numpy.all(numpy.asarray(a_val) == numpy.asarray(a_val_copy))
# verify gradient
def just_vals(v):
return T.Reshape(2)(v, theano._asarray([2, 3], dtype='int32'))
utt.verify_grad(just_vals, [a_val])
def run_conv_valid(self,
inputs_shape,
filters_shape,
border_mode='valid',
filter_dilation=(1, 1, 1),
subsample=(1, 1, 1),
verify_grad=False):
inputs_val = numpy.random.random(inputs_shape).astype('float32')
filters_val = numpy.random.random(filters_shape).astype('float32')
inputs = shared(inputs_val)
filters = shared(filters_val)
conv_ref = Corr3dMM(border_mode=border_mode,
filter_dilation=filter_dilation,
subsample=subsample)(inputs.dimshuffle(
0, 4, 1, 2,
3), filters.dimshuffle(0, 4, 1, 2, 3))
conv_ref = conv_ref.dimshuffle(0, 2, 3, 4, 1)
f_ref = theano.function([], conv_ref, mode='FAST_RUN')
conv = GpuCorr3dMM(border_mode=border_mode,
filter_dilation=filter_dilation,
subsample=subsample)(inputs.dimshuffle(
0, 4, 1, 2,
3), filters.dimshuffle(0, 4, 1, 2, 3))
conv = conv.dimshuffle(0, 2, 3, 4, 1)
f = theano.function([], conv, mode=mode_with_gpu)
res_ref = f_ref()
res = f()
utt.assert_allclose(res_ref, res)
if verify_grad:
utt.verify_grad(GpuCorr3dMM(border_mode=border_mode,
filter_dilation=filter_dilation,
subsample=subsample),
[
inputs_val.transpose(0, 4, 1, 2, 3),
filters_val.transpose(0, 4, 1, 2, 3)
],
mode=mode_with_gpu)
def run_gradinput(self, inputs_shape, filters_shape, output_shape, ref=dnn_gradinput,
subsample=(1, 1), filter_flip=True, verify_grad=True, mode=mode_without_gpu,
border_mode='valid', device='cpu', provide_shape=False):
output_val = numpy.random.random(output_shape).astype('float32')
filters_val = numpy.random.random(filters_shape).astype('float32')
if device == 'gpu':
output = gpu_shared(output_val)
filters = gpu_shared(filters_val)
else:
output = theano.tensor.as_tensor_variable(cpu_shared(output_val))
filters = theano.tensor.as_tensor_variable(cpu_shared(filters_val))
if provide_shape:
imshp = inputs_shape
kshp = filters_shape
else:
imshp = None
kshp = None
if filter_flip:
conv_mode = 'conv'
else:
conv_mode = 'cross'
c = conv.AbstractConv2d_gradInputs(border_mode=border_mode,
subsample=subsample,
filter_flip=filter_flip,
imshp=imshp, kshp=kshp)
c = c(filters, output, inputs_shape[-2:])
c_ref = ref(filters, output, inputs_shape,
border_mode=border_mode, subsample=subsample,
conv_mode=conv_mode)
f = theano.function([], c, mode)
f_ref = theano.function([], c_ref, mode)
res_ref = numpy.array(f_ref())
res = numpy.array(f())
utt.assert_allclose(res_ref, res)
def abstract_conv2d_gradinputs(filters_val, output_val):
conv_op = conv.AbstractConv2d_gradInputs(border_mode=border_mode, subsample=subsample)
return conv_op(filters_val, output_val, inputs_shape[-2:])
if verify_grad:
utt.verify_grad(abstract_conv2d_gradinputs, [filters_val, output_val],
mode=mode, eps=1)
def test_DownsampleFactorMaxPaddingStride_grad(self):
rng = numpy.random.RandomState(utt.fetch_seed())
imgsizes = ((10, 10), (10, 5), (5, 5))
maxpoolsizes = ((5, 3), (3, 5), (3, 3))
stridesizes = ((3, 2), (2, 3), (3, 3))
paddingsizes = ((2, 2), (2, 1), (2, 2))
for i in range(len(imgsizes)):
imgsize = imgsizes[i]
imval = rng.rand(1, 1, imgsize[0], imgsize[1]) * 10.0
maxpoolsize = maxpoolsizes[i]
stridesize = stridesizes[i]
paddingsize = paddingsizes[i]
def mp(input):
return DownsampleFactorMax(
maxpoolsize, ignore_border=True,
st=stridesize,
padding=paddingsize,
)(input)
utt.verify_grad(mp, [imval], rng=rng)
def test_grad(self):
np.random.seed(1234)
M_val = np.concatenate(
(
np.linspace(-10, 10, 100),
[
0.0,
-np.pi + 1e-3,
np.pi - 1e-3,
0.5 * np.pi,
-0.5 * np.pi,
1.5 * np.pi,
2 * np.pi + 1e-3,
],
)
)
e_val = np.random.uniform(0, 0.9, len(M_val))
a = lambda *args: tt.arctan2(*self.op(*args)) # NOQA
utt.verify_grad(a, [M_val, e_val], eps=1e-8)
def test_cross_map_norm_grad_simple():
rng = numpy.random.RandomState([2013, 2, 10])
op = CrossMapNorm(16, 15 / 16., 1, True)
make_graph = lambda inp: op(gpu_from_host(inp))[0]
verify = lambda array: verify_grad(make_graph, [array])
inputs = [
numpy.ones((16, 1, 1, 1), dtype='float32'),
rng.normal(size=(32, 5, 5, 10)).astype('float32')
]
for arr in inputs:
yield verify, arr
def test_max_pool_2d_2D(self):
rng = numpy.random.RandomState(utt.fetch_seed())
maxpoolshps = ((1, 1), (3, 2))
imval = rng.rand(4, 5)
images = tensor.dmatrix()
for maxpoolshp in maxpoolshps:
for ignore_border in [True, False]:
#print 'maxpoolshp =', maxpoolshp
#print 'ignore_border =', ignore_border
numpy_output_val = self.numpy_max_pool_2d(
imval, maxpoolshp, ignore_border)
output = max_pool_2d(images, maxpoolshp, ignore_border)
output_val = function([images], output)(imval)
assert numpy.all(output_val == numpy_output_val)
def mp(input):
return max_pool_2d(input, maxpoolshp, ignore_border)
utt.verify_grad(mp, [imval], rng=rng)
def test_maxpool(self):
# generate flatted images
maxpoolshps = ((2, 2), (3, 3), (4, 4), (5, 5), (6, 6))
imval = numpy.random.rand(4, 5, 10, 10)
images = tensor.dmatrix()
for maxpoolshp in maxpoolshps:
# symbolic stuff
output, outshp = sp.max_pool(images, imval.shape[1:], maxpoolshp)
f = function([
images,
], [
output,
])
output_val = f(imval.reshape(imval.shape[0], -1))
# numeric verification
my_output_val = numpy.zeros((imval.shape[0], imval.shape[1],
imval.shape[2] / maxpoolshp[0],
imval.shape[3] / maxpoolshp[1]))
assert numpy.prod(my_output_val.shape[1:]) == numpy.prod(
numpy.r_[imval.shape[1], outshp])
for n in range(imval.shape[0]):
for k in range(imval.shape[1]):
for i in range(imval.shape[2] / maxpoolshp[0]):
for j in range(imval.shape[3] / maxpoolshp[1]):
ii, jj = i * maxpoolshp[0], j * maxpoolshp[1]
patch = imval[n, k, ii:ii + maxpoolshp[0],
jj:jj + maxpoolshp[1]]
my_output_val[n, k, i, j] = numpy.max(patch)
my_output_val = my_output_val.reshape(imval.shape[0], -1)
assert numpy.all(output_val == my_output_val)
def mp(input):
output, outshp = sp.max_pool(input, imval.shape[1:],
maxpoolshp)
return output
utt.verify_grad(mp, [imval.reshape(imval.shape[0], -1)])
def test_grad(self):
np.random.seed(42)
def func(chol_vec, delta):
chol = tt.stack([
tt.stack([tt.exp(0.1 * chol_vec[0]), 0]),
tt.stack([chol_vec[1], 2 * tt.exp(chol_vec[2])]),
])
cov = tt.dot(chol, chol.T)
return MvNormalLogp()(cov, delta)
chol_vec_val = np.array([0.5, 1., -0.1])
delta_val = np.random.randn(1, 2)
try:
utt.verify_grad(func, [chol_vec_val, delta_val])
except ValueError as e:
print(e.args[0])
delta_val = np.random.randn(5, 2)
utt.verify_grad(func, [chol_vec_val, delta_val])
def test_DownsampleFactorMax_grad_st_extra(self):
"""checks the gradient for the case
that stride is used for extra examples"""
rng = numpy.random.RandomState(utt.fetch_seed())
maxpoolshps = ((5, 3), (5, 3), (5, 3), (5, 5), (3, 2), (7, 7), (9, 9))
stridesizes = ((3, 2), (7, 5), (10, 6), (1, 1),
(2, 3), (10, 10), (1, 1))
imvsizs = ((16, 16), (16, 16), (16, 16), (8, 5),
(8, 5), (8, 5), (8, 5))
for indx in numpy.arange(len(maxpoolshps)):
imvsize = imvsizs[indx]
imval = rng.rand(1, 2, imvsize[0], imvsize[1])
stride = stridesizes[indx]
maxpoolshp = maxpoolshps[indx]
for ignore_border in [True, False]:
def mp(input):
return DownsampleFactorMax(maxpoolshp,
ignore_border=ignore_border,
st=stride)(input)
utt.verify_grad(mp, [imval], rng=rng)
def test_dnn_conv_grad():
if not dnn.dnn_available(test_ctx_name):
raise SkipTest(dnn.dnn_available.msg)
b = 1
c = 4
f = 3
ih = 2
iw = 8
kh = 2
kw = 2
img_val = numpy.random.random((b, c, ih, iw)).astype('float32')
kern_val = numpy.random.random((f, c, kh, kw)).astype('float32')
out_val = numpy.random.random((b, f, ih - kw + 1,
iw - kw + 1)).astype('float32')
def dconv(img, kern, out):
desc = dnn.GpuDnnConvDesc(border_mode='valid', subsample=(1, 1),
conv_mode='conv')(kern.shape)
return dnn.GpuDnnConv()(img, kern, out, desc, alpha=0.5, beta=0.75)
def dconvi(img, kern, out):
desc = dnn.GpuDnnConvDesc(border_mode='valid', subsample=(1, 1),
conv_mode='conv')(kern.shape)
return dnn.GpuDnnConvGradI()(kern, out, img, desc, alpha=-1.0,
beta=0.0)
def dconvw(img, kern, out):
desc = dnn.GpuDnnConvDesc(border_mode='valid', subsample=(1, 1),
conv_mode='conv')(kern.shape)
return dnn.GpuDnnConvGradW()(img, out, kern, desc, alpha=0.75,
beta=-1.0)
utt.verify_grad(dconv, [img_val, kern_val, out_val])
utt.verify_grad(dconvi, [img_val, kern_val, out_val])
utt.verify_grad(dconvw, [img_val, kern_val, out_val])
def test_grad_nonnegative_axis_3d(self):
data = np.random.rand(2, 3, 4).astype(theano.config.floatX)
utt.verify_grad(lambda x: sort(x, 0), [data])
data = np.random.rand(2, 3, 4).astype(theano.config.floatX)
utt.verify_grad(lambda x: sort(x, 1), [data])
data = np.random.rand(2, 3, 4).astype(theano.config.floatX)
utt.verify_grad(lambda x: sort(x, 2), [data])
def test_grad_none_axis(self):
data = np.random.rand(10).astype(theano.config.floatX)
utt.verify_grad(lambda x: sort(x, None), [data])
utt.verify_grad(lambda x: sort(x, 0), [data])
data = np.random.rand(2, 3).astype(theano.config.floatX)
utt.verify_grad(lambda x: sort(x, None), [data])
def test_prod_no_zeros_in_input(self):
x = theano.tensor.dmatrix()
x_val = numpy.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype='float32')
pwz = Prod(axis=1, no_zeros_in_input=True)(x)
fn = theano.function([x], pwz, mode=self.mode)
assert numpy.allclose(fn(x_val), [6, 120, 504])
pwz = Prod(no_zeros_in_input=True)(x)
g = theano.grad(pwz, x)
gg = theano.grad(g.sum(), x)
fn = theano.function([x], g, mode=self.mode)
assert numpy.allclose(
fn(x_val), [[362880., 181440., 120960.], [90720., 72576., 60480.],
[51840., 45360., 40320.]])
fn = theano.function([x], gg, mode=self.mode)
assert numpy.allclose(
fn(x_val),
[[663696., 422568., 301872.], [233964., 190800., 161016.],
[139248., 122652., 109584.]])
unittest_tools.verify_grad(Prod(axis=1, no_zeros_in_input=True),
[x_val],
mode=self.mode)
unittest_tools.verify_grad(Prod(no_zeros_in_input=True), [x_val],
mode=self.mode)
def second_deriv(x):
return theano.grad(Prod(no_zeros_in_input=True)(x), x)
unittest_tools.verify_grad(second_deriv, [x_val], mode=self.mode)
def test_grad_inc_set(self):
def inc_slice(*s):
def just_numeric_args(a, b):
return tt.inc_subtensor(a[s], b)
return just_numeric_args
def set_slice(*s):
def just_numeric_args(a, b):
return tt.set_subtensor(a[s], b)
return just_numeric_args
for f_slice in [inc_slice, set_slice]:
# vector
utt.verify_grad(
f_slice(slice(2, 4, None)),
(numpy.asarray([0, 1, 2, 3, 4, 5.]),
numpy.asarray([9, 9.]), ))
# matrix
utt.verify_grad(
f_slice(slice(1, 2, None), slice(None, None, None)),
(numpy.asarray([[0, 1], [2, 3], [4, 5.]]),
numpy.asarray([[9, 9.]]), ))
#single element
utt.verify_grad(
f_slice(2, 1),
(numpy.asarray([[0, 1], [2, 3], [4, 5.]]),
numpy.asarray(9.),))
def test_rv(abs_tol=1e-5, rel_tol=1e-5, eps=1e-7):
with change_flags(compute_test_value="off"):
map = starry.Map(ydeg=2, rv=True)
theta = np.linspace(0, 30, 10)
xo = np.linspace(-1.5, 1.5, len(theta))
yo = np.ones_like(xo) * 0.3
zo = 1.0 * np.ones_like(xo)
ro = 0.1
inc = 85.0 * np.pi / 180.0
obl = 30.0 * np.pi / 180.0
veq = 0.5
alpha = 0.3
tau = 0.5
delta = 0.0
y = np.ones(9)
u = [-1.0]
# Just rotation
verify_grad(
map.ops.rv,
(theta, xo, yo, zo, 0.0, inc, obl, y, u, veq, alpha, tau, delta),
abs_tol=abs_tol,
rel_tol=rel_tol,
eps=eps,
n_tests=1,
)
# Just occultation
verify_grad(
map.ops.rv,
(
theta,
xo / 3,
yo,
zo,
ro,
inc,
obl,
y,
u,
veq,
alpha,
tau,
delta,
),
abs_tol=abs_tol,
rel_tol=rel_tol,
eps=eps,
n_tests=1,
)
# Rotation + occultation
verify_grad(
map.ops.rv,
(theta, xo, yo, zo, ro, inc, obl, y, u, veq, alpha, tau, delta),
abs_tol=abs_tol,
rel_tol=rel_tol,
eps=eps,
n_tests=1,
)
def test_batch_normalization_train_grad_grad():
utt.seed_rng()
for axes in ('per-activation', 'spatial', (1, 2, 3, 4)):
for vartype in (T.tensor5, T.tensor4, T.tensor3, T.matrix, T.vector):
# run these experiments with float64 for sufficient numerical stability
x, dy, scale, x_mean, x_invstd = (vartype(n, dtype='float64')
for n in ('x', 'dy', 'scale',
'x_mean', 'x_invstd'))
ndim = x.ndim
# reference forward pass
if axes == 'per-activation':
axes = (0, )
elif axes == 'spatial':
axes = (0, ) + tuple(range(2, ndim))
else:
# remove non-existing axes
axes = tuple(i for i in axes if i < ndim)
if len(axes) == 0:
continue
def bn_grad_wrt_inputs_f(x, dy, scale, x_mean, x_invstd):
g_inputs, g_scale, g_bias = bn.AbstractBatchNormTrainGrad(
axes)(x, dy, scale, x_mean, x_invstd)
return g_inputs
def bn_grad_wrt_scale_f(x, dy, scale, x_mean, x_invstd):
g_inputs, g_scale, g_bias = bn.AbstractBatchNormTrainGrad(
axes)(x, dy, scale, x_mean, x_invstd)
return g_scale
def bn_grad_wrt_bias_f(x, dy, scale, x_mean, x_invstd):
g_inputs, g_scale, g_bias = bn.AbstractBatchNormTrainGrad(
axes)(x, dy, scale, x_mean, x_invstd)
return g_bias
# run
for data_shape in ((4, 3, 3, 3, 3), (4, 3, 1, 1, 1), (2, 3, 5, 3,
2)):
data_shape = data_shape[:ndim]
param_shape = tuple(1 if d in axes else s
for d, s in enumerate(data_shape))
# force float64 for sufficient numerical stability
x_val = 4 + 3 * np.random.randn(*data_shape).astype('float64')
dy_val = -1 + 2 * np.random.randn(
*data_shape).astype('float64')
scale_val = np.random.randn(*param_shape).astype('float64')
x_mean_val = np.random.randn(*param_shape).astype('float64')
x_invstd_val = np.random.randn(*param_shape).astype('float64')
utt.verify_grad(
bn_grad_wrt_inputs_f,
[x_val, dy_val, scale_val, x_mean_val, x_invstd_val])
utt.verify_grad(
bn_grad_wrt_scale_f,
[x_val, dy_val, scale_val, x_mean_val, x_invstd_val])
utt.verify_grad(
bn_grad_wrt_bias_f,
[x_val, dy_val, scale_val, x_mean_val, x_invstd_val])
def verify_solve_grad(self, m, n, A_structure, lower, rng):
# ensure diagonal elements of A relatively large to avoid numerical
# precision issues
A_val = (rng.normal(size=(m, m)) * 0.5 + np.eye(m)).astype(
config.floatX)
if A_structure == 'lower_triangular':
A_val = np.tril(A_val)
elif A_structure == 'upper_triangular':
A_val = np.triu(A_val)
if n is None:
b_val = rng.normal(size=m).astype(config.floatX)
else:
b_val = rng.normal(size=(m, n)).astype(config.floatX)
eps = None
if config.floatX == "float64":
eps = 2e-8
if A_structure in ('lower_triangular', 'upper_triangular'):
solve_op = GpuCublasTriangularSolve(lower=lower)
else:
solve_op = GpuCusolverSolve(A_structure="general")
utt.verify_grad(solve_op, [A_val, b_val], 3, rng, eps=eps)
def test_AveragePoolGrad_grad(self):
rng = numpy.random.RandomState(utt.fetch_seed())
avgpoolshps = ((1, 1), (3, 2), (2, 3))
imval = rng.rand(2, 3, 3, 4) * 10.0
# more variance means numeric gradient will be more accurate
for avgpoolshp in avgpoolshps:
for ignore_border in [True, False]:
for mode in ['sum', 'average_inc_pad', 'average_exc_pad']:
# print 'maxpoolshp =', maxpoolshp
# print 'ignore_border =', ignore_border
# The shape of the gradient will be the shape of the output
grad_shape = Pool.out_shape(
imval.shape, avgpoolshp, ignore_border=ignore_border)
grad_val = rng.rand(*grad_shape) * 10.0
def mp(input, grad):
grad_op = AveragePoolGrad(
avgpoolshp, ignore_border=ignore_border, mode=mode)
return grad_op(input, grad)
utt.verify_grad(mp, [imval, grad_val], rng=rng)
def test_max_pool_2d_2D_same_size(self):
rng = numpy.random.RandomState(utt.fetch_seed())
test_input_array = numpy.array([[[
[1., 2., 3., 4.],
[5., 6., 7., 8.]
]]])
test_answer_array = numpy.array([[[
[0., 0., 0., 0.],
[0., 6., 0., 8.]
]]])
input = tensor.tensor4(name='input')
patch_size = (2, 2)
op = max_pool_2d_same_size(input, patch_size)
op_output = function([input], op)(test_input_array)
assert numpy.all(op_output == test_answer_array), (
"op_output is %s, test_answer_array is %s" % (
op_output, numpy_output_val
)
)
def mp(input):
return max_pool_2d_same_size(input, patch_size)
utt.verify_grad(mp, [test_input_array], rng=rng)
def test_verify_grad_with_zeros(self):
# including zeros, as the case with zeros is important
# (and special cases: 1 zero in the row, more than 1 zero in the row)
x_val = numpy.asarray([[1.,2.,3.],[0.,5.,6.],[0.,0.,9.]], dtype='float32')
x = theano.tensor.dmatrix()
# sanity check
x2 = theano.tensor.dmatrix()
p = Prod(axis=1)(x)
p2 = Prod(axis=1)(x2)
fn = theano.function([x,x2],[p-p2], mode=self.mode)
#print "hand computed diff for each row"
x2_val = numpy.asarray([[1., 2., 3.003], [0.003,5.,6], [0.,0.,9.01]])
#print fn(x_val, x2_val)
fn2 = theano.function([x],[theano.tensor.grad(p.sum(),x)], mode=self.mode)
#print "real grad"
#print fn2(x_val)
fn3 = theano.function([x],[p], mode=self.mode)
assert numpy.allclose(fn3(x_val), [6.,0.,0.])
# now with verify_grad
unittest_tools.verify_grad(Prod(axis=1), [x_val], mode=self.mode)
def test_intensity(abs_tol=1e-5, rel_tol=1e-5, eps=1e-7):
with change_flags(compute_test_value="off"):
map = starry.Map(ydeg=2, udeg=2)
np.random.seed(11)
lat = 180 * (np.random.random(10) - 0.5)
lon = 360 * (np.random.random(10) - 0.5)
y = [1.0] + list(np.random.randn(8))
u = [-1.0] + list(np.random.randn(2))
f = [np.pi]
wta = 0.0
def intensity(lat, lon, y, u, f, wta):
return map.ops.intensity(lat, lon, y, u, f, wta, np.array(True))
verify_grad(
intensity,
(lat, lon, y, u, f, wta),
abs_tol=abs_tol,
rel_tol=rel_tol,
eps=eps,
n_tests=1,
)
def test_DownsampleFactorMaxPaddingStride_grad(self):
rng = numpy.random.RandomState(utt.fetch_seed())
imgsizes = ((10, 10), (10, 5), (5, 5))
maxpoolsizes = ((5, 3), (3, 5), (3, 3))
stridesizes = ((3, 2), (2, 3), (3, 3))
paddingsizes = ((2, 2), (2, 1), (2, 2))
# average_inc_pad and average_exc_pad do not
# support grad with padding
for mode in ['max', 'sum']:
for i in range(len(imgsizes)):
imgsize = imgsizes[i]
imval = rng.rand(1, 1, imgsize[0], imgsize[1]) * 10.0
maxpoolsize = maxpoolsizes[i]
stridesize = stridesizes[i]
paddingsize = paddingsizes[i]
def mp(input):
return Pool(ignore_border=True,
mode=mode)(input, maxpoolsize, stridesize,
paddingsize)
utt.verify_grad(mp, [imval], rng=rng)
def test_DownsampleFactorMaxGrad_grad(self):
rng = numpy.random.RandomState(utt.fetch_seed())
maxpoolshps = ((1, 1), (3, 2), (2, 3))
imval = rng.rand(2, 3, 3, 4) * 10.0
# more variance means numeric gradient will be more accurate
for maxpoolshp in maxpoolshps:
for ignore_border in [True, False]:
# print 'maxpoolshp =', maxpoolshp
# print 'ignore_border =', ignore_border
# The shape of the gradient will be the shape of the output
grad_shape = Pool.out_shape(imval.shape,
maxpoolshp,
ignore_border=ignore_border)
grad_val = rng.rand(*grad_shape) * 10.0
def mp(input, grad):
out = Pool(ignore_border=ignore_border)(input, maxpoolshp)
grad_op = MaxPoolGrad(ignore_border=ignore_border)
return grad_op(input, out, grad, maxpoolshp)
utt.verify_grad(mp, [imval, grad_val], rng=rng)
def test_large_sparse_targets_grad(self):
V_mat, U_mat, UinvT_mat, Q_mat, H_mat, Y_indexes_mat, Y_values_mat, Y = \
TestLargeSparseTargets.generate_data()
V = shared(V_mat)
U = shared(U_mat)
UinvT = shared(UinvT_mat)
Q = shared(Q_mat)
H = T.matrix()
Y_indexes = T.imatrix()
Y_values = T.matrix()
learning_rate = 0.1
def f(H):
LST_grad = self.op(COST)(V, U, UinvT, Q, H, Y_indexes_mat,
Y_values_mat, learning_rate)
return LST_grad
utt.verify_grad(f, [H_mat])
def test_sparseblockgemv_grad(self):
W_val, h_val, iIdx_val, b_val, oIdx_val = \
BlockSparse_Gemv_and_Outer.gemv_data()
h_val = randn(1, 1, 1).astype('float32')
iIdx_val = numpy.random.permutation(1)[:1][None, :]
oIdx_val = numpy.random.permutation(1)[:1][None, :]
W_val = randn(1, 1, 1, 1).astype('float32')
b_val = randn(1, 1).astype('float32')
iIdx = theano.tensor.constant(iIdx_val)
oIdx = theano.tensor.constant(oIdx_val)
def metaop(b, h, W):
return sparse_block_dot(W, h, iIdx, b, oIdx)
def op(b, h, W):
return self.gemv_op(b.take(oIdx, axis=0), W, h, iIdx, oIdx)
utt.verify_grad(metaop, [b_val, h_val, W_val], mode=self.mode)
utt.verify_grad(op, [b_val, h_val, W_val], mode=self.mode)
