def test_blocksparse_grad_merge(): b = tensor.fmatrix() h = tensor.ftensor3() iIdx = tensor.lmatrix() oIdx = tensor.lmatrix() W_val, h_val, iIdx_val, b_val, oIdx_val = blocksparse_data() W = float32_shared_constructor(W_val) o = sparse_block_gemv_ss(b.take(oIdx, axis=0), W, h, iIdx, oIdx) gW = theano.grad(o.sum(), W) lr = numpy.asarray(0.05, dtype='float32') upd = W - lr * gW f1 = theano.function([h, iIdx, b, oIdx], updates=[(W, upd)], mode=mode_with_gpu) # not running with mode=gpu ensures that the elemwise is not merged in f2 = theano.function([h, iIdx, b, oIdx], updates=[(W, upd)]) f2(h_val, iIdx_val, b_val, oIdx_val) W_ref = W.get_value() # reset the var W.set_value(W_val) f1(h_val, iIdx_val, b_val, oIdx_val) W_opt = W.get_value() utt.assert_allclose(W_ref, W_opt)
def test_blocksparse_grad_merge(): b = tensor.fmatrix() h = tensor.ftensor3() iIdx = tensor.lmatrix() oIdx = tensor.lmatrix() W_val, h_val, iIdx_val, b_val, oIdx_val = blocksparse_data() W = float32_shared_constructor(W_val) o = sparse_block_gemv_ss(b.take(oIdx, axis=0), W, h, iIdx, oIdx) gW = theano.grad(o.sum(), W) lr = numpy.asarray(0.05, dtype='float32') upd = W - lr * gW f1 = theano.function([h, iIdx, b, oIdx], updates=[(W, upd)], mode=mode_with_gpu) # not running with mode=gpu ensures that the elemwise is not merged in mode = None if theano.config.mode == 'FAST_COMPILE': mode = theano.compile.mode.get_mode('FAST_RUN') f2 = theano.function([h, iIdx, b, oIdx], updates=[(W, upd)], mode=mode) f2(h_val, iIdx_val, b_val, oIdx_val) W_ref = W.get_value() # reset the var W.set_value(W_val) f1(h_val, iIdx_val, b_val, oIdx_val) W_opt = W.get_value() utt.assert_allclose(W_ref, W_opt)
def Xtest_blocksparse_grad_merge(): b = tensor.fmatrix() h = tensor.ftensor3() iIdx = tensor.lmatrix() oIdx = tensor.lmatrix() W_val, h_val, iIdx_val, b_val, oIdx_val = blocksparse_data() W = float32_shared_constructor(W_val) o = sparse_block_gemv_ss(b.take(oIdx, axis=0), W, h, iIdx, oIdx) gW = theano.grad(o.sum(), W) lr = numpy.asarray(0.05, dtype='float32') upd = W - lr * gW f1 = theano.function([h, iIdx, b, oIdx], updates=[(W, upd)], mode=mode_with_gpu) # Make sure the lr update was merged. assert isinstance(f1.maker.fgraph.outputs[0].owner.op, SparseBlockOuterSS) # Exclude the merge optimizations. mode = mode_with_gpu.excluding('local_merge_blocksparse_alpha') mode = mode.excluding('local_merge_blocksparse_output') f2 = theano.function([h, iIdx, b, oIdx], updates=[(W, upd)], mode=mode) # Make sure the lr update is not merged. assert not isinstance(f2.maker.fgraph.outputs[0].owner.op, SparseBlockOuterSS) f2(h_val, iIdx_val, b_val, oIdx_val) W_ref = W.get_value() # reset the var W.set_value(W_val) f1(h_val, iIdx_val, b_val, oIdx_val) W_opt = W.get_value() utt.assert_allclose(W_ref, W_opt)
def test_blocksparse_grad_shape(): b = tensor.fmatrix() W = tensor.ftensor4() h = tensor.ftensor3() iIdx = tensor.lmatrix() oIdx = tensor.lmatrix() o = sparse_block_gemv_ss(b.take(oIdx, axis=0), W, h, iIdx, oIdx) go = theano.grad(o.sum(), [b, W, h]) f = theano.function([W, h, iIdx, b, oIdx], go, mode=mode_with_gpu) W_val, h_val, iIdx_val, b_val, oIdx_val = blocksparse_data() # just make sure that it runs correcly and all the shapes are ok. b_g, W_g, h_g = f(W_val, h_val, iIdx_val, b_val, oIdx_val) assert b_g.shape == b_val.shape assert h_g.shape == h_val.shape assert W_g.shape == W_val.shape
def test_blocksparse_grad_shape(): b = tensor.fmatrix() W = tensor.ftensor4() h = tensor.ftensor3() iIdx = tensor.lmatrix() oIdx = tensor.lmatrix() o = sparse_block_gemv_ss(b.take(oIdx, axis=0), W, h, iIdx, oIdx) go = theano.grad(o.sum(), [b, W, h]) f = theano.function([W, h, iIdx, b, oIdx], go, mode=mode_with_gpu) W_val, h_val, iIdx_val, b_val, oIdx_val = blocksparse_data() # just make sure that it runs correcly and all the shapes are ok. b_g, W_g, h_g = f(W_val, h_val, iIdx_val, b_val, oIdx_val) assert b_g.shape == b_val.shape assert h_g.shape == h_val.shape assert W_g.shape == W_val.shape
def Xtest_blocksparse_grad_merge(): b = tensor.fmatrix() h = tensor.ftensor3() iIdx = tensor.lmatrix() oIdx = tensor.lmatrix() W_val, h_val, iIdx_val, b_val, oIdx_val = blocksparse_data() W = float32_shared_constructor(W_val) o = sparse_block_gemv_ss(b.take(oIdx, axis=0), W, h, iIdx, oIdx) gW = theano.grad(o.sum(), W) lr = numpy.asarray(0.05, dtype='float32') upd = W - lr * gW f1 = theano.function([h, iIdx, b, oIdx], updates=[(W, upd)], mode=mode_with_gpu) # Make sure the lr update was merged. assert isinstance(f1.maker.fgraph.outputs[0].owner.op, SparseBlockOuterSS) # Exclude the merge optimizations. mode = mode_with_gpu.excluding('local_merge_blocksparse_alpha') mode = mode.excluding('local_merge_blocksparse_output') f2 = theano.function([h, iIdx, b, oIdx], updates=[(W, upd)], mode=mode) # Make sure the lr update is not merged. assert not isinstance(f2.maker.fgraph.outputs[0].owner.op, SparseBlockOuterSS) f2(h_val, iIdx_val, b_val, oIdx_val) W_ref = W.get_value() # reset the var W.set_value(W_val) f1(h_val, iIdx_val, b_val, oIdx_val) W_opt = W.get_value() utt.assert_allclose(W_ref, W_opt)
def f(b, h, W): return sparse_block_gemv_ss(b.take(oIdx, axis=0), W, h, iIdx, oIdx)
def f(b, h, W): return sparse_block_gemv_ss(b.take(oIdx, axis=0), W, h, iIdx, oIdx)