def check_rop_lop(self, y, out_shape): """ As check_mat_rop_lop, except the input is self.x which is a vector. The output is still a vector. """ # TEST ROP vx = np.asarray(self.rng.uniform(size=self.in_shape), aesara.config.floatX) vv = np.asarray(self.rng.uniform(size=self.in_shape), aesara.config.floatX) yv = tensor.Rop(y, self.x, self.v) rop_f = function([self.x, self.v], yv, on_unused_input="ignore") J, _ = aesara.scan( lambda i, y, x: tensor.grad(y[i], x), sequences=tensor.arange(y.shape[0]), non_sequences=[y, self.x], ) sy = tensor.dot(J, self.v) scan_f = function([self.x, self.v], sy, on_unused_input="ignore") v1 = rop_f(vx, vv) v2 = scan_f(vx, vv) assert np.allclose(v1, v2), "ROP mismatch: {} {}".format(v1, v2) try: tensor.Rop( aesara.clone(y, replace={self.x: break_op(self.x)}), self.x, self.v ) except ValueError: pytest.skip( "Rop does not handle non-differentiable inputs " "correctly. Bug exposed by fixing Add.grad method." ) vx = np.asarray(self.rng.uniform(size=self.in_shape), aesara.config.floatX) vv = np.asarray(self.rng.uniform(size=out_shape), aesara.config.floatX) yv = tensor.Lop(y, self.x, self.v) lop_f = function([self.x, self.v], yv, on_unused_input="ignore") J, _ = aesara.scan( lambda i, y, x: tensor.grad(y[i], x), sequences=tensor.arange(y.shape[0]), non_sequences=[y, self.x], ) sy = tensor.dot(self.v, J) scan_f = function([self.x, self.v], sy) v1 = lop_f(vx, vv) v2 = scan_f(vx, vv) assert np.allclose(v1, v2), "LOP mismatch: {} {}".format(v1, v2)
def check_nondiff_rop(self, y): """ If your op is not differentiable(so you can't define Rop) test that an error is raised. """ with pytest.raises(ValueError): tensor.Rop(y, self.x, self.v)
def test_Rop_dot_bug_18Oct2013_Jeremiah(self): # This test refers to a bug reported by Jeremiah Lowin on 18th Oct # 2013. The bug consists when through a dot operation there is only # one differentiable path (i.e. there is no gradient wrt to one of # the inputs). x = tensor.arange(20.0).reshape([1, 20]) v = aesara.shared(np.ones([20])) d = tensor.dot(x, v).sum() tensor.Rop(tensor.grad(d, v), v, v)
def test_rop_lop(): mx = tensor.matrix("mx") mv = tensor.matrix("mv") v = tensor.vector("v") y = matrix_inverse(mx).sum(axis=0) yv = tensor.Rop(y, mx, mv) rop_f = function([mx, mv], yv) sy, _ = aesara.scan( lambda i, y, x, v: (tensor.grad(y[i], x) * v).sum(), sequences=tensor.arange(y.shape[0]), non_sequences=[y, mx, mv], ) scan_f = function([mx, mv], sy) rng = np.random.RandomState(utt.fetch_seed()) vx = np.asarray(rng.randn(4, 4), aesara.config.floatX) vv = np.asarray(rng.randn(4, 4), aesara.config.floatX) v1 = rop_f(vx, vv) v2 = scan_f(vx, vv) assert _allclose(v1, v2), "ROP mismatch: {} {}".format(v1, v2) raised = False try: tensor.Rop(aesara.clone(y, replace={mx: break_op(mx)}), mx, mv) except ValueError: raised = True if not raised: raise Exception("Op did not raised an error even though the function" " is not differentiable") vv = np.asarray(rng.uniform(size=(4, )), aesara.config.floatX) yv = tensor.Lop(y, mx, v) lop_f = function([mx, v], yv) sy = tensor.grad((v * y).sum(), mx) scan_f = function([mx, v], sy) v1 = lop_f(vx, vv) v2 = scan_f(vx, vv) assert _allclose(v1, v2), "LOP mismatch: {} {}".format(v1, v2)
def test_invalid_input(self): success = False try: tensor.Rop(0.0, [tensor.matrix()], [tensor.vector()]) success = True except ValueError: pass assert not success
def test_conv(self): for conv_op in [conv.conv2d, conv2d]: for border_mode in ["valid", "full"]: image_shape = (2, 2, 4, 5) filter_shape = (2, 2, 2, 3) image_dim = len(image_shape) filter_dim = len(filter_shape) input = tensor.TensorType(aesara.config.floatX, [False] * image_dim)( name="input" ) filters = tensor.TensorType(aesara.config.floatX, [False] * filter_dim)( name="filter" ) ev_input = tensor.TensorType(aesara.config.floatX, [False] * image_dim)( name="ev_input" ) ev_filters = tensor.TensorType( aesara.config.floatX, [False] * filter_dim )(name="ev_filters") def sym_conv2d(input, filters): return conv_op(input, filters, border_mode=border_mode) output = sym_conv2d(input, filters).flatten() yv = tensor.Rop(output, [input, filters], [ev_input, ev_filters]) mode = None if aesara.config.mode == "FAST_COMPILE": mode = "FAST_RUN" rop_f = function( [input, filters, ev_input, ev_filters], yv, on_unused_input="ignore", mode=mode, ) sy, _ = aesara.scan( lambda i, y, x1, x2, v1, v2: (tensor.grad(y[i], x1) * v1).sum() + (tensor.grad(y[i], x2) * v2).sum(), sequences=tensor.arange(output.shape[0]), non_sequences=[output, input, filters, ev_input, ev_filters], mode=mode, ) scan_f = function( [input, filters, ev_input, ev_filters], sy, on_unused_input="ignore", mode=mode, ) dtype = aesara.config.floatX image_data = np.random.random(image_shape).astype(dtype) filter_data = np.random.random(filter_shape).astype(dtype) ev_image_data = np.random.random(image_shape).astype(dtype) ev_filter_data = np.random.random(filter_shape).astype(dtype) v1 = rop_f(image_data, filter_data, ev_image_data, ev_filter_data) v2 = scan_f(image_data, filter_data, ev_image_data, ev_filter_data) assert np.allclose(v1, v2), "Rop mismatch: {} {}".format(v1, v2)
def test_downsample(self): rng = np.random.RandomState(utt.fetch_seed()) # ws, shp examples = ( ((2,), (16,)), ( (2,), ( 4, 16, ), ), ( (2,), ( 4, 2, 16, ), ), ((1, 1), (4, 2, 16, 16)), ((2, 2), (4, 2, 16, 16)), ((3, 3), (4, 2, 16, 16)), ((3, 2), (4, 2, 16, 16)), ((3, 2, 2), (3, 2, 16, 16, 16)), ((2, 3, 2), (3, 2, 16, 16, 16)), ((2, 2, 3), (3, 2, 16, 16, 16)), ((2, 2, 3, 2), (3, 2, 6, 6, 6, 5)), ) for example, ignore_border in itertools.product(examples, [True, False]): (ws, shp) = example vx = rng.rand(*shp) vex = rng.rand(*shp) x = aesara.shared(vx) ex = aesara.shared(vex) maxpool_op = Pool(ignore_border, ndim=len(ws)) a_pooled = maxpool_op(x, ws).flatten() yv = tensor.Rop(a_pooled, x, ex) mode = None if aesara.config.mode == "FAST_COMPILE": mode = "FAST_RUN" rop_f = function([], yv, on_unused_input="ignore", mode=mode) sy, _ = aesara.scan( lambda i, y, x, v: (tensor.grad(y[i], x) * v).sum(), sequences=tensor.arange(a_pooled.shape[0]), non_sequences=[a_pooled, x, ex], mode=mode, ) scan_f = function([], sy, on_unused_input="ignore", mode=mode) v1 = rop_f() v2 = scan_f() assert np.allclose(v1, v2), "Rop mismatch: {} {}".format(v1, v2)
def __call__(self, v, cost, parameters, damp): # compute Gauss-Newton Matrix right-multiplied by `v` Jv = tt.Rop(self._s, parameters, v) HJv = tt.grad( tt.sum(tt.grad(cost, self._s) * Jv), self._s, consider_constant=[Jv] ) JHJv = tt.grad(tt.sum(HJv * self._s), parameters, consider_constant=[HJv, Jv]) # apply Tikhonov damping JHJv = [JHJvi + damp * vi for JHJvi, vi in zip(JHJv, v)] return JHJv
def test_multiple_outputs(self): m = tensor.matrix("m") v = tensor.vector("v") m_ = tensor.matrix("m_") v_ = tensor.vector("v_") mval = self.rng.uniform(size=(3, 7)).astype(aesara.config.floatX) vval = self.rng.uniform(size=(7,)).astype(aesara.config.floatX) m_val = self.rng.uniform(size=(3, 7)).astype(aesara.config.floatX) v_val = self.rng.uniform(size=(7,)).astype(aesara.config.floatX) rop_out1 = tensor.Rop([m, v, m + v], [m, v], [m_, v_]) assert isinstance(rop_out1, list) assert len(rop_out1) == 3 rop_out2 = tensor.Rop((m, v, m + v), [m, v], [m_, v_]) assert isinstance(rop_out2, tuple) assert len(rop_out2) == 3 all_outs = [] for o in rop_out1, rop_out2: all_outs.extend(o) f = aesara.function([m, v, m_, v_], all_outs) f(mval, vval, m_val, v_val)
def check_mat_rop_lop(self, y, out_shape): """ Test the Rop/Lop when input is a matrix and the output is a vector :param y: the output variable of the op applied to self.mx :param out_shape: Used to generate a random tensor corresponding to the evaluation point of the Rop (i.e. the tensor with which you multiply the Jacobian). It should be a tuple of ints. If the Op has more than 1 input, one of them must be mx, while others must be shared variables / constants. We will test only against the input self.mx, so you must call check_mat_rop_lop/check_rop_lop for the other inputs. We expect all inputs/outputs have dtype floatX. If you want to test an Op with an output matrix, add a sum after the Op you want to test. """ vx = np.asarray(self.rng.uniform(size=self.mat_in_shape), aesara.config.floatX) vv = np.asarray(self.rng.uniform(size=self.mat_in_shape), aesara.config.floatX) yv = tensor.Rop(y, self.mx, self.mv) rop_f = function([self.mx, self.mv], yv, on_unused_input="ignore") sy, _ = aesara.scan( lambda i, y, x, v: (tensor.grad(y[i], x) * v).sum(), sequences=tensor.arange(y.shape[0]), non_sequences=[y, self.mx, self.mv], ) scan_f = function([self.mx, self.mv], sy, on_unused_input="ignore") v1 = rop_f(vx, vv) v2 = scan_f(vx, vv) assert np.allclose(v1, v2), "ROP mismatch: {} {}".format(v1, v2) self.check_nondiff_rop(aesara.clone(y, replace={self.mx: break_op(self.mx)})) vv = np.asarray(self.rng.uniform(size=out_shape), aesara.config.floatX) yv = tensor.Lop(y, self.mx, self.v) lop_f = function([self.mx, self.v], yv) sy = tensor.grad((self.v * y).sum(), self.mx) scan_f = function([self.mx, self.v], sy) v1 = lop_f(vx, vv) v2 = scan_f(vx, vv) assert np.allclose(v1, v2), "LOP mismatch: {} {}".format(v1, v2)
def test_rop(self, cls_ofg): a = tt.vector() M = tt.matrix() b = tt.dot(a, M) op_matmul = cls_ofg([a, M], [b]) x = tt.vector() W = tt.matrix() y = op_matmul(x, W) du = tt.vector() dv = tt.Rop(y, x, du) fn = function([x, W, du], dv) xval = np.random.rand(16).astype(config.floatX) Wval = np.random.rand(16, 16).astype(config.floatX) duval = np.random.rand(16).astype(config.floatX) dvval = np.dot(duval, Wval) dvval2 = fn(xval, Wval, duval) assert np.allclose(dvval2, dvval)
def test_rop_override(self, cls_ofg): x, y = tt.vectors("xy") def ro(inps, epts): x, y = inps u, v = epts return [u * y * 2.0 + x * v * 1.5] u, v = tt.vectors("uv") op_mul_rop = cls_ofg([x, y, u, v], ro([x, y], [u, v])) op_mul = cls_ofg([x, y], [x * y], rop_overrides=ro) op_mul2 = cls_ofg([x, y], [x * y], rop_overrides=op_mul_rop) # single override case xx, yy = tt.vector("xx"), tt.vector("yy") du, dv = tt.vector("du"), tt.vector("dv") for op in [op_mul, op_mul2]: zz = op_mul(xx, yy) dw = tt.Rop(zz, [xx, yy], [du, dv]) fn = function([xx, yy, du, dv], dw) vals = np.random.rand(4, 32).astype(config.floatX) dwval = fn(*vals) assert np.allclose( dwval, vals[0] * vals[3] * 1.5 + vals[1] * vals[2] * 2.0)
def test_pool2d(): shps = [ (1, 12), (1, 1, 12), (1, 1, 1, 12), (1, 1, 2, 2), (1, 1, 1, 1), (1, 1, 4, 4), (1, 1, 10, 11), (1, 2, 2, 2), (3, 5, 4, 4), (25, 1, 7, 7), (1, 1, 12, 12), (1, 1, 2, 14), (1, 1, 12, 14), (1, 1, 14, 14), (1, 1, 16, 16), (1, 1, 18, 18), (1, 1, 24, 24), (1, 6, 24, 24), (10, 1, 24, 24), (10, 6, 24, 24), (30, 6, 12, 12), (30, 2, 24, 24), (30, 6, 24, 24), (10, 10, 10, 11), (1, 1, 10, 1025), (1, 1, 10, 1023), (1, 1, 1025, 10), (1, 1, 1023, 10), (3, 2, 16, 16, 16), (3, 2, 6, 6, 6, 5), (3, 2, 6, 6, 6, 5, 7), ] np.random.RandomState(utt.fetch_seed()).shuffle(shps) test_ws = (2, 2), (3, 2), (1, 1) test_st = (2, 2), (3, 2), (1, 1) test_mode = ["max", "sum", "average_inc_pad", "average_exc_pad"] ref_mode = copy.copy(mode_without_gpu) ref_mode.check_py_code = False gpu_mode = mode_with_gpu.excluding("cudnn") gpu_mode.check_py_code = False for shp in shps: for mode, ws, st in itertools.product(test_mode, test_ws, test_st): if ws[0] > shp[-2] or ws[1] > shp[-1]: continue for ignore_border, pad in zip((True, False), [(1, 1), (0, 0)]): if pad[0] >= ws[0] or pad[1] >= ws[1]: continue if mode == "average_exc_pad" and (pad[0] > 0 or pad[1] > 0): continue # print('test_pool2d', shp, ws, st, pad, mode, ignore_border) ds_op = Pool(ndim=len(ws), mode=mode, ignore_border=ignore_border) a = aesara.shared(rand(*shp), "a") a_pooled = ds_op(tensor.as_tensor_variable(a), ws, st, pad) f = aesara.function([], a_pooled, mode=gpu_mode) f2 = aesara.function([], a_pooled, mode=ref_mode) assert any( [isinstance(node.op, GpuPool) for node in f.maker.fgraph.toposort()] ) assert any( [isinstance(node.op, Pool) for node in f2.maker.fgraph.toposort()] ) assert np.allclose(f(), f2()), (shp, ws, st, pad, mode, ignore_border) a_pooled_grad = tensor.grad(a_pooled.sum(), a) g = aesara.function([], a_pooled_grad, mode=gpu_mode) g2 = aesara.function([], a_pooled_grad, mode=ref_mode) if mode == "max": gop = GpuMaxPoolGrad gop2 = MaxPoolGrad else: gop = GpuAveragePoolGrad gop2 = AveragePoolGrad assert any( [isinstance(node.op, gop) for node in g.maker.fgraph.toposort()] ) assert any( [isinstance(node.op, gop2) for node in g2.maker.fgraph.toposort()] ) assert np.allclose(g(), g2()), (shp, ws, st, pad, mode, ignore_border) # test rop and grad grad for max pooling # for average pooling grad grad is just average pooling grad if mode != "max": continue ea = aesara.shared(rand(*shp), "ea") gr = aesara.function([], tensor.Rop(a_pooled, a, ea), mode=gpu_mode) gr2 = aesara.function([], tensor.Rop(a_pooled, a, ea), mode=ref_mode) assert any( [ isinstance(node.op, GpuDownsampleFactorMaxGradGrad) for node in gr.maker.fgraph.toposort() ] ) assert any( [ isinstance(node.op, DownsampleFactorMaxGradGrad) for node in gr2.maker.fgraph.toposort() ] ) assert np.allclose(gr(), gr2()), (shp, ws, st, pad, mode, ignore_border) ggf = gradient.Lop(tensor.grad((a_pooled ** 2).sum(), a), a, a) gg = aesara.function([], ggf, mode=gpu_mode) gg2 = aesara.function([], ggf, mode=ref_mode) assert any( [ isinstance(node.op, GpuDownsampleFactorMaxGradGrad) for node in gg.maker.fgraph.toposort() ] ) assert any( [ isinstance(node.op, DownsampleFactorMaxGradGrad) for node in gg2.maker.fgraph.toposort() ] ) assert np.allclose(gg(), gg2()), (shp, ws, st, pad, mode, ignore_border)