def test_basic(self): # Reported in https://github.com/Aesara/Aesara/issues/5730 x = tensor.tensor3() y = tensor.tensor3() z = tensor.batched_dot(x, y[:, 0, :, np.newaxis]) f = aesara.function([x, y], z, mode=mode_with_gpu) x_num = np.arange(32 * 19 * 600, dtype=config.floatX).reshape( (32, 19, 600)) y_num = np.arange(7 * 32 * 600, dtype=config.floatX).reshape( (32, 7, 600)) f(x_num, y_num) assert f.maker.fgraph.toposort()[-2].op.inplace
def test_non_zero_init(self): # Test the case where the initial value for the nitsot output is non-zero input1 = tt.tensor3() input2 = tt.tensor3() input3 = tt.tensor3() W = aesara.shared(np.random.normal(size=(4, 5))).astype(config.floatX) U = aesara.shared(np.random.normal(size=(6, 7))).astype(config.floatX) def inner_fct(seq1, seq2, seq3, previous_output): temp1 = tt.dot(seq1, W) + seq3 temp2 = tt.dot(seq2, U) dot_output = tt.dot(temp1, temp2) return previous_output + dot_output init = tt.as_tensor_variable(np.random.normal(size=(3, 7))) # Compile the function twice, once with the optimization and once # without opt_mode = mode.including("scan") h, _ = aesara.scan( inner_fct, sequences=[input1, input2, input3], outputs_info=init, mode=opt_mode, ) output = h[-1] f_opt = aesara.function([input1, input2, input3], output, mode=opt_mode) no_opt_mode = mode.excluding("scanOp_pushout_output") h, _ = aesara.scan( inner_fct, sequences=[input1, input2, input3], outputs_info=init, mode=no_opt_mode, ) output = h[-1] f_no_opt = aesara.function([input1, input2, input3], output, mode=no_opt_mode) # Ensure that the optimization has been applied for f_opt # TODO # Compare the outputs of the 2 functions input1_value = np.random.random((2, 3, 4)).astype(config.floatX) input2_value = np.random.random((2, 5, 6)).astype(config.floatX) input3_value = np.random.random((2, 3, 5)).astype(config.floatX) output_opt = f_opt(input1_value, input2_value, input3_value) output_no_opt = f_no_opt(input1_value, input2_value, input3_value) utt.assert_allclose(output_opt, output_no_opt)
def test_infer_shape(self, mode): op_class = partial(self.op_class, mode=mode) x = tt.tensor3("x") a = np.random.random((3, 5, 2)).astype(aesara.config.floatX) for axis in range(-len(a.shape), len(a.shape)): self._compile_and_check([x], [op_class(axis=axis)(x)], [a], GpuCumOp)
def test_cum_op(self): x = tt.tensor3("x") a = np.random.random((3, 5, 2)).astype(config.floatX) # Test axis out of bounds with pytest.raises(ValueError): cumsum(x, axis=3) with pytest.raises(ValueError): cumsum(x, axis=-4) with pytest.raises(ValueError): cumprod(x, axis=3) with pytest.raises(ValueError): cumprod(x, axis=-4) f = aesara.function([x], [cumsum(x), cumprod(x)]) s, p = f(a) assert np.allclose(np.cumsum(a), s) # Test axis=None assert np.allclose(np.cumprod(a), p) # Test axis=None for axis in range(-len(a.shape), len(a.shape)): f = aesara.function([x], [cumsum(x, axis=axis), cumprod(x, axis=axis)]) s, p = f(a) assert np.allclose(np.cumsum(a, axis=axis), s) assert np.allclose(np.cumprod(a, axis=axis), p)
def setup_method(self): super().setup_method() self.A = tensor.tensor4("A", dtype=aesara.config.floatX) self.B = tensor.tensor3("B", dtype=aesara.config.floatX) self.a = np.random.rand(4, 6, 8, 3).astype(aesara.config.floatX) self.b = np.random.rand(2, 15, 30).astype(aesara.config.floatX) self.b1 = np.random.rand(30, 2, 15).astype( aesara.config.floatX ) # for ind=1 since we need prod(b1.shape[:ind]) == prod(b1.shape[ind:])
def test_infer_shape(self): x = tt.tensor3("x") a = np.random.random((3, 5, 2)).astype(config.floatX) # Test axis=None self._compile_and_check([x], [self.op(x)], [a], self.op_class) for axis in range(-len(a.shape), len(a.shape)): self._compile_and_check([x], [cumsum(x, axis=axis)], [a], self.op_class)
def test_NanGuardMode(): # Tests if NanGuardMode is working by feeding in numpy.inf and numpy.nans # intentionally. A working implementation should be able to capture all # the abnormalties. x = tt.matrix() w = aesara.shared(np.random.randn(5, 7).astype(aesara.config.floatX)) y = tt.dot(x, w) fun = aesara.function( [x], y, mode=NanGuardMode(nan_is_error=True, inf_is_error=True) ) a = np.random.randn(3, 5).astype(aesara.config.floatX) infa = np.tile((np.asarray(100.0) ** 1000000).astype(aesara.config.floatX), (3, 5)) nana = np.tile(np.asarray(np.nan).astype(aesara.config.floatX), (3, 5)) biga = np.tile(np.asarray(1e20).astype(aesara.config.floatX), (3, 5)) fun(a) # normal values # Temporarily silence logger _logger = logging.getLogger("aesara.compile.nanguardmode") try: _logger.propagate = False with pytest.raises(AssertionError): fun(infa) # INFs with pytest.raises(AssertionError): fun(nana) # NANs with pytest.raises(AssertionError): fun(biga) # big values finally: _logger.propagate = True # slices a = np.random.randn(3, 4, 5).astype(aesara.config.floatX) infa = np.tile( (np.asarray(100.0) ** 1000000).astype(aesara.config.floatX), (3, 4, 5) ) nana = np.tile(np.asarray(np.nan).astype(aesara.config.floatX), (3, 4, 5)) biga = np.tile(np.asarray(1e20).astype(aesara.config.floatX), (3, 4, 5)) x = tt.tensor3() y = x[:, tt.arange(2), tt.arange(2), None] fun = aesara.function( [x], y, mode=NanGuardMode(nan_is_error=True, inf_is_error=True) ) fun(a) # normal values try: _logger.propagate = False with pytest.raises(AssertionError): fun(infa) # INFs with pytest.raises(AssertionError): fun(nana) # NANs with pytest.raises(AssertionError): fun(biga) # big values finally: _logger.propagate = True
def setup_method(self): super().setup_method() self.op_class = SearchsortedOp self.op = SearchsortedOp() self.x = tt.vector("x") self.v = tt.tensor3("v") self.a = 30 * np.random.random(50).astype(config.floatX) self.b = 30 * np.random.random((8, 10, 5)).astype(config.floatX) self.idx_sorted = np.argsort(self.a).astype("int32")
def test_multiple_out_crash(self): # This test failed up to commit 2faeb62c38 p0 = self.shared(np.asarray(np.random.random([4, 8]), dtype=self.dtype)) p1 = self.shared(np.asarray(np.random.random(8), dtype=self.dtype)) p2 = self.shared(np.asarray(np.random.random([8, 3]), dtype=self.dtype)) p3 = self.shared(np.asarray(np.random.random(3), dtype=self.dtype)) p = [p0, p1, p2, p3] # in my code these vars are the result of applying scan ften0 = tensor.tensor3("ft0", dtype=self.dtype) fmat1 = tensor.matrix("fm1", dtype=self.dtype) ften2 = tensor.tensor3("ft2", dtype=self.dtype) fmat3 = tensor.matrix("fm3", dtype=self.dtype) # then I keep only the last iteration fsub0 = ften0[-1] fsub1 = fmat1[-1] fsub2 = ften2[-1] fsub3 = fmat3[-1] fsub = [fsub0, fsub1, fsub2, fsub3] acc = aesara.tensor.constant(1, "int8") >= 0 new_positions = aesara.ifelse.ifelse(acc, fsub, p) new_updates = [(p[0], new_positions[0])] f = aesara.function([ften0, fmat1, ften2, fmat3], [], updates=new_updates, mode=self.mode) self.assertFunctionContains1(f, self.get_ifelse(4)) i1 = np.asarray(np.random.random([19, 4, 8]), dtype=self.dtype) i2 = np.asarray(np.random.random([19, 8]), dtype=self.dtype) i3 = np.asarray(np.random.random([19, 8, 3]), dtype=self.dtype) i4 = np.asarray(np.random.random([19, 3]), dtype=self.dtype) f(i1, i2, i3, i4)
def test_correct_answer(self): a = tt.matrix() b = tt.matrix() x = tt.tensor3() y = tt.tensor3() A = np.cast[aesara.config.floatX](np.random.rand(5, 3)) B = np.cast[aesara.config.floatX](np.random.rand(7, 2)) X = np.cast[aesara.config.floatX](np.random.rand(5, 6, 1)) Y = np.cast[aesara.config.floatX](np.random.rand(1, 9, 3)) make_list((3.0, 4.0)) c = make_list((a, b)) z = make_list((x, y)) fc = aesara.function([a, b], c) fz = aesara.function([x, y], z) for m, n in zip(fc(A, B), [A, B]): assert (m == n).all() for m, n in zip(fz(X, Y), [X, Y]): assert (m == n).all()
def test_perform(self): x = tt.matrix() y = tt.scalar() f = function([x, y], fill_diagonal(x, y)) for shp in [(8, 8), (5, 8), (8, 5)]: a = np.random.rand(*shp).astype(config.floatX) val = np.cast[config.floatX](np.random.rand()) out = f(a, val) # We can't use np.fill_diagonal as it is bugged. assert np.allclose(np.diag(out), val) assert (out == val).sum() == min(a.shape) # test for 3dtt a = np.random.rand(3, 3, 3).astype(config.floatX) x = tt.tensor3() y = tt.scalar() f = function([x, y], fill_diagonal(x, y)) val = np.cast[config.floatX](np.random.rand() + 10) out = f(a, val) # We can't use np.fill_diagonal as it is bugged. assert out[0, 0, 0] == val assert out[1, 1, 1] == val assert out[2, 2, 2] == val assert (out == val).sum() == min(a.shape)
def make_node(self, diag): diag = at.as_tensor_variable(diag) if diag.type.ndim != 2: raise TypeError("data argument must be a matrix", diag.type) return Apply(self, [diag], [at.tensor3(dtype=diag.dtype)])
def test_machine_translation(self): # This test case comes from https://github.com/rizar/scan-grad-speed and # is an example of actual computation done with scan in the context of # machine translation # # 'dim' has been reduced from 1000 to 5 to make the test run faster # Parameters from an actual machine tranlation run batch_size = 80 seq_len = 50 dim = 5 # Weight matrices U = aesara.shared( np.random.normal(size=(dim, dim), scale=0.0001).astype(config.floatX) ) U.name = "U" V = aesara.shared(U.get_value()) V.name = "V" W = aesara.shared(U.get_value()) W.name = "W" # Variables and their values x = tt.tensor3("x") x_value = np.random.normal( size=(seq_len, batch_size, dim), scale=0.0001 ).astype(config.floatX) ri = tt.tensor3("ri") ri_value = x_value zi = tt.tensor3("zi") zi_value = x_value init = tt.alloc(np.cast[config.floatX](0), batch_size, dim) def rnn_step1( # sequences x, ri, zi, # outputs_info h, ): pre_r = ri + h.dot(U) pre_z = zi + h.dot(V) r = tt.nnet.sigmoid(pre_r) z = tt.nnet.sigmoid(pre_z) after_r = r * h pre_h = x + after_r.dot(W) new_h = tt.tanh(pre_h) res_h = z * new_h + (1 - z) * h return res_h # Compile the function twice, once with the optimization and once # without opt_mode = mode.including("scan") h, _ = aesara.scan( rnn_step1, sequences=[x, ri, zi], n_steps=seq_len, outputs_info=init, name="fpass1", mode=opt_mode, ) cost = h[-1].sum() grad1 = tt.grad(cost, [U, V, W]) f_opt = aesara.function(inputs=[x, ri, zi], outputs=grad1, mode=opt_mode) no_opt_mode = mode.excluding("scanOp_pushout_output") h, _ = aesara.scan( rnn_step1, sequences=[x, ri, zi], n_steps=seq_len, outputs_info=init, name="fpass1", mode=no_opt_mode, ) cost = h[-1].sum() grad1 = tt.grad(cost, [U, V, W]) f_no_opt = aesara.function(inputs=[x, ri, zi], outputs=grad1, mode=no_opt_mode) # Validate that the optimization has been applied scan_node_grad = [ node for node in f_opt.maker.fgraph.toposort() if isinstance(node.op, Scan) ][1] for output in scan_node_grad.op.outputs: assert not ( isinstance(output.owner.op, tt.elemwise.Elemwise) and any([isinstance(i, tt.Dot) for i in output.owner.inputs]) ) # Compare the outputs of the two functions on the same input data. f_opt_output = f_opt(x_value, ri_value, zi_value) f_no_opt_output = f_no_opt(x_value, ri_value, zi_value) utt.assert_allclose(f_opt_output, f_no_opt_output)
def _run(self, num_features, num_timesteps, batch_size, mode): # determine shapes of inputs and targets depending on the batch size if batch_size == 1: inputs_size = (num_timesteps, num_features) targets_size = (num_timesteps, 1) else: inputs_size = (num_timesteps, batch_size, num_features) targets_size = (num_timesteps, batch_size, 1) # make inputs and targets shared variables inputs = aesara.shared( self.rng.uniform(size=inputs_size).astype(config.floatX), borrow=True ) targets = aesara.shared( self.rng.uniform(size=targets_size).astype(config.floatX), borrow=True ) # create symbolic inputs and targets variables if batch_size == 1: x = tt.matrix("inputs") t = tt.matrix("targets") else: x = tt.tensor3("inputs") t = tt.tensor3("inputs") x.tag.test_value = inputs.get_value(borrow=True) t.tag.test_value = targets.get_value(borrow=True) # create a set of parameters for a simple RNN W_xh = aesara.shared( (0.01 * self.rng.uniform(size=(num_features, 10))).astype(config.floatX), borrow=True, ) W_hh = aesara.shared( (0.01 * self.rng.uniform(size=(10, 10))).astype(config.floatX), borrow=True ) W_hy = aesara.shared( (0.01 * self.rng.uniform(size=(10, 1))).astype(config.floatX), borrow=True ) b_h = aesara.shared(np.zeros(10).astype(config.floatX), borrow=True) b_y = aesara.shared(np.zeros(1).astype(config.floatX), borrow=True) params = [W_xh, W_hh, W_hy, b_h, b_y] # recurrent function def step(x_t, h_tm1): h = tt.tanh(tt.dot(h_tm1, W_hh) + tt.dot(x_t, W_xh) + b_h) return h # build recurrent graph if batch_size == 1: h_0 = tt.alloc(0.0, 10).astype(config.floatX) else: h_0 = tt.alloc(0.0, batch_size, 10).astype(config.floatX) h, updates = aesara.scan(step, sequences=[x], outputs_info=[h_0]) # network output y = tt.dot(h, W_hy) + b_y # Create Gauss-Newton-Matrix object. Not really of any use here, but I # need it for Hessian-Free optimization. gn = GaussNewtonMatrix(y) # compute MSE cost = ((t - y) ** 2).sum(axis=1).mean() # Compute the cost at some other point in the parameter # space. Not really of any use here, but this is how I do it # during certain iterations of CG in the HF algorithm. There, # it's in fact `pi + current update proposal`. For simplicity, # I just multiply by 2 here. cost_ = aesara.clone(cost, replace={pi: 2 * pi for pi in params}) # Compute Gauss-Newton-Matrix times some vector `v` which is `p` in CG, # but for simplicity, I just take the parameters vector because it's # already there. Gv = gn(v=params, cost=cost, parameters=params, damp=tt.constant(1.0)) # compile Aesara function f = aesara.function([], [cost_] + Gv, givens={x: inputs, t: targets}, mode=mode) # execute f()