def compare_optimizer_variable_select(opt_ng, opt_ref): # Set up data placeholders C = ng.make_axis(20) N = ng.make_axis(32, name='N') data = ng.placeholder([C, N]) target = ng.placeholder([N]) # params to be updated using optimizer to be tested np_W1 = np.random.rand(C.length) np_W2 = np.random.rand(C.length) W1 = ng.variable([C], initial_value=np_W1) W2 = ng.variable([C], initial_value=np_W2) # Set up op graph cost = ng.sum(target - ng.dot(W1, data) - ng.dot(W2, data), out_axis=()) updated_weights = ng.sequential([opt_ng(cost, variables=[W1]), W1]) # Set up the computation and run the "train" loop with ExecutorFactory() as ex: opt_ng_comp = ex.transformer.computation([updated_weights, W2], data, target) mock_dataset = data_generator(20, C.length, N.length) for x, y in mock_dataset: [ng_W1, ng_W2] = opt_ng_comp(x, y) # updated weights for ngraph optimizer np_W1 = opt_ref(x, np_W1) # updated weights for reference optimizer ng.testing.assert_allclose(np_W1, ng_W1, rtol=1e-3) ng.testing.assert_allclose(np_W2, ng_W2, rtol=1e-3)
def test_deriv_missing_connection(N): """ Taking the derivative of an expression with respect to a variable not used to compute the expression should raise an exception. """ x = ng.variable([N]) y = ng.variable([N]) z = ng.variable([N]) with pytest.raises(ValueError): ng.deriv(x + y, z)
def test_variable(): input_axes = ng.make_axes([ng.make_axis(10), ng.make_axis(3)]) var = ng.variable(axes=input_axes) assign_val = np.random.rand(10, 3) var_assign = ng.AssignOp(tensor=var, val=assign_val) var_seq = ng.sequential([var_assign, var]) var_comp = ng.computation(var_seq, "all") results = dict() weight_saver = Saver() with closing(ngt.make_transformer()) as transformer: var_func = transformer.add_computation(var_comp) weight_saver.setup_save(transformer=transformer, computation=var_comp) results['saved'] = var_func().copy() weight_saver.save(filename="test_variable") reassign_val = np.random.rand(10, 3) var_reassign = ng.AssignOp(tensor=var, val=reassign_val) var_recomp = ng.computation(var_reassign, "all") var_read = ng.computation(var, "all") with closing(ngt.make_transformer()) as restore_transformer: var_recompfunc = restore_transformer.add_computation(var_recomp) weight_saver.setup_restore(transformer=restore_transformer, computation=var_recomp, filename="test_variable") var_readfunc = restore_transformer.add_computation(var_read) var_recompfunc() results['reassigned'] = var_readfunc().copy() weight_saver.restore() results['restored'] = var_readfunc().copy() os.remove("test_variable.npz") assert np.allclose(results['saved'], assign_val, atol=0) assert np.allclose(results['reassigned'], reassign_val, atol=0) assert np.allclose(results['saved'], results['restored'], atol=0)
def test_weight_clipping(w_clip, optimizer): opt_ng = optimizer(0.1, weight_clip_value=w_clip) # Set up data placeholders C = ng.make_axis(20) N = ng.make_axis(32, name='N') data = ng.placeholder([C, N]) target = ng.placeholder([N]) # params to be updated using optimizer to be tested # make sure initial values are higher than clip values np_W = 10 * w_clip * (2 * np.random.rand(C.length) - 1) W = ng.variable([C], initial_value=np_W) # double check generated initial W value assert np.max(np_W) > w_clip assert np.min(np_W) < -w_clip # Set up op graph cost = ng.sum(target - ng.dot(W, data), out_axis=()) updated_weights = ng.sequential([opt_ng(cost), W]) epsilon = w_clip * 1e-3 # Set up the computation and run the "train" loop with ExecutorFactory() as ex: opt_ng_comp = ex.transformer.computation(updated_weights, data, target) mock_dataset = data_generator(20, C.length, N.length) for x, y in mock_dataset: ng_W = opt_ng_comp(x, y) # updated weights for ngraph optimizer assert np.max(ng_W) < w_clip + epsilon assert np.min(ng_W) > -w_clip - epsilon
def test_variable_init(C): w_init = np.random.rand(C.length) W = ng.variable(ng.make_axes([C]), initial_value=w_init) with ExecutorFactory() as ex: result = ex.executor(W)() ng.testing.assert_allclose(result, w_init)
def test_pad_0(N): """ pad with length 0 should be a nop """ x = ng.variable([N]) assert ng.pad(x, [0]).axes == x.axes
def test_initial_value(): # Test work-around for issue #1138 w = [3, 4, 5] x = ng.constant(w) y = ng.variable([ng.make_axis(length=len(w))], initial_value=x) with ExecutorFactory() as ex: result = ex.executor(y)() ng.testing.assert_allclose(result, np.asarray(w, dtype=np.float32))
def test_pad_invalid_paddings_length(N): """ pad should raise an exception if the paddings length is not the same as the input dimensionality. """ x = ng.variable([N]) with pytest.raises(ValueError): ng.pad(x, [1, 0])
def test_modify_state(): with ExecutorFactory() as ex: N = ng.make_axis(3, name='N') x_np = np.ones((N.length)) * 4 x = ng.variable([N], initial_value=x_np).named('x') val = ng.sequential([ng.assign(x, x + x), x]) f = ex.executor(val) x_val = f() assert np.allclose(x_np + x_np, x_val)
def test_sign(): x_np = np.array([-1.2, 2.3, 0.0, 1.2]) N = ng.make_axis(len(x_np)) x = ng.variable([N]) y = ng.sign(x) y_np = np.sign(x_np) with ExecutorFactory() as ex: y_val = ex.executor(y, x)(x_np) assert np.allclose(y_val, y_np)
def test_sequential(N): x = ng.variable([N], initial_value=0) x0 = x + x x1 = x + x p = ng.sequential([x0, ng.assign(x, 2), x1, x0]) with ExecutorFactory() as ex: x0_val, x1_val, p_val = ex.executor([x0, x1, p])() assert x0_val == 0 assert x1_val == 4 assert p_val == 0
def test_read_state(): """ This just reads back a tensor. No code is generated. """ with ExecutorFactory() as ex: N = ng.make_axis(3, name='N') x_np = np.ones((N.length)) * 4 x = ng.variable([N], initial_value=x_np).named('x') f = ex.executor(x) x_val = f() assert np.allclose(x_np, x_val)
def test_sequential_side(M): x1_np = 2 x2_np = 3 b_np = 1 x_np = np.array([1, 2, 3], dtype=np.float32) x = ng.variable([M], initial_value=x_np) x1 = ng.persistent_tensor(axes=(), initial_value=x1_np) x2 = ng.persistent_tensor(axes=(), initial_value=x2_np) x1_vo = ng.value_of(x1) x2_vo = ng.value_of(x2) b = ng.persistent_tensor(axes=(), initial_value=b_np) y = ng.sequential([ x1_vo, x2_vo, ng.assign(x1, ng.sum(x, out_axes=()) + x1 * b + (1 - b)), ng.assign(x2, ng.mean(x, out_axes=()) + x2 * b + (1 - b)), x * 2 ]) with ExecutorFactory() as ex: main_effect = ex.executor((y, x1_vo, x2_vo, x1, x2)) current_values = ex.executor((x1, x2)) # Run main path #1 y_val, x1_init_val, x2_init_val, x1_final_val, x2_final_val = main_effect( ) y_np = x_np * 2 assert np.allclose(y_val, y_np) assert np.allclose(x1_init_val, x1_np) assert np.allclose(x2_init_val, x2_np) x1_np = np.sum(x_np) + x1_np * b_np + (1 - b_np) x2_np = np.mean(x_np) + x2_np * b_np + (1 - b_np) assert np.allclose(x1_final_val, x1_np) assert np.allclose(x2_final_val, x2_np) x1_val, x2_val = current_values() assert np.allclose(x1_val, x1_np) assert np.allclose(x2_val, x2_np) # Run main path #2 (Should be the same as before) y_val, x1_init_val, x2_init_val, x1_final_val, x2_final_val = main_effect( ) y_np = x_np * 2 assert np.allclose(y_val, y_np) assert np.allclose(x1_init_val, x1_np) assert np.allclose(x2_init_val, x2_np) x1_np = np.sum(x_np) + x1_np * b_np + (1 - b_np) x2_np = np.mean(x_np) + x2_np * b_np + (1 - b_np) assert np.allclose(x1_final_val, x1_np) assert np.allclose(x2_final_val, x2_np)
def test_use_state(): """ Uses the value of a tensor in a computation. """ with ExecutorFactory() as ex: N = ng.make_axis(3, name='N') x_np = np.ones((N.length)) * 4 x = ng.variable([N], initial_value=x_np).named('x') xx = x + x f = ex.executor(xx) xx_val = f() assert np.allclose(x_np + x_np, xx_val)
def test_subgraph_scopes_attribute(input_placeholder): """ Subgraphs should have a scopes attribute that contains scope-subgraph pairs. """ subgraph = SubGraph() with scope_ops("scope1", subgraph=subgraph): w1 = ng.variable(ng.make_axis(), initial_value=1) w1 * input_placeholder assert len(subgraph.scopes) == 1 assert "scope1" in subgraph.scopes assert len(subgraph.scopes["scope1"].scopes) == 1 with scope_ops("scope2", subgraph=subgraph): w2 = ng.variable(ng.make_axis(), initial_value=1) w2 * input_placeholder assert len(subgraph.scopes) == 2 assert "scope2" in subgraph.scopes assert len(subgraph.scopes["scope2"].scopes) == 1
def __call__(self, in_obj): if not self.initialized: w_axis = ng.make_axis() self.weight = ng.variable(axes=[w_axis], initial_value=2, metadata={"label": LABELS["weight"]}, name="W") self.side_effect = ng.persistent_tensor(axes=[w_axis], initial_value=0) return ng.sequential([ng.assign(self.side_effect, self.weight), self.weight * in_obj])
def test_pad_mixed(): """ mix 0 padding with non-0 padding """ input_axes = ng.make_axes([ng.make_axis(1), ng.make_axis(1)]) x = ng.variable(input_axes) pad = ng.pad(x, [0, 1]) assert pad.axes[0].name == x.axes[0].name assert pad.axes[1].name == x.axes[1].name assert pad.axes[0].length == x.axes[0].length assert pad.axes[1].length != x.axes[1].length
def test_execute_non_placeholder(): """ Expect a failure if a non-input (Variable) is used as an argument to executor. """ N = ng.make_axis(length=1) x = ng.temporary([N]) y = ng.variable([N]) with pytest.raises(ValueError): with executor(x + y, x, y) as ex: ex
def test_computational_graph_capture(input_placeholder): """ ComputationalGraph object should capture all ops created after it is instantiated. """ cg = ComputationalGraph() w = ng.variable(ng.make_axis(), initial_value=1) z = w * input_placeholder for op in [w, z]: assert op in cg.ops assert input_placeholder not in cg.ops assert w.name in cg.variables
def test_slice_nop(): """ slicing an axis shouldn't change the name """ input_axes = ng.make_axes([ng.make_axis(1), ng.make_axis(1)]) x = ng.variable(input_axes) s = ng.tensor_slice(x, [ slice(None, None, None), slice(None, None, 1), ]) assert s.axes[0] == x.axes[0] assert s.axes[1] == x.axes[1]
def test_sequential_reduce(M): x = ng.variable([M], initial_value=1) x0 = x + x x1 = ng.sum(x0, out_axes=()) x2 = ng.sum(x0, out_axes=()) + x0 p = ng.sequential([x0, x1, x2]) with ExecutorFactory() as ex: x0_val, x1_val, x2_val, p_val, x_val = ex.executor([x0, x1, x2, p, x])() x0_np = x_val + x_val x1_np = np.sum(x0_np) x2_np = x1_np + x0_np assert np.allclose(x0_val, x0_np) assert np.allclose(x1_val, x1_np) assert np.allclose(x2_val, x2_np) assert np.allclose(p_val, x2_np)
def ngraph_l2_norm(np_array): """ TODO. Arguments: np_array: TODO Returns: TODO """ axes = () for i, l in enumerate(np_array.shape): axes |= (ng.make_axis(length=l).named('axis%s' % i),) np_tensor = ng.constant(np_array, axes) var = ng.variable(axes, initial_value=np_tensor) with executor(ng.sqrt(ng.squared_L2(var))) as ex: return ex()
def test_idempotent_axes_c(): """ Test test axes transformations with autodiff, case c, with broadcast, slice, cast and dim-shuffle """ with ExecutorFactory() as ex: axes = ng.make_axes([ng.make_axis(3), ng.make_axis(1)]) result_axes = [ng.make_axis(length=axis.length) for axis in axes] # variable w = ng.variable(axes, initial_value=np.ones((3, 1))) # broadcast l / r, introducing dummy length 1 axes l = ng.broadcast(w, axes) r = ng.broadcast(w, axes) # slice axes_slice = [slice(None, None, None), slice(None, None, None)] l_sliced = ng.tensor_slice(l, axes_slice) r_sliced = ng.tensor_slice(r, axes_slice) # cast r r_sliced_casted = ng.cast_axes(r_sliced, axes) # perform add result = ng.add(l_sliced, r_sliced_casted) # cast / dimshuffle result = ng.cast_axes(result, result_axes) result = ng.axes_with_order(result, result_axes) # cost and grad cost = ng.sum(result, reduction_axes=result.axes) grad = ng.deriv(cost, w) grad_comp = ex.executor(grad) cost_comp = ex.executor(cost) cost_comp_ng = cost_comp() grad_comp_ng = grad_comp() grad_comp_np = np.ones((3, 1)) * 2. assert cost_comp_ng == 6.0 assert np.array_equal(grad_comp_ng, grad_comp_np)
def test_mode_setting(input_placeholder): """ Modes can be specified when scoping ops. The `modes` attribute should return all ops in the subgraph + any on which they depend. """ w = ng.variable(ng.make_axis(), initial_value=1, name="W") with scope_ops(name="mode_scope", mode="test") as subgraph: w * input_placeholder for op in subgraph: if not isinstance(op, ng.TensorValueOp): assert "mode" in op.metadata assert op.metadata["mode"] == "test" assert "test" in subgraph.modes assert w not in subgraph.ops # Make sure all ops in subgraph are in mode for op in [w] + subgraph.ops: assert op in subgraph.modes["test"].ops
def test_logreg(): # xs: (C, N), y: (N,) xs = np.array([[0.52, 0.88, 0.52, 0.74], [1.12, -1.08, 0.06, -2.49], [0.77, 0.15, -1.3, 1.39]]) ys = np.array([1, 1, 0, 1]) max_iter = 10 alpha = 0.1 thetas = np.array([0., 0., 0.]) np_logreg = NumpyLogreg(xs, ys, thetas) C, N = ng.make_axis(length=3), ng.make_axis(length=4) # input tensors xs_v = ng.placeholder((C, N)) ys_v = ng.placeholder([N]) alpha_v = ng.placeholder(()) thetas_var = ng.variable([C], initial_value=thetas) # define ops ys_pred = ng.sigmoid(ng.dot(thetas_var, xs_v)) log_likelihoods = ng.log(ys_pred) * ys_v + ng.log(1 - ys_pred) * (1 - ys_v) loss = -ng.sum(log_likelihoods, reduction_axes=[N]) grad_comp = ng.deriv(loss, thetas_var) weight_update = ng.sequential( [ng.assign(thetas_var, thetas_var - alpha_v * grad_comp), thetas_var]) # transformer with ExecutorFactory() as ex: train_eval_func = ex.executor([grad_comp, loss, weight_update], xs_v, ys_v, alpha_v) # evaluate for i in range(max_iter): grad_np, loss_np, thetas_np = np_logreg.optimize(alpha) grad_ng, loss_ng, thetas_ng = train_eval_func(xs, ys, alpha) ng.testing.assert_allclose(loss_np, loss_ng, rtol=1e-05, atol=1e-05, \ transformer_overwrite=False) ng.testing.assert_allclose(grad_np, grad_ng, rtol=1e-05, atol=1e-05, \ transformer_overwrite=False) ng.testing.assert_allclose(thetas_np, thetas_ng, rtol=1e-05, atol=1e-05, \ transformer_overwrite=False)
def test_idempotent_axes_b(): """ Test test axes transformations with autodiff, case b, with broadcast applied to the same tensor """ with ExecutorFactory() as ex: axes = ng.make_axes([ng.make_axis(3), ng.make_axis(1)]) w = ng.variable(axes, initial_value=np.ones((3, 1))) l = ng.broadcast(w, axes) r = ng.broadcast(w, axes) result = ng.add(l, r) result = ng.cast_axes(result, axes) cost = ng.sum(result, reduction_axes=axes) grad = ng.deriv(cost, w) grad_comp = ex.executor(grad) cost_comp = ex.executor(cost) assert cost_comp() == 6.0 assert np.array_equal(grad_comp(), np.ones((3, 1)) * 2.)
def test_idempotent_axes_a(): """ Test test axes transformations with autodiff, case a, reference test """ with ExecutorFactory() as ex: axes = ng.make_axes([ng.make_axis(3), ng.make_axis(1)]) w = ng.variable(axes, initial_value=np.ones((3, 1))) result = w + w result = ng.cast_axes(result, axes) cost = ng.sum(result, reduction_axes=axes) grad = ng.deriv(cost, w) grad_comp = ex.executor(grad) cost_comp = ex.executor(cost) cost_comp_val = cost_comp() grad_comp_val = grad_comp() grad_comp_np = np.ones((3, 1)) * 2. assert cost_comp_val == 6.0 assert np.array_equal(grad_comp_val, grad_comp_np)
def test_scope_ops(input_placeholder): """ Test scope_ops creates a subgraph with correct attributes """ with scope_ops(name="foo") as subgraph: w = ng.variable(ng.make_axis(), initial_value=1, name="W") y = w * input_placeholder z = y + 4 v1 = ng.persistent_tensor(w.axes, initial_value=0, name="effect1") v2 = ng.persistent_tensor(w.axes, initial_value=0, name="effect2") ng.sequential([ng.assign(v1, w), ng.assign(v2, w), z.named("output")]) assert len(subgraph.inputs) == 1 assert input_placeholder.unscoped_name in subgraph.inputs assert len(subgraph.variables) == 1 assert "W" in subgraph.variables assert len(subgraph.outputs) == 1 assert "output" in subgraph.outputs assert len(subgraph.side_effects) == 2
def test_one(): # Test that the cacheing on constant one used in DerivOp works. op = ng.variable([]) one_0 = op.one one_1 = op.one assert one_0 is one_1