def __call__(self, in_obj): in_axes = in_obj.axes if in_axes.channel_axis() is None: red_axes = ng.make_axes(in_axes.recurrent_axis()) + in_axes.batch_axes() else: red_axes = in_axes - in_axes.channel_axis() out_axes = in_axes - red_axes in_obj = ng.flatten(in_obj, out_axes | red_axes.flatten(force=True)) if self.gamma is None: self.gvar = self.gvar or ng.persistent_tensor(axes=out_axes, initial_value=1.0) self.gmean = self.gmean or ng.persistent_tensor(axes=out_axes, initial_value=0.0) self.gamma = ng.variable(axes=out_axes, initial_value=self.init_gamma, scope=self.scope).named('gamma') self.beta = ng.variable(axes=out_axes, initial_value=self.init_beta, scope=self.scope).named('beta') xmean = ng.mean(in_obj, out_axes=out_axes) xvar = ng.variance(in_obj, out_axes=out_axes) if Layer.inference_mode: return ng.unflatten(self.gamma * ((in_obj - self.gmean) * ng.reciprocal(ng.sqrt(self.gvar + self.eps))) + self.beta) else: return ng.sequential([ ng.assign(self.gmean, self.gmean * self.rho + xmean * (1.0 - self.rho)), ng.assign(self.gvar, self.gvar * self.rho + xvar * (1.0 - self.rho)), ng.unflatten(self.gamma * ((in_obj - xmean) * ng.reciprocal(ng.sqrt(xvar + self.eps))) + self.beta) ])
def test_allreduce_hint(hetr_device, config): if hetr_device == 'gpu': if 'gpu' not in ngt.transformer_choices(): pytest.skip("GPUTransformer not available") input = config['input'] device_id = config['device_id'] axis_A = ng.make_axis(length=4, name='axis_A') parallel_axis = ng.make_axis(name='axis_parallel', length=16) with ng.metadata(device=hetr_device, device_id=device_id, parallel=parallel_axis): var_A = ng.variable(axes=[axis_A], initial_value=UniformInit(1, 1)) var_B = ng.variable(axes=[axis_A], initial_value=UniformInit(input, input)) var_B.metadata['reduce_func'] = 'sum' var_B_mean = var_B / len(device_id) var_minus = (var_A - var_B_mean) with closing(ngt.make_transformer_factory('hetr', device=hetr_device)()) as hetr: out_comp = hetr.computation(var_minus) result = out_comp() np_result = np.full((axis_A.length), config['expected_result'], np.float32) np.testing.assert_array_equal(result, np_result)
def compare_optimizer_variable_select(opt_ng, opt_ref): # Set up data placeholders C = ng.make_axis(20) N = ng.make_axis(32, name='N') data = ng.placeholder([C, N]) target = ng.placeholder([N]) # params to be updated using optimizer to be tested np_W1 = np.random.rand(C.length) np_W2 = np.random.rand(C.length) W1 = ng.variable([C], initial_value=np_W1) W2 = ng.variable([C], initial_value=np_W2) # Set up op graph cost = ng.sum(target - ng.dot(W1, data) - ng.dot(W2, data), out_axis=()) updated_weights = ng.sequential([opt_ng(cost, variables=[W1]), W1]) # Set up the computation and run the "train" loop with ExecutorFactory() as ex: opt_ng_comp = ex.transformer.computation([updated_weights, W2], data, target) mock_dataset = data_generator(20, C.length, N.length) for x, y in mock_dataset: [ng_W1, ng_W2] = opt_ng_comp(x, y) # updated weights for ngraph optimizer np_W1 = opt_ref(x, np_W1) # updated weights for reference optimizer ng.testing.assert_allclose(np_W1, ng_W1, rtol=1e-3) ng.testing.assert_allclose(np_W2, ng_W2, rtol=1e-3)
def train_outputs(self, in_obj): in_axes = in_obj.axes.sample_axes() red_axes = ng.make_axes() if len(in_axes.role_axes(ar.features_input)) != 0: red_axes += in_axes.sample_axes() - in_axes.role_axes( ar.features_input) red_axes += in_obj.axes.batch_axes() out_axes = in_axes - red_axes self.gamma = self.gamma or ng.variable( axes=out_axes, initial_value=1.0).named('gamma') self.beta = self.beta or ng.variable(axes=out_axes, initial_value=0.0).named('beta') self.gvar = self.gvar or ng.persistent_tensor(axes=out_axes, initial_value=1.0) self.gmean = self.gmean or ng.persistent_tensor(axes=out_axes, initial_value=0.0) xmean = ng.mean(in_obj, reduction_axes=red_axes) xvar = ng.variance(in_obj, reduction_axes=red_axes) return ng.sequential([ ng.assign(self.gmean, self.gmean * self.rho + xmean * (1.0 - self.rho)), ng.assign(self.gvar, self.gvar * self.rho + xvar * (1.0 - self.rho)), self.gamma * (in_obj - xmean) / ng.sqrt(xvar + self.eps) + self.beta ])
def test_allreduce_hint_gpu(config): pytest.xfail("Multi-GPU testing not enabled yet") if 'gpu' not in ngt.transformer_choices(): pytest.skip("GPUTransformer not available") c = config os.environ["HETR_SERVER_GPU_NUM"] = str(len(c['device_id'])) ax_A_length = 32 ax_B_length = 16 np_result = [np.full((ax_A_length, ax_B_length), c['expected_result'], np.float32)] parallel_axis = ng.make_axis(name='axis_parallel', length=16) with ng.metadata(device_id=c['device_id'], parallel=parallel_axis): axis_A = ng.make_axis(length=ax_A_length, name='axis_A') axis_B = ng.make_axis(length=ax_B_length, name='axis_B') var_A = ng.variable(axes=[axis_A], initial_value=UniformInit(1, 1)).named('var_A') var_B = ng.variable(initial_value=UniformInit(c['input'], c['input']), axes=[axis_B]).named('var_B') var_B.metadata['reduce_func'] = c['func'] var_minus = (var_A - var_B).named('var_minus') with closing(ngt.make_transformer_factory('hetr', device='gpu')()) as hetr: out_comp = hetr.computation([var_minus]).named('out_comp') result = out_comp() np.testing.assert_array_equal(result, np_result)
def train_outputs(self, in_obj, init_state=None): """ Sets shape based parameters of this layer given an input tuple or int or input layer. Arguments: in_obj (int, tuple, Layer or Tensor): object that provides shape information for layer init_state (Tensor): object that provides initial state Returns: rnn_out (Tensor): output """ # try to understand the axes from the input self.interpret_axes(in_obj, init_state) # initialize the hidden states if init_state is not None: self.h_init = init_state else: if self.reset_cells: self.h_init = ng.constant( const=0, axes=self.hidden_state_axes).named('h_init') else: self.h_init = ng.variable( initial_value=0, axes=self.hidden_state_axes).named('h_init') self.W_input = ng.variable(axes=self.w_in_axes, initial_value=self.init).named("W_in") self.W_recur = ng.variable(axes=self.w_re_axes, initial_value=self.init_inner).named("W_re") self.b = ng.variable(axes=self.hidden_axes, initial_value=0).named("bias") h = self.h_init h_list = [] # slice the inputs into time slices in_s = get_steps(in_obj, self.recurrent_axis, self.backward) # unrolling computations for i in range(self.recurrent_axis.length): with ng.metadata(recurrent_step=str(i)): h = self._step(in_s[i], h) h_list.append(h) if self.return_sequence is True: # only when returning a sequence, need to reverse the output h_list = h_list[::-1] if self.backward else h_list rnn_out = ng.stack(h_list, self.recurrent_axis, pos=self.recurrent_axis_idx) else: rnn_out = h_list[-1] return rnn_out
def train_outputs(self, in_obj): """ Sets shape based parameters of this layer given an input tuple or int or input layer. Arguments: in_obj (int, tuple, Layer or Tensor): object that provides shape information for layer Returns: (Tensor): output """ in_axes = in_obj.axes self.time_axis = in_axes.recurrent_axes()[0] def get_steps(x, time_axis): return [ ng.slice_along_axis(x, time_axis, i) for i in range(time_axis.length) ] if self.axes is not None: hidden_axes = self.axes - self.axes.recurrent_axes() else: hidden_axes = ng.make_axes( [ng.make_axis(self.nout).named('Hidden_in')]) w_in_axes = hidden_axes + [ axis - 1 for axis in in_axes.sample_axes() - in_axes.recurrent_axes() ] w_re_axes = hidden_axes + [axis - 1 for axis in hidden_axes] self.W_input = ng.variable(axes=w_in_axes, initial_value=self.init( w_in_axes.lengths)).named("W_in") self.W_recur = ng.variable(axes=w_re_axes, initial_value=self.init_inner( w_re_axes.lengths)).named("W_re") self.b = ng.variable(axes=hidden_axes, initial_value=0).named("bias") h_ff_buf = ng.dot(self.W_input, in_obj).named("W_in_dot_in") h_ff_s = get_steps(h_ff_buf, self.time_axis) self.h_init = ng.constant(np.zeros(h_ff_s[0].axes.lengths), axes=h_ff_s[0].axes).named('h_init') hprev = [self.h_init] for i in range(self.time_axis.length): with ng.metadata(recurrent_step=str(i)): d = ng.dot(self.W_recur, hprev[i]).named("W_rec_dot_h{}".format(i)) h = self.activation(d + h_ff_s[i] + self.b) h.name = "activ{}".format(i) hprev.append(h) rnn_out = ng.stack(hprev[1:], self.time_axis, pos=1) return rnn_out
def get_ng_variable(self): # type: () -> TensorOp """Create an ngraph variable node for this value.""" axes = self.get_ng_axes() dtype = self.get_dtype() if self.has_initializer: initializer = self.get_initializer() return ng.variable(axes=axes, dtype=dtype, initial_value=initializer.to_array()).named(self.name) return ng.variable(axes=axes, dtype=dtype).named(self.name)
def test_conv_flatten_deriv(n4_hw12_c3_5x5): """ Test deriv of conv followed by flatten """ cf = ConvParams(**n4_hw12_c3_5x5) axes_rsck = ng.make_axes([cf.ax_f[2], cf.ax_f[3], cf.ax_f[0], cf.ax_f[-1]]) axes_rsck_prime = ng.make_axes([ng.make_axis(name=ax.name + 'p', length=ax.length) for ax in axes_rsck]) axes_nmpqk = ng.make_axes([cf.ax_o[-1], cf.ax_o[1], cf.ax_o[2], cf.ax_o[3], cf.ax_o[0]]) # broadcast input / filter axes input_var = ng.variable(cf.ax_i).named('input') input_val = np.ones(input_var.axes.lengths) filter_rsck_prime = ng.variable(axes_rsck_prime).named('filter') filter_var = filter_rsck_prime filter_rsck = ng.cast_axes(filter_rsck_prime, axes_rsck).named('frsck') filter_trsck = ng.expand_dims(filter_rsck, cf.ax_f[1], 0).named('ftrsck') filter_ctrsk = ng.axes_with_order(filter_trsck, axes=cf.ax_f).named('ctrsk') # convolution output_kmpqn = ng.convolution(cf.conv_params, input_var, filter_ctrsk, axes=cf.ax_o) output_nmpqk = ng.axes_with_order(output_kmpqn, axes=axes_nmpqk) # slice away the oD out_slicing = [slice(None), 0, slice(None), slice(None), slice(None)] output_npqk = ng.tensor_slice(output_nmpqk, out_slicing) output = ng.flatten_at(output_npqk, idx=1) # cost and grad cost = ng.sum(output, out_axes=()) filter_val = np.ones(filter_var.axes.lengths) with ExecutorFactory() as factory: conv_comp = factory.executor(output, filter_var, input_var) grad_filter_num_comp = factory.numeric_derivative(cost, filter_var, 1.0, input_var) grad_filter_sym_comp = factory.derivative(cost, filter_var, input_var) grad_input_num_comp = factory.numeric_derivative(cost, input_var, 1.0, filter_var) grad_input_sym_comp = factory.derivative(cost, input_var, filter_var) conv_val = conv_comp(filter_val, input_val) conv_val_num = np.empty_like(conv_val) conv_val_num.fill(np.prod(cf.ax_f.lengths[:-1])) ng.testing.assert_allclose(conv_val, conv_val_num) grad_filter_num_val = grad_filter_num_comp(filter_val, input_val) grad_filter_sym_val = grad_filter_sym_comp(filter_val, input_val) ng.testing.assert_allclose(grad_filter_num_val, grad_filter_sym_val) grad_input_num_val = grad_input_num_comp(input_val, filter_val) grad_input_sym_val = grad_input_sym_comp(input_val, filter_val) ng.testing.assert_allclose(grad_input_num_val, grad_input_sym_val)
def test_deriv_missing_connection(N): """ Taking the derivative of an expression with respect to a variable not used to compute the expression should raise an exception. """ x = ng.variable([N]) y = ng.variable([N]) z = ng.variable([N]) with pytest.raises(ValueError): ng.deriv(x + y, z)
def test_execute_non_placeholder(): """ Expect a failure if a non-input (Variable) is used as an argument to executor. """ N = ng.make_axis(1) x = ng.variable([N]) y = ng.variable([N]) with pytest.raises(ValueError): executor(x + y, x, y)
def train_outputs(self, in_obj): cpm = self.convparams.copy() in_obj = ng.axes_with_role_order(in_obj, self.role_order) in_axes = in_obj.axes if self.f_axes is None: self.f_axes = ng.make_axes([in_axes[0]]) for nm, role in zip('TRSK', self.filter_roles[1:]): self.f_axes += ng.make_axis(roles=[role], length=cpm[nm]).named(nm) self.W = ng.variable(axes=self.f_axes, initial_value=self.init) if self.o_axes is None: self.o_axes = ng.make_axes([ ng.make_axis(roles=a.roles).named(a.short_name) for a in in_axes if not a.is_batch ]) # set lengths out_shape = [ self.f_axes[-1].length, output_dim(in_axes[1].length, cpm['T'], cpm['pad_d'], cpm['str_d'], False, cpm['dil_d']), output_dim(in_axes[2].length, cpm['R'], cpm['pad_h'], cpm['str_h'], False, cpm['dil_h']), output_dim(in_axes[3].length, cpm['S'], cpm['pad_w'], cpm['str_w'], False, cpm['dil_w']) ] self.o_axes.set_shape(out_shape) self.o_axes += in_axes.batch_axes() return ng.convolution(cpm, in_obj, self.W, axes=self.o_axes)
def test_gemm(transformer_factory): """ TODO: make this more interesting """ n, c = 32, 32 N = ng.make_axis(length=n, name='N') C = ng.make_axis(length=c) X = ng.placeholder(axes=[C, N]) Y = ng.placeholder(axes=[N]) W = ng.variable(axes=[C - 1], initial_value=0.1) Y_hat = ng.dot(W, X) with executor(Y_hat, X) as ex: mm_executor = ex w = np.ones(c) * 0.1 xs = np.ones(n * c).reshape(c, n) for ii in range(3): y_hat_val = mm_executor(xs) # 8.8 fixed point test # assert np.allclose(np.dot(xs, w) - y_hat_val, 0.075*np.ones(n)) # autoflex test assert_allclose(np.dot(xs, w), y_hat_val)
def test_setting(M): with ExecutorFactory() as ex: axes = ng.make_axes([M]) np_x = np.array([1, 2, 3], dtype=np.float32) np_y = np.array([1, 3, 5], dtype=np.float32) x = ng.constant(np_x, axes) y = ng.constant(np_y, axes) v = ng.variable(axes, initial_value=x) f_v = ex.executor(v) vset = ng.sequential([ng.assign(v, v + y), v]) f_v1 = ex.executor(vset) f_v2 = ex.executor(v) e_v = f_v().copy() assert ng.testing.allclose(e_v, np_x) e_v1 = f_v1().copy() assert ng.testing.allclose(e_v1, np_x + np_y) e_v2 = f_v2().copy() assert ng.testing.allclose(e_v2, np_x + np_y)
def test_one_dot_bprop_allreduce(config): c = config pytest.xfail( "GPU child transformers generate errors during AssignLayouts graph pass #1651" ) H_axis = ng.make_axis(length=4, name='height') W_axis = ng.make_axis(length=6, name='width') with ng.metadata(step='input'): X = ng.placeholder(axes=[H_axis, W_axis]) target = ng.constant(1, axes=[W_axis]) with ng.metadata(device_id=c['device_id'], parallel=W_axis): W = ng.variable(axes=[H_axis], initial_value=UniformInit(1, 1)) dot = ng.dot(W, X) L = ng.squared_L2(target - dot, out_axes=()) grad = ng.deriv(L, W) grad.metadata['reduce_func'] = c['func'] update = (W - grad) with closing(ngt.make_transformer_factory('hetr')()) as hetr: out_comp = hetr.computation([update], X) result = out_comp(c['input']) np.testing.assert_array_equal(result, c['expected_result'])
def test_set_op_values(transformer_factory): NUM_OPS = 3 # set up an op and Assign a value to it so we can read it out axes = ng.make_axes([ ng.make_axis(name='A', length=2), ng.make_axis(name='B', length=3), ]) variable_ops = [ng.variable(axes) for i in range(NUM_OPS)] values = [np.ones(axes.lengths) * i for i in range(NUM_OPS)] with closing(ngt.make_transformer()) as t: serde_weights.set_op_values( t, variable_ops, { op.uuid.bytes: value for op, value in zip(variable_ops, values) } ) # extract values out of it and make sure they match expected results funcs = [t.computation(op) for op in variable_ops] op_values = [func() for func in funcs] np.testing.assert_allclose(values, op_values)
def build_graphs(L, BS): """ TODO. Arguments: L: TODO BS: TODO Returns: TODO """ # Axes L = [ng.make_axis(length=N, name='L%d' % i) for i, N in enumerate(L)] BS = ng.make_axis(length=BS, name='BS') # Builds Network activations = [ng.tanh for i in range(len(L) - 2)] + [ng.softmax] X = ng.placeholder((L[0], BS)).named('X') Y = ng.placeholder((L[-1], )).named('Y') W = [ ng.variable((L_np1, L_n - 1)).named('W%d' % i) for i, (L_np1, L_n) in enumerate(zip(L[1:], L[:-1])) ] A = [] for i, f in enumerate(activations): Aim1 = A[i - 1] if i > 0 else X A.append(f(ng.dot(W[i], Aim1))) Error = ng.cross_entropy_multi(A[-1], Y) dW = [ng.deriv(Error, w) for w in W] transformer = ngt.make_transformer() dfg = an.DataFlowGraph(transformer, dW) ifg = an.InterferenceGraph(dfg.liveness()) return dfg, ifg
def test_setting(): ex = ExecutorFactory() X = ng.make_axis(name='X', length=3) axes = ng.make_axes([X]) np_x = np.array([1, 2, 3], dtype=np.float32) np_y = np.array([1, 3, 5], dtype=np.float32) x = ng.constant(np_x, axes) y = ng.constant(np_y, axes) v = ng.variable(axes, initial_value=x) f_v = ex.executor(v) with ng.Op.saved_user_deps(): ng.assign(v, v + y) f_v1 = ex.executor(v) f_v2 = ex.executor(v) e_v = f_v().copy() assert np.allclose(e_v, np_x) e_v1 = f_v1().copy() assert np.allclose(e_v1, np_x + np_y) e_v2 = f_v2().copy() assert np.allclose(e_v2, np_x + np_y)
def train_outputs(self, in_obj): """ Arguments: in_obj (Tensor): object that provides the lookup indices """ in_obj.axes.find_by_short_name('time')[0].add_role(ar.time) in_obj.axes.find_by_short_name('time')[0].is_recurrent = True in_obj = ng.axes_with_role_order(in_obj, self.role_order) in_obj = ng.flatten(in_obj) in_axes = in_obj.axes self.lut_v_axis = ng.make_axis(self.vocab_size).named('V') self.lut_f_axis = ng.make_axis(self.embed_dim).named('F') self.w_axes = ng.make_axes([self.lut_v_axis, self.lut_f_axis]) self.lut_o_axes = in_axes + ng.make_axes([self.lut_f_axis]) self.o_axes = ng.make_axes([self.lut_f_axis]) + in_axes[0].axes self.W = ng.variable(axes=self.w_axes, initial_value=self.lut_init( self.w_axes, self.lut_v_axis, self.pad_idx)).named('W') lut_result = ng.lookuptable(self.W, in_obj, self.lut_o_axes, update=self.update, pad_idx=self.pad_idx) return ng.axes_with_order(ng.unflatten(lut_result), self.o_axes)
def test_variable_init(transformer_factory, C): w_init = np.random.rand(C.length) W = ng.variable(ng.make_axes([C]), initial_value=w_init) with ExecutorFactory() as ex: result = ex.executor(W)() ng.testing.assert_allclose(result, w_init)
def test_extract_many_ops(): """ Create NUM_OPS, fill them with 0, 1, 2, ... then check that serde_weights.extract_ops is able to extract the correct uuid/value pairs. """ NUM_OPS = 3 # set up an op and Assign a value to it so we can read it out axes = ng.make_axes([ ng.make_axis(name='A', length=2), ng.make_axis(name='B', length=3), ]) variable_ops = [ng.variable(axes) for _ in range(NUM_OPS)] assign_sequential = assign_ops(variable_ops, range(NUM_OPS)) with executor(assign_sequential) as assign_computation: t = assign_computation.transformer # Set values manually assign_computation() # extract values out of it and make sure they match expected results weights = serde_weights.extract_ops(t, variable_ops) for i, variable_op in enumerate(variable_ops): np.testing.assert_allclose(weights[variable_op.uuid.bytes], i)
def test_allreduce_hint_cpu(config): c = config parallel_axis = ng.make_axis(name='axis_parallel', length=16) with ng.metadata(device_id=c['device_id'], parallel=parallel_axis): axis_A = ng.make_axis(length=4, name='axis_A') axis_B = ng.make_axis(length=2, name='axis_B') var_A = ng.variable(axes=[axis_A], initial_value=UniformInit(1, 1)).named('var_A') var_B = ng.variable(initial_value=UniformInit(c['input'], c['input']), axes=[axis_B]).named('var_B') var_B.metadata['reduce_func'] = c['func'] var_minus = (var_A - var_B).named('var_minus') with closing(ngt.make_transformer_factory('hetr')()) as hetr: out_comp = hetr.computation([var_minus]).named('out_comp') result = out_comp() np.testing.assert_array_equal(result, c['expected_result'])
def test_pad_0(N): """ pad with length 0 should be a nop """ x = ng.variable([N]) assert ng.pad(x, [0]).axes == x.axes
def test_round_trip(): # set up an op and Assign a value to it so we can read it out axes = ng.make_axes([ ng.make_axis(name='A', length=2), ng.make_axis(name='B', length=3), ]) x_op = ng.variable(axes) assign_op = ng.AssignOp(x_op, 1) with executor(assign_op) as assign_computation: t = assign_computation.transformer # Set initial value assign_computation() # Test value np.testing.assert_allclose(serde_weights.extract_op(t, x_op), 1) # write out values in x and graph f = BytesIO() # ## EXAMPLE OF HOW TO FULLY SERIALIZE A GRAPH ### serde_weights.serialize_weights(t, [x_op], f) graph_string = serde.serialize_graph([x_op]) # ## /EXAMPLE OF HOW TO FULLY SERIALIZE A GRAPH ### f.seek(0) # ## EXAMPLE OF HOW TO FULLY DESERIALIZE A GRAPH ### new_ops = serde.deserialize_graph(graph_string) serde_weights.deserialize_weights(t, new_ops, f) # ## /EXAMPLE OF HOW TO FULLY DESERIALIZE A GRAPH ### np.testing.assert_allclose(serde_weights.extract_op(t, new_ops[0]), 1)
def import_operation(self, cntk_op): """ Recursively import and translate CNTK operations. Arguments: cntk_op: CNTK operation to be imported. Returns: Translated operation. """ if self.debug: for _ in range(len(inspect.stack())): print(' ', end="") print("Importing: " + cntk_op.uid + "(", end="") for i in cntk_op.inputs: print(i.uid + str(i.shape) + ",", end="") print(")") inputs = [] for i in cntk_op.inputs: axes = [ng.make_axis(dim) for dim in i.shape] dtype = np.dtype(i.dtype) if i.is_output: uid = i.owner.root_function.uid temp = self.uid_op_map[uid] if isinstance(temp, C.Function): temp = self.import_operation(temp) if temp is None: raise ValueError("Error translating: " + uid) else: if self.debug: for _ in range(len(inspect.stack()) + 1): print(' ', end="") print("Finished importing: " + uid + str(cntk_op.shape) + " -> " + temp.name + str(temp.shape.full_lengths)) self.uid_op_map[uid] = temp inputs.append(temp) elif i.is_input: if self.batch_size > 1: axes.append(ng.make_axis(self.batch_size, 'N')) temp = ng.placeholder(axes, dtype).named(i.uid) inputs.append(temp) self.placeholders.append(temp) else: try: input_value = i.value except AttributeError: input_value = C.plus(i, np.zeros(i.shape)).eval() if i.is_constant: inputs.append( ng.constant(input_value, axes, dtype).named(i.uid)) elif i.is_parameter: inputs.append( ng.variable(axes, dtype, input_value).named(i.uid)) else: raise ValueError("Unknown input: " + i.uid) return self.ops_bridge(cntk_op, inputs)
def test_pad_invalid_paddings_length(N): """ pad should raise an exception if the paddings length is not the same as the input dimensionality. """ x = ng.variable([N]) with pytest.raises(ValueError): ng.pad(x, [1, 0])
def test_initial_value(transformer_factory): # Test work-around for issue #1138 w = [3, 4, 5] x = ng.constant(w) y = ng.variable([ng.make_axis(length=len(w))], initial_value=x) with ExecutorFactory() as ex: result = ex.executor(y)() ng.testing.assert_allclose(result, np.asarray(w, dtype=np.float32))
def train_outputs(self, in_obj): out_axes = ng.make_axes(self.axes or [ng.make_axis(self.nout).named('Hidden')]) in_axes = in_obj.axes.sample_axes() in_axes = in_axes - in_axes.recurrent_axes() w_axes = out_axes - out_axes.recurrent_axes() + [axis - 1 for axis in in_axes] if self.W is None: self.W = ng.variable(axes=w_axes, initial_value=self.init(w_axes.lengths)) return ng.dot(self.W, in_obj)
def test_sign(): x_np = np.array([-1.2, 2.3, 0.0, 1.2]) N = ng.make_axis(len(x_np)) x = ng.variable([N]) y = ng.sign(x) y_np = np.sign(x_np) with ExecutorFactory() as ex: y_val = ex.executor(y, x)(x_np) assert np.allclose(y_val, y_np)
def test_fill_state(): with ExecutorFactory() as ex: N = ng.make_axis(3, name='N') x_np = np.ones((N.length)) * 4 x = ng.variable([N], initial_value=x_np).named('x') val = ng.sequential([ng.fill(x, -1), x]) f = ex.executor(val) x_val = f() assert np.allclose(-1, x_val)
def __call__(self, in_obj, **kwargs): """ Arguments: in_obj (Tensor): object that provides the lookup indices """ in_obj = ng.flatten(in_obj) in_axes = in_obj.axes # label lut_v_axis as shadow axis for initializers ... once #1158 is # in, shadow axis will do more than just determine fan in/out for # initializers. self.lut_v_axis = ng.make_axis(self.vocab_size).named('V') self.axes_map = shadow_axes_map([self.lut_v_axis]) self.lut_v_axis = list(self.axes_map.values())[0] self.lut_f_axis = ng.make_axis(self.embed_dim).named('F') self.w_axes = ng.make_axes([self.lut_v_axis, self.lut_f_axis]) self.lut_o_axes = in_axes | ng.make_axes([self.lut_f_axis]) self.o_axes = ng.make_axes([self.lut_f_axis]) | in_axes[0].axes if not self.initialized: self.W = ng.variable( axes=self.w_axes, initial_value=self.lut_init( self.w_axes, self.lut_v_axis, self.pad_idx), metadata={ "label": LABELS["weight"]}, ).named('LutW') lut_result = ng.lookuptable( self.W, in_obj, self.lut_o_axes, update=self.update, pad_idx=self.pad_idx) return ng.map_roles(ng.unflatten(lut_result), self.axes_map)
def __init__( self, cands, num_cands, max_cand_len, memory_size, max_utt_len, vocab_size, emb_size, batch_size, use_match_type=False, kb_ents_to_type=None, kb_ents_to_cand_idxs=None, match_type_idxs=None, nhops=3, eps=1e-6, init=GaussianInit( mean=0.0, std=0.1)): super(MemN2N_Dialog, self).__init__() self.cands = cands self.memory_size = memory_size self.max_utt_len = max_utt_len self.vocab_size = vocab_size self.num_cands = num_cands self.max_cand_len = max_cand_len self.batch_size = batch_size self.use_match_type = use_match_type self.kb_ents_to_type = kb_ents_to_type self.kb_ents_to_cand_idxs = kb_ents_to_cand_idxs self.match_type_idxs = match_type_idxs self.nhops = nhops self.eps = eps self.init = init # Make axes self.batch_axis = ng.make_axis(length=batch_size, name='N') self.sentence_rec_axis = ng.make_axis(length=max_utt_len, name='REC') self.memory_axis = ng.make_axis(length=memory_size, name='memory_axis') self.embedding_axis = ng.make_axis(length=emb_size, name='F') self.embedding_axis_proj = ng.make_axis(length=emb_size, name='F_proj') self.cand_axis = ng.make_axis(length=num_cands, name='cand_axis') self.cand_rec_axis = ng.make_axis(length=max_cand_len, name='REC') # Weight sharing of A's accross all hops input and output self.LUT_A = ModifiedLookupTable( vocab_size, emb_size, init, update=True, pad_idx=0) # Use lookuptable W to embed the candidate answers self.LUT_W = ModifiedLookupTable( vocab_size, emb_size, init, update=True, pad_idx=0) # Initialize projection matrix between internal model states self.R_proj = ng.variable( axes=[ self.embedding_axis, self.embedding_axis_proj], initial_value=init) if not self.use_match_type: # Initialize constant matrix of all candidate answers self.cands_mat = ng.constant( self.cands, axes=[ self.cand_axis, self.cand_rec_axis])