def __call__(self, in_obj):
in_axes = in_obj.axes
if in_axes.channel_axis() is None:
red_axes = ng.make_axes(in_axes.recurrent_axis()) + in_axes.batch_axes()
else:
red_axes = in_axes - in_axes.channel_axis()
out_axes = in_axes - red_axes
in_obj = ng.flatten(in_obj, out_axes | red_axes.flatten(force=True))
if self.gamma is None:
self.gvar = self.gvar or ng.persistent_tensor(axes=out_axes, initial_value=1.0)
self.gmean = self.gmean or ng.persistent_tensor(axes=out_axes, initial_value=0.0)
self.gamma = ng.variable(axes=out_axes,
initial_value=self.init_gamma,
scope=self.scope).named('gamma')
self.beta = ng.variable(axes=out_axes,
initial_value=self.init_beta,
scope=self.scope).named('beta')
xmean = ng.mean(in_obj, out_axes=out_axes)
xvar = ng.variance(in_obj, out_axes=out_axes)
if Layer.inference_mode:
return ng.unflatten(self.gamma * ((in_obj - self.gmean) *
ng.reciprocal(ng.sqrt(self.gvar + self.eps))) + self.beta)
else:
return ng.sequential([
ng.assign(self.gmean, self.gmean * self.rho + xmean * (1.0 - self.rho)),
ng.assign(self.gvar, self.gvar * self.rho + xvar * (1.0 - self.rho)),
ng.unflatten(self.gamma * ((in_obj - xmean) *
ng.reciprocal(ng.sqrt(xvar + self.eps))) + self.beta)
])
def test_allreduce_hint(hetr_device, config):
if hetr_device == 'gpu':
if 'gpu' not in ngt.transformer_choices():
pytest.skip("GPUTransformer not available")
input = config['input']
device_id = config['device_id']
axis_A = ng.make_axis(length=4, name='axis_A')
parallel_axis = ng.make_axis(name='axis_parallel', length=16)
with ng.metadata(device=hetr_device,
device_id=device_id,
parallel=parallel_axis):
var_A = ng.variable(axes=[axis_A], initial_value=UniformInit(1, 1))
var_B = ng.variable(axes=[axis_A],
initial_value=UniformInit(input, input))
var_B.metadata['reduce_func'] = 'sum'
var_B_mean = var_B / len(device_id)
var_minus = (var_A - var_B_mean)
with closing(ngt.make_transformer_factory('hetr',
device=hetr_device)()) as hetr:
out_comp = hetr.computation(var_minus)
result = out_comp()
np_result = np.full((axis_A.length), config['expected_result'],
np.float32)
np.testing.assert_array_equal(result, np_result)
def compare_optimizer_variable_select(opt_ng, opt_ref):
# Set up data placeholders
C = ng.make_axis(20)
N = ng.make_axis(32, name='N')
data = ng.placeholder([C, N])
target = ng.placeholder([N])
# params to be updated using optimizer to be tested
np_W1 = np.random.rand(C.length)
np_W2 = np.random.rand(C.length)
W1 = ng.variable([C], initial_value=np_W1)
W2 = ng.variable([C], initial_value=np_W2)
# Set up op graph
cost = ng.sum(target - ng.dot(W1, data) - ng.dot(W2, data), out_axis=())
updated_weights = ng.sequential([opt_ng(cost, variables=[W1]), W1])
# Set up the computation and run the "train" loop
with ExecutorFactory() as ex:
opt_ng_comp = ex.transformer.computation([updated_weights, W2], data,
target)
mock_dataset = data_generator(20, C.length, N.length)
for x, y in mock_dataset:
[ng_W1,
ng_W2] = opt_ng_comp(x, y) # updated weights for ngraph optimizer
np_W1 = opt_ref(x,
np_W1) # updated weights for reference optimizer
ng.testing.assert_allclose(np_W1, ng_W1, rtol=1e-3)
ng.testing.assert_allclose(np_W2, ng_W2, rtol=1e-3)
def train_outputs(self, in_obj):
in_axes = in_obj.axes.sample_axes()
red_axes = ng.make_axes()
if len(in_axes.role_axes(ar.features_input)) != 0:
red_axes += in_axes.sample_axes() - in_axes.role_axes(
ar.features_input)
red_axes += in_obj.axes.batch_axes()
out_axes = in_axes - red_axes
self.gamma = self.gamma or ng.variable(
axes=out_axes, initial_value=1.0).named('gamma')
self.beta = self.beta or ng.variable(axes=out_axes,
initial_value=0.0).named('beta')
self.gvar = self.gvar or ng.persistent_tensor(axes=out_axes,
initial_value=1.0)
self.gmean = self.gmean or ng.persistent_tensor(axes=out_axes,
initial_value=0.0)
xmean = ng.mean(in_obj, reduction_axes=red_axes)
xvar = ng.variance(in_obj, reduction_axes=red_axes)
return ng.sequential([
ng.assign(self.gmean,
self.gmean * self.rho + xmean * (1.0 - self.rho)),
ng.assign(self.gvar,
self.gvar * self.rho + xvar * (1.0 - self.rho)),
self.gamma * (in_obj - xmean) / ng.sqrt(xvar + self.eps) +
self.beta
])
def test_allreduce_hint_gpu(config):
pytest.xfail("Multi-GPU testing not enabled yet")
if 'gpu' not in ngt.transformer_choices():
pytest.skip("GPUTransformer not available")
c = config
os.environ["HETR_SERVER_GPU_NUM"] = str(len(c['device_id']))
ax_A_length = 32
ax_B_length = 16
np_result = [np.full((ax_A_length, ax_B_length), c['expected_result'], np.float32)]
parallel_axis = ng.make_axis(name='axis_parallel', length=16)
with ng.metadata(device_id=c['device_id'], parallel=parallel_axis):
axis_A = ng.make_axis(length=ax_A_length, name='axis_A')
axis_B = ng.make_axis(length=ax_B_length, name='axis_B')
var_A = ng.variable(axes=[axis_A], initial_value=UniformInit(1, 1)).named('var_A')
var_B = ng.variable(initial_value=UniformInit(c['input'], c['input']),
axes=[axis_B]).named('var_B')
var_B.metadata['reduce_func'] = c['func']
var_minus = (var_A - var_B).named('var_minus')
with closing(ngt.make_transformer_factory('hetr', device='gpu')()) as hetr:
out_comp = hetr.computation([var_minus]).named('out_comp')
result = out_comp()
np.testing.assert_array_equal(result, np_result)
def train_outputs(self, in_obj, init_state=None):
"""
Sets shape based parameters of this layer given an input tuple or int
or input layer.
Arguments:
in_obj (int, tuple, Layer or Tensor): object that provides shape
information for layer
init_state (Tensor): object that provides initial state
Returns:
rnn_out (Tensor): output
"""
# try to understand the axes from the input
self.interpret_axes(in_obj, init_state)
# initialize the hidden states
if init_state is not None:
self.h_init = init_state
else:
if self.reset_cells:
self.h_init = ng.constant(
const=0, axes=self.hidden_state_axes).named('h_init')
else:
self.h_init = ng.variable(
initial_value=0,
axes=self.hidden_state_axes).named('h_init')
self.W_input = ng.variable(axes=self.w_in_axes,
initial_value=self.init).named("W_in")
self.W_recur = ng.variable(axes=self.w_re_axes,
initial_value=self.init_inner).named("W_re")
self.b = ng.variable(axes=self.hidden_axes,
initial_value=0).named("bias")
h = self.h_init
h_list = []
# slice the inputs into time slices
in_s = get_steps(in_obj, self.recurrent_axis, self.backward)
# unrolling computations
for i in range(self.recurrent_axis.length):
with ng.metadata(recurrent_step=str(i)):
h = self._step(in_s[i], h)
h_list.append(h)
if self.return_sequence is True:
# only when returning a sequence, need to reverse the output
h_list = h_list[::-1] if self.backward else h_list
rnn_out = ng.stack(h_list,
self.recurrent_axis,
pos=self.recurrent_axis_idx)
else:
rnn_out = h_list[-1]
return rnn_out
def train_outputs(self, in_obj):
"""
Sets shape based parameters of this layer given an input tuple or int
or input layer.
Arguments:
in_obj (int, tuple, Layer or Tensor): object that provides shape
information for layer
Returns:
(Tensor): output
"""
in_axes = in_obj.axes
self.time_axis = in_axes.recurrent_axes()[0]
def get_steps(x, time_axis):
return [
ng.slice_along_axis(x, time_axis, i)
for i in range(time_axis.length)
]
if self.axes is not None:
hidden_axes = self.axes - self.axes.recurrent_axes()
else:
hidden_axes = ng.make_axes(
[ng.make_axis(self.nout).named('Hidden_in')])
w_in_axes = hidden_axes + [
axis - 1
for axis in in_axes.sample_axes() - in_axes.recurrent_axes()
]
w_re_axes = hidden_axes + [axis - 1 for axis in hidden_axes]
self.W_input = ng.variable(axes=w_in_axes,
initial_value=self.init(
w_in_axes.lengths)).named("W_in")
self.W_recur = ng.variable(axes=w_re_axes,
initial_value=self.init_inner(
w_re_axes.lengths)).named("W_re")
self.b = ng.variable(axes=hidden_axes, initial_value=0).named("bias")
h_ff_buf = ng.dot(self.W_input, in_obj).named("W_in_dot_in")
h_ff_s = get_steps(h_ff_buf, self.time_axis)
self.h_init = ng.constant(np.zeros(h_ff_s[0].axes.lengths),
axes=h_ff_s[0].axes).named('h_init')
hprev = [self.h_init]
for i in range(self.time_axis.length):
with ng.metadata(recurrent_step=str(i)):
d = ng.dot(self.W_recur,
hprev[i]).named("W_rec_dot_h{}".format(i))
h = self.activation(d + h_ff_s[i] + self.b)
h.name = "activ{}".format(i)
hprev.append(h)
rnn_out = ng.stack(hprev[1:], self.time_axis, pos=1)
return rnn_out
def get_ng_variable(self): # type: () -> TensorOp
"""Create an ngraph variable node for this value."""
axes = self.get_ng_axes()
dtype = self.get_dtype()
if self.has_initializer:
initializer = self.get_initializer()
return ng.variable(axes=axes, dtype=dtype,
initial_value=initializer.to_array()).named(self.name)
return ng.variable(axes=axes, dtype=dtype).named(self.name)
def test_conv_flatten_deriv(n4_hw12_c3_5x5):
"""
Test deriv of conv followed by flatten
"""
cf = ConvParams(**n4_hw12_c3_5x5)
axes_rsck = ng.make_axes([cf.ax_f[2], cf.ax_f[3], cf.ax_f[0], cf.ax_f[-1]])
axes_rsck_prime = ng.make_axes([ng.make_axis(name=ax.name + 'p', length=ax.length)
for ax in axes_rsck])
axes_nmpqk = ng.make_axes([cf.ax_o[-1], cf.ax_o[1], cf.ax_o[2], cf.ax_o[3], cf.ax_o[0]])
# broadcast input / filter axes
input_var = ng.variable(cf.ax_i).named('input')
input_val = np.ones(input_var.axes.lengths)
filter_rsck_prime = ng.variable(axes_rsck_prime).named('filter')
filter_var = filter_rsck_prime
filter_rsck = ng.cast_axes(filter_rsck_prime, axes_rsck).named('frsck')
filter_trsck = ng.expand_dims(filter_rsck, cf.ax_f[1], 0).named('ftrsck')
filter_ctrsk = ng.axes_with_order(filter_trsck, axes=cf.ax_f).named('ctrsk')
# convolution
output_kmpqn = ng.convolution(cf.conv_params, input_var, filter_ctrsk, axes=cf.ax_o)
output_nmpqk = ng.axes_with_order(output_kmpqn, axes=axes_nmpqk)
# slice away the oD
out_slicing = [slice(None), 0, slice(None), slice(None), slice(None)]
output_npqk = ng.tensor_slice(output_nmpqk, out_slicing)
output = ng.flatten_at(output_npqk, idx=1)
# cost and grad
cost = ng.sum(output, out_axes=())
filter_val = np.ones(filter_var.axes.lengths)
with ExecutorFactory() as factory:
conv_comp = factory.executor(output, filter_var, input_var)
grad_filter_num_comp = factory.numeric_derivative(cost, filter_var, 1.0, input_var)
grad_filter_sym_comp = factory.derivative(cost, filter_var, input_var)
grad_input_num_comp = factory.numeric_derivative(cost, input_var, 1.0, filter_var)
grad_input_sym_comp = factory.derivative(cost, input_var, filter_var)
conv_val = conv_comp(filter_val, input_val)
conv_val_num = np.empty_like(conv_val)
conv_val_num.fill(np.prod(cf.ax_f.lengths[:-1]))
ng.testing.assert_allclose(conv_val, conv_val_num)
grad_filter_num_val = grad_filter_num_comp(filter_val, input_val)
grad_filter_sym_val = grad_filter_sym_comp(filter_val, input_val)
ng.testing.assert_allclose(grad_filter_num_val, grad_filter_sym_val)
grad_input_num_val = grad_input_num_comp(input_val, filter_val)
grad_input_sym_val = grad_input_sym_comp(input_val, filter_val)
ng.testing.assert_allclose(grad_input_num_val, grad_input_sym_val)
def test_deriv_missing_connection(N):
"""
Taking the derivative of an expression with respect to a variable not
used to compute the expression should raise an exception.
"""
x = ng.variable([N])
y = ng.variable([N])
z = ng.variable([N])
with pytest.raises(ValueError):
ng.deriv(x + y, z)
def test_execute_non_placeholder():
"""
Expect a failure if a non-input (Variable) is used as an argument to
executor.
"""
N = ng.make_axis(1)
x = ng.variable([N])
y = ng.variable([N])
with pytest.raises(ValueError):
executor(x + y, x, y)
def train_outputs(self, in_obj):
cpm = self.convparams.copy()
in_obj = ng.axes_with_role_order(in_obj, self.role_order)
in_axes = in_obj.axes
if self.f_axes is None:
self.f_axes = ng.make_axes([in_axes[0]])
for nm, role in zip('TRSK', self.filter_roles[1:]):
self.f_axes += ng.make_axis(roles=[role],
length=cpm[nm]).named(nm)
self.W = ng.variable(axes=self.f_axes, initial_value=self.init)
if self.o_axes is None:
self.o_axes = ng.make_axes([
ng.make_axis(roles=a.roles).named(a.short_name)
for a in in_axes if not a.is_batch
])
# set lengths
out_shape = [
self.f_axes[-1].length,
output_dim(in_axes[1].length, cpm['T'], cpm['pad_d'],
cpm['str_d'], False, cpm['dil_d']),
output_dim(in_axes[2].length, cpm['R'], cpm['pad_h'],
cpm['str_h'], False, cpm['dil_h']),
output_dim(in_axes[3].length, cpm['S'], cpm['pad_w'],
cpm['str_w'], False, cpm['dil_w'])
]
self.o_axes.set_shape(out_shape)
self.o_axes += in_axes.batch_axes()
return ng.convolution(cpm, in_obj, self.W, axes=self.o_axes)
def test_gemm(transformer_factory):
"""
TODO: make this more interesting
"""
n, c = 32, 32
N = ng.make_axis(length=n, name='N')
C = ng.make_axis(length=c)
X = ng.placeholder(axes=[C, N])
Y = ng.placeholder(axes=[N])
W = ng.variable(axes=[C - 1], initial_value=0.1)
Y_hat = ng.dot(W, X)
with executor(Y_hat, X) as ex:
mm_executor = ex
w = np.ones(c) * 0.1
xs = np.ones(n * c).reshape(c, n)
for ii in range(3):
y_hat_val = mm_executor(xs)
# 8.8 fixed point test
# assert np.allclose(np.dot(xs, w) - y_hat_val, 0.075*np.ones(n))
# autoflex test
assert_allclose(np.dot(xs, w), y_hat_val)
def test_setting(M):
with ExecutorFactory() as ex:
axes = ng.make_axes([M])
np_x = np.array([1, 2, 3], dtype=np.float32)
np_y = np.array([1, 3, 5], dtype=np.float32)
x = ng.constant(np_x, axes)
y = ng.constant(np_y, axes)
v = ng.variable(axes, initial_value=x)
f_v = ex.executor(v)
vset = ng.sequential([ng.assign(v, v + y), v])
f_v1 = ex.executor(vset)
f_v2 = ex.executor(v)
e_v = f_v().copy()
assert ng.testing.allclose(e_v, np_x)
e_v1 = f_v1().copy()
assert ng.testing.allclose(e_v1, np_x + np_y)
e_v2 = f_v2().copy()
assert ng.testing.allclose(e_v2, np_x + np_y)
def test_one_dot_bprop_allreduce(config):
c = config
pytest.xfail(
"GPU child transformers generate errors during AssignLayouts graph pass #1651"
)
H_axis = ng.make_axis(length=4, name='height')
W_axis = ng.make_axis(length=6, name='width')
with ng.metadata(step='input'):
X = ng.placeholder(axes=[H_axis, W_axis])
target = ng.constant(1, axes=[W_axis])
with ng.metadata(device_id=c['device_id'], parallel=W_axis):
W = ng.variable(axes=[H_axis], initial_value=UniformInit(1, 1))
dot = ng.dot(W, X)
L = ng.squared_L2(target - dot, out_axes=())
grad = ng.deriv(L, W)
grad.metadata['reduce_func'] = c['func']
update = (W - grad)
with closing(ngt.make_transformer_factory('hetr')()) as hetr:
out_comp = hetr.computation([update], X)
result = out_comp(c['input'])
np.testing.assert_array_equal(result, c['expected_result'])
def test_set_op_values(transformer_factory):
NUM_OPS = 3
# set up an op and Assign a value to it so we can read it out
axes = ng.make_axes([
ng.make_axis(name='A', length=2),
ng.make_axis(name='B', length=3),
])
variable_ops = [ng.variable(axes) for i in range(NUM_OPS)]
values = [np.ones(axes.lengths) * i for i in range(NUM_OPS)]
with closing(ngt.make_transformer()) as t:
serde_weights.set_op_values(
t,
variable_ops, {
op.uuid.bytes: value
for op, value in zip(variable_ops, values)
}
)
# extract values out of it and make sure they match expected results
funcs = [t.computation(op) for op in variable_ops]
op_values = [func() for func in funcs]
np.testing.assert_allclose(values, op_values)
def build_graphs(L, BS):
"""
TODO.
Arguments:
L: TODO
BS: TODO
Returns:
TODO
"""
# Axes
L = [ng.make_axis(length=N, name='L%d' % i) for i, N in enumerate(L)]
BS = ng.make_axis(length=BS, name='BS')
# Builds Network
activations = [ng.tanh for i in range(len(L) - 2)] + [ng.softmax]
X = ng.placeholder((L[0], BS)).named('X')
Y = ng.placeholder((L[-1], )).named('Y')
W = [
ng.variable((L_np1, L_n - 1)).named('W%d' % i)
for i, (L_np1, L_n) in enumerate(zip(L[1:], L[:-1]))
]
A = []
for i, f in enumerate(activations):
Aim1 = A[i - 1] if i > 0 else X
A.append(f(ng.dot(W[i], Aim1)))
Error = ng.cross_entropy_multi(A[-1], Y)
dW = [ng.deriv(Error, w) for w in W]
transformer = ngt.make_transformer()
dfg = an.DataFlowGraph(transformer, dW)
ifg = an.InterferenceGraph(dfg.liveness())
return dfg, ifg
def test_setting():
ex = ExecutorFactory()
X = ng.make_axis(name='X', length=3)
axes = ng.make_axes([X])
np_x = np.array([1, 2, 3], dtype=np.float32)
np_y = np.array([1, 3, 5], dtype=np.float32)
x = ng.constant(np_x, axes)
y = ng.constant(np_y, axes)
v = ng.variable(axes, initial_value=x)
f_v = ex.executor(v)
with ng.Op.saved_user_deps():
ng.assign(v, v + y)
f_v1 = ex.executor(v)
f_v2 = ex.executor(v)
e_v = f_v().copy()
assert np.allclose(e_v, np_x)
e_v1 = f_v1().copy()
assert np.allclose(e_v1, np_x + np_y)
e_v2 = f_v2().copy()
assert np.allclose(e_v2, np_x + np_y)
def train_outputs(self, in_obj):
"""
Arguments:
in_obj (Tensor): object that provides the lookup indices
"""
in_obj.axes.find_by_short_name('time')[0].add_role(ar.time)
in_obj.axes.find_by_short_name('time')[0].is_recurrent = True
in_obj = ng.axes_with_role_order(in_obj, self.role_order)
in_obj = ng.flatten(in_obj)
in_axes = in_obj.axes
self.lut_v_axis = ng.make_axis(self.vocab_size).named('V')
self.lut_f_axis = ng.make_axis(self.embed_dim).named('F')
self.w_axes = ng.make_axes([self.lut_v_axis, self.lut_f_axis])
self.lut_o_axes = in_axes + ng.make_axes([self.lut_f_axis])
self.o_axes = ng.make_axes([self.lut_f_axis]) + in_axes[0].axes
self.W = ng.variable(axes=self.w_axes,
initial_value=self.lut_init(
self.w_axes, self.lut_v_axis,
self.pad_idx)).named('W')
lut_result = ng.lookuptable(self.W,
in_obj,
self.lut_o_axes,
update=self.update,
pad_idx=self.pad_idx)
return ng.axes_with_order(ng.unflatten(lut_result), self.o_axes)
def test_variable_init(transformer_factory, C):
w_init = np.random.rand(C.length)
W = ng.variable(ng.make_axes([C]), initial_value=w_init)
with ExecutorFactory() as ex:
result = ex.executor(W)()
ng.testing.assert_allclose(result, w_init)
def test_extract_many_ops():
"""
Create NUM_OPS, fill them with 0, 1, 2, ... then check that
serde_weights.extract_ops is able to extract the correct uuid/value pairs.
"""
NUM_OPS = 3
# set up an op and Assign a value to it so we can read it out
axes = ng.make_axes([
ng.make_axis(name='A', length=2),
ng.make_axis(name='B', length=3),
])
variable_ops = [ng.variable(axes) for _ in range(NUM_OPS)]
assign_sequential = assign_ops(variable_ops, range(NUM_OPS))
with executor(assign_sequential) as assign_computation:
t = assign_computation.transformer
# Set values manually
assign_computation()
# extract values out of it and make sure they match expected results
weights = serde_weights.extract_ops(t, variable_ops)
for i, variable_op in enumerate(variable_ops):
np.testing.assert_allclose(weights[variable_op.uuid.bytes], i)
def test_allreduce_hint_cpu(config):
c = config
parallel_axis = ng.make_axis(name='axis_parallel', length=16)
with ng.metadata(device_id=c['device_id'], parallel=parallel_axis):
axis_A = ng.make_axis(length=4, name='axis_A')
axis_B = ng.make_axis(length=2, name='axis_B')
var_A = ng.variable(axes=[axis_A], initial_value=UniformInit(1, 1)).named('var_A')
var_B = ng.variable(initial_value=UniformInit(c['input'], c['input']),
axes=[axis_B]).named('var_B')
var_B.metadata['reduce_func'] = c['func']
var_minus = (var_A - var_B).named('var_minus')
with closing(ngt.make_transformer_factory('hetr')()) as hetr:
out_comp = hetr.computation([var_minus]).named('out_comp')
result = out_comp()
np.testing.assert_array_equal(result, c['expected_result'])
def test_pad_0(N):
"""
pad with length 0 should be a nop
"""
x = ng.variable([N])
assert ng.pad(x, [0]).axes == x.axes
def test_round_trip():
# set up an op and Assign a value to it so we can read it out
axes = ng.make_axes([
ng.make_axis(name='A', length=2),
ng.make_axis(name='B', length=3),
])
x_op = ng.variable(axes)
assign_op = ng.AssignOp(x_op, 1)
with executor(assign_op) as assign_computation:
t = assign_computation.transformer
# Set initial value
assign_computation()
# Test value
np.testing.assert_allclose(serde_weights.extract_op(t, x_op), 1)
# write out values in x and graph
f = BytesIO()
# ## EXAMPLE OF HOW TO FULLY SERIALIZE A GRAPH ###
serde_weights.serialize_weights(t, [x_op], f)
graph_string = serde.serialize_graph([x_op])
# ## /EXAMPLE OF HOW TO FULLY SERIALIZE A GRAPH ###
f.seek(0)
# ## EXAMPLE OF HOW TO FULLY DESERIALIZE A GRAPH ###
new_ops = serde.deserialize_graph(graph_string)
serde_weights.deserialize_weights(t, new_ops, f)
# ## /EXAMPLE OF HOW TO FULLY DESERIALIZE A GRAPH ###
np.testing.assert_allclose(serde_weights.extract_op(t, new_ops[0]), 1)
def import_operation(self, cntk_op):
"""
Recursively import and translate CNTK operations.
Arguments:
cntk_op: CNTK operation to be imported.
Returns:
Translated operation.
"""
if self.debug:
for _ in range(len(inspect.stack())):
print(' ', end="")
print("Importing: " + cntk_op.uid + "(", end="")
for i in cntk_op.inputs:
print(i.uid + str(i.shape) + ",", end="")
print(")")
inputs = []
for i in cntk_op.inputs:
axes = [ng.make_axis(dim) for dim in i.shape]
dtype = np.dtype(i.dtype)
if i.is_output:
uid = i.owner.root_function.uid
temp = self.uid_op_map[uid]
if isinstance(temp, C.Function):
temp = self.import_operation(temp)
if temp is None:
raise ValueError("Error translating: " + uid)
else:
if self.debug:
for _ in range(len(inspect.stack()) + 1):
print(' ', end="")
print("Finished importing: " + uid +
str(cntk_op.shape) + " -> " + temp.name +
str(temp.shape.full_lengths))
self.uid_op_map[uid] = temp
inputs.append(temp)
elif i.is_input:
if self.batch_size > 1:
axes.append(ng.make_axis(self.batch_size, 'N'))
temp = ng.placeholder(axes, dtype).named(i.uid)
inputs.append(temp)
self.placeholders.append(temp)
else:
try:
input_value = i.value
except AttributeError:
input_value = C.plus(i, np.zeros(i.shape)).eval()
if i.is_constant:
inputs.append(
ng.constant(input_value, axes, dtype).named(i.uid))
elif i.is_parameter:
inputs.append(
ng.variable(axes, dtype, input_value).named(i.uid))
else:
raise ValueError("Unknown input: " + i.uid)
return self.ops_bridge(cntk_op, inputs)
def test_pad_invalid_paddings_length(N):
"""
pad should raise an exception if the paddings length is not the same as the
input dimensionality.
"""
x = ng.variable([N])
with pytest.raises(ValueError):
ng.pad(x, [1, 0])
def test_initial_value(transformer_factory):
# Test work-around for issue #1138
w = [3, 4, 5]
x = ng.constant(w)
y = ng.variable([ng.make_axis(length=len(w))], initial_value=x)
with ExecutorFactory() as ex:
result = ex.executor(y)()
ng.testing.assert_allclose(result, np.asarray(w, dtype=np.float32))
def train_outputs(self, in_obj):
out_axes = ng.make_axes(self.axes or [ng.make_axis(self.nout).named('Hidden')])
in_axes = in_obj.axes.sample_axes()
in_axes = in_axes - in_axes.recurrent_axes()
w_axes = out_axes - out_axes.recurrent_axes() + [axis - 1 for axis in in_axes]
if self.W is None:
self.W = ng.variable(axes=w_axes, initial_value=self.init(w_axes.lengths))
return ng.dot(self.W, in_obj)
def test_sign():
x_np = np.array([-1.2, 2.3, 0.0, 1.2])
N = ng.make_axis(len(x_np))
x = ng.variable([N])
y = ng.sign(x)
y_np = np.sign(x_np)
with ExecutorFactory() as ex:
y_val = ex.executor(y, x)(x_np)
assert np.allclose(y_val, y_np)
def test_fill_state():
with ExecutorFactory() as ex:
N = ng.make_axis(3, name='N')
x_np = np.ones((N.length)) * 4
x = ng.variable([N], initial_value=x_np).named('x')
val = ng.sequential([ng.fill(x, -1), x])
f = ex.executor(val)
x_val = f()
assert np.allclose(-1, x_val)
def __call__(self, in_obj, **kwargs):
"""
Arguments:
in_obj (Tensor): object that provides the lookup indices
"""
in_obj = ng.flatten(in_obj)
in_axes = in_obj.axes
# label lut_v_axis as shadow axis for initializers ... once #1158 is
# in, shadow axis will do more than just determine fan in/out for
# initializers.
self.lut_v_axis = ng.make_axis(self.vocab_size).named('V')
self.axes_map = shadow_axes_map([self.lut_v_axis])
self.lut_v_axis = list(self.axes_map.values())[0]
self.lut_f_axis = ng.make_axis(self.embed_dim).named('F')
self.w_axes = ng.make_axes([self.lut_v_axis, self.lut_f_axis])
self.lut_o_axes = in_axes | ng.make_axes([self.lut_f_axis])
self.o_axes = ng.make_axes([self.lut_f_axis]) | in_axes[0].axes
if not self.initialized:
self.W = ng.variable(
axes=self.w_axes,
initial_value=self.lut_init(
self.w_axes,
self.lut_v_axis,
self.pad_idx),
metadata={
"label": LABELS["weight"]},
).named('LutW')
lut_result = ng.lookuptable(
self.W,
in_obj,
self.lut_o_axes,
update=self.update,
pad_idx=self.pad_idx)
return ng.map_roles(ng.unflatten(lut_result), self.axes_map)
def __init__(
self,
cands,
num_cands,
max_cand_len,
memory_size,
max_utt_len,
vocab_size,
emb_size,
batch_size,
use_match_type=False,
kb_ents_to_type=None,
kb_ents_to_cand_idxs=None,
match_type_idxs=None,
nhops=3,
eps=1e-6,
init=GaussianInit(
mean=0.0,
std=0.1)):
super(MemN2N_Dialog, self).__init__()
self.cands = cands
self.memory_size = memory_size
self.max_utt_len = max_utt_len
self.vocab_size = vocab_size
self.num_cands = num_cands
self.max_cand_len = max_cand_len
self.batch_size = batch_size
self.use_match_type = use_match_type
self.kb_ents_to_type = kb_ents_to_type
self.kb_ents_to_cand_idxs = kb_ents_to_cand_idxs
self.match_type_idxs = match_type_idxs
self.nhops = nhops
self.eps = eps
self.init = init
# Make axes
self.batch_axis = ng.make_axis(length=batch_size, name='N')
self.sentence_rec_axis = ng.make_axis(length=max_utt_len, name='REC')
self.memory_axis = ng.make_axis(length=memory_size, name='memory_axis')
self.embedding_axis = ng.make_axis(length=emb_size, name='F')
self.embedding_axis_proj = ng.make_axis(length=emb_size, name='F_proj')
self.cand_axis = ng.make_axis(length=num_cands, name='cand_axis')
self.cand_rec_axis = ng.make_axis(length=max_cand_len, name='REC')
# Weight sharing of A's accross all hops input and output
self.LUT_A = ModifiedLookupTable(
vocab_size, emb_size, init, update=True, pad_idx=0)
# Use lookuptable W to embed the candidate answers
self.LUT_W = ModifiedLookupTable(
vocab_size, emb_size, init, update=True, pad_idx=0)
# Initialize projection matrix between internal model states
self.R_proj = ng.variable(
axes=[
self.embedding_axis,
self.embedding_axis_proj],
initial_value=init)
if not self.use_match_type:
# Initialize constant matrix of all candidate answers
self.cands_mat = ng.constant(
self.cands, axes=[
self.cand_axis, self.cand_rec_axis])
