def make_weights(
input_placeholder,
hidden_size,
weight_initializer,
bias_initializer,
init_state=False):
gates = ['i', 'f', 'o', 'g']
# input axis + any extra axes of length 1
in_feature_axes = tuple(input_placeholder.axes)[:-2]
out_feature_axes = ng.make_axes([ng.make_axis(hidden_size)])
batch_axis = input_placeholder.axes.batch_axis()
hidden_axis = ng.make_axis(hidden_size)
w_in_axes = ng.make_axes(hidden_axis) + in_feature_axes
w_rec_axes = ng.make_axes(hidden_axis) + out_feature_axes
W_in = {gate: weight_initializer(w_in_axes) for gate in gates}
W_rec = {gate: weight_initializer(w_rec_axes) for gate in gates}
b = {gate: bias_initializer(hidden_axis) for gate in gates}
if init_state is True:
ax_s = ng.make_axes([hidden_axis, batch_axis])
init_state = {name: ng.placeholder(ax_s) for name in ['h', 'c']}
init_state_value = {
name: rng.uniform(-1, 1, ax_s) for name in ['h', 'c']}
else:
init_state = None
init_state_value = None
return W_in, W_rec, b, init_state, init_state_value
def test_lut(lut_args):
"""
test lut fprop and bprop
"""
pad_idx = 0
with ExecutorFactory() as ex:
vocab_size, embed_dim, bsz, seq_len, mem_size = lut_args
V = ng.make_axis(vocab_size)
F = ng.make_axis(embed_dim)
M = ng.make_axis(mem_size)
ax.N.length = bsz
ax.REC.length = seq_len
# Multi-axis input to LUT
ax_idx = ng.make_axes([M, ax.REC, ax.N])
ax_lut = ng.make_axes([V, F])
lut = ng.placeholder(ax_lut)
idx = ng.placeholder(ax_idx)
idx_flat = ng.flatten(idx)
ax_out = idx_flat.axes | ng.make_axes([F])
# fprop
lut_out_ng = ng.lookuptable(lut, idx_flat, ax_out, pad_idx=pad_idx)
fprop_fun = ex.executor(lut_out_ng, lut, idx)
# bprop
update_error = ng.placeholder(ax_out)
update_out_ng = lookuptable_update(update_error, lut, idx, lut_out_ng)
update_fun = ex.executor(update_out_ng, update_error, lut, idx)
# provide actual inputs and execute the graph
lut_value = rng.uniform(-1, 1, lut.axes)
idx_value = rng.random_integers(0, vocab_size - 1, idx.axes)
fprop_lut = fprop_fun(lut_value, idx_value).copy()
# compare fprop
fprop_ref = lut_fprop_ref(lut_value, idx_value)
ng.testing.assert_allclose(fprop_lut, fprop_ref, rtol=0.0, atol=1.0e-5)
# provide actual delta and execute the update op
update_value = rng.uniform(-1, 1, update_error.axes)
update_lut = update_fun(update_value, lut_value, idx_value).copy()
# compare bprop (udpate)
update_ref = lut_update_ref(
update_value,
lut_value,
idx_value,
pad_idx=pad_idx)
ng.testing.assert_allclose(
update_lut, update_ref, rtol=0.0, atol=1.0e-5)
def make_placeholder(input_size, sequence_length, batch_size, extra_axes=0):
input_axis = ng.make_axis(name='features')
recurrent_axis = ng.make_axis(name='REC_REP')
batch_axis = ng.make_axis(name='N')
input_axes = ng.make_axes([input_axis, recurrent_axis, batch_axis])
input_axes.set_shape((input_size, sequence_length, batch_size))
input_axes = ng.make_axes([ng.make_axis(length=1, name='features_' + str(i))
for i in range(extra_axes)]) + input_axes
input_placeholder = ng.placeholder(input_axes)
rng = RandomTensorGenerator()
input_value = rng.uniform(-0.01, 0.01, input_axes)
return input_placeholder, input_value
def __call__(self, in_obj, **kwargs):
"""
Arguments:
in_obj (Tensor): object that provides the lookup indices
"""
in_obj = ng.flatten(in_obj)
in_axes = in_obj.axes
# label lut_v_axis as shadow axis for initializers ... once #1158 is
# in, shadow axis will do more than just determine fan in/out for
# initializers.
self.lut_v_axis = ng.make_axis(self.vocab_size).named('V')
self.axes_map = shadow_axes_map([self.lut_v_axis])
self.lut_v_axis = list(self.axes_map.values())[0]
self.lut_f_axis = ng.make_axis(self.embed_dim).named('F')
self.w_axes = ng.make_axes([self.lut_v_axis, self.lut_f_axis])
self.lut_o_axes = in_axes | ng.make_axes([self.lut_f_axis])
self.o_axes = ng.make_axes([self.lut_f_axis]) | in_axes[0].axes
if not self.initialized:
self.W = ng.variable(
axes=self.w_axes,
initial_value=self.lut_init(
self.w_axes,
self.lut_v_axis,
self.pad_idx),
metadata={
"label": LABELS["weight"]},
).named('LutW')
lut_result = ng.lookuptable(
self.W,
in_obj,
self.lut_o_axes,
update=self.update,
pad_idx=self.pad_idx)
return ng.map_roles(ng.unflatten(lut_result), self.axes_map)
time_steps=seq_len,
batch_size=batch_size,
stride=stride,
include_iteration=True,
tgt_key='y',
shuffle=False)
# Our input is of size (batch_size, seq_len)
# batch_axis must be named N
batch_axis = ng.make_axis(length=batch_size, name="N")
# time_axis must be named REC
time_axis = ng.make_axis(length=seq_len, name="REC")
# Output is of size (vocab_size + 1,1)
# +1 is for unknown token
out_axis = ng.make_axis(length=len(shakes.vocab) + 1, name="out_feature_axis")
in_axes = ng.make_axes([batch_axis, time_axis])
out_axes = ng.make_axes([batch_axis, time_axis])
# Build placeholders for the created axes
inputs = {
'X': ng.placeholder(in_axes),
'y': ng.placeholder(out_axes),
'iteration': ng.placeholder(axes=())
}
# Network Definition
if (use_embedding is False):
seq1 = Sequential([
Preprocess(functor=expand_onehot),
LSTM(nout=recurrent_units,
init=init_uni,
def __init__(self, C=1, N=1, K=1, D=1, H=1, W=1, T=1, R=1, S=1,
pad_d=0, pad_h=0, pad_w=0,
str_d=1, str_h=1, str_w=1,
dil_d=1, dil_h=1, dil_w=1, deconv=False):
if deconv:
M = deconv_output_dim(D, T, pad_d, str_d)
P = deconv_output_dim(H, R, pad_h, str_h)
Q = deconv_output_dim(W, S, pad_w, str_w)
else:
M = conv_output_dim(D, T, pad_d, str_d)
P = conv_output_dim(H, R, pad_h, str_h)
Q = conv_output_dim(W, S, pad_w, str_w)
self.dimO = (K, M, P, Q, N)
self.dimI = (C, D, H, W, N)
if deconv:
self.dimF = (K, T, R, S, C)
else:
self.dimF = (C, T, R, S, K)
self.conv_params = dict(
pad_d=pad_d, pad_h=pad_h, pad_w=pad_w,
str_d=str_d, str_h=str_h, str_w=str_w,
dil_d=dil_d, dil_h=dil_h, dil_w=dil_w
)
batch_axis = ng.make_axis(name='N', length=N)
self.ax_i = ng.make_axes([
ng.make_axis(name='C', length=C),
ng.make_axis(name='D', length=D),
ng.make_axis(name='H', length=H),
ng.make_axis(name='W', length=W),
batch_axis
])
if deconv:
self.ax_f = ng.make_axes([
ng.make_axis(name='C', length=K),
ng.make_axis(name='D', length=T),
ng.make_axis(name='H', length=R),
ng.make_axis(name='W', length=S),
ng.make_axis(name='K', length=C),
])
else:
self.ax_f = ng.make_axes([
ng.make_axis(name='C', length=C),
ng.make_axis(name='D', length=T),
ng.make_axis(name='H', length=R),
ng.make_axis(name='W', length=S),
ng.make_axis(name='K', length=K),
])
self.ax_o = ng.make_axes([
ng.make_axis(name='C', length=K),
ng.make_axis(name='D', length=M),
ng.make_axis(name='H', length=P),
ng.make_axis(name='W', length=Q),
batch_axis
])
def test_axes_equal():
""" Test axes == operator """
a1 = ng.make_axes([ax.A, ax.B, ax.C])
a2 = ng.make_axes([ax.A, ax.B, ax.C])
assert a1 == a2
def recurrent_input_tensor(feature_axis, recurrent_axis, batch_axis):
axes = ng.make_axes([feature_axis, batch_axis, recurrent_axis])
return ng.placeholder(axes)
def train_outputs(self, in_obj, init_state=None):
"""
Sets shape based parameters of this layer given an input tuple or int
or input layer.
Arguments:
in_obj (int, tuple, Layer or Tensor): object that provides shape
information for layer
init_state (Tensor): object that provides initial state
Returns:
(Tensor): output
"""
def get_steps(x, time_axis):
return [ng.slice_along_axis(x, time_axis, i) for i in range(time_axis.length)]
in_axes = in_obj.axes
self.time_axis = in_axes.recurrent_axes()[0]
self.time_axis_idx = in_axes.index(self.time_axis)
if self.axes is not None:
hidden_axes = self.axes - self.axes.recurrent_axes()
elif init_state:
hidden_axes = init_state.axes.sample_axes() - init_state.axes.recurrent_axes()
else:
hidden_axes = ng.make_axes([ng.make_axis(self.nout).named('Hidden')])
w_in_axes = hidden_axes + [axis - 1 for axis in in_axes.sample_axes() -
in_axes.recurrent_axes()]
w_re_axes = hidden_axes + [axis - 1 for axis in hidden_axes]
hidden_state_axes = hidden_axes + in_axes.batch_axes()
self.W_input = ng.variable(axes=w_in_axes,
initial_value=self.init(w_in_axes.lengths)
).named("W_in")
self.W_recur = ng.variable(axes=w_re_axes,
initial_value=self.init_inner(w_re_axes.lengths)
).named("W_re")
self.b = ng.variable(axes=hidden_axes, initial_value=0).named("bias")
if init_state:
self.h_init = init_state
else:
if self.reset_cells:
self.h_init = ng.constant(np.zeros(hidden_state_axes.lengths),
axes=hidden_state_axes).named('h_init')
else:
self.h_init = ng.variable(initial_value=np.zeros(hidden_state_axes.lengths),
axes=hidden_state_axes).named('h_init')
hprev = [self.h_init]
h_ff_buf = ng.dot(self.W_input, in_obj).named("W_in_dot_in")
h_ff_s = get_steps(h_ff_buf, self.time_axis)
for i in range(self.time_axis.length):
with ng.metadata(recurrent_step=str(i)):
d = ng.dot(self.W_recur, hprev[i]).named("W_rec_dot_h{}".format(i))
h = self.activation(d + h_ff_s[i] + self.b)
h.name = "activ{}".format(i)
hprev.append(h)
if self.return_sequence is True:
rnn_out = ng.stack(hprev[1:], self.time_axis, pos=self.time_axis_idx)
else:
rnn_out = hprev[-1]
return rnn_out
def symmetric_tensor():
axes = ng.make_axes([ng.make_axis(length=10), ng.make_axis(length=10)])
return ng.placeholder(axes)
def get_simple_graph():
ax = ng.make_axes([ng.make_axis(name='C', length=1)])
base_op = ng.constant(5.0, ax)
simple_graph = ng.log(ng.exp(base_op))
return base_op, simple_graph
def test_recvop_tensorupdate(transformer_factory):
"""
The tensor (RecvOp_#_#) associated with the following conv op has two views:
1) Non-flat view (e.g. RecvOp_#_#_1_1_1_1_4.shape=(1,1,1,1,4))
2) Flat view (e.g. RecvOp_#_#_1_4.shape = (1,4))
This test ensures that inside RecvOp code generation, the generated code
should make sure both views get updated (e.g. by using update_RecvOp_#_# API)
In this test, ng.dot operation tends to use the flat view (i.e. RecvOp_#_#_1_4)
And previously RecvOp with RecvOp_#_#_1_1_1_1_4 = recv_from_send(send_id) failed
to update both two views (i.e. flat and non-flat view of the same buffer/tensor)
"""
class ConvParams(object):
def __init__(self, C=1, N=1, K=1, D=1, H=1, W=1, T=1, R=1, S=1,
pad_d=0, pad_h=0, pad_w=0,
str_d=1, str_h=1, str_w=1):
from ngraph.frontends.neon.layer import output_dim
M = output_dim(D, T, pad_d, str_d)
P = output_dim(H, R, pad_h, str_h)
Q = output_dim(W, S, pad_w, str_w)
self.dimO = (K, M, P, Q, N)
self.dimI = (C, D, H, W, N)
self.dimF = (C, T, R, S, K)
self.conv_params = dict(
pad_d=pad_d, pad_h=pad_h, pad_w=pad_w,
str_d=str_d, str_h=str_h, str_w=str_w,
dil_d=1, dil_h=1, dil_w=1
)
self.batch_axis = ng.make_axis(name='N', length=N)
self.ax_i = ng.make_axes([
ng.make_axis(name='C', length=C),
ng.make_axis(name='D', length=D),
ng.make_axis(name='H', length=H),
ng.make_axis(name='W', length=W),
self.batch_axis
])
self.ax_f = ng.make_axes([
ng.make_axis(name='C', length=C),
ng.make_axis(name='D', length=T),
ng.make_axis(name='H', length=R),
ng.make_axis(name='W', length=S),
ng.make_axis(name='K', length=K),
])
self.ax_o = ng.make_axes([
ng.make_axis(name='C', length=K),
ng.make_axis(name='D', length=M),
ng.make_axis(name='H', length=P),
ng.make_axis(name='W', length=Q),
self.batch_axis
])
# Layer 1, using convolutation introduces multi/flatten view of tensors
cf = ConvParams(C=2, N=4, K=1, H=2, W=2, R=2, S=2)
inputs = ng.placeholder(axes=cf.ax_i)
filters = ng.placeholder(axes=cf.ax_f)
# randomly initialize
from ngraph.testing import RandomTensorGenerator
rng = RandomTensorGenerator(0, np.float32)
# put value 1 into inputs/filters for conv
input_value = rng.uniform(1, 1, cf.ax_i)
filter_value = rng.uniform(1, 1, cf.ax_f)
conv = ng.convolution(cf.conv_params, inputs, filters, axes=cf.ax_o)
# Layer 2, using dot to ensure recv_op.axes == send_op.axes
from ngraph.frontends.neon import UniformInit
# put value 1 into weights for dot
init_uni = UniformInit(1, 1)
W_A = ng.make_axis(length=2)
w_axes = ng.make_axes(W_A) + conv.axes.feature_axes()
w = ng.variable(axes=w_axes, initial_value=init_uni)
with ng.metadata(device_id='1'):
dot = ng.dot(w, conv)
with ExecutorFactory() as ex:
dot_comp = ex.executor(dot, filters, inputs)
dot_val = dot_comp(filter_value, input_value)
np.testing.assert_array_equal(dot_val, [[8., 8., 8., 8.],
[8., 8., 8., 8.]])
assert len(active_children()) == 1
comp()
assert len(active_children()) == 2
assert len(active_children()) == len(baseline)
ax_A = ng.make_axis(4)
ax_B = ng.make_axis(6)
ax_C = ng.make_axis(12)
ax_D = ng.make_axis(24)
@pytest.mark.hetr_gpu_only
@pytest.mark.parametrize('config', [
{
'axes': ng.make_axes([ax_A]),
'device_id': ('1', '2'),
'parallel_axis': ax_A,
},
{
'axes': ng.make_axes([ax_A, ax_B]),
'device_id': ('1', '2'),
'parallel_axis': ax_A,
},
{
'axes': ng.make_axes([ax_A, ax_B]),
'device_id': ('1', '2'),
'parallel_axis': ax_B,
},
{
'axes': ng.make_axes([ax_A, ax_B, ax_C]),
train_iterator = ArrayIterator(turbofan_dataset.train,
batch_size,
total_iterations=num_iterations,
shuffle=True)
test_iterator = ArrayIterator(turbofan_dataset.test, batch_size)
train_set_one_epoch = ArrayIterator(turbofan_dataset.train,
batch_size,
shuffle=False)
# Name and create axes
batch_axis = ng.make_axis(length=batch_size, name="N")
time_axis = ng.make_axis(length=seq_len, name="REC")
feature_axis = ng.make_axis(length=n_features, name="F")
out_axis = ng.make_axis(length=n_output_features, name="Fo")
in_axes = ng.make_axes([batch_axis, time_axis, feature_axis])
out_axes = ng.make_axes([batch_axis, out_axis])
# Build placeholders for the created axes
inputs = dict(X=ng.placeholder(in_axes),
y=ng.placeholder(out_axes),
iteration=ng.placeholder(axes=()))
# take only the last timepoint of output sequence to predict RUL
last_timepoint = [
lambda op: ng.tensor_slice(op, [
slice(seq_len - 1, seq_len, 1) if ax.name == "W" else slice(None)
for ax in op.axes
])
]
affine_layer = Affine(axes=out_axis,
def parse_net_def(self, net_def, init_net_def=None, c2_workspace=None, verbose=False):
"""
Imports a net_def to ngraph.
Arguments:
net_def: GraphDef object
verbose: Prints net_def at each node if True.
"""
def _register_op(op, c2_op):
# convert to list for convenience
if isinstance(op, tuple):
op = list(op)
else:
op = [op]
# post-process output ops
for idx in range(len(op)):
op[idx] = self.post_process_op(op[idx])
# convert back to tuple or op
if len(op) > 1:
op = tuple(op)
else:
op = op[0]
# TODO: what if some c2_op have more than one output?
key = c2_op.name if c2_op.name != '' else c2_op.output[0]
self.name_op_map[key] = op
self.init_net_def = init_net_def
if init_net_def:
self.net_def = copy.deepcopy(init_net_def)
self.net_def.op.extend(net_def.op)
else:
self.net_def = net_def
# process nodes
for c2_op in self.net_def.op:
# print node
if verbose:
print("------")
print(c2_op)
# resolve inputs
input_ops = []
for name in c2_op.input:
try:
input_ops.append(self.get_op_handle_by_name(name))
except KeyError as e:
if not c2_workspace.HasBlob(e.message):
raise e
c2_blob = c2_workspace.FetchBlob(e.message)
external_input = ng.persistent_tensor(
axes=ng.make_axes([ng.make_axis(i) for i in c2_blob.shape]),
dtype=c2_blob.dtype,
initial_value=c2_blob).named(e.message)
external_input.axes[0]._Axis__is_batch = True # TODO: find nice way to do it
input_ops.append(external_input)
class mock_c2_op:
def __init__(self, name):
self.name = name
mock_obj = mock_c2_op(e.message)
_register_op(external_input, mock_obj)
# get output op
if None in input_ops:
# ignored
print("!!! IGNORED:{} !!!".format(c2_op.name))
output_op = None
else:
# call bridge op
output_op = self.ops_bridge(c2_op, input_ops)
if output_op is None:
print("!!! Unknown Operation '{}' of type '{}' !!!"
.format(c2_op.name, c2_op.type))
_register_op(output_op, c2_op)
def input_axes(feature_axis, batch_axis):
return ng.make_axes([feature_axis, batch_axis])
def test_tensor_description_init():
with pytest.raises(ValueError):
# TensorDescription axes require lengths
TensorDescription(ng.make_axes(ng.make_axis()))