total_iterations=args.num_iterations)
valid_set = SequentialArrayIterator(ptb_data['valid'],
batch_size=args.batch_size,
time_steps=time_steps)
inputs = train_set.make_placeholders()
ax.Y.length = len(tree_bank_data.vocab)
def expand_onehot(x):
return ng.one_hot(x, axis=ax.Y)
# weight initialization
init = UniformInit(low=-0.08, high=0.08)
if args.layer_type == "lstm":
rlayer = LSTM(hidden_size,
init,
activation=Tanh(),
gate_activation=Logistic(),
return_sequence=True)
# model initialization
seq1 = Sequential([
Preprocess(functor=expand_onehot), rlayer,
Affine(init, activation=Softmax(), bias_init=init, axes=(ax.Y, ))
])
optimizer = RMSProp(gradient_clip_value=gradient_clip_value)
type=int,
help="Number of recurrent units in the network",
default=256)
parser.set_defaults(num_iterations=20000)
args = parser.parse_args()
use_embedding = args.use_embedding
recurrent_units = args.recurrent_units
batch_size = args.batch_size
seq_len = args.seq_len
num_iterations = args.num_iterations
# Ratio of the text to use for training
train_ratio = 0.95
# Define initialization method of neurons in the network
init_uni = UniformInit(-0.1, 0.1)
# Create the object that includes the sample text
shakes = Shakespeare(train_split=train_ratio)
# Build an iterator that gets seq_len long chunks of characters
# Stride is by how many characters the window moves in each step
# if stride is set to seq_len, windows are non-overlapping
stride = seq_len // 8
shakes_data_train = {
'X': np.copy(shakes.train),
'y': np.roll(np.copy(shakes.train), shift=-1)
}
shakes_data_test = {
'X': np.copy(shakes.test),
'y': np.roll(np.copy(shakes.test), shift=-1)
def test_recvop_tensorupdate(transformer_factory):
"""
The tensor (RecvOp_#_#) associated with the following conv op has two views:
1) Non-flat view (e.g. RecvOp_#_#_1_1_1_1_4.shape=(1,1,1,1,4))
2) Flat view (e.g. RecvOp_#_#_1_4.shape = (1,4))
This test ensures that inside RecvOp code generation, the generated code
should make sure both views get updated (e.g. by using update_RecvOp_#_# API)
In this test, ng.dot operation tends to use the flat view (i.e. RecvOp_#_#_1_4)
And previously RecvOp with RecvOp_#_#_1_1_1_1_4 = recv_from_send(send_id) failed
to update both two views (i.e. flat and non-flat view of the same buffer/tensor)
"""
class ConvParams(object):
def __init__(self, C=1, N=1, K=1, D=1, H=1, W=1, T=1, R=1, S=1,
pad_d=0, pad_h=0, pad_w=0,
str_d=1, str_h=1, str_w=1):
from ngraph.frontends.neon.layer import output_dim
M = output_dim(D, T, pad_d, str_d)
P = output_dim(H, R, pad_h, str_h)
Q = output_dim(W, S, pad_w, str_w)
self.dimO = (K, M, P, Q, N)
self.dimI = (C, D, H, W, N)
self.dimF = (C, T, R, S, K)
self.conv_params = dict(
pad_d=pad_d, pad_h=pad_h, pad_w=pad_w,
str_d=str_d, str_h=str_h, str_w=str_w,
dil_d=1, dil_h=1, dil_w=1
)
self.batch_axis = ng.make_axis(name='N', length=N)
self.ax_i = ng.make_axes([
ng.make_axis(name='C', length=C),
ng.make_axis(name='D', length=D),
ng.make_axis(name='H', length=H),
ng.make_axis(name='W', length=W),
self.batch_axis
])
self.ax_f = ng.make_axes([
ng.make_axis(name='C', length=C),
ng.make_axis(name='D', length=T),
ng.make_axis(name='H', length=R),
ng.make_axis(name='W', length=S),
ng.make_axis(name='K', length=K),
])
self.ax_o = ng.make_axes([
ng.make_axis(name='C', length=K),
ng.make_axis(name='D', length=M),
ng.make_axis(name='H', length=P),
ng.make_axis(name='W', length=Q),
self.batch_axis
])
# Layer 1, using convolutation introduces multi/flatten view of tensors
cf = ConvParams(C=2, N=4, K=1, H=2, W=2, R=2, S=2)
inputs = ng.placeholder(axes=cf.ax_i)
filters = ng.placeholder(axes=cf.ax_f)
# randomly initialize
from ngraph.testing import RandomTensorGenerator
rng = RandomTensorGenerator(0, np.float32)
# put value 1 into inputs/filters for conv
input_value = rng.uniform(1, 1, cf.ax_i)
filter_value = rng.uniform(1, 1, cf.ax_f)
conv = ng.convolution(cf.conv_params, inputs, filters, axes=cf.ax_o)
# Layer 2, using dot to ensure recv_op.axes == send_op.axes
from ngraph.frontends.neon import UniformInit
# put value 1 into weights for dot
init_uni = UniformInit(1, 1)
W_A = ng.make_axis(length=2)
w_axes = ng.make_axes(W_A) + conv.axes.feature_axes()
w = ng.variable(axes=w_axes, initial_value=init_uni)
with ng.metadata(device_id='1'):
dot = ng.dot(w, conv)
with ExecutorFactory() as ex:
dot_comp = ex.executor(dot, filters, inputs)
dot_val = dot_comp(filter_value, input_value)
np.testing.assert_array_equal(dot_val, [[8., 8., 8., 8.],
[8., 8., 8., 8.]])
def test_linear_invalid_batch_axes():
with pytest.raises(ValueError):
Linear(axes=ng.make_axis(1, name='N'), init=UniformInit(1.0, 1.0))
def test_linear_invalid_shadow_axes():
with pytest.raises(ValueError):
Linear(axes=make_shadow_axis(ng.make_axis(1, name='A')),
init=UniformInit(1.0, 1.0))
def test_linear_accepts_axes_axis():
""" Ensure that Linear.__init__ accepts an Axis as axes """
Linear(axes=ng.make_axis(1), init=UniformInit(1.0, 1.0))
