total_iterations=args.num_iterations) valid_set = SequentialArrayIterator(ptb_data['valid'], batch_size=args.batch_size, time_steps=time_steps) inputs = train_set.make_placeholders() ax.Y.length = len(tree_bank_data.vocab) def expand_onehot(x): return ng.one_hot(x, axis=ax.Y) # weight initialization init = UniformInit(low=-0.08, high=0.08) if args.layer_type == "lstm": rlayer = LSTM(hidden_size, init, activation=Tanh(), gate_activation=Logistic(), return_sequence=True) # model initialization seq1 = Sequential([ Preprocess(functor=expand_onehot), rlayer, Affine(init, activation=Softmax(), bias_init=init, axes=(ax.Y, )) ]) optimizer = RMSProp(gradient_clip_value=gradient_clip_value)
type=int, help="Number of recurrent units in the network", default=256) parser.set_defaults(num_iterations=20000) args = parser.parse_args() use_embedding = args.use_embedding recurrent_units = args.recurrent_units batch_size = args.batch_size seq_len = args.seq_len num_iterations = args.num_iterations # Ratio of the text to use for training train_ratio = 0.95 # Define initialization method of neurons in the network init_uni = UniformInit(-0.1, 0.1) # Create the object that includes the sample text shakes = Shakespeare(train_split=train_ratio) # Build an iterator that gets seq_len long chunks of characters # Stride is by how many characters the window moves in each step # if stride is set to seq_len, windows are non-overlapping stride = seq_len // 8 shakes_data_train = { 'X': np.copy(shakes.train), 'y': np.roll(np.copy(shakes.train), shift=-1) } shakes_data_test = { 'X': np.copy(shakes.test), 'y': np.roll(np.copy(shakes.test), shift=-1)
def test_recvop_tensorupdate(transformer_factory): """ The tensor (RecvOp_#_#) associated with the following conv op has two views: 1) Non-flat view (e.g. RecvOp_#_#_1_1_1_1_4.shape=(1,1,1,1,4)) 2) Flat view (e.g. RecvOp_#_#_1_4.shape = (1,4)) This test ensures that inside RecvOp code generation, the generated code should make sure both views get updated (e.g. by using update_RecvOp_#_# API) In this test, ng.dot operation tends to use the flat view (i.e. RecvOp_#_#_1_4) And previously RecvOp with RecvOp_#_#_1_1_1_1_4 = recv_from_send(send_id) failed to update both two views (i.e. flat and non-flat view of the same buffer/tensor) """ class ConvParams(object): def __init__(self, C=1, N=1, K=1, D=1, H=1, W=1, T=1, R=1, S=1, pad_d=0, pad_h=0, pad_w=0, str_d=1, str_h=1, str_w=1): from ngraph.frontends.neon.layer import output_dim M = output_dim(D, T, pad_d, str_d) P = output_dim(H, R, pad_h, str_h) Q = output_dim(W, S, pad_w, str_w) self.dimO = (K, M, P, Q, N) self.dimI = (C, D, H, W, N) self.dimF = (C, T, R, S, K) self.conv_params = dict( pad_d=pad_d, pad_h=pad_h, pad_w=pad_w, str_d=str_d, str_h=str_h, str_w=str_w, dil_d=1, dil_h=1, dil_w=1 ) self.batch_axis = ng.make_axis(name='N', length=N) self.ax_i = ng.make_axes([ ng.make_axis(name='C', length=C), ng.make_axis(name='D', length=D), ng.make_axis(name='H', length=H), ng.make_axis(name='W', length=W), self.batch_axis ]) self.ax_f = ng.make_axes([ ng.make_axis(name='C', length=C), ng.make_axis(name='D', length=T), ng.make_axis(name='H', length=R), ng.make_axis(name='W', length=S), ng.make_axis(name='K', length=K), ]) self.ax_o = ng.make_axes([ ng.make_axis(name='C', length=K), ng.make_axis(name='D', length=M), ng.make_axis(name='H', length=P), ng.make_axis(name='W', length=Q), self.batch_axis ]) # Layer 1, using convolutation introduces multi/flatten view of tensors cf = ConvParams(C=2, N=4, K=1, H=2, W=2, R=2, S=2) inputs = ng.placeholder(axes=cf.ax_i) filters = ng.placeholder(axes=cf.ax_f) # randomly initialize from ngraph.testing import RandomTensorGenerator rng = RandomTensorGenerator(0, np.float32) # put value 1 into inputs/filters for conv input_value = rng.uniform(1, 1, cf.ax_i) filter_value = rng.uniform(1, 1, cf.ax_f) conv = ng.convolution(cf.conv_params, inputs, filters, axes=cf.ax_o) # Layer 2, using dot to ensure recv_op.axes == send_op.axes from ngraph.frontends.neon import UniformInit # put value 1 into weights for dot init_uni = UniformInit(1, 1) W_A = ng.make_axis(length=2) w_axes = ng.make_axes(W_A) + conv.axes.feature_axes() w = ng.variable(axes=w_axes, initial_value=init_uni) with ng.metadata(device_id='1'): dot = ng.dot(w, conv) with ExecutorFactory() as ex: dot_comp = ex.executor(dot, filters, inputs) dot_val = dot_comp(filter_value, input_value) np.testing.assert_array_equal(dot_val, [[8., 8., 8., 8.], [8., 8., 8., 8.]])
def test_linear_invalid_batch_axes(): with pytest.raises(ValueError): Linear(axes=ng.make_axis(1, name='N'), init=UniformInit(1.0, 1.0))
def test_linear_invalid_shadow_axes(): with pytest.raises(ValueError): Linear(axes=make_shadow_axis(ng.make_axis(1, name='A')), init=UniformInit(1.0, 1.0))
def test_linear_accepts_axes_axis(): """ Ensure that Linear.__init__ accepts an Axis as axes """ Linear(axes=ng.make_axis(1), init=UniformInit(1.0, 1.0))