def test_ref_stacked(reflstmargs):
seq_len, input_size, hidden_size, batch_size, num_iter, reset_cells = reflstmargs
check_stacked_lstm(seq_len,
input_size,
hidden_size,
batch_size,
GaussianInit(0.0, 0.1),
reset_cells=reset_cells,
num_iter=num_iter)
def test_ref_compare_rand(reflstmargs):
# run comparison with reference code
# for Gaussian random init
seq_len, input_size, hidden_size, batch_size, num_iter, reset_cells = reflstmargs
check_lstm(seq_len,
input_size,
hidden_size,
batch_size,
GaussianInit(0.0, 0.1),
reset_cells=reset_cells,
num_iter=num_iter)
def test_conv1d(transformer_factory, filter_width, num_filters, strides,
padding, time_steps, feature_dimension, batch_size):
dilation = 1 # reference conv does not support dilation
F = ng.make_axis(name='F', length=feature_dimension)
REC = ng.make_axis(name='REC', length=time_steps)
N = ng.make_axis(name='N', length=batch_size)
in_axes = ng.make_axes([F, REC, N])
inputs = ng.placeholder(axes=in_axes)
input_vals = np.random.randn(*in_axes.lengths)
filter_init = GaussianInit()
conv1d = Convolution((filter_width, num_filters),
filter_init,
strides=strides,
padding=padding,
dilation=dilation,
bias_init=None,
activation=Rectlin(),
batch_norm=None)
result_op = conv1d(inputs, channel_axes='F', spatial_axes={'W': 'REC'})
with closing(ngt.make_transformer()) as transformer:
result_comp = transformer.add_computation(
ng.computation(result_op, inputs))
filter_vals = transformer.add_computation(ng.computation(
conv1d.conv.W))()
result_ng = result_comp(input_vals)
result_np = np.squeeze(
reference_conv1d(input_vals, filter_vals,
lambda x: np.maximum(0, x)))
ng.testing.assert_allclose(result_ng, result_np)
# the output needs transpose as well
Hout_ref_2 = Hout_ref_2.reshape(seq_len * batch_size,
hidden_size + 1).T
fprop_ref_2_list.append(Hout_ref_2)
for i in range(num_iter):
ng.testing.assert_allclose(fprop_neon_2_list[i],
fprop_ref_2_list[i],
rtol=rtol,
atol=atol)
if __name__ == '__main__':
seq_len, input_size, hidden_size, batch_size, reset_cells = (8, 5, 16, 1,
True)
init = GaussianInit(0.0, 1)
check_lstm(seq_len,
input_size,
hidden_size,
batch_size,
init,
reset_cells=reset_cells)
check_stacked_lstm(seq_len,
input_size,
hidden_size,
batch_size,
init,
reset_cells=reset_cells)
'axes': ('N', )
}
}
train_set = ArrayIterator(train_data,
batch_size=args.batch_size,
total_iterations=args.num_iterations)
inputs = train_set.make_placeholders(include_iteration=True)
ax.Y.length = 1000 # number of outputs of last layer.
# weight initialization
init = UniformInit(low=-0.08, high=0.08)
# Setup model
seq1 = Sequential([
Convolution((3, 3, 64),
filter_init=GaussianInit(std=0.01),
bias_init=init,
activation=Rectlin(),
padding=1),
Pooling((2, 2), strides=2),
Convolution((3, 3, 128),
filter_init=GaussianInit(std=0.01),
bias_init=init,
activation=Rectlin(),
padding=1),
Pooling((2, 2), strides=2),
Convolution((3, 3, 256),
filter_init=GaussianInit(std=0.01),
bias_init=init,
activation=Rectlin(),
padding=1),
# Create the dataloader
train_data, valid_data = MNIST(args.data_dir).load_data()
train_set = ArrayIterator(train_data,
args.batch_size,
total_iterations=args.num_iterations)
valid_set = ArrayIterator(valid_data, args.batch_size)
inputs = train_set.make_placeholders()
ax.Y.length = 10
######################
# Model specification
seq1 = Sequential([
Preprocess(functor=lambda x: x / 255.),
Affine(nout=100, weight_init=GaussianInit(), activation=Rectlin()),
Affine(axes=ax.Y, weight_init=GaussianInit(), activation=Logistic())
])
optimizer = GradientDescentMomentum(0.1, 0.9)
train_prob = seq1(inputs['image'])
train_loss = ng.cross_entropy_binary(train_prob,
ng.one_hot(inputs['label'], axis=ax.Y))
batch_cost = ng.sequential(
[optimizer(train_loss),
ng.mean(train_loss, out_axes=())])
train_outputs = dict(batch_cost=batch_cost)
with Layer.inference_mode_on():
inference_prob = seq1(inputs['image'])
y_train = np.ones(shape=(args.batch_size), dtype=np.int32)
train_data = {'image': {'data': X_train,
'axes': ('N', 'C', 'H', 'W')},
'label': {'data': y_train,
'axes': ('N',)}}
train_set = ArrayIterator(train_data,
batch_size=args.batch_size,
total_iterations=args.num_iterations)
inputs = train_set.make_placeholders(include_iteration=True)
ax.Y.length = 1000 # number of outputs of last layer.
# weight initialization
init = UniformInit(low=-0.08, high=0.08)
# Setup model
seq1 = Sequential([Convolution((11, 11, 64), filter_init=GaussianInit(std=0.01),
bias_init=init,
activation=Rectlin(), padding=3, strides=4),
Pooling((3, 3), strides=2),
Convolution((5, 5, 192), filter_init=GaussianInit(std=0.01),
bias_init=init,
activation=Rectlin(), padding=2),
Pooling((3, 3), strides=2),
Convolution((3, 3, 384), filter_init=GaussianInit(std=0.03),
bias_init=init,
activation=Rectlin(), padding=1),
Convolution((3, 3, 256), filter_init=GaussianInit(std=0.03),
bias_init=init,
activation=Rectlin(), padding=1),
Convolution((3, 3, 256), filter_init=GaussianInit(std=0.03),
bias_init=init,