def test_variable():
input_axes = ng.make_axes([ng.make_axis(10), ng.make_axis(3)])
var = ng.variable(axes=input_axes)
assign_val = np.random.rand(10, 3)
var_assign = ng.AssignOp(tensor=var, val=assign_val)
var_seq = ng.sequential([var_assign, var])
var_comp = ng.computation(var_seq, "all")
results = dict()
weight_saver = Saver()
with closing(ngt.make_transformer()) as transformer:
var_func = transformer.add_computation(var_comp)
weight_saver.setup_save(transformer=transformer, computation=var_comp)
results['saved'] = var_func().copy()
weight_saver.save(filename="test_variable")
reassign_val = np.random.rand(10, 3)
var_reassign = ng.AssignOp(tensor=var, val=reassign_val)
var_recomp = ng.computation(var_reassign, "all")
var_read = ng.computation(var, "all")
with closing(ngt.make_transformer()) as restore_transformer:
var_recompfunc = restore_transformer.add_computation(var_recomp)
weight_saver.setup_restore(transformer=restore_transformer,
computation=var_recomp,
filename="test_variable")
var_readfunc = restore_transformer.add_computation(var_read)
var_recompfunc()
results['reassigned'] = var_readfunc().copy()
weight_saver.restore()
results['restored'] = var_readfunc().copy()
os.remove("test_variable.npz")
assert np.allclose(results['saved'], assign_val, atol=0)
assert np.allclose(results['reassigned'], reassign_val, atol=0)
assert np.allclose(results['saved'], results['restored'], atol=0)
def test_conv1d(transformer_factory, filter_width, num_filters, strides,
padding, time_steps, feature_dimension, batch_size):
dilation = 1 # reference conv does not support dilation
F = ng.make_axis(name='F', length=feature_dimension)
REC = ng.make_axis(name='REC', length=time_steps)
N = ng.make_axis(name='N', length=batch_size)
in_axes = ng.make_axes([F, REC, N])
inputs = ng.placeholder(axes=in_axes)
input_vals = np.random.randn(*in_axes.lengths)
filter_init = GaussianInit()
conv1d = Convolution((filter_width, num_filters),
filter_init,
strides=strides,
padding=padding,
dilation=dilation,
bias_init=None,
activation=Rectlin(),
batch_norm=None)
result_op = conv1d(inputs, channel_axes='F', spatial_axes={'W': 'REC'})
with closing(ngt.make_transformer()) as transformer:
result_comp = transformer.add_computation(
ng.computation(result_op, inputs))
filter_vals = transformer.add_computation(ng.computation(
conv1d.conv.W))()
result_ng = result_comp(input_vals)
result_np = np.squeeze(
reference_conv1d(input_vals, filter_vals,
lambda x: np.maximum(0, x)))
ng.testing.assert_allclose(result_ng, result_np)
def test_deconv():
"""
basic test of deconv fprop.
ngraph/tests/test_conv.py tests ng.deconvolution bprop
"""
# filter params
R, S = 5, 5
fshape = (R, S, 1)
strides = 2
filter_val_nz = np.arange(1, R * S + 1).reshape(R, S)
filter_val = np.zeros(fshape)
filter_val[:, :, 0] = filter_val_nz
deconv = Deconvolution(fshape,
filter_init=ConstantInit(filter_val),
strides=strides,
padding=0,
dilation=1)
N = ng.make_axis(name='N', length=1) # batch
image_shape = (1, 8, 8) # CHW
image_axes = ng.make_axes(
[ng.make_axis(name=nm, length=l) for nm, l in zip('CHW', image_shape)])
image_axes |= N
image = ng.placeholder(axes=image_axes)
output = deconv(image)
with closing(ngt.make_transformer()) as transformer:
comp = transformer.add_computation(ng.computation(output, image))
input_val = np.zeros(image_shape + (N.length, ), dtype=float)
input_val[0, 0, 0] = 1
input_val[0, 5, 5] = 1
input_val[0, 7, 7] = 1
result = comp(input_val)
feature_map = np.squeeze(result)
assert (feature_map[:5, :5] == filter_val_nz).all()
result2 = filter_val_nz.copy()
result2[-1, -1] = 26
assert (feature_map[10:15, 10:15] == result2).all()
result3 = filter_val_nz.copy()
result3[0, 0] = 26
assert (feature_map[-5:, -5:] == result3).all()
def test_persistent_tensor():
input_axes = ng.make_axes([ng.make_axis(10), ng.make_axis(3)])
bgr = ng.persistent_tensor(axes=input_axes,
initial_value=np.array([113.9, 123.0, 125.3]))
bgr_comp = ng.computation(bgr, "all")
results = dict()
weight_saver = Saver()
with closing(ngt.make_transformer()) as transformer:
bgr_func = transformer.add_computation(bgr_comp)
weight_saver.setup_save(transformer=transformer, computation=bgr_comp)
results['saved'] = bgr_func().copy()
weight_saver.save(filename="test_persistent_tensor")
with closing(ngt.make_transformer()) as restore_transformer:
bgr_refunc = restore_transformer.add_computation(bgr_comp)
weight_saver.setup_restore(transformer=restore_transformer,
computation=bgr_comp,
filename="test_persistent_tensor")
weight_saver.restore()
results['restored'] = bgr_refunc().copy()
os.remove("test_persistent_tensor.npz")
assert np.allclose(results['saved'], results['restored'], atol=0)
learning_rate_policy = {'name': 'schedule',
'schedule': schedule,
'gamma': 0.95,
'base_lr': 0.01}
optimizer = GradientDescentMomentum(learning_rate=learning_rate_policy,
iteration=inputs['iteration'])
# Define the loss function (Cross entropy loss)
# Note that we convert the integer values of input['y'] to one hot here
fwd_prop = seq1(inputs['X'])
train_loss = ng.cross_entropy_multi(fwd_prop,
ng.one_hot(inputs['y'], axis=out_axis),
usebits=True)
# Train cost computation
batch_cost = ng.sequential([optimizer(train_loss), ng.mean(train_loss, out_axes=())])
train_computation = ng.computation([batch_cost, fwd_prop], "all")
train_outputs = dict(batch_cost=batch_cost)
# Forward prop of evaluation set
# Required for correct functioning of batch norm and dropout layers during inference mode
with Layer.inference_mode_on():
inference_prop = seq1(inputs['X'])
eval_loss = ng.cross_entropy_multi(inference_prop,
ng.one_hot(inputs['y'], axis=out_axis),
usebits=True)
eval_computation = ng.computation([ng.mean(eval_loss, out_axes=()), inference_prop], "all")
eval_outputs = dict(x_ent_loss=eval_loss)
# Computation for text generation - this is pure inference (fwd prop)
gen_computation = ng.computation(inference_prop, "all")
# Cost calculation
batch_cost = ng.sequential(
[optimizer(train_loss),
ng.mean(train_loss, out_axes=())])
train_outputs = dict(batch_cost=batch_cost)
# Forward prop of test set
# Required for correct functioning of batch norm and dropout layers during inference mode
with Layer.inference_mode_on():
inference_prob = seq1(inputs['X'])
eval_loss = ng.squared_L2(inference_prob - inputs['y'])
eval_outputs = dict(l2_loss=eval_loss)
# Define computations
print('Start training')
eval_computation = ng.computation(inference_prob, "all")
with closing(ngt.make_transformer()) as transformer:
# transformer = ngt.make_transformer()
train_computation = make_bound_computation(transformer, train_outputs,
inputs)
loss_computation = make_bound_computation(transformer, eval_outputs,
inputs)
eval_function = transformer.add_computation(eval_computation)
# Printout interval of the validation set loss during training
iter_interval = num_iterations // 10
cbs = make_default_callbacks(transformer=transformer,
output_file=args.output_file,
frequency=iter_interval,
train_computation=train_computation,
optimizer = GradientDescentMomentum(learning_rate=learning_rate_policy,
momentum_coef=momentum_coef,
wdecay=wdecay,
nesterov=False,
iteration=input_ph['iteration'])
label_indices = input_ph['label']
# Make a prediction
prediction = resnet(input_ph['image'])
# Calculate loss
train_loss = ng.cross_entropy_multi(prediction,
ng.one_hot(label_indices, axis=ax.Y))
# Average loss over the batch
batch_cost = ng.sequential(
[optimizer(train_loss),
ng.mean(train_loss, out_axes=())])
train_computation = ng.computation(batch_cost, "all")
# Instantiate the Saver object to save weights
weight_saver = Saver()
# Inference
with Layer.inference_mode_on():
# Doing inference
inference_prob = resnet(input_ph['image'])
eval_loss = ng.cross_entropy_multi(inference_prob,
ng.one_hot(label_indices, axis=ax.Y))
# Computation for inference
eval_computation = ng.computation([inference_prob, eval_loss], "all")
# Doing inference
if (args.inference is not None):
# inference graph
with Layer.inference_mode_on():
enc_out_inference = enc(one_hot_enc_out)
# Create decoder placeholders
axes = one_hot_dec_out.axes
axes = axes - axes.recurrent_axis() + ng.make_axis(length=1, name="REC")
decoder_input_inference = ng.placeholder(axes, name="input")
decoder_state_inference = ng.placeholder(enc_out_inference.axes,
name="state")
dec_out_inference = dec(decoder_input_inference,
init_state=decoder_state_inference)
inference_out = linear(dec_out_inference)
encoder_computation = ng.computation(enc_out_inference, inputs["inp_txt"])
decoder_computation = ng.computation([inference_out, dec_out_inference],
decoder_input_inference,
decoder_state_inference)
######################
# Train Loop
# Now bind the computations we are interested in
with closing(ngt.make_transformer()) as transformer:
# training computations
train_computation = make_bound_computation(transformer, train_outputs,
inputs)
loss_computation = make_bound_computation(transformer, loss_outputs,
inputs)