def test_linear_keep_batch_axis():
feature_axis = ng.make_axis(1, name='A')
batch_axis = ng.make_axis(2, name='N')
x = ng.placeholder([batch_axis])
linear = Linear(axes=feature_axis,
keep_axes=[batch_axis],
init=UniformInit(1.0, 1.0))(x)
assert linear.axes == ng.make_axes([feature_axis, batch_axis])
def test_linear_axes_nout():
feature_axis = ng.make_axis(1, name='A')
batch_axis = ng.make_axis(2, name='N')
x = ng.placeholder([feature_axis, batch_axis])
linear = Linear(nout=3, init=UniformInit(1.0, 1.0))(x)
assert feature_axis not in linear.axes
assert batch_axis in linear.axes
assert linear.axes.batch_axis().length == 2
assert linear.axes.sample_axes().lengths == (3, )
def test_linear_zeros(input_placeholder, output_size):
# basic sanity check with 0 weights random inputs
x = np.random.random(input_placeholder.axes.lengths)
layer = Linear(nout=output_size, init=UniformInit(0.0, 0.0))
with ExecutorFactory() as ex:
if ex.transformer.transformer_name == 'hetr':
pytest.xfail("hetr fork-safe issue on mac")
comp = ex.executor(layer(input_placeholder), input_placeholder)
output_values = comp(x)
assert np.min(output_values) == 0.0 and np.max(output_values) == 0.0
def test_linear_W_axes_nout():
feature_axis = ng.make_axis(1, name='A')
batch_axis = ng.make_axis(2, name='N')
x = ng.placeholder([feature_axis, batch_axis])
linear = Linear(nout=3, init=UniformInit(1.0, 1.0))
linear(x)
assert linear.W.axes.batch_axis() is None
assert feature_axis in linear.W.axes
assert len(linear.W.axes - feature_axis) == 1
assert (linear.W.axes - feature_axis)[0].length == 3
def test_linear_ones(input_size, input_placeholder, output_size):
# basic sanity check with all ones on the inputs and weights, check that
# each row in output is the sum of the weights for that output this check
# will confirm that the correct number of operations is being run
x = np.ones(input_placeholder.axes.lengths)
layer = Linear(nout=output_size, init=UniformInit(1.0, 1.0))
with ExecutorFactory() as ex:
if ex.transformer.transformer_name == 'hetr':
pytest.xfail("hetr fork-safe issue on mac")
out = layer(input_placeholder)
comp = ex.executor([out, layer.W], input_placeholder)
output_values, w = comp(x)
ng.testing.assert_allclose(np.ones(out.axes.lengths) * input_size,
output_values,
atol=0.0,
rtol=0.0)
def __init__(self,
branch_units,
activation=Rectlin(),
bias_init=UniformInit(low=-0.08, high=0.08),
filter_init=XavierInit()):
(p1, p2, p3, p4) = branch_units
self.branch_1 = Convolution((1, 1, p1[0]),
activation=activation,
bias_init=bias_init,
filter_init=filter_init)
self.branch_2 = [
Convolution((1, 1, p2[0]),
activation=activation,
bias_init=bias_init,
filter_init=filter_init),
Convolution((3, 3, p2[1]),
activation=activation,
bias_init=bias_init,
filter_init=filter_init,
padding=1)
]
self.branch_3 = [
Convolution((1, 1, p3[0]),
activation=activation,
bias_init=bias_init,
filter_init=filter_init),
Convolution((5, 5, p3[1]),
activation=activation,
bias_init=bias_init,
filter_init=filter_init,
padding=2)
]
self.branch_4 = [
Pooling(pool_shape=(3, 3), padding=1, strides=1, pool_type="max"),
Convolution((1, 1, p3[0]),
activation=activation,
bias_init=bias_init,
filter_init=filter_init)
]
'data': X_train,
'axes': ('N', 'C', 'H', 'W')
},
'label': {
'data': y_train,
'axes': ('N', )
}
}
train_set = ArrayIterator(train_data,
batch_size=args.batch_size,
total_iterations=args.num_iterations)
inputs = train_set.make_placeholders(include_iteration=True)
ax.Y.length = 1000 # number of outputs of last layer.
# weight initialization
bias_init = UniformInit(low=-0.08, high=0.08)
class Inception(Sequential):
def __init__(self,
branch_units,
activation=Rectlin(),
bias_init=UniformInit(low=-0.08, high=0.08),
filter_init=XavierInit()):
(p1, p2, p3, p4) = branch_units
self.branch_1 = Convolution((1, 1, p1[0]),
activation=activation,
bias_init=bias_init,
filter_init=filter_init)
parser.add_argument('--recurrent_units', type=int,
help="Number of recurrent units in the network",
default=256)
parser.set_defaults(num_iterations=20000)
args = parser.parse_args()
use_embedding = args.use_embedding
recurrent_units = args.recurrent_units
batch_size = args.batch_size
seq_len = args.seq_len
num_iterations = args.num_iterations
# Ratio of the text to use for training
train_ratio = 0.95
# Define initialization method of neurons in the network
init_uni = UniformInit(-0.1, 0.1)
# Create the object that includes the sample text
shakes = Shakespeare(train_split=train_ratio)
# Build an iterator that gets seq_len long chunks of characters
# Stride is by how many characters the window moves in each step
# if stride is set to seq_len, windows are non-overlapping
stride = seq_len // 8
shakes_data_train = {'X': np.copy(shakes.train), 'y': np.roll(np.copy(shakes.train), shift=-1)}
shakes_data_test = {'X': np.copy(shakes.test), 'y': np.roll(np.copy(shakes.test), shift=-1)}
shakes_train = SequentialArrayIterator(data_arrays=shakes_data_train,
total_iterations=num_iterations,
time_steps=seq_len, batch_size=batch_size, stride=stride,
include_iteration=True, tgt_key='y', shuffle=False)
shakes_test = SequentialArrayIterator(data_arrays=shakes_data_test,
inputs = train_set.make_placeholders()
ax.Y.length = 10
######################
# Model specification
def cifar_mean_subtract(x):
bgr_mean = ng.persistent_tensor(
axes=x.axes.find_by_name('C'),
initial_value=np.array([104., 119., 127.]))
return (x - bgr_mean) / 255.
seq1 = Sequential([Preprocess(functor=cifar_mean_subtract),
Affine(nout=200, weight_init=UniformInit(-0.1, 0.1), activation=Rectlin()),
Affine(axes=ax.Y, weight_init=UniformInit(-0.1, 0.1), activation=Softmax())])
optimizer = GradientDescentMomentum(0.1, 0.9)
train_prob = seq1(inputs['image'])
train_loss = ng.cross_entropy_multi(train_prob, ng.one_hot(inputs['label'], axis=ax.Y))
batch_cost = ng.sequential([optimizer(train_loss), ng.mean(train_loss, out_axes=())])
train_outputs = dict(batch_cost=batch_cost)
with Layer.inference_mode_on():
inference_prob = seq1(inputs['image'])
eval_loss = ng.cross_entropy_multi(inference_prob, ng.one_hot(inputs['label'], axis=ax.Y))
eval_outputs = dict(results=inference_prob, cross_ent_loss=eval_loss)
# Now bind the computations we are interested in
with closing(ngt.make_transformer()) as transformer:
def test_linear_invalid_batch_axes():
with pytest.raises(ValueError):
Linear(axes=ng.make_axis(1, name='N'), init=UniformInit(1.0, 1.0))
def test_linear_invalid_shadow_axes():
with pytest.raises(ValueError):
Linear(axes=make_shadow_axis(ng.make_axis(1, name='A')),
init=UniformInit(1.0, 1.0))
def test_linear_accepts_axes_axis():
""" Ensure that Linear.__init__ accepts an Axis as axes """
Linear(axes=ng.make_axis(1), init=UniformInit(1.0, 1.0))