def create_network():
'''
Define 3D convolutional network
'''
# Define for weight initialization
g1 = GaussianInit(mean=0., var=0.01)
g5 = GaussianInit(mean=0., var=0.005)
c0 = ConstantInit(val=0.)
c1 = ConstantInit(val=1.)
ax.Y.length = 101
padding = {'D': 1, 'H': 1, 'W': 1, 'C': 0}
strides = {'D': 2, 'H': 2, 'W': 2, 'C': 1}
layers = [
Convolution((3, 3, 3, 64),
padding=padding,
filter_init=g1,
bias_init=c0,
activation=Rectlin()),
Pooling((1, 2, 2), strides={
'D': 1,
'H': 2,
'W': 2,
'C': 1
}),
Convolution((3, 3, 3, 128),
padding=padding,
filter_init=g1,
bias_init=c1,
activation=Rectlin()),
Pooling((2, 2, 2), strides=strides),
Convolution((3, 3, 3, 256),
padding=padding,
filter_init=g1,
bias_init=c1,
activation=Rectlin()),
Pooling((2, 2, 2), strides=strides),
Convolution((3, 3, 3, 256),
padding=padding,
filter_init=g1,
bias_init=c1,
activation=Rectlin()),
Pooling((2, 2, 2), strides=strides),
Convolution((3, 3, 3, 256),
padding=padding,
filter_init=g1,
bias_init=c1,
activation=Rectlin()),
Pooling((2, 2, 2), strides=strides),
Affine(nout=2048, weight_init=g5, bias_init=c1, activation=Rectlin()),
Dropout(keep=0.5),
Affine(nout=2048, weight_init=g5, bias_init=c1, activation=Rectlin()),
Dropout(keep=0.5),
Affine(axes=ax.Y, weight_init=g1, bias_init=c0, activation=Softmax())
]
return Sequential(layers)
def __init__(self,
nfilters,
filter_width,
str_w,
nbands,
depth,
hidden_size,
batch_norm=False,
batch_norm_affine=False,
batch_norm_conv=False,
to_ctc=True):
self.to_ctc = to_ctc
# Initializers
gauss = GaussianInit(0.01)
glorot = GlorotInit()
# 1D Convolution layer
padding = dict(pad_h=0, pad_w=filter_width // 2, pad_d=0)
strides = dict(str_h=1, str_w=str_w, str_d=1)
dilation = dict(dil_d=1, dil_h=1, dil_w=1)
conv_layer = Convolution((nbands, filter_width, nfilters),
gauss,
bias_init=ConstantInit(0),
padding=padding,
strides=strides,
dilation=dilation,
activation=Rectlin(),
batch_norm=batch_norm_conv)
# Add BiRNN layers
deep_birnn = DeepBiRNN(depth,
hidden_size,
glorot,
Rectlinclip(),
batch_norm=batch_norm)
# Add a single affine layer
fc = Affine(nout=hidden_size,
weight_init=glorot,
activation=Rectlinclip(),
batch_norm=batch_norm_affine)
# Add the final affine layer
# Softmax output is computed within the CTC cost function, so no activation is needed here.
if self.to_ctc is False:
activation = Softmax()
else:
activation = None
final = Affine(axes=ax.Y, weight_init=glorot, activation=activation)
layers = [conv_layer, deep_birnn, fc, final]
super(Deepspeech, self).__init__(layers=layers)
def make_layers(use_large, vocab_size):
if use_large:
init = GaussianInit(0., 0.02)
else:
init = GaussianInit(0., 0.05)
layers = []
layers.append(make_embedding_layer(vocab_size))
layers.append(lambda op: ng.map_roles(op, {'REC': 'W', 'F': 'C'}))
kernel_sizes = [7, 7, 3, 3, 3, 3]
pool_layer_idxs = [0, 1, 5]
conv_nout = 1024 if use_large else 256
fc_nout = 2048 if use_large else 1024
for i in range(6):
conv_layer = Convolution(
**conv_params(kernel_sizes[i], conv_nout, init))
layers.append(conv_layer)
if i in pool_layer_idxs:
pool_layer = Pooling(pool_shape=(3, ), strides=3)
layers.append(pool_layer)
layers.append(
Affine(nout=fc_nout,
weight_init=init,
bias_init=ConstantInit(0.),
activation=Rectlin()))
layers.append(Dropout(keep=0.5))
layers.append(
Affine(nout=fc_nout,
weight_init=init,
bias_init=ConstantInit(0.),
activation=Rectlin()))
layers.append(Dropout(keep=0.5))
layers.append(
Affine(axes=(ax.Y, ),
weight_init=init,
bias_init=ConstantInit(0.),
activation=Softmax()))
return layers
def test_dilated_conv(dilation):
"""Test that the dilated convolution layer output matches expected. This test compares
the maximum output value to an expected max output value. The expected value is computed
based on the dilation parameter. The test also checks that the output size matches the
expected size based on the dilaton parameter value."""
image_size = 3
batch_size = 1
init_val = 0.1
conv_size = 3
pad = 3
N_filters = 1
image_channels = 3
model = Sequential([
Convolution((conv_size, conv_size, N_filters),
filter_init=ConstantInit(val=init_val),
padding=pad,
dilation=dilation)
])
X = np.ones(shape=(batch_size, 3, image_size,
image_size)) # Create dummy image
data = {'image': X, 'iteration': 1}
data_size = OrderedDict([('N', batch_size), ('C', 3), ('H', image_size),
('W', image_size)])
ax = [
ng.make_axis(length=data_size[k], name=k)
for k in list(data_size.keys())
]
p_axes = ng.make_axes(ax)
named_inputs = {'image': ng.placeholder(p_axes)}
outputs = model(named_inputs['image'])
named_outputs = {outputs.name: outputs}
with closing(ngt.make_transformer()) as transformer:
m = make_bound_computation(transformer, named_outputs, named_inputs)
output = m(data)[list(m(data).keys())[0]]
filter_size = dilation * (conv_size -
1) + 1 # Compute expected filter size
# Compute the expected output size based on convolution parameters
out_size = (image_size + 2 * pad - filter_size) + 1
filt_tmp = np.zeros(filter_size)
filt_tmp[0::dilation] = 1
# max overlap between dilated filter and image (in 1-d)
max_overlap = int(np.min([filter_size, image_size]))
exp_max_output = init_val * image_channels * (np.sum(
filt_tmp[0:max_overlap]))**2
# Expected max output changes for different dilation parameter values#
assert int(10 * np.max(output)) == int(10 * exp_max_output), \
("Dilated conv max outputs do not match expected: "
"{} != {}").format(np.max(output),
init_val * conv_size * ((image_size - (dilation - 1))**2))
assert np.shape(output) == (batch_size, N_filters, out_size, out_size), \
("Dilated conv output is not expected size: "
"{} != {}").format(np.shape(output), (batch_size, N_filters, out_size, out_size))
def make_embedding_layer(vocab_size):
vectors = []
vectors.append(np.zeros((1, vocab_size)))
vectors.append(np.eye(vocab_size))
vectors = np.concatenate(vectors)
embed_init = ConstantInit(vectors)
embed_layer = LookupTable(vocab_size + 1,
vocab_size,
embed_init,
update=False,
pad_idx=0)
return embed_layer
def __init__(self):
super(ConvolutionLayer, self).__init__()
self.layer = Convolution({
'T': 1,
'R': 1,
'S': 1,
'K': 1
},
ConstantInit(0.0), {
'str_d': 1,
'str_h': 1,
'str_w': 1
}, {
'pad_d': 0,
'pad_h': 0,
'pad_w': 0
}, {
'dil_d': 0,
'dil_h': 0,
'dil_w': 0
},
bias_init=ConstantInit(0.0),
batch_norm=True)
def test_deconv():
"""
basic test of deconv fprop.
ngraph/tests/test_conv.py tests ng.deconvolution bprop
"""
# filter params
R, S = 5, 5
fshape = (1, R, S, 1) # TRSK
strides = 2
filter_val_nz = np.arange(1, R * S + 1).reshape(R, S)
filter_val = np.zeros(fshape)
filter_val[0, :, :, 0] = filter_val_nz
deconv = Deconvolution(fshape,
filter_init=ConstantInit(filter_val),
strides=strides,
padding=0,
dilation=1)
N = ng.make_axis(name='N', length=1) # batch
image_shape = (1, 1, 8, 8) # CDHW
image_axes = ng.make_axes([ng.make_axis(name=nm, length=l)
for nm, l in zip('CDHW', image_shape)])
image_axes |= N
image = ng.placeholder(axes=image_axes)
output = deconv(image)
with closing(ngt.make_transformer()) as transformer:
comp = transformer.computation(output, image)
input_val = np.zeros(image_shape + (N.length, ), dtype=float)
input_val[0, 0, 0, 0, 0] = 1
input_val[0, 0, 5, 5, 0] = 1
input_val[0, 0, 7, 7, 0] = 1
result = comp(input_val)
feature_map = np.squeeze(result)
assert (feature_map[:5, :5] == filter_val_nz).all()
result2 = filter_val_nz.copy()
result2[-1, -1] = 26
assert (feature_map[10:15, 10:15] == result2).all()
result3 = filter_val_nz.copy()
result3[0, 0] = 26
assert (feature_map[-5:, -5:] == result3).all()
def test_change_recurrent_axis_length(recurrent_layer_cls, batch_size,
sequence_length, input_size,
hidden_size):
"""
Recurrent layer support for changing REC axis length
(needed by seq2seq inference)
"""
# create three identical recurrent layers with same weights
W_input_val = np.random.normal(size=(hidden_size, input_size))
W_recur_val = np.random.normal(size=(hidden_size, hidden_size))
rec1 = recurrent_layer_cls(nout=hidden_size,
init=ConstantInit(W_input_val),
init_inner=ConstantInit(W_recur_val),
activation=Tanh())
rec2 = recurrent_layer_cls(nout=hidden_size,
init=ConstantInit(W_input_val),
init_inner=ConstantInit(W_recur_val),
activation=Tanh())
rec3 = recurrent_layer_cls(nout=hidden_size,
init=ConstantInit(W_input_val),
init_inner=ConstantInit(W_recur_val),
activation=Tanh())
# create input placeholders and values
# sequence length greater than 1
N = ng.make_axis(length=batch_size, name='N')
REC = ng.make_axis(length=sequence_length, name='REC')
M = ng.make_axis(length=input_size, name='M')
xn_axes = ng.make_axes([M, REC, N])
xn = ng.placeholder(axes=xn_axes)
xn_val = np.random.normal(size=(input_size, sequence_length, batch_size))
# sequence length 1
REC1 = ng.make_axis(length=1, name='REC')
x1_axes = ng.make_axes([M, REC1, N])
x1 = ng.placeholder(axes=x1_axes)
x1_val = np.random.normal(size=(input_size, 1, batch_size))
# check results of switching REC axis of a layer's input
# computations switching REC axis
y1_n = rec1(xn)
y1_1 = rec1(x1)
# check against not switching
y2_n = rec2(xn)
y3_1 = rec3(x1)
with ExecutorFactory() as ex:
y1_n_comp = ex.executor(y1_n, xn)
y1_1_comp = ex.executor(y1_1, x1)
y2_n_comp = ex.executor(y2_n, xn)
y3_1_comp = ex.executor(y3_1, x1)
ng.testing.assert_allclose(y1_n_comp(xn_val), y2_n_comp(xn_val))
ng.testing.assert_allclose(y1_1_comp(x1_val), y3_1_comp(x1_val))
parser.add_argument('--dim', type=int, default=512,
help='Hidden layer dimension for the model')
parser.add_argument('--num_critic', type=int, default=5,
help='Number of discriminator iterations per generator iteration')
parser.add_argument('--plot_dir', type=str, default='WGAN_Toy_Plots',
help='Directory name to save the results')
args = parser.parse_args()
np.random.seed(args.rng_seed)
dim = args.dim
if not os.path.isdir(args.plot_dir):
os.makedirs(args.plot_dir)
w_init = KaimingInit()
b_init = ConstantInit()
def make_optimizer(name=None, weight_clip_value=None):
optimizer = Adam(learning_rate=1e-4, beta_1=0.5, beta_2=0.9,
epsilon=1e-8, weight_clip_value=weight_clip_value)
return optimizer
def make_generator(out_axis):
generator = [Affine(nout=dim, weight_init=w_init, bias_init=b_init, activation=Rectlin()),
Affine(nout=dim, weight_init=w_init, bias_init=b_init, activation=Rectlin()),
Affine(nout=dim, weight_init=w_init, bias_init=b_init, activation=Rectlin()),
Affine(axes=out_axis, weight_init=w_init, bias_init=b_init, activation=None)]
def affine_layer(h_dim, activation, name):
return Affine(nout=h_dim,
activation=activation,
weight_init=GaussianInit(std=1.0),
bias_init=ConstantInit(val=0.0),
name=name)
def conv_params(ksize, nout, init):
return dict(filter_shape=(ksize, nout),
activation=Rectlin(),
filter_init=init,
bias_init=ConstantInit(0.))
def __init__(self):
super(LSTMLayer, self).__init__()
self.layer = LSTM(nout=16,
init=ConstantInit(0.0),
activation=Tanh(),
gate_activation=Tanh())
def __init__(self):
super(AffineLayer, self).__init__()
self.layer = Affine(ConstantInit(0.0),
nout=10,
bias_init=ConstantInit(0.0),
activation=Rectlin())
def __init__(self):
super(LinearLayer, self).__init__()
self.layer = Linear(ConstantInit(0.0), nout=10)
def __init__(self):
super(BiRNNLayer, self).__init__()
self.layer = BiRNN(nout=16, init=ConstantInit(0.0), activation=Tanh())
