def define_recurrent_layers(out_axes=None,
celltype='RNN',
recurrent_units=[32],
init=GlorotInit(),
return_sequence=True):
layers = []
for e, i in enumerate(recurrent_units):
layer_return_sequence = e < len(recurrent_units) - 1 or return_sequence
if celltype == 'RNN':
layers.append(
Recurrent(nout=i,
init=init,
backward=False,
activation=Tanh(),
return_sequence=layer_return_sequence))
elif celltype == 'LSTM':
layers.append(
LSTM(nout=i,
init=init,
backward=False,
activation=Tanh(),
gate_activation=Logistic(),
return_sequence=layer_return_sequence))
if out_axes is not None:
affine_layer = Affine(weight_init=init,
bias_init=init,
activation=Identity(),
axes=out_axes)
layers.append(affine_layer)
return layers
def __init__(self,
nfilters,
filter_width,
str_w,
nbands,
depth,
hidden_size,
batch_norm=False,
batch_norm_affine=False,
batch_norm_conv=False,
to_ctc=True):
self.to_ctc = to_ctc
# Initializers
gauss = GaussianInit(0.01)
glorot = GlorotInit()
# 1D Convolution layer
padding = dict(pad_h=0, pad_w=filter_width // 2, pad_d=0)
strides = dict(str_h=1, str_w=str_w, str_d=1)
dilation = dict(dil_d=1, dil_h=1, dil_w=1)
conv_layer = Convolution((nbands, filter_width, nfilters),
gauss,
bias_init=ConstantInit(0),
padding=padding,
strides=strides,
dilation=dilation,
activation=Rectlin(),
batch_norm=batch_norm_conv)
# Add BiRNN layers
deep_birnn = DeepBiRNN(depth,
hidden_size,
glorot,
Rectlinclip(),
batch_norm=batch_norm)
# Add a single affine layer
fc = Affine(nout=hidden_size,
weight_init=glorot,
activation=Rectlinclip(),
batch_norm=batch_norm_affine)
# Add the final affine layer
# Softmax output is computed within the CTC cost function, so no activation is needed here.
if self.to_ctc is False:
activation = Softmax()
else:
activation = None
final = Affine(axes=ax.Y, weight_init=glorot, activation=activation)
layers = [conv_layer, deep_birnn, fc, final]
super(Deepspeech, self).__init__(layers=layers)
def define_model(out_axes=None,
celltype='RNN',
recurrent_units=[32],
init=GlorotInit(),
return_sequence=True):
layers = define_recurrent_layers(out_axes=out_axes,
celltype=celltype,
recurrent_units=recurrent_units,
init=init,
return_sequence=return_sequence)
return Sequential(layers)
def __init__(self,
celltype='RNN',
recurrent_units=[32],
init=GlorotInit(),
bottleneck=False,
*args,
**kwargs):
layers = define_recurrent_layers(celltype=celltype,
recurrent_units=recurrent_units,
init=init,
return_sequence=True)
super(RecurrentEncoder, self).__init__(layers, *args, **kwargs)
self.bottleneck = bottleneck
def __init__(self,
out_axes=None,
celltype='RNN',
recurrent_units=[32],
init=GlorotInit(),
*args,
**kwargs):
layers = define_recurrent_layers(out_axes=out_axes,
celltype=celltype,
recurrent_units=recurrent_units,
init=init,
return_sequence=True)
super(RecurrentDecoder, self).__init__(layers, *args, **kwargs)
self.celltype = celltype
self.recurrent_units = recurrent_units
from datetime import datetime
out_folder = os.path.join(args.results_dir, "results-turbofan-LSTM-{}".format(datetime.strftime(datetime.now(), "%Y-%m-%d_%H%M%S")))
if not os.path.exists(out_folder):
os.mkdir(out_folder)
# Plot the inference / generation results
if args.save_plots:
try:
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
except ImportError:
args.save_plots = False
# Define initialization
init_uni = GlorotInit()
batch_size = args.batch_size
seq_len = args.seq_len
no_epochs = args.epochs
output_dim = 1
dataset = TurboFan(data_dir="../../data/", T=args.seq_len, skip=args.skip, max_rul_predictable=130)
feature_dim = dataset.n_features
if args.save_plots:
dataset.plot_sample(out_folder, trajectory_id=10)
# Build input data iterables
# Yields an input array of Shape (batch_size, seq_len, input_feature_dim)
train_samples = len(dataset.train['X']['data'])
hidden_size = 150
gradient_clip_value = 15
embed_size = 300
params_dict = {}
params_dict['batch_size'] = args.batch_size_squad
params_dict['embed_size'] = 300
params_dict['pad_idx'] = 0
params_dict['hs'] = hidden_size
params_dict['glove_dim'] = 300
params_dict['iter_interval'] = 8000
params_dict['num_iterations'] = 500000
params_dict['ax'] = ax
# Initialzer
init = GlorotInit()
params_dict['init'] = init
validate_existing_directory(args.data_path)
path_gen = sanitize_path(args.data_path)
path_gen = os.path.join(path_gen + "/")
file_name_dict = {}
file_name_dict['train_para_ids'] = 'train.ids.context'
file_name_dict['train_ques_ids'] = 'train.ids.question'
file_name_dict['train_answer'] = 'train.span'
file_name_dict['val_para_ids'] = 'dev.ids.context'
file_name_dict['val_ques_ids'] = 'dev.ids.question'
file_name_dict['val_ans'] = 'dev.span'
file_name_dict['vocab_file'] = 'vocab.dat'