def call(self, inputs, mask=None, **kwargs):
input_fw = inputs
input_bw = inputs
for i in range(self.layers):
output_fw = self.fw_lstm[i](input_fw)
output_bw = self.bw_lstm[i](input_bw)
output_bw = Lambda(lambda x: K.reverse(x, 1),
mask=lambda inputs, mask: mask)(output_bw)
if i >= self.layers - self.res_layers:
output_fw += input_fw
output_bw += input_bw
input_fw = output_fw
input_bw = output_bw
output_fw = input_fw
output_bw = input_bw
if self.merge_mode == "fw":
output = output_fw
elif self.merge_mode == "bw":
output = output_bw
elif self.merge_mode == 'concat':
output = K.concatenate([output_fw, output_bw])
elif self.merge_mode == 'sum':
output = output_fw + output_bw
elif self.merge_mode == 'ave':
output = (output_fw + output_bw) / 2
elif self.merge_mode == 'mul':
output = output_fw * output_bw
elif self.merge_mode is None:
output = [output_fw, output_bw]
return output
def _defun_gru_call(self, inputs, initial_state, training):
# Use the new defun approach for backend implementation swap.
# Note that different implementations need to have same function
# signature, eg, the tensor parameters need to have same shape and dtypes.
if self.go_backwards:
# Reverse time axis.
inputs = K.reverse(inputs, 0 if self.time_major else 1)
self.reset_dropout_mask()
dropout_mask = self.get_dropout_mask_for_cell(inputs,
training,
count=3)
if dropout_mask is not None:
inputs *= dropout_mask[0]
if ops.executing_eagerly_outside_functions():
# Under eager context, the device placement is already known. Prefer the
# GPU implementation when GPU is available.
if context.num_gpus() > 0:
last_output, outputs, new_h, runtime = cudnn_gru(
inputs=inputs,
init_h=initial_state[0],
kernel=self.cell.kernel,
recurrent_kernel=self.cell.recurrent_kernel,
bias=self.cell.bias,
time_major=self.time_major)
else:
last_output, outputs, new_h, runtime = standard_gru(
inputs=inputs,
init_h=initial_state[0],
kernel=self.cell.kernel,
recurrent_kernel=self.cell.recurrent_kernel,
bias=self.cell.bias,
activation=self.activation,
recurrent_activation=self.recurrent_activation,
time_major=self.time_major)
else:
api_name = 'gru_' + str(uuid.uuid4())
defun_standard_gru = _generate_defun_backend(
api_name, _CPU_DEVICE_NAME, standard_gru)
defun_cudnn_gru = _generate_defun_backend(api_name,
_GPU_DEVICE_NAME,
cudnn_gru)
# Call the normal GRU impl and register the CuDNN impl function. The
# grappler will kick in during session execution to optimize the graph.
last_output, outputs, new_h, runtime = defun_standard_gru(
inputs=inputs,
init_h=initial_state[0],
kernel=self.cell.kernel,
recurrent_kernel=self.cell.recurrent_kernel,
bias=self.cell.bias,
activation=self.activation,
recurrent_activation=self.recurrent_activation,
time_major=self.time_major)
function.register(defun_cudnn_gru, inputs, initial_state[0],
self.cell.kernel, self.cell.recurrent_kernel,
self.cell.bias, self.time_major)
states = [new_h]
return last_output, outputs, runtime, states
def call(self,
inputs,
training=None,
mask=None,
initial_state=None,
constants=None):
"""`Bidirectional.call` implements the same API as the wrapped `RNN`."""
kwargs = {}
if generic_utils.has_arg(self.layer.call, 'training'):
kwargs['training'] = training
if generic_utils.has_arg(self.layer.call, 'mask'):
kwargs['mask'] = mask
if generic_utils.has_arg(self.layer.call, 'constants'):
kwargs['constants'] = constants
if initial_state is not None and generic_utils.has_arg(
self.layer.call, 'initial_state'):
forward_state = initial_state[:len(initial_state) // 2]
backward_state = initial_state[len(initial_state) // 2:]
y = self.forward_layer.call(inputs, initial_state=forward_state, **kwargs)
y_rev = self.backward_layer.call(
inputs, initial_state=backward_state, **kwargs)
else:
y = self.forward_layer.call(inputs, **kwargs)
y_rev = self.backward_layer.call(inputs, **kwargs)
if self.return_state:
states = y[1:] + y_rev[1:]
y = y[0]
y_rev = y_rev[0]
if self.return_sequences:
y_rev = K.reverse(y_rev, 1)
if self.merge_mode == 'concat':
output = K.concatenate([y, y_rev])
elif self.merge_mode == 'sum':
output = y + y_rev
elif self.merge_mode == 'ave':
output = (y + y_rev) / 2
elif self.merge_mode == 'mul':
output = y * y_rev
elif self.merge_mode is None:
output = [y, y_rev]
# Properly set learning phase
if (getattr(y, '_uses_learning_phase', False) or
getattr(y_rev, '_uses_learning_phase', False)):
if self.merge_mode is None:
for out in output:
out._uses_learning_phase = True
else:
output._uses_learning_phase = True
if self.return_state:
if self.merge_mode is None:
return output + states
return [output] + states
return output
def call(self, inputs, mask=None, training=None, initial_state=None):
if isinstance(inputs, list):
initial_state = inputs[1:]
inputs = inputs[0]
elif initial_state is not None:
pass
elif self.stateful:
initial_state = self.states
else:
initial_state = self.get_initial_state(inputs)
if len(initial_state) != len(self.states):
raise ValueError('Layer has ' + str(len(self.states)) +
' states but was passed ' + str(len(initial_state)) +
' initial states.')
if self.go_backwards:
# Reverse time axis.
inputs = K.reverse(inputs, 1)
if ops.executing_eagerly_outside_functions():
if context.num_gpus() > 0:
outputs, [new_h, new_c], runtime = cudnn_lstm(
inputs, initial_state[0], initial_state[1], self.kernel,
self.recurrent_kernel, self.bias, self.units)
else:
outputs, [new_h, new_c], runtime = normal_lstm(
inputs, initial_state[0], initial_state[1], self.kernel,
self.recurrent_kernel, self.bias, self.units, self.activation,
self.recurrent_activation)
else:
outputs, [new_h, new_c], runtime = normal_lstm(
inputs, initial_state[0], initial_state[1], self.kernel,
self.recurrent_kernel, self.bias, self.units, self.activation,
self.recurrent_activation)
function.register(cudnn_lstm, inputs, initial_state[0], initial_state[1],
self.kernel, self.recurrent_kernel, self.bias,
self.units)
states = [new_h, new_c]
if self.stateful:
updates = []
for i in range(len(states)):
updates.append(state_ops.assign(self.states[i], states[i]))
self.add_update(updates, inputs)
if self.return_sequences:
output = outputs
else:
output = outputs[:, -1, :]
if self.return_state:
return [output] + states
else:
return output, runtime
def call(self, inputs,
training=None,
mask=None,
initial_state=None,
constants=None):
"""`Bidirectional.call` implements the same API as the wrapped `RNN`."""
kwargs = {}
if generic_utils.has_arg(self.layer.call, 'training'):
kwargs['training'] = training
if generic_utils.has_arg(self.layer.call, 'mask'):
kwargs['mask'] = mask
if generic_utils.has_arg(self.layer.call, 'constants'):
kwargs['constants'] = constants
if initial_state is not None and generic_utils.has_arg(
self.layer.call, 'initial_state'):
forward_state = initial_state[:len(initial_state) // 2]
backward_state = initial_state[len(initial_state) // 2:]
y = self.forward_layer.call(inputs, initial_state=forward_state, **kwargs)
y_rev = self.backward_layer.call(
inputs, initial_state=backward_state, **kwargs)
else:
y = self.forward_layer.call(inputs, **kwargs)
y_rev = self.backward_layer.call(inputs, **kwargs)
if self.return_state:
states = y[1:] + y_rev[1:]
y = y[0]
y_rev = y_rev[0]
if self.return_sequences:
y_rev = K.reverse(y_rev, 1)
if self.merge_mode == 'concat':
output = K.concatenate([y, y_rev])
elif self.merge_mode == 'sum':
output = y + y_rev
elif self.merge_mode == 'ave':
output = (y + y_rev) / 2
elif self.merge_mode == 'mul':
output = y * y_rev
elif self.merge_mode is None:
output = [y, y_rev]
# Properly set learning phase
if (getattr(y, '_uses_learning_phase', False) or
getattr(y_rev, '_uses_learning_phase', False)):
if self.merge_mode is None:
for out in output:
out._uses_learning_phase = True
else:
output._uses_learning_phase = True
if self.return_state:
if self.merge_mode is None:
return output + states
return [output] + states
return output
def call(self, inputs, mask=None, training=None, initial_state=None):
if isinstance(mask, list):
mask = mask[0]
if mask is not None:
raise ValueError('Masking is not supported for CuDNN RNNs.')
# input shape: `(samples, time (padded with zeros), input_dim)`
# note that the .build() method of subclasses MUST define
# self.input_spec and self.state_spec with complete input shapes.
if isinstance(inputs, list):
initial_state = inputs[1:]
inputs = inputs[0]
elif initial_state is not None:
pass
elif self.stateful:
initial_state = self.states
else:
initial_state = self.get_initial_state(inputs)
if len(initial_state) != len(self.states):
raise ValueError('Layer has ' + str(len(self.states)) +
' states but was passed ' +
str(len(initial_state)) + ' initial states.')
if self.go_backwards:
# Reverse time axis.
inputs = K.reverse(inputs, 1)
output, states = self._process_batch(inputs, initial_state)
if self.stateful:
updates = [
state_ops.assign(self_state, state)
for self_state, state in zip(self.states, states)
]
self.add_update(updates)
if self.return_state:
return [output] + states
else:
return output
def call(self, inputs, mask=None, training=None, initial_state=None):
if isinstance(mask, list):
mask = mask[0]
if mask is not None:
raise ValueError('Masking is not supported for CuDNN RNNs.')
# input shape: `(samples, time (padded with zeros), input_dim)`
# note that the .build() method of subclasses MUST define
# self.input_spec and self.state_spec with complete input shapes.
if isinstance(inputs, list):
initial_state = inputs[1:]
inputs = inputs[0]
elif initial_state is not None:
pass
elif self.stateful:
initial_state = self.states
else:
initial_state = self.get_initial_state(inputs)
if len(initial_state) != len(self.states):
raise ValueError('Layer has ' + str(len(self.states)) +
' states but was passed ' + str(len(initial_state)) +
' initial states.')
if self.go_backwards:
# Reverse time axis.
inputs = K.reverse(inputs, 1)
output, states = self._process_batch(inputs, initial_state)
if self.stateful:
updates = []
for i in range(len(states)):
updates.append(state_ops.assign(self.states[i], states[i]))
self.add_update(updates, inputs)
if self.return_state:
return [output] + states
else:
return output
def get_model(self):
input_current = Input((self.maxlen,))
input_left = Input((self.maxlen,))
input_right = Input((self.maxlen,))
embedder = Embedding(self.max_tokens, self.embedding_size, input_length=self.maxlen)
embedding_current = embedder(input_current)
embedding_left = embedder(input_left)
embedding_right = embedder(input_right)
x_left = self._RNN(128, return_sequences=True)(embedding_left)
x_right = self._RNN(128, return_sequences=True, go_backwards=True)(embedding_right)
x_right = Lambda(lambda x: K.reverse(x, axes=1))(x_right)
x = Concatenate(axis=2)([x_left, embedding_current, x_right])
x = Conv1D(64, kernel_size=1, activation='tanh')(x)
x = GlobalMaxPooling1D()(x)
output = Dense(self.num_class, activation=self.last_activation)(x)
model = Model(inputs=[input_current, input_left, input_right], outputs=output)
return model
def call(self,
inputs,
mask=None,
**kwargs): # 双向堆叠LSTM，每个方向互不影响，各自经过多层处理之后，进行整合
input_fw = inputs
input_bw = inputs
for i in range(self.layers):
output_fw = self.fw_lstm[i](input_fw)
output_bw = self.bw_lstm[i](input_bw)
output_bw = Lambda(lambda x: K.reverse(x, 1),
mask=lambda inputs, mask: mask)(
output_bw) # 反向的，要注意转向 # 有问题？？？输入先转向？？？
if i >= self.layers - self.res_layers:
output_fw += input_fw
output_bw += input_bw
input_fw = output_fw
input_bw = output_bw
output_fw = input_fw
output_bw = input_bw
if self.merge_mode == "fw":
output = output_fw
elif self.merge_mode == "bw":
output = output_bw
elif self.merge_mode == 'concat':
output = K.concatenate([output_fw, output_bw])
elif self.merge_mode == 'sum':
output = output_fw + output_bw
elif self.merge_mode == 'ave':
output = (output_fw + output_bw) / 2
elif self.merge_mode == 'mul':
output = output_fw * output_bw
elif self.merge_mode is None:
output = [output_fw, output_bw]
return output
def call(self,
inputs,
training=None,
mask=None,
initial_state=None,
constants=None):
"""`Bidirectional.call` implements the same API as the wrapped `RNN`."""
kwargs = {}
if generic_utils.has_arg(self.layer.call, 'training'):
kwargs['training'] = training
if generic_utils.has_arg(self.layer.call, 'mask'):
kwargs['mask'] = mask
if generic_utils.has_arg(self.layer.call, 'constants'):
kwargs['constants'] = constants
if initial_state is not None and generic_utils.has_arg(
self.layer.call, 'initial_state'):
forward_inputs = [inputs[0]]
backward_inputs = [inputs[0]]
pivot = len(initial_state) // 2 + 1
# add forward initial state
forward_state = inputs[1:pivot]
forward_inputs += forward_state
if self._num_constants is None:
# add backward initial state
backward_state = inputs[pivot:]
backward_inputs += backward_state
else:
# add backward initial state
backward_state = inputs[pivot:-self._num_constants]
backward_inputs += backward_state
# add constants for forward and backward layers
forward_inputs += inputs[-self._num_constants:]
backward_inputs += inputs[-self._num_constants:]
y = self.forward_layer.call(forward_inputs,
initial_state=forward_state,
**kwargs)
y_rev = self.backward_layer.call(backward_inputs,
initial_state=backward_state,
**kwargs)
else:
y = self.forward_layer.call(inputs, **kwargs)
y_rev = self.backward_layer.call(inputs, **kwargs)
if self.return_state:
states = y[1:] + y_rev[1:]
y = y[0]
y_rev = y_rev[0]
if self.return_sequences:
y_rev = K.reverse(y_rev, 1)
if self.merge_mode == 'concat':
output = K.concatenate([y, y_rev])
elif self.merge_mode == 'sum':
output = y + y_rev
elif self.merge_mode == 'ave':
output = (y + y_rev) / 2
elif self.merge_mode == 'mul':
output = y * y_rev
elif self.merge_mode is None:
output = [y, y_rev]
else:
raise ValueError('Unrecognized value for `merge_mode`: %s' %
(self.merge_mode))
if self.return_state:
if self.merge_mode is None:
return output + states
return [output] + states
return output
def call(self,
inputs,
training=None,
mask=None,
initial_state=None,
constants=None):
"""`Bidirectional.call` implements the same API as the wrapped `RNN`."""
kwargs = {}
if generic_utils.has_arg(self.layer.call, 'training'):
kwargs['training'] = training
if generic_utils.has_arg(self.layer.call, 'mask'):
kwargs['mask'] = mask
if generic_utils.has_arg(self.layer.call, 'constants'):
kwargs['constants'] = constants
if generic_utils.has_arg(self.layer.call, 'initial_state'):
if isinstance(inputs, list) and len(inputs) > 1:
# initial_states are keras tensors, which means they are passed in
# together with inputs as list. The initial_states need to be split into
# forward and backward section, and be feed to layers accordingly.
forward_inputs = [inputs[0]]
backward_inputs = [inputs[0]]
pivot = (len(inputs) - self._num_constants) // 2 + 1
# add forward initial state
forward_inputs += inputs[1:pivot]
if not self._num_constants:
# add backward initial state
backward_inputs += inputs[pivot:]
else:
# add backward initial state
backward_inputs += inputs[pivot:-self._num_constants]
# add constants for forward and backward layers
forward_inputs += inputs[-self._num_constants:]
backward_inputs += inputs[-self._num_constants:]
forward_state, backward_state = None, None
if 'constants' in kwargs:
kwargs['constants'] = None
elif initial_state is not None:
# initial_states are not keras tensors, eg eager tensor from np array.
# They are only passed in from kwarg initial_state, and should be passed
# to forward/backward layer via kwarg initial_state as well.
forward_inputs, backward_inputs = inputs, inputs
half = len(initial_state) // 2
forward_state = initial_state[:half]
backward_state = initial_state[half:]
else:
forward_inputs, backward_inputs = inputs, inputs
forward_state, backward_state = None, None
y = self.forward_layer(forward_inputs,
initial_state=forward_state,
**kwargs)
y_rev = self.backward_layer(backward_inputs,
initial_state=backward_state,
**kwargs)
else:
y = self.forward_layer(inputs, **kwargs)
y_rev = self.backward_layer(inputs, **kwargs)
if self.return_state:
states = y[1:] + y_rev[1:]
y = y[0]
y_rev = y_rev[0]
if self.return_sequences:
time_dim = 0 if getattr(self.forward_layer, 'time_major',
False) else 1
y_rev = K.reverse(y_rev, time_dim)
if self.merge_mode == 'concat':
output = K.concatenate([y, y_rev])
elif self.merge_mode == 'sum':
output = y + y_rev
elif self.merge_mode == 'ave':
output = (y + y_rev) / 2
elif self.merge_mode == 'mul':
output = y * y_rev
elif self.merge_mode is None:
output = [y, y_rev]
else:
raise ValueError('Unrecognized value for `merge_mode`: %s' %
(self.merge_mode))
if self.return_state:
if self.merge_mode is None:
return output + states
return [output] + states
return output
def call(self,
inputs, # inputs: batch_size*max_len*embedding_dim
input_lengths, # input_lengths: batch_size
dependencies, # dependencies: 2*batch_size*max_len*recurrent_size
edge_types, # edge_types: 2*batch_size*max_len*recurrent_size
mask=None, # mask: batch_size*max_len
cell_mask=None, # mask: 2*batch_size*max_len*recurrent_size
initial_state=None, # initial_state: 2*4*batch_size*embedding_dim
training=True):
def swap_pad(info, lengths):
batch_size = info.shape[0]
max_len = info.shape[1]
new_info = []
for i in range(batch_size):
if lengths[i].numpy() == max_len:
new_info.append(info[i])
continue
stack = []
sentence_info = info[i, 0:lengths[i], :]
pad_info = info[i, lengths[i]:, :]
sentence_info_slices = tf.split(sentence_info, num_or_size_splits=sentence_info.shape[0], axis=0)
pad_info_slices = tf.split(pad_info, num_or_size_splits=pad_info.shape[0], axis=0)
for tensor in pad_info_slices:
stack.append(tensor)
for tensor in sentence_info_slices:
stack.append(tensor)
new_info.append(tf.squeeze(tf.stack(stack, axis=0), axis=1))
new_info = tf.stack(new_info, axis=0)
return new_info
if initial_state is not None:
forward_inputs, backward_inputs = inputs, inputs
forward_state, backward_state = initial_state[0], initial_state[1]
forward_dependencies, backward_dependencies = dependencies[0], dependencies[1]
forward_edge_types, backward_edge_types = edge_types[0], edge_types[1]
forward_cell_mask, backward_cell_mask = cell_mask[0], cell_mask[1]
backward_dependencies = swap_pad(backward_dependencies, input_lengths)
backward_edge_types = swap_pad(backward_edge_types, input_lengths)
backward_cell_mask = swap_pad(backward_cell_mask, input_lengths)
else:
raise ValueError("Please provide initial states for RNN.")
y = self.forward_layer(forward_inputs, input_lengths, forward_dependencies, forward_edge_types, mask, forward_cell_mask, forward_state, training)
y_rev = self.backward_layer(backward_inputs, input_lengths, backward_dependencies, backward_edge_types, mask, backward_cell_mask, backward_state, training)
if self.return_state:
states = y[1:] + y_rev[1:]
y = y[0]
y_rev = y_rev[0]
if self.return_sequences:
y_rev = K.reverse(y_rev, 1)
if self.merge_mode == 'concat':
output = K.concatenate([y, y_rev])
elif self.merge_mode == 'sum':
output = y + y_rev
elif self.merge_mode == 'ave':
output = (y + y_rev) / 2
elif self.merge_mode == 'mul':
output = y * y_rev
elif self.merge_mode is None:
output = [y, y_rev]
else:
raise ValueError('Unrecognized value for `merge_mode`: %s' % (self.merge_mode))
if self.return_state:
if self.merge_mode is None:
return output + states
return [output] + states
return output
def call(self,
inputs,
training=None,
mask=None,
initial_state=None,
constants=None):
"""`Bidirectional.call` implements the same API as the wrapped `RNN`."""
kwargs = {}
if generic_utils.has_arg(self.layer.call, 'training'):
kwargs['training'] = training
if generic_utils.has_arg(self.layer.call, 'mask'):
kwargs['mask'] = mask
if generic_utils.has_arg(self.layer.call, 'constants'):
kwargs['constants'] = constants
if initial_state is not None and generic_utils.has_arg(
self.layer.call, 'initial_state'):
forward_inputs = [inputs[0]]
backward_inputs = [inputs[0]]
pivot = len(initial_state) // 2 + 1
# add forward initial state
forward_state = inputs[1:pivot]
forward_inputs += forward_state
if self._num_constants is None:
# add backward initial state
backward_state = inputs[pivot:]
backward_inputs += backward_state
else:
# add backward initial state
backward_state = inputs[pivot:-self._num_constants]
backward_inputs += backward_state
# add constants for forward and backward layers
forward_inputs += inputs[-self._num_constants:]
backward_inputs += inputs[-self._num_constants:]
y = self.forward_layer.call(forward_inputs,
initial_state=forward_state, **kwargs)
y_rev = self.backward_layer.call(backward_inputs,
initial_state=backward_state, **kwargs)
else:
y = self.forward_layer.call(inputs, **kwargs)
y_rev = self.backward_layer.call(inputs, **kwargs)
if self.return_state:
states = y[1:] + y_rev[1:]
y = y[0]
y_rev = y_rev[0]
if self.return_sequences:
y_rev = K.reverse(y_rev, 1)
if self.merge_mode == 'concat':
output = K.concatenate([y, y_rev])
elif self.merge_mode == 'sum':
output = y + y_rev
elif self.merge_mode == 'ave':
output = (y + y_rev) / 2
elif self.merge_mode == 'mul':
output = y * y_rev
elif self.merge_mode is None:
output = [y, y_rev]
else:
raise ValueError(
'Unrecognized value for `merge_mode`: %s' % (self.merge_mode))
if self.return_state:
if self.merge_mode is None:
return output + states
return [output] + states
return output
def get_Model(training):
input_shape = (img_w, img_h, 1) # (128, 64, 1)
# Make Networkw
inputs = Input(name='the_input', shape=input_shape,
dtype='float32') # (None, 128, 64, 1)
# Convolution layer (VGG)
inner = Conv2D(64, (3, 3),
padding='same',
name='conv1',
kernel_initializer='he_normal')(
inputs) # (None, 128, 64, 64)
inner = BatchNormalization()(inner)
inner = Activation('relu')(inner)
inner = MaxPooling2D(pool_size=(2, 2),
name='max1')(inner) # (None,64, 32, 64)
inner = Conv2D(128, (3, 3),
padding='same',
name='conv2',
kernel_initializer='he_normal')(
inner) # (None, 64, 32, 128)
inner = BatchNormalization()(inner)
inner = Activation('relu')(inner)
inner = MaxPooling2D(pool_size=(2, 2),
name='max2')(inner) # (None, 32, 16, 128)
inner = Conv2D(256, (3, 3),
padding='same',
name='conv3',
kernel_initializer='he_normal')(
inner) # (None, 32, 16, 256)
inner = BatchNormalization()(inner)
inner = Activation('relu')(inner)
inner = Conv2D(256, (3, 3),
padding='same',
name='conv4',
kernel_initializer='he_normal')(
inner) # (None, 32, 16, 256)
inner = BatchNormalization()(inner)
inner = Activation('relu')(inner)
inner = MaxPooling2D(pool_size=(1, 2),
name='max3')(inner) # (None, 32, 8, 256)
inner = Conv2D(512, (3, 3),
padding='same',
name='conv5',
kernel_initializer='he_normal')(inner) # (None, 32, 8, 512)
inner = BatchNormalization()(inner)
inner = Activation('relu')(inner)
inner = Conv2D(512, (3, 3), padding='same',
name='conv6')(inner) # (None, 32, 8, 512)
inner = BatchNormalization()(inner)
inner = Activation('relu')(inner)
inner = MaxPooling2D(pool_size=(1, 2),
name='max4')(inner) # (None, 32, 4, 512)
inner = Conv2D(512, (2, 2),
padding='same',
kernel_initializer='he_normal',
name='con7')(inner) # (None, 32, 4, 512)
inner = BatchNormalization()(inner)
inner = Activation('relu')(inner)
# CNN to RNN
inner = Reshape(target_shape=((32, 2048)),
name='reshape')(inner) # (None, 32, 2048)
inner = Dense(64,
activation='relu',
kernel_initializer='he_normal',
name='dense1')(inner) # (None, 32, 64)
# RNN layer
lstm_1 = LSTM(256,
return_sequences=True,
kernel_initializer='he_normal',
name='lstm1')(inner) # (None, 32, 512)
lstm_1b = LSTM(256,
return_sequences=True,
go_backwards=True,
kernel_initializer='he_normal',
name='lstm1_b')(inner)
reversed_lstm_1b = Lambda(
lambda inputTensor: K.reverse(inputTensor, axes=1))(lstm_1b)
lstm1_merged = add([lstm_1, reversed_lstm_1b]) # (None, 32, 512)
lstm1_merged = BatchNormalization()(lstm1_merged)
lstm_2 = LSTM(256,
return_sequences=True,
kernel_initializer='he_normal',
name='lstm2')(lstm1_merged)
lstm_2b = LSTM(256,
return_sequences=True,
go_backwards=True,
kernel_initializer='he_normal',
name='lstm2_b')(lstm1_merged)
reversed_lstm_2b = Lambda(
lambda inputTensor: K.reverse(inputTensor, axes=1))(lstm_2b)
lstm2_merged = concatenate([lstm_2, reversed_lstm_2b]) # (None, 32, 1024)
lstm2_merged = BatchNormalization()(lstm2_merged)
# transforms RNN output to character activations:
inner = Dense(num_classes, kernel_initializer='he_normal',
name='dense2')(lstm2_merged) #(None, 32, 63)
y_pred = Activation('softmax', name='softmax')(inner)
labels = Input(name='the_labels', shape=[max_text_len],
dtype='float32') # (None ,8)
input_length = Input(name='input_length', shape=[1],
dtype='int64') # (None, 1)
label_length = Input(name='label_length', shape=[1],
dtype='int64') # (None, 1)
# Keras doesn't currently support loss funcs with extra parameters
# so CTC loss is implemented in a lambda layer
loss_out = Lambda(focal_ctc_lambda_func, output_shape=(1, ),
name='ctc')([labels, y_pred, input_length,
label_length]) #(None, 1)
if training:
return Model(inputs=[inputs, labels, input_length, label_length],
outputs=loss_out)
else:
return Model(inputs=[inputs], outputs=y_pred)
def lemol_rnn(step_function,
inputs,
initial_states,
go_backwards=False,
mask=None,
constants=None,
unroll=False,
input_length=None,
time_major=False,
zero_output_for_mask=False):
print('Using alternative rnn function designed for LeMOL')
def swap_batch_timestep(input_t):
# Swap the batch and timestep dim for the incoming tensor.
axes = list(range(len(input_t.shape)))
axes[0], axes[1] = 1, 0
return array_ops.transpose(input_t, axes)
if not time_major:
inputs = nest.map_structure(swap_batch_timestep, inputs)
flatted_inputs = nest.flatten(inputs)
time_steps = flatted_inputs[0].shape[0]
batch = flatted_inputs[0].shape[1]
time_steps_t = array_ops.shape(flatted_inputs[0])[0]
for input_ in flatted_inputs:
input_.shape.with_rank_at_least(3)
if mask is not None:
ValueError('Mask Not Supported.\n' +
'Perhaps you want the original rnn function from tf.keras.backend')
if constants is None:
constants = []
if unroll:
if not time_steps:
raise ValueError('Unrolling requires a fixed number of timesteps.')
states = tuple(initial_states)
successive_states = []
successive_outputs = []
successive_learning_features = []
# Process the input tensors. The input tensor need to be split on the
# time_step dim, and reverse if go_backwards is True. In the case of nested
# input, the input is flattened and then transformed individually.
# The result of this will be a tuple of lists, each of the item in tuple is
# list of the tensor with shape (batch, feature)
def _process_single_input_t(input_t):
input_t = array_ops.unstack(input_t) # unstack for time_step dim
if go_backwards:
input_t.reverse()
return input_t
if nest.is_sequence(inputs):
processed_input = nest.map_structure(
_process_single_input_t, inputs)
else:
processed_input = (_process_single_input_t(inputs),)
def _get_input_tensor(time):
inp = [t_[time] for t_ in processed_input]
return nest.pack_sequence_as(inputs, inp)
for i in range(time_steps):
inp = _get_input_tensor(i)
output, states, learning_feature = step_function(
inp, tuple(states) + tuple(constants))
successive_outputs.append(output)
successive_states.append(states)
successive_learning_features.append(learning_feature)
last_output = successive_outputs[-1]
new_states = successive_states[-1]
outputs = array_ops.stack(successive_outputs)
all_learning_featues = array_ops.stack(successive_learning_features)
else:
states = tuple(initial_states)
# Create input tensor array, if the inputs is nested tensors, then it will
# be flattened first, and tensor array will be created one per flattened
# tensor.
input_ta = tuple(
tensor_array_ops.TensorArray(
dtype=inp.dtype,
size=time_steps_t,
tensor_array_name='input_ta_%s' % i)
for i, inp in enumerate(flatted_inputs))
input_ta = tuple(
ta.unstack(input_) if not go_backwards else ta
.unstack(reverse(input_, 0))
for ta, input_ in zip(input_ta, flatted_inputs))
# Get the time(0) input and compute the output for that, the output will be
# used to determine the dtype of output tensor array. Don't read from
# input_ta due to TensorArray clear_after_read default to True.
input_time_zero = nest.pack_sequence_as(inputs,
[inp[0] for inp in flatted_inputs])
# output_time_zero is used to determine the cell output shape and its dtype.
# the value is discarded.
output_time_zero, _, lf_time_zero = step_function(
input_time_zero, initial_states + constants)
output_ta = tuple(
tensor_array_ops.TensorArray(
dtype=out.dtype,
size=time_steps_t,
tensor_array_name='output_ta_%s' % i)
for i, out in enumerate(nest.flatten(output_time_zero)))
lf_out_ta = tuple(
tensor_array_ops.TensorArray(
dtype=out.dtype,
size=time_steps_t,
tensor_array_name='output_ta_%s' % i)
for i, out in enumerate(nest.flatten(lf_time_zero)))
time = constant_op.constant(0, dtype='int32', name='time')
while_loop_kwargs = {
'cond': lambda time, *_: time < time_steps_t,
'maximum_iterations': input_length,
'parallel_iterations': 32,
'swap_memory': True,
}
def _step(time, output_ta_t, output_lf_t, *states):
'''RNN step function.
Arguments:
time: Current timestep value.
output_ta_t: TensorArray.
*states: List of states.
Returns:
Tuple: `(time + 1,output_ta_t) + tuple(new_states)`
'''
current_input = tuple(ta.read(time) for ta in input_ta)
current_input = nest.pack_sequence_as(inputs, current_input)
output, new_states, new_lf = step_function(current_input,
tuple(states) + tuple(constants))
flat_state = nest.flatten(states)
flat_new_state = nest.flatten(new_states)
for state, new_state in zip(flat_state, flat_new_state):
if hasattr(new_state, 'set_shape'):
new_state.set_shape(state.shape)
flat_output = nest.flatten(output)
output_ta_t = tuple(
ta.write(time, out) for ta, out in zip(output_ta_t, flat_output))
flat_lf = nest.flatten(new_lf)
output_lf_t = tuple(
ta.write(time, out) for ta, out in zip(output_lf_t, flat_lf))
new_states = nest.pack_sequence_as(
initial_states, flat_new_state)
return (time + 1, output_ta_t, output_lf_t) + tuple(new_states)
final_outputs = control_flow_ops.while_loop(
body=_step,
loop_vars=(time, output_ta, lf_out_ta) + states,
**while_loop_kwargs)
new_states = final_outputs[3:]
output_ta = final_outputs[1]
lf_ta = final_outputs[2]
lfs = tuple(o.stack() for o in lf_ta)
outputs = tuple(o.stack() for o in output_ta)
last_output = tuple(o[-1] for o in outputs)
last_lf = tuple(o[-1] for o in lfs)
outputs = nest.pack_sequence_as(output_time_zero, outputs)
lfs = nest.pack_sequence_as(lf_time_zero, lfs)
last_output = nest.pack_sequence_as(output_time_zero, last_output)
last_lf = nest.pack_sequence_as(lf_time_zero, last_lf)
# static shape inference
def set_shape(output_):
if hasattr(output_, 'set_shape'):
shape = output_.shape.as_list()
shape[0] = time_steps
shape[1] = batch
output_.set_shape(shape)
return output_
outputs = nest.map_structure(set_shape, outputs)
lfs = nest.map_structure(set_shape, lfs)
if not time_major:
outputs = nest.map_structure(swap_batch_timestep, outputs)
lfs = nest.map_structure(swap_batch_timestep, lfs)
return last_output, outputs, new_states, last_lf, lfs
def call(self, inputs, mask=None, training=None, initial_state=None):
# LSTM does not support constants. Ignore it during process.
inputs, initial_state, _ = self._process_inputs(
inputs, initial_state, None)
if isinstance(mask, list):
mask = mask[0]
input_shape = K.int_shape(inputs)
timesteps = input_shape[0] if self.time_major else input_shape[1]
if mask is not None or not self.could_use_cudnn:
# CuDNN does not support masking, fall back to use the normal LSTM.
kwargs = {'training': training}
def step(inputs, states):
return self.cell.call(inputs, states, **kwargs)
last_output, outputs, states = K.rnn(
step,
inputs,
initial_state,
constants=None,
go_backwards=self.go_backwards,
mask=mask,
unroll=self.unroll,
input_length=timesteps,
time_major=self.time_major,
zero_output_for_mask=self.zero_output_for_mask)
runtime = _runtime('unknown')
else:
# Use the new defun approach for backend implementation swap.
# Note that different implementations need to have same function
# signature, eg, the tensor parameters need to have same shape and dtypes.
# Since the CuDNN has an extra set of bias, those bias will be passed to
# both normal and CuDNN implementations.
if self.go_backwards:
# Reverse time axis.
inputs = K.reverse(inputs, 0 if self.time_major else 1)
self.reset_dropout_mask()
dropout_mask = self.get_dropout_mask_for_cell(inputs,
training,
count=4)
if dropout_mask is not None:
inputs *= dropout_mask[0]
if context.executing_eagerly():
device_type = _get_context_device_type()
if device_type == _GPU_DEVICE_NAME or (device_type is None and
context.num_gpus() > 0):
# Under eager context, check the device placement and prefer the
# GPU implementation when GPU is available.
last_output, outputs, new_h, new_c, runtime = cudnn_lstm(
inputs, initial_state[0], initial_state[1],
self.cell.kernel, self.cell.recurrent_kernel,
self.cell.bias, self.time_major)
else:
last_output, outputs, new_h, new_c, runtime = standard_lstm(
inputs, initial_state[0], initial_state[1],
self.cell.kernel, self.cell.recurrent_kernel,
self.cell.bias, self.activation,
self.recurrent_activation, self.time_major)
else:
# Each time a `tf.function` is called, we will give it a unique
# identifiable API name, so that Grappler won't get confused when it
# sees multiple LSTM layers added into same graph, and it will be able
# to pair up the different implementations across them.
api_name = 'lstm_' + str(uuid.uuid4())
defun_standard_lstm = _generate_defun_backend(
api_name, _CPU_DEVICE_NAME, standard_lstm)
defun_cudnn_lstm = _generate_defun_backend(
api_name, _GPU_DEVICE_NAME, cudnn_lstm)
# Call the normal LSTM impl and register the CuDNN impl function. The
# grappler will kick in during session execution to optimize the graph.
last_output, outputs, new_h, new_c, runtime = defun_standard_lstm(
inputs, initial_state[0], initial_state[1],
self.cell.kernel, self.cell.recurrent_kernel,
self.cell.bias, self.activation, self.recurrent_activation,
self.time_major)
function.register(defun_cudnn_lstm, inputs, initial_state[0],
initial_state[1], self.cell.kernel,
self.cell.recurrent_kernel, self.cell.bias,
self.time_major)
states = [new_h, new_c]
if self.stateful:
updates = []
for i in range(len(states)):
updates.append(state_ops.assign(self.states[i], states[i]))
self.add_update(updates, inputs)
if self.return_sequences:
output = outputs
else:
output = last_output
if self.return_state:
return [output] + list(states)
elif self.return_runtime:
return output, runtime
else:
return output
