def call(self, inputs, mask=None, **kwargs): input_fw = inputs input_bw = inputs for i in range(self.layers): output_fw = self.fw_lstm[i](input_fw) output_bw = self.bw_lstm[i](input_bw) output_bw = Lambda(lambda x: K.reverse(x, 1), mask=lambda inputs, mask: mask)(output_bw) if i >= self.layers - self.res_layers: output_fw += input_fw output_bw += input_bw input_fw = output_fw input_bw = output_bw output_fw = input_fw output_bw = input_bw if self.merge_mode == "fw": output = output_fw elif self.merge_mode == "bw": output = output_bw elif self.merge_mode == 'concat': output = K.concatenate([output_fw, output_bw]) elif self.merge_mode == 'sum': output = output_fw + output_bw elif self.merge_mode == 'ave': output = (output_fw + output_bw) / 2 elif self.merge_mode == 'mul': output = output_fw * output_bw elif self.merge_mode is None: output = [output_fw, output_bw] return output
def _defun_gru_call(self, inputs, initial_state, training): # Use the new defun approach for backend implementation swap. # Note that different implementations need to have same function # signature, eg, the tensor parameters need to have same shape and dtypes. if self.go_backwards: # Reverse time axis. inputs = K.reverse(inputs, 0 if self.time_major else 1) self.reset_dropout_mask() dropout_mask = self.get_dropout_mask_for_cell(inputs, training, count=3) if dropout_mask is not None: inputs *= dropout_mask[0] if ops.executing_eagerly_outside_functions(): # Under eager context, the device placement is already known. Prefer the # GPU implementation when GPU is available. if context.num_gpus() > 0: last_output, outputs, new_h, runtime = cudnn_gru( inputs=inputs, init_h=initial_state[0], kernel=self.cell.kernel, recurrent_kernel=self.cell.recurrent_kernel, bias=self.cell.bias, time_major=self.time_major) else: last_output, outputs, new_h, runtime = standard_gru( inputs=inputs, init_h=initial_state[0], kernel=self.cell.kernel, recurrent_kernel=self.cell.recurrent_kernel, bias=self.cell.bias, activation=self.activation, recurrent_activation=self.recurrent_activation, time_major=self.time_major) else: api_name = 'gru_' + str(uuid.uuid4()) defun_standard_gru = _generate_defun_backend( api_name, _CPU_DEVICE_NAME, standard_gru) defun_cudnn_gru = _generate_defun_backend(api_name, _GPU_DEVICE_NAME, cudnn_gru) # Call the normal GRU impl and register the CuDNN impl function. The # grappler will kick in during session execution to optimize the graph. last_output, outputs, new_h, runtime = defun_standard_gru( inputs=inputs, init_h=initial_state[0], kernel=self.cell.kernel, recurrent_kernel=self.cell.recurrent_kernel, bias=self.cell.bias, activation=self.activation, recurrent_activation=self.recurrent_activation, time_major=self.time_major) function.register(defun_cudnn_gru, inputs, initial_state[0], self.cell.kernel, self.cell.recurrent_kernel, self.cell.bias, self.time_major) states = [new_h] return last_output, outputs, runtime, states
def call(self, inputs, training=None, mask=None, initial_state=None, constants=None): """`Bidirectional.call` implements the same API as the wrapped `RNN`.""" kwargs = {} if generic_utils.has_arg(self.layer.call, 'training'): kwargs['training'] = training if generic_utils.has_arg(self.layer.call, 'mask'): kwargs['mask'] = mask if generic_utils.has_arg(self.layer.call, 'constants'): kwargs['constants'] = constants if initial_state is not None and generic_utils.has_arg( self.layer.call, 'initial_state'): forward_state = initial_state[:len(initial_state) // 2] backward_state = initial_state[len(initial_state) // 2:] y = self.forward_layer.call(inputs, initial_state=forward_state, **kwargs) y_rev = self.backward_layer.call( inputs, initial_state=backward_state, **kwargs) else: y = self.forward_layer.call(inputs, **kwargs) y_rev = self.backward_layer.call(inputs, **kwargs) if self.return_state: states = y[1:] + y_rev[1:] y = y[0] y_rev = y_rev[0] if self.return_sequences: y_rev = K.reverse(y_rev, 1) if self.merge_mode == 'concat': output = K.concatenate([y, y_rev]) elif self.merge_mode == 'sum': output = y + y_rev elif self.merge_mode == 'ave': output = (y + y_rev) / 2 elif self.merge_mode == 'mul': output = y * y_rev elif self.merge_mode is None: output = [y, y_rev] # Properly set learning phase if (getattr(y, '_uses_learning_phase', False) or getattr(y_rev, '_uses_learning_phase', False)): if self.merge_mode is None: for out in output: out._uses_learning_phase = True else: output._uses_learning_phase = True if self.return_state: if self.merge_mode is None: return output + states return [output] + states return output
def call(self, inputs, mask=None, training=None, initial_state=None): if isinstance(inputs, list): initial_state = inputs[1:] inputs = inputs[0] elif initial_state is not None: pass elif self.stateful: initial_state = self.states else: initial_state = self.get_initial_state(inputs) if len(initial_state) != len(self.states): raise ValueError('Layer has ' + str(len(self.states)) + ' states but was passed ' + str(len(initial_state)) + ' initial states.') if self.go_backwards: # Reverse time axis. inputs = K.reverse(inputs, 1) if ops.executing_eagerly_outside_functions(): if context.num_gpus() > 0: outputs, [new_h, new_c], runtime = cudnn_lstm( inputs, initial_state[0], initial_state[1], self.kernel, self.recurrent_kernel, self.bias, self.units) else: outputs, [new_h, new_c], runtime = normal_lstm( inputs, initial_state[0], initial_state[1], self.kernel, self.recurrent_kernel, self.bias, self.units, self.activation, self.recurrent_activation) else: outputs, [new_h, new_c], runtime = normal_lstm( inputs, initial_state[0], initial_state[1], self.kernel, self.recurrent_kernel, self.bias, self.units, self.activation, self.recurrent_activation) function.register(cudnn_lstm, inputs, initial_state[0], initial_state[1], self.kernel, self.recurrent_kernel, self.bias, self.units) states = [new_h, new_c] if self.stateful: updates = [] for i in range(len(states)): updates.append(state_ops.assign(self.states[i], states[i])) self.add_update(updates, inputs) if self.return_sequences: output = outputs else: output = outputs[:, -1, :] if self.return_state: return [output] + states else: return output, runtime
def call(self, inputs, training=None, mask=None, initial_state=None, constants=None): """`Bidirectional.call` implements the same API as the wrapped `RNN`.""" kwargs = {} if generic_utils.has_arg(self.layer.call, 'training'): kwargs['training'] = training if generic_utils.has_arg(self.layer.call, 'mask'): kwargs['mask'] = mask if generic_utils.has_arg(self.layer.call, 'constants'): kwargs['constants'] = constants if initial_state is not None and generic_utils.has_arg( self.layer.call, 'initial_state'): forward_state = initial_state[:len(initial_state) // 2] backward_state = initial_state[len(initial_state) // 2:] y = self.forward_layer.call(inputs, initial_state=forward_state, **kwargs) y_rev = self.backward_layer.call( inputs, initial_state=backward_state, **kwargs) else: y = self.forward_layer.call(inputs, **kwargs) y_rev = self.backward_layer.call(inputs, **kwargs) if self.return_state: states = y[1:] + y_rev[1:] y = y[0] y_rev = y_rev[0] if self.return_sequences: y_rev = K.reverse(y_rev, 1) if self.merge_mode == 'concat': output = K.concatenate([y, y_rev]) elif self.merge_mode == 'sum': output = y + y_rev elif self.merge_mode == 'ave': output = (y + y_rev) / 2 elif self.merge_mode == 'mul': output = y * y_rev elif self.merge_mode is None: output = [y, y_rev] # Properly set learning phase if (getattr(y, '_uses_learning_phase', False) or getattr(y_rev, '_uses_learning_phase', False)): if self.merge_mode is None: for out in output: out._uses_learning_phase = True else: output._uses_learning_phase = True if self.return_state: if self.merge_mode is None: return output + states return [output] + states return output
def call(self, inputs, mask=None, training=None, initial_state=None): if isinstance(mask, list): mask = mask[0] if mask is not None: raise ValueError('Masking is not supported for CuDNN RNNs.') # input shape: `(samples, time (padded with zeros), input_dim)` # note that the .build() method of subclasses MUST define # self.input_spec and self.state_spec with complete input shapes. if isinstance(inputs, list): initial_state = inputs[1:] inputs = inputs[0] elif initial_state is not None: pass elif self.stateful: initial_state = self.states else: initial_state = self.get_initial_state(inputs) if len(initial_state) != len(self.states): raise ValueError('Layer has ' + str(len(self.states)) + ' states but was passed ' + str(len(initial_state)) + ' initial states.') if self.go_backwards: # Reverse time axis. inputs = K.reverse(inputs, 1) output, states = self._process_batch(inputs, initial_state) if self.stateful: updates = [ state_ops.assign(self_state, state) for self_state, state in zip(self.states, states) ] self.add_update(updates) if self.return_state: return [output] + states else: return output
def call(self, inputs, mask=None, training=None, initial_state=None): if isinstance(mask, list): mask = mask[0] if mask is not None: raise ValueError('Masking is not supported for CuDNN RNNs.') # input shape: `(samples, time (padded with zeros), input_dim)` # note that the .build() method of subclasses MUST define # self.input_spec and self.state_spec with complete input shapes. if isinstance(inputs, list): initial_state = inputs[1:] inputs = inputs[0] elif initial_state is not None: pass elif self.stateful: initial_state = self.states else: initial_state = self.get_initial_state(inputs) if len(initial_state) != len(self.states): raise ValueError('Layer has ' + str(len(self.states)) + ' states but was passed ' + str(len(initial_state)) + ' initial states.') if self.go_backwards: # Reverse time axis. inputs = K.reverse(inputs, 1) output, states = self._process_batch(inputs, initial_state) if self.stateful: updates = [] for i in range(len(states)): updates.append(state_ops.assign(self.states[i], states[i])) self.add_update(updates, inputs) if self.return_state: return [output] + states else: return output
def get_model(self): input_current = Input((self.maxlen,)) input_left = Input((self.maxlen,)) input_right = Input((self.maxlen,)) embedder = Embedding(self.max_tokens, self.embedding_size, input_length=self.maxlen) embedding_current = embedder(input_current) embedding_left = embedder(input_left) embedding_right = embedder(input_right) x_left = self._RNN(128, return_sequences=True)(embedding_left) x_right = self._RNN(128, return_sequences=True, go_backwards=True)(embedding_right) x_right = Lambda(lambda x: K.reverse(x, axes=1))(x_right) x = Concatenate(axis=2)([x_left, embedding_current, x_right]) x = Conv1D(64, kernel_size=1, activation='tanh')(x) x = GlobalMaxPooling1D()(x) output = Dense(self.num_class, activation=self.last_activation)(x) model = Model(inputs=[input_current, input_left, input_right], outputs=output) return model
def call(self, inputs, mask=None, **kwargs): # 双向堆叠LSTM,每个方向互不影响,各自经过多层处理之后,进行整合 input_fw = inputs input_bw = inputs for i in range(self.layers): output_fw = self.fw_lstm[i](input_fw) output_bw = self.bw_lstm[i](input_bw) output_bw = Lambda(lambda x: K.reverse(x, 1), mask=lambda inputs, mask: mask)( output_bw) # 反向的,要注意转向 # 有问题???输入先转向??? if i >= self.layers - self.res_layers: output_fw += input_fw output_bw += input_bw input_fw = output_fw input_bw = output_bw output_fw = input_fw output_bw = input_bw if self.merge_mode == "fw": output = output_fw elif self.merge_mode == "bw": output = output_bw elif self.merge_mode == 'concat': output = K.concatenate([output_fw, output_bw]) elif self.merge_mode == 'sum': output = output_fw + output_bw elif self.merge_mode == 'ave': output = (output_fw + output_bw) / 2 elif self.merge_mode == 'mul': output = output_fw * output_bw elif self.merge_mode is None: output = [output_fw, output_bw] return output
def call(self, inputs, training=None, mask=None, initial_state=None, constants=None): """`Bidirectional.call` implements the same API as the wrapped `RNN`.""" kwargs = {} if generic_utils.has_arg(self.layer.call, 'training'): kwargs['training'] = training if generic_utils.has_arg(self.layer.call, 'mask'): kwargs['mask'] = mask if generic_utils.has_arg(self.layer.call, 'constants'): kwargs['constants'] = constants if initial_state is not None and generic_utils.has_arg( self.layer.call, 'initial_state'): forward_inputs = [inputs[0]] backward_inputs = [inputs[0]] pivot = len(initial_state) // 2 + 1 # add forward initial state forward_state = inputs[1:pivot] forward_inputs += forward_state if self._num_constants is None: # add backward initial state backward_state = inputs[pivot:] backward_inputs += backward_state else: # add backward initial state backward_state = inputs[pivot:-self._num_constants] backward_inputs += backward_state # add constants for forward and backward layers forward_inputs += inputs[-self._num_constants:] backward_inputs += inputs[-self._num_constants:] y = self.forward_layer.call(forward_inputs, initial_state=forward_state, **kwargs) y_rev = self.backward_layer.call(backward_inputs, initial_state=backward_state, **kwargs) else: y = self.forward_layer.call(inputs, **kwargs) y_rev = self.backward_layer.call(inputs, **kwargs) if self.return_state: states = y[1:] + y_rev[1:] y = y[0] y_rev = y_rev[0] if self.return_sequences: y_rev = K.reverse(y_rev, 1) if self.merge_mode == 'concat': output = K.concatenate([y, y_rev]) elif self.merge_mode == 'sum': output = y + y_rev elif self.merge_mode == 'ave': output = (y + y_rev) / 2 elif self.merge_mode == 'mul': output = y * y_rev elif self.merge_mode is None: output = [y, y_rev] else: raise ValueError('Unrecognized value for `merge_mode`: %s' % (self.merge_mode)) if self.return_state: if self.merge_mode is None: return output + states return [output] + states return output
def call(self, inputs, training=None, mask=None, initial_state=None, constants=None): """`Bidirectional.call` implements the same API as the wrapped `RNN`.""" kwargs = {} if generic_utils.has_arg(self.layer.call, 'training'): kwargs['training'] = training if generic_utils.has_arg(self.layer.call, 'mask'): kwargs['mask'] = mask if generic_utils.has_arg(self.layer.call, 'constants'): kwargs['constants'] = constants if generic_utils.has_arg(self.layer.call, 'initial_state'): if isinstance(inputs, list) and len(inputs) > 1: # initial_states are keras tensors, which means they are passed in # together with inputs as list. The initial_states need to be split into # forward and backward section, and be feed to layers accordingly. forward_inputs = [inputs[0]] backward_inputs = [inputs[0]] pivot = (len(inputs) - self._num_constants) // 2 + 1 # add forward initial state forward_inputs += inputs[1:pivot] if not self._num_constants: # add backward initial state backward_inputs += inputs[pivot:] else: # add backward initial state backward_inputs += inputs[pivot:-self._num_constants] # add constants for forward and backward layers forward_inputs += inputs[-self._num_constants:] backward_inputs += inputs[-self._num_constants:] forward_state, backward_state = None, None if 'constants' in kwargs: kwargs['constants'] = None elif initial_state is not None: # initial_states are not keras tensors, eg eager tensor from np array. # They are only passed in from kwarg initial_state, and should be passed # to forward/backward layer via kwarg initial_state as well. forward_inputs, backward_inputs = inputs, inputs half = len(initial_state) // 2 forward_state = initial_state[:half] backward_state = initial_state[half:] else: forward_inputs, backward_inputs = inputs, inputs forward_state, backward_state = None, None y = self.forward_layer(forward_inputs, initial_state=forward_state, **kwargs) y_rev = self.backward_layer(backward_inputs, initial_state=backward_state, **kwargs) else: y = self.forward_layer(inputs, **kwargs) y_rev = self.backward_layer(inputs, **kwargs) if self.return_state: states = y[1:] + y_rev[1:] y = y[0] y_rev = y_rev[0] if self.return_sequences: time_dim = 0 if getattr(self.forward_layer, 'time_major', False) else 1 y_rev = K.reverse(y_rev, time_dim) if self.merge_mode == 'concat': output = K.concatenate([y, y_rev]) elif self.merge_mode == 'sum': output = y + y_rev elif self.merge_mode == 'ave': output = (y + y_rev) / 2 elif self.merge_mode == 'mul': output = y * y_rev elif self.merge_mode is None: output = [y, y_rev] else: raise ValueError('Unrecognized value for `merge_mode`: %s' % (self.merge_mode)) if self.return_state: if self.merge_mode is None: return output + states return [output] + states return output
def call(self, inputs, # inputs: batch_size*max_len*embedding_dim input_lengths, # input_lengths: batch_size dependencies, # dependencies: 2*batch_size*max_len*recurrent_size edge_types, # edge_types: 2*batch_size*max_len*recurrent_size mask=None, # mask: batch_size*max_len cell_mask=None, # mask: 2*batch_size*max_len*recurrent_size initial_state=None, # initial_state: 2*4*batch_size*embedding_dim training=True): def swap_pad(info, lengths): batch_size = info.shape[0] max_len = info.shape[1] new_info = [] for i in range(batch_size): if lengths[i].numpy() == max_len: new_info.append(info[i]) continue stack = [] sentence_info = info[i, 0:lengths[i], :] pad_info = info[i, lengths[i]:, :] sentence_info_slices = tf.split(sentence_info, num_or_size_splits=sentence_info.shape[0], axis=0) pad_info_slices = tf.split(pad_info, num_or_size_splits=pad_info.shape[0], axis=0) for tensor in pad_info_slices: stack.append(tensor) for tensor in sentence_info_slices: stack.append(tensor) new_info.append(tf.squeeze(tf.stack(stack, axis=0), axis=1)) new_info = tf.stack(new_info, axis=0) return new_info if initial_state is not None: forward_inputs, backward_inputs = inputs, inputs forward_state, backward_state = initial_state[0], initial_state[1] forward_dependencies, backward_dependencies = dependencies[0], dependencies[1] forward_edge_types, backward_edge_types = edge_types[0], edge_types[1] forward_cell_mask, backward_cell_mask = cell_mask[0], cell_mask[1] backward_dependencies = swap_pad(backward_dependencies, input_lengths) backward_edge_types = swap_pad(backward_edge_types, input_lengths) backward_cell_mask = swap_pad(backward_cell_mask, input_lengths) else: raise ValueError("Please provide initial states for RNN.") y = self.forward_layer(forward_inputs, input_lengths, forward_dependencies, forward_edge_types, mask, forward_cell_mask, forward_state, training) y_rev = self.backward_layer(backward_inputs, input_lengths, backward_dependencies, backward_edge_types, mask, backward_cell_mask, backward_state, training) if self.return_state: states = y[1:] + y_rev[1:] y = y[0] y_rev = y_rev[0] if self.return_sequences: y_rev = K.reverse(y_rev, 1) if self.merge_mode == 'concat': output = K.concatenate([y, y_rev]) elif self.merge_mode == 'sum': output = y + y_rev elif self.merge_mode == 'ave': output = (y + y_rev) / 2 elif self.merge_mode == 'mul': output = y * y_rev elif self.merge_mode is None: output = [y, y_rev] else: raise ValueError('Unrecognized value for `merge_mode`: %s' % (self.merge_mode)) if self.return_state: if self.merge_mode is None: return output + states return [output] + states return output
def call(self, inputs, training=None, mask=None, initial_state=None, constants=None): """`Bidirectional.call` implements the same API as the wrapped `RNN`.""" kwargs = {} if generic_utils.has_arg(self.layer.call, 'training'): kwargs['training'] = training if generic_utils.has_arg(self.layer.call, 'mask'): kwargs['mask'] = mask if generic_utils.has_arg(self.layer.call, 'constants'): kwargs['constants'] = constants if initial_state is not None and generic_utils.has_arg( self.layer.call, 'initial_state'): forward_inputs = [inputs[0]] backward_inputs = [inputs[0]] pivot = len(initial_state) // 2 + 1 # add forward initial state forward_state = inputs[1:pivot] forward_inputs += forward_state if self._num_constants is None: # add backward initial state backward_state = inputs[pivot:] backward_inputs += backward_state else: # add backward initial state backward_state = inputs[pivot:-self._num_constants] backward_inputs += backward_state # add constants for forward and backward layers forward_inputs += inputs[-self._num_constants:] backward_inputs += inputs[-self._num_constants:] y = self.forward_layer.call(forward_inputs, initial_state=forward_state, **kwargs) y_rev = self.backward_layer.call(backward_inputs, initial_state=backward_state, **kwargs) else: y = self.forward_layer.call(inputs, **kwargs) y_rev = self.backward_layer.call(inputs, **kwargs) if self.return_state: states = y[1:] + y_rev[1:] y = y[0] y_rev = y_rev[0] if self.return_sequences: y_rev = K.reverse(y_rev, 1) if self.merge_mode == 'concat': output = K.concatenate([y, y_rev]) elif self.merge_mode == 'sum': output = y + y_rev elif self.merge_mode == 'ave': output = (y + y_rev) / 2 elif self.merge_mode == 'mul': output = y * y_rev elif self.merge_mode is None: output = [y, y_rev] else: raise ValueError( 'Unrecognized value for `merge_mode`: %s' % (self.merge_mode)) if self.return_state: if self.merge_mode is None: return output + states return [output] + states return output
def get_Model(training): input_shape = (img_w, img_h, 1) # (128, 64, 1) # Make Networkw inputs = Input(name='the_input', shape=input_shape, dtype='float32') # (None, 128, 64, 1) # Convolution layer (VGG) inner = Conv2D(64, (3, 3), padding='same', name='conv1', kernel_initializer='he_normal')( inputs) # (None, 128, 64, 64) inner = BatchNormalization()(inner) inner = Activation('relu')(inner) inner = MaxPooling2D(pool_size=(2, 2), name='max1')(inner) # (None,64, 32, 64) inner = Conv2D(128, (3, 3), padding='same', name='conv2', kernel_initializer='he_normal')( inner) # (None, 64, 32, 128) inner = BatchNormalization()(inner) inner = Activation('relu')(inner) inner = MaxPooling2D(pool_size=(2, 2), name='max2')(inner) # (None, 32, 16, 128) inner = Conv2D(256, (3, 3), padding='same', name='conv3', kernel_initializer='he_normal')( inner) # (None, 32, 16, 256) inner = BatchNormalization()(inner) inner = Activation('relu')(inner) inner = Conv2D(256, (3, 3), padding='same', name='conv4', kernel_initializer='he_normal')( inner) # (None, 32, 16, 256) inner = BatchNormalization()(inner) inner = Activation('relu')(inner) inner = MaxPooling2D(pool_size=(1, 2), name='max3')(inner) # (None, 32, 8, 256) inner = Conv2D(512, (3, 3), padding='same', name='conv5', kernel_initializer='he_normal')(inner) # (None, 32, 8, 512) inner = BatchNormalization()(inner) inner = Activation('relu')(inner) inner = Conv2D(512, (3, 3), padding='same', name='conv6')(inner) # (None, 32, 8, 512) inner = BatchNormalization()(inner) inner = Activation('relu')(inner) inner = MaxPooling2D(pool_size=(1, 2), name='max4')(inner) # (None, 32, 4, 512) inner = Conv2D(512, (2, 2), padding='same', kernel_initializer='he_normal', name='con7')(inner) # (None, 32, 4, 512) inner = BatchNormalization()(inner) inner = Activation('relu')(inner) # CNN to RNN inner = Reshape(target_shape=((32, 2048)), name='reshape')(inner) # (None, 32, 2048) inner = Dense(64, activation='relu', kernel_initializer='he_normal', name='dense1')(inner) # (None, 32, 64) # RNN layer lstm_1 = LSTM(256, return_sequences=True, kernel_initializer='he_normal', name='lstm1')(inner) # (None, 32, 512) lstm_1b = LSTM(256, return_sequences=True, go_backwards=True, kernel_initializer='he_normal', name='lstm1_b')(inner) reversed_lstm_1b = Lambda( lambda inputTensor: K.reverse(inputTensor, axes=1))(lstm_1b) lstm1_merged = add([lstm_1, reversed_lstm_1b]) # (None, 32, 512) lstm1_merged = BatchNormalization()(lstm1_merged) lstm_2 = LSTM(256, return_sequences=True, kernel_initializer='he_normal', name='lstm2')(lstm1_merged) lstm_2b = LSTM(256, return_sequences=True, go_backwards=True, kernel_initializer='he_normal', name='lstm2_b')(lstm1_merged) reversed_lstm_2b = Lambda( lambda inputTensor: K.reverse(inputTensor, axes=1))(lstm_2b) lstm2_merged = concatenate([lstm_2, reversed_lstm_2b]) # (None, 32, 1024) lstm2_merged = BatchNormalization()(lstm2_merged) # transforms RNN output to character activations: inner = Dense(num_classes, kernel_initializer='he_normal', name='dense2')(lstm2_merged) #(None, 32, 63) y_pred = Activation('softmax', name='softmax')(inner) labels = Input(name='the_labels', shape=[max_text_len], dtype='float32') # (None ,8) input_length = Input(name='input_length', shape=[1], dtype='int64') # (None, 1) label_length = Input(name='label_length', shape=[1], dtype='int64') # (None, 1) # Keras doesn't currently support loss funcs with extra parameters # so CTC loss is implemented in a lambda layer loss_out = Lambda(focal_ctc_lambda_func, output_shape=(1, ), name='ctc')([labels, y_pred, input_length, label_length]) #(None, 1) if training: return Model(inputs=[inputs, labels, input_length, label_length], outputs=loss_out) else: return Model(inputs=[inputs], outputs=y_pred)
def lemol_rnn(step_function, inputs, initial_states, go_backwards=False, mask=None, constants=None, unroll=False, input_length=None, time_major=False, zero_output_for_mask=False): print('Using alternative rnn function designed for LeMOL') def swap_batch_timestep(input_t): # Swap the batch and timestep dim for the incoming tensor. axes = list(range(len(input_t.shape))) axes[0], axes[1] = 1, 0 return array_ops.transpose(input_t, axes) if not time_major: inputs = nest.map_structure(swap_batch_timestep, inputs) flatted_inputs = nest.flatten(inputs) time_steps = flatted_inputs[0].shape[0] batch = flatted_inputs[0].shape[1] time_steps_t = array_ops.shape(flatted_inputs[0])[0] for input_ in flatted_inputs: input_.shape.with_rank_at_least(3) if mask is not None: ValueError('Mask Not Supported.\n' + 'Perhaps you want the original rnn function from tf.keras.backend') if constants is None: constants = [] if unroll: if not time_steps: raise ValueError('Unrolling requires a fixed number of timesteps.') states = tuple(initial_states) successive_states = [] successive_outputs = [] successive_learning_features = [] # Process the input tensors. The input tensor need to be split on the # time_step dim, and reverse if go_backwards is True. In the case of nested # input, the input is flattened and then transformed individually. # The result of this will be a tuple of lists, each of the item in tuple is # list of the tensor with shape (batch, feature) def _process_single_input_t(input_t): input_t = array_ops.unstack(input_t) # unstack for time_step dim if go_backwards: input_t.reverse() return input_t if nest.is_sequence(inputs): processed_input = nest.map_structure( _process_single_input_t, inputs) else: processed_input = (_process_single_input_t(inputs),) def _get_input_tensor(time): inp = [t_[time] for t_ in processed_input] return nest.pack_sequence_as(inputs, inp) for i in range(time_steps): inp = _get_input_tensor(i) output, states, learning_feature = step_function( inp, tuple(states) + tuple(constants)) successive_outputs.append(output) successive_states.append(states) successive_learning_features.append(learning_feature) last_output = successive_outputs[-1] new_states = successive_states[-1] outputs = array_ops.stack(successive_outputs) all_learning_featues = array_ops.stack(successive_learning_features) else: states = tuple(initial_states) # Create input tensor array, if the inputs is nested tensors, then it will # be flattened first, and tensor array will be created one per flattened # tensor. input_ta = tuple( tensor_array_ops.TensorArray( dtype=inp.dtype, size=time_steps_t, tensor_array_name='input_ta_%s' % i) for i, inp in enumerate(flatted_inputs)) input_ta = tuple( ta.unstack(input_) if not go_backwards else ta .unstack(reverse(input_, 0)) for ta, input_ in zip(input_ta, flatted_inputs)) # Get the time(0) input and compute the output for that, the output will be # used to determine the dtype of output tensor array. Don't read from # input_ta due to TensorArray clear_after_read default to True. input_time_zero = nest.pack_sequence_as(inputs, [inp[0] for inp in flatted_inputs]) # output_time_zero is used to determine the cell output shape and its dtype. # the value is discarded. output_time_zero, _, lf_time_zero = step_function( input_time_zero, initial_states + constants) output_ta = tuple( tensor_array_ops.TensorArray( dtype=out.dtype, size=time_steps_t, tensor_array_name='output_ta_%s' % i) for i, out in enumerate(nest.flatten(output_time_zero))) lf_out_ta = tuple( tensor_array_ops.TensorArray( dtype=out.dtype, size=time_steps_t, tensor_array_name='output_ta_%s' % i) for i, out in enumerate(nest.flatten(lf_time_zero))) time = constant_op.constant(0, dtype='int32', name='time') while_loop_kwargs = { 'cond': lambda time, *_: time < time_steps_t, 'maximum_iterations': input_length, 'parallel_iterations': 32, 'swap_memory': True, } def _step(time, output_ta_t, output_lf_t, *states): '''RNN step function. Arguments: time: Current timestep value. output_ta_t: TensorArray. *states: List of states. Returns: Tuple: `(time + 1,output_ta_t) + tuple(new_states)` ''' current_input = tuple(ta.read(time) for ta in input_ta) current_input = nest.pack_sequence_as(inputs, current_input) output, new_states, new_lf = step_function(current_input, tuple(states) + tuple(constants)) flat_state = nest.flatten(states) flat_new_state = nest.flatten(new_states) for state, new_state in zip(flat_state, flat_new_state): if hasattr(new_state, 'set_shape'): new_state.set_shape(state.shape) flat_output = nest.flatten(output) output_ta_t = tuple( ta.write(time, out) for ta, out in zip(output_ta_t, flat_output)) flat_lf = nest.flatten(new_lf) output_lf_t = tuple( ta.write(time, out) for ta, out in zip(output_lf_t, flat_lf)) new_states = nest.pack_sequence_as( initial_states, flat_new_state) return (time + 1, output_ta_t, output_lf_t) + tuple(new_states) final_outputs = control_flow_ops.while_loop( body=_step, loop_vars=(time, output_ta, lf_out_ta) + states, **while_loop_kwargs) new_states = final_outputs[3:] output_ta = final_outputs[1] lf_ta = final_outputs[2] lfs = tuple(o.stack() for o in lf_ta) outputs = tuple(o.stack() for o in output_ta) last_output = tuple(o[-1] for o in outputs) last_lf = tuple(o[-1] for o in lfs) outputs = nest.pack_sequence_as(output_time_zero, outputs) lfs = nest.pack_sequence_as(lf_time_zero, lfs) last_output = nest.pack_sequence_as(output_time_zero, last_output) last_lf = nest.pack_sequence_as(lf_time_zero, last_lf) # static shape inference def set_shape(output_): if hasattr(output_, 'set_shape'): shape = output_.shape.as_list() shape[0] = time_steps shape[1] = batch output_.set_shape(shape) return output_ outputs = nest.map_structure(set_shape, outputs) lfs = nest.map_structure(set_shape, lfs) if not time_major: outputs = nest.map_structure(swap_batch_timestep, outputs) lfs = nest.map_structure(swap_batch_timestep, lfs) return last_output, outputs, new_states, last_lf, lfs
def call(self, inputs, mask=None, training=None, initial_state=None): # LSTM does not support constants. Ignore it during process. inputs, initial_state, _ = self._process_inputs( inputs, initial_state, None) if isinstance(mask, list): mask = mask[0] input_shape = K.int_shape(inputs) timesteps = input_shape[0] if self.time_major else input_shape[1] if mask is not None or not self.could_use_cudnn: # CuDNN does not support masking, fall back to use the normal LSTM. kwargs = {'training': training} def step(inputs, states): return self.cell.call(inputs, states, **kwargs) last_output, outputs, states = K.rnn( step, inputs, initial_state, constants=None, go_backwards=self.go_backwards, mask=mask, unroll=self.unroll, input_length=timesteps, time_major=self.time_major, zero_output_for_mask=self.zero_output_for_mask) runtime = _runtime('unknown') else: # Use the new defun approach for backend implementation swap. # Note that different implementations need to have same function # signature, eg, the tensor parameters need to have same shape and dtypes. # Since the CuDNN has an extra set of bias, those bias will be passed to # both normal and CuDNN implementations. if self.go_backwards: # Reverse time axis. inputs = K.reverse(inputs, 0 if self.time_major else 1) self.reset_dropout_mask() dropout_mask = self.get_dropout_mask_for_cell(inputs, training, count=4) if dropout_mask is not None: inputs *= dropout_mask[0] if context.executing_eagerly(): device_type = _get_context_device_type() if device_type == _GPU_DEVICE_NAME or (device_type is None and context.num_gpus() > 0): # Under eager context, check the device placement and prefer the # GPU implementation when GPU is available. last_output, outputs, new_h, new_c, runtime = cudnn_lstm( inputs, initial_state[0], initial_state[1], self.cell.kernel, self.cell.recurrent_kernel, self.cell.bias, self.time_major) else: last_output, outputs, new_h, new_c, runtime = standard_lstm( inputs, initial_state[0], initial_state[1], self.cell.kernel, self.cell.recurrent_kernel, self.cell.bias, self.activation, self.recurrent_activation, self.time_major) else: # Each time a `tf.function` is called, we will give it a unique # identifiable API name, so that Grappler won't get confused when it # sees multiple LSTM layers added into same graph, and it will be able # to pair up the different implementations across them. api_name = 'lstm_' + str(uuid.uuid4()) defun_standard_lstm = _generate_defun_backend( api_name, _CPU_DEVICE_NAME, standard_lstm) defun_cudnn_lstm = _generate_defun_backend( api_name, _GPU_DEVICE_NAME, cudnn_lstm) # Call the normal LSTM impl and register the CuDNN impl function. The # grappler will kick in during session execution to optimize the graph. last_output, outputs, new_h, new_c, runtime = defun_standard_lstm( inputs, initial_state[0], initial_state[1], self.cell.kernel, self.cell.recurrent_kernel, self.cell.bias, self.activation, self.recurrent_activation, self.time_major) function.register(defun_cudnn_lstm, inputs, initial_state[0], initial_state[1], self.cell.kernel, self.cell.recurrent_kernel, self.cell.bias, self.time_major) states = [new_h, new_c] if self.stateful: updates = [] for i in range(len(states)): updates.append(state_ops.assign(self.states[i], states[i])) self.add_update(updates, inputs) if self.return_sequences: output = outputs else: output = last_output if self.return_state: return [output] + list(states) elif self.return_runtime: return output, runtime else: return output