def _streaming_internal_state(self, inputs): if self.use_one_step: # The time dimenstion always has to equal 1 in streaming mode. if inputs.shape[1] != 1: raise ValueError('inputs.shape[1]: %d must be 1 ' % inputs.shape[1]) # remove latest row [batch_size, (memory_size-1), feature_dim, channel] memory = self.states[:, 1:self.ring_buffer_size_in_time_dim, :] # add new row [batch_size, memory_size, feature_dim, channel] memory = tf.keras.backend.concatenate([memory, inputs], 1) assign_states = self.states.assign(memory) with tf.control_dependencies([assign_states]): return self.cell(memory) else: # add new row [batch_size, memory_size, feature_dim, channel] if self.ring_buffer_size_in_time_dim: memory = tf.keras.backend.concatenate([self.states, inputs], 1) state_update = memory[:, -self.ring_buffer_size_in_time_dim:, :] # pylint: disable=invalid-unary-operand-type assign_states = self.states.assign(state_update) with tf.control_dependencies([assign_states]): return self.cell(memory) else: return self.cell(inputs)
def _streaming_internal_state(self, inputs): outputs = super(Conv1DTranspose, self).call(inputs) if self.overlap == 0: if self.crop_output: return tf.identity(outputs[:, 0:self.output_time_dim, :]) else: return tf.identity(outputs) output_shape = outputs.shape.as_list() # need to add remainder state to a specific region of output as below: # outputs[:,0:self.overlap,:] = outputs[:,0:self.overlap,:] + self.states # but 'Tensor' object does not support item assignment, # so doing it through full summation below output_shape[1] -= self.state_shape[1] padded_remainder = tf.concat( [self.states, tf.zeros(output_shape, tf.float32)], 1) outputs = outputs + padded_remainder # extract remainder state and substruct bias if it is used: # bias will be added in the next iteration again and remainder # should have only convolution part, so that bias is not added twice if self.use_bias: new_state = outputs[:, -self.overlap:, :] - self.bias else: new_state = outputs[:, -self.overlap:, :] assign_states = self.states.assign(new_state) with tf.control_dependencies([assign_states]): if self.crop_output: return tf.identity(outputs[:, 0:self.output_time_dim, :]) else: return tf.identity(outputs)
def _streaming_internal_state(self, inputs): # depthwise 1D convolution in streaming mode with internal state # it is used for streaming inference if inputs.shape[1] != 1: # [batch, 1, feature] raise ValueError('inputs.shape[1]: %d must be 1 ' % inputs.shape[1]) # remove latest row [batch_size, (memory_size-1), feature_dim] memory = self.states[:, 1:self.memory_size, :] # add new row [batch_size, memory_size, feature_dim] memory = tf.keras.backend.concatenate([memory, inputs], 1) assign_states = self.states.assign(memory) with tf.control_dependencies([assign_states]): # elementwise multiplication [batch_size, memory_size, feature_dim] output = memory * self.time_kernel # [batch_size, feature_dim] output_sum = tf.keras.backend.sum(output, axis=1) if self.use_bias: output_sum = output_sum + self.bias output_sum = tf.keras.backend.expand_dims(output_sum, -2) return output_sum # [batch, 1, feature]
def _streaming_internal_state(self, inputs): # first dimension is batch size if inputs.shape[0] != self.inference_batch_size: raise ValueError( 'inputs.shape[0]:%d must be = self.inference_batch_size:%d' % (inputs.shape[0], self.inference_batch_size)) # second dimension is frame_step if inputs.shape[1] != self.frame_step: raise ValueError( 'inputs.shape[1]:%d must be = self.frame_step:%d' % (inputs.shape[1], self.frame_step)) # remove latest rows [batch_size, (frame_size-frame_step)] memory = self.states[:, self.frame_step:self.frame_size] # add new rows [batch_size, memory_size] memory = tf.keras.backend.concatenate([memory, inputs], 1) assign_states = self.states.assign(memory) with tf.control_dependencies([assign_states]): # add time dim output_frame = tf.keras.backend.expand_dims(memory, -2) return output_frame
def _streaming_internal_state(self, inputs): inversed_frames, new_states = self._streaming_external_state( inputs, self.states) assign_states = self.states.assign(new_states) with tf.control_dependencies([assign_states]): # use tf.identity to ensure that assign_states is executed return tf.identity(inversed_frames)
def _streaming_internal_state(self, inputs): outputs = super(Conv1DTranspose, self).call(inputs) if self.overlap == 0: if self.crop_output: return tf.identity(outputs[:, 0:self.output_time_dim, :]) else: return tf.identity(outputs) output_shape = outputs.shape.as_list() # need to add remainder state to a specific region of output as below: # outputs[:,0:self.overlap,:] = outputs[:,0:self.overlap,:] + self.states # but 'Tensor' object does not support item assignment, # so doing it through full summation below output_shape[1] -= self.state_shape[1] padded_remainder = tf.concat( [self.states, tf.zeros(output_shape, tf.float32)], 1) outputs = outputs + padded_remainder new_state = outputs[:, -self.overlap:, :] assign_states = self.states.assign(new_state) with tf.control_dependencies([assign_states]): if self.crop_output: return tf.identity(outputs[:, 0:self.output_time_dim, :]) else: return tf.identity(outputs)
def _streaming_internal_state(self, inputs): memory = tf.keras.backend.concatenate([self.states, inputs], 1) outputs = memory[:, 0:inputs.shape.as_list()[1], :] new_memory = memory[:, -self.delay:, :] assign_states = self.states.assign(new_memory) with tf.control_dependencies([assign_states]): return tf.identity(outputs)
def _streaming_internal_state(self, inputs): new_state = self.state + 1.0 new_state = tf.math.minimum(new_state, self.max_counter + 1) assign_state = self.state.assign(new_state) with tf.control_dependencies([assign_state]): multiplier = tf.keras.activations.relu(new_state[0][0][0], max_value=1.0, threshold=self.max_counter) outputs = tf.multiply(inputs, multiplier) return outputs
def _streaming_internal_state(self, inputs): # first dimension is batch size if inputs.shape[0] != self.inference_batch_size: raise ValueError( 'inputs.shape[0]:%d must be = self.inference_batch_size:%d' % (inputs.shape[0], self.inference_batch_size)) if self.use_one_step: # use_one_step is used only for backward compatibility # it assumes that frame_size and frame_step overlap # below version with tf.signal.frame is more generic # second dimension is frame_step if inputs.shape[1] != self.frame_step: raise ValueError( 'inputs.shape[1]:%d must be = self.frame_step:%d' % (inputs.shape[1], self.frame_step)) # remove latest rows [batch_size, (frame_size-frame_step)] memory = self.states[:, self.frame_step:self.frame_size] # add new rows [batch_size, memory_size] memory = tf.keras.backend.concatenate([memory, inputs], 1) assign_states = self.states.assign(memory) with tf.control_dependencies([assign_states]): # add time dim output_frame = tf.keras.backend.expand_dims(memory, -2) return output_frame else: memory = tf.keras.backend.concatenate([self.states, inputs], 1) state_update = memory[:, -self.ring_buffer_size_in_time_dim:] assign_states = self.states.assign(state_update) with tf.control_dependencies([assign_states]): output_frame = tf.signal.frame(memory, frame_length=self.frame_size, frame_step=self.frame_step) return output_frame
def _streaming_internal_state(self, inputs): # The time dimenstion always has to equal 1 in streaming mode. if inputs.shape[1] != 1: raise ValueError('inputs.shape[1]: %d must be 1 ' % inputs.shape[1]) # remove latest row [batch_size, (memory_size-1), feature_dim, channel] memory = self.states[:, 1:self.effective_ksize_tdim, :] # add new row [batch_size, memory_size, feature_dim, channel] memory = tf.keras.backend.concatenate([memory, inputs], 1) assign_states = self.states.assign(memory) with tf.control_dependencies([assign_states]): return self.cell(memory)
def _streaming_internal_state(self, inputs): # first dimension is batch size if inputs.shape[0] != self.inference_batch_size: raise ValueError( 'inputs.shape[0]:%d must be = self.inference_batch_size:%d' % (inputs.shape[0], self.inference_batch_size)) # receive inputs: [batch, 1, feature] # convert it for gru cell to inputs1: [batch, feature] inputs = tf.keras.backend.squeeze(inputs, axis=1) output, states = self.gru_cell(inputs, [self.input_state]) # update internal states assign_state = self.input_state.assign(states[0]) with tf.control_dependencies([assign_state]): # output [batch, 1, feature] output = tf.keras.backend.expand_dims(output, axis=1) return output
def _streaming_internal_state(self, inputs): if isinstance(self.get_core_layer(), tf.keras.layers.Conv2DTranspose): outputs = self.cell(inputs) if self.ring_buffer_size_in_time_dim == 0: if self.transposed_conv_crop_output: outputs = outputs[:, 0:self.output_time_dim] return outputs output_shape = outputs.shape.as_list() # need to add remainder state to a specific region of output as below: # outputs[:,0:self.ring_buffer_size_in_time_dim,:] = # outputs[:,0:self.ring_buffer_size_in_time_dim,:] + self.states # but 'Tensor' object does not support item assignment, # so doing it through full summation below output_shape[1] -= self.state_shape[1] padded_remainder = tf.concat( [self.states, tf.zeros(output_shape, tf.float32)], 1) outputs = outputs + padded_remainder # extract remainder state and subtract bias if it is used: # bias will be added in the next iteration again and remainder # should have only convolution part, so that bias is not added twice if self.get_core_layer().get_config()['use_bias']: # need to access bias of the cell layer, # where cell can be wrapped by wrapper layer bias = self.get_core_layer().bias new_state = outputs[:, -self.ring_buffer_size_in_time_dim:, :] - bias # pylint: disable=invalid-unary-operand-type else: new_state = outputs[:, -self.ring_buffer_size_in_time_dim:, :] # pylint: disable=invalid-unary-operand-type assign_states = self.states.assign(new_state) with tf.control_dependencies([assign_states]): if self.transposed_conv_crop_output: return tf.identity(outputs[:, 0:self.output_time_dim, :]) else: return tf.identity(outputs) else: if self.use_one_step: # The time dimenstion always has to equal 1 in streaming mode. if inputs.shape[1] != 1: raise ValueError('inputs.shape[1]: %d must be 1 ' % inputs.shape[1]) # remove latest row [batch_size, (memory_size-1), feature_dim, channel] memory = self.states[:, 1:self.ring_buffer_size_in_time_dim, :] # add new row [batch_size, memory_size, feature_dim, channel] memory = tf.keras.backend.concatenate([memory, inputs], 1) assign_states = self.states.assign(memory) with tf.control_dependencies([assign_states]): return self.cell(memory) else: # add new row [batch_size, memory_size, feature_dim, channel] if self.ring_buffer_size_in_time_dim: memory = tf.keras.backend.concatenate([self.states, inputs], 1) state_update = memory[:, -self.ring_buffer_size_in_time_dim:, :] # pylint: disable=invalid-unary-operand-type assign_states = self.states.assign(state_update) with tf.control_dependencies([assign_states]): return self.cell(memory) else: return self.cell(inputs)