def build_model(self): data_input = keras.engine.Input( shape=(self.input_length,), name='data_input') data_expanded = layers.AddSingletonDepth()(data_input) data_out = keras.layers.Convolution1D(self.config['model']['filters']['depths']['res'], self.config['model']['filters']['lengths']['res'], padding='same', use_bias=False, name='initial_causal_conv')(data_expanded) skip_connections = [] res_block_i = 0 for stack_i in range(self.num_stacks): layer_in_stack = 0 for dilation in self.dilations: res_block_i += 1 data_out, skip_out = self.dilated_residual_block(data_out, res_block_i, layer_in_stack, dilation, stack_i) if skip_out is not None: skip_connections.append(skip_out) layer_in_stack += 1 data_out = keras.layers.Add()(skip_connections) data_out = self.activation(data_out) data_out = keras.layers.Convolution1D(self.config['model']['filters']['depths']['final'][0], self.config['model']['filters']['lengths']['final'][0], padding='same', use_bias=False)(data_out) data_out = self.activation(data_out) data_out = keras.layers.Convolution1D(self.config['model']['filters']['depths']['final'][1], self.config['model']['filters']['lengths']['final'][1], padding='same', use_bias=False)(data_out) data_out = keras.layers.Convolution1D(3, 1)(data_out) data_out_vocals = layers.Slice((Ellipsis, slice(0, 1)), (self.padded_target_field_length, 1), name='slice_data_output_1')(data_out) data_out_drums = layers.Slice((Ellipsis, slice(1, 2)), (self.padded_target_field_length, 1), name='slice_data_output_2')(data_out) data_out_bass = layers.Slice((Ellipsis, slice(2, 3)), (self.padded_target_field_length, 1), name='slice_data_output_3')(data_out) data_out_vocals = keras.layers.Lambda(lambda x: keras.backend.squeeze(x, 2), output_shape=lambda shape: (shape[0], shape[1]), name='data_output_1')( data_out_vocals) data_out_drums = keras.layers.Lambda(lambda x: keras.backend.squeeze(x, 2), output_shape=lambda shape: (shape[0], shape[1]), name='data_output_2')( data_out_drums) data_out_bass = keras.layers.Lambda(lambda x: keras.backend.squeeze(x, 2), output_shape=lambda shape: (shape[0], shape[1]), name='data_output_3')( data_out_bass) return keras.engine.Model(inputs=[data_input], outputs=[data_out_vocals, data_out_drums, data_out_bass])
def dilated_residual_block(self, data_x, res_block_i, layer_i, dilation, stack_i): original_x = data_x data_out = keras.layers.Conv1D(2 * self.config['model']['filters']['depths']['res'], self.config['model']['filters']['lengths']['res'], dilation_rate=dilation, padding='same', use_bias=False, name='res_%d_dilated_conv_d%d_s%d' % ( res_block_i, dilation, stack_i), activation=None)(data_x) data_out_1 = layers.Slice( (Ellipsis, slice(0, self.config['model']['filters']['depths']['res'])), (self.input_length, self.config['model']['filters']['depths']['res']), name='res_%d_data_slice_1_d%d_s%d' % (self.num_residual_blocks, dilation, stack_i))(data_out) data_out_2 = layers.Slice( (Ellipsis, slice(self.config['model']['filters']['depths']['res'], 2 * self.config['model']['filters']['depths']['res'])), (self.input_length, self.config['model']['filters']['depths']['res']), name='res_%d_data_slice_2_d%d_s%d' % (self.num_residual_blocks, dilation, stack_i))(data_out) tanh_out = keras.layers.Activation('tanh')(data_out_1) sigm_out = keras.layers.Activation('sigmoid')(data_out_2) data_x = keras.layers.Multiply(name='res_%d_gated_activation_%d_s%d' % (res_block_i, layer_i, stack_i))( [tanh_out, sigm_out]) data_x = keras.layers.Convolution1D( self.config['model']['filters']['depths']['res'] + self.config['model']['filters']['depths']['skip'], 1, padding='same', use_bias=False)(data_x) res_x = layers.Slice((Ellipsis, slice(0, self.config['model']['filters']['depths']['res'])), (self.input_length, self.config['model']['filters']['depths']['res']), name='res_%d_data_slice_3_d%d_s%d' % (res_block_i, dilation, stack_i))(data_x) skip_x = layers.Slice((Ellipsis, slice(self.config['model']['filters']['depths']['res'], self.config['model']['filters']['depths']['res'] + self.config['model']['filters']['depths']['skip'])), (self.input_length, self.config['model']['filters']['depths']['skip']), name='res_%d_data_slice_4_d%d_s%d' % (res_block_i, dilation, stack_i))(data_x) skip_x = layers.Slice((slice(self.samples_of_interest_indices[0], self.samples_of_interest_indices[-1] + 1, 1), Ellipsis), (self.padded_target_field_length, self.config['model']['filters']['depths']['skip']), name='res_%d_keep_samples_of_interest_d%d_s%d' % (res_block_i, dilation, stack_i))(skip_x) res_x = keras.layers.Add()([original_x, res_x]) return res_x, skip_x
def dilated_residual_block(self, data_x, condition_x, res_block_i, layer_i, dilation, stack_i): original_x = data_x # Data sub-block data_out = keras.layers.AtrousConv1D(2 * self.config['model']['filters']['depths']['res'], self.config['model']['filters']['lengths']['res'], atrous_rate=dilation, border_mode='same', bias=False, name='res_%d_dilated_conv_d%d_s%d' % ( res_block_i, dilation, stack_i), activation=None)(data_x) data_out_1 = layers.Slice( (Ellipsis, slice(0, self.config['model']['filters']['depths']['res'])), (self.input_length, self.config['model']['filters']['depths']['res']), name='res_%d_data_slice_1_d%d_s%d' % (self.num_residual_blocks, dilation, stack_i))(data_out) data_out_2 = layers.Slice( (Ellipsis, slice(self.config['model']['filters']['depths']['res'], 2 * self.config['model']['filters']['depths']['res'])), (self.input_length, self.config['model']['filters']['depths']['res']), name='res_%d_data_slice_2_d%d_s%d' % (self.num_residual_blocks, dilation, stack_i))(data_out) # Condition sub-block condition_out = keras.layers.Dense(2 * self.config['model']['filters']['depths']['res'], name='res_%d_dense_condition_%d_s%d' % (res_block_i, layer_i, stack_i), use_bias=False)(condition_x) condition_out = keras.layers.Reshape((self.config['model']['filters']['depths']['res'], 2), name='res_%d_condition_reshape_d%d_s%d' % ( res_block_i, dilation, stack_i))(condition_out) condition_out_1 = layers.Slice((Ellipsis, 0), (self.config['model']['filters']['depths']['res'],), name='res_%d_condition_slice_1_d%d_s%d' % ( res_block_i, dilation, stack_i))(condition_out) condition_out_2 = layers.Slice((Ellipsis, 1), (self.config['model']['filters']['depths']['res'],), name='res_%d_condition_slice_2_d%d_s%d' % ( res_block_i, dilation, stack_i))(condition_out) condition_out_1 = keras.layers.RepeatVector(int(self.input_length), name='res_%d_condition_repeat_1_d%d_s%d' % ( res_block_i, dilation, stack_i))(condition_out_1) condition_out_2 = keras.layers.RepeatVector(int(self.input_length), name='res_%d_condition_repeat_2_d%d_s%d' % ( res_block_i, dilation, stack_i))(condition_out_2) data_out_1 = keras.layers.Add(name='res_%d_merge_1_d%d_s%d' % (res_block_i, dilation, stack_i))([data_out_1, condition_out_1]) data_out_2 = keras.layers.Add(name='res_%d_merge_2_d%d_s%d' % (res_block_i, dilation, stack_i))\ ([data_out_2, condition_out_2]) tanh_out = keras.layers.Activation('tanh')(data_out_1) sigm_out = keras.layers.Activation('sigmoid')(data_out_2) data_x = keras.layers.Multiply(name='res_%d_gated_activation_%d_s%d' % (res_block_i, layer_i, stack_i))( [tanh_out, sigm_out]) data_x = keras.layers.Conv1D( self.config['model']['filters']['depths']['res'] + self.config['model']['filters']['depths']['skip'], 1, padding='same', use_bias=False)(data_x) res_x = layers.Slice((Ellipsis, slice(0, self.config['model']['filters']['depths']['res'])), (self.input_length, self.config['model']['filters']['depths']['res']), name='res_%d_data_slice_3_d%d_s%d' % (res_block_i, dilation, stack_i))(data_x) skip_x = layers.Slice((Ellipsis, slice(self.config['model']['filters']['depths']['res'], self.config['model']['filters']['depths']['res'] + self.config['model']['filters']['depths']['skip'])), (self.input_length, self.config['model']['filters']['depths']['skip']), name='res_%d_data_slice_4_d%d_s%d' % (res_block_i, dilation, stack_i))(data_x) skip_x = layers.Slice((slice(self.samples_of_interest_indices[0], self.samples_of_interest_indices[-1] + 1, 1), Ellipsis), (self.padded_target_field_length, self.config['model']['filters']['depths']['skip']), name='res_%d_keep_samples_of_interest_d%d_s%d' % (res_block_i, dilation, stack_i))(skip_x) res_x = keras.layers.Add()([original_x, res_x]) return res_x, skip_x
def build_model(self): data_input = keras.engine.Input( shape=(self.input_length,), name='data_input') condition_input = keras.engine.Input(shape=(self.condition_input_length,), name='condition_input') data_expanded = layers.AddSingletonDepth()(data_input) data_input_target_field_length = layers.Slice( (slice(self.samples_of_interest_indices[0], self.samples_of_interest_indices[-1] + 1, 1), Ellipsis), (self.padded_target_field_length,1), name='data_input_target_field_length')(data_expanded) data_out = keras.layers.Conv1D(self.config['model']['filters']['depths']['res'], self.config['model']['filters']['lengths']['res'], padding='same', use_bias=False, name='initial_causal_conv')(data_expanded) condition_out = keras.layers.Dense(self.config['model']['filters']['depths']['res'], name='initial_dense_condition', use_bias=False)(condition_input) condition_out = keras.layers.RepeatVector(int(self.input_length), name='initial_condition_repeat')(condition_out) data_out = keras.layers.Add(name='initial_data_condition_merge')([data_out, condition_out]) skip_connections = [] res_block_i = 0 for stack_i in range(self.num_stacks): layer_in_stack = 0 for dilation in self.dilations: res_block_i += 1 data_out, skip_out = self.dilated_residual_block(data_out, condition_input, res_block_i, layer_in_stack, dilation, stack_i) if skip_out is not None: skip_connections.append(skip_out) layer_in_stack += 1 data_out = keras.layers.Add()(skip_connections) data_out = self.activation(data_out) data_out = keras.layers.Conv1D(self.config['model']['filters']['depths']['final'][0], self.config['model']['filters']['lengths']['final'][0], padding='same', bias=False)(data_out) condition_out = keras.layers.Dense(self.config['model']['filters']['depths']['final'][0], bias=False, name='penultimate_conv_1d_condition')(condition_input) condition_out = keras.layers.RepeatVector(self.padded_target_field_length, name='penultimate_conv_1d_condition_repeat')(condition_out) data_out = keras.layers.Add(name='penultimate_conv_1d_condition_merge')([data_out, condition_out]) data_out = self.activation(data_out) data_out = keras.layers.Conv1D(self.config['model']['filters']['depths']['final'][1], self.config['model']['filters']['lengths']['final'][1], padding='same', use_bias=False)(data_out) condition_out = keras.layers.Dense(self.config['model']['filters']['depths']['final'][1], use_bias=False, name='final_conv_1d_condition')(condition_input) condition_out = keras.layers.RepeatVector(self.padded_target_field_length, name='final_conv_1d_condition_repeat')(condition_out) data_out = keras.layers.Add(name='final_conv_1d_condition_merge')([data_out, condition_out]) data_out = keras.layers.Conv1D(1, 1)(data_out) data_out_speech = data_out data_out_noise = layers.Subtract(name='subtract_layer')([data_input_target_field_length, data_out_speech]) data_out_speech = keras.layers.Lambda(lambda x: keras.backend.squeeze(x, 2), output_shape=lambda shape: (shape[0], shape[1]), name='data_output_1')( data_out_speech) data_out_noise = keras.layers.Lambda(lambda x: keras.backend.squeeze(x, 2), output_shape=lambda shape: (shape[0], shape[1]), name='data_output_2')( data_out_noise) return keras.engine.Model(input=[data_input, condition_input], output=[data_out_speech, data_out_noise])
def build_model(self): data_input = keras.Input(shape=(self.input_length, ), name='data_input') data_expanded = layers.AddSingletonDepth()(data_input) data_input_target_field_length = layers.Slice( (slice(self.samples_of_interest_indices[0], self.samples_of_interest_indices[-1] + 1, 1), Ellipsis), (self.padded_target_field_length, 1), name='data_input_target_field_length')(data_expanded) data_out = keras.layers.Convolution1D( self.config['model']['filters']['depths']['res'], self.config['model']['filters']['lengths']['res'], padding='same', use_bias=False, name='initial_causal_conv_1', kernel_constraint=keras.constraints.max_norm(30))(data_expanded) skip_connections = [] res_block_i = 0 for stack_i in range(self.num_stacks): layer_in_stack = 0 for dilation in self.dilations: res_block_i += 1 data_out, skip_out = self.dilated_residual_block( data_out, res_block_i, layer_in_stack, dilation, stack_i) if skip_out is not None: skip_connections.append(skip_out) layer_in_stack += 1 data_out = keras.layers.Add()(skip_connections) # data_out = keras.layers.Concatenate(axis=2)(skip_connections) # data_out = keras.layers.Convolution1D(self.config['model']['filters']['depths']['res'], 1)(data_out) data_out = self.activation(data_out) data_out = keras.layers.Convolution1D( self.config['model']['filters']['depths']['final'][0], self.config['model']['filters']['lengths']['final'][0], padding='same', use_bias=False)(data_out) data_out = self.activation(data_out) data_out = keras.layers.Convolution1D( self.config['model']['filters']['depths']['final'][1], self.config['model']['filters']['lengths']['final'][1], padding='same', use_bias=False)(data_out) data_out = keras.layers.Convolution1D(1, 1)(data_out) data_out_speech = self.fft_layer(data_out) data_out_noise = layers.Subtract(name='subtract_layer')( [data_input_target_field_length, data_out_speech]) data_out_speech = keras.layers.Lambda( lambda x: keras.backend.squeeze(x, -1), output_shape=lambda shape: (shape[0], shape[1]), name='data_output_1')(data_out_speech) data_out_noise = keras.layers.Lambda( lambda x: keras.backend.squeeze(x, -1), output_shape=lambda shape: (shape[0], shape[1]), name='data_output_2')(data_out_noise) return keras.Model(inputs=[data_input], outputs=[data_out_speech, data_out_noise])