def build_model(self):
data_input = keras.engine.Input(
shape=(self.input_length,),
name='data_input')
data_expanded = layers.AddSingletonDepth()(data_input)
data_out = keras.layers.Convolution1D(self.config['model']['filters']['depths']['res'],
self.config['model']['filters']['lengths']['res'], padding='same',
use_bias=False,
name='initial_causal_conv')(data_expanded)
skip_connections = []
res_block_i = 0
for stack_i in range(self.num_stacks):
layer_in_stack = 0
for dilation in self.dilations:
res_block_i += 1
data_out, skip_out = self.dilated_residual_block(data_out, res_block_i, layer_in_stack, dilation, stack_i)
if skip_out is not None:
skip_connections.append(skip_out)
layer_in_stack += 1
data_out = keras.layers.Add()(skip_connections)
data_out = self.activation(data_out)
data_out = keras.layers.Convolution1D(self.config['model']['filters']['depths']['final'][0],
self.config['model']['filters']['lengths']['final'][0],
padding='same',
use_bias=False)(data_out)
data_out = self.activation(data_out)
data_out = keras.layers.Convolution1D(self.config['model']['filters']['depths']['final'][1],
self.config['model']['filters']['lengths']['final'][1], padding='same',
use_bias=False)(data_out)
data_out = keras.layers.Convolution1D(3, 1)(data_out)
data_out_vocals = layers.Slice((Ellipsis, slice(0, 1)), (self.padded_target_field_length, 1),
name='slice_data_output_1')(data_out)
data_out_drums = layers.Slice((Ellipsis, slice(1, 2)), (self.padded_target_field_length, 1),
name='slice_data_output_2')(data_out)
data_out_bass = layers.Slice((Ellipsis, slice(2, 3)), (self.padded_target_field_length, 1),
name='slice_data_output_3')(data_out)
data_out_vocals = keras.layers.Lambda(lambda x: keras.backend.squeeze(x, 2),
output_shape=lambda shape: (shape[0], shape[1]), name='data_output_1')(
data_out_vocals)
data_out_drums = keras.layers.Lambda(lambda x: keras.backend.squeeze(x, 2),
output_shape=lambda shape: (shape[0], shape[1]), name='data_output_2')(
data_out_drums)
data_out_bass = keras.layers.Lambda(lambda x: keras.backend.squeeze(x, 2),
output_shape=lambda shape: (shape[0], shape[1]), name='data_output_3')(
data_out_bass)
return keras.engine.Model(inputs=[data_input], outputs=[data_out_vocals, data_out_drums, data_out_bass])
def dilated_residual_block(self, data_x, res_block_i, layer_i, dilation, stack_i):
original_x = data_x
data_out = keras.layers.Conv1D(2 * self.config['model']['filters']['depths']['res'],
self.config['model']['filters']['lengths']['res'],
dilation_rate=dilation, padding='same',
use_bias=False,
name='res_%d_dilated_conv_d%d_s%d' % (
res_block_i, dilation, stack_i),
activation=None)(data_x)
data_out_1 = layers.Slice(
(Ellipsis, slice(0, self.config['model']['filters']['depths']['res'])),
(self.input_length, self.config['model']['filters']['depths']['res']),
name='res_%d_data_slice_1_d%d_s%d' % (self.num_residual_blocks, dilation, stack_i))(data_out)
data_out_2 = layers.Slice(
(Ellipsis, slice(self.config['model']['filters']['depths']['res'],
2 * self.config['model']['filters']['depths']['res'])),
(self.input_length, self.config['model']['filters']['depths']['res']),
name='res_%d_data_slice_2_d%d_s%d' % (self.num_residual_blocks, dilation, stack_i))(data_out)
tanh_out = keras.layers.Activation('tanh')(data_out_1)
sigm_out = keras.layers.Activation('sigmoid')(data_out_2)
data_x = keras.layers.Multiply(name='res_%d_gated_activation_%d_s%d' % (res_block_i, layer_i, stack_i))(
[tanh_out, sigm_out])
data_x = keras.layers.Convolution1D(
self.config['model']['filters']['depths']['res'] + self.config['model']['filters']['depths']['skip'], 1,
padding='same', use_bias=False)(data_x)
res_x = layers.Slice((Ellipsis, slice(0, self.config['model']['filters']['depths']['res'])),
(self.input_length, self.config['model']['filters']['depths']['res']),
name='res_%d_data_slice_3_d%d_s%d' % (res_block_i, dilation, stack_i))(data_x)
skip_x = layers.Slice((Ellipsis, slice(self.config['model']['filters']['depths']['res'],
self.config['model']['filters']['depths']['res'] +
self.config['model']['filters']['depths']['skip'])),
(self.input_length, self.config['model']['filters']['depths']['skip']),
name='res_%d_data_slice_4_d%d_s%d' % (res_block_i, dilation, stack_i))(data_x)
skip_x = layers.Slice((slice(self.samples_of_interest_indices[0], self.samples_of_interest_indices[-1] + 1, 1),
Ellipsis), (self.padded_target_field_length, self.config['model']['filters']['depths']['skip']),
name='res_%d_keep_samples_of_interest_d%d_s%d' % (res_block_i, dilation, stack_i))(skip_x)
res_x = keras.layers.Add()([original_x, res_x])
return res_x, skip_x
def dilated_residual_block(self, data_x, condition_x, res_block_i, layer_i, dilation, stack_i):
original_x = data_x
# Data sub-block
data_out = keras.layers.AtrousConv1D(2 * self.config['model']['filters']['depths']['res'],
self.config['model']['filters']['lengths']['res'],
atrous_rate=dilation, border_mode='same',
bias=False,
name='res_%d_dilated_conv_d%d_s%d' % (
res_block_i, dilation, stack_i),
activation=None)(data_x)
data_out_1 = layers.Slice(
(Ellipsis, slice(0, self.config['model']['filters']['depths']['res'])),
(self.input_length, self.config['model']['filters']['depths']['res']),
name='res_%d_data_slice_1_d%d_s%d' % (self.num_residual_blocks, dilation, stack_i))(data_out)
data_out_2 = layers.Slice(
(Ellipsis, slice(self.config['model']['filters']['depths']['res'],
2 * self.config['model']['filters']['depths']['res'])),
(self.input_length, self.config['model']['filters']['depths']['res']),
name='res_%d_data_slice_2_d%d_s%d' % (self.num_residual_blocks, dilation, stack_i))(data_out)
# Condition sub-block
condition_out = keras.layers.Dense(2 * self.config['model']['filters']['depths']['res'],
name='res_%d_dense_condition_%d_s%d' % (res_block_i, layer_i, stack_i),
use_bias=False)(condition_x)
condition_out = keras.layers.Reshape((self.config['model']['filters']['depths']['res'], 2),
name='res_%d_condition_reshape_d%d_s%d' % (
res_block_i, dilation, stack_i))(condition_out)
condition_out_1 = layers.Slice((Ellipsis, 0), (self.config['model']['filters']['depths']['res'],),
name='res_%d_condition_slice_1_d%d_s%d' % (
res_block_i, dilation, stack_i))(condition_out)
condition_out_2 = layers.Slice((Ellipsis, 1), (self.config['model']['filters']['depths']['res'],),
name='res_%d_condition_slice_2_d%d_s%d' % (
res_block_i, dilation, stack_i))(condition_out)
condition_out_1 = keras.layers.RepeatVector(int(self.input_length), name='res_%d_condition_repeat_1_d%d_s%d' % (
res_block_i, dilation, stack_i))(condition_out_1)
condition_out_2 = keras.layers.RepeatVector(int(self.input_length), name='res_%d_condition_repeat_2_d%d_s%d' % (
res_block_i, dilation, stack_i))(condition_out_2)
data_out_1 = keras.layers.Add(name='res_%d_merge_1_d%d_s%d' %
(res_block_i, dilation, stack_i))([data_out_1, condition_out_1])
data_out_2 = keras.layers.Add(name='res_%d_merge_2_d%d_s%d' % (res_block_i, dilation, stack_i))\
([data_out_2, condition_out_2])
tanh_out = keras.layers.Activation('tanh')(data_out_1)
sigm_out = keras.layers.Activation('sigmoid')(data_out_2)
data_x = keras.layers.Multiply(name='res_%d_gated_activation_%d_s%d' % (res_block_i, layer_i, stack_i))(
[tanh_out, sigm_out])
data_x = keras.layers.Conv1D(
self.config['model']['filters']['depths']['res'] + self.config['model']['filters']['depths']['skip'], 1,
padding='same', use_bias=False)(data_x)
res_x = layers.Slice((Ellipsis, slice(0, self.config['model']['filters']['depths']['res'])),
(self.input_length, self.config['model']['filters']['depths']['res']),
name='res_%d_data_slice_3_d%d_s%d' % (res_block_i, dilation, stack_i))(data_x)
skip_x = layers.Slice((Ellipsis, slice(self.config['model']['filters']['depths']['res'],
self.config['model']['filters']['depths']['res'] +
self.config['model']['filters']['depths']['skip'])),
(self.input_length, self.config['model']['filters']['depths']['skip']),
name='res_%d_data_slice_4_d%d_s%d' % (res_block_i, dilation, stack_i))(data_x)
skip_x = layers.Slice((slice(self.samples_of_interest_indices[0], self.samples_of_interest_indices[-1] + 1, 1),
Ellipsis), (self.padded_target_field_length, self.config['model']['filters']['depths']['skip']),
name='res_%d_keep_samples_of_interest_d%d_s%d' % (res_block_i, dilation, stack_i))(skip_x)
res_x = keras.layers.Add()([original_x, res_x])
return res_x, skip_x
def build_model(self):
data_input = keras.engine.Input(
shape=(self.input_length,),
name='data_input')
condition_input = keras.engine.Input(shape=(self.condition_input_length,),
name='condition_input')
data_expanded = layers.AddSingletonDepth()(data_input)
data_input_target_field_length = layers.Slice(
(slice(self.samples_of_interest_indices[0], self.samples_of_interest_indices[-1] + 1, 1), Ellipsis),
(self.padded_target_field_length,1),
name='data_input_target_field_length')(data_expanded)
data_out = keras.layers.Conv1D(self.config['model']['filters']['depths']['res'],
self.config['model']['filters']['lengths']['res'], padding='same',
use_bias=False,
name='initial_causal_conv')(data_expanded)
condition_out = keras.layers.Dense(self.config['model']['filters']['depths']['res'],
name='initial_dense_condition',
use_bias=False)(condition_input)
condition_out = keras.layers.RepeatVector(int(self.input_length),
name='initial_condition_repeat')(condition_out)
data_out = keras.layers.Add(name='initial_data_condition_merge')([data_out, condition_out])
skip_connections = []
res_block_i = 0
for stack_i in range(self.num_stacks):
layer_in_stack = 0
for dilation in self.dilations:
res_block_i += 1
data_out, skip_out = self.dilated_residual_block(data_out, condition_input, res_block_i, layer_in_stack, dilation, stack_i)
if skip_out is not None:
skip_connections.append(skip_out)
layer_in_stack += 1
data_out = keras.layers.Add()(skip_connections)
data_out = self.activation(data_out)
data_out = keras.layers.Conv1D(self.config['model']['filters']['depths']['final'][0],
self.config['model']['filters']['lengths']['final'][0],
padding='same',
bias=False)(data_out)
condition_out = keras.layers.Dense(self.config['model']['filters']['depths']['final'][0],
bias=False,
name='penultimate_conv_1d_condition')(condition_input)
condition_out = keras.layers.RepeatVector(self.padded_target_field_length,
name='penultimate_conv_1d_condition_repeat')(condition_out)
data_out = keras.layers.Add(name='penultimate_conv_1d_condition_merge')([data_out, condition_out])
data_out = self.activation(data_out)
data_out = keras.layers.Conv1D(self.config['model']['filters']['depths']['final'][1],
self.config['model']['filters']['lengths']['final'][1], padding='same',
use_bias=False)(data_out)
condition_out = keras.layers.Dense(self.config['model']['filters']['depths']['final'][1], use_bias=False,
name='final_conv_1d_condition')(condition_input)
condition_out = keras.layers.RepeatVector(self.padded_target_field_length,
name='final_conv_1d_condition_repeat')(condition_out)
data_out = keras.layers.Add(name='final_conv_1d_condition_merge')([data_out, condition_out])
data_out = keras.layers.Conv1D(1, 1)(data_out)
data_out_speech = data_out
data_out_noise = layers.Subtract(name='subtract_layer')([data_input_target_field_length, data_out_speech])
data_out_speech = keras.layers.Lambda(lambda x: keras.backend.squeeze(x, 2),
output_shape=lambda shape: (shape[0], shape[1]), name='data_output_1')(
data_out_speech)
data_out_noise = keras.layers.Lambda(lambda x: keras.backend.squeeze(x, 2),
output_shape=lambda shape: (shape[0], shape[1]), name='data_output_2')(
data_out_noise)
return keras.engine.Model(input=[data_input, condition_input], output=[data_out_speech, data_out_noise])
def build_model(self):
data_input = keras.Input(shape=(self.input_length, ),
name='data_input')
data_expanded = layers.AddSingletonDepth()(data_input)
data_input_target_field_length = layers.Slice(
(slice(self.samples_of_interest_indices[0],
self.samples_of_interest_indices[-1] + 1, 1), Ellipsis),
(self.padded_target_field_length, 1),
name='data_input_target_field_length')(data_expanded)
data_out = keras.layers.Convolution1D(
self.config['model']['filters']['depths']['res'],
self.config['model']['filters']['lengths']['res'],
padding='same',
use_bias=False,
name='initial_causal_conv_1',
kernel_constraint=keras.constraints.max_norm(30))(data_expanded)
skip_connections = []
res_block_i = 0
for stack_i in range(self.num_stacks):
layer_in_stack = 0
for dilation in self.dilations:
res_block_i += 1
data_out, skip_out = self.dilated_residual_block(
data_out, res_block_i, layer_in_stack, dilation, stack_i)
if skip_out is not None:
skip_connections.append(skip_out)
layer_in_stack += 1
data_out = keras.layers.Add()(skip_connections)
# data_out = keras.layers.Concatenate(axis=2)(skip_connections)
# data_out = keras.layers.Convolution1D(self.config['model']['filters']['depths']['res'], 1)(data_out)
data_out = self.activation(data_out)
data_out = keras.layers.Convolution1D(
self.config['model']['filters']['depths']['final'][0],
self.config['model']['filters']['lengths']['final'][0],
padding='same',
use_bias=False)(data_out)
data_out = self.activation(data_out)
data_out = keras.layers.Convolution1D(
self.config['model']['filters']['depths']['final'][1],
self.config['model']['filters']['lengths']['final'][1],
padding='same',
use_bias=False)(data_out)
data_out = keras.layers.Convolution1D(1, 1)(data_out)
data_out_speech = self.fft_layer(data_out)
data_out_noise = layers.Subtract(name='subtract_layer')(
[data_input_target_field_length, data_out_speech])
data_out_speech = keras.layers.Lambda(
lambda x: keras.backend.squeeze(x, -1),
output_shape=lambda shape: (shape[0], shape[1]),
name='data_output_1')(data_out_speech)
data_out_noise = keras.layers.Lambda(
lambda x: keras.backend.squeeze(x, -1),
output_shape=lambda shape: (shape[0], shape[1]),
name='data_output_2')(data_out_noise)
return keras.Model(inputs=[data_input],
outputs=[data_out_speech, data_out_noise])
