def orchestra_embedding(self, orch_past):
conv_layer = self.weights["orch_embed_conv"]
conv_layer_timeDist = TimeDistributed(
conv_layer,
input_shape=(self.temporal_order - 1, self.orch_dim, 1))
with tf.name_scope("build_orch_input"):
# Format as a 4D
input_seq = tf.reshape(
orch_past, [-1, self.temporal_order - 1, self.orch_dim, 1])
with tf.name_scope("conv_orch"):
o0 = conv_layer_timeDist(input_seq)
keras_layer_summary(conv_layer)
# Remove the last useless dimension
cropped_orch_dim = self.orch_dim - self.kernel_size_orch + 1
o0 = tf.reshape(o0, [-1, self.temporal_order - 1, cropped_orch_dim])
with tf.name_scope("gru"):
orchestra_embedding = stacked_rnn(
o0,
self.hs_orch,
rnn_type='gru',
weight_decay_coeff=self.weight_decay_coeff,
dropout_probability=self.dropout_probability,
activation='relu')
return orchestra_embedding
def predict(self, inputs_ph):
piano_t, piano_past, piano_future, orch_past, _ = inputs_ph
with tf.name_scope("piano_embedding"):
piano_embedding_past, piano_embedding_fut = self.piano_embedding(piano_t, piano_past, piano_future)
with tf.name_scope("orchestra_embedding"):
orchestra_embedding = self.orchestra_embedding(orch_past)
#####################
# Concatenate and predict
with tf.name_scope("top_level_prediction"):
with tf.name_scope("layer_0"):
top_input = keras.layers.concatenate([orchestra_embedding, piano_embedding_past, piano_embedding_fut], axis=1)
dense_layer = Dense(1000, activation='relu', name='orch_pred_0')
top_0 = dense_layer(top_input)
keras_layer_summary(dense_layer, collections=['weights'])
with tf.name_scope("layer_1"):
dense_layer = Dense(self.orch_dim, activation='sigmoid', name='orch_pred')
orch_prediction = dense_layer(top_0)
keras_layer_summary(dense_layer, collections=['weights'])
#####################
return orch_prediction, top_input
def predict(self, inputs_ph):
piano_t, piano_past, piano_future, orch_past, _ = inputs_ph
with tf.name_scope("piano_embedding"):
piano_t_emb, piano_past_emb, piano_future_emb = self.piano_embedding(
piano_t, piano_past, piano_future)
with tf.name_scope("orchestra_embedding"):
orchestra_emb = self.orchestra_embedding(orch_past)
#####################
# Concatenate and predict
with tf.name_scope("top_level_prediction"):
embedding_concat = tf.concat(
[orchestra_emb, piano_past_emb, piano_future_emb, piano_t_emb],
axis=1)
if self.dropout_probability > 0:
top_input_drop = Dropout(
self.dropout_probability)(embedding_concat)
else:
top_input_drop = embedding_concat
dense_layer = Dense(self.orch_dim,
activation='sigmoid',
name='orch_pred')
orch_prediction = dense_layer(top_input_drop)
keras_layer_summary(dense_layer)
#####################
return orch_prediction, orch_prediction
def piano_embedding(self, piano_t, piano_seq, reverse):
with tf.name_scope("build_piano_input"):
# Add a time axis to piano_t
piano_t_time = tf.reshape(piano_t, [-1, 1, self.piano_dim])
# Concatenate t and future
input_seq = tf.concat([piano_t_time, piano_seq], 1)
if reverse:
# Flip the matrix along the time axis so that the last time index is t
input_seq = tf.reverse(input_seq, [1])
# Format as a 4D
input_seq = tf.reshape(input_seq, [-1, self.temporal_order, self.piano_dim, 1])
with tf.name_scope("conv_piano"):
conv_layer = Conv1D(1, self.kernel_size_piano, activation='relu')
conv_layer_timeDist = TimeDistributed(conv_layer, input_shape=(self.temporal_order, self.piano_dim, 1))
p0 = conv_layer_timeDist(input_seq)
keras_layer_summary(conv_layer)
# Remove the last useless dimension
cropped_piano_dim = self.piano_dim-self.kernel_size_piano+1
p0 = tf.reshape(p0, [-1, self.temporal_order, cropped_piano_dim])
with tf.name_scope("gru"):
piano_emb = stacked_rnn(p0, self.hs_piano,
rnn_type='gru',
weight_decay_coeff=self.weight_decay_coeff,
dropout_probability=self.dropout_probability,
activation='relu'
)
return piano_emb
def MLP(x, layers, activation='relu'):
for layer_ind, num_unit in enumerate(layers):
with tf.variable_scope(str(layer_ind)):
dense_layer = Dense(num_unit, activation=activation)
x = dense_layer(x)
keras_layer_summary(dense_layer)
return x
def predict(self, inputs_ph):
piano_t, _, _, orch_past, _ = inputs_ph
FiLM_Coeff = self.FiLM_generator(orch_past,
self.weights['FiLM_generator'])
# Unpack
ind_start = 0
ind_end = ind_start + self.FilM_dim_0
beta_0 = FiLM_Coeff[:, ind_start:ind_end]
ind_start = ind_end
ind_end = ind_start + self.FilM_dim_0
gamma_0 = FiLM_Coeff[:, ind_start:ind_end]
ind_start = ind_end
ind_end = ind_start + self.FilM_dim_1
beta_1 = FiLM_Coeff[:, ind_start:ind_end]
ind_start = ind_end
ind_end = ind_start + self.FilM_dim_1
gamma_1 = FiLM_Coeff[:, ind_start:ind_end]
# Adapt input dimension
first_layer = self.weights["first_layer"]
x = first_layer(piano_t)
keras_layer_summary(first_layer, collections=["weights"])
x = self.residual_FiLM(x, gamma_0, beta_0, self.weights["block_0"])
x = self.residual_FiLM(x, gamma_1, beta_1, self.weights["block_1"])
last_layer = self.weights["last_layer"]
orch_prediction = last_layer(x)
keras_layer_summary(last_layer, collections=["weights"])
return orch_prediction, orch_prediction
def orchestra_embedding(self, orch_past):
with tf.name_scope("past"):
# Concatenate t and future
x = orch_past
for layer_ind, gru_layer in enumerate(
self.weights["past_orchestra"]):
with tf.name_scope("l_" + str(layer_ind)):
x = gru_layer(x)
keras_layer_summary(gru_layer)
return x
def predict(self, inputs_ph, pitch_mask):
piano_t, _, _, orch_t, orch_past, orch_future = inputs_ph
with tf.name_scope("orch_past"):
x = orch_past
for layer in self.weights["orch_past"]:
x = layer(x)
keras_layer_summary(layer, collections=["weights"])
orch_past_embedding = x
with tf.name_scope("orch_future"):
x = orch_future
for layer in self.weights["orch_future"]:
x = layer(x)
keras_layer_summary(layer, collections=["weights"])
orch_future_embedding = x
with tf.name_scope("present_piano"):
x = piano_t
for layer in self.weights["piano_present"]:
x = layer(x)
keras_layer_summary(layer, collections=["weights"])
piano_t_embedding = x
with tf.name_scope("present_orchestra"):
with tf.name_scope("build_input"):
# pitch_mask_reshape = tf.reshape(pitch_mask, [1, self.orch_dim])
pitch_mask_reshape = pitch_mask
masked_orch_t = tf.multiply(orch_t, pitch_mask_reshape)
x = tf.concat([masked_orch_t, pitch_mask_reshape], axis=1)
for ind_layer, layer in enumerate(self.weights["orch_present"]):
x = layer(x)
keras_layer_summary(layer, collections=["weights"])
orch_t_embedding = x
with tf.name_scope("final_pred"):
# Perhaps concatenate the mask ?
x = tf.concat([
orch_past_embedding, orch_future_embedding, piano_t_embedding,
orch_t_embedding
],
axis=1)
# for layer_ind in range(len(self.weights["final_pred"])-1):
# layer = self.weights["final_pred"][layer_ind]
# x = layer(x)
# keras_layer_summary(layer, collections=["weights"])
# W, b = self.weights["final_pred"][-1]
# pred = tf.sigmoid(tf.matmul(x, W) + b)
for layer in self.weights["final_pred"]:
x = layer(x)
keras_layer_summary(layer, collections=["weights"])
pred = x
return pred, pred
def predict(self, inputs_ph, pitch_mask):
piano_t, _, _, orch_t, orch_past, orch_future = inputs_ph
with tf.name_scope("orch_past"):
x = orch_past
for layer in self.weights["orch_past"]:
x = layer(x)
keras_layer_summary(layer, collections=["weights"])
orch_past_embedding = x
with tf.name_scope("orch_future"):
x = orch_future
for layer in self.weights["orch_future"]:
x = layer(x)
keras_layer_summary(layer, collections=["weights"])
orch_future_embedding = x
with tf.name_scope("present_piano"):
x = piano_t
for layer in self.weights["piano_present"]:
x = layer(x)
keras_layer_summary(layer, collections=["weights"])
piano_t_embedding = x
with tf.name_scope("present_orchestra"):
for ind_layer, layer in enumerate(self.weights["orch_present"]):
if ind_layer == 0:
# Remove pitch from orch and weight
W, b = layer
with tf.name_scope("masking_orch_t"):
orch_t_masked = tf.boolean_mask(orch_t,
pitch_mask,
axis=1)
W_masked = tf.boolean_mask(W, pitch_mask, axis=0)
# Mask out the column corresponding to the predicted pitch
x = tf.matmul(orch_t_masked, W_masked) + b
else:
x = layer(x)
keras_layer_summary(layer, collections=["weights"])
orch_t_embedding = x
with tf.name_scope("final_pred"):
# Perhaps concatenate the mask ?
x = tf.concat([
orch_past_embedding, orch_future_embedding, piano_t_embedding,
orch_t_embedding
],
axis=1)
for layer in self.weights["final_pred"]:
x = layer(x)
keras_layer_summary(layer, collections=["weights"])
pred = x
# pred has size (batch, 1)
return pred, pred
def residual_FiLM(self, x, gamma, beta, layers):
original = x
# Stacked MLPs
for layer in layers:
x = layer(x)
keras_layer_summary(layer, collections=["weights"])
# FiLM
x = tf.multiply(gamma, x) + beta
# Relu
x = keras.activations.relu(x)
return x
def predict(self, inputs_ph):
piano_t, _, _, _, _ = inputs_ph
#####################
# Embedding piano
with tf.name_scope("piano_embedding"):
dense_layer = Dense(
self.n_hs, activation='relu'
) # fully-connected layer with 128 units and ReLU activation
piano_embedding_ = dense_layer(piano_t)
if self.dropout_probability > 0:
piano_embedding = Dropout(
self.dropout_probability)(piano_embedding_)
else:
piano_embedding = piano_embedding_
keras_layer_summary(dense_layer, collections=["weights"])
# num_filter = 30
# conv_layer = Conv1D(num_filter, 12, activation='relu')
# piano_t_reshape = tf.reshape(piano_t, [-1, self.piano_dim, 1])
# piano_embedding = conv_layer(piano_t_reshape)
# piano_embedding = tf.reshape(piano_embedding, [-1, (self.piano_dim-12+1) * num_filter])
# keras_layer_summary(conv_layer, collections=["weights"])
#####################
#####################
# Concatenate and predict
with tf.name_scope("top_layer_prediction"):
embedding_concat = piano_embedding
if self.dropout_probability > 0:
top_input_drop = Dropout(
self.dropout_probability)(embedding_concat)
else:
top_input_drop = embedding_concat
# dense_layer = Dense(self.orch_dim, activation='relu', name='orch_pred')
dense_layer = Dense(self.orch_dim,
activation='sigmoid',
name='orch_pred')
orch_prediction = dense_layer(top_input_drop)
keras_layer_summary(dense_layer, collections=["weights"])
#####################
return orch_prediction, embedding_concat
# 'temporal_order' : 5,
# 'dropout_probability' : 0,
# 'weight_decay_coeff' : 0,
# 'n_hidden': [500, 500],
def predict_knowing_context(self, context_embedding_precomputed, orch_pred, mask):
with tf.name_scope("predict_knowing_context"):
# Build input
x = tf.multiply(orch_pred - 0.5, mask)
for dense in self.weights["MLP_NADE"]:
x = dense(x)
keras_layer_summary(dense, collections=["weights"])
if self.dropout_probability > 0:
x = Dropout(self.dropout_probability)(x)
# concatenate
x = tf.concat([context_embedding_precomputed, x], axis=1)
dense = self.weights["last_MLP"]
orch_prediction = dense(x)
keras_layer_summary(dense, collections=["weights"])
return orch_prediction, x
def embed_context(self, inputs_ph):
# MLP on concatenation of past orchestra and present piano
with tf.name_scope("embed_context"):
piano_t, _, _, orch_past, _ = inputs_ph
orch_past_flat = tf.reshape(orch_past, [-1, (self.temporal_order-1) * self.orch_dim])
x = tf.concat([piano_t, orch_past_flat], axis=1)
for dense in self.weights["MLP_embed"]:
x = dense(x)
keras_layer_summary(dense, collections=["weights"])
if self.dropout_probability > 0:
x = Dropout(self.dropout_probability)(x)
context_embedding = x
if self.context_embedding_size is None:
self.context_embedding_size = context_embedding.get_shape()[1]
return context_embedding
def predict_knowing_context(self, context_embedding, orch_pred, mask):
with tf.name_scope("predict_knowing_context"):
# Build input
with tf.name_scope("build_input"):
masked_orch_t = tf.multiply(orch_pred, mask)
x = tf.concat([context_embedding, masked_orch_t, mask], axis=1)
# Propagate as in a normal network
for i, dense in enumerate(self.weights["NADE_mlp"]):
with tf.name_scope("layer_" + str(i)):
x = dense(x)
keras_layer_summary(dense, collections=["weights"])
x = Dropout(self.dropout_probability)(x)
# Last layer NADE_mlp
dense = self.weights["NADE_mlp_last"]
orch_prediction = dense(x)
keras_layer_summary(dense, collections=["weights"])
return orch_prediction, x
def predict(self, inputs_ph):
piano_t, _, _, orch_past, _ = inputs_ph
# x = piano_t
orch_past_flat = tf.reshape(orch_past, [-1, (self.temporal_order-1) * self.orch_dim])
x = tf.concat([piano_t, orch_past_flat], axis=1)
for i, l in enumerate(self.layers):
with tf.name_scope("layer_" + str(i)):
dense = Dense(l, activation='relu', kernel_regularizer=regularizers.l2(self.weight_decay_coeff))
x = dense(x)
keras_layer_summary(dense)
x = Dropout(self.dropout_probability)(x)
with tf.name_scope("predictive_layer"):
dense = Dense(self.orch_dim, activation='sigmoid')
orch_prediction = dense(x)
keras_layer_summary(dense)
return orch_prediction, None
def embed_context(self, inputs_ph):
with tf.name_scope("embed_context"):
piano_t, _, _, orch_past, _ = inputs_ph
#####################
# GRU for modelling past orchestra
# First layer
with tf.name_scope("orchestra_past_embedding"):
x = orch_past
for layer_ind, gru_layer in enumerate(
self.weights["orchestra_emb_GRUs"]):
with tf.name_scope("orch_rnn_" + str(layer_ind)):
x = gru_layer(x)
keras_layer_summary(gru_layer, collections=["weights"])
lstm_out = x
#####################
#####################
# gru out and piano(t)
with tf.name_scope("piano_present_embedding"):
dense_layer = self.weights["piano_emb_MLP"]
piano_embedding_ = dense_layer(piano_t)
piano_embedding = Dropout(
self.dropout_probability)(piano_embedding_)
keras_layer_summary(dense_layer, collections=["weights"])
#####################
#####################
# Merge embeddings
with tf.name_scope("merge_embeddings"):
context_embedding = keras.layers.concatenate(
[lstm_out, piano_embedding], axis=1)
#####################
#####################
# Context embedding size
if self.context_embedding_size is None:
self.context_embedding_size = context_embedding.get_shape()[1]
#####################
return context_embedding
def predict(self, inputs_ph):
piano_ph, _ = inputs_ph
#####################
# GRU for modelling past orchestra
# First layer
with tf.name_scope("orch_rnn_0"):
gru_layer = GRU(self.n_hs[0], return_sequences=True, input_shape=(self.temporal_order, self.piano_dim), activation='relu', dropout=self.dropout_probability)
x = gru_layer(piano_ph)
keras_layer_summary(gru_layer, collections=["weights"])
if len(self.n_hs) > 1:
# Intermediates layers
for layer_ind in range(1, len(self.n_hs)):
with tf.name_scope("orch_rnn_" + str(layer_ind)):
gru_layer = GRU(self.n_hs[layer_ind], return_sequences=True, activation='relu', dropout=self.dropout_probability)
x = gru_layer(x)
keras_layer_summary(gru_layer, collections=["weights"])
orch_pred = x
return orch_pred, orch_pred
def predict_knowing_context(self, context_embedding_precomputed, orch_pred,
mask):
with tf.name_scope("predict_knowing_context"):
# Build input
with tf.name_scope("build_input"):
masked_orch_t = tf.multiply(orch_pred, mask)
x = tf.concat(
[context_embedding_precomputed, masked_orch_t, mask],
axis=1)
with tf.name_scope("NADE_prediction"):
# Propagate as in a normal network
for dense in self.weights["MLP"]:
x = dense(x)
keras_layer_summary(dense, collections=["weights"])
x = Dropout(self.dropout_probability)(x)
dense = self.weights["last_MLP"]
orch_prediction = dense(x)
keras_layer_summary(dense, collections=["weights"])
return orch_prediction, x
def predict(self, inputs_ph, orch_pred, mask):
piano_t, _, _, orch_past, _ = inputs_ph
# Build input
with tf.name_scope("build_input"):
orch_past_flat = tf.reshape(orch_past, [-1, (self.temporal_order-1) * self.orch_dim])
masked_orch_t = tf.multiply(orch_pred, mask)
x = tf.concat([piano_t, orch_past_flat, masked_orch_t, mask], axis=1)
# Propagate as in a normal network
for i, l in enumerate(self.layers):
with tf.name_scope("layer_" + str(i)):
dense = Dense(l, activation='relu')
x = dense(x)
keras_layer_summary(dense)
x = Dropout(self.dropout_probability)(x)
dense = Dense(self.orch_dim, activation='sigmoid')
orch_prediction = dense(x)
keras_layer_summary(dense)
return orch_prediction, x
def stacked_rnn(input_seq,
layers,
rnn_type='gru',
dropout_probability=0,
activation='relu'):
def layer_rnn(layer, rnn_type, return_sequences):
if rnn_type is 'gru':
this_layer = GRU(layer,
return_sequences=return_sequences,
activation=activation,
dropout=dropout_probability)
return this_layer
if len(layers) > 1:
return_sequences = True
else:
return_sequences = False
with tf.name_scope("0"):
this_layer = layer_rnn(layers[0], rnn_type, return_sequences)
x = this_layer(input_seq)
keras_layer_summary(this_layer)
if len(layers) > 1:
# Intermediates layers
for layer_ind in range(1, len(layers)):
# Last layer ?
if layer_ind == len(layers) - 1:
return_sequences = False
else:
return_sequences = True
with tf.name_scope(str(layer_ind)):
this_layer = layer_rnn(layers[layer_ind], rnn_type,
return_sequences)
x = this_layer(x)
keras_layer_summary(this_layer)
return x
def predict(self, inputs_ph):
piano_t, _, _, orch_past, _ = inputs_ph
gamma, beta = self.FiLM_generator(orch_past,
self.weights['FiLM_generator'])
# Unpack
ind_end = 0
x = piano_t
layer = self.weights["MLP"]
x = layer(x)
keras_layer_summary(layer, collections=["weights"])
# FiLM layer
x = tf.multiply(gamma, x) + beta
x = keras.activations.relu(x)
last_layer = self.weights["last_layer"]
orch_prediction = last_layer(x)
keras_layer_summary(last_layer, collections=["weights"])
return orch_prediction, orch_prediction
def predict(self, inputs_ph):
piano_t, piano_past, piano_future, orch_past, _ = inputs_ph
with tf.name_scope("piano_embedding_past"):
piano_embedding_past = self.piano_embedding(piano_t, piano_past, reverse=False)
with tf.name_scope("piano_embedding_future"):
piano_embedding_future = self.piano_embedding(piano_t, piano_future, reverse=True)
with tf.name_scope("orchestra_embedding"):
orchestra_embedding = self.orchestra_embedding(orch_past)
#####################
# Concatenate and predict
with tf.name_scope("top_layer_prediction_0"):
top_input = keras.layers.concatenate([orchestra_embedding, piano_embedding_past, piano_embedding_future], axis=1)
dense_layer = Dense(1000, activation='relu', name='orch_pred_0')
top_0 = dense_layer(top_input)
keras_layer_summary(dense_layer)
with tf.name_scope("top_layer_prediction_1"):
dense_layer = Dense(self.orch_dim, activation='sigmoid', name='orch_pred')
orch_prediction = dense_layer(top_0)
keras_layer_summary(dense_layer)
#####################
return orch_prediction, top_input
# 'temporal_order' : 5,
# 'dropout_probability' : 0,
# 'weight_decay_coeff' : 0,
# 'kernel_size_piano': 12,
# 'kernel_size_orch': 12,
# 'hs_piano': [500],
# 'hs_orch': [600],
# 'embeddings_size': 500,
def FiLM_generator(self, x, layers):
for gru_layer in layers:
x = gru_layer(x)
keras_layer_summary(gru_layer, collections=["weights"])
gamma_layer = self.weights["gamma"]
beta_layer = self.weights["beta"]
keras_layer_summary(gamma_layer, collections=["weights"])
keras_layer_summary(beta_layer, collections=["weights"])
gamma = gamma_layer(x)
beta = beta_layer(x)
return gamma, beta
def predict(self, inputs_ph, pitch_mask):
piano_t, _, _, orch_t, orch_past, orch_future = inputs_ph
with tf.name_scope("orch_past"):
x = orch_past
for layer in self.weights["orch_past"]:
x = layer(x)
keras_layer_summary(layer, collections=["weights"])
orch_past_embedding = x
with tf.name_scope("orch_future"):
x = orch_future
for layer in self.weights["orch_future"]:
x = layer(x)
keras_layer_summary(layer, collections=["weights"])
orch_future_embedding = x
with tf.name_scope("present_piano"):
x = piano_t
for layer in self.weights["piano_present"]:
x = layer(x)
keras_layer_summary(layer, collections=["weights"])
piano_t_embedding = x
with tf.name_scope("present_orchestra"):
x = orch_t
for ind_layer, layer in enumerate(self.weights["orch_present"]):
if ind_layer == 0:
# Remove pitch from orch and weight
W, b = layer
# Mask out the column corresponding to the predicted pitch
x = tf.matmul(x, tf.multiply(W, pitch_mask)) + b
else:
import pdb
pdb.set_trace()
x = layer(x)
keras_layer_summary(layer, collections=["weights"])
orch_t_embedding = x
return x, x
def piano_embedding(self, piano_t, piano_past, piano_future):
# Build input
# Add a time axis to piano_t
piano_t_time = tf.reshape(piano_t, [-1, 1, self.piano_dim])
with tf.name_scope("present"):
# Concatenate t and future
x = piano_t
for layer_ind, gru_layer in enumerate(
self.weights["present_piano"]):
with tf.name_scope("l_" + str(layer_ind)):
x = gru_layer(x)
keras_layer_summary(gru_layer)
piano_t_emb = x
with tf.name_scope("past"):
# Concatenate t and future
x = tf.concat([piano_past, piano_t_time], 1)
for layer_ind, gru_layer in enumerate(self.weights["past_piano"]):
with tf.name_scope("l_" + str(layer_ind)):
x = gru_layer(x)
keras_layer_summary(gru_layer)
piano_past_emb = x
with tf.name_scope("future"):
# Concatenate t and future
input_seq_fut = tf.concat([piano_t_time, piano_future], axis=1)
# Flip the matrix along the time axis so that the last time index is t
x = tf.reverse(input_seq_fut, axis=[1])
for layer_ind, gru_layer in enumerate(
self.weights["future_piano"]):
with tf.name_scope("l_" + str(layer_ind)):
x = gru_layer(x)
keras_layer_summary(gru_layer)
piano_future_emb = x
return piano_t_emb, piano_past_emb, piano_future_emb
def predict(self, inputs_ph):
piano_t, _, _, orch_past, _ = inputs_ph
#####################
# Batch norm
# if self.binarize_piano:
# piano_t = BatchNorm()
#####################
#####################
# GRU for modelling past orchestra
# First layer
if len(self.n_hs) > 1:
return_sequences = True
else:
return_sequences = False
with tf.name_scope("orch_rnn_0"):
gru_layer = GRU(self.n_hs[0], return_sequences=return_sequences, input_shape=(self.temporal_order, self.orch_dim),
activation='relu', dropout=self.dropout_probability)
x = gru_layer(orch_past)
keras_layer_summary(gru_layer)
if len(self.n_hs) > 1:
# Intermediates layers
for layer_ind in range(1, len(self.n_hs)):
# Last layer ?
if layer_ind == len(self.n_hs)-1:
return_sequences = False
else:
return_sequences = True
with tf.name_scope("orch_rnn_" + str(layer_ind)):
gru_layer = GRU(self.n_hs[layer_ind], return_sequences=return_sequences,
activation='relu', dropout=self.dropout_probability)
x = gru_layer(x)
keras_layer_summary(gru_layer)
lstm_out = x
#####################
#####################
# gru out and piano(t)
with tf.name_scope("piano_embedding"):
# fully-connected layer with 128 units and ReLU activation
piano_t_drop = Dropout(self.dropout_probability)(piano_t)
dense_layer = Dense(self.n_hs[-1], activation='relu')
piano_embedding = dense_layer(piano_t_drop)
keras_layer_summary(dense_layer)
#####################
#####################
# Concatenate and predict
with tf.name_scope("top_layer_prediction"):
top_input = keras.layers.concatenate([lstm_out, piano_embedding], axis=1)
top_input_drop = Dropout(self.dropout_probability)(top_input)
# First, just the linear part
dense_layer = Dense(self.orch_dim, activation='linear', name='orch_pred')
biased_linear_pred = dense_layer(top_input_drop)
keras_layer_summary(dense_layer)
# Add the pre-computed static biases
precomputed_biases = tf.constant(self.static_bias, dtype=tf.float32, name='precomputed_static_biases')
fix_linear_pred = biased_linear_pred + precomputed_biases
# Now pass in the non-linearity
orch_prediction = Activation('sigmoid')(fix_linear_pred)
#####################
embedding_concat = top_input
return orch_prediction, embedding_concat
def predict(self, inputs_ph):
piano_t, _, _, orch_past, _ = inputs_ph
#####################
# Batch norm
# if self.binarize_piano:
# piano_t = BatchNorm()
#####################
#####################
# GRU for modelling past orchestra
# First layer
if len(self.n_hs) > 1:
return_sequences = True
else:
return_sequences = False
with tf.name_scope("orch_rnn_0"):
gru_layer = GRU(self.n_hs[0],
return_sequences=return_sequences,
input_shape=(self.temporal_order, self.orch_dim),
activation='relu',
dropout=self.dropout_probability)
x = gru_layer(orch_past)
keras_layer_summary(gru_layer)
if len(self.n_hs) > 1:
# Intermediates layers
for layer_ind in range(1, len(self.n_hs)):
# Last layer ?
if layer_ind == len(self.n_hs) - 1:
return_sequences = False
else:
return_sequences = True
with tf.name_scope("orch_rnn_" + str(layer_ind)):
gru_layer = GRU(self.n_hs[layer_ind],
return_sequences=return_sequences,
activation='relu',
dropout=self.dropout_probability)
x = gru_layer(x)
keras_layer_summary(gru_layer)
lstm_out = x
#####################
#####################
# gru out and piano(t)
with tf.name_scope("piano_embedding"):
piano_t_ = Dropout(self.dropout_probability)(piano_t)
dense_layer = Dense(
self.n_hs[-1], activation='relu'
) # fully-connected layer with 128 units and ReLU activation
piano_embedding = dense_layer(piano_t_)
keras_layer_summary(dense_layer)
#####################
#####################
# Concatenate and predict
with tf.name_scope("top_layer_prediction"):
top_input = keras.layers.concatenate([lstm_out, piano_embedding],
axis=1)
top_input_drop = Dropout(self.dropout_probability)(top_input)
dense_layer = Dense(self.orch_dim,
activation='sigmoid',
name='orch_pred')
orch_prediction = dense_layer(top_input_drop)
keras_layer_summary(dense_layer)
#####################
embedding_concat = top_input
return orch_prediction, embedding_concat
def FiLM_generator(self, x, layers):
for gru_layer in layers:
x = gru_layer(x)
keras_layer_summary(gru_layer, collections=["weights"])
return x
def predict(self, inputs_ph):
piano_t, _, _, orch_past, _ = inputs_ph
#####################
# GRU for modelling past orchestra
# First layer
if len(self.GRU_orch_emb) > 1:
return_sequences = True
else:
return_sequences = False
with tf.name_scope("orchestra_embedding"):
gru_layer = GRU(self.GRU_orch_emb[0],
return_sequences=return_sequences,
input_shape=(self.temporal_order, self.orch_dim),
activation='relu',
dropout=self.dropout_probability)
x = gru_layer(orch_past)
keras_layer_summary(gru_layer, collections=["weights"])
if len(self.GRU_orch_emb) > 1:
# Intermediates layers
for layer_ind in range(1, len(self.GRU_orch_emb)):
# Last layer ?
if layer_ind == len(self.GRU_orch_emb) - 1:
return_sequences = False
else:
return_sequences = True
with tf.name_scope("orch_rnn_" + str(layer_ind)):
gru_layer = GRU(self.GRU_orch_emb[layer_ind],
return_sequences=return_sequences,
activation='relu',
dropout=self.dropout_probability)
x = gru_layer(x)
keras_layer_summary(gru_layer, collections=["weights"])
lstm_out = x
#####################
#####################
# gru out and piano(t)
with tf.name_scope("piano_embedding"):
x = piano_t
for num_unit in self.MLP_piano_emb:
dense_layer = Dense(
num_unit, activation='relu'
) # fully-connected layer with 128 units and ReLU activation
x = dense_layer(x)
keras_layer_summary(dense_layer, collections=["weights"])
x = Dropout(self.dropout_probability)(x)
piano_embedding = x
#####################
#####################
# Concatenate and predict
with tf.name_scope("top_layer_prediction"):
x = keras.layers.concatenate([lstm_out, piano_embedding], axis=1)
for num_unit in self.last_MLP:
dense_layer = Dense(num_unit, activation='relu')
x = dense_layer(x)
keras_layer_summary(dense_layer, collections=["weights"])
x = Dropout(self.dropout_probability)(x)
# Residual
dense_layer = Dense(self.orch_dim, name='orch_pred')
residue = dense_layer(x)
keras_layer_summary(dense_layer, collections=["weights"])
# Input
orch_tm1 = orch_past[:, -1, :]
# Sum
# activations = (orch_tm1 - 0.5) + residue
activations = orch_tm1 + residue
# Activation
orch_prediction = keras.activations.sigmoid(activations)
#####################
embedding_concat = x
return orch_prediction, embedding_concat
def predict(self, piano_t, orch_past):
#####################
# Piano embedding
with tf.name_scope("conv_piano"):
conv_layer = Conv1D(self.num_filter_piano,
self.kernel_size_piano,
activation='relu')
p0 = conv_layer(tf.reshape(piano_t, [-1, self.piano_dim, 1]))
keras_layer_summary(conv_layer)
with tf.name_scope("weighted_sum_piano"):
W = tf.get_variable("W", shape=(self.num_filter_piano, ))
p1 = tf.scalar_mul(1 / tf.reduce_sum(W),
tf.tensordot(p0, W, [[2], [0]]))
variable_summary(W)
piano_embedding = MLP(p1,
self.mlp_piano,
"mlp_piano",
activation='relu')
#####################
#####################
# GRU for modelling past orchestra
# First layer
if len(self.gru_orch) > 1:
return_sequences = True
else:
return_sequences = False
with tf.name_scope("orch_rnn_0"):
x = GRU(self.gru_orch[0],
return_sequences=return_sequences,
input_shape=(self.temporal_order, self.orch_dim),
activation='relu',
dropout=self.dropout_probability)(orch_past)
if len(self.gru_orch) > 1:
# Intermediates layers
for layer_ind in range(1, len(self.gru_orch)):
# Last layer ?
if layer_ind == len(self.gru_orch) - 1:
return_sequences = False
else:
return_sequences = True
with tf.name_scope("orch_rnn_" + str(layer_ind)):
x = GRU(self.gru_orch[layer_ind],
return_sequences=return_sequences,
activation='relu',
dropout=self.dropout_probability)(x)
orch_embedding = x
#####################
#####################
# Prediction
input_pred = tf.concat([piano_embedding, orch_embedding], axis=1)
top_input = MLP(input_pred,
self.mlp_pred,
"mlp_pred",
activation='relu')
# Dense layers on top
with tf.name_scope("last_MLP"):
orch_prediction = Dense(self.orch_dim,
activation='sigmoid',
name='orch_pred')(top_input)
#####################
return orch_prediction
# 'batch_size' : 200,
# 'temporal_order' : 5,
# 'dropout_probability' : 0,
# 'weight_decay_coeff' : 0,
# 'num_filter_piano': 20,
# 'kernel_size_piano': 12,
# 'mlp_piano': [500, 500],
# 'mlp_pred': [500, 500],
# 'gru_orch': [500, 500],
