def _attention(self, lstm_outputs):
with tf.variable_scope('Attention', initializer=self.init):
weights = tf.get_variable('weights',
[self.lstm_units, self.output_units],
regularizer=self.regularizer)
biases = tf.get_variable('biases', [1, self.output_units])
u_w = tf.get_variable('u_w', [self.output_units, 1])
outputs, scores = [], []
for v in lstm_outputs:
hidden_rep = nn.tanh(tf.add(tf.matmul(v, weights), biases))
scores.append(tf.matmul(hidden_rep, u_w))
# list -> tensor
scores = tf.concat(scores, axis=1)
# softmax
scores = nn.softmax(scores, dim=-1)
# tensor -> list
scores = tf.unstack(scores, axis=1)
for i, v in enumerate(scores):
# v: (64,) -> (64,1)
v = tf.reshape(v, [-1, 1])
# v: (64,1) -> [(64,1), (64,1), ...]
v = [v] * self.lstm_units
# v: (64,self.lstm_units)
v = tf.concat(v, axis=1)
outputs.append(tf.multiply(v, lstm_outputs[i]))
return tf.add_n(outputs)
def call(self, x, training=True, return_lowres=False):
if len(x.shape) == 2:
x = self._low_res_generator(x, training=False)
lowres = x
for block in self.blocks:
x = block(x, training=training)
hires = tanh(self.final_conv(x))
return (hires, lowres) if return_lowres else hires
def call(self, x, training=True):
x = self.fc(x)
x = self.initial_norm(x, training=training)
x = tf.nn.relu(x)
x = tf.reshape(x, shape=(-1, 15, 15, 64))
for block in self.blocks:
x = block(x, training=training)
return tanh(self.final_conv(x))
def call(self, features, hidden):
hidden_with_time_axis = expand_dims(hidden, 1)
score = nn.tanh(self.W1(features) + self.W2(hidden_with_time_axis))
attention_weights = nn.softmax(self.V(score), axis=1)
context_vector = attention_weights * features
context_vector = reduce_sum(context_vector, axis=1)
return context_vector, attention_weights
def ACT(inputs, act_fn):
if act_fn == 'relu':
act = relu(inputs)
elif act_fn == 'lrelu':
act = leaky_relu(inputs)
elif act_fn == 'sigmoid':
act = sigmoid(inputs)
elif act_fn == 'tanh':
act = tanh(inputs)
else:
act = inputs
return act
def call(self, q, val):
#make q i.e. the hidden value into the same shape
q_with_time_axis = tf.expand_dims(q, 1)
score = self.V(nn.tanh(self.W1(q_with_time_axis) + self.W2(val)))
attention_w = nn.softmax(score, axis=1)
context_vec = attention_w * val
context_vec = tf.reduce_sum(context_vec, axis=1)
return context_vec, attention_w
def __init__(self, game, tensor_logs_dir):
self.game = game
self.shape = game.board_size
self.possible_moves_size = self.shape[1]
self.log_writer = tf.summary.FileWriter(tensor_logs_dir + "/losses")
self.checkpoint_dir = "checkpoints"
create_dir(self.checkpoint_dir)
self.graph = tf.Graph()
self.session = tf.Session(graph=self.graph)
self.saver = None
with tf.Session() as temp_session:
temp_session.run(tf.global_variables_initializer())
with self.graph.as_default():
self.input_boards = tf.placeholder(tf.float32, shape=[None, self.game.board.rows, self.game.board.cols])
self.dropout = tf.placeholder(tf.float32)
self.isTraining = tf.placeholder(tf.bool, name="is_training")
self.pi_losses_var = tf.Variable(0, dtype=tf.float32)
self.v_losses_var = tf.Variable(0, dtype=tf.float32)
X = tf.reshape(self.input_boards, [-1, self.shape[0], self.shape[1], 1])
h_conv1 = relu(batch_normalization(conv2d(X, 'same'), axis=3, training=self.isTraining))
h_conv2 = relu(batch_normalization(conv2d(h_conv1, 'same'), axis=3, training=self.isTraining))
h_conv3 = relu(batch_normalization(conv2d(h_conv2, 'valid'), axis=3, training=self.isTraining))
h_conv4 = relu(batch_normalization(conv2d(h_conv3, 'valid'), axis=3, training=self.isTraining))
h_conv4_flat = tf.reshape(h_conv4, [-1, config.num_channels * (self.shape[0] - 4) * (self.shape[1] - 4)])
s_fc1 = dropout(relu(batch_normalization(dense(h_conv4_flat, 1024), axis=1, training=self.isTraining)),
rate=self.dropout)
s_fc2 = dropout(relu(batch_normalization(dense(s_fc1, 512), axis=1, training=self.isTraining)),
rate=self.dropout)
self.pi = dense(s_fc2, self.possible_moves_size)
self.prob = tf.nn.softmax(self.pi)
self.v = tanh(dense(s_fc2, 1))
# Place holders for Predicted (pi, v)s
self.predicted_pis = tf.placeholder(dtype=tf.float32, shape=[None, self.possible_moves_size])
self.predicted_vs = tf.placeholder(dtype=tf.float32, shape=[None])
# Real Losses
self.loss_pi = tf.losses.softmax_cross_entropy(self.predicted_pis, self.pi)
self.loss_v = tf.losses.mean_squared_error(self.predicted_vs, tf.reshape(self.v, shape=[-1, ]))
self.total_loss = self.loss_pi + self.loss_v
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
self.train_step = tf.train.AdamOptimizer(config.lr).minimize(self.total_loss)
self.session.run(tf.variables_initializer(self.graph.get_collection('variables')))
def get_attention(units,
lstm_units,
n_layers_att,
p_dropout=0,
l1_reg=0,
l2_reg=0):
# W1 = Dense(units, kernel_regularizer=l1_l2(l1_reg, l2_reg), name = 'W_feats')
W1 = [
Dense(units, activation='tanh', name='W_feats_{}'.format(i))
for i in range(n_layers_att)
]
W2 = Dense(units,
kernel_regularizer=l1_l2(l1_reg, l2_reg),
name='W_hidden')
V = Dense(1, kernel_regularizer=l1_l2(l1_reg, l2_reg), name='V')
f_beta = Dense(1, kernel_regularizer=l1_l2(l1_reg, l2_reg), name='f_beta')
dropout = Dropout(p_dropout, name='dropout')
# shape = (batch, attn_features, features_shape)
encoder_output = Input(feature_vector_shape, name='image_features')
# shape = (batch, lstm_units)
hidden_last = Input(lstm_units, name='last_hidden_state')
projected_features = dropout(W1[0](encoder_output))
for i in range(1, n_layers_att):
projected_features = dropout(W1[i](projected_features))
# shape = (batch, attn_features, 1)
score = V(tanh(
projected_features + \
dropout(W2(tf.expand_dims(hidden_last, axis = 1)))
))
# shape = (batch, attn_features)
score = dropout(tf.reduce_sum(score, axis=2))
attention_weights = Activation('softmax', dtype='float32')(score)
# beta = f_beta(dropout(tf.expand_dims(hidden_last, axis = 1)))
# shape = (batch, 1)
beta = f_beta(hidden_last)
beta = Activation('sigmoid', dtype='float32')(beta)
# shape = (batch, attn_features, features_shape)
context_vector = tf.expand_dims(attention_weights, axis=2) * encoder_output
# shape = (batch, features_shape)
context_vector = beta * tf.reduce_sum(context_vector, axis=1)
context_vector = Activation('linear', dtype='float32')(context_vector)
return Model(inputs=[encoder_output, hidden_last],
outputs=[context_vector, attention_weights],
name='attention')
def forward_pass(self, input_data):
input_data = input_data / 255
pre_inception = InceptionNet._pre_inception_layer(
input_data, self.pre_inception_filters)
inception_1 = InceptionNet._inception_layer(pre_inception,
self.inception_1_filters)
inception_2 = InceptionNet._inception_layer(inception_1,
self.inception_2_filters)
# print(inception_2)
post_inception_pool = InceptionNet._max_pool(inception_2, (3, 3),
(2, 2))
inception_3 = InceptionNet._inception_layer(post_inception_pool,
self.inception_3_filters)
inception_4 = InceptionNet._inception_layer(inception_3,
self.inception_4_filters)
inception_5 = InceptionNet._inception_layer(inception_4,
self.inception_5_filters)
inception_6 = InceptionNet._inception_layer(inception_5,
self.inception_6_filters)
inception_7 = InceptionNet._inception_layer(inception_6,
self.inception_7_filters)
post_inception_pool_2 = InceptionNet._max_pool(inception_7, (3, 3),
(2, 2))
inception_8 = InceptionNet._inception_layer(post_inception_pool_2,
self.inception_8_filters)
inception_9 = InceptionNet._inception_layer(inception_8,
self.inception_9_filters)
post_inception_pool_3 = nn.avg_pool(inception_9, (7, 7),
strides=4,
padding="SAME")
flatten_layer = tf.reshape(
tf.keras.backend.flatten(post_inception_pool_3),
[1024, input_data.shape[0]])
relu_layer = nn.relu(
tf.matmul(tf.transpose(self.relu_weights), flatten_layer) +
self.relu_bias)
dropout_layer = nn.dropout(relu_layer, .4)
linear_layer = tf.matmul(tf.transpose(self.linear_weights),
dropout_layer) + self.linear_bias
return nn.softmax(tf.transpose(nn.tanh(linear_layer)))
def __call__(self, tensor_in, condition=None, training=None, slope=None):
norm = get_normalization(NORMALIZATION, training)
tconv_layer = lambda i, f, k, s: ACTIVATION(norm(tconv3d(i, f, k, s)))
with tf.variable_scope(self.name, reuse=tf.AUTO_REUSE):
h = tensor_in
h = tf.expand_dims(tf.expand_dims(tf.expand_dims(h, 1), 1), 1)
# Shared network
with tf.variable_scope('shared'):
h = tconv_layer(h, 512, (4, 1, 1), (4, 1, 1)) # 4, 1, 1
h = tconv_layer(h, 256, (1, 4, 1), (1, 4, 1)) # 4, 4, 1
h = tconv_layer(h, 128, (1, 4, 1), (1, 4, 1)) # 4, 16, 1
h = tconv_layer(h, 64, (1, 1, 12), (1, 1, 12)) # 4, 16, 12
# Pitch-time private network
with tf.variable_scope('pitch_time_private'):
s1 = [
tconv_layer(h, 32, (1, 1, 7), (1, 1, 7)) # 4, 16, 84
for _ in range(self.n_tracks)
]
s1 = [
tconv_layer(s1[i], 16, (1, 3, 1), (1, 3, 1)) # 4, 48, 84
for i in range(self.n_tracks)
]
# Time-pitch private network
with tf.variable_scope('time_pitch_private'):
s2 = [
tconv_layer(h, 32, (1, 3, 1), (1, 3, 1)) # 4, 48, 12
for _ in range(self.n_tracks)
]
s2 = [
tconv_layer(s2[i], 16, (1, 1, 7), (1, 1, 7)) # 4, 48, 84
for i in range(self.n_tracks)
]
h = [tf.concat((s1[i], s2[i]), -1) for i in range(self.n_tracks)]
# Merged private network
with tf.variable_scope('merged_private'):
h = [
norm(tconv3d(h[i], 1, (1, 1, 1), (1, 1, 1))) # 4, 48, 84
for i in range(self.n_tracks)
]
h = tf.concat(h, -1)
return tanh(h)
def __call__(self, v, u):
"""
Input:
- v: N x D x H x W
- u: N x D
Returns:
- next_u: N x D
"""
N, K = v.shape(0), self.hidden_dim
D, H, W = v.shape(1), v.shape(2), v.shape(3)
v_proj = self.Wv(v) # N x K x H x W
u_proj = self.Wu(u) # N x K
u_proj_expand = u_proj.reshape(N, K, 1, 1).expand(N, K, H, W)
h = nn.tanh(v_proj + u_proj_expand)
p = nn.softmax(self.Wp(h).reshape(N, H * W)).reshape(N, 1, H, W)
self.attention_maps = tf.identity(p.data)
v_tilde = (p.expand_as(v) * v).sum(2).sum(3).reshape(N, D)
next_u = u + v_tilde
return next_u
def __call__(self, tensor_in, condition=None, training=None, slope=None):
norm = get_normalization(NORMALIZATION, training)
tconv_layer = lambda i, f, k, s: ACTIVATION(norm(tconv3d(i, f, k, s)))
with tf.variable_scope(self.name, reuse=tf.AUTO_REUSE):
h = tensor_in
h = tf.expand_dims(tf.expand_dims(tf.expand_dims(h, 1), 1), 1)
# Shared network
with tf.variable_scope('shared'):
h = tconv_layer(h, 512, (4, 1, 1), (4, 1, 1)) # 4, 1, 1
h = tconv_layer(h, 256, (1, 4, 1), (1, 4, 1)) # 4, 4, 1
h = tconv_layer(h, 128, (1, 4, 1), (1, 4, 1)) # 4, 16, 1
h = tconv_layer(h, 64, (1, 1, 12), (1, 1, 12)) # 4, 16, 12
# Pitch-time private network
with tf.variable_scope('pitch_time_private'):
s1 = [tconv_layer(h, 32, (1, 1, 7), (1, 1, 7)) # 4, 16, 84
for _ in range(self.n_tracks)]
s1 = [tconv_layer(s1[i], 16, (1, 3, 1), (1, 3, 1)) # 4, 48, 84
for i in range(self.n_tracks)]
# Time-pitch private network
with tf.variable_scope('time_pitch_private'):
s2 = [tconv_layer(h, 32, (1, 3, 1), (1, 3, 1)) # 4, 48, 12
for _ in range(self.n_tracks)]
s2 = [tconv_layer(s2[i], 16, (1, 1, 7), (1, 1, 7)) # 4, 48, 84
for i in range(self.n_tracks)]
h = [tf.concat((s1[i], s2[i]), -1) for i in range(self.n_tracks)]
# Merged private network
with tf.variable_scope('merged_private'):
h = [norm(tconv3d(h[i], 1, (1, 1, 1), (1, 1, 1))) # 4, 48, 84
for i in range(self.n_tracks)]
h = tf.concat(h, -1)
return tanh(h)
def call(self, x, hidden, enc_output):
# Attention
hidden_with_time_axis = tf.expand_dims(hidden, 1)
score = self.Verdict(
tanh(self.W1(enc_output) + self.W2(hidden_with_time_axis)))
attention_weights = softmax(score, axis=1)
context_vector = attention_weights * enc_output
context_vector = tf.reduce_sum(context_vector, axis=1)
# forward pass
x = self.embedding(x)
x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)
output, state = self.gru(x)
output = tf.reshape(output, (-1, output.shape[2]))
x = self.final_output(output)
return x, state, attention_weights
def netG16_decoder(layers, lab=False):
enc_conv1, enc_conv2, enc_conv3, enc_conv4, enc_conv5, enc_conv6, enc_conv7, enc_conv8 = layers[0], layers[1], layers[2], layers[3], layers[4], layers[5], layers[6], layers[7]
# decoder, no batch norm
dec_conv1 = tcl.convolution2d_transpose(enc_conv8, 512, 4, 2, activation_fn=tf.identity, weights_initializer=tf.random_normal_initializer(stddev=0.02), scope='g_dec_conv1')
dec_conv1 = relu(dec_conv1)
dec_conv1 = tf.concat([dec_conv1, enc_conv7], axis=3)
print (dec_conv1)
dec_conv2 = tcl.convolution2d_transpose(dec_conv1, 512, 4, 2, activation_fn=tf.identity, weights_initializer=tf.random_normal_initializer(stddev=0.02), scope='g_dec_conv2')
dec_conv2 = relu(dec_conv2)
dec_conv2 = tf.concat([dec_conv2, enc_conv6], axis=3)
print (dec_conv2)
dec_conv3 = tcl.convolution2d_transpose(dec_conv2, 512, 4, 2, activation_fn=tf.identity, weights_initializer=tf.random_normal_initializer(stddev=0.02), scope='g_dec_conv3')
dec_conv3 = relu(dec_conv3)
dec_conv3 = tf.concat([dec_conv3, enc_conv5], axis=3)
print (dec_conv3)
dec_conv4 = tcl.convolution2d_transpose(dec_conv3, 512, 4, 2, activation_fn=tf.identity, weights_initializer=tf.random_normal_initializer(stddev=0.02), scope='g_dec_conv4')
dec_conv4 = relu(dec_conv4)
dec_conv4 = tf.concat([dec_conv4, enc_conv4], axis=3)
print (dec_conv4)
dec_conv5 = tcl.convolution2d_transpose(dec_conv4, 256, 4, 2, activation_fn=tf.identity, weights_initializer=tf.random_normal_initializer(stddev=0.02), scope='g_dec_conv5')
dec_conv5 = relu(dec_conv5)
dec_conv5 = tf.concat([dec_conv5, enc_conv3], axis=3)
print (dec_conv5)
dec_conv6 = tcl.convolution2d_transpose(dec_conv5, 128, 4, 2, activation_fn=tf.identity, weights_initializer=tf.random_normal_initializer(stddev=0.02), scope='g_dec_conv6')
dec_conv6 = relu(dec_conv6)
dec_conv6 = tf.concat([dec_conv6, enc_conv2], axis=3)
print (dec_conv6)
dec_conv7 = tcl.convolution2d_transpose(dec_conv6, 64, 4, 2, activation_fn=tf.identity, weights_initializer=tf.random_normal_initializer(stddev=0.02), scope='g_dec_conv7')
dec_conv7 = relu(dec_conv7)
dec_conv7 = tf.concat([dec_conv7, enc_conv1], axis=3)
print (dec_conv7)
c = 2 if lab else 3
dec_conv8 = tcl.convolution2d_transpose(dec_conv7, c, 4, 2, activation_fn=tf.identity, weights_initializer=tf.random_normal_initializer(stddev=0.02), scope='g_dec_conv8')
dec_conv8 = tanh(dec_conv8)
print (dec_conv1)
return dec_conv8
def netG(x):
print("\nnetG\n")
print(x)
conv1 = tcl.conv2d(
x,
64,
7,
1,
activation_fn=tf.nn.relu,
normalizer_fn=tcl.batch_norm,
weights_initializer=tf.random_normal_initializer(stddev=0.02),
scope='g_conv1')
print(conv1)
conv2 = tcl.conv2d(
conv1,
128,
3,
2,
activation_fn=tf.nn.relu,
normalizer_fn=tcl.batch_norm,
weights_initializer=tf.random_normal_initializer(stddev=0.02),
scope='g_conv2')
print(conv2)
conv3 = tcl.conv2d(
conv2,
256,
3,
2,
activation_fn=tf.identity,
normalizer_fn=tcl.batch_norm,
weights_initializer=tf.random_normal_initializer(stddev=0.02),
scope='g_conv3')
print(conv3)
res1 = resBlock(conv3, 1)
res2 = resBlock(res1, 2)
res3 = resBlock(res2, 3)
res4 = resBlock(res3, 4)
res5 = resBlock(res4, 5)
res6 = resBlock(res5, 6)
res7 = resBlock(res6, 7)
res8 = resBlock(res7, 8)
res9 = resBlock(res8, 9)
res10 = resBlock(res9, 10)
res11 = resBlock(res10, 11)
res12 = resBlock(res11, 12)
res13 = resBlock(res12, 13)
res14 = resBlock(res13, 14)
res15 = resBlock(res14, 15)
res16 = resBlock(res15, 16)
res17 = resBlock(res16, 17)
res18 = resBlock(res17, 18)
res19 = resBlock(res18, 19)
res20 = resBlock(res19, 20)
res21 = resBlock(res20, 21)
res22 = resBlock(res21, 22)
res23 = resBlock(res22, 23)
res24 = resBlock(res23, 24)
res25 = resBlock(res24, 25)
res26 = resBlock(res25, 26)
res27 = resBlock(res26, 27)
res28 = resBlock(res27, 28)
res29 = resBlock(res28, 29)
res30 = resBlock(res29, 30)
conv4 = tcl.conv2d_transpose(
res30,
128,
3,
2,
activation_fn=tf.nn.relu,
normalizer_fn=tcl.batch_norm,
weights_initializer=tf.random_normal_initializer(stddev=0.02),
scope='g_conv4')
print(conv4)
conv5 = tcl.conv2d_transpose(
conv4,
64,
3,
2,
activation_fn=tf.nn.relu,
normalizer_fn=tcl.batch_norm,
weights_initializer=tf.random_normal_initializer(stddev=0.02),
scope='g_conv5')
print(conv5)
conv6 = tcl.conv2d(
conv5,
3,
7,
1,
activation_fn=tf.identity,
normalizer_fn=tcl.batch_norm,
weights_initializer=tf.random_normal_initializer(stddev=0.02),
scope='g_conv6')
conv6 = tanh(conv6)
print(conv6)
return conv6
def get_decoder(embedding_dim,
units,
lstm_units,
vocab_size,
n_layers_att,
attn_dropout=0,
lstm_dropout=0,
logit_dropout=0,
l1_reg=0,
l2_reg=0):
attention = get_attention(units, lstm_units, n_layers_att, attn_dropout,
l1_reg, l2_reg)
embedding = Embedding(input_dim=vocab_size,
output_dim=embedding_dim,
input_length=1,
name='embedding')
lstm = LSTM(lstm_units,
# return_sequences = True,
return_state = True,
recurrent_initializer = 'glorot_uniform',
dropout = lstm_dropout,
# recurrent_dropout = p_dropout,
kernel_regularizer = l1_l2(l1_reg, l2_reg),\
dtype = 'float32')
last_layer_hidden = Dense(embedding_dim,
kernel_regularizer=l1_l2(l1_reg, l2_reg),
name='last_hidden')
last_layer_context = Dense(embedding_dim,
kernel_regularizer=l1_l2(l1_reg, l2_reg),
name='last_context')
logits_kernel = Dense(vocab_size,
kernel_regularizer=l1_l2(l1_reg, l2_reg),
name='logits_kernel')
dropout = Dropout(logit_dropout, name='dropout')
# shape = (batch, 1)
word_input = Input(1, name='bow_input')
# shape = (batch, attn_features, features_shape)
encoder_output = Input(feature_vector_shape, name='image_features')
# shape = (batch, lstm_units)
hidden_last = Input(lstm_units, name='last_hidden_state')
cell_last = Input(lstm_units, name='last_cell_state')
# see keras doc on Embedding layer: if input shape is (batch, input_length)
# then output shape is (batch, input_length, embedding_dim).
# In this case, input_length is always 1, so the embedded_word shape is
# (batch, 1, embedding_dim). We will use this axis 1 for the lstm input
embedded_word = embedding(word_input)
#this just for caompatibility with mixed precision
embedded_word = Activation('linear')(embedded_word)
# context_vector shape = (batch, features_shape)
context_vector, attention_weights = attention(
[encoder_output, hidden_last])
# RNN input shape must be (batch, time_steps, features). In our case,
# time_steps is 1, so we must expand the context vector dimensions
# lstm_output shape = (batch, lstm_units)
# shape = (batch, 1, embedding_dim + features_shape)
lstm_input = Concatenate(axis=-1)(
[embedded_word, tf.expand_dims(context_vector, axis=1)])
# shape = (batch, lstm_units)
lstm_output, hidden, cell = lstm(lstm_input,
initial_state=[hidden_last, cell_last])
# shape = (batch, embedding_dim)
embedded_hidden = last_layer_hidden(dropout(lstm_output))
embedded_context = last_layer_context(context_vector)
# Now we finally drop the extra 1 axis in the embedded_word
logits_kernel_input = tanh(tf.reduce_sum(embedded_word,axis=1) + \
embedded_hidden + embedded_context)
# logits_kernel_input = Concatenate(axis=-1)([tf.reduce_sum(embedded_word,
# axis=1),
# context_vector,
# lstm_output])
# shape = (batch, vocab_size)
logits = logits_kernel(dropout(logits_kernel_input))
logits = Activation('linear', dtype='float32')(logits)
return Model(inputs=[word_input, encoder_output, hidden_last, cell_last],
outputs=[logits, attention_weights, hidden, cell])
def call(self, x, training=True):
for block in self.blocks:
x = block(x, training=training)
return tanh(self.final_conv(x))
def train_layers(self, train_x, train_y, test_x, test_y):
params = {}
X = tf.placeholder(tf.float32, [None, self.opt['n_dim']])
Y = tf.placeholder(tf.float32, [None, self.opt['n_classes']])
keep_prob = tf.placeholder(tf.float32) #for dropout
params['W1'] = tf.Variable(
tf.random_normal([self.opt['n_dim'], self.opt['num_hidden1']],
mean=0,
stddev=self.opt['std']))
params['b1'] = tf.Variable(
tf.random_normal([self.opt['num_hidden1']],
mean=0,
stddev=self.opt['std']))
params['a1'] = nn.sigmoid(tf.matmul(X, params['W1']) + params['b1'])
params['dropout1'] = nn.dropout(params['a1'], keep_prob)
params['W2'] = tf.Variable(
tf.random_normal(
[self.opt['num_hidden1'], self.opt['num_hidden2']],
mean=0,
stddev=self.opt['std']))
params['b2'] = tf.Variable(
tf.random_normal([self.opt['num_hidden2']],
mean=0,
stddev=self.opt['std']))
params['a2'] = nn.relu(
tf.matmul(params['dropout1'], params['W2']) + params['b2'])
params['dropout2'] = nn.dropout(params['a2'], keep_prob)
params['W3'] = tf.Variable(
tf.random_normal(
[self.opt['num_hidden2'], self.opt['num_hidden3']],
mean=0,
stddev=self.opt['std']))
params['b3'] = tf.Variable(
tf.random_normal([self.opt['num_hidden3']],
mean=0,
stddev=self.opt['std']))
params['a3'] = nn.tanh(
tf.matmul(params['dropout2'], params['W3']) + params['b3'])
params['dropout3'] = nn.dropout(params['a3'], keep_prob)
params['outW'] = tf.Variable(
tf.random_normal([self.opt['num_hidden3'], self.opt['n_classes']],
mean=0,
stddev=self.opt['std']))
params['outb'] = tf.Variable(
tf.random_normal([self.opt['n_classes']],
mean=0,
stddev=self.opt['std']))
out = nn.softmax(
tf.matmul(params['dropout3'], params['outW']) + params['outb'])
cost = tf.reduce_mean(
-tf.reduce_sum(Y * tf.log(out), reduction_indices=[1]))
optimizer = tf.train.AdamOptimizer(
self.opt['learning_rate']).minimize(cost)
correct_pred = tf.equal(tf.argmax(out, 1), tf.argmax(Y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
cost_history = np.empty(shape=[1], dtype=float)
y, y_pred = None, None
#reshape labels into a one hot vector
f = FeatureParser()
train_y = f.one_hot_encode(train_y)
test_y = f.one_hot_encode(test_y)
print('TRAIN_ONE_HOT_LABEL{}'.format(train_y))
print('Training...')
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for epoch in range(training_epochs):
_, loss, acc = sess.run([optimizer, cost, accuracy],
feed_dict={
X: train_x,
Y: train_y,
keep_prob: 0.5
})
cost_history = np.append(cost_history, loss)
if epoch % 50 == 0:
print('Epoch#', epoch, 'Cost:', loss, 'Train acc.:', acc)
y_pred = sess.run(tf.argmax(out, 1),
feed_dict={
X: test_x,
keep_prob: 1.0
})
y = sess.run(tf.argmax(test_y, 1))
print(
"Test accuracy: ",
round(
sess.run(accuracy,
feed_dict={
X: test_x,
Y: test_y,
keep_prob: 1.0
}), 3))
fig = plt.figure(figsize=(10, 8))
plt.plot(cost_history)
plt.xlabel('Iterations')
plt.ylabel('Cost')
plt.axis([0, training_epochs, 0, np.max(cost_history)])
plt.show()
precision, recall, f_score, s = precision_recall_fscore_support(
y, y_pred, average='micro')
print('F score:', round(f_score, 3))
def __call__(self, tensor_in, condition=None, condition_track=None,
training=None, slope=None):
norm = get_normalization(NORMALIZATION, training)
conv_layer = lambda i, f, k, s: ACTIVATION(norm(conv3d(i, f, k, s)))
tconv_layer = lambda i, c, f, k, s: ACTIVATION(norm(tconv3d(
tf.concat((i, c), -1), f, k, s)))
with tf.variable_scope(self.name, reuse=tf.AUTO_REUSE):
# ---------------------------- Encoder -----------------------------
c = condition_track
# Pitch-time private network
with tf.variable_scope('encoder_pitch_time'):
pt_1 = conv_layer(c, 16, (1, 1, 12), (1, 1, 12)) # 4, 48, 7
pt_2 = conv_layer(pt_1, 32, (1, 3, 1), (1, 3, 1)) # 4, 16, 7
# Time-pitch private network
with tf.variable_scope('encoder_time_pitch'):
tp_1 = conv_layer(c, 16, (1, 3, 1), (1, 3, 1)) # 4, 16, 84
tp_2 = conv_layer(tp_1, 32, (1, 1, 12), (1, 1, 12)) # 4, 16, 7
shared = tf.concat((tp_2, pt_2), -1)
# Shared network
with tf.variable_scope('encoder_shared'):
s1 = conv_layer(shared, 64, (1, 4, 3), (1, 4, 2)) # 4, 4, 3
s2 = conv_layer(s1, 128, (1, 4, 3), (1, 4, 3)) # 4, 1, 1
s3 = conv_layer(s2, 256, (4, 1, 1), (4, 1, 1)) # 1, 1, 1
# --------------------------- Generator ----------------------------
h = tensor_in
h = tf.expand_dims(tf.expand_dims(tf.expand_dims(h, 1), 1), 1)
# Shared network
with tf.variable_scope('shared'):
h = tconv_layer(h, s3, 512, (4, 1, 1), (4, 1, 1)) # 4, 1, 1
h = tconv_layer(h, s2, 256, (1, 4, 3), (1, 4, 3)) # 4, 4, 3
h = tconv_layer(h, s1, 128, (1, 4, 3), (1, 4, 2)) # 4, 16, 7
# Pitch-time private network
with tf.variable_scope('pitch_time_private'):
s1 = [tconv_layer(h, tp_2, 32, (1, 1, 12), (1, 1, 12))
for _ in range(self.n_tracks)] # 4, 16, 84
s1 = [tconv_layer(s1[i], tp_1, 16, (1, 3, 1), (1, 3, 1))
for i in range(self.n_tracks)] # 4, 48, 84
# Time-pitch private network
with tf.variable_scope('time_pitch_private'):
s2 = [tconv_layer(h, pt_2, 32, (1, 3, 1), (1, 3, 1))
for _ in range(self.n_tracks)] # 4, 48, 7
s2 = [tconv_layer(s2[i], pt_1, 16, (1, 1, 12), (1, 1, 12))
for i in range(self.n_tracks)] # 4, 48, 84
h = [tf.concat((s1[i], s2[i]), -1) for i in range(self.n_tracks)]
# Merged private network
with tf.variable_scope('merged_private'):
h = [tanh(norm(tconv3d(h[i], 1, (1, 1, 1), (1, 1, 1))))
for i in range(self.n_tracks)] # 4, 48, 84
return tf.concat((
h[:self.condition_track_idx] + [c] +
h[self.condition_track_idx:]), -1)
def _gate(self, x):
tanh_preactivation, sigmoid_preactivation = tf.split(x, 2, axis=-1)
return nn.tanh(tanh_preactivation) * nn.sigmoid(sigmoid_preactivation)
def generator(x):
with tf.variable_scope("generator_sr",
reuse=tf.AUTO_REUSE,
initializer=tf.initializers.random_normal(
0.0, 0.02)):
with tf.variable_scope("conv1"):
conv1 = layers.conv2d(x, 64, 3, strides=1, padding="same")
conv1 = layers.batch_normalization(conv1, scale=False)
conv1 = nn.relu(conv1)
with tf.variable_scope("conv2"):
# residual block with (up to) 8 convolution layers
n_conv_layers = 8
previous = conv1
for i in range(n_conv_layers):
conv2 = layers.conv2d(previous,
64,
3,
strides=1,
padding="same")
conv2 = layers.batch_normalization(conv2, scale=False)
conv2 += previous
conv2 = nn.relu(conv2)
previous = conv2
with tf.variable_scope("conv3"):
conv3 = layers.conv2d(conv2, 64, 3, strides=1, padding="same")
conv3 = layers.batch_normalization(conv3, scale=False)
conv3 = nn.relu(conv3)
with tf.variable_scope("deconv4"):
deconv4 = layers.conv2d_transpose(conv3,
256,
3,
strides=2,
padding="same")
deconv4 = layers.batch_normalization(deconv4, scale=False)
deconv4 = nn.relu(deconv4)
with tf.variable_scope("deconv5"):
deconv5 = layers.conv2d_transpose(deconv4,
256,
3,
strides=2,
padding="same")
deconv5 = layers.batch_normalization(deconv5, scale=False)
deconv5 = nn.relu(deconv5)
with tf.variable_scope("conv6"):
conv6 = layers.conv2d(deconv5, 3, 1, strides=1, padding="same")
with tf.variable_scope("out"):
out = nn.tanh(conv6)
#Adding Refinement network
with tf.variable_scope("generator_rf",
reuse=tf.AUTO_REUSE,
initializer=tf.initializers.random_normal(
0.0, 0.02)):
with tf.variable_scope("conv1"):
conv6 = layers.batch_normalization(conv6, scale=False)
conv6 = nn.relu(conv6)
conv1 = layers.conv2d(conv1, 64, 3, strides=1, padding="same")
conv1 = layers.batch_normalization(conv1, scale=False)
conv1 = nn.relu(conv1)
with tf.variable_scope("conv2"):
# residual block with (up to) 8 convolution layers
n_conv_layers = 8
previous = conv1
for i in range(n_conv_layers):
conv2 = layers.conv2d(previous,
64,
3,
strides=1,
padding="same")
conv2 = layers.batch_normalization(conv2, scale=False)
conv2 += previous
conv2 = nn.relu(conv2)
previous = conv2
with tf.variable_scope("conv3"):
conv3 = layers.conv2d(conv2, 64, 3, strides=1, padding="same")
conv3 = layers.batch_normalization(conv3, scale=False)
conv3 = nn.relu(conv3)
with tf.variable_scope("conv4"):
conv4 = layers.conv2d(conv3, 256, 3, strides=1, padding="same")
conv4 = layers.batch_normalization(conv4, scale=False)
conv4 = nn.relu(conv4)
with tf.variable_scope("conv5"):
conv5 = layers.conv2d(conv4, 256, 3, strides=1, padding="same")
conv5 = layers.batch_normalization(conv5, scale=False)
conv5 = nn.relu(deconv5)
with tf.variable_scope("conv6"):
conv6 = layers.conv2d(conv5, 3, 3, strides=1, padding="same")
with tf.variable_scope("out"):
out2 = nn.tanh(conv6)
return out, out2
def __call__(self,
tensor_in,
condition=None,
condition_track=None,
training=None,
slope=None):
norm = get_normalization(NORMALIZATION, training)
conv_layer = lambda i, f, k, s: ACTIVATION(norm(conv3d(i, f, k, s)))
tconv_layer = lambda i, c, f, k, s: ACTIVATION(
norm(tconv3d(tf.concat((i, c), -1), f, k, s)))
with tf.variable_scope(self.name, reuse=tf.AUTO_REUSE):
# ---------------------------- Encoder -----------------------------
c = condition_track
# Pitch-time private network
with tf.variable_scope('encoder_pitch_time'):
pt_1 = conv_layer(c, 16, (1, 1, 12), (1, 1, 12)) # 4, 48, 7
pt_2 = conv_layer(pt_1, 32, (1, 3, 1), (1, 3, 1)) # 4, 16, 7
# Time-pitch private network
with tf.variable_scope('encoder_time_pitch'):
tp_1 = conv_layer(c, 16, (1, 3, 1), (1, 3, 1)) # 4, 16, 84
tp_2 = conv_layer(tp_1, 32, (1, 1, 12), (1, 1, 12)) # 4, 16, 7
shared = tf.concat((tp_2, pt_2), -1)
# Shared network
with tf.variable_scope('encoder_shared'):
s1 = conv_layer(shared, 64, (1, 4, 3), (1, 4, 2)) # 4, 4, 3
s2 = conv_layer(s1, 128, (1, 4, 3), (1, 4, 3)) # 4, 1, 1
s3 = conv_layer(s2, 256, (4, 1, 1), (4, 1, 1)) # 1, 1, 1
# --------------------------- Generator ----------------------------
h = tensor_in
h = tf.expand_dims(tf.expand_dims(tf.expand_dims(h, 1), 1), 1)
# Shared network
with tf.variable_scope('shared'):
h = tconv_layer(h, s3, 512, (4, 1, 1), (4, 1, 1)) # 4, 1, 1
h = tconv_layer(h, s2, 256, (1, 4, 3), (1, 4, 3)) # 4, 4, 3
h = tconv_layer(h, s1, 128, (1, 4, 3), (1, 4, 2)) # 4, 16, 7
# Pitch-time private network
with tf.variable_scope('pitch_time_private'):
s1 = [
tconv_layer(h, tp_2, 32, (1, 1, 12), (1, 1, 12))
for _ in range(self.n_tracks)
] # 4, 16, 84
s1 = [
tconv_layer(s1[i], tp_1, 16, (1, 3, 1), (1, 3, 1))
for i in range(self.n_tracks)
] # 4, 48, 84
# Time-pitch private network
with tf.variable_scope('time_pitch_private'):
s2 = [
tconv_layer(h, pt_2, 32, (1, 3, 1), (1, 3, 1))
for _ in range(self.n_tracks)
] # 4, 48, 7
s2 = [
tconv_layer(s2[i], pt_1, 16, (1, 1, 12), (1, 1, 12))
for i in range(self.n_tracks)
] # 4, 48, 84
h = [tf.concat((s1[i], s2[i]), -1) for i in range(self.n_tracks)]
# Merged private network
with tf.variable_scope('merged_private'):
h = [
tanh(norm(tconv3d(h[i], 1, (1, 1, 1), (1, 1, 1))))
for i in range(self.n_tracks)
] # 4, 48, 84
return tf.concat((h[:self.condition_track_idx] + [c] +
h[self.condition_track_idx:]), -1)
def generator_1(self, x):
print('making G1-network')
sess = tf.Session()
with tf.variable_scope('G1'):
#Cv0
conv0 = conv_layer(x, [self.KERNEL_SIZE, self.KERNEL_SIZE, 3, 64],
1, 'g_wc0', True, False)
print('Cv0')
print(sess.run(tf.shape(conv0)))
#Cv1
conv1 = conv_layer(conv0,
[self.KERNEL_SIZE, self.KERNEL_SIZE, 64, 128],
1, 'g_wc1')
print('Cv1')
print(sess.run(tf.shape(conv1)))
#Cv2
conv2 = conv_layer(conv1,
[self.KERNEL_SIZE, self.KERNEL_SIZE, 128, 256],
1, 'g_wc2')
print('Cv2')
print(sess.run(tf.shape(conv2)))
#Cv3
conv3 = conv_layer(conv2,
[self.KERNEL_SIZE, self.KERNEL_SIZE, 256, 512],
1, 'g_wc3')
print('Cv3')
print(sess.run(tf.shape(conv3)))
#Cv4
conv4 = conv_layer(conv3,
[self.KERNEL_SIZE, self.KERNEL_SIZE, 512, 512],
1, 'g_wc4x1')
print('Cv4')
print(sess.run(tf.shape(conv4)))
conv4 = conv_layer(conv4,
[self.KERNEL_SIZE, self.KERNEL_SIZE, 512, 512],
1, 'g_wc4x2')
print('Cv4')
print(sess.run(tf.shape(conv4)))
conv4 = conv_layer(conv4,
[self.KERNEL_SIZE, self.KERNEL_SIZE, 512, 512],
1, 'g_wc4x3')
print('Cv4')
print(sess.run(tf.shape(conv4)))
#Cv5
conv5 = conv_layer(conv4,
[self.KERNEL_SIZE, self.KERNEL_SIZE, 512, 512],
1, 'g_wc5', False, False)
conv5 = nn.relu(conv5)
print('Cv5')
print(sess.run(tf.shape(conv5)))
#Cv6
conv6 = deconv2d(
conv5, [self.BATCH, self.INPUT_SIZE, self.INPUT_SIZE, 512],
self.KERNEL_SIZE, self.KERNEL_SIZE, 1, 1, "deConv6")
conv6 = bn(conv6, self.IS_TRAINING, 'g_bn6')
conv6 = nn.relu(conv6)
print('Cv6')
print(sess.run(tf.shape(conv6)))
#Cv7
conv7 = tf.concat([conv6, conv4], 3)
conv7 = deconv2d(
conv7, [self.BATCH, self.INPUT_SIZE, self.INPUT_SIZE, 512],
self.KERNEL_SIZE, self.KERNEL_SIZE, 1, 1, "deConv7x1")
conv7 = bn(conv7, self.IS_TRAINING, 'g_bn7_x1')
conv7 = nn.relu(conv7)
print('Cv7')
print(sess.run(tf.shape(conv7)))
conv7 = tf.concat([conv7, conv4], 3)
conv7 = deconv2d(
conv7, [self.BATCH, self.INPUT_SIZE, self.INPUT_SIZE, 512],
self.KERNEL_SIZE, self.KERNEL_SIZE, 1, 1, "deConv7x2")
conv7 = bn(conv7, self.IS_TRAINING, 'g_bn7_x2')
conv7 = nn.relu(conv7)
print('Cv7')
print(sess.run(tf.shape(conv7)))
conv7 = tf.concat([conv7, conv4], 3)
conv7 = deconv2d(
conv7, [self.BATCH, self.INPUT_SIZE, self.INPUT_SIZE, 512],
self.KERNEL_SIZE, self.KERNEL_SIZE, 1, 1, "deConv7x3")
conv7 = bn(conv7, self.IS_TRAINING, 'g_bn7_x3')
conv7 = nn.relu(conv7)
print('Cv7')
print(sess.run(tf.shape(conv7)))
#Cv8
conv8 = tf.concat([conv7, conv3], 3)
conv8 = deconv2d(
conv8, [self.BATCH, self.INPUT_SIZE, self.INPUT_SIZE, 256],
self.KERNEL_SIZE, self.KERNEL_SIZE, 1, 1, "deConv8")
conv8 = bn(conv8, self.IS_TRAINING, 'g_bn8')
conv8 = nn.relu(conv8)
print('Cv8')
print(sess.run(tf.shape(conv8)))
#Cv9
conv9 = tf.concat([conv8, conv2], 3)
conv9 = deconv2d(
conv9, [self.BATCH, self.INPUT_SIZE, self.INPUT_SIZE, 128],
self.KERNEL_SIZE, self.KERNEL_SIZE, 1, 1, "deConv9")
conv9 = bn(conv9, self.IS_TRAINING, 'g_bn9')
conv9 = nn.relu(conv9)
print('Cv9')
print(sess.run(tf.shape(conv9)))
#Cv10
conv10 = tf.concat([conv9, conv1], 3)
conv10 = deconv2d(
conv10, [self.BATCH, self.INPUT_SIZE, self.INPUT_SIZE, 64],
self.KERNEL_SIZE, self.KERNEL_SIZE, 1, 1, "deConv10")
conv10 = bn(conv10, self.IS_TRAINING, 'g_bn10')
conv10 = nn.relu(conv10)
print('Cv10')
print(sess.run(tf.shape(conv10)))
#Cv11
conv11 = tf.concat([conv10, conv0], 3)
conv11 = deconv2d(
conv11, [self.BATCH, self.INPUT_SIZE, self.INPUT_SIZE, 1],
self.KERNEL_SIZE, self.KERNEL_SIZE, 1, 1, "deConv11")
print('Cv11')
print(sess.run(tf.shape(conv11)))
conv11 = nn.tanh(conv11)
#conv11=nn.sigmoid(conv11)
return conv11
# tf.Tensor( # [[-0.2 0. 1. ] # [ 0. 0. 1. ]], shape=(2, 3), dtype=float32) # tf.Tensor( # [[-0.2 0. 1. ] # [ 0. 0. 1. ]], shape=(2, 3), dtype=float32) print(_sigmoid(x)) print(nn.sigmoid(x)) # tf.Tensor( # [[0.26894143 0.5 0.73105854] # [0.5 0.5 0.73105854]], shape=(2, 3), dtype=float32) # tf.Tensor( # [[0.26894143 0.5 0.73105854] # [0.5 0.5 0.7310586 ]], shape=(2, 3), dtype=float32) print(_tanh(x)) print(nn.tanh(x)) # tf.Tensor( # [[-0.7615942 0. 0.7615941] # [ 0. 0. 0.7615941]], shape=(2, 3), dtype=float32) # tf.Tensor( # [[-0.7615942 0. 0.7615942] # [ 0. 0. 0.7615942]], shape=(2, 3), dtype=float32) print(_softmax(x)) print(nn.softmax(x)) # tf.Tensor( # [[0.09003059 0.24472848 0.66524094] # [0.21194156 0.21194156 0.57611686]], shape=(2, 3), dtype=float32) # tf.Tensor( # [[0.09003057 0.24472848 0.6652409 ] # [0.21194157 0.21194157 0.57611686]], shape=(2, 3), dtype=float32) print(_silu(x))
