def _attention(self, lstm_outputs): with tf.variable_scope('Attention', initializer=self.init): weights = tf.get_variable('weights', [self.lstm_units, self.output_units], regularizer=self.regularizer) biases = tf.get_variable('biases', [1, self.output_units]) u_w = tf.get_variable('u_w', [self.output_units, 1]) outputs, scores = [], [] for v in lstm_outputs: hidden_rep = nn.tanh(tf.add(tf.matmul(v, weights), biases)) scores.append(tf.matmul(hidden_rep, u_w)) # list -> tensor scores = tf.concat(scores, axis=1) # softmax scores = nn.softmax(scores, dim=-1) # tensor -> list scores = tf.unstack(scores, axis=1) for i, v in enumerate(scores): # v: (64,) -> (64,1) v = tf.reshape(v, [-1, 1]) # v: (64,1) -> [(64,1), (64,1), ...] v = [v] * self.lstm_units # v: (64,self.lstm_units) v = tf.concat(v, axis=1) outputs.append(tf.multiply(v, lstm_outputs[i])) return tf.add_n(outputs)
def call(self, x, training=True, return_lowres=False): if len(x.shape) == 2: x = self._low_res_generator(x, training=False) lowres = x for block in self.blocks: x = block(x, training=training) hires = tanh(self.final_conv(x)) return (hires, lowres) if return_lowres else hires
def call(self, x, training=True): x = self.fc(x) x = self.initial_norm(x, training=training) x = tf.nn.relu(x) x = tf.reshape(x, shape=(-1, 15, 15, 64)) for block in self.blocks: x = block(x, training=training) return tanh(self.final_conv(x))
def call(self, features, hidden): hidden_with_time_axis = expand_dims(hidden, 1) score = nn.tanh(self.W1(features) + self.W2(hidden_with_time_axis)) attention_weights = nn.softmax(self.V(score), axis=1) context_vector = attention_weights * features context_vector = reduce_sum(context_vector, axis=1) return context_vector, attention_weights
def ACT(inputs, act_fn): if act_fn == 'relu': act = relu(inputs) elif act_fn == 'lrelu': act = leaky_relu(inputs) elif act_fn == 'sigmoid': act = sigmoid(inputs) elif act_fn == 'tanh': act = tanh(inputs) else: act = inputs return act
def call(self, q, val): #make q i.e. the hidden value into the same shape q_with_time_axis = tf.expand_dims(q, 1) score = self.V(nn.tanh(self.W1(q_with_time_axis) + self.W2(val))) attention_w = nn.softmax(score, axis=1) context_vec = attention_w * val context_vec = tf.reduce_sum(context_vec, axis=1) return context_vec, attention_w
def __init__(self, game, tensor_logs_dir): self.game = game self.shape = game.board_size self.possible_moves_size = self.shape[1] self.log_writer = tf.summary.FileWriter(tensor_logs_dir + "/losses") self.checkpoint_dir = "checkpoints" create_dir(self.checkpoint_dir) self.graph = tf.Graph() self.session = tf.Session(graph=self.graph) self.saver = None with tf.Session() as temp_session: temp_session.run(tf.global_variables_initializer()) with self.graph.as_default(): self.input_boards = tf.placeholder(tf.float32, shape=[None, self.game.board.rows, self.game.board.cols]) self.dropout = tf.placeholder(tf.float32) self.isTraining = tf.placeholder(tf.bool, name="is_training") self.pi_losses_var = tf.Variable(0, dtype=tf.float32) self.v_losses_var = tf.Variable(0, dtype=tf.float32) X = tf.reshape(self.input_boards, [-1, self.shape[0], self.shape[1], 1]) h_conv1 = relu(batch_normalization(conv2d(X, 'same'), axis=3, training=self.isTraining)) h_conv2 = relu(batch_normalization(conv2d(h_conv1, 'same'), axis=3, training=self.isTraining)) h_conv3 = relu(batch_normalization(conv2d(h_conv2, 'valid'), axis=3, training=self.isTraining)) h_conv4 = relu(batch_normalization(conv2d(h_conv3, 'valid'), axis=3, training=self.isTraining)) h_conv4_flat = tf.reshape(h_conv4, [-1, config.num_channels * (self.shape[0] - 4) * (self.shape[1] - 4)]) s_fc1 = dropout(relu(batch_normalization(dense(h_conv4_flat, 1024), axis=1, training=self.isTraining)), rate=self.dropout) s_fc2 = dropout(relu(batch_normalization(dense(s_fc1, 512), axis=1, training=self.isTraining)), rate=self.dropout) self.pi = dense(s_fc2, self.possible_moves_size) self.prob = tf.nn.softmax(self.pi) self.v = tanh(dense(s_fc2, 1)) # Place holders for Predicted (pi, v)s self.predicted_pis = tf.placeholder(dtype=tf.float32, shape=[None, self.possible_moves_size]) self.predicted_vs = tf.placeholder(dtype=tf.float32, shape=[None]) # Real Losses self.loss_pi = tf.losses.softmax_cross_entropy(self.predicted_pis, self.pi) self.loss_v = tf.losses.mean_squared_error(self.predicted_vs, tf.reshape(self.v, shape=[-1, ])) self.total_loss = self.loss_pi + self.loss_v update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): self.train_step = tf.train.AdamOptimizer(config.lr).minimize(self.total_loss) self.session.run(tf.variables_initializer(self.graph.get_collection('variables')))
def get_attention(units, lstm_units, n_layers_att, p_dropout=0, l1_reg=0, l2_reg=0): # W1 = Dense(units, kernel_regularizer=l1_l2(l1_reg, l2_reg), name = 'W_feats') W1 = [ Dense(units, activation='tanh', name='W_feats_{}'.format(i)) for i in range(n_layers_att) ] W2 = Dense(units, kernel_regularizer=l1_l2(l1_reg, l2_reg), name='W_hidden') V = Dense(1, kernel_regularizer=l1_l2(l1_reg, l2_reg), name='V') f_beta = Dense(1, kernel_regularizer=l1_l2(l1_reg, l2_reg), name='f_beta') dropout = Dropout(p_dropout, name='dropout') # shape = (batch, attn_features, features_shape) encoder_output = Input(feature_vector_shape, name='image_features') # shape = (batch, lstm_units) hidden_last = Input(lstm_units, name='last_hidden_state') projected_features = dropout(W1[0](encoder_output)) for i in range(1, n_layers_att): projected_features = dropout(W1[i](projected_features)) # shape = (batch, attn_features, 1) score = V(tanh( projected_features + \ dropout(W2(tf.expand_dims(hidden_last, axis = 1))) )) # shape = (batch, attn_features) score = dropout(tf.reduce_sum(score, axis=2)) attention_weights = Activation('softmax', dtype='float32')(score) # beta = f_beta(dropout(tf.expand_dims(hidden_last, axis = 1))) # shape = (batch, 1) beta = f_beta(hidden_last) beta = Activation('sigmoid', dtype='float32')(beta) # shape = (batch, attn_features, features_shape) context_vector = tf.expand_dims(attention_weights, axis=2) * encoder_output # shape = (batch, features_shape) context_vector = beta * tf.reduce_sum(context_vector, axis=1) context_vector = Activation('linear', dtype='float32')(context_vector) return Model(inputs=[encoder_output, hidden_last], outputs=[context_vector, attention_weights], name='attention')
def forward_pass(self, input_data): input_data = input_data / 255 pre_inception = InceptionNet._pre_inception_layer( input_data, self.pre_inception_filters) inception_1 = InceptionNet._inception_layer(pre_inception, self.inception_1_filters) inception_2 = InceptionNet._inception_layer(inception_1, self.inception_2_filters) # print(inception_2) post_inception_pool = InceptionNet._max_pool(inception_2, (3, 3), (2, 2)) inception_3 = InceptionNet._inception_layer(post_inception_pool, self.inception_3_filters) inception_4 = InceptionNet._inception_layer(inception_3, self.inception_4_filters) inception_5 = InceptionNet._inception_layer(inception_4, self.inception_5_filters) inception_6 = InceptionNet._inception_layer(inception_5, self.inception_6_filters) inception_7 = InceptionNet._inception_layer(inception_6, self.inception_7_filters) post_inception_pool_2 = InceptionNet._max_pool(inception_7, (3, 3), (2, 2)) inception_8 = InceptionNet._inception_layer(post_inception_pool_2, self.inception_8_filters) inception_9 = InceptionNet._inception_layer(inception_8, self.inception_9_filters) post_inception_pool_3 = nn.avg_pool(inception_9, (7, 7), strides=4, padding="SAME") flatten_layer = tf.reshape( tf.keras.backend.flatten(post_inception_pool_3), [1024, input_data.shape[0]]) relu_layer = nn.relu( tf.matmul(tf.transpose(self.relu_weights), flatten_layer) + self.relu_bias) dropout_layer = nn.dropout(relu_layer, .4) linear_layer = tf.matmul(tf.transpose(self.linear_weights), dropout_layer) + self.linear_bias return nn.softmax(tf.transpose(nn.tanh(linear_layer)))
def __call__(self, tensor_in, condition=None, training=None, slope=None): norm = get_normalization(NORMALIZATION, training) tconv_layer = lambda i, f, k, s: ACTIVATION(norm(tconv3d(i, f, k, s))) with tf.variable_scope(self.name, reuse=tf.AUTO_REUSE): h = tensor_in h = tf.expand_dims(tf.expand_dims(tf.expand_dims(h, 1), 1), 1) # Shared network with tf.variable_scope('shared'): h = tconv_layer(h, 512, (4, 1, 1), (4, 1, 1)) # 4, 1, 1 h = tconv_layer(h, 256, (1, 4, 1), (1, 4, 1)) # 4, 4, 1 h = tconv_layer(h, 128, (1, 4, 1), (1, 4, 1)) # 4, 16, 1 h = tconv_layer(h, 64, (1, 1, 12), (1, 1, 12)) # 4, 16, 12 # Pitch-time private network with tf.variable_scope('pitch_time_private'): s1 = [ tconv_layer(h, 32, (1, 1, 7), (1, 1, 7)) # 4, 16, 84 for _ in range(self.n_tracks) ] s1 = [ tconv_layer(s1[i], 16, (1, 3, 1), (1, 3, 1)) # 4, 48, 84 for i in range(self.n_tracks) ] # Time-pitch private network with tf.variable_scope('time_pitch_private'): s2 = [ tconv_layer(h, 32, (1, 3, 1), (1, 3, 1)) # 4, 48, 12 for _ in range(self.n_tracks) ] s2 = [ tconv_layer(s2[i], 16, (1, 1, 7), (1, 1, 7)) # 4, 48, 84 for i in range(self.n_tracks) ] h = [tf.concat((s1[i], s2[i]), -1) for i in range(self.n_tracks)] # Merged private network with tf.variable_scope('merged_private'): h = [ norm(tconv3d(h[i], 1, (1, 1, 1), (1, 1, 1))) # 4, 48, 84 for i in range(self.n_tracks) ] h = tf.concat(h, -1) return tanh(h)
def __call__(self, v, u): """ Input: - v: N x D x H x W - u: N x D Returns: - next_u: N x D """ N, K = v.shape(0), self.hidden_dim D, H, W = v.shape(1), v.shape(2), v.shape(3) v_proj = self.Wv(v) # N x K x H x W u_proj = self.Wu(u) # N x K u_proj_expand = u_proj.reshape(N, K, 1, 1).expand(N, K, H, W) h = nn.tanh(v_proj + u_proj_expand) p = nn.softmax(self.Wp(h).reshape(N, H * W)).reshape(N, 1, H, W) self.attention_maps = tf.identity(p.data) v_tilde = (p.expand_as(v) * v).sum(2).sum(3).reshape(N, D) next_u = u + v_tilde return next_u
def __call__(self, tensor_in, condition=None, training=None, slope=None): norm = get_normalization(NORMALIZATION, training) tconv_layer = lambda i, f, k, s: ACTIVATION(norm(tconv3d(i, f, k, s))) with tf.variable_scope(self.name, reuse=tf.AUTO_REUSE): h = tensor_in h = tf.expand_dims(tf.expand_dims(tf.expand_dims(h, 1), 1), 1) # Shared network with tf.variable_scope('shared'): h = tconv_layer(h, 512, (4, 1, 1), (4, 1, 1)) # 4, 1, 1 h = tconv_layer(h, 256, (1, 4, 1), (1, 4, 1)) # 4, 4, 1 h = tconv_layer(h, 128, (1, 4, 1), (1, 4, 1)) # 4, 16, 1 h = tconv_layer(h, 64, (1, 1, 12), (1, 1, 12)) # 4, 16, 12 # Pitch-time private network with tf.variable_scope('pitch_time_private'): s1 = [tconv_layer(h, 32, (1, 1, 7), (1, 1, 7)) # 4, 16, 84 for _ in range(self.n_tracks)] s1 = [tconv_layer(s1[i], 16, (1, 3, 1), (1, 3, 1)) # 4, 48, 84 for i in range(self.n_tracks)] # Time-pitch private network with tf.variable_scope('time_pitch_private'): s2 = [tconv_layer(h, 32, (1, 3, 1), (1, 3, 1)) # 4, 48, 12 for _ in range(self.n_tracks)] s2 = [tconv_layer(s2[i], 16, (1, 1, 7), (1, 1, 7)) # 4, 48, 84 for i in range(self.n_tracks)] h = [tf.concat((s1[i], s2[i]), -1) for i in range(self.n_tracks)] # Merged private network with tf.variable_scope('merged_private'): h = [norm(tconv3d(h[i], 1, (1, 1, 1), (1, 1, 1))) # 4, 48, 84 for i in range(self.n_tracks)] h = tf.concat(h, -1) return tanh(h)
def call(self, x, hidden, enc_output): # Attention hidden_with_time_axis = tf.expand_dims(hidden, 1) score = self.Verdict( tanh(self.W1(enc_output) + self.W2(hidden_with_time_axis))) attention_weights = softmax(score, axis=1) context_vector = attention_weights * enc_output context_vector = tf.reduce_sum(context_vector, axis=1) # forward pass x = self.embedding(x) x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1) output, state = self.gru(x) output = tf.reshape(output, (-1, output.shape[2])) x = self.final_output(output) return x, state, attention_weights
def netG16_decoder(layers, lab=False): enc_conv1, enc_conv2, enc_conv3, enc_conv4, enc_conv5, enc_conv6, enc_conv7, enc_conv8 = layers[0], layers[1], layers[2], layers[3], layers[4], layers[5], layers[6], layers[7] # decoder, no batch norm dec_conv1 = tcl.convolution2d_transpose(enc_conv8, 512, 4, 2, activation_fn=tf.identity, weights_initializer=tf.random_normal_initializer(stddev=0.02), scope='g_dec_conv1') dec_conv1 = relu(dec_conv1) dec_conv1 = tf.concat([dec_conv1, enc_conv7], axis=3) print (dec_conv1) dec_conv2 = tcl.convolution2d_transpose(dec_conv1, 512, 4, 2, activation_fn=tf.identity, weights_initializer=tf.random_normal_initializer(stddev=0.02), scope='g_dec_conv2') dec_conv2 = relu(dec_conv2) dec_conv2 = tf.concat([dec_conv2, enc_conv6], axis=3) print (dec_conv2) dec_conv3 = tcl.convolution2d_transpose(dec_conv2, 512, 4, 2, activation_fn=tf.identity, weights_initializer=tf.random_normal_initializer(stddev=0.02), scope='g_dec_conv3') dec_conv3 = relu(dec_conv3) dec_conv3 = tf.concat([dec_conv3, enc_conv5], axis=3) print (dec_conv3) dec_conv4 = tcl.convolution2d_transpose(dec_conv3, 512, 4, 2, activation_fn=tf.identity, weights_initializer=tf.random_normal_initializer(stddev=0.02), scope='g_dec_conv4') dec_conv4 = relu(dec_conv4) dec_conv4 = tf.concat([dec_conv4, enc_conv4], axis=3) print (dec_conv4) dec_conv5 = tcl.convolution2d_transpose(dec_conv4, 256, 4, 2, activation_fn=tf.identity, weights_initializer=tf.random_normal_initializer(stddev=0.02), scope='g_dec_conv5') dec_conv5 = relu(dec_conv5) dec_conv5 = tf.concat([dec_conv5, enc_conv3], axis=3) print (dec_conv5) dec_conv6 = tcl.convolution2d_transpose(dec_conv5, 128, 4, 2, activation_fn=tf.identity, weights_initializer=tf.random_normal_initializer(stddev=0.02), scope='g_dec_conv6') dec_conv6 = relu(dec_conv6) dec_conv6 = tf.concat([dec_conv6, enc_conv2], axis=3) print (dec_conv6) dec_conv7 = tcl.convolution2d_transpose(dec_conv6, 64, 4, 2, activation_fn=tf.identity, weights_initializer=tf.random_normal_initializer(stddev=0.02), scope='g_dec_conv7') dec_conv7 = relu(dec_conv7) dec_conv7 = tf.concat([dec_conv7, enc_conv1], axis=3) print (dec_conv7) c = 2 if lab else 3 dec_conv8 = tcl.convolution2d_transpose(dec_conv7, c, 4, 2, activation_fn=tf.identity, weights_initializer=tf.random_normal_initializer(stddev=0.02), scope='g_dec_conv8') dec_conv8 = tanh(dec_conv8) print (dec_conv1) return dec_conv8
def netG(x): print("\nnetG\n") print(x) conv1 = tcl.conv2d( x, 64, 7, 1, activation_fn=tf.nn.relu, normalizer_fn=tcl.batch_norm, weights_initializer=tf.random_normal_initializer(stddev=0.02), scope='g_conv1') print(conv1) conv2 = tcl.conv2d( conv1, 128, 3, 2, activation_fn=tf.nn.relu, normalizer_fn=tcl.batch_norm, weights_initializer=tf.random_normal_initializer(stddev=0.02), scope='g_conv2') print(conv2) conv3 = tcl.conv2d( conv2, 256, 3, 2, activation_fn=tf.identity, normalizer_fn=tcl.batch_norm, weights_initializer=tf.random_normal_initializer(stddev=0.02), scope='g_conv3') print(conv3) res1 = resBlock(conv3, 1) res2 = resBlock(res1, 2) res3 = resBlock(res2, 3) res4 = resBlock(res3, 4) res5 = resBlock(res4, 5) res6 = resBlock(res5, 6) res7 = resBlock(res6, 7) res8 = resBlock(res7, 8) res9 = resBlock(res8, 9) res10 = resBlock(res9, 10) res11 = resBlock(res10, 11) res12 = resBlock(res11, 12) res13 = resBlock(res12, 13) res14 = resBlock(res13, 14) res15 = resBlock(res14, 15) res16 = resBlock(res15, 16) res17 = resBlock(res16, 17) res18 = resBlock(res17, 18) res19 = resBlock(res18, 19) res20 = resBlock(res19, 20) res21 = resBlock(res20, 21) res22 = resBlock(res21, 22) res23 = resBlock(res22, 23) res24 = resBlock(res23, 24) res25 = resBlock(res24, 25) res26 = resBlock(res25, 26) res27 = resBlock(res26, 27) res28 = resBlock(res27, 28) res29 = resBlock(res28, 29) res30 = resBlock(res29, 30) conv4 = tcl.conv2d_transpose( res30, 128, 3, 2, activation_fn=tf.nn.relu, normalizer_fn=tcl.batch_norm, weights_initializer=tf.random_normal_initializer(stddev=0.02), scope='g_conv4') print(conv4) conv5 = tcl.conv2d_transpose( conv4, 64, 3, 2, activation_fn=tf.nn.relu, normalizer_fn=tcl.batch_norm, weights_initializer=tf.random_normal_initializer(stddev=0.02), scope='g_conv5') print(conv5) conv6 = tcl.conv2d( conv5, 3, 7, 1, activation_fn=tf.identity, normalizer_fn=tcl.batch_norm, weights_initializer=tf.random_normal_initializer(stddev=0.02), scope='g_conv6') conv6 = tanh(conv6) print(conv6) return conv6
def get_decoder(embedding_dim, units, lstm_units, vocab_size, n_layers_att, attn_dropout=0, lstm_dropout=0, logit_dropout=0, l1_reg=0, l2_reg=0): attention = get_attention(units, lstm_units, n_layers_att, attn_dropout, l1_reg, l2_reg) embedding = Embedding(input_dim=vocab_size, output_dim=embedding_dim, input_length=1, name='embedding') lstm = LSTM(lstm_units, # return_sequences = True, return_state = True, recurrent_initializer = 'glorot_uniform', dropout = lstm_dropout, # recurrent_dropout = p_dropout, kernel_regularizer = l1_l2(l1_reg, l2_reg),\ dtype = 'float32') last_layer_hidden = Dense(embedding_dim, kernel_regularizer=l1_l2(l1_reg, l2_reg), name='last_hidden') last_layer_context = Dense(embedding_dim, kernel_regularizer=l1_l2(l1_reg, l2_reg), name='last_context') logits_kernel = Dense(vocab_size, kernel_regularizer=l1_l2(l1_reg, l2_reg), name='logits_kernel') dropout = Dropout(logit_dropout, name='dropout') # shape = (batch, 1) word_input = Input(1, name='bow_input') # shape = (batch, attn_features, features_shape) encoder_output = Input(feature_vector_shape, name='image_features') # shape = (batch, lstm_units) hidden_last = Input(lstm_units, name='last_hidden_state') cell_last = Input(lstm_units, name='last_cell_state') # see keras doc on Embedding layer: if input shape is (batch, input_length) # then output shape is (batch, input_length, embedding_dim). # In this case, input_length is always 1, so the embedded_word shape is # (batch, 1, embedding_dim). We will use this axis 1 for the lstm input embedded_word = embedding(word_input) #this just for caompatibility with mixed precision embedded_word = Activation('linear')(embedded_word) # context_vector shape = (batch, features_shape) context_vector, attention_weights = attention( [encoder_output, hidden_last]) # RNN input shape must be (batch, time_steps, features). In our case, # time_steps is 1, so we must expand the context vector dimensions # lstm_output shape = (batch, lstm_units) # shape = (batch, 1, embedding_dim + features_shape) lstm_input = Concatenate(axis=-1)( [embedded_word, tf.expand_dims(context_vector, axis=1)]) # shape = (batch, lstm_units) lstm_output, hidden, cell = lstm(lstm_input, initial_state=[hidden_last, cell_last]) # shape = (batch, embedding_dim) embedded_hidden = last_layer_hidden(dropout(lstm_output)) embedded_context = last_layer_context(context_vector) # Now we finally drop the extra 1 axis in the embedded_word logits_kernel_input = tanh(tf.reduce_sum(embedded_word,axis=1) + \ embedded_hidden + embedded_context) # logits_kernel_input = Concatenate(axis=-1)([tf.reduce_sum(embedded_word, # axis=1), # context_vector, # lstm_output]) # shape = (batch, vocab_size) logits = logits_kernel(dropout(logits_kernel_input)) logits = Activation('linear', dtype='float32')(logits) return Model(inputs=[word_input, encoder_output, hidden_last, cell_last], outputs=[logits, attention_weights, hidden, cell])
def call(self, x, training=True): for block in self.blocks: x = block(x, training=training) return tanh(self.final_conv(x))
def train_layers(self, train_x, train_y, test_x, test_y): params = {} X = tf.placeholder(tf.float32, [None, self.opt['n_dim']]) Y = tf.placeholder(tf.float32, [None, self.opt['n_classes']]) keep_prob = tf.placeholder(tf.float32) #for dropout params['W1'] = tf.Variable( tf.random_normal([self.opt['n_dim'], self.opt['num_hidden1']], mean=0, stddev=self.opt['std'])) params['b1'] = tf.Variable( tf.random_normal([self.opt['num_hidden1']], mean=0, stddev=self.opt['std'])) params['a1'] = nn.sigmoid(tf.matmul(X, params['W1']) + params['b1']) params['dropout1'] = nn.dropout(params['a1'], keep_prob) params['W2'] = tf.Variable( tf.random_normal( [self.opt['num_hidden1'], self.opt['num_hidden2']], mean=0, stddev=self.opt['std'])) params['b2'] = tf.Variable( tf.random_normal([self.opt['num_hidden2']], mean=0, stddev=self.opt['std'])) params['a2'] = nn.relu( tf.matmul(params['dropout1'], params['W2']) + params['b2']) params['dropout2'] = nn.dropout(params['a2'], keep_prob) params['W3'] = tf.Variable( tf.random_normal( [self.opt['num_hidden2'], self.opt['num_hidden3']], mean=0, stddev=self.opt['std'])) params['b3'] = tf.Variable( tf.random_normal([self.opt['num_hidden3']], mean=0, stddev=self.opt['std'])) params['a3'] = nn.tanh( tf.matmul(params['dropout2'], params['W3']) + params['b3']) params['dropout3'] = nn.dropout(params['a3'], keep_prob) params['outW'] = tf.Variable( tf.random_normal([self.opt['num_hidden3'], self.opt['n_classes']], mean=0, stddev=self.opt['std'])) params['outb'] = tf.Variable( tf.random_normal([self.opt['n_classes']], mean=0, stddev=self.opt['std'])) out = nn.softmax( tf.matmul(params['dropout3'], params['outW']) + params['outb']) cost = tf.reduce_mean( -tf.reduce_sum(Y * tf.log(out), reduction_indices=[1])) optimizer = tf.train.AdamOptimizer( self.opt['learning_rate']).minimize(cost) correct_pred = tf.equal(tf.argmax(out, 1), tf.argmax(Y, 1)) accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) cost_history = np.empty(shape=[1], dtype=float) y, y_pred = None, None #reshape labels into a one hot vector f = FeatureParser() train_y = f.one_hot_encode(train_y) test_y = f.one_hot_encode(test_y) print('TRAIN_ONE_HOT_LABEL{}'.format(train_y)) print('Training...') with tf.Session() as sess: sess.run(tf.global_variables_initializer()) for epoch in range(training_epochs): _, loss, acc = sess.run([optimizer, cost, accuracy], feed_dict={ X: train_x, Y: train_y, keep_prob: 0.5 }) cost_history = np.append(cost_history, loss) if epoch % 50 == 0: print('Epoch#', epoch, 'Cost:', loss, 'Train acc.:', acc) y_pred = sess.run(tf.argmax(out, 1), feed_dict={ X: test_x, keep_prob: 1.0 }) y = sess.run(tf.argmax(test_y, 1)) print( "Test accuracy: ", round( sess.run(accuracy, feed_dict={ X: test_x, Y: test_y, keep_prob: 1.0 }), 3)) fig = plt.figure(figsize=(10, 8)) plt.plot(cost_history) plt.xlabel('Iterations') plt.ylabel('Cost') plt.axis([0, training_epochs, 0, np.max(cost_history)]) plt.show() precision, recall, f_score, s = precision_recall_fscore_support( y, y_pred, average='micro') print('F score:', round(f_score, 3))
def __call__(self, tensor_in, condition=None, condition_track=None, training=None, slope=None): norm = get_normalization(NORMALIZATION, training) conv_layer = lambda i, f, k, s: ACTIVATION(norm(conv3d(i, f, k, s))) tconv_layer = lambda i, c, f, k, s: ACTIVATION(norm(tconv3d( tf.concat((i, c), -1), f, k, s))) with tf.variable_scope(self.name, reuse=tf.AUTO_REUSE): # ---------------------------- Encoder ----------------------------- c = condition_track # Pitch-time private network with tf.variable_scope('encoder_pitch_time'): pt_1 = conv_layer(c, 16, (1, 1, 12), (1, 1, 12)) # 4, 48, 7 pt_2 = conv_layer(pt_1, 32, (1, 3, 1), (1, 3, 1)) # 4, 16, 7 # Time-pitch private network with tf.variable_scope('encoder_time_pitch'): tp_1 = conv_layer(c, 16, (1, 3, 1), (1, 3, 1)) # 4, 16, 84 tp_2 = conv_layer(tp_1, 32, (1, 1, 12), (1, 1, 12)) # 4, 16, 7 shared = tf.concat((tp_2, pt_2), -1) # Shared network with tf.variable_scope('encoder_shared'): s1 = conv_layer(shared, 64, (1, 4, 3), (1, 4, 2)) # 4, 4, 3 s2 = conv_layer(s1, 128, (1, 4, 3), (1, 4, 3)) # 4, 1, 1 s3 = conv_layer(s2, 256, (4, 1, 1), (4, 1, 1)) # 1, 1, 1 # --------------------------- Generator ---------------------------- h = tensor_in h = tf.expand_dims(tf.expand_dims(tf.expand_dims(h, 1), 1), 1) # Shared network with tf.variable_scope('shared'): h = tconv_layer(h, s3, 512, (4, 1, 1), (4, 1, 1)) # 4, 1, 1 h = tconv_layer(h, s2, 256, (1, 4, 3), (1, 4, 3)) # 4, 4, 3 h = tconv_layer(h, s1, 128, (1, 4, 3), (1, 4, 2)) # 4, 16, 7 # Pitch-time private network with tf.variable_scope('pitch_time_private'): s1 = [tconv_layer(h, tp_2, 32, (1, 1, 12), (1, 1, 12)) for _ in range(self.n_tracks)] # 4, 16, 84 s1 = [tconv_layer(s1[i], tp_1, 16, (1, 3, 1), (1, 3, 1)) for i in range(self.n_tracks)] # 4, 48, 84 # Time-pitch private network with tf.variable_scope('time_pitch_private'): s2 = [tconv_layer(h, pt_2, 32, (1, 3, 1), (1, 3, 1)) for _ in range(self.n_tracks)] # 4, 48, 7 s2 = [tconv_layer(s2[i], pt_1, 16, (1, 1, 12), (1, 1, 12)) for i in range(self.n_tracks)] # 4, 48, 84 h = [tf.concat((s1[i], s2[i]), -1) for i in range(self.n_tracks)] # Merged private network with tf.variable_scope('merged_private'): h = [tanh(norm(tconv3d(h[i], 1, (1, 1, 1), (1, 1, 1)))) for i in range(self.n_tracks)] # 4, 48, 84 return tf.concat(( h[:self.condition_track_idx] + [c] + h[self.condition_track_idx:]), -1)
def _gate(self, x): tanh_preactivation, sigmoid_preactivation = tf.split(x, 2, axis=-1) return nn.tanh(tanh_preactivation) * nn.sigmoid(sigmoid_preactivation)
def generator(x): with tf.variable_scope("generator_sr", reuse=tf.AUTO_REUSE, initializer=tf.initializers.random_normal( 0.0, 0.02)): with tf.variable_scope("conv1"): conv1 = layers.conv2d(x, 64, 3, strides=1, padding="same") conv1 = layers.batch_normalization(conv1, scale=False) conv1 = nn.relu(conv1) with tf.variable_scope("conv2"): # residual block with (up to) 8 convolution layers n_conv_layers = 8 previous = conv1 for i in range(n_conv_layers): conv2 = layers.conv2d(previous, 64, 3, strides=1, padding="same") conv2 = layers.batch_normalization(conv2, scale=False) conv2 += previous conv2 = nn.relu(conv2) previous = conv2 with tf.variable_scope("conv3"): conv3 = layers.conv2d(conv2, 64, 3, strides=1, padding="same") conv3 = layers.batch_normalization(conv3, scale=False) conv3 = nn.relu(conv3) with tf.variable_scope("deconv4"): deconv4 = layers.conv2d_transpose(conv3, 256, 3, strides=2, padding="same") deconv4 = layers.batch_normalization(deconv4, scale=False) deconv4 = nn.relu(deconv4) with tf.variable_scope("deconv5"): deconv5 = layers.conv2d_transpose(deconv4, 256, 3, strides=2, padding="same") deconv5 = layers.batch_normalization(deconv5, scale=False) deconv5 = nn.relu(deconv5) with tf.variable_scope("conv6"): conv6 = layers.conv2d(deconv5, 3, 1, strides=1, padding="same") with tf.variable_scope("out"): out = nn.tanh(conv6) #Adding Refinement network with tf.variable_scope("generator_rf", reuse=tf.AUTO_REUSE, initializer=tf.initializers.random_normal( 0.0, 0.02)): with tf.variable_scope("conv1"): conv6 = layers.batch_normalization(conv6, scale=False) conv6 = nn.relu(conv6) conv1 = layers.conv2d(conv1, 64, 3, strides=1, padding="same") conv1 = layers.batch_normalization(conv1, scale=False) conv1 = nn.relu(conv1) with tf.variable_scope("conv2"): # residual block with (up to) 8 convolution layers n_conv_layers = 8 previous = conv1 for i in range(n_conv_layers): conv2 = layers.conv2d(previous, 64, 3, strides=1, padding="same") conv2 = layers.batch_normalization(conv2, scale=False) conv2 += previous conv2 = nn.relu(conv2) previous = conv2 with tf.variable_scope("conv3"): conv3 = layers.conv2d(conv2, 64, 3, strides=1, padding="same") conv3 = layers.batch_normalization(conv3, scale=False) conv3 = nn.relu(conv3) with tf.variable_scope("conv4"): conv4 = layers.conv2d(conv3, 256, 3, strides=1, padding="same") conv4 = layers.batch_normalization(conv4, scale=False) conv4 = nn.relu(conv4) with tf.variable_scope("conv5"): conv5 = layers.conv2d(conv4, 256, 3, strides=1, padding="same") conv5 = layers.batch_normalization(conv5, scale=False) conv5 = nn.relu(deconv5) with tf.variable_scope("conv6"): conv6 = layers.conv2d(conv5, 3, 3, strides=1, padding="same") with tf.variable_scope("out"): out2 = nn.tanh(conv6) return out, out2
def __call__(self, tensor_in, condition=None, condition_track=None, training=None, slope=None): norm = get_normalization(NORMALIZATION, training) conv_layer = lambda i, f, k, s: ACTIVATION(norm(conv3d(i, f, k, s))) tconv_layer = lambda i, c, f, k, s: ACTIVATION( norm(tconv3d(tf.concat((i, c), -1), f, k, s))) with tf.variable_scope(self.name, reuse=tf.AUTO_REUSE): # ---------------------------- Encoder ----------------------------- c = condition_track # Pitch-time private network with tf.variable_scope('encoder_pitch_time'): pt_1 = conv_layer(c, 16, (1, 1, 12), (1, 1, 12)) # 4, 48, 7 pt_2 = conv_layer(pt_1, 32, (1, 3, 1), (1, 3, 1)) # 4, 16, 7 # Time-pitch private network with tf.variable_scope('encoder_time_pitch'): tp_1 = conv_layer(c, 16, (1, 3, 1), (1, 3, 1)) # 4, 16, 84 tp_2 = conv_layer(tp_1, 32, (1, 1, 12), (1, 1, 12)) # 4, 16, 7 shared = tf.concat((tp_2, pt_2), -1) # Shared network with tf.variable_scope('encoder_shared'): s1 = conv_layer(shared, 64, (1, 4, 3), (1, 4, 2)) # 4, 4, 3 s2 = conv_layer(s1, 128, (1, 4, 3), (1, 4, 3)) # 4, 1, 1 s3 = conv_layer(s2, 256, (4, 1, 1), (4, 1, 1)) # 1, 1, 1 # --------------------------- Generator ---------------------------- h = tensor_in h = tf.expand_dims(tf.expand_dims(tf.expand_dims(h, 1), 1), 1) # Shared network with tf.variable_scope('shared'): h = tconv_layer(h, s3, 512, (4, 1, 1), (4, 1, 1)) # 4, 1, 1 h = tconv_layer(h, s2, 256, (1, 4, 3), (1, 4, 3)) # 4, 4, 3 h = tconv_layer(h, s1, 128, (1, 4, 3), (1, 4, 2)) # 4, 16, 7 # Pitch-time private network with tf.variable_scope('pitch_time_private'): s1 = [ tconv_layer(h, tp_2, 32, (1, 1, 12), (1, 1, 12)) for _ in range(self.n_tracks) ] # 4, 16, 84 s1 = [ tconv_layer(s1[i], tp_1, 16, (1, 3, 1), (1, 3, 1)) for i in range(self.n_tracks) ] # 4, 48, 84 # Time-pitch private network with tf.variable_scope('time_pitch_private'): s2 = [ tconv_layer(h, pt_2, 32, (1, 3, 1), (1, 3, 1)) for _ in range(self.n_tracks) ] # 4, 48, 7 s2 = [ tconv_layer(s2[i], pt_1, 16, (1, 1, 12), (1, 1, 12)) for i in range(self.n_tracks) ] # 4, 48, 84 h = [tf.concat((s1[i], s2[i]), -1) for i in range(self.n_tracks)] # Merged private network with tf.variable_scope('merged_private'): h = [ tanh(norm(tconv3d(h[i], 1, (1, 1, 1), (1, 1, 1)))) for i in range(self.n_tracks) ] # 4, 48, 84 return tf.concat((h[:self.condition_track_idx] + [c] + h[self.condition_track_idx:]), -1)
def generator_1(self, x): print('making G1-network') sess = tf.Session() with tf.variable_scope('G1'): #Cv0 conv0 = conv_layer(x, [self.KERNEL_SIZE, self.KERNEL_SIZE, 3, 64], 1, 'g_wc0', True, False) print('Cv0') print(sess.run(tf.shape(conv0))) #Cv1 conv1 = conv_layer(conv0, [self.KERNEL_SIZE, self.KERNEL_SIZE, 64, 128], 1, 'g_wc1') print('Cv1') print(sess.run(tf.shape(conv1))) #Cv2 conv2 = conv_layer(conv1, [self.KERNEL_SIZE, self.KERNEL_SIZE, 128, 256], 1, 'g_wc2') print('Cv2') print(sess.run(tf.shape(conv2))) #Cv3 conv3 = conv_layer(conv2, [self.KERNEL_SIZE, self.KERNEL_SIZE, 256, 512], 1, 'g_wc3') print('Cv3') print(sess.run(tf.shape(conv3))) #Cv4 conv4 = conv_layer(conv3, [self.KERNEL_SIZE, self.KERNEL_SIZE, 512, 512], 1, 'g_wc4x1') print('Cv4') print(sess.run(tf.shape(conv4))) conv4 = conv_layer(conv4, [self.KERNEL_SIZE, self.KERNEL_SIZE, 512, 512], 1, 'g_wc4x2') print('Cv4') print(sess.run(tf.shape(conv4))) conv4 = conv_layer(conv4, [self.KERNEL_SIZE, self.KERNEL_SIZE, 512, 512], 1, 'g_wc4x3') print('Cv4') print(sess.run(tf.shape(conv4))) #Cv5 conv5 = conv_layer(conv4, [self.KERNEL_SIZE, self.KERNEL_SIZE, 512, 512], 1, 'g_wc5', False, False) conv5 = nn.relu(conv5) print('Cv5') print(sess.run(tf.shape(conv5))) #Cv6 conv6 = deconv2d( conv5, [self.BATCH, self.INPUT_SIZE, self.INPUT_SIZE, 512], self.KERNEL_SIZE, self.KERNEL_SIZE, 1, 1, "deConv6") conv6 = bn(conv6, self.IS_TRAINING, 'g_bn6') conv6 = nn.relu(conv6) print('Cv6') print(sess.run(tf.shape(conv6))) #Cv7 conv7 = tf.concat([conv6, conv4], 3) conv7 = deconv2d( conv7, [self.BATCH, self.INPUT_SIZE, self.INPUT_SIZE, 512], self.KERNEL_SIZE, self.KERNEL_SIZE, 1, 1, "deConv7x1") conv7 = bn(conv7, self.IS_TRAINING, 'g_bn7_x1') conv7 = nn.relu(conv7) print('Cv7') print(sess.run(tf.shape(conv7))) conv7 = tf.concat([conv7, conv4], 3) conv7 = deconv2d( conv7, [self.BATCH, self.INPUT_SIZE, self.INPUT_SIZE, 512], self.KERNEL_SIZE, self.KERNEL_SIZE, 1, 1, "deConv7x2") conv7 = bn(conv7, self.IS_TRAINING, 'g_bn7_x2') conv7 = nn.relu(conv7) print('Cv7') print(sess.run(tf.shape(conv7))) conv7 = tf.concat([conv7, conv4], 3) conv7 = deconv2d( conv7, [self.BATCH, self.INPUT_SIZE, self.INPUT_SIZE, 512], self.KERNEL_SIZE, self.KERNEL_SIZE, 1, 1, "deConv7x3") conv7 = bn(conv7, self.IS_TRAINING, 'g_bn7_x3') conv7 = nn.relu(conv7) print('Cv7') print(sess.run(tf.shape(conv7))) #Cv8 conv8 = tf.concat([conv7, conv3], 3) conv8 = deconv2d( conv8, [self.BATCH, self.INPUT_SIZE, self.INPUT_SIZE, 256], self.KERNEL_SIZE, self.KERNEL_SIZE, 1, 1, "deConv8") conv8 = bn(conv8, self.IS_TRAINING, 'g_bn8') conv8 = nn.relu(conv8) print('Cv8') print(sess.run(tf.shape(conv8))) #Cv9 conv9 = tf.concat([conv8, conv2], 3) conv9 = deconv2d( conv9, [self.BATCH, self.INPUT_SIZE, self.INPUT_SIZE, 128], self.KERNEL_SIZE, self.KERNEL_SIZE, 1, 1, "deConv9") conv9 = bn(conv9, self.IS_TRAINING, 'g_bn9') conv9 = nn.relu(conv9) print('Cv9') print(sess.run(tf.shape(conv9))) #Cv10 conv10 = tf.concat([conv9, conv1], 3) conv10 = deconv2d( conv10, [self.BATCH, self.INPUT_SIZE, self.INPUT_SIZE, 64], self.KERNEL_SIZE, self.KERNEL_SIZE, 1, 1, "deConv10") conv10 = bn(conv10, self.IS_TRAINING, 'g_bn10') conv10 = nn.relu(conv10) print('Cv10') print(sess.run(tf.shape(conv10))) #Cv11 conv11 = tf.concat([conv10, conv0], 3) conv11 = deconv2d( conv11, [self.BATCH, self.INPUT_SIZE, self.INPUT_SIZE, 1], self.KERNEL_SIZE, self.KERNEL_SIZE, 1, 1, "deConv11") print('Cv11') print(sess.run(tf.shape(conv11))) conv11 = nn.tanh(conv11) #conv11=nn.sigmoid(conv11) return conv11
# tf.Tensor( # [[-0.2 0. 1. ] # [ 0. 0. 1. ]], shape=(2, 3), dtype=float32) # tf.Tensor( # [[-0.2 0. 1. ] # [ 0. 0. 1. ]], shape=(2, 3), dtype=float32) print(_sigmoid(x)) print(nn.sigmoid(x)) # tf.Tensor( # [[0.26894143 0.5 0.73105854] # [0.5 0.5 0.73105854]], shape=(2, 3), dtype=float32) # tf.Tensor( # [[0.26894143 0.5 0.73105854] # [0.5 0.5 0.7310586 ]], shape=(2, 3), dtype=float32) print(_tanh(x)) print(nn.tanh(x)) # tf.Tensor( # [[-0.7615942 0. 0.7615941] # [ 0. 0. 0.7615941]], shape=(2, 3), dtype=float32) # tf.Tensor( # [[-0.7615942 0. 0.7615942] # [ 0. 0. 0.7615942]], shape=(2, 3), dtype=float32) print(_softmax(x)) print(nn.softmax(x)) # tf.Tensor( # [[0.09003059 0.24472848 0.66524094] # [0.21194156 0.21194156 0.57611686]], shape=(2, 3), dtype=float32) # tf.Tensor( # [[0.09003057 0.24472848 0.6652409 ] # [0.21194157 0.21194157 0.57611686]], shape=(2, 3), dtype=float32) print(_silu(x))