def __init(self, name, kernel_shape, bias_shape, padding, stride=1, activation_func=None): NetworkLayer.__init__(self, name) with tf.variable_scope(name): self.weights = tf.get_variable( 'w', kernel_shape, initializer=layers.xavier_initializer_conv2d(uniform=False)) self.bias = tf.get_variable( 'b', bias_shape, initializer=tf.random_normal_initializer) assert self.weights.name == name + '/w:0' and self.bias.name == name + '/b:0' tf.add_to_collection(self.weights.name, self.weights) tf.add_to_collection(self.bias.name, self.bias) self.__activation_func = activation_func self.__padding = padding self.__stride = stride
def encoder(self, x): shapes = [] n_hidden = [1] + self.n_hidden input = x for i, k_size in enumerate(self.kernel_size): w = tf.get_variable( 'enc_w{}'.format(i), shape=[k_size, k_size, n_hidden[i], n_hidden[i + 1]], initializer=layers.xavier_initializer_conv2d(), regularizer=self.reg) b = tf.get_variable('enc_b{}'.format(i), shape=[n_hidden[i + 1]], initializer=tf.zeros_initializer()) shapes.append(input.get_shape().as_list()) enc_i = tf.nn.conv2d(input, w, strides=[1, 2, 2, 1], padding='SAME') enc_i = tf.nn.bias_add(enc_i, b) enc_i = tf.nn.relu(enc_i) input = enc_i return input, shapes
def conv2d(input, filter_shape, strides=2, padding='SAME', name='conv'): with tf.variable_scope(name): w = tf.get_variable(name='w', shape=filter_shape, dtype=tf.float32, initializer=xavier_initializer_conv2d()) tf.summary.histogram('w_' + name, w) b = tf.get_variable(name='b', shape=[filter_shape[-1]], dtype=tf.float32, initializer=tf.zeros_initializer()) tf.summary.histogram('b_' + name, b) conv = tf.nn.conv2d(input=input, filter=w, strides=[1, strides, strides, 1], padding=padding, use_cudnn_on_gpu=True) return lrelu(conv + b, 0.02)
def __init__(self, label_size, network_architecture, activation=tf.nn.relu, max_grad_norm=1, learning_rate=0.001, batch_size=100, save_path=None, load_model=None): """ :param dict network_architecture: dictionary with following elements n_input: shape of input n_z: dimensionality of latent space :param activation: activation function (tensor flow function) :param float learning_rate: :param int batch_size: """ self.network_architecture = network_architecture self.max_grad_norm = max_grad_norm self.activation = activation self.learning_rate = learning_rate self.batch_size = batch_size self.label_size = label_size # Initializer if "relu" in self.activation.__name__: self.ini_c, self.ini = variance_scaling_initializer(), variance_scaling_initializer() else: self.ini_c, self.ini = xavier_initializer_conv2d(), xavier_initializer() # Create network self._create_network() # Summary tf.summary.scalar("loss", self.loss) # Launch the session self.sess = tf.Session(config=tf.ConfigProto(log_device_placement=False)) # Summary writer for tensor board self.summary = tf.summary.merge_all() if save_path: self.writer = tf.summary.FileWriter(save_path, self.sess.graph) # Load model if load_model: tf.reset_default_graph() self.saver.restore(self.sess, load_model)
def gen_nlayer_conv_decoder(X, decoder_para): ''' generate a n-layer convolutional encoder @encoder_para: dict, parameters of method, including: @@kernal_size: list, shape=[n:4], [[height, width, output_channel, input_channel], ... ]; notice that elements in kernal_size has the same order with that in encoder. That is to say input_channel of encoder is the output_channel of decoder. @@strides: list, shape=[n:4], [[batch_stride, height_stride, width_stride, channel_stride], ...] @@output_size: list, shape=[4:], [batch, height, width, channel] ''' kernal_size = decoder_para['kernal_size'] strides = decoder_para['strides'] output_size = decoder_para['output_size'] last_layer = X n_layers = len(kernal_size) for l_idx in xrange(n_layers): knl_w = tf.get_variable('dec_w_%d'%l_idx, shape=kernal_size[n_layers - l_idx - 1],\ initializer=layers.xavier_initializer_conv2d()) knl_b = tf.Variable(tf.zeros([kernal_size[n_layers - l_idx - 1][-2]])) stride = strides[n_layers - l_idx - 1] #calculate the output shape last_ly_shape = last_layer.get_shape().as_list() out_height = last_ly_shape[1] * stride[1] if l_idx!=n_layers-1 else output_size[1] out_width = last_ly_shape[2] * stride[2] if l_idx!=n_layers-1 else output_size[2] out_channel = kernal_size[n_layers - l_idx - 1][-2] if l_idx!=n_layers-1 else output_size[3] #generate last layer last_layer = tf.nn.bias_add(tf.nn.conv2d_transpose(last_layer, knl_w, \ tf.stack([output_size[0], out_height, out_width, out_channel]), strides=stride, padding='SAME'),\ knl_b) last_layer = tf.nn.relu(last_layer) return last_layer
def gen_conv_encoder(encoder_para): ''' generate a one layer convolutional encoder @encoder_para: dict, parameters of method, including: @@img_size: np.ndarray, shape=[4:], the size of input image @@kernal_size: np.ndarray, shape=[4:], [width, height, input_channel, output_channel] @@strides: np.ndarray, shape=[4:], [batch_stride, width_stride, height_stride, channel_stride] ''' img_size = encoder_para['img_size'] kernal_size = encoder_para['kernal_size'] strides = encoder_para['strides'] # generate X and weight X = tf.placeholder(tf.float32, img_size) #batch_size, img_width, img_height, channel enc_knl = tf.get_variable("enc_knl", shape=kernal_size, \ initializer=layers.xavier_initializer_conv2d()) enc_b = tf.Variable(tf.zeros([kernal_size[-1]], dtype=tf.float32)) #generate conv layer conv_layer = tf.nn.conv2d(X, enc_knl, strides, padding='SAME') conv_layer = tf.nn.relu(tf.nn.bias_add(conv_layer, enc_b)) return conv_layer, X
def conv2d(input_node, channels_n, level, is_training, stride=1, activation=True): input_shape = input_node.get_shape().as_list() assert len(input_shape) == 4, 'Tensor with rank 4 is expected.' in_channels_n = input_shape[3] with tf.variable_scope(f'conv_{level}'): Wconv1 = tf.get_variable(f"W_{level}", shape=[3, 3, in_channels_n, channels_n], initializer=xavier_initializer_conv2d()) bconv1 = tf.get_variable(f"b_{level}", shape=[channels_n], initializer=tf.zeros_initializer) conv = tf.nn.conv2d(input_node, Wconv1, strides=[1, stride, stride, 1], padding='SAME') + bconv1 batch_norm = tf.layers.batch_normalization(conv, training=is_training) output = tf.nn.relu(batch_norm) if activation else batch_norm return output
def conv2d(self, inputs, filters=16, kernel_size=3, strides=1, padding='VALID', activation=None, kernel_initializer=layers.xavier_initializer_conv2d(), bias_initializer=tf.constant_initializer(0.2), kernel_regularizer=layers.l2_regularizer(2e-4), use_bias=True, name="conv"): return tf.layers.conv2d(inputs, filters=filters, kernel_size=kernel_size, strides=strides, padding=padding, data_format="channels_first", activation=activation, kernel_initializer=kernel_initializer, kernel_regularizer=kernel_regularizer, bias_initializer=bias_initializer, use_bias=use_bias, name=name)
def __init_model__(self, input_shape): self.states = tf.placeholder(tf.float32, [None] + list(input_shape), name='states') self.actions = tf.placeholder(tf.float32, [None, 1], name='actions') self.weights = tf.placeholder(tf.float32, [None, 1], name='weights') self.learning_rate = tf.placeholder(tf.float32, name='learning_rate') self.beta = tf.placeholder(tf.float32, name='beta') conv1 = tf.layers.conv2d( inputs=self.states, filters=64, kernel_size=8, strides=4, data_format='channels_first', activation=tf.nn.tanh, kernel_initializer=xavier_initializer_conv2d()) h1 = tf.layers.dense(inputs=tf.layers.flatten(conv1), units=128, activation=tf.nn.tanh, kernel_initializer=xavier_initializer()) self.prob_1 = tf.layers.dense(inputs=h1, units=1, activation=tf.sigmoid, kernel_initializer=xavier_initializer()) self.prob_1 = self.prob_1 * 0.9998 + 0.0001 prob_0 = 1 - self.prob_1 entropy = self.prob_1 * tf.log(1 / self.prob_1) + prob_0 * tf.log( 1 / prob_0) regularizer = tf.reduce_mean(entropy) loss = tf.losses.log_loss(labels=self.actions, predictions=self.prob_1, weights=self.weights) optimizer = tf.train.AdamOptimizer(learning_rate=self.learning_rate) self.train_op = optimizer.minimize(loss - self.beta * regularizer)
def construct_conv_weights(self): weights = {} dtype = tf.float32 conv_initializer = contrib_layers.xavier_initializer_conv2d(dtype=dtype) fc_initializer = contrib_layers.xavier_initializer(dtype=dtype) k = 3 weights['conv1'] = tf.get_variable('conv1', [k, k, self.channels, self.dim_hidden], initializer=conv_initializer, dtype=dtype) weights['b1'] = tf.Variable(tf.zeros([self.dim_hidden])) weights['conv2'] = tf.get_variable('conv2', [k, k, self.dim_hidden, self.dim_hidden], initializer=conv_initializer, dtype=dtype) weights['b2'] = tf.Variable(tf.zeros([self.dim_hidden])) weights['conv3'] = tf.get_variable('conv3', [k, k, self.dim_hidden, self.dim_hidden], initializer=conv_initializer, dtype=dtype) weights['b3'] = tf.Variable(tf.zeros([self.dim_hidden])) weights['conv4'] = tf.get_variable('conv4', [k, k, self.dim_hidden, self.dim_hidden], initializer=conv_initializer, dtype=dtype) weights['b4'] = tf.Variable(tf.zeros([self.dim_hidden])) if FLAGS.datasource == 'miniimagenet' or FLAGS.datasource == 'dclaw': # assumes max pooling weights['w5'] = tf.get_variable('w5', [self.dim_hidden*5*5, self.dim_output], initializer=fc_initializer) weights['b5'] = tf.Variable(tf.zeros([self.dim_output]), name='b5') else: weights['w5'] = tf.Variable(tf.random_normal([self.dim_hidden, self.dim_output]), name='w5') weights['b5'] = tf.Variable(tf.zeros([self.dim_output]), name='b5') return weights
def encoder(self, imgs, activation, is_training, batch_size, img_shape, channels): with tf.variable_scope('encoder', reuse=tf.AUTO_REUSE, initializer=xavier_initializer_conv2d(), regularizer=l2_regularizer(0.01)): # stn >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> n_fc = 6 initial = np.array([[1., 0, 0], [0, 1., 0]]) initial = initial.astype('float32').flatten() W_fc1 = tf.Variable( tf.zeros(shape=[img_shape * img_shape * channels, n_fc]), name='W_fc1', validate_shape=False) b_fc1 = tf.Variable(initial_value=initial, name='b_fc1') h_fc1 = tf.matmul( tf.zeros([batch_size, img_shape * img_shape * channels]), W_fc1) + b_fc1 h_trans = transformer(imgs, h_fc1) # stn <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< e_conv = self.coord_conv(h_trans, 48, 3, padding='same', activation=None) e_conv = tf.layers.batch_normalization(e_conv, training=is_training, fused=True) e_conv = activation(e_conv) e_conv = tf.layers.max_pooling2d(e_conv, 2, 2) e_conv = self.coord_conv(e_conv, 92, 3, padding='same', activation=None) e_conv = tf.layers.batch_normalization(e_conv, training=is_training, fused=True) e_conv = activation(e_conv) e_conv = tf.layers.max_pooling2d(e_conv, 2, 2) e_conv = self.coord_conv(e_conv, 256, 3, padding='same', activation=None) e_conv = tf.layers.batch_normalization(e_conv, training=is_training, fused=True) e_conv = activation(e_conv) e_conv = tf.layers.max_pooling2d(e_conv, 2, 2) e_conv = self.coord_conv(e_conv, 256, 3, padding='same', activation=None) e_conv = tf.layers.batch_normalization(e_conv, training=is_training, fused=True) e_conv = activation(e_conv) e_conv = tf.layers.max_pooling2d(e_conv, 2, 2) e_conv = self.coord_conv(e_conv, 256, 3, padding='same', activation=None) e_conv = tf.layers.batch_normalization(e_conv, training=is_training, fused=True) e_conv = activation(e_conv) e_conv = tf.layers.max_pooling2d(e_conv, 2, 2) e_conv = self.coord_conv(e_conv, 256, 3, padding='same', activation=None) e_conv = tf.layers.batch_normalization(e_conv, training=is_training, fused=True) e_conv = activation(e_conv) e_conv = tf.layers.max_pooling2d(e_conv, 2, 2) lv = tf.layers.flatten(e_conv) return lv
def __init__(self, params, pretrained_model_path=""): self.params = params self.pretrained_model_path = pretrained_model_path dicts = helper.load_dictionaries(self.params.dicts_file) self.word2id = dicts["word2id"] self.id2word = dicts["id2word"] self.char2id = dicts["char2id"] self.id2char = dicts["id2char"] self.tag2id = dicts["tag2id"] self.id2tag = dicts["id2tag"] self.pretrained_emb = np.zeros(shape=(len(self.word2id), self.params.word_dim)) if self.pretrained_model_path == "" and self.params.train != "" and self.params.pretrained_emb != "": self.pretrained_emb = helper.load_word_emb( self.word2id, self.pretrained_emb, self.params.pretrained_emb) # build model self.tf_word_ids = tf.placeholder(dtype=tf.int32, shape=[None, None], name="word_ids") self.tf_sentence_lengths = tf.placeholder(dtype=tf.int32, shape=[None], name="sentence_lengths") self.tf_labels = tf.placeholder(dtype=tf.int32, shape=[None, None], name="labels") self.tf_dropout = tf.placeholder(dtype=tf.float32, shape=[], name="drop_out") self.tf_learning_rate = tf.placeholder(dtype=tf.float32, shape=[], name="learning_rate") self.tf_char_ids = tf.placeholder(dtype=tf.int32, shape=[None, None, None], name="char_ids") self.tf_word_lengths = tf.placeholder(dtype=tf.int32, shape=[None, None], name="word_lengths") self.tf_raw_word = tf.placeholder(dtype=tf.string, shape=[None, None], name="raw_word") with tf.variable_scope("word_embedding"): tf_word_embeddings = tf.Variable(self.pretrained_emb, dtype=tf.float32, trainable=True, name="word_embedding") self.input = tf.nn.embedding_lookup(tf_word_embeddings, self.tf_word_ids, name="embedded_words") with tf.variable_scope("char_cnn"): tf_char_embeddings = tf.get_variable( name="char_embeddings", dtype=tf.float32, shape=[len(self.char2id), self.params.char_dim], trainable=True, initializer=xavier_initializer()) embedded_cnn_chars = tf.nn.embedding_lookup( tf_char_embeddings, self.tf_char_ids, name="embedded_cnn_chars") conv1 = tf.layers.conv2d( inputs=embedded_cnn_chars, filters=self.params.nb_filters_1, kernel_size=(1, 3), strides=(1, 1), padding="same", name="conv1", kernel_initializer=xavier_initializer_conv2d()) conv2 = tf.layers.conv2d( inputs=conv1, filters=self.params.nb_filters_2, kernel_size=(1, 3), strides=(1, 1), padding="same", name="conv2", kernel_initializer=xavier_initializer_conv2d()) char_cnn = tf.reduce_max(conv2, axis=2) self.input = tf.concat([self.input, char_cnn], axis=-1) with tf.variable_scope("elmo_emb"): elmo = hub.Module("/elmo2", trainable=False) embeddings = \ elmo(inputs={"tokens": self.tf_raw_word, "sequence_len": self.tf_sentence_lengths}, signature="tokens", as_dict=True)["elmo"] # num_sent, max_sent_len, 1024 elmo_emb = tf.layers.dense(inputs=embeddings, units=self.params.elmo_dim, activation=None) self.input = tf.concat([self.input, elmo_emb], axis=-1) self.input = tf.nn.dropout(self.input, self.tf_dropout) with tf.variable_scope("bi_lstm_words"): cell_fw = tf.contrib.rnn.LSTMCell(self.params.word_hidden_size) cell_bw = tf.contrib.rnn.LSTMCell(self.params.word_hidden_size) (output_fw, output_bw), _ = tf.nn.bidirectional_dynamic_rnn( cell_fw, cell_bw, self.input, sequence_length=self.tf_sentence_lengths, dtype=tf.float32) self.output = tf.concat([output_fw, output_bw], axis=-1) ntime_steps = tf.shape(self.output)[1] self.output = tf.reshape(self.output, [-1, 2 * params.word_hidden_size]) layer1 = tf.nn.dropout( tf.layers.dense(inputs=self.output, units=params.word_hidden_size, activation=None, kernel_initializer=xavier_initializer()), self.tf_dropout) pred = tf.layers.dense(inputs=layer1, units=len(self.tag2id), activation=None, kernel_initializer=xavier_initializer()) self.logits = tf.reshape(pred, [-1, ntime_steps, len(self.tag2id)]) # compute loss value using crf log_likelihood, self.transition_params = tf.contrib.crf.crf_log_likelihood( self.logits, self.tf_labels, self.tf_sentence_lengths) with tf.variable_scope("loss_and_opt"): self.tf_loss = tf.reduce_mean(-log_likelihood) optimizer = tf.train.AdamOptimizer( learning_rate=self.tf_learning_rate) self.tf_train_op = optimizer.minimize(self.tf_loss)
def __init__(self, params=None): self.params = params self.keep_prob = params["keep_prob"] self.nb_epochs = params["nb_epochs"] # load word and char dictionaries dicts = pickle.load(open(params["dicts_file"], "rb")) self.w2i = dicts["w2i"] self.i2w = dicts["i2w"] self.c2i = dicts["c2i"] self.i2c = dicts["i2c"] self.word_dim = params["word_dim"] self.word_vocab_size = len(self.w2i) self.char_dim = params["char_dim"] self.char_vocab_size = len(self.c2i) print("Sizes of word and char dictionaries: {}, {}".format( self.word_vocab_size, self.char_vocab_size)) self.word_emb = np.zeros(shape=(self.word_vocab_size, self.word_dim)) # load word embedding if "word_emb" in params: print("pre-trained word embedding {} is being loaded ...".format( params["word_emb"])) self.load_word_emb(params["word_emb"]) tf.reset_default_graph() # [sent, word] self.tf_word_ids = tf.placeholder(dtype=tf.int32, shape=[None, None], name="word_ids") # real length sents self.tf_sentence_lengths = tf.placeholder(dtype=tf.int32, shape=[None], name="sentence_lengths") # [sent, word, char] self.tf_char_ids = tf.placeholder(dtype=tf.int32, shape=[None, None, None], name="char_ids") # binary matrix representing the relationship between sents: [sent, sent] self.tf_target_matrix = tf.placeholder(dtype=tf.int32, shape=[None], name="target_matrix") # keep_prob self.tf_keep_prob = tf.placeholder(dtype=tf.float32, shape=[], name="keep_prob") # learning rate self.tf_learning_rate = tf.placeholder(dtype=tf.float32, shape=[], name="learning_rate") # load word embedding with tf.variable_scope("word_embedding"): tf_word_embeddings = tf.Variable(self.word_emb, dtype=tf.float32, trainable=True, name="word_embedding") embedded_words = tf.nn.embedding_lookup(tf_word_embeddings, self.tf_word_ids, name="embedded_words") self.input = embedded_words # sent, word, word_dim # CNN network to capture character-level features with tf.variable_scope("char_cnn"): tf_char_embeddings = tf.get_variable( name="char_embeddings", dtype=tf.float32, shape=[self.char_vocab_size, self.char_dim], trainable=True, initializer=xavier_initializer()) conv = tf.nn.embedding_lookup(tf_char_embeddings, self.tf_char_ids, name="embedded_cnn_chars") for i, (ks, fil) in enumerate(self.params["conv"]): conv = tf.layers.conv2d( inputs=conv, # sent, word, char, feature filters=fil, kernel_size=(1, ks), strides=(1, 1), padding="same", name="conv_{}".format(i), kernel_initializer=xavier_initializer_conv2d()) self.char_cnn = tf.reduce_max(conv, axis=2) # sent, word, cnn_feature self.input = tf.nn.dropout( tf.concat([self.input, self.char_cnn], axis=-1), self.tf_keep_prob) # [sents, words, word_dim + cnn_features] # Bi-LSTM to generate final input representation in combination with both left and right contexts with tf.variable_scope("bi_lstm_words"): cell_fw = tf.contrib.rnn.LSTMCell(self.params["word_lstm_units"]) cell_bw = tf.contrib.rnn.LSTMCell(self.params["word_lstm_units"]) (output_fw, output_bw), _ = tf.nn.bidirectional_dynamic_rnn( cell_fw, cell_bw, self.input, sequence_length=self.tf_sentence_lengths, dtype=tf.float32) bilstm_output = tf.concat([output_fw, output_bw], axis=-1) # [sents, words, 2*lstm_units] mask = tf.where( condition=tf.equal(self.tf_word_ids, self.w2i["<PAD>"]), x=-1e10 * tf.ones_like(self.tf_word_ids, dtype=tf.float32), y=tf.zeros_like(self.tf_word_ids, dtype=tf.float32)) mask = tf.tile(tf.expand_dims(mask, -1), (1, 1, 2 * self.params["word_lstm_units"])) bilstm_output = bilstm_output + mask bilstm_output = tf.reduce_max(bilstm_output, axis=1) # [sents, 2*word_lstm_units] with tf.variable_scope("att"): c = bilstm_output[:-1, :] q = bilstm_output[-1:, :] de = c.get_shape()[-1] w = tf.get_variable(dtype=tf.float32, shape=[1, de], trainable=True, name="w") w1 = tf.get_variable(dtype=tf.float32, shape=[de, de], trainable=True, name="w1") w2 = tf.get_variable(dtype=tf.float32, shape=[de, de], trainable=True, name="w2") b1 = tf.get_variable(dtype=tf.float32, shape=[de, 1], trainable=True, name="b1") b = tf.get_variable(dtype=tf.float32, shape=[1], trainable=True, name="b") w1cT = tf.matmul(w1, tf.transpose(c)) w2q = tf.matmul(w2, tf.transpose(q)) s = tf.add(tf.add(w1cT, w2q), b1) f = tf.squeeze(tf.matmul(w, tf.tanh(s)) + b, axis=0) e = tf.nn.softmax(f) h = c * e[:, None] h = tf.add(h, tf.squeeze(q, axis=0)) with tf.variable_scope("loss_and_opt"): self.logits = tf.nn.dropout( tf.layers.dense(inputs=h, units=2, activation=None), self.tf_keep_prob) self.loss = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits_v2( labels=tf.one_hot(self.tf_target_matrix, 2), logits=self.logits, name="loss_function")) self.pred = tf.argmax(self.logits, axis=-1) eq = tf.cast( tf.equal(tf.cast(self.pred, tf.int32), self.tf_target_matrix), tf.float32) self.acc = tf.reduce_mean(eq) self.opt = tf.train.AdamOptimizer( learning_rate=self.tf_learning_rate).minimize(self.loss)
def _initialize_weights(self): all_weights = dict() all_weights['enc_w0'] = tf.get_variable( "enc_w0", shape=[ self.kernel_size[0], self.kernel_size[0], 1, self.n_hidden[0] ], initializer=layers.xavier_initializer_conv2d(), regularizer=self.reg) all_weights['enc_b0'] = tf.Variable( tf.zeros([self.n_hidden[0]], dtype=tf.float32)) all_weights['enc_w1'] = tf.get_variable( "enc_w1", shape=[ self.kernel_size[1], self.kernel_size[1], self.n_hidden[0], self.n_hidden[1] ], initializer=layers.xavier_initializer_conv2d(), regularizer=self.reg) all_weights['enc_b1'] = tf.Variable( tf.zeros([self.n_hidden[1]], dtype=tf.float32)) all_weights['enc_w2'] = tf.get_variable( "enc_w2", shape=[ self.kernel_size[2], self.kernel_size[2], self.n_hidden[1], self.n_hidden[2] ], initializer=layers.xavier_initializer_conv2d(), regularizer=self.reg) all_weights['enc_b2'] = tf.Variable( tf.zeros([self.n_hidden[2]], dtype=tf.float32)) all_weights['dec_w0'] = tf.get_variable( "dec_w0", shape=[ self.kernel_size[2], self.kernel_size[2], self.n_hidden[1], self.n_hidden[2] ], initializer=layers.xavier_initializer_conv2d(), regularizer=self.reg) all_weights['dec_b0'] = tf.Variable( tf.zeros([self.n_hidden[1]], dtype=tf.float32)) all_weights['dec_w1'] = tf.get_variable( "dec_w1", shape=[ self.kernel_size[1], self.kernel_size[1], self.n_hidden[0], self.n_hidden[1] ], initializer=layers.xavier_initializer_conv2d(), regularizer=self.reg) all_weights['dec_b1'] = tf.Variable( tf.zeros([self.n_hidden[0]], dtype=tf.float32)) all_weights['dec_w2'] = tf.get_variable( "dec_w2", shape=[ self.kernel_size[0], self.kernel_size[0], 1, self.n_hidden[0] ], initializer=layers.xavier_initializer_conv2d(), regularizer=self.reg) all_weights['dec_b2'] = tf.Variable(tf.zeros([1], dtype=tf.float32)) return all_weights
def conv(X, in_ch, out_ch, name): with tf.variable_scope(name) as scope: W_conv = tf.get_variable(name='weights', shape=[3, 3, in_ch, out_ch], initializer=xavier_initializer_conv2d()) h_bn = tf.layers.batch_normalization(tf.nn.conv2d(X, W_conv, strides=[1, 1, 1, 1], padding='SAME'), training =trainphase) h_conv = my_relu(h_bn) return h_conv
def _initialize_weights(self): all_weights = dict() all_weights['enc_w0'] = tf.get_variable( "enc_w0", shape=[ self.kernel_size[0], self.kernel_size[0], 1, self.n_hidden[0] ], initializer=layers.xavier_initializer_conv2d(), regularizer=self.reg) all_weights['enc_b0'] = tf.Variable( tf.zeros([self.n_hidden[0]], dtype=tf.float32)) all_weights['enc_w1'] = tf.get_variable( "enc_w1", shape=[ self.kernel_size[1], self.kernel_size[1], self.n_hidden[0], self.n_hidden[1] ], initializer=layers.xavier_initializer_conv2d(), regularizer=self.reg) all_weights['enc_b1'] = tf.Variable( tf.zeros([self.n_hidden[1]], dtype=tf.float32)) all_weights['enc_w2'] = tf.get_variable( "enc_w2", shape=[ self.kernel_size[2], self.kernel_size[2], self.n_hidden[1], self.n_hidden[2] ], initializer=layers.xavier_initializer_conv2d(), regularizer=self.reg) all_weights['enc_b2'] = tf.Variable( tf.zeros([self.n_hidden[2]], dtype=tf.float32)) all_weights['Coef'] = tf.Variable( 1.0e-4 * tf.ones([self.batch_size, self.batch_size], tf.float32), name='Coef') all_weights['dec_w0'] = tf.get_variable( "dec_w0", shape=[ self.kernel_size[2], self.kernel_size[2], self.n_hidden[1], self.n_hidden[2] ], initializer=layers.xavier_initializer_conv2d(), regularizer=self.reg) all_weights['dec_b0'] = tf.Variable( tf.zeros([self.n_hidden[1]], dtype=tf.float32)) all_weights['dec_w1'] = tf.get_variable( "dec_w1", shape=[ self.kernel_size[1], self.kernel_size[1], self.n_hidden[0], self.n_hidden[1] ], initializer=layers.xavier_initializer_conv2d(), regularizer=self.reg) all_weights['dec_b1'] = tf.Variable( tf.zeros([self.n_hidden[0]], dtype=tf.float32)) all_weights['dec_w2'] = tf.get_variable( "dec_w2", shape=[ self.kernel_size[0], self.kernel_size[0], 1, self.n_hidden[0] ], initializer=layers.xavier_initializer_conv2d(), regularizer=self.reg) all_weights['dec_b2'] = tf.Variable(tf.zeros([1], dtype=tf.float32)) all_weights['oenc_w0'] = tf.get_variable( "oenc_w0", shape=[ self.kernel_size[0], self.kernel_size[0], self.n_hidden[0], 1 ], initializer=layers.xavier_initializer_conv2d(), regularizer=self.reg) all_weights['oenc_b0'] = tf.Variable(tf.zeros([1], dtype=tf.float32)) all_weights['oenc_w1'] = tf.get_variable( "oenc_w1", shape=[ self.kernel_size[1], self.kernel_size[1], self.n_hidden[2], self.n_hidden[0] ], initializer=layers.xavier_initializer_conv2d(), regularizer=self.reg) all_weights['oenc_b1'] = tf.Variable( tf.zeros([self.n_hidden[2]], dtype=tf.float32)) all_weights['oenc_w2'] = tf.get_variable( "oenc_w2", shape=[ self.kernel_size[2], self.kernel_size[2], self.n_hidden[2], self.n_hidden[1] ], initializer=layers.xavier_initializer_conv2d(), regularizer=self.reg) all_weights['oenc_b2'] = tf.Variable( tf.zeros([self.n_hidden[2]], dtype=tf.float32)) all_weights['oCoef'] = tf.Variable( 1.0e-4 * tf.ones([self.batch_size, self.batch_size], tf.float32), name='Coef') all_weights['odec_w0'] = tf.get_variable( "odec_w0", shape=[ self.kernel_size[2], self.kernel_size[2], self.n_hidden[1], self.n_hidden[2] ], initializer=layers.xavier_initializer_conv2d(), regularizer=self.reg) all_weights['odec_b0'] = tf.Variable( tf.zeros([self.n_hidden[2]], dtype=tf.float32)) all_weights['odec_w1'] = tf.get_variable( "odec_w1", shape=[ self.kernel_size[1], self.kernel_size[1], self.n_hidden[0], self.n_hidden[2] ], initializer=layers.xavier_initializer_conv2d(), regularizer=self.reg) all_weights['odec_b1'] = tf.Variable( tf.zeros([self.n_hidden[2]], dtype=tf.float32)) all_weights['odec_w2'] = tf.get_variable( "odec_w2", shape=[self.kernel_size[0], self.kernel_size[0], 3, 3], initializer=layers.xavier_initializer_conv2d(), regularizer=self.reg) all_weights['odec_b2'] = tf.Variable(tf.zeros([3], dtype=tf.float32)) all_weights['odec_w3'] = tf.get_variable( "odec_w3", shape=[self.kernel_size[0], self.kernel_size[0], 3, 3], initializer=layers.xavier_initializer_conv2d(), regularizer=self.reg) all_weights['odec_b3'] = tf.Variable(tf.zeros([3], dtype=tf.float32)) return all_weights
def __conv_relu(self, input, kernel_shape, bias_shape, name): weights = tf.get_variable("W"+name, kernel_shape, initializer=xavier_initializer_conv2d()) biases = tf.get_variable("b"+name, bias_shape, initializer=tf.constant_initializer(0.0)) conv = tf.nn.conv2d(input, weights, strides=[1, 1, 1, 1], padding='SAME', name=name+"_conv") return tf.nn.relu(conv + biases)
def siamese_net(input, reuse=False, is_training=True): # without batch norm with tf.name_scope("model"): with tf.variable_scope("conv1_1") as scope: # ,normalizer_fn=slim.batch_norm,normalizer_params={'is_training':is_training} # tf.truncated_normal_initializer(stddev=0.001) conv1_1 = layers.conv2d( input, 48, [3, 3], activation_fn=tf.nn.relu, weights_initializer=layers.xavier_initializer_conv2d(), weights_regularizer=slim.l2_regularizer(0.0001), scope=scope, reuse=reuse) with tf.variable_scope("conv1_2") as scope: conv1_2 = layers.conv2d( conv1_1, 48, [3, 3], activation_fn=tf.nn.relu, weights_initializer=layers.xavier_initializer_conv2d(), weights_regularizer=slim.l2_regularizer(0.0001), scope=scope, reuse=reuse) with tf.variable_scope("conv2_1") as scope: conv2_1 = layers.avg_pool2d(conv1_2, [2, 2]) with tf.variable_scope("conv2_2") as scope: conv2_2 = layers.conv2d( conv2_1, 96, [3, 3], activation_fn=tf.nn.relu, weights_initializer=layers.xavier_initializer_conv2d(), weights_regularizer=slim.l2_regularizer(0.0001), scope=scope, reuse=reuse) with tf.variable_scope("conv2_3") as scope: conv2_3 = layers.conv2d( conv2_2, 96, [3, 3], activation_fn=tf.nn.relu, weights_initializer=layers.xavier_initializer_conv2d(), weights_regularizer=slim.l2_regularizer(0.0001), scope=scope, reuse=reuse) with tf.variable_scope("conv3_1") as scope: conv3_1 = layers.avg_pool2d(conv2_3, [2, 2]) with tf.variable_scope("conv3_2") as scope: conv3_2 = layers.conv2d( conv3_1, 192, [3, 3], activation_fn=tf.nn.relu, weights_initializer=layers.xavier_initializer_conv2d(), weights_regularizer=slim.l2_regularizer(0.0001), scope=scope, reuse=reuse) with tf.variable_scope("conv3_3") as scope: conv3_3 = layers.conv2d( conv3_2, 192, [3, 3], activation_fn=tf.nn.relu, weights_initializer=layers.xavier_initializer_conv2d(), weights_regularizer=slim.l2_regularizer(0.0001), scope=scope, reuse=reuse) with tf.variable_scope("conv4_1") as scope: conv4_1 = layers.avg_pool2d(conv3_3, [2, 2]) with tf.variable_scope("conv4_2") as scope: conv4_2 = layers.conv2d( conv4_1, 384, [3, 3], activation_fn=tf.nn.relu, weights_initializer=layers.xavier_initializer_conv2d(), weights_regularizer=slim.l2_regularizer(0.0001), scope=scope, reuse=reuse) with tf.variable_scope("conv4_3") as scope: conv4_3 = layers.conv2d( conv4_2, 384, [3, 3], activation_fn=tf.nn.relu, weights_initializer=layers.xavier_initializer_conv2d(), weights_regularizer=slim.l2_regularizer(0.0001), scope=scope, reuse=reuse) with tf.variable_scope("conv43") as scope: conv43 = layers.conv2d_transpose( conv4_3, 192, [2, 2], stride=2, activation_fn=None, weights_initializer=layers.xavier_initializer_conv2d(), weights_regularizer=slim.l2_regularizer(0.0001), scope=scope, reuse=reuse) with tf.variable_scope("conv3_4") as scope: conv3_4 = tf.concat([conv43, conv3_3], 3) with tf.variable_scope("conv3_5") as scope: conv3_5 = layers.conv2d( conv3_4, 192, [3, 3], activation_fn=tf.nn.relu, weights_initializer=layers.xavier_initializer_conv2d(), weights_regularizer=slim.l2_regularizer(0.0001), scope=scope, reuse=reuse) with tf.variable_scope("conv3_6") as scope: conv3_6 = layers.conv2d( conv3_5, 192, [3, 3], activation_fn=tf.nn.relu, weights_initializer=layers.xavier_initializer_conv2d(), weights_regularizer=slim.l2_regularizer(0.0001), scope=scope, reuse=reuse) with tf.variable_scope("conv32") as scope: conv32 = layers.conv2d_transpose( conv3_6, 64, [2, 2], stride=2, activation_fn=None, weights_initializer=layers.xavier_initializer_conv2d(), weights_regularizer=slim.l2_regularizer(0.0001), scope=scope, reuse=reuse) with tf.variable_scope("conv2_4") as scope: conv2_4 = tf.concat([conv32, conv2_3], 3) with tf.variable_scope("conv2_5") as scope: conv2_5 = layers.conv2d( conv2_4, 96, [3, 3], activation_fn=tf.nn.relu, weights_initializer=layers.xavier_initializer_conv2d(), weights_regularizer=slim.l2_regularizer(0.0001), scope=scope, reuse=reuse) with tf.variable_scope("conv2_6") as scope: conv2_6 = layers.conv2d( conv2_5, 96, [3, 3], activation_fn=tf.nn.relu, weights_initializer=layers.xavier_initializer_conv2d(), weights_regularizer=slim.l2_regularizer(0.0001), scope=scope, reuse=reuse) with tf.variable_scope("conv21") as scope: conv21 = layers.conv2d_transpose( conv2_6, 48, [2, 2], stride=2, activation_fn=None, weights_initializer=layers.xavier_initializer_conv2d(), weights_regularizer=slim.l2_regularizer(0.0001), scope=scope, reuse=reuse) with tf.variable_scope("conv1_3") as scope: conv1_3 = tf.concat([conv21, conv1_2], 3) with tf.variable_scope("conv1_4") as scope: conv1_4 = layers.conv2d( conv1_3, 48, [3, 3], activation_fn=tf.nn.relu, weights_initializer=layers.xavier_initializer_conv2d(), weights_regularizer=slim.l2_regularizer(0.0001), scope=scope, reuse=reuse) with tf.variable_scope("conv1_5") as scope: conv1_5 = layers.conv2d( conv1_4, 24, [3, 3], activation_fn=tf.nn.relu, weights_initializer=layers.xavier_initializer_conv2d(), weights_regularizer=slim.l2_regularizer(0.0001), scope=scope, reuse=reuse) with tf.variable_scope("conv1_6") as scope: conv1_6 = layers.conv2d( conv1_5, 1, [3, 3], activation_fn=None, weights_initializer=layers.xavier_initializer_conv2d(), weights_regularizer=slim.l2_regularizer(0.0001), scope=scope, reuse=reuse) return conv1_6
def mru_conv_block_v3(inp, ht, filter_depth, sn, stride, dilate=1, activation_fn=tf.nn.relu, normalizer_fn=None, normalizer_params=None, weights_initializer=ly.xavier_initializer_conv2d(), biases_initializer_mask=tf.constant_initializer( value=0.5), biases_initializer_h=tf.constant_initializer(value=-1), data_format='NCHW', weight_decay_rate=1e-8, norm_mask=False, norm_input=True, deconv=False): def norm_activ(tensor_in): if normalizer_fn is not None: _normalizer_params = normalizer_params or {} tensor_normed = normalizer_fn(tensor_in, **_normalizer_params) else: tensor_normed = tf.identity(tensor_in) if activation_fn is not None: tensor_normed = activation_fn(tensor_normed) return tensor_normed channel_index = 1 if data_format == 'NCHW' else 3 reduce_dim = [2, 3] if data_format == 'NCHW' else [1, 2] hidden_depth = ht.get_shape().as_list()[channel_index] regularizer = ly.l2_regularizer( weight_decay_rate) if weight_decay_rate > 0 else None weights_initializer_mask = weights_initializer biases_initializer = tf.zeros_initializer() if norm_mask: mask_normalizer_fn = normalizer_fn mask_normalizer_params = normalizer_params else: mask_normalizer_fn = None mask_normalizer_params = None if deconv: if stride == 2: ht = upsample(ht, data_format=data_format) elif stride != 1: raise NotImplementedError ht_orig = tf.identity(ht) # Normalize hidden state with tf.variable_scope('norm_activation_in') as sc: if norm_input: full_inp = tf.concat([norm_activ(ht), inp], axis=channel_index) else: full_inp = tf.concat([ht, inp], axis=channel_index) # update gate rg = conv2d2(full_inp, hidden_depth, 3, sn=sn, stride=1, rate=dilate, data_format=data_format, activation_fn=lrelu, normalizer_fn=mask_normalizer_fn, normalizer_params=mask_normalizer_params, weights_regularizer=regularizer, weights_initializer=weights_initializer_mask, biases_initializer=biases_initializer_mask, scope='update_gate') rg = (rg - tf.reduce_min(rg, axis=reduce_dim, keep_dims=True)) / ( tf.reduce_max(rg, axis=reduce_dim, keep_dims=True) - tf.reduce_min(rg, axis=reduce_dim, keep_dims=True)) # Input Image conv img_new = conv2d2(inp, hidden_depth, 3, sn=sn, stride=1, rate=dilate, data_format=data_format, activation_fn=None, normalizer_fn=None, normalizer_params=None, biases_initializer=biases_initializer, weights_regularizer=regularizer, weights_initializer=weights_initializer) ht_plus = ht + rg * img_new with tf.variable_scope('norm_activation_merge_1') as sc: ht_new_in = norm_activ(ht_plus) # new hidden state h_new = conv2d2(ht_new_in, filter_depth, 3, sn=sn, stride=1, rate=dilate, data_format=data_format, activation_fn=activation_fn, normalizer_fn=normalizer_fn, normalizer_params=normalizer_params, biases_initializer=biases_initializer, weights_regularizer=regularizer, weights_initializer=weights_initializer) h_new = conv2d2(h_new, filter_depth, 3, sn=sn, stride=1, rate=dilate, data_format=data_format, activation_fn=None, normalizer_fn=None, normalizer_params=None, biases_initializer=biases_initializer, weights_regularizer=regularizer, weights_initializer=weights_initializer) # new hidden state out # linear project for filter depth if ht.get_shape().as_list()[channel_index] != filter_depth: ht_orig = conv2d2(ht_orig, filter_depth, 1, sn=sn, stride=1, data_format=data_format, activation_fn=None, normalizer_fn=None, normalizer_params=None, biases_initializer=biases_initializer, weights_regularizer=regularizer, weights_initializer=weights_initializer) ht_new = ht_orig + h_new if not deconv: if stride == 2: ht_new = mean_pool(ht_new, data_format=data_format) elif stride != 1: raise NotImplementedError return ht_new
def model(x, n_classes, keep_prob): mu = 0 sigma = 0.1 depth = { 'D_1': 16, 'D_2': 32, 'D_3': 512, 'D_4': 8192, 'D_5': 256, } weights = { # 'W_conv1':tf.Variable(tf.truncated_normal(shape=[5, 5, 1, depth["D_1"]], mean = mu, stddev = sigma, name = 'weight1')), # 'W_conv2':tf.Variable(tf.truncated_normal(shape=[5, 5, depth["D_1"], depth["D_2"]], mean = mu, stddev = sigma, name = 'weight2')), # 'W_conv3':tf.Variable(tf.truncated_normal(shape=[5, 5, depth["D_2"], depth["D_3"]], mean = mu, stddev = sigma, name = 'weight3')), # 'W_fc1': tf.Variable(tf.truncated_normal(shape=[8192, depth["D_4"]], mean = mu, stddev = sigma, name = 'weight4')), # 'W_fc2': tf.Variable(tf.truncated_normal(shape=[depth["D_4"], depth["D_5"]], mean = mu, stddev = sigma, name = 'weight5')), # 'W_out': tf.Variable(tf.truncated_normal(shape=[depth["D_5"], n_classes], mean = mu, stddev = sigma, name = 'weight_out')), 'W_conv1': tf.Variable(xavier_initializer_conv2d()([5, 5, 1, depth["D_1"]]), name='weight1'), 'W_conv2': tf.Variable( xavier_initializer_conv2d()([5, 5, depth["D_1"], depth["D_2"]]), name='weight2'), 'W_conv3': tf.Variable( xavier_initializer_conv2d()([5, 5, depth["D_2"], depth["D_3"]]), name='weight3'), 'W_fc1': tf.Variable(xavier_initializer_conv2d()([8192, depth["D_4"]]), name='weight4'), 'W_fc2': tf.Variable(xavier_initializer_conv2d()([depth["D_4"], depth["D_5"]]), name='weight5'), 'W_out': tf.Variable(xavier_initializer_conv2d()([depth["D_5"], n_classes]), name='weight_out'), } biases = { 'B_1': tf.Variable(tf.zeros(depth['D_1']), name='bias_1'), 'B_2': tf.Variable(tf.zeros(depth['D_2']), name='bias_2'), 'B_3': tf.Variable(tf.zeros(depth['D_3']), name='bias_3'), 'B_4': tf.Variable(tf.zeros(depth['D_4']), name='bias_4'), 'B_5': tf.Variable(tf.zeros(depth['D_5']), name='bias_5'), 'B_out': tf.Variable(tf.zeros(n_classes), name='bias_out') } # print(x) # Layer 1: Convolutional. Input = 32x32x1. Output = 32x32x1. conv1 = tf.nn.conv2d( x, weights['W_conv1'], strides=[1, 1, 1, 1 ], padding='SAME') + biases['B_1'] conv1 = tf.nn.relu(conv1) # Activation. # Pooling. Input = 32x32x1. Output = 14x14x16. conv1 = tf.nn.max_pool(conv1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME') # Layer 2: Convolutional. Output = 10x10x32. conv2 = tf.nn.conv2d( conv1, weights['W_conv2'], strides=[1, 1, 1, 1 ], padding='SAME') + biases['B_2'] conv2 = tf.nn.relu(conv2) # Activation. # Pooling. Input = 10x10x32. Output = 5x5x32. conv2 = tf.nn.max_pool(conv2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME') # Layer 2: Convolutional. Output = 10x10x32. conv3 = tf.nn.conv2d( conv2, weights['W_conv3'], strides=[1, 1, 1, 1 ], padding='SAME') + biases['B_3'] conv3 = tf.nn.relu(conv3) # Activation. # Pooling. Input = 10x10x32. Output = 5x5x32. conv3 = tf.nn.max_pool(conv3, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME') # Flatten. Input = 5x5x32. Output = 800. fc0 = flatten(conv3) # print("fc0", fc0, "weights:", weights["W_fc1"]) # Layer 3: Fully Connected. Input = 800. Output = 120. fc1 = tf.matmul(fc0, weights['W_fc1']) + biases['B_4'] fc1 = tf.nn.relu(fc1) # Activation. # Layer 4: Fully Connected. Input = 120. Output = 84. fc2 = tf.matmul(fc1, weights['W_fc2']) + biases['B_5'] fc2 = tf.nn.relu(fc2) # Activation. # Layer 5: Fully Connected. Input = 84. Output = 10. logits = tf.matmul(fc2, weights['W_out']) + biases['B_out'] return logits
def apply_logit(self): with tf.variable_scope(self.modelname + "conv_1") as scope: self.W_conv1 = tf.get_variable( name='weights', shape=[1, 5, 1, 5], initializer=xavier_initializer_conv2d()) self.h_bn1 = tf.layers.batch_normalization( tf.nn.conv2d(self.X, self.W_conv1, strides=[1, 1, 1, 1], padding='VALID'), training=self.trainphase) self.h_conv1 = tf.nn.leaky_relu(self.h_bn1) with tf.variable_scope(self.modelname + "conv_2") as scope: self.W_conv2 = tf.get_variable( name='weights', shape=[2, 5, 1, 5], initializer=xavier_initializer_conv2d()) self.h_bn2 = tf.layers.batch_normalization( tf.nn.conv2d(self.X, self.W_conv2, strides=[1, 1, 1, 1], padding='VALID'), training=self.trainphase) self.h_conv2 = tf.nn.leaky_relu(self.h_bn2) with tf.variable_scope(self.modelname + "conv_3") as scope: self.W_conv3 = tf.get_variable( name='weights', shape=[3, 5, 1, 5], initializer=xavier_initializer_conv2d()) self.h_bn3 = tf.layers.batch_normalization( tf.nn.conv2d(self.X, self.W_conv3, strides=[1, 1, 1, 1], padding='VALID'), training=self.trainphase) self.h_conv3 = tf.nn.leaky_relu(self.h_bn3) with tf.variable_scope(self.modelname + "conv_4") as scope: self.W_conv4 = tf.get_variable( name='weights', shape=[4, 5, 1, 5], initializer=xavier_initializer_conv2d()) self.h_bn4 = tf.layers.batch_normalization( tf.nn.conv2d(self.X, self.W_conv4, strides=[1, 1, 1, 1], padding='VALID'), training=self.trainphase) self.h_conv4 = tf.nn.leaky_relu(self.h_bn4) self.pooled1 = tf.reshape(self.h_conv1, shape=[-1, int(self.duration), 5]) self.pooled1 = tf.reduce_mean(self.pooled1, reduction_indices=2) self.pooled1 = tf.squeeze(self.pooled1) self.pooled2 = tf.reshape(self.h_conv2, shape=[-1, int(self.duration - 1), 5]) self.pooled2 = tf.reduce_mean(self.pooled2, reduction_indices=2) self.pooled2 = tf.squeeze(self.pooled2) self.pooled3 = tf.reshape(self.h_conv3, shape=[-1, int(self.duration - 2), 5]) self.pooled3 = tf.reduce_mean(self.pooled3, reduction_indices=2) self.pooled3 = tf.squeeze(self.pooled3) ''' self.pooled4 = tf.reshape(self.h_conv4, shape=[-1, int(self.duration - 3), 5]) self.pooled4 = tf.reduce_mean(self.pooled4, reduction_indices=2) self.pooled4 = tf.squeeze(self.pooled4) ''' ''' self.pooled1_flatten = tf.reshape(self.h_conv1,shape=[-1,int(self.duration)*5]) self.pooled2_flatten = tf.reshape(self.h_conv2,shape=[-1,int(self.duration-1)*5]) self.pooled3_flatten = tf.reshape(self.h_conv3, shape=[-1, int(self.duration-2)*5]) self.pooled4_flatten = tf.reshape(self.h_conv4, shape=[-1, int(self.duration-3)*5]) ''' #self.pooled1_flatten = tf.reshape(self.pooled1,shape=[-1,self.duration]) self.pooled2_flatten = tf.reshape(self.pooled2, shape=[-1, self.duration - 1]) self.pooled3_flatten = tf.reshape(self.pooled3, shape=[-1, self.duration - 2]) #self.pooled4_flatten = tf.reshape(self.pooled4, shape=[-1, self.duration - 3]) #self.flatten = tf.concat([self.pooled1_flatten,self.pooled2_flatten,self.pooled3_flatten,self.pooled4_flatten],1) self.flatten = tf.concat([self.pooled2_flatten, self.pooled3_flatten], 1) num_unit = 300 H = self.linear(self.flatten, self.flatten.shape[1], num_unit, 'L1') H = self.linear(H, num_unit, num_unit, "L2") #H = self.linear(H,num_unit,num_unit,"L3") #H = self.linear(H, num_unit, num_unit, "L4") #H = self.linear(H, num_unit, num_unit, "L5") #H = self.linear(H, num_unit, num_unit, "L6") #H = self.linear(H,num_unit,num_unit,"L7") #H = self.linear(H,num_unit,num_unit,"L8") #H = self.linear(H,num_unit,num_unit,"L9") #H = self.linear(H,num_unit,num_unit,"L10") #H = self.linear(H,100,100,"L7") #H = self.linear(H,100,100,"L8") #H = self.linear(H,100,100,"L9") self.output_features = self.linear(H, num_unit, 128, "Llast") self.output = self.linear(self.output_features, 128, 1, "output", active_f=None) return self.output
def __init__(self, n_tags: int, word_vocab, word_dim: int, word_emb_path: str, word_emb_name: str = None, char_vocab_size: int = None, pos_vocab_size: int = None, chunk_vocab_size: int = None, char_dim: int = None, elmo_dim: int = None, pos_dim: int = None, chunk_dim: int = None, cap_dim: int = None, cap_vocab_size: int = 5, lstm_hidden_size: int = 256, dropout_keep_prob: float = 0.5, **kwargs) -> None: assert n_tags != 0, 'Number of classes equal 0! It seems that vocabularies is not loaded.' \ ' Check that all vocabulary files are downloaded!' if 'learning_rate_drop_div' not in kwargs: kwargs['learning_rate_drop_div'] = 10.0 if 'learning_rate_drop_patience' not in kwargs: kwargs['learning_rate_drop_patience'] = 5.0 if 'clip_norm' not in kwargs: kwargs['clip_norm'] = 5.0 super().__init__(**kwargs) word2id = word_vocab.t2i self._dropout_ph = tf.placeholder_with_default(dropout_keep_prob, shape=[], name='dropout') self.training_ph = tf.placeholder_with_default(False, shape=[], name='is_training') self._y_ph = tf.placeholder(tf.int32, [None, None], name='y_ph') self._xs_ph_list = [] self._input_features = [] # use for word contextualized bi-lstm, elmo self.real_sent_lengths_ph = tf.placeholder(tf.int32, [None], name="real_sent_lengths") self._xs_ph_list.append(self.real_sent_lengths_ph) # Word emb with tf.variable_scope("word_emb"): word_ids_ph = tf.placeholder(tf.int32, [None, None], name="word_ids") self._xs_ph_list.append(word_ids_ph) word_embeddings = self.load_pretrained_word_emb( word_emb_path, word_emb_name, word_dim, word2id) word_lookup_table = tf.Variable(word_embeddings, dtype=tf.float32, trainable=True, name="word_embeddings") word_emb = tf.nn.embedding_lookup(word_lookup_table, word_ids_ph, name="embedded_word") self._input_features.append(word_emb) # POS feature if pos_dim is not None: with tf.variable_scope("pos_emb"): pos_ph = tf.placeholder(tf.int32, [None, None], name="pos_ids") self._xs_ph_list.append(pos_ph) tf_pos_embeddings = tf.get_variable( name="pos_embeddings", dtype=tf.float32, shape=[pos_vocab_size, pos_dim], trainable=True, initializer=xavier_initializer()) embedded_pos = tf.nn.embedding_lookup(tf_pos_embeddings, pos_ph, name="embedded_pos") self._input_features.append(embedded_pos) # Chunk feature if chunk_dim is not None: with tf.variable_scope("chunk_emb"): chunk_ph = tf.placeholder(tf.int32, [None, None], name="chunk_ids") self._xs_ph_list.append(chunk_ph) tf_chunk_embeddings = tf.get_variable( name="chunk_embeddings", dtype=tf.float32, shape=[chunk_vocab_size, chunk_dim], trainable=True, initializer=xavier_initializer()) embedded_chunk = tf.nn.embedding_lookup(tf_chunk_embeddings, chunk_ph, name="embedded_chunk") self._input_features.append(embedded_chunk) # Capitalization feature if cap_dim is not None: with tf.variable_scope("cap_emb"): cap_ph = tf.placeholder(tf.int32, [None, None], name="cap_ids") self._xs_ph_list.append(cap_ph) tf_cap_embeddings = tf.get_variable( name="cap_embeddings", dtype=tf.float32, shape=[cap_vocab_size, cap_dim], trainable=True, initializer=xavier_initializer()) embedded_cap = tf.nn.embedding_lookup(tf_cap_embeddings, cap_ph, name="embedded_cap") self._input_features.append(embedded_cap) # Character feature if char_dim is not None: with tf.variable_scope("char_emb"): char_ids_ph = tf.placeholder(tf.int32, [None, None, None], name="char_ids") self._xs_ph_list.append(char_ids_ph) tf_char_embeddings = tf.get_variable( name="char_embeddings", dtype=tf.float32, shape=[char_vocab_size, char_dim], trainable=True, initializer=xavier_initializer()) embedded_cnn_chars = tf.nn.embedding_lookup( tf_char_embeddings, char_ids_ph, name="embedded_cnn_chars") conv1 = tf.layers.conv2d( inputs=embedded_cnn_chars, filters=128, kernel_size=(1, 3), strides=(1, 1), padding="same", name="conv1", kernel_initializer=xavier_initializer_conv2d()) conv2 = tf.layers.conv2d( inputs=conv1, filters=128, kernel_size=(1, 3), strides=(1, 1), padding="same", name="conv2", kernel_initializer=xavier_initializer_conv2d()) char_cnn = tf.reduce_max(conv2, axis=2) self._input_features.append(char_cnn) # ELMo if elmo_dim is not None: with tf.variable_scope("elmo_emb"): padded_x_tokens_ph = tf.placeholder(tf.string, [None, None], name="padded_x_tokens") self._xs_ph_list.append(padded_x_tokens_ph) elmo = hub.Module("https://tfhub.dev/google/elmo/2", trainable=True) emb = elmo(inputs={ "tokens": padded_x_tokens_ph, "sequence_len": self.real_sent_lengths_ph }, signature="tokens", as_dict=True)["elmo"] elmo_emb = tf.layers.dense(emb, elmo_dim, activation=None) self._input_features.append(elmo_emb) features = tf.nn.dropout(tf.concat(self._input_features, axis=2), self._dropout_ph) with tf.variable_scope("bi_lstm_words"): cell_fw = tf.contrib.rnn.LSTMCell(lstm_hidden_size) cell_bw = tf.contrib.rnn.LSTMCell(lstm_hidden_size) (output_fw, output_bw), _ = tf.nn.bidirectional_dynamic_rnn( cell_fw, cell_bw, features, sequence_length=self.real_sent_lengths_ph, dtype=tf.float32) self.output = tf.concat([output_fw, output_bw], axis=-1) ntime_steps = tf.shape(self.output)[1] self.output = tf.reshape(self.output, [-1, 2 * lstm_hidden_size]) layer1 = tf.nn.dropout( tf.layers.dense(inputs=self.output, units=lstm_hidden_size, activation=None, kernel_initializer=xavier_initializer()), self._dropout_ph) pred = tf.layers.dense(inputs=layer1, units=n_tags, activation=None, kernel_initializer=xavier_initializer()) self.logits = tf.reshape(pred, [-1, ntime_steps, n_tags]) log_likelihood, self.transition_params = tf.contrib.crf.crf_log_likelihood( self.logits, self._y_ph, self.real_sent_lengths_ph) # loss and opt with tf.variable_scope("loss_and_opt"): self.loss = tf.reduce_mean(-log_likelihood) self.train_op = self.get_train_op(self.loss) self.sess = tf.Session() self.sess.run(tf.global_variables_initializer()) self.load()
def redResidua(input): out = tf.reshape(input, [-1, 128, 128, 1]) out = conv2d(out, 64, 3, padding='same', activation=tf.nn.relu, kernel_initializer=xavier_initializer_conv2d()) out = batch_normalization(out) out = conv2d(out, 64, 3, padding='same', activation=tf.nn.relu, kernel_initializer=xavier_initializer_conv2d()) out = batch_normalization(out) out = max_pool2d(out, 3, stride=2) max1 = out out = conv2d(out, 64, 3, padding='same', activation=tf.nn.relu, kernel_initializer=xavier_initializer_conv2d()) out = batch_normalization(out) out = conv2d(out, 64, 3, padding='same', activation=tf.nn.relu, kernel_initializer=xavier_initializer_conv2d()) out = batch_normalization(out) out = out + max1 r1 = out out = conv2d(out, 64, 3, padding='same', activation=tf.nn.relu, kernel_initializer=xavier_initializer_conv2d()) out = batch_normalization(out) out = conv2d(out, 64, 3, padding='same', activation=tf.nn.relu, kernel_initializer=xavier_initializer_conv2d()) out = batch_normalization(out) out = out + r1 out = conv2d(out, 128, 3, padding='same', activation=tf.nn.relu, kernel_initializer=xavier_initializer_conv2d()) out = batch_normalization(out) out = max_pool2d(out, 3, stride=2) max2 = out out = conv2d(out, 128, 3, padding='same', activation=tf.nn.relu, kernel_initializer=xavier_initializer_conv2d()) out = batch_normalization(out) out = conv2d(out, 128, 3, padding='same', activation=tf.nn.relu, kernel_initializer=xavier_initializer_conv2d()) out = batch_normalization(out) out = out + max2 r3 = out out = conv2d(out, 128, 3, padding='same', activation=tf.nn.relu, kernel_initializer=xavier_initializer_conv2d()) out = batch_normalization(out) out = conv2d(out, 128, 3, padding='same', activation=tf.nn.relu, kernel_initializer=xavier_initializer_conv2d()) out = batch_normalization(out) out = out + r3 out = conv2d(out, 256, 3, padding='same', activation=tf.nn.relu, kernel_initializer=xavier_initializer_conv2d()) out = batch_normalization(out) out = max_pool2d(out, 3, stride=2) max3 = out out = conv2d(out, 256, 3, padding='same', activation=tf.nn.relu, kernel_initializer=xavier_initializer_conv2d()) out = batch_normalization(out) out = conv2d(out, 256, 3, padding='same', activation=tf.nn.relu, kernel_initializer=xavier_initializer_conv2d()) out = batch_normalization(out) out = out + max3 r5 = out out = conv2d(out, 256, 3, padding='same', activation=tf.nn.relu, kernel_initializer=xavier_initializer_conv2d()) out = batch_normalization(out) out = conv2d(out, 256, 3, padding='same', activation=tf.nn.relu, kernel_initializer=xavier_initializer_conv2d()) out = batch_normalization(out) out = out + r5 out = conv2d(out, 256, 3, padding='same', activation=tf.nn.relu, kernel_initializer=xavier_initializer_conv2d()) out = batch_normalization(out) out = max_pool2d(out, 3, stride=2) out = tf.contrib.layers.flatten(out) out = fully_connected(out, 1024, activation_fn=tf.nn.relu, weights_initializer=xavier_initializer()) out = fully_connected(out, 100, activation_fn=None, weights_initializer=xavier_initializer()) return out
def inference(self, img): self.padding = 'SAME' # initializer = tf.truncated_normal_initializer(stddev=0.01) self.initializer = layers.xavier_initializer_conv2d() self.initializer_b = layers.xavier_initializer() self.regularizer = slim.l2_regularizer(0.0005) self.activation = slim.layers.nn.leaky_relu self.activation = slim.layers.nn.relu print(self.n_filters_first_conv) with tf.variable_scope("first-conv"): stack = slim.conv2d(img, num_outputs=self.n_filters_first_conv, kernel_size=[3, 3], weights_initializer=self.initializer, activation_fn=self.activation, padding=self.padding, weights_regularizer=self.regularizer, biases_initializer=self.initializer_b) print(stack.get_shape()) n_filters = self.n_filters_first_conv pool_heads = [] for i in range(self.n_pool + 1): print("Dense Block ", i + 1) with tf.variable_scope("block" + str(i + 1)): for j in range(self.n_layers_per_block[i]): with tf.variable_scope("layer" + str(j + 1)): #with tf.variable_scope("1x1conv"): #conv=self.conv(stack,self.growth_rate*4,kernel_size=1) with tf.variable_scope("3x3conv"): conv = self.conv(stack, self.growth_rate, kernel_size=3) stack = tf.concat([stack, conv], axis=-1) n_filters += self.growth_rate print(stack.get_shape(), " n=", n_filters) pool_heads.append(stack) if self.n_pool == i: continue with tf.variable_scope("Downsample" + str(i + 1)): stack = self.downSample(stack, int(n_filters * 0.5)) print(n_filters / 2) stack = pool_heads[len(pool_heads) - 1] print("Turning up:", stack.get_shape()) for i in range(self.n_pool): print("Dense Block ", i + self.n_pool + 1) with tf.variable_scope("Upsample" + str(i + 1)): concat = pool_heads[len(pool_heads) - i - 2] n_filters_keep = self.growth_rate * self.n_layers_per_block[ self.n_pool + i] print(n_filters) stack = self.upsample(stack, concat, int(n_filters_keep * self.compression)) with tf.variable_scope("unpool-block" + str(i + 1)): for j in range(self.n_layers_per_block[self.n_pool + i + 1]): with tf.variable_scope("layer" + str(j + 1)): #with tf.variable_scope("1x1conv" + str(j + 1)): #conv=self.conv(stack,self.growth_rate*4,kernel_size=1) with tf.variable_scope("3x3conv" + str(j + 1)): conv = self.conv(stack, self.growth_rate) stack = tf.concat([stack, conv], axis=-1) print(stack.get_shape()) with tf.variable_scope("last-conv"): batchnorm = tf.layers.batch_normalization(stack) logits = slim.conv2d(inputs=batchnorm, num_outputs=self.n_classes + 1, kernel_size=1, padding='SAME', weights_initializer=self.initializer, biases_initializer=self.initializer_b, activation_fn=None) print("logits=", logits.get_shape()) with tf.variable_scope("softmax"): pred = tf.argmax(tf.nn.softmax(logits), axis=-1) print(pred.get_shape()) p = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES) #print(p) for i, a in enumerate(p): if (i % 8) == 3: #print(i) tf.summary.histogram( a.name, tf.Graph.get_tensor_by_name(tf.get_default_graph(), a.name)) pred = tf.expand_dims(pred, axis=-1) print("pred==", pred.get_shape()) tf.summary.image( "pred", tf.multiply(tf.constant(255, dtype=tf.uint8), tf.cast(pred, tf.uint8))) # tf.summary.histogram("bias", kernel) return logits, pred
def _build_model(self, inputs): self.inputs = inputs if self.data_format == 'NCHW': channel_axis = 1 _inputs = tf.cast(tf.transpose(inputs, [0, 3, 1, 2]), tf.float32) else: channel_axis = 3 _inputs = tf.cast(inputs, tf.float32) self.L = [] with arg_scope([layers.avg_pool2d], \ padding='VALID', data_format=self.data_format): with tf.variable_scope('SRM_preprocess'): W_SRM = tf.get_variable('W', initializer=SRM_Kernels, \ dtype=tf.float32, \ regularizer=None) b = tf.get_variable('b', shape=[30], dtype=tf.float32, \ initializer=tf.constant_initializer(0.)) self.L.append(tf.nn.bias_add( \ tf.nn.conv2d(_inputs, \ W_SRM, [1,1,1,1], 'VALID', \ data_format=self.data_format), b, \ data_format=self.data_format, name='Layer1')) self.L.append(tf.clip_by_value(self.L[-1], \ -self.tlu_threshold, self.tlu_threshold, \ name='TLU')) with tf.variable_scope('ConvNetwork'): with arg_scope([my_layers.conv2d], num_outputs=30, \ kernel_size=3, stride=1, padding='VALID', \ data_format=self.data_format, \ activation_fn=tf.nn.relu, \ weights_initializer=layers.xavier_initializer_conv2d(), \ weights_regularizer=layers.l2_regularizer(5e-4), \ biases_initializer=tf.constant_initializer(0.2), \ biases_regularizer=None), arg_scope([layers.batch_norm], \ decay=0.9, center=True, scale=True, \ updates_collections=None, is_training=self.is_training, \ fused=True, data_format=self.data_format): if self.with_bn: self.L.append(layers.batch_norm(self.L[-1], \ scope='Norm1')) self.L.append(my_layers.conv2d(self.L[-1], \ scope='Layer2')) if self.with_bn: self.L.append(layers.batch_norm(self.L[-1], \ scope='Norm2')) self.L.append(my_layers.conv2d(self.L[-1], \ scope='Layer3')) if self.with_bn: self.L.append(layers.batch_norm(self.L[-1], \ scope='Norm3')) self.L.append(my_layers.conv2d(self.L[-1], \ scope='Layer4')) if self.with_bn: self.L.append(layers.batch_norm(self.L[-1], \ scope='Norm4')) self.L.append(layers.avg_pool2d(self.L[-1], \ kernel_size=[2,2], scope='Stride1')) with arg_scope([my_layers.conv2d], kernel_size=5, \ num_outputs=32): self.L.append(my_layers.conv2d(self.L[-1], \ scope='Layer5')) if self.with_bn: self.L.append(layers.batch_norm(self.L[-1], \ scope='Norm5')) self.L.append(layers.avg_pool2d(self.L[-1], \ kernel_size=[3,3], \ scope='Stride2')) self.L.append(my_layers.conv2d(self.L[-1], \ scope='Layer6')) if self.with_bn: self.L.append(layers.batch_norm(self.L[-1], \ scope='Norm6')) self.L.append(layers.avg_pool2d(self.L[-1], \ kernel_size=[3,3], \ scope='Stride3')) self.L.append(my_layers.conv2d(self.L[-1], \ scope='Layer7')) if self.with_bn: self.L.append(layers.batch_norm(self.L[-1], \ scope='Norm7')) self.L.append(layers.avg_pool2d(self.L[-1], \ kernel_size=[3,3], \ scope='Stride4')) self.L.append(my_layers.conv2d(self.L[-1], \ num_outputs=16, \ scope='Layer8')) if self.with_bn: self.L.append(layers.batch_norm(self.L[-1], \ scope='Norm8')) self.L.append(my_layers.conv2d(self.L[-1], \ num_outputs=16, stride=3, \ scope='Layer9')) if self.with_bn: self.L.append(layers.batch_norm(self.L[-1], \ scope='Norm9')) self.L.append(layers.flatten(self.L[-1])) self.L.append(layers.fully_connected(self.L[-1], num_outputs=2, \ activation_fn=None, normalizer_fn=None, \ weights_initializer=tf.random_normal_initializer(mean=0., stddev=0.01), \ biases_initializer=tf.constant_initializer(0.), scope='ip')) self.outputs = self.L[-1] return self.outputs
def _build_critic_network(self): """ Common shared critic. Take the observation(state) from all agencies First observation state_inputs[0] must be the observation of 'self' agent. Variable: num_assist (int) : number of assisting state critic_state_input (placeholder) : state placeholder for critic evaluation assist_states (list placeholder) : state placeholder for assists critic evaluation assist_mask (boolean placeholder) : mask of assist inputs (Block if flowpath is blocked) """ num_assist = self.num_agent-1 self.critic_state_input = tf.placeholder(shape=self.in_size, dtype=tf.float32, name='cr_state_hold') self.assist_states = [tf.placeholder(shape=self.in_size, dtype=tf.float32, name='assist_states') for _ in range(num_assist)] self.mask = tf.placeholder(shape=[None, self.num_agent], dtype=tf.float32, name='mask') scope = 'critic' critic_evaluations = [] with tf.variable_scope(scope): for input_tensor in [self.critic_state_input]+self.assist_states: net = layers.conv2d(input_tensor, 32, [3,3], activation_fn=tf.nn.relu, weights_initializer=layers.xavier_initializer_conv2d(), biases_initializer=tf.zeros_initializer(), padding='VALID', scope='conv1', reuse=tf.AUTO_REUSE) net = layers.max_pool2d(net, [2,2]) net = layers.conv2d(net, 64, [2,2], activation_fn=tf.nn.relu, weights_initializer=layers.xavier_initializer_conv2d(), biases_initializer=tf.zeros_initializer(), padding='VALID', scope='conv2', reuse=tf.AUTO_REUSE) net = layers.flatten(net) net = layers.fully_connected(net, 128, scope='dense1', reuse=tf.AUTO_REUSE) net = layers.fully_connected(net, 1, activation_fn=None, scope='dense2', reuse=tf.AUTO_REUSE) #if input_tensor in self.assist_states: # net = tf.stop_gradient(net) critic_evaluations.append(tf.reshape(net,[-1,1])) with tf.name_scope('Concat'): net = tf.concat(critic_evaluations, 1) # [None, 4] reweight = tf.get_variable(name='critic_reweight', shape=[self.num_agent], dtype=tf.float32, initializer=tf.constant_initializer(value=1.0/self.num_agent) ) shift = tf.get_variable(name='critic_shift', shape=[1], dtype=tf.float32, initializer=tf.zeros_initializer ) net = tf.multiply(net, reweight) + shift net = tf.reduce_sum(tf.multiply(net, self.mask), axis=1) vars_list = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=self.scope+'/'+scope) return net, vars_list
def _create_weight(self, name, shape): init = layers.xavier_initializer_conv2d(dtype=tf.float32) return tf.Variable(init(shape=shape), name=name)
def _initialize_weights(self): all_weights = dict() n_layers = len(self.n_hidden) all_weights['Coef'] = tf.Variable( 0 * tf.ones([self.batch_size, self.batch_size], tf.float32), name='Coef') all_weights['enc_w0'] = tf.get_variable( "enc_w0", shape=[ self.kernel_size[0], self.kernel_size[0], 1, self.n_hidden[0] ], initializer=layers.xavier_initializer_conv2d(), regularizer=self.reg) all_weights['enc_b0'] = tf.Variable( tf.zeros([self.n_hidden[0]], dtype=tf.float32)) # , name = 'enc_b0' iter_i = 1 while iter_i < n_layers: enc_name_wi = 'enc_w' + str(iter_i) all_weights[enc_name_wi] = tf.get_variable(enc_name_wi, shape=[self.kernel_size[iter_i], self.kernel_size[iter_i], self.n_hidden[iter_i-1], \ self.n_hidden[iter_i]], initializer=layers.xavier_initializer_conv2d(),regularizer = self.reg) enc_name_bi = 'enc_b' + str(iter_i) all_weights[enc_name_bi] = tf.Variable( tf.zeros([self.n_hidden[iter_i]], dtype=tf.float32)) # , name = enc_name_bi iter_i = iter_i + 1 iter_i = 1 while iter_i < n_layers: dec_name_wi = 'dec_w' + str(iter_i - 1) all_weights[dec_name_wi] = tf.get_variable( dec_name_wi, shape=[ self.kernel_size[n_layers - iter_i], self.kernel_size[n_layers - iter_i], self.n_hidden[n_layers - iter_i - 1], self.n_hidden[n_layers - iter_i] ], initializer=layers.xavier_initializer_conv2d(), regularizer=self.reg) dec_name_bi = 'dec_b' + str(iter_i - 1) all_weights[dec_name_bi] = tf.Variable( tf.zeros([self.n_hidden[n_layers - iter_i - 1]], dtype=tf.float32)) # , name = dec_name_bi iter_i = iter_i + 1 dec_name_wi = 'dec_w' + str(iter_i - 1) all_weights[dec_name_wi] = tf.get_variable( dec_name_wi, shape=[ self.kernel_size[0], self.kernel_size[0], 1, self.n_hidden[0] ], initializer=layers.xavier_initializer_conv2d(), regularizer=self.reg) dec_name_bi = 'dec_b' + str(iter_i - 1) all_weights[dec_name_bi] = tf.Variable(tf.zeros( [1], dtype=tf.float32)) # , name = dec_name_bi return all_weights
def __init__(self, params=None): self.log_dis = {} self.log_dis[0] = [0, 0] for i in range(1, 10): self.log_dis[i] = [self.log_dis[i-1][1] + 1, self.log_dis[i-1][1] + 2**i] self.params = params self.keep_prob = params["keep_prob"] self.nb_epochs = params["nb_epochs"] # load word and char dictionaries dicts = pickle.load(open(params["dicts_file"], "rb")) self.w2i = dicts["w2i"] self.i2w = dicts["i2w"] self.c2i = dicts["c2i"] self.i2c = dicts["i2c"] self.word_dim = params["word_dim"] self.word_vocab_size = len(self.w2i) self.char_dim = params["char_dim"] self.char_vocab_size = len(self.c2i) print("Sizes of word and char dictionaries: {}, {}".format(self.word_vocab_size, self.char_vocab_size)) self.word_emb = np.zeros(shape=(self.word_vocab_size, self.word_dim)) # load word embedding if "word_emb" in params: print("pre-trained word embedding {} is being loaded ...".format(params["word_emb"])) self.load_word_emb(params["word_emb"]) tf.reset_default_graph() # [sent, word] self.tf_word_ids = tf.placeholder(dtype=tf.int32, shape=[None, None], name="word_ids") # real length sents self.tf_sentence_lengths= tf.placeholder(dtype=tf.int32, shape=[None], name="sentence_lengths") # [sent, word, char] self.tf_char_ids = tf.placeholder(dtype=tf.int32, shape=[None, None, None], name="char_ids") # binary matrix representing the relationship between sents: [sent, sent] self.tf_target_matrix = tf.placeholder(dtype=tf.int32, shape=[None, None], name="target_matrix") # keep_prob self.tf_keep_prob = tf.placeholder(dtype=tf.float32, shape=[], name="keep_prob") # learning rate self.tf_learning_rate= tf.placeholder(dtype=tf.float32, shape=[], name="learning_rate") # distant sentences self.tf_distant_sents = tf.placeholder(dtype=tf.int32, shape=[None, None], name="tf_distant_sents") # load word embedding with tf.variable_scope("word_embedding"): tf_word_embeddings = tf.Variable(self.word_emb, dtype=tf.float32, trainable=True, name="word_embedding") embedded_words = tf.nn.embedding_lookup(tf_word_embeddings, self.tf_word_ids, name="embedded_words") self.input = embedded_words # sent, word, word_dim # CNN network to capture character-level features with tf.variable_scope("char_cnn"): tf_char_embeddings = tf.get_variable(name="char_embeddings", dtype=tf.float32, shape=[self.char_vocab_size, self.char_dim], trainable=True, initializer=xavier_initializer()) conv = tf.nn.embedding_lookup(tf_char_embeddings, self.tf_char_ids, name="embedded_cnn_chars") for i, (ks, fil) in enumerate(self.params["conv"]): conv = tf.layers.conv2d(inputs=conv, # sent, word, char, feature filters=fil, kernel_size=(1, ks), strides=(1, 1), padding="same", name="conv_{}".format(i), kernel_initializer=xavier_initializer_conv2d()) # sent, word, char, cnn_feature # shape = tf.shape(conv) # conv = tf.nn.dropout(x=conv, keep_prob=self.tf_keep_prob, noise_shape=[shape[0], 1, shape[2], shape[3]]) # conv = tf.nn.dropout(x=conv, keep_prob=self.tf_keep_prob) self.char_cnn = tf.reduce_max(conv, axis=2) # sent, word, cnn_feature self.input = tf.nn.dropout(tf.concat([self.input, self.char_cnn], axis=-1), self.tf_keep_prob) # [sents, words, word_dim + cnn_features] # Bi-LSTM to generate final input representation in combination with both left and right contexts with tf.variable_scope("bi_lstm_words"): cell_fw = tf.contrib.rnn.LSTMCell(self.params["word_lstm_units"]) cell_bw = tf.contrib.rnn.LSTMCell(self.params["word_lstm_units"]) (output_fw, output_bw), _ = tf.nn.bidirectional_dynamic_rnn(cell_fw, cell_bw, self.input, sequence_length=self.tf_sentence_lengths, dtype=tf.float32) bilstm_output = tf.concat([output_fw, output_bw], axis=-1) # [sents, words, 2*lstm_units] mask = tf.where(condition=tf.equal(self.tf_word_ids, self.w2i["<PAD>"]), x=-1e10*tf.ones_like(self.tf_word_ids, dtype=tf.float32), y=tf.zeros_like(self.tf_word_ids, dtype=tf.float32)) mask = tf.tile(tf.expand_dims(mask, -1), (1, 1, 2*self.params["word_lstm_units"])) bilstm_output = bilstm_output + mask # bilstm_output = tf.nn.dropout(tf.reduce_max(bilstm_output, axis=1), self.tf_keep_prob) # [sents, 2*word_lstm_units] bilstm_output = tf.reduce_max(bilstm_output, axis=1) # [sents, 2*word_lstm_units] # represent sents in their contexts with tf.variable_scope("sent_representation"): # sent_rep = tf.layers.conv1d(inputs=sent_rep[None, :, :], filters=2*self.lstm_num_units, kernel_size=3, padding='same') # sent_rep = tf.squeeze(sent_rep, axis=0) sent_fw = tf.contrib.rnn.LSTMCell(self.params["sent_lstm_units"]) sent_bw = tf.contrib.rnn.LSTMCell(self.params["sent_lstm_units"]) (output_sent_fw, output_sent_bw), _ = tf.nn.bidirectional_dynamic_rnn(sent_fw, sent_bw, bilstm_output[None, :, :], # sequence_length=self.tf_sentence_lengths, dtype=tf.float32) sent_bilstm = tf.concat([output_sent_fw, output_sent_bw], axis=-1) sent_rep = tf.squeeze(sent_bilstm, axis=0) # [sents, 2*sent_lstm_units] with tf.variable_scope("self_att"): distant_embeddings = tf.get_variable(name="distant_embeddings", dtype=tf.float32, shape=[10, 30], trainable=True, initializer=xavier_initializer() ) embedded_disant_sent = tf.nn.embedding_lookup(distant_embeddings, self.tf_distant_sents, name="embedded_disant_sent") # [sent, sent, 20] x = sent_rep de = x.get_shape()[-1] w = tf.get_variable(dtype=tf.float32, shape=[de], trainable=True, name="w") w1 = tf.get_variable(dtype=tf.float32, shape=[de, de], trainable=True, name="W1") w2 = tf.get_variable(dtype=tf.float32, shape=[de, de], trainable=True, name="W2") b1 = tf.get_variable(dtype=tf.float32, shape=[de], trainable=True, name="b1") b = tf.get_variable(dtype=tf.float32, shape=[], trainable=True, name="b") x_w1 = tf.matmul(x, w1) # n, de s = tf.map_fn(lambda xj: x_w1 + tf.tensordot(w2, xj, axes=1) + b1, x) # n, n, de s = s + tf.layers.dense(embedded_disant_sent, de, use_bias=False, reuse=tf.AUTO_REUSE) f = tf.tensordot(tf.tanh(s), w, axes=1) + b # n, n weight = tf.nn.softmax(f, axis=1) # n, n h = tf.transpose(tf.map_fn(lambda weight_j: tf.transpose(x)*weight_j, weight), [0, 2, 1]) # [sents, sents, 4*sent_lstm_units] with tf.variable_scope("loss_and_opt"): self.logits = tf.nn.dropout(tf.layers.dense(inputs=h, units=2, activation=None), self.tf_keep_prob) self.logits = 0.5*(self.logits + tf.transpose(self.logits, [1, 0, 2])) self.loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=tf.one_hot(self.tf_target_matrix, 2), logits=self.logits, name="loss_function")) self.pred = tf.argmax(self.logits, axis=-1) eq = tf.cast(tf.equal(tf.cast(self.pred, tf.int32), self.tf_target_matrix), tf.float32) self.acc = tf.reduce_mean(eq) self.opt = tf.train.AdamOptimizer(learning_rate=self.tf_learning_rate).minimize(self.loss)
def redSimple(input): out = tf.reshape(input, [-1, 128, 128, 1]) out = conv2d(out, 64, 3, activation_fn=tf.nn.relu, weights_initializer=xavier_initializer_conv2d()) out = batch_normalization(out) out = conv2d(out, 64, 3, activation_fn=tf.nn.relu, weights_initializer=xavier_initializer_conv2d()) out = batch_normalization(out) out = max_pool2d(out, 3, stride=2) out = conv2d(out, 128, 3, activation_fn=tf.nn.relu, weights_initializer=xavier_initializer_conv2d()) out = batch_normalization(out) out = conv2d(out, 128, 3, activation_fn=tf.nn.relu, weights_initializer=xavier_initializer_conv2d()) out = batch_normalization(out) out = max_pool2d(out, 3, stride=2) out = conv2d(out, 128, 3, activation_fn=tf.nn.relu, weights_initializer=xavier_initializer_conv2d()) out = batch_normalization(out) out = conv2d(out, 128, 3, activation_fn=tf.nn.relu, weights_initializer=xavier_initializer_conv2d()) out = batch_normalization(out) out = max_pool2d(out, 3, stride=2) out = conv2d(out, 256, 3, activation_fn=tf.nn.relu, weights_initializer=xavier_initializer_conv2d()) out = batch_normalization(out) out = conv2d(out, 256, 3, activation_fn=tf.nn.relu, weights_initializer=xavier_initializer_conv2d()) out = batch_normalization(out) out = max_pool2d(out, 3, stride=2) out = tf.contrib.layers.flatten(out) out = fully_connected(out, 1024, activation_fn=tf.nn.relu, weights_initializer=xavier_initializer()) out = fully_connected(out, 100, activation_fn=None, weights_initializer=xavier_initializer()) return out
def generator_real(self, emg_data, z, reuse=False): # (None, 300, 16) -> (1, 100) # input = self.lstm(emg_data) _ = self.lstm(emg_data, reuse) input = z print(input.shape) # input = tf.concat([input, z], 1) # print(input.shape) test = True # Generator with tf.variable_scope('generator_real', reuse=reuse): net = slim.fully_connected( input, 64 * 8 * 8, activation_fn=tf.nn.relu, weights_initializer=tflayers.xavier_initializer(), reuse=reuse) print(net) net = tf.reshape(net, [-1, 8, 8, 64]) net = slim.conv2d(net, num_outputs=128, kernel_size=1, stride=1) print(net) if not test: with slim.arg_scope( [slim.conv2d_transpose], kernel_size=3, stride=2, weights_initializer=tflayers.xavier_initializer()): with slim.arg_scope([slim.batch_norm], activation_fn=tf.nn.relu, is_training=(self.mode == 'train')): net = slim.conv2d_transpose(net, num_outputs=128) net = slim.batch_norm(net) print(net) net = slim.conv2d_transpose( net, num_outputs=256) # output : 32 x 32 net = slim.batch_norm(net) print(net) net = slim.conv2d_transpose( net, num_outputs=512) # output : 64 x 64 net = slim.batch_norm(net) print(net) net = slim.conv2d_transpose( net, num_outputs=256) # output : 128 x 128 net = slim.batch_norm(net) print(net) net = slim.conv2d_transpose( net, num_outputs=3, kernel_size=1, stride=1, weights_initializer=tflayers.xavier_initializer()) print(net) else: with slim.arg_scope([slim.conv2d_transpose], kernel_size=3, stride=1, padding='SAME', weights_initializer=tflayers. xavier_initializer_conv2d()): with slim.arg_scope([slim.batch_norm], activation_fn=tf.nn.relu): # 2x Upsampling -> Conv2D # Xavier -> Truncated normal # 16 x 16 net = tf.image.resize_images(net, [16, 16]) net = slim.conv2d_transpose(net, num_outputs=128) net = slim.batch_norm(net) print(net) # 32 x 32 net = tf.image.resize_images(net, [32, 32]) net = slim.conv2d_transpose(net, num_outputs=256) net = slim.batch_norm(net) print(net) # 64 x 64 net = tf.image.resize_images(net, [64, 64]) net = slim.conv2d_transpose(net, num_outputs=512) net = slim.batch_norm(net) print(net) # 128 x 128 net = tf.image.resize_images(net, [128, 128]) net = slim.conv2d_transpose(net, num_outputs=256) net = slim.batch_norm(net) print(net) net = slim.conv2d_transpose( net, num_outputs=1, kernel_size=1, stride=1, padding='SAME', activation_fn=tf.nn.tanh, weights_initializer=tflayers.xavier_initializer_conv2d()) print(net) return net