def __init__(self, state_size, action_size): self.states = tf.placeholder(tf.float32, shape=[None, *state_size], name="states") self.labels = tf.placeholder( tf.int32, shape=[None, 1], name="labels") # size 1 because sparse loss is used. # conv1 = layers.conv2d(self.states, filters=16, kernel_size=(8, 8), strides=(4, 4), activation='relu', # name="conv1"), # conv2 = layers.conv2d(conv1, filters=32, kernel_size=(4, 4), strides=(2, 2), activation='relu', name="conv2"), # flatten = layers.flatten(conv2), # dense = layers.dense(flatten, 256, activation='relu', name="features"), # # self.logits = layers.dense(dense, action_size, name="logits") # self.value = layers.dense(dense, 1, name="values") # conv1 = conv2d(self.states, filters=32, kernel_size=(3, 3), name='conv1') with tf.variable_scope('layers'): conv1 = layers.conv2d(self.states, filters=32, kernel_size=(3, 3), activation='relu', name='conv1') conv2 = layers.conv2d(conv1, filters=64, kernel_size=(3, 3), activation='relu', name='conv2') max_pool = layers.max_pooling2d(conv2, 2, 1, name='max_pool') drop_1 = layers.dropout(max_pool, 0.25, name='drop1') flatten = layers.flatten(drop_1, name='flatten') dense = layers.dense(flatten, 128, activation='relu', name='dense') drop2 = layers.dropout(dense, 0.5, name='drop2') logits = layers.dense(drop2, action_size, activation='softmax', name='logits') self.output = tf.nn.softmax(logits, name='output') # tf.one_hot(tf.arg_max(self.output, 1), depth=10) print(tf.arg_max(self.output, 1)) self.test = tf.one_hot(tf.arg_max(self.output, 1), depth=10) # input() self.cost = tf.losses.sparse_softmax_cross_entropy(self.labels, logits) self.acc, self.acc_op = tf.metrics.accuracy(self.labels, tf.arg_max(self.output, 1)) # self.grad = tf.gradients(self.cost, self.states, stop_gradients=[self.states]) self.grad = tf.gradients(self.cost, tf.trainable_variables()) self.optimizer = tf.train.AdamOptimizer() # print(tf.trainable_variables()) # print(tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='layers')) # print(self.grad) self.apply_grad = self.optimizer.apply_gradients( zip( self.grad, tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='layers')))
def __init__(self, game, tensor_logs_dir): self.game = game self.shape = game.board_size self.possible_moves_size = self.shape[1] self.log_writer = tf.summary.FileWriter(tensor_logs_dir + "/losses") self.checkpoint_dir = "checkpoints" create_dir(self.checkpoint_dir) self.graph = tf.Graph() self.session = tf.Session(graph=self.graph) self.saver = None with tf.Session() as temp_session: temp_session.run(tf.global_variables_initializer()) with self.graph.as_default(): self.input_boards = tf.placeholder(tf.float32, shape=[None, self.game.board.rows, self.game.board.cols]) self.dropout = tf.placeholder(tf.float32) self.isTraining = tf.placeholder(tf.bool, name="is_training") self.pi_losses_var = tf.Variable(0, dtype=tf.float32) self.v_losses_var = tf.Variable(0, dtype=tf.float32) X = tf.reshape(self.input_boards, [-1, self.shape[0], self.shape[1], 1]) h_conv1 = relu(batch_normalization(conv2d(X, 'same'), axis=3, training=self.isTraining)) h_conv2 = relu(batch_normalization(conv2d(h_conv1, 'same'), axis=3, training=self.isTraining)) h_conv3 = relu(batch_normalization(conv2d(h_conv2, 'valid'), axis=3, training=self.isTraining)) h_conv4 = relu(batch_normalization(conv2d(h_conv3, 'valid'), axis=3, training=self.isTraining)) h_conv4_flat = tf.reshape(h_conv4, [-1, config.num_channels * (self.shape[0] - 4) * (self.shape[1] - 4)]) s_fc1 = dropout(relu(batch_normalization(dense(h_conv4_flat, 1024), axis=1, training=self.isTraining)), rate=self.dropout) s_fc2 = dropout(relu(batch_normalization(dense(s_fc1, 512), axis=1, training=self.isTraining)), rate=self.dropout) self.pi = dense(s_fc2, self.possible_moves_size) self.prob = tf.nn.softmax(self.pi) self.v = tanh(dense(s_fc2, 1)) # Place holders for Predicted (pi, v)s self.predicted_pis = tf.placeholder(dtype=tf.float32, shape=[None, self.possible_moves_size]) self.predicted_vs = tf.placeholder(dtype=tf.float32, shape=[None]) # Real Losses self.loss_pi = tf.losses.softmax_cross_entropy(self.predicted_pis, self.pi) self.loss_v = tf.losses.mean_squared_error(self.predicted_vs, tf.reshape(self.v, shape=[-1, ])) self.total_loss = self.loss_pi + self.loss_v update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): self.train_step = tf.train.AdamOptimizer(config.lr).minimize(self.total_loss) self.session.run(tf.variables_initializer(self.graph.get_collection('variables')))
def _dense_block(self, x, nb_layers, nb_filter, grow_nb_filters=True, return_concat_list=False): """ Build a dense_block where the output of each conv_block is fed to subsequent ones Args: x: tensor nb_layers: the number of layers of conv_block to append to the model. nb_filter: number of filters grow_nb_filters: flag to decide to allow number of filters to grow return_concat_list: return the list of feature maps along with the actual output Returns: tensor with nb_layers of conv_block appended """ x_list = [x] for i in range(nb_layers): with tf.variable_scope('denselayer_{}'.format(i), use_resource=True): cb = self._conv_block(x, self.growth_rate) x_list.append(cb) x = concatenate([x, cb], self.axis) if grow_nb_filters: nb_filter += self.growth_rate if self.dropout_rate: x = dropout(x, self.dropout_rate, training=self.training) if return_concat_list: return x, nb_filter, x_list else: return x, nb_filter
def _x(ip): x = batch_normalization(ip, **self.bn_kwargs) x = tf.nn.relu(x) if self.bottleneck: inter_channel = nb_filter * 4 x = conv2d(x, inter_channel, (1, 1), kernel_initializer='he_normal', padding='same', use_bias=False, **self.conv_kwargs) x = batch_normalization(x, **self.bn_kwargs) x = tf.nn.relu(x) x = conv2d(x, nb_filter, (3, 3), kernel_initializer='he_normal', padding='same', use_bias=False, **self.conv_kwargs) if self.dropout_rate: x = dropout(x, self.dropout_rate, training=self.training) return x
def deconv_block(self, input, filters, conv, padding, scope): with tf.variable_scope(scope): deconv1 = conv2d_transpose(input, filters, kernel_size=(3, 3), strides=[2, 2], padding=padding) deconv_shape = tf.shape(deconv1) conv_shape = tf.shape(conv) offsets = [ 0, (conv_shape[1] - deconv_shape[1]) // 2, (conv_shape[2] - deconv_shape[2]) // 2, 0 ] size = [-1, deconv_shape[1], deconv_shape[2], filters] conv_crop = tf.slice(conv, offsets, size) conv1 = tf.concat([deconv1, conv_crop], 3) bn = batch_normalization(conv1) drop = dropout(bn, .25) conv2 = conv2d(drop, filters, kernel_size=(3, 3), activation=tf.nn.relu, name='middle1', padding="SAME") conv3 = conv2d(conv2, filters, kernel_size=(3, 3), activation=tf.nn.relu, name='middle2', padding="SAME") return conv3
def misconception_model(inputs, filters_list, kernel_size, strides_list, training, objective_functions, sub_filters=128, sub_layers=2, dropout_rate=0.5, virtual_batch_size=None, feature_means=None, feature_stds=None): """ A misconception tower. Args: input: a tensor of size [batch_size, 1, width, depth]. window_size: the width of the conv and pooling filters to apply. depth: the depth of the output tensor. levels: the height of the tower in misconception layers. objective_functions: a list of objective functions to add to the top of the network. is_training: whether the network is training. Returns: a tensor of size [batch_size, num_classes]. """ layers = [] net = inputs if feature_means is not None: net = net - tf.constant(feature_means)[None, None, :] if feature_stds is not None: net = net / (tf.constant(feature_stds) + 1e-6) layers.append(net) for filters, strides in zip(filters_list, strides_list): net = misconception_with_bypass(net, filters, kernel_size, strides, training, virtual_batch_size=virtual_batch_size) layers.append(net) outputs = [] for ofunc in objective_functions: onet = net for _ in range(sub_layers - 1): onet = ly.conv1d(onet, sub_filters, 1, activation=None, use_bias=False) onet = ly.batch_normalization( onet, training=training, virtual_batch_size=virtual_batch_size) onet = tf.nn.relu(onet) onet = ly.conv1d(onet, sub_filters, 1, activation=tf.nn.relu) onet = ly.flatten(onet) # onet = ly.dropout(onet, training=training, rate=dropout_rate) outputs.append(ofunc.build(onet)) return outputs, layers
def build_model(embs, lr_w, dropout_on=tf.constant(False), min_cut=min_cut, max_cut=max_cut, cut_size=cut_size): if embedding_dropout_type == 'none' or dropout_rate<0.01: pass elif embedding_dropout_type=='standard': embs = tf.layaers.dropout(embs, rate=dropout_rate, training=dropout_on) elif embedding_dropout_type=='field': # random dropout some fields embs = tf.layers.dropout( embs, rate=dropout_rate, training=dropout_on, noise_shape=[embs.shape[1], tf.constant(1)]) kwargs['scaling_factor'] = tf.cond(dropout_on, lambda: 1.-dropout_rate, lambda: 1.) elif embedding_dropout_type=='embedding': # random dropout some dimensions of embeddings embs = tf.layers.dropout( embs, rate=dropout_rate, training=dropout_on, noise_shape=[tf.constant(1), embs.shape[2]]) kwargs['scaling_factor'] = tf.cond(dropout_on, lambda: 1.-dropout_rate, lambda: 1.) else: print("Unknown embedding_dropout_type! Embedding Dropout is bypassed.") self.embs = embs latent = get_latent(embs, lr_w, **kwargs) latent = dropout(latent, dropout_rate, training=dropout_on) logit = tf.layers.dense(latent, 1, name='last_fc', kernel_initializer=tf.initializers.he_uniform()) return logit, logit
def add_inference(self, inputs, training, nclass): df = 'channels_first' if self.data_format == 'NCHW' else 'channels_last' conv1 = layers.conv2d(inputs=inputs, filters=32, kernel_size=[5, 5], padding="same", data_format=df, activation=tf.nn.relu) pool1 = layers.max_pooling2d(inputs=conv1, pool_size=[2, 2], strides=2, data_format=df) conv2 = layers.conv2d(inputs=pool1, filters=64, kernel_size=[3, 3], padding="same", data_format=df, activation=tf.nn.relu) pool2 = layers.max_pooling2d(inputs=conv2, pool_size=[2, 2], strides=2, data_format=df) pool2_flat = tf.reshape(pool2, [-1, 7 * 7 * 64]) dense = layers.dense(inputs=pool2_flat, units=1024, activation=tf.nn.relu) dropout = layers.dropout(inputs=dense, rate=0.4, training=training) logits = layers.dense(inputs=dropout, units=nclass) return logits
def build_discriminator(image, reuse=False): with tf.variable_scope("discriminator") as scope: if reuse: tf.get_variable_scope().reuse_variables() # t = image t = conv2d(inputs=t, filters=64, kernel_size=[5, 5], strides=2, padding="same", activation=my_leaky_relu) t = batch_normalization(inputs=t) t = conv2d(inputs=t, filters=128, kernel_size=[5, 5], strides=2, padding="same", activation=my_leaky_relu) t = batch_normalization(inputs=t) t = dropout(inputs=t, rate=DROP_RATE) t = flatten(inputs=t) t = dense(inputs=t, units=1, activation=tf.sigmoid) decision = t #print("\nD output shape: {}".format(decision.shape)) return decision
def mixed_7a(self, x): with tf.variable_scope("mixed_7a"): with tf.variable_scope("branch0"): x0 = self.basic_conv2d(x, 256, kernel_size=1, stride=1, padding="same", namescope="conv1", use_bias=False) x0 = self.basic_conv2d(x0, 384, kernel_size=3, stride=2, padding="same", namescope="conv2", use_bias=False) with tf.variable_scope("branch1"): x1 = self.basic_conv2d(x, 256, kernel_size=1, stride=1, padding="same", namescope="conv1", use_bias=False) x1 = self.basic_conv2d(x1, 288, kernel_size=3, stride=2, padding="same", namescope="conv2", use_bias=False) with tf.variable_scope("branch2"): x2 = self.basic_conv2d(x, 256, kernel_size=1, stride=1, padding="same", namescope="conv1", use_bias=False) x2 = self.basic_conv2d(x2, 288, kernel_size=3, stride=1, padding="same", namescope="conv2", use_bias=False) x2 = self.basic_conv2d(x2, 320, kernel_size=3, stride=2, padding="same", namescope="conv3", use_bias=False) with tf.variable_scope("branch3"): x3 = layers.max_pooling2d(x, 3, strides=2, padding="same") x = tf.concat([x0, x1, x2, x3], axis=-1) x = layers.dropout(x, noise_shape=[None, 1, 1, None]) return x
def mixed_5b(self, x): with tf.variable_scope("mixed_5b"): with tf.variable_scope("branch0"): x0 = self.basic_conv2d(x, 96, kernel_size=1, stride=1, padding="same", namescope="conv1", use_bias=False) with tf.variable_scope("branch1"): x1 = self.basic_conv2d(x, 48, kernel_size=1, stride=1, padding="same", namescope="conv1", use_bias=False) x1 = self.basic_conv2d(x1, 64, kernel_size=5, stride=1, padding="same", namescope="conv2", use_bias=False) with tf.variable_scope("branch2"): x2 = self.basic_conv2d(x, 64, kernel_size=1, stride=1, padding="same", namescope="conv1", use_bias=False) x2 = self.basic_conv2d(x2, 96, kernel_size=3, stride=1, padding="same", namescope="conv2", use_bias=False) x2 = self.basic_conv2d(x2, 96, kernel_size=3, stride=1, padding="same", namescope="conv3", use_bias=False) with tf.variable_scope("branch3"): x3 = layers.average_pooling2d(x, 3, strides=1, padding="same") x3 = self.basic_conv2d(x3, 64, kernel_size=1, stride=1, padding="same", namescope="conv1", use_bias=False) x = tf.concat([x0, x1, x2, x3], axis=-1) x = layers.dropout(x, noise_shape=[None, 1, 1, None]) return x
def create_discriminator(self, img, rate, reuse=None): # hard code all layer params # 1. small kernels would lead to loss become 0 quickly # try a very large one # 2. More filters help the performance # especially a lot more filters in generator with tf.variable_scope("discriminator", reuse=reuse): x = conv2d(img, 128, (5, 5), strides=(2, 2), padding='same', data_format='channels_last', dilation_rate=(1, 1), activation=tf.nn.leaky_relu, use_bias=True) x = dropout(x, rate) # assert tf.shape(x) == (64,64,1) x = conv2d(x, 128, (5, 5), strides=(2, 2), padding='same', data_format='channels_last', dilation_rate=(1, 1), activation=tf.nn.leaky_relu, use_bias=True) x = dropout(x, rate) x = conv2d(x, 128, (5, 5), strides=(1, 1), padding='same', data_format='channels_last', dilation_rate=(1, 1), activation=tf.nn.leaky_relu, use_bias=True) x = dropout(x, rate) x = flatten(x) x = fully_connected(x, 512, activation_fn=tf.nn.leaky_relu) d_logits = fully_connected(x, 1, activation_fn=None) d_prob = tf.sigmoid(d_logits) return d_prob, d_logits
def create_generator(self, z, rate, is_training, reuse=None): # hard code all layer params upsample_power = 5 # 2**5=32 momentum = 0.99 with tf.variable_scope("generator", reuse=reuse): # Since "mode collapse" issue, feature of FC needs to be small # each layer come with dropout and batch_norm to improve the performance x = fully_connected(z, 4 * 4 * 1, activation_fn=tf.nn.leaky_relu) # x = dropout(x, rate = rate,training=is_training) x = dropout(x, rate=rate) x = tf.contrib.layers.batch_norm(x, is_training=is_training, decay=momentum) x = tf.reshape(x, shape=(-1, 4, 4, 1)) for _ in range(upsample_power): x = conv2d_transpose(x, 128, (5, 5), strides=(2, 2), padding='same', data_format='channels_last', activation=tf.nn.leaky_relu, use_bias=True) # x = dropout(x, rate = rate,training=is_training) x = dropout(x, rate=rate) x = tf.contrib.layers.batch_norm(x, is_training=is_training, decay=momentum) x = conv2d_transpose(x, 1, (5, 5), strides=(1, 1), padding='same', data_format='channels_last', activation=tf.sigmoid, use_bias=True) # assert tf.shape(x) == (128,128,1) # output is a matrix with -1 to 1 return x
def transpose_conv1d_layer(x, n_out, kernel_size, stride=1, activation=ACTIVATION, regularize=True, use_bias=True, drop_rate=0.0, batch_norm=BATCH_NORM, training=True, name=None, reuse=None): if batch_norm: if name: x = batch_norm_layer(x, training, name=name + '_bn', reuse=reuse) else: x = batch_norm_layer(x, training, name=name, reuse=reuse) #wt_init = tf.truncated_normal_initializer(stddev=0.2) #bi_init = tf.truncated_normal_initializer(mean=BIAS_SHIFT,stddev=0.01) wt_init = None bi_init = None if regularize: wt_reg = WT_REG bi_reg = BI_REG else: wt_reg = None bi_reg = None x_tmp = tf.expand_dims(x, 3) x_tmp = tf.transpose(x_tmp, (0, 1, 3, 2)) y_tmp = layers.conv2d_transpose(x_tmp, n_out, kernel_size=[kernel_size, 1], strides=(stride, 1), padding='same', data_format='channels_last', activation=activation, use_bias=add_bias, kernel_initializer=wt_init, bias_initializer=bi_init, kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, trainable=True, name=None, reuse=None) y = tf.reduce_sum(y_tmp, axis=2) y = layers.dropout(y, rate=drop_rate, training=training) return y
def model(inputs, is_training, data_format, num_classes): net = conv2d(inputs=inputs, filters=96, kernel_size=[7, 7], strides=2, padding='valid', data_format=data_format, activation=tf.nn.relu, use_bias=True, kernel_initializer=tf.variance_scaling_initializer(), bias_initializer=tf.zeros_initializer()) net = max_pooling2d(inputs=net, pool_size=[3, 3], strides=2, data_format=data_format) net = fire_module(net, 16, 64, data_format) net = fire_module(net, 16, 64, data_format) net = fire_module(net, 32, 128, data_format) net = max_pooling2d(inputs=net, pool_size=[3, 3], strides=2, data_format=data_format) net = fire_module(net, 32, 128, data_format) net = fire_module(net, 48, 192, data_format) net = fire_module(net, 48, 192, data_format) net = fire_module(net, 64, 256, data_format) net = max_pooling2d(inputs=net, pool_size=[3, 3], strides=2, data_format=data_format) net = fire_module(net, 64, 256, data_format) net = dropout(inputs=net, rate=0.5, training=is_training) net = conv2d( inputs=net, filters=num_classes, kernel_size=[1, 1], strides=1, padding='valid', # no padding eqv. to pad=1 for 1x1 conv? data_format=data_format, activation=tf.nn.relu, use_bias=True, kernel_initializer=tf.initializers.random_normal(mean=0.0, stddev=0.01), bias_initializer=tf.zeros_initializer()) net = average_pooling2d(inputs=net, pool_size=[13, 13], strides=1, data_format=data_format) # TODO fix for data_format later logits = tf.squeeze(net, [2, 3]) return logits
def dense_layer(x, n_out, activation=ACTIVATION, drop_rate=0.0, reuse=None, name=None, batch_norm=False, regularize=False, training=True): wt_init = None bi_init = None #wt_init = tf.truncated_normal_initializer(stddev=0.15) #bi_init = tf.truncated_normal_initializer(mean=BIAS_SHIFT,stddev=0.25) #wt_init = tf.truncated_normal_initializer(stddev=0.05) #bi_init = tf.truncated_normal_initializer(mean=BIAS_SHIFT,stddev=0.35) if regularize: wt_reg = WT_REG bi_reg = BI_REG else: wt_reg = None bi_reg = None # Apply dense layer y = layers.dense(x, n_out, activation=None, use_bias=True, kernel_initializer=wt_init, bias_initializer=bi_init, kernel_regularizer=wt_reg, bias_regularizer=bi_reg, trainable=True, name=name, reuse=reuse) # Apply batch normalization if batch_norm: if name: y = batch_norm_layer(y, training, name=name + '_bn', reuse=reuse) else: y = batch_norm_layer(y, training, name=name, reuse=reuse) # Apply dropout y = layers.dropout(y, rate=drop_rate, training=training) # Apply activation if activation is not None: y = activation(y) return y
def misconception_fishing(inputs, filters_list, kernel_size, strides_list, objective_function, training, pre_filters=128, post_filters=128, post_layers=1, dropout_rate=0.5, internal_dropout_rate=0.5, other_objectives=(), feature_means=None, feature_stds=None): _, layers = misconception_model(inputs, filters_list, kernel_size, strides_list, training, other_objectives, sub_filters=post_filters, sub_layers=2, dropout_rate=internal_dropout_rate, feature_means=feature_means, feature_stds=feature_stds) expanded_layers = [] for i, lyr in enumerate(layers): lyr = ly.conv1d(lyr, pre_filters, 1, activation=None) lyr = ly.batch_normalization(lyr, training=training) lyr = tf.nn.relu(lyr) expanded_layers.append(repeat_tensor(lyr, 2**i)) embedding = tf.add_n(expanded_layers) for _ in range(post_layers - 1): embedding = ly.conv1d(embedding, post_filters, 1, activation=None, use_bias=False) embedding = ly.batch_normalization(embedding, training=training) embedding = tf.nn.relu(embedding) embedding = ly.conv1d(embedding, post_filters, 1, activation=tf.nn.relu) embedding = ly.dropout(embedding, training=training, rate=dropout_rate) fishing_outputs = ly.conv1d(embedding, 1, 1, activation=None) return objective_function.build(fishing_outputs)
def conv1d_layer(x, n_out, kernel_size, stride=1, activation=ACTIVATION, regularize=True, use_bias=True, drop_rate=0.0, batch_norm=BATCH_NORM, training=True, name=None, reuse=None): if batch_norm: if name: x = batch_norm_layer(x, training, name=name + '_bn', reuse=reuse) else: x = batch_norm_layer(x, training, name=name, reuse=reuse) #wt_init = tf.truncated_normal_initializer(stddev=0.2) #bi_init = tf.truncated_normal_initializer(mean=BIAS_SHIFT,stddev=0.01) wt_init = None bi_init = None if regularize: wt_reg = WT_REG bi_reg = BI_REG else: wt_reg = None bi_reg = None y = layers.conv1d(x, n_out, kernel_size, strides=stride, padding='same', data_format='channels_last', dilation_rate=1, activation=activation, use_bias=use_bias, kernel_initializer=wt_init, bias_initializer=bi_init, kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, trainable=True, name=None, reuse=None) y = layers.dropout(y, rate=drop_rate, training=training) return y
def build(self, input): self.conv1 = self.conv_block(input, 64, "conv1") pool1 = max_pooling2d(self.conv1, pool_size=(2, 2), strides=2) drop1 = dropout(pool1, .25) self.conv2 = self.conv_block(drop1, 128, 'conv2') pool2 = max_pooling2d(self.conv2, pool_size=(2, 2), strides=2) drop2 = dropout(pool2, .25) self.conv3 = self.conv_block(drop2, 256, 'conv3') pool3 = max_pooling2d(self.conv3, pool_size=(2, 2), strides=2) drop3 = dropout(pool3, .25) self.conv4 = self.conv_block(drop3, 512, 'conv4') pool4 = max_pooling2d(self.conv4, pool_size=(2, 2), strides=2) drop4 = dropout(pool4, .25) self.conv5 = conv2d(drop4, 1024, kernel_size=(3, 3), activation=tf.nn.relu, padding='SAME') self.conv5_2 = conv2d(self.conv5, 1024, kernel_size=(3, 3), activation=tf.nn.relu, padding='SAME') self.deconv4 = self.deconv_block(self.conv5_2, 512, self.conv4, "SAME", 'deconv4') self.deconv3 = self.deconv_block(self.deconv4, 256, self.conv3, "VALID", 'deconv3') self.deconv2 = self.deconv_block(self.deconv3, 128, self.conv2, "SAME", 'deconv2') self.deconv1 = self.deconv_block(self.deconv2, 64, self.conv1, "VALID", 'deconv1') self.output = conv2d(self.deconv1, filters=1, kernel_size=1, name='logits')
def block8(self, x): initial = x with tf.variable_scope("block8"): with tf.variable_scope("branch0"): x0 = self.basic_conv2d(x, 192, kernel_size=1, stride=1, padding="same", namescope="conv1", use_bias=False) with tf.variable_scope("branch1"): x1 = self.basic_conv2d(x, 192, kernel_size=1, stride=1, padding="same", namescope="conv1", use_bias=False) x1 = self.basic_conv2d(x1, 224, kernel_size=(1, 3), stride=1, padding="same", namescope="conv2", use_bias=False) x1 = self.basic_conv2d(x1, 256, kernel_size=(3, 1), stride=1, padding="same", namescope="conv3", use_bias=False) x = tf.concat([x0, x1], axis=-1) x = layers.conv2d(x, filters=2080, kernel_size=1, padding="same", strides=1, use_bias=True) x = initial + x x = layers.dropout(x, noise_shape=[None, 1, 1, None]) return x
def vgg(inputs, ys, knn_inputs, knn_ys): # block 1 x = conv2d(inputs, filters=64, kernel_size=(3, 3), padding='same', activation=ReLU) x = conv2d(x, filters=64, kernel_size=(3, 3), padding='same', activation=ReLU) x = max_pooling2d(x, pool_size=(2, 2), strides=(2, 2)) # block 2 x = conv2d(x, filters=128, kernel_size=(3, 3), padding='same', activation=ReLU) x = conv2d(x, filters=128, kernel_size=(3, 3), padding='same', activation=ReLU) x = max_pooling2d(x, pool_size=(2, 2), strides=(2, 2)) # final x = flatten(x) x = dense(x, units=1024, activation=ReLU) x = dropout(x, rate=0.5, training=) x = dense(x, units=1024, activation=ReLU) x = dense(x, units=self.data_loader.num_outputs) return x
def dense1d_layer(x, n_out, activation=ACTIVATION, regularize=True, use_bias=True, drop_rate=0.0, batch_norm=BATCH_NORM, training=True, name=None, reuse=None): if batch_norm: if name: x = batch_norm_layer(x, training, name=name + '_bn', reuse=reuse) else: x = batch_norm_layer(x, training, name=name, reuse=reuse) #wt_init = tf.truncated_normal_initializer(stddev=0.2) #bi_init = tf.truncated_normal_initializer(mean=BIAS_SHIFT,stddev=0.01) wt_init = None bi_init = None if regularize: wt_reg = WT_REG bi_reg = BI_REG else: wt_reg = None bi_reg = None y = layers.dense(x, n_out, activation=activation, use_bias=True, kernel_initializer=wt_init, bias_initializer=bi_init, kernel_regularizer=wt_reg, bias_regularizer=bi_reg, trainable=True, name=name, reuse=reuse) y = layers.dropout(y, rate=drop_rate, training=training) return y
def lstm_model(features, mode): """ cnn model structure :param features: images :return: predicts """ input_layer = tf.unstack(value=features, num=28, axis=1, name='input') lstm_cell = rnn.BasicLSTMCell(num_units=128, name='lstm') lstm_out, _ = rnn.static_rnn( lstm_cell, input_layer, dtype=tf.float32, ) flatten_layer = layers.flatten(lstm_out[-1], name='flatten') dense_layer = layers.dense(inputs=flatten_layer, units=512, name='dense') dropout = layers.dropout(inputs=dense_layer, rate=0.5, training=(mode == tf.estimator.ModeKeys.TRAIN), name='dropout') logits = layers.dense(inputs=dropout, units=10, name='logits') return logits
def cnn_model(features, mode): """ cnn model structure :param features: images :return: predicts """ input_layer = tf.reshape(features, shape=[-1, 28, 28, 1], name='input') # conv1 # trainable can change in middle of training conv1 = layers.conv2d(inputs=input_layer, filters=32, kernel_size=[3, 3], padding="same", activation=tf.nn.relu, name='conv1') pool1 = layers.max_pooling2d(inputs=conv1, pool_size=[2, 2], strides=2, name='pool1') # conv2 conv2 = layers.conv2d(inputs=pool1, filters=64, kernel_size=[5, 5], padding="same", activation=tf.nn.relu, name='conv2') pool2 = layers.max_pooling2d(inputs=conv2, pool_size=[2, 2], strides=2, name='pool2') # fully connected layer pool2_flat = layers.flatten(pool2, name='flatten') dense = layers.dense(inputs=pool2_flat, units=512, activation=tf.nn.relu, name='dense_layer') dropout = layers.dropout(inputs=dense, rate=0.4, training=(mode == tf.estimator.ModeKeys.TRAIN), name='dropout') # output layer logits = tf.layers.dense(inputs=dropout, units=10, name='logits') return logits
def inference(self, input): output = conv2d(input, self.output_channels, kernel_size=(3, 1), strides=1, padding="SAME") output = conv2d(output, self.output_channels, kernel_size=(1, 3), strides=1, padding="SAME", activation=None) output = batch_normalization(output, self.training) output = tf.nn.relu(output) output = conv2d(output, self.output_channels, kernel_size=(3, 1), strides=1, dilation_rate=(self.dilated, self.dilated), padding="SAME") output = conv2d(output, self.output_channels, kernel_size=(1, 3), strides=1, dilation_rate=(self.dilated, self.dilated), padding="SAME", activation=None) output = batch_normalization(output, self.training) if self.dropprob != 0: output = dropout(output, rate=self.dropprob, training=self.training) return tf.nn.relu(output + input)
def googlenet(X, config): """ googlenet implementation. """ with tf.name_scope('googlenet'): block1 = conv2d(X, 64, 7, strides=2, padding='SAME') block1 = tf.nn.relu(block1) block1 = max_pooling2d(block1, 3, 2, padding='SAME') block1 = tf.nn.local_response_normalization(block1) block2 = conv2d(block1, 64, 1, padding='SAME') block2 = tf.nn.relu(block2) block2 = conv2d(block2, 192, 3, padding='SAME') block2 = tf.nn.relu(block2) block2 = tf.nn.local_response_normalization(block2) block2 = max_pooling2d(block2, 3, 2, padding='SAME') block2 = tf.nn.relu(block2) # inception x2 block3 = inceptionBlock(block2, c1=64, c3_r=96, c3=128, c5_r=16, c5=32, p3_r=32) block3 = inceptionBlock(block3, c1=128, c3_r=128, c3=192, c5_r=32, c5=96, p3_r=64) block3 = max_pooling2d(block3, 3, 2, padding='SAME') # inception x5 block4 = inceptionBlock(block3, c1=192, c3_r=96, c3=208, c5_r=16, c5=48, p3_r=64) block4 = inceptionBlock(block4, c1=160, c3_r=112, c3=224, c5_r=24, c5=64, p3_r=64) block4 = inceptionBlock(block4, c1=128, c3_r=128, c3=256, c5_r=24, c5=64, p3_r=64) block4 = inceptionBlock(block4, c1=112, c3_r=144, c3=288, c5_r=32, c5=64, p3_r=64) block4 = inceptionBlock(block4, c1=256, c3_r=160, c3=320, c5_r=32, c5=128, p3_r=128) block4 = max_pooling2d(block4, 3, 2, padding='SAME') # inception x2 with average pooling block5 = inceptionBlock(block4, c1=256, c3_r=160, c3=320, c5_r=32, c5=128, p3_r=128) block5 = inceptionBlock(block5, c1=384, c3_r=192, c3=384, c5_r=48, c5=128, p3_r=128) block5 = average_pooling2d(block5, 7, 1, padding='SAME') block5 = dropout(block5, rate=0.4, training=config.training) logits = flatten(block5) logits = dense(logits, config.NUM_CLASSES) return logits, None
def cnn_model(features, labels, mode): """ CNN MODEL""" """ ((Conv2D)*2-->MaxPool-->Dropout)*2-->Dense-->dropout-->softmax """ # input layer input_layer = tf.reshape(features["x"], (-1, 28, 28, 1)) # conv1 conv1 = layers.conv2d(inputs=input_layer, filters=32, kernel_size=(5, 5), padding="same", data_format="channels_last", activation=tf.nn.relu) # conv2 conv2 = layers.conv2d(inputs=conv1, filters=32, kernel_size=(5, 5), padding="same", data_format="channels_last", activation=tf.nn.relu) # max1 max1 = layers.max_pooling2d(inputs=conv2, pool_size=(2, 2), strides=2) # drop1 drop1 = layers.dropout(inputs=max1, rate=0.25, training=(mode == tf.estimator.ModeKeys.TRAIN)) # conv3 conv3 = layers.conv2d(inputs=drop1, filters=64, kernel_size=(3, 3), padding="same", data_format="channels_last", activation=tf.nn.relu) # conv4 conv4 = layers.conv2d(inputs=conv3, filters=64, kernel_size=(3, 3), padding="same", data_format="channels_last", activation=tf.nn.relu) # max1 max2 = layers.max_pooling2d(inputs=conv4, pool_size=(2, 2), strides=2) # drop1 drop2 = layers.dropout(inputs=max2, rate=0.25, training=(mode == tf.estimator.ModeKeys.TRAIN)) # Dense layer flat1 = tf.reshape(drop2, (-1, 7 * 7 * 64)) dense1 = layers.dense(inputs=flat1, units=256, activation=tf.nn.relu) drop3 = layers.dropout(inputs=dense1, rate=0.5) logits = layers.dense(inputs=drop3, units=10) predictions = { "classes": tf.argmax(input=logits, axis=1), "probabilities": tf.nn.softmax(logits=logits, name="softmax_tensor"), } # type: Dict[str, Any] # if mode is eval of predict return this dict if mode == tf.estimator.ModeKeys.PREDICT: return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions) # Calculate the loss loss = tf.losses.softmax_cross_entropy(onehot_labels=tf.one_hot( indices=tf.cast(labels, tf.int32), depth=10), logits=logits) # Optimize for training process if mode == tf.estimator.ModeKeys.TRAIN: optimizer = tf.train.AdamOptimizer() train_op = optimizer.minimize(loss=loss, global_step=tf.train.get_global_step()) return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op) eval_metric_ops = { "accuracy": tf.metrics.accuracy(labels=labels, predictions=predictions["classes"]) } return tf.estimator.EstimatorSpec(mode=mode, loss=loss, eval_metric_ops=eval_metric_ops)
def __call__(self, inputs): return layers.dropout(inputs, training=self._is_training)
def VGG_net(x, is_training, params): with tf.variable_scope('vgg_16', 'vgg_16', [x]) as sc: layer1 = ConvBatch(x, 64, is_training, "layer1") layer2 = ConvBatch(layer1, 64, is_training, "layer2") maxpool_2 = layers.max_pooling2d(layer2, pool_size=2, strides=2) dropout_2 = layers.dropout(maxpool_2, rate=params['dropout'], training=is_training) layer3 = ConvBatch(dropout_2, 128, is_training, "layer3") layer4 = ConvBatch(layer3, 128, is_training, "layer4") maxpool_4 = layers.max_pooling2d(layer4, pool_size=2, strides=2) dropout_4 = layers.dropout(maxpool_4, rate=params['dropout'], training=is_training) layer5 = ConvBatch(dropout_4, 256, is_training, "layer5") layer6 = ConvBatch(layer5, 256, is_training, "layer6") layer7 = ConvBatch(layer6, 256, is_training, "layer7") maxpool_7 = layers.max_pooling2d(layer7, pool_size=2, strides=2) dropout_7 = layers.dropout(maxpool_7, rate=params['dropout'], training=is_training) layer8 = ConvBatch(dropout_7, 512, is_training, "layer8") layer9 = ConvBatch(layer8, 512, is_training, "layer9") layer10 = ConvBatch(layer9, 512, is_training, "layer10") maxpool_10 = layers.max_pooling2d(layer10, pool_size=2, strides=2) dropout_10 = layers.dropout(maxpool_10, rate=params['dropout'], training=is_training) layer11 = ConvBatch(dropout_10, 512, is_training, "layer11") layer12 = ConvBatch(layer11, 512, is_training, "layer12") layer13 = ConvBatch(layer12, 512, is_training, "layer13") maxpool_13 = layers.max_pooling2d(layer13, pool_size=2, strides=2) dropout_13 = layers.dropout(maxpool_13, rate=params['dropout'], training=is_training) flat_output = layers.flatten(dropout_13) dense14 = DenseBatch(flat_output, 4096, is_training, "dense14") dropout_14 = layers.dropout(dense14, rate=params['dropout'], training=is_training) dense15 = DenseBatch(dropout_14, 4096, is_training, "dense15") logits = layers.dense( dense15, 100, activation=None, kernel_initializer=tf.contrib.layers.xavier_initializer()) return logits
def main(): # parsing parser = argparse.ArgumentParser() parser.add_argument( "input", help="pretrained sqznet weight file, e.g., sqz_full.mat") parser.add_argument("output", help="folder for generated model") parser.add_argument( "train", help="train folder dir with images in different subfolders") parser.add_argument("--test", help="optional test folder") args = parser.parse_args() # data loading label2class = {"green": 0, "yellow": 1, "red": 2, "nolight": 3} train_dir = args.train test_dir = args.test or args.train train_imgs, train_targets = load_imgs_labels(train_dir, label2class) print("loaded training images", train_imgs.shape) if test_dir != train_dir: test_imgs, test_targets = load_imgs_labels(test_dir, label2class) else: test_imgs, test_targets = train_imgs, train_targets print("loaded test images", test_imgs.shape) ## model # load weights as numpy array sqz_wts, sqz_mean = load_net(args.input) params = {"dropout_rate": 0.5, "n_classes": 4, "learning_rate": 5e-4} tf.reset_default_graph() image = tf.placeholder(tf.float32, shape=[None, 227, 227, 3]) labels = tf.placeholder(tf.int32, shape=[None]) training = tf.placeholder(tf.bool, shape=[]) sqz_net = net_preloaded(sqz_wts, image, 'max', False) # weights as tf.constant bottleneck_feats = sqz_net['fire9/concat_conc'] # ?x13x13x512 output = layers.dropout(bottleneck_feats, rate=params["dropout_rate"], training=training) output = layers.conv2d(output, params["n_classes"], kernel_size=[13, 13], strides=[1, 1]) logits = tf.reshape(output, [-1, 4]) predictions = tf.argmax(logits, axis=1) loss = tf.reduce_mean( tf.losses.sparse_softmax_cross_entropy(labels, logits)) match = tf.nn.in_top_k(logits, labels, 1) accuracy = tf.reduce_mean(tf.cast(match, tf.float32)) optimizer = tf.train.AdamOptimizer(learning_rate=params["learning_rate"]) train_op = optimizer.minimize(loss, global_step=tf.train.get_global_step()) saver = tf.train.Saver() tf.add_to_collection("model_input", image) tf.add_to_collection("model_input", labels) tf.add_to_collection("model_input", training) tf.add_to_collection("model_output", logits) tf.add_to_collection("model_output", predictions) ## training batch_size = 256 n_epochs = train_imgs.shape[0] // batch_size * 10 train_batches = make_tl_generator(train_imgs, train_targets, batch_size=batch_size) test_batches = make_tl_generator(test_imgs, test_targets, batch_size=batch_size) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) for epoch in range(n_epochs): x_batch, y_batch = next(train_batches) _, loss_val = sess.run([train_op, loss], feed_dict={ image: x_batch, labels: y_batch, training: True }) if epoch % 50 == 0: print(epoch, loss_val) model_file = saver.save(sess, path.join(args.output, "tl_model")) print("model saved to", model_file) ## test accuracy test_preds, test_accuracy = [], [] for i in range(0, test_imgs.shape[0], batch_size): tp, ta = sess.run( [predictions, accuracy], feed_dict={ image: test_imgs[i:i + batch_size], labels: test_targets[i:i + batch_size], training: False }) test_preds.append(tp) test_accuracy.append(ta) test_yhat = np.concatenate(test_preds) test_accuracy = np.mean(test_yhat == test_targets) print("test accuracy:", test_accuracy)