def build_model(self): X_list = [tf.placeholder(tf.float32, [None, self.dims[0]], name="X_%d" % (t + 1)) for t in range(self.n_tasks)] Y_list = [tf.placeholder(tf.float32, [None, self.n_classes], name="Y_%d" % (t + 1)) for t in range(self.n_tasks)] is_training = tf.placeholder(tf.bool, name="is_training") for t, (X, Y) in enumerate(zip(X_list, Y_list)): bottom = X excl_bottom = X if self.use_set_based_mask else None for i in range(1, self.n_layers): w = self.get_variable('layer%d' % i, 'weight', True) b = self.get_variable('layer%d' % i, 'biases', True) bottom = tf.nn.relu(tf.matmul(bottom, w) + b) if self.use_set_based_mask: bottom, residuals = Mask( i - 1, self.mask_alpha[i - 1], self.mask_not_alpha[i - 1], mask_type=self.mask_type, ).get_masked_tensor(bottom, is_training, with_residuals=True) mask = residuals["cond_mask"] with tf.control_dependencies([mask]): excl_bottom = tf.nn.relu(tf.matmul(excl_bottom, w) + b) excl_bottom = Mask.get_exclusive_masked_tensor(excl_bottom, mask, is_training) w = self.get_variable("layer%d" % self.n_layers, "weight_%d" % (t + 1), True) b = self.get_variable("layer%d" % self.n_layers, "biases_%d" % (t + 1), True) y = tf.matmul(bottom, w) + b yhat = tf.nn.sigmoid(y) self.yhat_list.append(yhat) loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=y, labels=Y)) self.partial_loss_list.append(loss) if self.use_set_based_mask: excl_y = tf.matmul(excl_bottom, w) + b excl_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=excl_y, labels=Y)) self.excl_partial_loss_list.append(excl_loss) return X_list, Y_list, is_training
def build_model_for_retraining(self, flags): xn_filters, xsize = self.conv_dims[0], self.conv_dims[1] X = tf.get_default_graph().get_tensor_by_name("X:0") Y = tf.get_default_graph().get_tensor_by_name("Y:0") keep_prob = tf.get_default_graph().get_tensor_by_name("keep_prob:0") is_training = tf.get_default_graph().get_tensor_by_name("is_training:0") excl_X = tf.placeholder(tf.float32, [None, xsize, xsize, xn_filters], name="excl_X") excl_Y = tf.placeholder(tf.float32, [None, self.n_classes], name="excl_Y") h_conv = X excl_h_conv = excl_X if self.use_set_based_mask else None for conv_num, i in enumerate(range(2, len(self.conv_dims), 2)): w_conv = self.get_variable("conv%d" % (i // 2), "weight", True) b_conv = self.get_variable("conv%d" % (i // 2), "biases", True) h_conv = tf.nn.conv2d(h_conv, w_conv, strides=[1, 1, 1, 1], padding="SAME") + b_conv if self.use_set_based_mask: excl_h_conv = tf.nn.conv2d(excl_h_conv, w_conv, strides=[1, 1, 1, 1], padding="SAME") + b_conv # activation h_conv = tf.nn.relu(h_conv) if self.use_set_based_mask: ndarray_hard_mask = np.ones(shape=h_conv.get_shape().as_list()[-1:]) indices_to_zero = np.asarray(list(self.layer_to_removed_unit_set["conv{}".format(i // 2)])) ndarray_hard_mask[indices_to_zero] = 0 tf_hard_mask = tf.constant(ndarray_hard_mask, dtype=tf.float32) h_conv = Mask( 100 + conv_num, 0, 0, mask_type=MaskType.HARD, hard_mask=tf_hard_mask, ).get_masked_tensor(h_conv) h_conv, residuals = Mask( 200 + conv_num, self.mask_alpha[conv_num], self.mask_not_alpha[conv_num], mask_type=self.mask_type, ).get_masked_tensor(h_conv, is_training, with_residuals=True) mask = residuals["cond_mask"] with tf.control_dependencies([mask]): excl_h_conv = tf.nn.relu(excl_h_conv) excl_h_conv = Mask.get_exclusive_masked_tensor(excl_h_conv, mask, is_training) # max pooling if conv_num + 1 in self.pool_pos_to_dims: pool_dim = self.pool_pos_to_dims[conv_num + 1] pool_ksize = [1, pool_dim, pool_dim, 1] h_conv = tf.nn.max_pool(h_conv, ksize=pool_ksize, strides=[1, 2, 2, 1], padding="SAME") if self.use_set_based_mask: excl_h_conv = tf.nn.max_pool(excl_h_conv, ksize=pool_ksize, strides=[1, 2, 2, 1], padding="SAME") h_fc = tf.reshape(h_conv, (-1, self.dims[0])) excl_h_fc = tf.reshape(excl_h_conv, (-1, self.dims[0])) if self.use_set_based_mask else None for i in range(1, len(self.dims)): w_fc = self.get_variable("fc%d" % i, "weight", True) b_fc = self.get_variable("fc%d" % i, "biases", True) h_fc = tf.matmul(h_fc, w_fc) + b_fc if self.use_set_based_mask: excl_h_fc = tf.matmul(excl_h_fc, w_fc) + b_fc if i < len(self.dims) - 1: # Do not activate the last layer. h_fc = tf.nn.relu(h_fc) if self.use_set_based_mask: excl_h_fc = tf.nn.relu(excl_h_fc) if self.dropout_type == "dropout": h_fc = tf.nn.dropout(h_fc, keep_prob) if self.use_set_based_mask: excl_h_fc = tf.nn.dropout(excl_h_fc, keep_prob) else: raise ValueError loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=h_fc, labels=Y)) excl_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=excl_h_fc, labels=excl_Y)) l1_l2_regularizer = tf.contrib.layers.l1_l2_regularizer(scale_l1=self.l1_lambda, scale_l2=self.l2_lambda) regularization_loss = tf.contrib.layers.apply_regularization( l1_l2_regularizer, [v for v in tf.trainable_variables() if "conv" in v.name or "fc" in v.name], ) loss_with_excl = loss + excl_loss + regularization_loss opt_with_excl = tf.train.AdamOptimizer(self.init_lr).minimize(loss_with_excl) self.sess.run(tf.global_variables_initializer()) return X, Y, excl_X, excl_Y, opt_with_excl, loss_with_excl, keep_prob, is_training
def build_model(self): xn_filters, xsize = self.conv_dims[0], self.conv_dims[1] X = tf.placeholder(tf.float32, [None, xsize, xsize, xn_filters], name="X") Y = tf.placeholder(tf.float32, [None, self.n_classes], name="Y") keep_prob = tf.placeholder(tf.float32, name="keep_prob") is_training = tf.placeholder(tf.bool, name="is_training") excl_X = tf.placeholder(tf.float32, [None, xsize, xsize, xn_filters], name="excl_X") excl_Y = tf.placeholder(tf.float32, [None, self.n_classes], name="excl_Y") h_conv = X excl_h_conv = excl_X if self.use_set_based_mask else None for conv_num, i in enumerate(range(2, len(self.conv_dims), 2)): w_conv = self.get_variable("conv%d" % (i // 2), "weight", True) b_conv = self.get_variable("conv%d" % (i // 2), "biases", True) h_conv = tf.nn.conv2d(h_conv, w_conv, strides=[1, 1, 1, 1], padding="SAME") + b_conv if self.use_set_based_mask: excl_h_conv = tf.nn.conv2d(excl_h_conv, w_conv, strides=[1, 1, 1, 1], padding="SAME") + b_conv # batch normalization if self.use_batch_normalization: beta = self.get_variable("bn%d" % (i // 2), "beta", True) gamma = self.get_variable("bn%d" % (i // 2), "gamma", True) h_conv = _get_batch_normalized_conv(beta, gamma, h_conv, is_training) if self.use_set_based_mask: excl_h_conv = _get_batch_normalized_conv(beta, gamma, excl_h_conv, is_training) # activation h_conv = tf.nn.relu(h_conv) # Mask if self.use_set_based_mask: h_conv, residuals = Mask( conv_num, self.mask_alpha[conv_num], self.mask_not_alpha[conv_num], mask_type=self.mask_type, ).get_masked_tensor(h_conv, is_training, with_residuals=True) mask = residuals["cond_mask"] with tf.control_dependencies([mask]): excl_h_conv = tf.nn.relu(excl_h_conv) excl_h_conv = Mask.get_exclusive_masked_tensor(excl_h_conv, mask, is_training) elif self.use_filter_dropout: h_conv = Mask(conv_num, mask_type=MaskType.INDEPENDENT).get_masked_tensor(h_conv, is_training) # max pooling if conv_num + 1 in self.pool_pos_to_dims: pool_dim = self.pool_pos_to_dims[conv_num + 1] pool_ksize = [1, pool_dim, pool_dim, 1] h_conv = tf.nn.max_pool(h_conv, ksize=pool_ksize, strides=[1, 2, 2, 1], padding="SAME") if self.use_set_based_mask: excl_h_conv = tf.nn.max_pool(excl_h_conv, ksize=pool_ksize, strides=[1, 2, 2, 1], padding="SAME") h_fc = tf.reshape(h_conv, (-1, self.dims[0])) excl_h_fc = tf.reshape(excl_h_conv, (-1, self.dims[0])) if self.use_set_based_mask else None for i in range(1, len(self.dims)): w_fc = self.get_variable("fc%d" % i, "weight", True) b_fc = self.get_variable("fc%d" % i, "biases", True) h_fc = tf.matmul(h_fc, w_fc) + b_fc if self.use_set_based_mask: excl_h_fc = tf.matmul(excl_h_fc, w_fc) + b_fc if i < len(self.dims) - 1: # Do not activate the last layer. h_fc = tf.nn.relu(h_fc) if self.use_set_based_mask: excl_h_fc = tf.nn.relu(excl_h_fc) if self.dropout_type == "dropout": h_fc = tf.nn.dropout(h_fc, keep_prob) if self.use_set_based_mask: excl_h_fc = tf.nn.dropout(excl_h_fc, keep_prob) else: raise ValueError self.yhat = tf.nn.sigmoid(h_fc) self.loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=h_fc, labels=Y)) if self.use_set_based_mask: self.excl_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=excl_h_fc, labels=excl_Y)) return X, Y, excl_X, excl_Y, keep_prob, is_training