def build_model(self):
X_list = [tf.placeholder(tf.float32, [None, self.dims[0]], name="X_%d" % (t + 1))
for t in range(self.n_tasks)]
Y_list = [tf.placeholder(tf.float32, [None, self.n_classes], name="Y_%d" % (t + 1))
for t in range(self.n_tasks)]
is_training = tf.placeholder(tf.bool, name="is_training")
for t, (X, Y) in enumerate(zip(X_list, Y_list)):
bottom = X
excl_bottom = X if self.use_set_based_mask else None
for i in range(1, self.n_layers):
w = self.get_variable('layer%d' % i, 'weight', True)
b = self.get_variable('layer%d' % i, 'biases', True)
bottom = tf.nn.relu(tf.matmul(bottom, w) + b)
if self.use_set_based_mask:
bottom, residuals = Mask(
i - 1, self.mask_alpha[i - 1], self.mask_not_alpha[i - 1], mask_type=self.mask_type,
).get_masked_tensor(bottom, is_training, with_residuals=True)
mask = residuals["cond_mask"]
with tf.control_dependencies([mask]):
excl_bottom = tf.nn.relu(tf.matmul(excl_bottom, w) + b)
excl_bottom = Mask.get_exclusive_masked_tensor(excl_bottom, mask, is_training)
w = self.get_variable("layer%d" % self.n_layers, "weight_%d" % (t + 1), True)
b = self.get_variable("layer%d" % self.n_layers, "biases_%d" % (t + 1), True)
y = tf.matmul(bottom, w) + b
yhat = tf.nn.sigmoid(y)
self.yhat_list.append(yhat)
loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=y, labels=Y))
self.partial_loss_list.append(loss)
if self.use_set_based_mask:
excl_y = tf.matmul(excl_bottom, w) + b
excl_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=excl_y, labels=Y))
self.excl_partial_loss_list.append(excl_loss)
return X_list, Y_list, is_training
def build_model_for_retraining(self, flags):
xn_filters, xsize = self.conv_dims[0], self.conv_dims[1]
X = tf.get_default_graph().get_tensor_by_name("X:0")
Y = tf.get_default_graph().get_tensor_by_name("Y:0")
keep_prob = tf.get_default_graph().get_tensor_by_name("keep_prob:0")
is_training = tf.get_default_graph().get_tensor_by_name("is_training:0")
excl_X = tf.placeholder(tf.float32, [None, xsize, xsize, xn_filters], name="excl_X")
excl_Y = tf.placeholder(tf.float32, [None, self.n_classes], name="excl_Y")
h_conv = X
excl_h_conv = excl_X if self.use_set_based_mask else None
for conv_num, i in enumerate(range(2, len(self.conv_dims), 2)):
w_conv = self.get_variable("conv%d" % (i // 2), "weight", True)
b_conv = self.get_variable("conv%d" % (i // 2), "biases", True)
h_conv = tf.nn.conv2d(h_conv, w_conv, strides=[1, 1, 1, 1], padding="SAME") + b_conv
if self.use_set_based_mask:
excl_h_conv = tf.nn.conv2d(excl_h_conv, w_conv, strides=[1, 1, 1, 1], padding="SAME") + b_conv
# activation
h_conv = tf.nn.relu(h_conv)
if self.use_set_based_mask:
ndarray_hard_mask = np.ones(shape=h_conv.get_shape().as_list()[-1:])
indices_to_zero = np.asarray(list(self.layer_to_removed_unit_set["conv{}".format(i // 2)]))
ndarray_hard_mask[indices_to_zero] = 0
tf_hard_mask = tf.constant(ndarray_hard_mask, dtype=tf.float32)
h_conv = Mask(
100 + conv_num, 0, 0, mask_type=MaskType.HARD, hard_mask=tf_hard_mask,
).get_masked_tensor(h_conv)
h_conv, residuals = Mask(
200 + conv_num, self.mask_alpha[conv_num], self.mask_not_alpha[conv_num], mask_type=self.mask_type,
).get_masked_tensor(h_conv, is_training, with_residuals=True)
mask = residuals["cond_mask"]
with tf.control_dependencies([mask]):
excl_h_conv = tf.nn.relu(excl_h_conv)
excl_h_conv = Mask.get_exclusive_masked_tensor(excl_h_conv, mask, is_training)
# max pooling
if conv_num + 1 in self.pool_pos_to_dims:
pool_dim = self.pool_pos_to_dims[conv_num + 1]
pool_ksize = [1, pool_dim, pool_dim, 1]
h_conv = tf.nn.max_pool(h_conv, ksize=pool_ksize, strides=[1, 2, 2, 1], padding="SAME")
if self.use_set_based_mask:
excl_h_conv = tf.nn.max_pool(excl_h_conv, ksize=pool_ksize, strides=[1, 2, 2, 1], padding="SAME")
h_fc = tf.reshape(h_conv, (-1, self.dims[0]))
excl_h_fc = tf.reshape(excl_h_conv, (-1, self.dims[0])) if self.use_set_based_mask else None
for i in range(1, len(self.dims)):
w_fc = self.get_variable("fc%d" % i, "weight", True)
b_fc = self.get_variable("fc%d" % i, "biases", True)
h_fc = tf.matmul(h_fc, w_fc) + b_fc
if self.use_set_based_mask:
excl_h_fc = tf.matmul(excl_h_fc, w_fc) + b_fc
if i < len(self.dims) - 1: # Do not activate the last layer.
h_fc = tf.nn.relu(h_fc)
if self.use_set_based_mask:
excl_h_fc = tf.nn.relu(excl_h_fc)
if self.dropout_type == "dropout":
h_fc = tf.nn.dropout(h_fc, keep_prob)
if self.use_set_based_mask:
excl_h_fc = tf.nn.dropout(excl_h_fc, keep_prob)
else:
raise ValueError
loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=h_fc, labels=Y))
excl_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=excl_h_fc, labels=excl_Y))
l1_l2_regularizer = tf.contrib.layers.l1_l2_regularizer(scale_l1=self.l1_lambda, scale_l2=self.l2_lambda)
regularization_loss = tf.contrib.layers.apply_regularization(
l1_l2_regularizer,
[v for v in tf.trainable_variables() if "conv" in v.name or "fc" in v.name],
)
loss_with_excl = loss + excl_loss + regularization_loss
opt_with_excl = tf.train.AdamOptimizer(self.init_lr).minimize(loss_with_excl)
self.sess.run(tf.global_variables_initializer())
return X, Y, excl_X, excl_Y, opt_with_excl, loss_with_excl, keep_prob, is_training
def build_model(self):
xn_filters, xsize = self.conv_dims[0], self.conv_dims[1]
X = tf.placeholder(tf.float32, [None, xsize, xsize, xn_filters], name="X")
Y = tf.placeholder(tf.float32, [None, self.n_classes], name="Y")
keep_prob = tf.placeholder(tf.float32, name="keep_prob")
is_training = tf.placeholder(tf.bool, name="is_training")
excl_X = tf.placeholder(tf.float32, [None, xsize, xsize, xn_filters], name="excl_X")
excl_Y = tf.placeholder(tf.float32, [None, self.n_classes], name="excl_Y")
h_conv = X
excl_h_conv = excl_X if self.use_set_based_mask else None
for conv_num, i in enumerate(range(2, len(self.conv_dims), 2)):
w_conv = self.get_variable("conv%d" % (i // 2), "weight", True)
b_conv = self.get_variable("conv%d" % (i // 2), "biases", True)
h_conv = tf.nn.conv2d(h_conv, w_conv, strides=[1, 1, 1, 1], padding="SAME") + b_conv
if self.use_set_based_mask:
excl_h_conv = tf.nn.conv2d(excl_h_conv, w_conv, strides=[1, 1, 1, 1], padding="SAME") + b_conv
# batch normalization
if self.use_batch_normalization:
beta = self.get_variable("bn%d" % (i // 2), "beta", True)
gamma = self.get_variable("bn%d" % (i // 2), "gamma", True)
h_conv = _get_batch_normalized_conv(beta, gamma, h_conv, is_training)
if self.use_set_based_mask:
excl_h_conv = _get_batch_normalized_conv(beta, gamma, excl_h_conv, is_training)
# activation
h_conv = tf.nn.relu(h_conv)
# Mask
if self.use_set_based_mask:
h_conv, residuals = Mask(
conv_num, self.mask_alpha[conv_num], self.mask_not_alpha[conv_num], mask_type=self.mask_type,
).get_masked_tensor(h_conv, is_training, with_residuals=True)
mask = residuals["cond_mask"]
with tf.control_dependencies([mask]):
excl_h_conv = tf.nn.relu(excl_h_conv)
excl_h_conv = Mask.get_exclusive_masked_tensor(excl_h_conv, mask, is_training)
elif self.use_filter_dropout:
h_conv = Mask(conv_num, mask_type=MaskType.INDEPENDENT).get_masked_tensor(h_conv, is_training)
# max pooling
if conv_num + 1 in self.pool_pos_to_dims:
pool_dim = self.pool_pos_to_dims[conv_num + 1]
pool_ksize = [1, pool_dim, pool_dim, 1]
h_conv = tf.nn.max_pool(h_conv, ksize=pool_ksize, strides=[1, 2, 2, 1], padding="SAME")
if self.use_set_based_mask:
excl_h_conv = tf.nn.max_pool(excl_h_conv, ksize=pool_ksize, strides=[1, 2, 2, 1], padding="SAME")
h_fc = tf.reshape(h_conv, (-1, self.dims[0]))
excl_h_fc = tf.reshape(excl_h_conv, (-1, self.dims[0])) if self.use_set_based_mask else None
for i in range(1, len(self.dims)):
w_fc = self.get_variable("fc%d" % i, "weight", True)
b_fc = self.get_variable("fc%d" % i, "biases", True)
h_fc = tf.matmul(h_fc, w_fc) + b_fc
if self.use_set_based_mask:
excl_h_fc = tf.matmul(excl_h_fc, w_fc) + b_fc
if i < len(self.dims) - 1: # Do not activate the last layer.
h_fc = tf.nn.relu(h_fc)
if self.use_set_based_mask:
excl_h_fc = tf.nn.relu(excl_h_fc)
if self.dropout_type == "dropout":
h_fc = tf.nn.dropout(h_fc, keep_prob)
if self.use_set_based_mask:
excl_h_fc = tf.nn.dropout(excl_h_fc, keep_prob)
else:
raise ValueError
self.yhat = tf.nn.sigmoid(h_fc)
self.loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=h_fc, labels=Y))
if self.use_set_based_mask:
self.excl_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=excl_h_fc, labels=excl_Y))
return X, Y, excl_X, excl_Y, keep_prob, is_training
