def mobilenet_v1_arg_scope(
is_training=True,
weight_decay=0.00004,
stddev=0.09,
regularize_depthwise=False,
batch_norm_decay=0.9997,
batch_norm_epsilon=0.001,
batch_norm_updates_collections=tf.compat.v1.GraphKeys.UPDATE_OPS,
normalizer_fn=slim.batch_norm):
"""Defines the default MobilenetV1 arg scope.
Args:
is_training: Whether or not we're training the model. If this is set to
None, the parameter is not added to the batch_norm arg_scope.
weight_decay: The weight decay to use for regularizing the model.
stddev: The standard deviation of the trunctated normal weight initializer.
regularize_depthwise: Whether or not apply regularization on depthwise.
batch_norm_decay: Decay for batch norm moving average.
batch_norm_epsilon: Small float added to variance to avoid dividing by zero
in batch norm.
batch_norm_updates_collections: Collection for the update ops for
batch norm.
normalizer_fn: Normalization function to apply after convolution.
Returns:
An `arg_scope` to use for the mobilenet v1 model.
"""
batch_norm_params = {
'center': True,
'scale': True,
'decay': batch_norm_decay,
'epsilon': batch_norm_epsilon,
'updates_collections': batch_norm_updates_collections,
}
if is_training is not None:
batch_norm_params['is_training'] = is_training
# Set weight_decay for weights in Conv and DepthSepConv layers.
weights_init = tf.truncated_normal_initializer(stddev=stddev)
regularizer = slim.layers.l2_regularizer(weight_decay)
if regularize_depthwise:
depthwise_regularizer = regularizer
else:
depthwise_regularizer = None
with slim.arg_scope([slim.conv2d, slim.separable_conv2d],
weights_initializer=weights_init,
activation_fn=tf.nn.relu6,
normalizer_fn=normalizer_fn):
with slim.arg_scope([slim.batch_norm], **batch_norm_params):
with slim.arg_scope([slim.conv2d],
weights_regularizer=regularizer):
with slim.arg_scope(
[slim.separable_conv2d],
weights_regularizer=depthwise_regularizer) as sc:
return sc
def discriminator(self, x, y_fill, isTrain=True, reuse=False):
with tf.variable_scope('discriminator', reuse=reuse):
# cat1 = tf.concat([x, y_fill], 3)
# net, endpoint = inception_resnet_v2(cat1, is_training=False)
# inception_out = endpoint['Logits']
# initializer
w_init = tf.truncated_normal_initializer(mean=0.0, stddev=0.02)
b_init = tf.constant_initializer(0.0)
# tensor_std = tf.math.reduce_std(x, axis=0)
# tensor_std = tf.math.reduce_mean(tensor_std)
# concat layer
x = tf.keras.layers.GaussianNoise(0.01)(x)
# std_channel = y_fill*0 + tensor_std
# cat1 = tf.concat([x, y_fill, std_channel], 3)
cat1 = tf.concat([x, y_fill], 3)
conv_256 = half_conv(cat1, 256, 5)
conv_256 = same_size_conv(conv_256, 5)
conv_256 = same_size_conv(conv_256, 5)
conv_128 = half_conv(conv_256, 128, 5)
conv_128 = same_size_conv(conv_128, 5)
conv_128 = same_size_conv(conv_128, 5)
# conv_64 = half_conv(conv_128, 128, 5)
# conv_32 = half_conv(conv_64, 64, 5)
# conv_32 = same_size_conv(conv_32, 5)
# conv_16 = half_conv(conv_32, 32, 5)
# conv_8 = half_conv(conv_16, 32, 5)
# conv_4 = half_conv(conv_8, 16, 5)
conv_2 = half_conv(conv_128, 64, 2, do_BN=False)
# activated = tf.math.tanh(conv_2)
# # 1st hidden layer
# conv1 = tf.layers.conv2d(cat1, 128, [5, 5], strides=(2, 2), padding='same', kernel_initializer=w_init, bias_initializer=b_init)
# lrelu1 = self.lrelu(conv1, 0.2)
# # 2nd hidden layer
# conv2 = tf.layers.conv2d(lrelu1, 256, [5, 5], strides=(2, 2), padding='same', kernel_initializer=w_init, bias_initializer=b_init)
# lrelu2 = self.lrelu(tf.layers.batch_normalization(conv2, training=isTrain), 0.2)
# # output layer
# conv3 = tf.layers.conv2d(lrelu2, 1, [7, 7], strides=(1, 1), padding='valid', kernel_initializer=w_init)
# reshaped_conv_out = tf.reshape(conv_4, [config.batch_size, shape_conv[1]*shape_conv[2]*shape_conv[3]])
logit = tf.layers.dense(conv_2, 1)
# logit = conv_2
print(logit)
shape_logit = logit.shape
logit = tf.reshape(logit, [
config.batch_size,
shape_logit[1] * shape_logit[2] * shape_logit[3]
])
o = tf.nn.sigmoid(logit)
return o, logit
def build_network(self, sess, is_training=True):
with tf.variable_scope('vgg_16', 'vgg_16'):
# select initializer
if cfg.FLAGS.initializer == "truncated":
initializer = tf.truncated_normal_initializer(mean=0.0,
stddev=0.01)
initializer_bbox = tf.truncated_normal_initializer(
mean=0.0, stddev=0.001)
else:
initializer = tf.random_normal_initializer(mean=0.0,
stddev=0.01)
initializer_bbox = tf.random_normal_initializer(mean=0.0,
stddev=0.001)
# Build head
net = self.build_head(is_training)
# Build rpn
rpn_cls_prob, rpn_bbox_pred, rpn_cls_score, rpn_cls_score_reshape = self.build_rpn(
net, is_training, initializer)
# Build proposals
rois = self.build_proposals(is_training, rpn_cls_prob,
rpn_bbox_pred, rpn_cls_score)
# Build predictions
cls_score, cls_prob, bbox_pred = self.build_predictions(
net, rois, is_training, initializer, initializer_bbox)
self._predictions["rpn_cls_score"] = rpn_cls_score
self._predictions["rpn_cls_score_reshape"] = rpn_cls_score_reshape
self._predictions["rpn_cls_prob"] = rpn_cls_prob
self._predictions["rpn_bbox_pred"] = rpn_bbox_pred
self._predictions["cls_score"] = cls_score
self._predictions["cls_prob"] = cls_prob
self._predictions["bbox_pred"] = bbox_pred
self._predictions["rois"] = rois
self._score_summaries.update(self._predictions)
return rois, cls_prob, bbox_pred
def create_model(bert_config, is_training, input_ids, input_mask, segment_ids,
labels, num_labels, use_one_hot_embeddings, max_seq_length,
dropout_prob):
"""Creates a classification model."""
bsz_per_core = tf.shape(input_ids)[0]
model = modeling.BertModel(
config=bert_config,
is_training=is_training,
input_ids=tf.reshape(input_ids,
[bsz_per_core * num_labels, max_seq_length]),
input_mask=tf.reshape(input_mask,
[bsz_per_core * num_labels, max_seq_length]),
token_type_ids=tf.reshape(segment_ids,
[bsz_per_core * num_labels, max_seq_length]),
use_one_hot_embeddings=use_one_hot_embeddings)
# In the demo, we are doing a simple classification task on the entire
# segment.
#
# If you want to use the token-level output, use model.get_sequence_output()
# instead.
output_layer = model.get_pooled_output()
hidden_size = output_layer.shape[-1].value
output_weights = tf.get_variable(
"output_weights", [1, hidden_size],
initializer=tf.truncated_normal_initializer(stddev=0.02))
output_bias = tf.get_variable("output_bias", [1],
initializer=tf.zeros_initializer())
with tf.variable_scope("loss"):
if is_training:
# I.e., 0.1 dropout
output_layer = tf.nn.dropout(output_layer,
keep_prob=1 - dropout_prob)
logits = tf.matmul(output_layer, output_weights, transpose_b=True)
logits = tf.nn.bias_add(logits, output_bias)
logits = tf.reshape(logits, [bsz_per_core, num_labels])
probabilities = tf.nn.softmax(logits, axis=-1)
predictions = tf.argmax(probabilities, axis=-1, output_type=tf.int32)
log_probs = tf.nn.log_softmax(logits, axis=-1)
one_hot_labels = tf.one_hot(labels,
depth=tf.cast(num_labels, dtype=tf.int32),
dtype=tf.float32)
per_example_loss = -tf.reduce_sum(one_hot_labels * log_probs, axis=-1)
loss = tf.reduce_mean(per_example_loss)
return (loss, per_example_loss, probabilities, logits, predictions)
def _create_test_variables(outer_scope, inner_scope, var_c_name, var_e_name,
var_n_name):
# Keras layers can cause problems for `tf.train.init_from_checkpoint`
# if not handled properly. Here we intentionally use Dense layers
# to test whether the ckpt loading logic works.
dense_layer = tf.keras.layers.Dense(10, name="dense")
with tf.variable_scope(outer_scope):
var_c = tf.get_variable(var_c_name,
shape=[2, 4],
initializer=tf.truncated_normal_initializer())
var_d = dense_layer(var_c)
with tf.variable_scope(inner_scope):
var_e = tf.get_variable(
var_e_name,
shape=[2, 3],
initializer=tf.truncated_normal_initializer())
_ = tf.get_variable(var_n_name,
shape=[3, 5],
initializer=tf.truncated_normal_initializer())
return var_c, var_d, var_e
def autoencoder(input, hidden_layers=None, name=None, reuse=None):
if hidden_layers is None:
hidden_layers = [32, 16]
with tf.variable_scope(name_or_scope=name, reuse=reuse):
current = input
for i in range(len(hidden_layers)):
layer_name = 'layer_' + str(i)
current = tf.layers.dense(current, hidden_layers[i], tf.nn.relu, name=layer_name,
kernel_initializer=tf.truncated_normal_initializer(stddev=1e-3, seed=2020),
bias_initializer=tf.zeros_initializer)
return current
def deconv2d(input_, output_dim, ks=4, s=2, stddev=0.02, name="deconv2d"):
with tf.variable_scope(name):
return slim.conv2d_transpose(
input_,
output_dim,
ks,
s,
padding='SAME',
activation_fn=None,
weights_initializer=tf.truncated_normal_initializer(stddev=stddev),
biases_initializer=None)
def hidden_layer(layer_input,output_depth,scope='hidden_layer',reuse=None): input_depth = layer_input.get_shape()[-1] with tf.variable_scope(scope,reuse=reuse): #初始化方法是truncated_normal 产生截断正态分布随机数 w = tf.get_variable(initializer=tf.truncated_normal_initializer(stddev=0.1),shape=(input_depth,output_depth),name='weights') #用0.1对偏置进行初始化 b = tf.get_variable(initializer=tf.constant_initializer(0.1),shape=(output_depth),name='bias') net = tf.matmul(layer_input,w) + b return net
def create_model(albert_config, is_training, input_ids, input_mask,
segment_ids, labels, num_labels, use_one_hot_embeddings,
task_name):
"""Creates a classification model."""
model = modeling.AlbertModel(config=albert_config,
is_training=is_training,
input_ids=input_ids,
input_mask=input_mask,
token_type_ids=segment_ids,
use_one_hot_embeddings=use_one_hot_embeddings)
# In the demo, we are doing a simple classification task on the entire
# segment.
#
# If you want to use the token-level output, use model.get_sequence_output()
# instead.
output_layer = model.get_pooled_output()
hidden_size = output_layer.shape[-1].value
output_weights = tf.get_variable(
"output_weights", [num_labels, hidden_size],
initializer=tf.truncated_normal_initializer(stddev=0.02))
output_bias = tf.get_variable("output_bias", [num_labels],
initializer=tf.zeros_initializer())
with tf.variable_scope("loss"):
if is_training:
# I.e., 0.1 dropout
output_layer = tf.nn.dropout(output_layer, keep_prob=0.9)
logits = tf.matmul(output_layer, output_weights, transpose_b=True)
logits = tf.nn.bias_add(logits, output_bias)
if task_name != "sts-b":
probabilities = tf.nn.softmax(logits, axis=-1)
predictions = tf.argmax(probabilities,
axis=-1,
output_type=tf.int32)
log_probs = tf.nn.log_softmax(logits, axis=-1)
one_hot_labels = tf.one_hot(labels,
depth=num_labels,
dtype=tf.float32)
per_example_loss = -tf.reduce_sum(one_hot_labels * log_probs,
axis=-1)
else:
probabilities = logits
logits = tf.squeeze(logits, [-1])
predictions = logits
per_example_loss = tf.square(logits - labels)
loss = tf.reduce_mean(per_example_loss)
return (loss, per_example_loss, probabilities, logits, predictions)
def conv3d(input_, output_dim, k_h=4, k_w=4, k_d=4, d_h=2, d_w=2, d_d=2, stddev=0.02,
name="conv3d"):
with tf.variable_scope(name):
w = tf.get_variable('w', [k_d, k_h, k_w, input_.get_shape()[-1], output_dim],
initializer=tf.truncated_normal_initializer(stddev=stddev))
conv = tf.nn.conv3d(input_, w, strides=[1, d_d, d_h, d_w, 1], padding='SAME')
biases = tf.get_variable('biases', [output_dim], initializer=tf.constant_initializer(0.0))
conv = tf.reshape(tf.nn.bias_add(conv, biases), conv.get_shape())
return conv
def extract_features(self, preprocessed_inputs):
"""Extract features from preprocessed inputs.
Args:
preprocessed_inputs: a [batch, height, width, channels] float tensor
representing a batch of images.
Returns:
feature_maps: a list of tensors where the ith tensor has shape
[batch, height_i, width_i, depth_i]
"""
preprocessed_inputs = shape_utils.check_min_image_dim(
33, preprocessed_inputs)
nodes_dict = lookup_spaghetti_arch(self._spaghettinet_arch_name)
with tf.variable_scope(self._spaghettinet_arch_name,
reuse=self._reuse_weights):
with slim.arg_scope(
[slim.conv2d],
weights_initializer=tf.truncated_normal_initializer(
mean=0.0, stddev=0.03),
weights_regularizer=slim.l2_regularizer(1e-5)):
with slim.arg_scope(
[slim.separable_conv2d],
weights_initializer=tf.truncated_normal_initializer(
mean=0.0, stddev=0.03),
weights_regularizer=slim.l2_regularizer(1e-5)):
with slim.arg_scope([slim.batch_norm],
is_training=self._is_training,
epsilon=0.001,
decay=0.97,
center=True,
scale=True):
spaghetti_net = SpaghettiNet(
node_specs=nodes_dict,
is_training=self._is_training,
use_native_resize_op=self._use_native_resize_op,
use_explicit_padding=self._use_explicit_padding,
name=self._spaghettinet_arch_name)
feature_maps = spaghetti_net.apply(preprocessed_inputs)
return feature_maps
def encoder_vgg(x, enc_final_size, reuse=False, scope_prefix='', hparams=None,
is_training=True):
"""VGG network to use as encoder without the top few layers.
Can be pretrained.
Args:
x: The image to encode. In the range 0 to 1.
enc_final_size: The desired size of the encoding.
reuse: To reuse in variable scope or not.
scope_prefix: The prefix before the scope name.
hparams: The python hparams.
is_training: boolean value indicating if training is happening.
Returns:
The generated image.
"""
with tf.variable_scope(scope_prefix + 'encoder', reuse=reuse):
# Preprocess input
x *= 256
x = x - COLOR_NORMALIZATION_VECTOR
with arg_scope(vgg.vgg_arg_scope()):
# Padding because vgg_16 accepts images of size at least VGG_IMAGE_SIZE.
x = tf.pad(x, [[0, 0], [0, VGG_IMAGE_SIZE - IMG_WIDTH],
[0, VGG_IMAGE_SIZE - IMG_HEIGHT], [0, 0]])
_, end_points = vgg.vgg_16(
x,
num_classes=enc_final_size,
is_training=is_training)
pool5_key = [key for key in end_points.keys() if 'pool5' in key]
assert len(pool5_key) == 1
enc = end_points[pool5_key[0]]
# Undoing padding.
enc = tf.slice(enc, [0, 0, 0, 0], [-1, 2, 2, -1])
enc_shape = enc.get_shape().as_list()
enc_shape[0] = -1
enc_size = enc_shape[1] * enc_shape[2] * enc_shape[3]
enc_flat = tf.reshape(enc, (-1, enc_size))
enc_flat = tf.nn.dropout(enc_flat, hparams.enc_keep_prob)
enc_flat = tf.layers.dense(
enc_flat,
enc_final_size,
kernel_initializer=tf.truncated_normal_initializer(stddev=1e-4,))
if hparams.enc_pred_use_l2norm:
enc_flat = tf.nn.l2_normalize(enc_flat, 1)
return enc_flat
def setup(self):
self.XHolder = tf.placeholder(dtype=tf.int32, shape=[None])
self.YHolder = tf.placeholder(dtype=tf.int32, shape=[None])
self.params['W_embed'] = tf.get_variable(name='W_embed',
shape=[self.vocab_size, self.embed_dim],
dtype=tf.float32,
initializer=tf.truncated_normal_initializer(stddev=0.1))
# self.params['b_embed'] = tf.get_variable('b_embed',
# shape=[self.embed_dim],
# dtype=tf.float32,
# initializer=tf.zeros_initializer())
self.params['W_nce'] = tf.get_variable(name='W_nce',
shape=[self.vocab_size, self.embed_dim],
dtype=tf.float32,
initializer=tf.truncated_normal_initializer(stddev=0.1))
self.params['b_nce'] = tf.get_variable('b_nce',
shape=[self.vocab_size],
dtype=tf.float32,
initializer=tf.zeros_initializer())
def training_scope(is_training=True,
weight_decay=0.00004,
stddev=0.09,
dropout_keep_prob=0.8,
bn_decay=0.997):
"""Defines Mobilenet training scope.
Usage:
with tf.contrib.slim.arg_scope(mobilenet.training_scope()):
logits, endpoints = mobilenet_v2.mobilenet(input_tensor)
# the network created will be trainble with dropout/batch norm
# initialized appropriately.
Args:
is_training: if set to False this will ensure that all customizations are
set to non-training mode. This might be helpful for code that is reused
across both training/evaluation, but most of the time training_scope with
value False is not needed. If this is set to None, the parameters is not
added to the batch_norm arg_scope.
weight_decay: The weight decay to use for regularizing the model.
stddev: Standard deviation for initialization, if negative uses xavier.
dropout_keep_prob: dropout keep probability (not set if equals to None).
bn_decay: decay for the batch norm moving averages (not set if equals to
None).
Returns:
An argument scope to use via arg_scope.
"""
# Note: do not introduce parameters that would change the inference
# model here (for example whether to use bias), modify conv_def instead.
batch_norm_params = {
'decay': bn_decay,
'is_training': is_training
}
if stddev < 0:
weight_intitializer = slim.initializers.xavier_initializer()
else:
weight_intitializer = tf.truncated_normal_initializer(stddev=stddev)
# Set weight_decay for weights in Conv and FC layers.
with slim.arg_scope(
[slim.conv2d, slim.fully_connected, slim.separable_conv2d],
weights_initializer=weight_intitializer,
normalizer_fn=slim.batch_norm), \
slim.arg_scope([mobilenet_base, mobilenet], is_training=is_training),\
safe_arg_scope([slim.batch_norm], **batch_norm_params), \
safe_arg_scope([slim.dropout], is_training=is_training,
keep_prob=dropout_keep_prob), \
slim.arg_scope([slim.conv2d], \
weights_regularizer=slim.l2_regularizer(weight_decay)), \
slim.arg_scope([slim.separable_conv2d], weights_regularizer=None) as s:
return s
def linear(x, state_dim, name='linear', reuse=True):
with tf.variable_scope(name) as scope:
if reuse:
scope.reuse_variables()
weight = tf.get_variable(
'weight', [x.get_shape()[-1], state_dim],
initializer=tf.truncated_normal_initializer(stddev=0.02))
bias = tf.get_variable('bias', [state_dim],
initializer=tf.constant_initializer(0))
weighted_sum = tf.matmul(x, weight) + bias
return weighted_sum
def create_model(albert_config, is_training, input_ids, input_mask,
segment_ids, labels, num_labels, use_one_hot_embeddings,
max_seq_length, dropout_prob, hub_module):
"""Creates a classification model."""
bsz_per_core = tf.shape(input_ids)[0]
input_ids = tf.reshape(input_ids,
[bsz_per_core * num_labels, max_seq_length])
input_mask = tf.reshape(input_mask,
[bsz_per_core * num_labels, max_seq_length])
token_type_ids = tf.reshape(segment_ids,
[bsz_per_core * num_labels, max_seq_length])
(output_layer, _) = fine_tuning_utils.create_albert(
albert_config=albert_config,
is_training=is_training,
input_ids=input_ids,
input_mask=input_mask,
segment_ids=token_type_ids,
use_one_hot_embeddings=use_one_hot_embeddings,
use_einsum=True,
hub_module=hub_module)
hidden_size = output_layer.shape[-1].value
output_weights = tf.get_variable(
"output_weights", [1, hidden_size],
initializer=tf.truncated_normal_initializer(stddev=0.02))
output_bias = tf.get_variable("output_bias", [1],
initializer=tf.zeros_initializer())
with tf.variable_scope("loss"):
if is_training:
# I.e., 0.1 dropout
output_layer = tf.nn.dropout(output_layer,
keep_prob=1 - dropout_prob)
logits = tf.matmul(output_layer, output_weights, transpose_b=True)
logits = tf.nn.bias_add(logits, output_bias)
logits = tf.reshape(logits, [bsz_per_core, num_labels])
probabilities = tf.nn.softmax(logits, axis=-1)
predictions = tf.argmax(probabilities, axis=-1, output_type=tf.int32)
log_probs = tf.nn.log_softmax(logits, axis=-1)
one_hot_labels = tf.one_hot(labels,
depth=tf.cast(num_labels, dtype=tf.int32),
dtype=tf.float32)
per_example_loss = -tf.reduce_sum(one_hot_labels * log_probs, axis=-1)
loss = tf.reduce_mean(per_example_loss)
return (loss, per_example_loss, probabilities, logits, predictions)
def instance_norm(x, name):
with tf.variable_scope(name):
epsilon = 1e-5
mean, var = tf.nn.moments(x, [1, 2], keep_dims=True)
scale = tf.get_variable('scale', [x.get_shape()[-1]],
initializer=tf.truncated_normal_initializer(
mean=1.0, stddev=0.02))
offset = tf.get_variable('offset', [x.get_shape()[-1]],
initializer=tf.constant_initializer(0.0))
out = scale * tf.div(x - mean, tf.sqrt(var + epsilon)) + offset
return out
def get_crit_p(attacked_data):
is_training = False
with tf.Graph().as_default():
with tf.device('/gpu:' + str(GPU_INDEX)):
pointclouds_pl, labels_pl = MODEL.placeholder_inputs(
BATCH_SIZE, NUM_POINT)
is_training_pl = tf.placeholder(tf.bool, shape=())
pert = tf.get_variable(
name='pert',
shape=[BATCH_SIZE, NUM_ADD, 3],
initializer=tf.truncated_normal_initializer(stddev=0.01))
initial_point_pl = tf.placeholder(shape=[BATCH_SIZE, NUM_ADD, 3],
dtype=tf.float32)
point_added = initial_point_pl + pert
pointclouds_input = tf.concat([pointclouds_pl, point_added],
axis=1)
pred, end_points = MODEL.get_model(pointclouds_input,
is_training_pl)
vl = tf.global_variables()
vl = [x for x in vl if 'pert' not in x.name]
saver = tf.train.Saver(vl)
# Create a session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
sess = tf.Session(config=config)
sess.run(tf.global_variables_initializer())
ops = {
'pointclouds_pl': pointclouds_pl,
'labels_pl': labels_pl,
'is_training_pl': is_training_pl,
'pointclouds_input': pointclouds_input,
'initial_point_pl': initial_point_pl,
'pert': pert,
'point_added': point_added,
'pre_max': end_points['pre_max'],
'post_max': end_points['post_max'],
'pred': pred,
}
saver.restore(sess, MODEL_PATH)
#print('model restored!')
# Critical points:
#attacked_data = attacked_data_all[35][:1]
crit_p = MODEL.get_critical_points(sess, ops, attacked_data,
BATCH_SIZE, NUM_ADD, NUM_POINT)
return crit_p
def model(x):
# Example model
with tf.variable_scope('fully_connected', use_resource=True):
weights = tf.get_variable(
'weights', [NUM_UNITS_IN, NUM_UNITS_OUT],
initializer=tf.truncated_normal_initializer(stddev=0.01),
dtype=datatype)
biases = tf.get_variable('biases', [NUM_UNITS_OUT],
initializer=tf.constant_initializer(0.0),
dtype=datatype)
return tf.nn.xw_plus_b(x, weights, biases)
def weight_variable(name, shape, dtype):
"""Xavier initialization"""
stddev = np.sqrt(2.0 / (sum(shape)))
# Do not use a constant as the initializer, that will cause the
# variable to be stored in wrong dtype.
weights = tf.get_variable(name,
shape,
dtype=dtype,
initializer=tf.truncated_normal_initializer(
stddev=stddev, dtype=dtype))
tf.add_to_collection(tf.GraphKeys.WEIGHTS, weights)
return weights
def weight_variable(shape):
"""
Create a weight variable with appropriate initialization
:param name: weight name
:param shape: weight shape
:return: initialized weight variable
"""
initer = tf.truncated_normal_initializer(stddev=0.01)
return tf.get_variable('W',
dtype=tf.float32,
shape=shape,
initializer=initer)
def compute_relative_attention_scores(
queries,
relative_att_ids,
relative_vocab_size,
num_heads,
key_size_per_head,
use_relative_scalar_only = True,
):
"""Computes relative attention scores.
Args:
queries: <float32>[batch_size, query_len, num_heads, key_size_per_head].
Queries for the attention head module.
relative_att_ids: <int32>[batch_size, query_len, key_len]. Relative
attention ids.
relative_vocab_size: Attention bias vocab size.
num_heads: Number of attention heads.
key_size_per_head: Bias embeding dimension.
use_relative_scalar_only: Whether to use bias scalar only.
use_one_hot_lookup: Whether to use bias table lookup or matrix multiplying.
Returns:
<float32>[batch_size, query_len, key_len, num_heads] Tensor.
"""
gather_op = tf.gather
# <float32>[relative_vocab_size, num_heads]
relative_bias_table = tf.get_variable(
name="relative_bias_table",
shape=[relative_vocab_size, num_heads],
initializer=tf.zeros_initializer,
trainable=True)
# <float32>[batch_size, num_heads, query_len, key_len]
relative_att_scores = tf.transpose(
gather_op(relative_bias_table, relative_att_ids), [0, 3, 1, 2])
if use_relative_scalar_only:
return relative_att_scores
# <float32>[relative_vocab_size, num_heads, key_size_per_head]
relative_emb_table = tf.get_variable(
name="relative_emb_table",
shape=[relative_vocab_size, num_heads, key_size_per_head],
initializer=tf.truncated_normal_initializer(stddev=0.02),
trainable=True)
# <float32>[batch_size, query_len, key_len, num_heads, key_size_per_head]
gathered_relative_emb = gather_op(relative_emb_table, relative_att_ids)
# <float32>[batch_size, num_heads, query_len, key_len]
return tf.transpose(
relative_att_scores +
tf.einsum("bqhd,bqkhd->bhqk", queries, gathered_relative_emb),
[0, 3, 1, 2])
def create_model(albert_config, is_training, input_ids, input_mask,
segment_ids, labels, num_labels, use_one_hot_embeddings,
task_name, hub_module):
"""Creates a classification model."""
if hub_module:
tf.logging.info("creating model from hub_module: %s", hub_module)
output_layer = _create_model_from_hub(hub_module, is_training,
input_ids, input_mask,
segment_ids)
else:
tf.logging.info("creating model from albert_config")
output_layer = _create_model_from_scratch(albert_config, is_training,
input_ids, input_mask,
segment_ids,
use_one_hot_embeddings)
hidden_size = output_layer.shape[-1].value
output_weights = tf.get_variable(
"output_weights", [num_labels, hidden_size],
initializer=tf.truncated_normal_initializer(stddev=0.02))
output_bias = tf.get_variable("output_bias", [num_labels],
initializer=tf.zeros_initializer())
with tf.variable_scope("loss"):
if is_training:
# I.e., 0.1 dropout
output_layer = tf.nn.dropout(output_layer, keep_prob=0.9)
logits = tf.matmul(output_layer, output_weights, transpose_b=True)
logits = tf.nn.bias_add(logits, output_bias)
if task_name != "sts-b":
probabilities = tf.nn.softmax(logits, axis=-1)
predictions = tf.argmax(probabilities,
axis=-1,
output_type=tf.int32)
log_probs = tf.nn.log_softmax(logits, axis=-1)
one_hot_labels = tf.one_hot(labels,
depth=num_labels,
dtype=tf.float32)
per_example_loss = -tf.reduce_sum(one_hot_labels * log_probs,
axis=-1)
else:
probabilities = logits
logits = tf.squeeze(logits, [-1])
predictions = logits
per_example_loss = tf.square(logits - labels)
loss = tf.reduce_mean(per_example_loss)
return (loss, per_example_loss, probabilities, logits, predictions)
def initialize_parameters():
"""
Initializes parameters to build a neural network with tensorflow. The shapes are:
W1 : [25, 12288]
b1 : [25, 1]
W2 : [12, 25]
b2 : [12, 1]
W3 : [6, 12]
b3 : [6, 1]
Returns:
parameters -- a dictionary of tensors containing W1, b1, W2, b2, W3, b3
"""
tf.set_random_seed(1) # so that your "random" numbers match ours
### START CODE HERE ### (approx. 6 lines of code)
W1 = tf.get_variable(
"W1", [25, 12288],
initializer=tf.truncated_normal_initializer(stddev=0.1)
) #as contrib deppreciated in tf 2 so X tf.contrib.layers.xavier_initializer(seed = 1)) ##None
b1 = tf.get_variable("b1", [25, 1],
initializer=tf.zeros_initializer()) ##None
W2 = tf.get_variable(
"W2", [12, 25],
initializer=tf.truncated_normal_initializer(stddev=0.1)
) #tf.contrib.layers.xavier_initializer(seed = 1)) ##None
b2 = tf.get_variable("b2", [12, 1],
initializer=tf.zeros_initializer()) ##None
W3 = tf.get_variable(
"W3", [6, 12], initializer=tf.truncated_normal_initializer(stddev=0.1)
) #tf.contrib.layers.xavier_initializer(seed = 1)) ##None
b3 = tf.get_variable("b3", [6, 1],
initializer=tf.zeros_initializer()) ##None
### END CODE HERE ###
parameters = {"W1": W1, "b1": b1, "W2": W2, "b2": b2, "W3": W3, "b3": b3}
return parameters
def affine(inpOp, nIn, nOut, name, weight_decay=0.0):
with tf.variable_scope(name):
l2_regularizer = lambda t: l2_loss(t, weight=weight_decay)
weights = tf.get_variable(
"weights", [nIn, nOut],
initializer=tf.truncated_normal_initializer(stddev=1e-1),
regularizer=l2_regularizer,
dtype=inpOp.dtype)
biases = tf.get_variable("biases", [nOut],
initializer=tf.constant_initializer(),
dtype=inpOp.dtype)
affine1 = tf.nn.relu_layer(inpOp, weights, biases)
return affine1
def test_transformer_decoder_decode_v(self):
vs = 9
h = 4
f = 12
z = 8
es = 4
bs = 5
nl = 4
nh = 2
k = 3
alpha = 0.6
for cond_by_addition in [True, False]:
if cond_by_addition:
z = es
tf.reset_default_graph()
emb_matrix_VxE = tf.get_variable(
'emb',
shape=[vs, es],
initializer=tf.truncated_normal_initializer(stddev=0.01))
decoder = m.TransformerDecoder(num_layers=nl,
num_heads=nh,
hidden_size=h,
filter_size=f,
attention_dropout=0.1,
relu_dropout=0.1,
postprocess_dropout=0.1,
embed_VxE=emb_matrix_VxE,
vocab_size=vs,
max_steps=10,
latent_size=z,
tie_embeddings=False,
cond_by_addition=cond_by_addition)
symb = decoder.decode_v(tf.random.normal((bs, z)))
symbb = decoder.decode_v(tf.random.normal((bs, z)),
method='beam',
first_token=0,
beam_size=k,
alpha=alpha)
with self.session() as ss:
ss.run(tf.initializers.global_variables())
symb_np, symbb_np = ss.run([symb, symbb])
# Check shape
self.assertEqual(bs, symb_np.shape[0])
self.assertEqual(bs, symbb_np.shape[0])
# Check decoded symbols are legit IDs
self.assertIntsInRange(symb_np, 0, vs)
self.assertIntsInRange(symbb_np, 0, vs)
def create_model(bert_config, is_training, input_ids, input_mask, segment_ids,
labels, num_labels, use_one_hot_embeddings, num_segments,
aggregation_method,
pretrained_model='bert', from_distilled_student=False,):
"""Creates a classification model."""
scope = ""
if from_distilled_student:
scope = "student"
parade_model = Parade(
bert_config=bert_config,
is_training=is_training,
input_ids=input_ids,
input_mask=input_mask,
segment_ids=segment_ids,
num_segments=num_segments,
pretrained_model=pretrained_model,
use_one_hot_embeddings=use_one_hot_embeddings,
scope=scope
)
output_layer = None
if aggregation_method == 'cls_attn':
output_layer = parade_model.reduced_by_attn()
elif aggregation_method == 'cls_avg':
output_layer = parade_model.reduced_by_avg()
elif aggregation_method == 'cls_max':
output_layer = parade_model.reduced_by_max()
elif aggregation_method == 'cls_transformer':
output_layer = parade_model.reduced_by_transformer(is_training, num_transformer_layers=2)
else:
raise ValueError("Un-supported model type: {}".format(aggregation_method))
with tf.variable_scope(scope):
output_weights = tf.get_variable(
"output_weights", [num_labels, parade_model.hidden_size],
initializer=tf.truncated_normal_initializer(stddev=0.02))
output_bias = tf.get_variable(
"output_bias", [num_labels], initializer=tf.zeros_initializer())
with tf.variable_scope("loss"):
if is_training:
# I.e., 0.1 dropout
output_layer = tf.nn.dropout(output_layer, keep_prob=0.9)
logits = tf.tensordot(output_layer, output_weights, axes=[-1, -1])
logits = tf.nn.bias_add(logits, output_bias)
log_probs = tf.nn.log_softmax(logits, axis=-1)
one_hot_labels = tf.one_hot(labels, depth=num_labels, dtype=tf.float32)
per_example_loss = -tf.reduce_sum(one_hot_labels * log_probs, axis=-1)
loss = tf.reduce_mean(per_example_loss)
return (loss, per_example_loss, log_probs)
def create_model(albert_config, is_training, input_ids, input_mask,
segment_ids, labels, num_labels, use_one_hot_embeddings,
task_name, hub_module):
"""Creates a classification model."""
(output_layer, _) = fine_tuning_utils.create_albert(
albert_config=albert_config,
is_training=is_training,
input_ids=input_ids,
input_mask=input_mask,
segment_ids=segment_ids,
use_one_hot_embeddings=use_one_hot_embeddings,
use_einsum=False,
hub_module=hub_module)
hidden_size = output_layer.shape[-1]
output_weights = tf.get_variable(
"output_weights", [num_labels, hidden_size],
initializer=tf.truncated_normal_initializer(stddev=0.02))
output_bias = tf.get_variable("output_bias", [num_labels],
initializer=tf.zeros_initializer())
with tf.variable_scope("loss"):
if is_training and not FLAGS.deterministic_run:
# I.e., 0.1 dropout
output_layer = tf.nn.dropout(output_layer, keep_prob=0.9)
logits = tf.matmul(output_layer, output_weights, transpose_b=True)
logits = tf.nn.bias_add(logits, output_bias)
if task_name != "sts-b":
probabilities = tf.nn.softmax(logits, axis=-1)
predictions = tf.argmax(probabilities,
axis=-1,
output_type=tf.int32)
log_probs = tf.nn.log_softmax(logits, axis=-1)
one_hot_labels = tf.one_hot(labels,
depth=num_labels,
dtype=tf.float32)
per_example_loss = -tf.reduce_sum(one_hot_labels * log_probs,
axis=-1)
else:
probabilities = logits
logits = tf.squeeze(logits, [-1])
predictions = logits
per_example_loss = tf.square(logits - labels)
loss = tf.reduce_mean(per_example_loss)
return (loss, per_example_loss, probabilities, logits, predictions)
def f_T(num_action):
# get transition kernel
initializer = tf.truncated_normal_initializer(mean=1.0 / 9.0,
stddev=1.0 / 90.0,
dtype=tf.float32)
kernel = tf.get_variable("w_T_conv", [3 * 3, num_action],
initializer=initializer,
dtype=tf.float32)
# enforce proper probability distribution (i.e. values must sum to one) by softmax
kernel = tf.nn.softmax(kernel, axis=0)
kernel = tf.reshape(kernel, [3, 3, 1, num_action], name="T_w")
return kernel
def __init__(self,
batchnorm_training,
default_batchnorm_momentum=0.999,
conv_hyperparams=None,
use_explicit_padding=False,
alpha=1.0,
min_depth=None,
conv_defs=None):
"""Alternative tf.keras.layers interface, for use by the Keras MobileNetV1.
It is used by the Keras applications kwargs injection API to
modify the MobilenetV1 Keras application with changes required by
the Object Detection API.
These injected interfaces make the following changes to the network:
- Applies the Object Detection hyperparameter configuration
- Supports FreezableBatchNorms
- Adds support for a min number of filters for each layer
- Makes the `alpha` parameter affect the final convolution block even if it
is less than 1.0
- Adds support for explicit padding of convolutions
Args:
batchnorm_training: Bool. Assigned to Batch norm layer `training` param
when constructing `freezable_batch_norm.FreezableBatchNorm` layers.
default_batchnorm_momentum: Float. When 'conv_hyperparams' is None,
batch norm layers will be constructed using this value as the momentum.
conv_hyperparams: A `hyperparams_builder.KerasLayerHyperparams` object
containing hyperparameters for convolution ops. Optionally set to `None`
to use default mobilenet_v1 layer builders.
use_explicit_padding: If True, use 'valid' padding for convolutions,
but explicitly pre-pads inputs so that the output dimensions are the
same as if 'same' padding were used. Off by default.
alpha: The width multiplier referenced in the MobileNetV1 paper. It
modifies the number of filters in each convolutional layer. It's called
depth multiplier in Keras application MobilenetV1.
min_depth: Minimum number of filters in the convolutional layers.
conv_defs: Network layout to specify the mobilenet_v1 body. Default is
`None` to use the default mobilenet_v1 network layout.
"""
self._alpha = alpha
self._batchnorm_training = batchnorm_training
self._default_batchnorm_momentum = default_batchnorm_momentum
self._conv_hyperparams = conv_hyperparams
self._use_explicit_padding = use_explicit_padding
self._min_depth = min_depth
self._conv_defs = conv_defs
self.regularizer = tf.keras.regularizers.l2(0.00004 * 0.5)
self.initializer = tf.truncated_normal_initializer(stddev=0.09)
