def _weight_and_bias(in_size, out_size):
weight = tf.truncated_normal([in_size, out_size], stddev=0.01)
bias = tf.constant(0.1, shape=[out_size])
return tf.Variable(weight), tf.Variable(bias)
def draw_bounding_boxes_on_image_tensors(images,
boxes,
classes,
scores,
category_index,
original_image_spatial_shape=None,
true_image_shape=None,
instance_masks=None,
keypoints=None,
max_boxes_to_draw=20,
min_score_thresh=0.2,
use_normalized_coordinates=True):
"""Draws bounding boxes, masks, and keypoints on batch of image tensors.
Args:
images: A 4D uint8 image tensor of shape [N, H, W, C]. If C > 3, additional
channels will be ignored. If C = 1, then we convert the images to RGB
images.
boxes: [N, max_detections, 4] float32 tensor of detection boxes.
classes: [N, max_detections] int tensor of detection classes. Note that
classes are 1-indexed.
scores: [N, max_detections] float32 tensor of detection scores.
category_index: a dict that maps integer ids to category dicts. e.g.
{1: {1: 'dog'}, 2: {2: 'cat'}, ...}
original_image_spatial_shape: [N, 2] tensor containing the spatial size of
the original image.
true_image_shape: [N, 3] tensor containing the spatial size of unpadded
original_image.
instance_masks: A 4D uint8 tensor of shape [N, max_detection, H, W] with
instance masks.
keypoints: A 4D float32 tensor of shape [N, max_detection, num_keypoints, 2]
with keypoints.
max_boxes_to_draw: Maximum number of boxes to draw on an image. Default 20.
min_score_thresh: Minimum score threshold for visualization. Default 0.2.
use_normalized_coordinates: Whether to assume boxes and kepoints are in
normalized coordinates (as opposed to absolute coordiantes).
Default is True.
Returns:
4D image tensor of type uint8, with boxes drawn on top.
"""
# Additional channels are being ignored.
if images.shape[3] > 3:
images = images[:, :, :, 0:3]
elif images.shape[3] == 1:
images = tf.image.grayscale_to_rgb(images)
visualization_keyword_args = {
'use_normalized_coordinates': use_normalized_coordinates,
'max_boxes_to_draw': max_boxes_to_draw,
'min_score_thresh': min_score_thresh,
'agnostic_mode': False,
'line_thickness': 4
}
if true_image_shape is None:
true_shapes = tf.constant(-1, shape=[images.shape.as_list()[0], 3])
else:
true_shapes = true_image_shape
if original_image_spatial_shape is None:
original_shapes = tf.constant(-1, shape=[images.shape.as_list()[0], 2])
else:
original_shapes = original_image_spatial_shape
if instance_masks is not None and keypoints is None:
visualize_boxes_fn = functools.partial(_visualize_boxes_and_masks,
category_index=category_index,
**visualization_keyword_args)
elems = [
true_shapes, original_shapes, images, boxes, classes, scores,
instance_masks
]
elif instance_masks is None and keypoints is not None:
visualize_boxes_fn = functools.partial(_visualize_boxes_and_keypoints,
category_index=category_index,
**visualization_keyword_args)
elems = [
true_shapes, original_shapes, images, boxes, classes, scores,
keypoints
]
elif instance_masks is not None and keypoints is not None:
visualize_boxes_fn = functools.partial(
_visualize_boxes_and_masks_and_keypoints,
category_index=category_index,
**visualization_keyword_args)
elems = [
true_shapes, original_shapes, images, boxes, classes, scores,
instance_masks, keypoints
]
else:
visualize_boxes_fn = functools.partial(_visualize_boxes,
category_index=category_index,
**visualization_keyword_args)
elems = [true_shapes, original_shapes, images, boxes, classes, scores]
def draw_boxes(image_and_detections):
"""Draws boxes on image."""
true_shape = image_and_detections[0]
original_shape = image_and_detections[1]
if true_image_shape is not None:
image = shape_utils.pad_or_clip_nd(
image_and_detections[2], [true_shape[0], true_shape[1], 3])
if original_image_spatial_shape is not None:
image_and_detections[2] = _resize_original_image(
image, original_shape)
image_with_boxes = tf.py_func(visualize_boxes_fn,
image_and_detections[2:], tf.uint8)
return image_with_boxes
images = tf.map_fn(draw_boxes, elems, dtype=tf.uint8, back_prop=False)
return images
def _model_fn(features, labels, mode, params, model):
"""Model defination for the SSD model based on ResNet-50.
Args:
features: the input image tensor with shape [batch_size, height, width, 3].
The height and width are fixed and equal.
labels: the input labels in a dictionary. The labels include class targets
and box targets which are dense label maps. The labels are generated from
get_input_fn function in data/dataloader.py
mode: the mode of TPUEstimator including TRAIN, EVAL, and PREDICT.
params: the dictionary defines hyperparameters of model. The default
settings are in default_hparams function in this file.
model: the SSD model outputs class logits and box regression outputs.
Returns:
spec: the EstimatorSpec or TPUEstimatorSpec to run training, evaluation,
or prediction.
"""
if mode == tf.estimator.ModeKeys.PREDICT:
labels = features
features = labels.pop('image')
features -= tf.constant(constants.NORMALIZATION_MEAN,
shape=[1, 1, 3],
dtype=features.dtype)
COEF_STD = 1.0 / tf.constant(
constants.NORMALIZATION_STD, shape=[1, 1, 3], dtype=features.dtype)
features *= COEF_STD
def _model_outputs():
return model(features,
params,
is_training_bn=(mode == tf.estimator.ModeKeys.TRAIN))
if params['dtype'] == 'bf16':
with tf.compat.v1.tpu.bfloat16_scope():
cls_outputs, box_outputs = _model_outputs()
levels = cls_outputs.keys()
for level in levels:
cls_outputs[level] = tf.cast(cls_outputs[level], tf.float32)
box_outputs[level] = tf.cast(box_outputs[level], tf.float32)
else:
cls_outputs, box_outputs = _model_outputs()
levels = cls_outputs.keys()
# First check if it is in PREDICT mode.
if mode == tf.estimator.ModeKeys.PREDICT:
flattened_cls, flattened_box = concat_outputs(cls_outputs, box_outputs,
True)
ssd_box_coder = faster_rcnn_box_coder.FasterRcnnBoxCoder(
scale_factors=constants.BOX_CODER_SCALES)
anchors = box_list.BoxList(
tf.convert_to_tensor(dataloader.DefaultBoxes()('ltrb')))
decoded_boxes = box_coder.batch_decode(encoded_boxes=flattened_box,
box_coder=ssd_box_coder,
anchors=anchors)
pred_scores = tf.nn.softmax(flattened_cls, axis=2)
pred_scores, indices = select_top_k_scores(
pred_scores, constants.MAX_NUM_EVAL_BOXES)
predictions = dict(
labels,
indices=indices,
pred_scores=pred_scores,
pred_box=decoded_boxes,
)
if params['visualize_dataloader']:
# this is for inference visualization.
predictions['image'] = features
return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions)
# Load pretrained model from checkpoint.
if params['resnet_checkpoint'] and mode == tf.estimator.ModeKeys.TRAIN:
def scaffold_fn():
"""Loads pretrained model through scaffold function."""
tf.train.init_from_checkpoint(
params['resnet_checkpoint'], {
'/': 'resnet%s/' % constants.RESNET_DEPTH,
})
return tf.train.Scaffold()
else:
scaffold_fn = None
# Set up training loss and learning rate.
update_learning_rate_schedule_parameters(params)
global_step = tf.train.get_or_create_global_step()
learning_rate = learning_rate_schedule(params, global_step)
# cls_loss and box_loss are for logging. only total_loss is optimized.
loss, cls_loss, box_loss = detection_loss(cls_outputs, box_outputs, labels)
total_loss = loss + params['weight_decay'] * tf.add_n(
[tf.nn.l2_loss(v) for v in tf.trainable_variables()])
if mode == tf.estimator.ModeKeys.TRAIN:
optimizer = tf.train.MomentumOptimizer(learning_rate,
momentum=constants.MOMENTUM)
if params['distributed_optimizer']:
optimizer = params['distributed_optimizer'](optimizer)
# Batch norm requires update_ops to be added as a train_op dependency.
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
train_op = tf.group(optimizer.minimize(total_loss, global_step),
update_ops)
save_summary_steps = params['save_summary_steps']
if save_summary_steps is not None:
tf.summary.scalar("learning_rate", learning_rate)
tf.summary.scalar("class_loss", cls_loss)
tf.summary.scalar("box_loss", box_loss)
return model_fn_lib.EstimatorSpec(mode=mode,
loss=loss,
train_op=train_op,
training_hooks=None,
scaffold=scaffold_fn())
if mode == tf.estimator.ModeKeys.EVAL:
raise NotImplementedError
def when_empty():
return tf.constant([], shape=(0, 3), dtype=tf.float64)
def bias_variable(self,shape): initial = tf.constant(0.01, shape = shape) return tf.Variable(initial)
def create_optimizer(loss,
init_lr,
num_train_steps,
num_warmup_steps,
use_tpu,
optimizer="adamw",
poly_power=1.0,
start_warmup_step=0,
colocate_gradients_with_ops=False):
"""Creates an optimizer training op."""
global_step = tf.train.get_or_create_global_step()
learning_rate = tf.constant(value=init_lr, shape=[], dtype=tf.float32)
# Implements linear decay of the learning rate.
learning_rate = tf.train.polynomial_decay(learning_rate,
global_step,
num_train_steps,
end_learning_rate=0.0,
power=poly_power,
cycle=False)
# Implements linear warmup. I.e., if global_step - start_warmup_step <
# num_warmup_steps, the learning rate will be
# `(global_step - start_warmup_step)/num_warmup_steps * init_lr`.
if num_warmup_steps:
tf.logging.info("++++++ warmup starts at step " +
str(start_warmup_step) + ", for " +
str(num_warmup_steps) + " steps ++++++")
global_steps_int = tf.cast(global_step, tf.int32)
start_warm_int = tf.constant(start_warmup_step, dtype=tf.int32)
global_steps_int = global_steps_int - start_warm_int
warmup_steps_int = tf.constant(num_warmup_steps, dtype=tf.int32)
global_steps_float = tf.cast(global_steps_int, tf.float32)
warmup_steps_float = tf.cast(warmup_steps_int, tf.float32)
warmup_percent_done = global_steps_float / warmup_steps_float
warmup_learning_rate = init_lr * warmup_percent_done
is_warmup = tf.cast(global_steps_int < warmup_steps_int, tf.float32)
learning_rate = ((1.0 - is_warmup) * learning_rate +
is_warmup * warmup_learning_rate)
# It is OK that you use this optimizer for finetuning, since this
# is how the model was trained (note that the Adam m/v variables are NOT
# loaded from init_checkpoint.)
# It is OK to use AdamW in the finetuning even the model is trained by LAMB.
# As report in the Bert pulic github, the learning rate for SQuAD 1.1 finetune
# is 3e-5, 4e-5 or 5e-5. For LAMB, the users can use 3e-4, 4e-4,or 5e-4 for a
# batch size of 64 in the finetune.
optimizer_name = optimizer
if optimizer == "adamw":
tf.logging.info("using adamw")
optimizer = AdamWeightDecayOptimizer(
learning_rate=learning_rate,
weight_decay_rate=0.01,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-6,
exclude_from_weight_decay=["LayerNorm", "layer_norm", "bias"])
elif optimizer == "lamb":
tf.logging.info("using lamb")
optimizer = lamb_optimizer.LAMBOptimizer(
learning_rate=learning_rate,
weight_decay_rate=0.01,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-6,
exclude_from_weight_decay=["LayerNorm", "layer_norm", "bias"])
elif optimizer == "adafactor":
tf.logging.info("using adafactor")
optimizer = AdafactorOptimizer(multiply_by_parameter_scale=True,
learning_rate=learning_rate,
decay_rate=None,
beta1=0.0,
clipping_threshold=1.0,
factored=True,
simulated_quantize_bits=None,
parameter_encoding=None,
use_locking=False,
name="Adafactor",
epsilon1=1e-30,
epsilon2=1e-3)
else:
raise ValueError("Not supported optimizer: ", optimizer)
if use_tpu:
optimizer = contrib_tpu.CrossShardOptimizer(optimizer)
tvars = tf.trainable_variables()
grads = tf.gradients(
loss, tvars, colocate_gradients_with_ops=colocate_gradients_with_ops)
# This is how the model was pre-trained.
(grads, _) = tf.clip_by_global_norm(grads, clip_norm=1.0)
train_op = optimizer.apply_gradients(list(zip(grads, tvars)),
global_step=global_step)
# Normally the global step update is done inside of `apply_gradients`.
# However, neither `AdamWeightDecayOptimizer` nor `LAMBOptimizer` do this.
# But if you use a different optimizer, you should probably take this line
# out.
if optimizer_name != "adafactor":
new_global_step = global_step + 1
train_op = tf.group(train_op, [global_step.assign(new_global_step)])
return train_op
def main(unused_argv=None):
with tf.Graph().as_default():
# Force all input processing onto CPU in order to reserve the GPU for the
# forward inference and back-propagation.
device = '/cpu:0' if not FLAGS.ps_tasks else '/job:worker/cpu:0'
with tf.device(
tf.train.replica_device_setter(FLAGS.ps_tasks,
worker_device=device)):
inputs, _ = image_utils.imagenet_inputs(FLAGS.batch_size,
FLAGS.image_size)
# Load style images and select one at random (for each graph execution, a
# new random selection occurs)
style_images, style_labels, \
style_gram_matrices = image_utils.style_image_inputs(
os.path.expanduser(FLAGS.style_dataset_file),
batch_size=FLAGS.batch_size,
image_size=FLAGS.image_size,
square_crop=True,
shuffle=True)
with tf.device(tf.train.replica_device_setter(FLAGS.ps_tasks)):
# Process style and weight flags
num_styles = FLAGS.num_styles
if FLAGS.style_coefficients is None:
style_coefficients = [1.0 for _ in range(num_styles)]
else:
style_coefficients = ast.literal_eval(FLAGS.style_coefficients)
if len(style_coefficients) != num_styles:
raise ValueError(
'number of style coefficients differs from number of styles'
)
content_weights = ast.literal_eval(FLAGS.content_weights)
style_weights = ast.literal_eval(FLAGS.style_weights)
# Rescale style weights dynamically based on the current style image
style_coefficient = tf.gather(tf.constant(style_coefficients),
style_labels)
style_weights = dict((key, style_coefficient * style_weights[key])
for key in style_weights)
# Define the model
stylized_inputs = model.transform(inputs,
alpha=FLAGS.alpha,
normalizer_params={
'labels': style_labels,
'num_categories': num_styles,
'center': True,
'scale': True
})
# Compute losses.
total_loss, loss_dict = learning.total_loss(
inputs, stylized_inputs, style_gram_matrices, content_weights,
style_weights)
for key in loss_dict:
tf.summary.scalar(key, loss_dict[key])
# Adding Image summaries to the tensorboard.
tf.summary.image('image/0_inputs', inputs, 3)
tf.summary.image('image/1_styles', style_images, 3)
tf.summary.image('image/2_styled_inputs', stylized_inputs, 3)
# Set up training
optimizer = tf.train.AdamOptimizer(FLAGS.learning_rate)
train_op = slim.learning.create_train_op(
total_loss,
optimizer,
clip_gradient_norm=FLAGS.clip_gradient_norm,
summarize_gradients=False)
# Function to restore VGG16 parameters.
init_fn_vgg = slim.assign_from_checkpoint_fn(
vgg.checkpoint_file(), slim.get_variables('vgg_16'))
# Run training
slim.learning.train(train_op=train_op,
logdir=os.path.expanduser(FLAGS.train_dir),
master=FLAGS.master,
is_chief=FLAGS.task == 0,
number_of_steps=FLAGS.train_steps,
init_fn=init_fn_vgg,
save_summaries_secs=FLAGS.save_summaries_secs,
save_interval_secs=FLAGS.save_interval_secs)
def dataset_parser(self, value):
"""See base class."""
if not self.data_dir:
return value, tf.constant(0., tf.float32, (1000, ))
return super(ImageNetInput, self).dataset_parser(value)
def _update_mask(self, weights, threshold, gradients): # pylint: disable=unused-argument
"""Updates the mask for a given weight tensor.
This functions first computes the cdf of the weight tensor, and estimates
the threshold value such that 'desired_sparsity' fraction of weights
have magnitude less than the threshold.
Args:
weights: The weight tensor that needs to be masked.
threshold: The current threshold value. The function will compute a new
threshold and return the exponential moving average using the current
value of threshold
gradients: The gradient tensor that is used for salience calculation.
Returns:
new_threshold: The new value of the threshold based on weights, and
sparsity at the current global_step
new_mask: A numpy array of the same size and shape as weights containing
0 or 1 to indicate which of the values in weights falls below
the threshold
Raises:
ValueError: if sparsity is not defined
"""
if self._sparsity is None:
raise ValueError('Sparsity variable undefined')
sparsity = self._get_sparsity(weights.op.name)
with tf.name_scope(weights.op.name + '_pruning_ops'):
tf.logging.info('Applying option %s pruning',
self._spec.prune_option)
if self._spec.prune_option == 'weight':
abs_weights = tf.abs(weights)
elif self._spec.prune_option in ('first_order_gradient',
'second_order_gradient'):
if gradients is None:
raise ValueError('gradient tensor cannot be None.')
# gradient variable stores absolute value already
abs_weights = tf.multiply(tf.abs(weights), gradients)
else:
raise ValueError('undefined option')
k = tf.cast(
tf.round(
tf.cast(tf.size(abs_weights), tf.float32) *
(1 - sparsity)), tf.int32)
# Generate a random shuffling of the weights s.t. the tie-breaker on
# weight magnitude is random uniform.
shuffling = tf.random_shuffle(tf.range(tf.size(abs_weights)))
shuffling = tf.reshape(shuffling, [-1, 1])
# Flatten the weights and scatter the values randomly.
abs_weights = tf.reshape(abs_weights, [-1])
abs_weights = tf.scatter_nd(shuffling, abs_weights,
tf.shape(abs_weights))
# Sort the entire array
_, indices = tf.nn.top_k(abs_weights, k=tf.size(abs_weights))
# `k` is how many non-zero weights we're going to have. Create a new
# mask where the first `k` elements are set to one and all others are
# set to zero.
mask_staging = tf.range(tf.size(abs_weights))
mask_staging = tf.cast(tf.less(mask_staging, k), tf.float32)
# Scatter the mask back into the proper positions for the weight matrix.
indices = tf.reshape(indices, [-1, 1])
new_mask = tf.scatter_nd(indices, mask_staging,
tf.shape(mask_staging))
# Un-shuffle the newly created mask.
new_mask = tf.reshape(tf.gather_nd(new_mask, shuffling),
tf.shape(weights))
return tf.constant(0, tf.float32), new_mask
def constant_3d_image(self):
return tf.constant([[[-100, 2], [2, 3]], [[4, 35], [-1024, 7]]],
dtype=tf.float32)
def constant_3d_label(self):
return tf.constant([[[0, 0], [0, 1]], [[1, 1], [2, 2]]],
dtype=tf.float32)
net = tf_util.max_pool2d(net, [num_point,1],
padding='VALID', scope='maxpool')
# MLP on global point cloud vector
net = tf.reshape(net, [batch_size, -1])
net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training,
scope='fc1', bn_decay=bn_decay)
net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training,
scope='fc2', bn_decay=bn_decay)
net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training,
scope='dp1')
net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3')
return net, end_points
def get_loss(pred, label, end_points):
""" pred: B*NUM_CLASSES,
label: B, """
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=pred, labels=label)
classify_loss = tf.reduce_mean(loss)
tf.summary.scalar('classify loss', classify_loss)
return classify_loss
if __name__=='__main__':
with tf.Graph().as_default():
inputs = tf.zeros((32,1024,3))
outputs = get_model(inputs, tf.constant(True))
print(outputs)
def _greedy_decode(input_embeddings,
output_vocab_size,
target_end_id,
target_start_id,
output_vocab_embeddings_table,
source_len,
model_config,
mode,
input_copy_mask=None):
"""Fast decoding."""
encoder_output = common_layers.linear_transform(
input_embeddings,
output_size=model_config.model_parameters.encoder_dims,
scope="bert_to_transformer")
decode_length = model_config.data_options.max_decode_length
# Expand the inputs in to the beam width.
def symbols_to_logits_fn(logit_indices, current_index):
"""Go from targets to logits."""
logit_indices = tf.expand_dims(logit_indices, 0)
decode_steps = decode_utils.get_decode_steps(logit_indices,
output_vocab_size,
model_config)
target_embeddings = _get_target_embeddings(
input_embeddings, output_vocab_embeddings_table, decode_steps,
model_config)
decoder_output = _build_transformer_decoder(
encoder_output,
source_len,
target_embeddings,
mode,
model_config,
single_step_index=current_index)
logits = _get_action_logits(encoder_output,
decoder_output,
output_vocab_embeddings_table,
output_vocab_size,
model_config,
input_copy_mask=input_copy_mask)
# Squeeze batch dimension and length dimension, as both should be 1.
logits = tf.squeeze(logits, axis=[0, 1])
# Shape of logits should now be (output_vocab_size).
return logits
def loop_cond(i, decoded_ids, unused_logprobs):
"""Loop conditional that returns false to stop loop."""
return tf.logical_and(
tf.reduce_all(tf.not_equal(decoded_ids, target_end_id)),
tf.less(i, decode_length))
def inner_loop(i, decoded_ids, logprobs):
"""Decoder function invoked on each while loop iteration."""
logits = symbols_to_logits_fn(decoded_ids, i)
next_id = tf.argmax(logits, axis=0)
softmax = tf.nn.softmax(logits)
extended_vocab_size = tf.shape(softmax)[-1]
mask = tf.one_hot(next_id, extended_vocab_size)
prob = tf.reduce_sum(softmax * mask)
logprob = tf.log(prob)
# Add one-hot values to output Tensors, since values at index > i+1 should
# still be zero.
logprobs += tf.one_hot(i + 1,
decode_length + 1,
on_value=logprob,
dtype=tf.float32)
decoded_ids += tf.one_hot(i + 1,
decode_length + 1,
on_value=next_id,
dtype=tf.int64)
return i + 1, decoded_ids, logprobs
initial_ids = tf.zeros(dtype=tf.int64, shape=[decode_length + 1])
initial_ids += tf.one_hot(0,
decode_length + 1,
on_value=tf.cast(target_start_id, tf.int64))
initial_logprob = tf.zeros(dtype=tf.float32, shape=[decode_length + 1])
initial_i = tf.constant(0)
initial_values = [initial_i, initial_ids, initial_logprob]
_, decoded_ids, logprobs = tf.while_loop(loop_cond, inner_loop,
initial_values)
# Remove <START> symbol.
decoded_ids = decoded_ids[1:]
logprobs = logprobs[1:]
# Sum logprobs to get scores for overall sequence.
logprobs = tf.reduce_sum(logprobs, axis=0)
# Expand decoded_ids and logprobs to reflect beam width dimension of 1.
decoded_ids = tf.expand_dims(decoded_ids, 0)
logprobs = tf.expand_dims(logprobs, 0)
# This is the output dict that the function returns.
output_decode_steps = decode_utils.get_decode_steps(
decoded_ids, output_vocab_size, model_config)
predictions = decode_utils.get_predictions(output_decode_steps)
predictions[constants.SCORES_KEY] = logprobs
return predictions
def _conv_layer(input, weights, bias):
conv = tf.nn.conv2d(input, tf.constant(weights), strides=(1, 1, 1, 1), padding='SAME')
return tf.nn.bias_add(conv, bias)
def test_dense_feature_scaler(self):
tensor = tf.constant([-15., -12., -20., 5., -100.])
expected = [0, 0.6, -1., 1., -1.]
scaler = run_finetuning_lib.DenseFeatureScaler(min_value=-20, max_value=-10)
self.assertAllClose(expected, scaler.transform(tensor))
def _bias_variable(shape):
initial = tf.constant(0.1, shape = shape)
return tf.Variable(initial)
def _build(self, z):
if isinstance(z, np.ndarray):
z = tf.constant(z)
return self.mlp(z)
def test_train(args):
"""Trains the model."""
if args.verbose:
tf.logging.set_verbosity(tf.logging.INFO)
# Create input data pipeline.
with tf.device("/cpu:0"):
train_files = glob.glob(args.train_glob)
if not train_files:
raise RuntimeError(
"No training images found with glob '{}'.".format(
args.train_glob))
train_dataset = tf.data.Dataset.from_tensor_slices(train_files)
train_dataset = train_dataset.shuffle(
buffer_size=len(train_files)).repeat()
train_dataset = train_dataset.map(
read_png, num_parallel_calls=args.preprocess_threads)
train_dataset = train_dataset.map(
lambda x: tf.random_crop(x, (args.patchsize, args.patchsize, 3)))
train_dataset = train_dataset.batch(args.batchsize)
train_dataset = train_dataset.prefetch(32)
num_pixels = args.batchsize * args.patchsize**2
# Get training patch from dataset.
x = train_dataset.make_one_shot_iterator().get_next()
# Instantiate model.
analysis_transform = AnalysisTransform(args.num_filters)
synthesis_transform = SynthesisTransform(args.num_filters)
hyper_analysis_transform = HyperAnalysisTransform(args.num_filters)
hyper_synthesis_transform = HyperSynthesisTransform(args.num_filters)
entropy_bottleneck = DynamicEntropyBottleneck(name="entropy_bottleneck")
# Build autoencoder and hyperprior.
y = analysis_transform(x)
z = hyper_analysis_transform(abs(y))
z_tilde, z_likelihoods = entropy_bottleneck(z, training=True)
sigma = hyper_synthesis_transform(z_tilde)
scale_table = np.exp(
np.linspace(np.log(SCALES_MIN), np.log(SCALES_MAX), SCALES_LEVELS))
conditional_bottleneck = tfc.GaussianConditional(sigma, scale_table)
y_tilde, y_likelihoods = conditional_bottleneck(y, training=True)
incep = Intercept(32, 256, 1)
y_incep = incep(y_tilde)
x_tilde = synthesis_transform(y_incep)
train_bpp = (tf.reduce_sum(tf.log(y_likelihoods)) + tf.reduce_sum(
tf.log(z_likelihoods))) / (-np.log(2) * num_pixels)
train_mse = tf.reduce_mean(tf.squared_difference(x, x_tilde)) * (255**2)
def f(W):
return 10**(-W / 32.0 + 4.5)
dist = np.zeros(256)
dist[:32] = f(32)
for i in range(32, 256):
dist[i] = f(i + 1)
lambda_dist = tf.constant(dist, dtype=tf.float32)
train_bpp_dist = ( tf.reduce_sum(tf.log(y_likelihoods), axis=(0,1,2)) + \
tf.reduce_sum(tf.log(z_likelihoods), axis=(0,1,2)) ) / (-np.log(2) * num_pixels)
# The rate-distortion cost.
train_loss = tf.reduce_sum(lambda_dist * train_bpp_dist) + train_mse
# Minimize loss and auxiliary loss, and execute update op.
#with tf.Session() as sess:
# latest = tf.train.latest_checkpoint(checkpoint_dir="./e2")
# tf.train.Saver().restore(sess, save_path=latest)
step = tf.train.create_global_step()
main_optimizer = tf.train.AdamOptimizer(learning_rate=1e-4)
aux_optimizer = tf.train.AdamOptimizer(learning_rate=1e-3)
main_step = main_optimizer.minimize(train_loss, global_step=step)
aux_step = aux_optimizer.minimize(entropy_bottleneck.losses[0])
train_op = tf.group(main_step, aux_step, entropy_bottleneck.updates[0])
tf.summary.scalar("loss", train_loss)
tf.summary.scalar("bpp", train_bpp)
tf.summary.scalar("mse", train_mse)
tf.summary.image("original", quantize_image(x))
tf.summary.image("reconstruction", quantize_image(x_tilde))
hooks = [
tf.train.StopAtStepHook(last_step=args.last_step),
tf.train.NanTensorHook(train_loss),
]
with tf.train.MonitoredTrainingSession(hooks=hooks,
checkpoint_dir=args.checkpoint_dir,
save_checkpoint_secs=300,
save_summaries_secs=60) as sess:
while not sess.should_stop():
sess.run(train_op)
def interpolate(alpha_tf, x, y):
x = tf.constant(x, dtype=tf.float32)
y = tf.constant(y, dtype=tf.float32)
range = tf.math.subtract(y, x)
delta = tf.math.multiply(alpha_tf, range)
return tf.math.add(x, delta)
def test_compress(args):
"""Compresses an image."""
fn = tf.placeholder(tf.string, [])
# Load input image and add batch dimension.
x = read_png(fn)
x = tf.expand_dims(x, 0)
x.set_shape([1, None, None, 3])
x_shape = tf.shape(x)
# Instantiate model.
analysis_transform = AnalysisTransform(args.num_filters)
synthesis_transform = SynthesisTransform(args.num_filters)
hyper_analysis_transform = HyperAnalysisTransform(args.num_filters)
hyper_synthesis_transform = HyperSynthesisTransform(args.num_filters)
entropy_bottleneck = tfc.EntropyBottleneck()
# Transform and compress the image.
y = analysis_transform(x)
y_shape = tf.shape(y)
z = hyper_analysis_transform(abs(y))
z_hat, z_likelihoods = entropy_bottleneck(z, training=False)
sigma = hyper_synthesis_transform(z_hat)
sigma = sigma[:, :y_shape[1], :y_shape[2], :]
scale_table = np.exp(
np.linspace(np.log(SCALES_MIN), np.log(SCALES_MAX), SCALES_LEVELS))
conditional_bottleneck = DynamicGaussianConditional(
sigma, scale_table, name="gaussian_conditional")
side_string = entropy_bottleneck.compress(z)
string = conditional_bottleneck.compress(y)
# Transform the quantized image back (if requested).
y_hat, y_likelihoods = conditional_bottleneck(y, training=False)
x_hat = synthesis_transform(y_hat)
x_hat = x_hat[:, :x_shape[1], :x_shape[2], :]
num_pixels = tf.cast(tf.reduce_prod(tf.shape(x)[:-1]), dtype=tf.float32)
# Total number of bits divided by number of pixels.
eval_bpp = (tf.reduce_sum(tf.log(y_likelihoods)) + tf.reduce_sum(
tf.log(z_likelihoods))) / (-np.log(2) * num_pixels)
# Bring both images back to 0..255 range.
x *= 255
x_hat = tf.clip_by_value(x_hat, 0, 1)
x_hat = tf.round(x_hat * 255)
mse = tf.reduce_mean(tf.squared_difference(x, x_hat))
psnr = tf.squeeze(tf.image.psnr(x_hat, x, 255))
msssim = tf.squeeze(tf.image.ssim_multiscale(x_hat, x, 255))
with tf.Session() as sess:
# Load the latest model checkpoint, get the compressed string and the tensor
# shapes.
latest = tf.train.latest_checkpoint(checkpoint_dir=args.checkpoint_dir)
tf.train.Saver().restore(sess, save_path=latest)
#a = sess.run( tf.reduce_sum(tf.log(y_likelihoods), axis=(0,1,2)) / (-np.log(2) * num_pixels))
#b = sess.run( tf.reduce_sum(tf.log(z_likelihoods), axis=(0,1,2)) / (-np.log(2) * num_pixels))
#np.savetxt('ay.csv', a, delimiter = ',')
#np.savetxt('bz.csv', b, delimiter = ',')
#return
const = tf.constant([1] * 256 + [0] * 224, dtype=tf.float32)
f = open("e5.csv", "w")
print("active, fn, bpp, mse, np", file=f)
for active in range(256, 31, -16):
#conditional_bottleneck.input_spec = tf.keras.layers.InputSpec(ndim=4, axes={3: active})
mask = const[256 - active:512 - active]
rate = tf.reduce_sum(mask) / 256
y_itc = y * mask / rate
string = conditional_bottleneck.compress(y_itc)
y_itc_hat = conditional_bottleneck.decompress(string)
# Transform the quantized image back (if requested).
x_hat = synthesis_transform(y_itc_hat)
x_hat = x_hat[:, :x_shape[1], :x_shape[2], :]
eval_bpp = (tf.reduce_sum(tf.log(y_likelihoods[:, :, :, :active]))
+ tf.reduce_sum(tf.log(z_likelihoods))) / (-np.log(2) *
num_pixels)
x_hat = tf.clip_by_value(x_hat, 0, 1)
x_hat = tf.round(x_hat * 255)
mse = tf.reduce_mean(tf.squared_difference(x, x_hat))
psnr = tf.squeeze(tf.image.psnr(x_hat, x, 255))
msssim = tf.squeeze(tf.image.ssim_multiscale(x_hat, x, 255))
#tensors = [string, side_string,
# tf.shape(x)[1:-1], tf.shape(y)[1:-1], tf.shape(z)[1:-1]]
#arrays = sess.run(tensors)
# Write a binary file with the shape information and the compressed string.
#packed = tfc.PackedTensors()
#packed.pack(tensors, arrays)
for filename in glob.glob("kodak/*.png"):
v_eval_bpp, v_mse, v_num_pixels = sess.run(
[eval_bpp, mse, num_pixels], feed_dict={fn: filename})
print("%.2f, %s, %.4f, %.4f, %d" %
(active, filename, v_eval_bpp, v_mse, v_num_pixels),
file=f)
f.close()
import tensorflow.compat.v1 as tf
# 构建待拼接的二维张量t1
t1 = tf.constant([[1, 2, 3], [4, 5, 6]], tf.float32)
# 构建待拼接的二维张量t2
t2 = tf.constant([[4, 5], [7, 8]], tf.float32)
# 利用函数concat来实现对t1和t2的拼接
t = tf.concat([t1, t2], axis=1)
session = tf.Session()
# 打印结果
print("拼接后的张量：\n", session.run(t))
#1.0 버전으로 변경
import tensorflow.compat.v1 as tf
tf.disable_v2_behavior()
# 2.0 버전
# import tensorflow as tf
hello = tf.constant("hello")
sess = tf.Session()
print(sess.run(hello))
def build_network(self):
with tf.variable_scope(self.net_name):
#Model Placeholders
self.num_input = tf.placeholder(tf.float32,
shape=[None, *self.input_dims[0]],
name='num_inputs')
self.img_input = tf.placeholder(tf.float32,
shape=[None, *self.input_dims[1]],
name='img_inputs')
self.img_size = tf.constant([45, 80], dtype=tf.int32)
self.action_gradient = tf.placeholder(tf.float32,
shape=[None, self.n_actions],
name='action_gradient')
#Model building layers
c1 = 1 / np.sqrt(self.conv1_filters * (self.conv1_kernel_size + 1))
self.conv_layer_1 = k_conv2d(
filters=self.conv1_filters,
kernel_size=self.conv1_kernel_size,
kernel_initializer=k_rand_uniform(-c1, c1),
bias_initializer=k_rand_uniform(-c1, c1))
c2 = 1 / np.sqrt(self.conv2_filters * (self.conv2_kernel_size + 1))
self.conv_layer_2 = k_conv2d(
filters=self.conv2_filters,
kernel_size=self.conv2_kernel_size,
kernel_initializer=k_rand_uniform(-c2, c2),
bias_initializer=k_rand_uniform(-c2, c2))
c3 = 1 / np.sqrt(self.conv3_filters * (self.conv3_kernel_size + 1))
self.conv_layer_3 = k_conv2d(
filters=self.conv3_filters,
kernel_size=self.conv3_kernel_size,
kernel_initializer=k_rand_uniform(-c3, c3),
bias_initializer=k_rand_uniform(-c3, c3))
f4 = 1 / np.sqrt(self.fc1_dims)
self.dense_layer_4_img = k_dense(
units=self.fc1_dims,
kernel_initializer=k_rand_uniform(-f4, f4),
bias_initializer=k_rand_uniform(-f4, f4))
self.dense_layer_4_num = k_dense(
units=self.fc1_dims,
input_shape=(13, ), #
kernel_initializer=k_rand_uniform(-f4, f4),
bias_initializer=k_rand_uniform(-f4, f4))
f5 = 1 / np.sqrt(self.fc2_dims)
self.dense_layer_5 = k_dense(
units=self.fc2_dims,
kernel_initializer=k_rand_uniform(-f5, f5),
bias_initializer=k_rand_uniform(-f5, f5))
f6 = 0.0003
self.dense_layer_6 = k_dense(
units=self.n_actions,
kernel_initializer=k_rand_uniform(-f6, f6),
bias_initializer=k_rand_uniform(-f6, f6))
#The Model
#Conv2D 1
img_cnn = tf.image.resize(
images=self.img_input,
size=self.img_size,
method=tf.image.ResizeMethod.NEAREST_NEIGHBOR,
preserve_aspect_ratio=True)
conv1 = self.conv_layer_1(img_cnn)
batch1 = k_batch_norm()(conv1)
layer1_activation = k_relu(batch1)
#Conv2D 2
conv2 = self.conv_layer_2(layer1_activation)
batch2 = k_batch_norm()(conv2)
layer2_activation = k_relu(batch2)
#Conv2D 3
conv3 = self.conv_layer_3(layer2_activation)
batch3 = k_batch_norm()(conv3)
layer3_activation = k_relu(batch3)
layer3_flat = k_flatten()(layer3_activation)
#Dense 4 Flattened Image Input + Numeric Input
img_in = self.dense_layer_4_img(layer3_flat)
img_batch = k_batch_norm()(img_in)
num_in = self.dense_layer_4_num(self.num_input)
batch4 = k_batch_norm()(num_in)
input_batch = tf.add(batch4, img_batch)
layer4_activation = k_relu(input_batch)
#Dense 5
dense5 = self.dense_layer_5(layer4_activation)
batch5 = k_batch_norm()(dense5)
layer5_activation = k_relu(batch5)
#Dense 6 Actions
mu = self.dense_layer_6(layer5_activation)
self.mu = tf.math.multiply(mu, self.action_bound)
def build(input_reader_config,
model_config,
lstm_config,
unroll_length,
data_augmentation_options=None,
batch_size=1):
"""Builds a tensor dictionary based on the InputReader config.
Args:
input_reader_config: An input_reader_builder.InputReader object.
model_config: A model.proto object containing the config for the desired
DetectionModel.
lstm_config: LSTM specific configs.
unroll_length: Unrolled length for LSTM training.
data_augmentation_options: A list of tuples, where each tuple contains a
data augmentation function and a dictionary containing arguments and their
values (see preprocessor.py).
batch_size: Batch size for queue outputs.
Returns:
A dictionary of tensors based on items in the input_reader_config.
Raises:
ValueError: On invalid input reader proto.
ValueError: If no input paths are specified.
"""
if not isinstance(input_reader_config, input_reader_pb2.InputReader):
raise ValueError('input_reader_config not of type '
'input_reader_pb2.InputReader.')
external_reader_config = input_reader_config.external_input_reader
external_input_reader_config = external_reader_config.Extensions[
input_reader_google_pb2.GoogleInputReader.google_input_reader]
input_reader_type = external_input_reader_config.WhichOneof('input_reader')
if input_reader_type == 'tf_record_video_input_reader':
config = external_input_reader_config.tf_record_video_input_reader
reader_type_class = tf.TFRecordReader
else:
raise ValueError(
'Unsupported reader in input_reader_config: %s' % input_reader_type)
if not config.input_path:
raise ValueError('At least one input path must be specified in '
'`input_reader_config`.')
key, value = parallel_reader.parallel_read(
config.input_path[:], # Convert `RepeatedScalarContainer` to list.
reader_class=reader_type_class,
num_epochs=(input_reader_config.num_epochs
if input_reader_config.num_epochs else None),
num_readers=input_reader_config.num_readers,
shuffle=input_reader_config.shuffle,
dtypes=[tf.string, tf.string],
capacity=input_reader_config.queue_capacity,
min_after_dequeue=input_reader_config.min_after_dequeue)
# TODO(yinxiao): Add loading instance mask option.
decoder = tf_sequence_example_decoder.TFSequenceExampleDecoder()
keys_to_decode = [
fields.InputDataFields.image, fields.InputDataFields.groundtruth_boxes,
fields.InputDataFields.groundtruth_classes
]
tensor_dict = decoder.decode(value, items=keys_to_decode)
tensor_dict['image'].set_shape([None, None, None, 3])
tensor_dict['groundtruth_boxes'].set_shape([None, None, 4])
height = model_config.ssd.image_resizer.fixed_shape_resizer.height
width = model_config.ssd.image_resizer.fixed_shape_resizer.width
# If data augmentation is specified in the config file, the preprocessor
# will be called here to augment the data as specified. Most common
# augmentations include horizontal flip and cropping.
if data_augmentation_options:
images_pre = tf.split(tensor_dict['image'], config.video_length, axis=0)
bboxes_pre = tf.split(
tensor_dict['groundtruth_boxes'], config.video_length, axis=0)
labels_pre = tf.split(
tensor_dict['groundtruth_classes'], config.video_length, axis=0)
images_proc, bboxes_proc, labels_proc = [], [], []
cache = preprocessor_cache.PreprocessorCache()
for i, _ in enumerate(images_pre):
image_dict = {
fields.InputDataFields.image:
images_pre[i],
fields.InputDataFields.groundtruth_boxes:
tf.squeeze(bboxes_pre[i], axis=0),
fields.InputDataFields.groundtruth_classes:
tf.squeeze(labels_pre[i], axis=0),
}
image_dict = preprocessor.preprocess(
image_dict,
data_augmentation_options,
func_arg_map=preprocessor.get_default_func_arg_map(),
preprocess_vars_cache=cache)
# Pads detection count to _PADDING_SIZE.
image_dict[fields.InputDataFields.groundtruth_boxes] = tf.pad(
image_dict[fields.InputDataFields.groundtruth_boxes],
[[0, _PADDING_SIZE], [0, 0]])
image_dict[fields.InputDataFields.groundtruth_boxes] = tf.slice(
image_dict[fields.InputDataFields.groundtruth_boxes], [0, 0],
[_PADDING_SIZE, -1])
image_dict[fields.InputDataFields.groundtruth_classes] = tf.pad(
image_dict[fields.InputDataFields.groundtruth_classes],
[[0, _PADDING_SIZE]])
image_dict[fields.InputDataFields.groundtruth_classes] = tf.slice(
image_dict[fields.InputDataFields.groundtruth_classes], [0],
[_PADDING_SIZE])
images_proc.append(image_dict[fields.InputDataFields.image])
bboxes_proc.append(image_dict[fields.InputDataFields.groundtruth_boxes])
labels_proc.append(image_dict[fields.InputDataFields.groundtruth_classes])
tensor_dict['image'] = tf.concat(images_proc, axis=0)
tensor_dict['groundtruth_boxes'] = tf.stack(bboxes_proc, axis=0)
tensor_dict['groundtruth_classes'] = tf.stack(labels_proc, axis=0)
else:
# Pads detection count to _PADDING_SIZE per frame.
tensor_dict['groundtruth_boxes'] = tf.pad(
tensor_dict['groundtruth_boxes'], [[0, 0], [0, _PADDING_SIZE], [0, 0]])
tensor_dict['groundtruth_boxes'] = tf.slice(
tensor_dict['groundtruth_boxes'], [0, 0, 0], [-1, _PADDING_SIZE, -1])
tensor_dict['groundtruth_classes'] = tf.pad(
tensor_dict['groundtruth_classes'], [[0, 0], [0, _PADDING_SIZE]])
tensor_dict['groundtruth_classes'] = tf.slice(
tensor_dict['groundtruth_classes'], [0, 0], [-1, _PADDING_SIZE])
tensor_dict['image'], _ = preprocessor.resize_image(
tensor_dict['image'], new_height=height, new_width=width)
num_steps = config.video_length / unroll_length
init_states = {
'lstm_state_c':
tf.zeros([height / 32, width / 32, lstm_config.lstm_state_depth]),
'lstm_state_h':
tf.zeros([height / 32, width / 32, lstm_config.lstm_state_depth]),
'lstm_state_step':
tf.constant(num_steps, shape=[]),
}
batch = sqss.batch_sequences_with_states(
input_key=key,
input_sequences=tensor_dict,
input_context={},
input_length=None,
initial_states=init_states,
num_unroll=unroll_length,
batch_size=batch_size,
num_threads=batch_size,
make_keys_unique=True,
capacity=batch_size * batch_size)
return _build_training_batch_dict(batch, unroll_length, batch_size)
def lagrangian_optimizer_kld(train_set, additive_slack, learning_rate,
learning_rate_constraint, loops):
"""Implements surrogate-based Lagrangian optimizer (Algorithm 2).
Specifically solves:
min_{theta} sum_{G = 0, 1} KLD(p, pprG(theta))
s.t. error_rate <= additive_slack,
where p is the overall proportion of positives and pprG is the positive
prediction rate for group G.
We frame this as a constrained optimization problem:
min_{theta, xi_pos0, xi_pos1, xi_neg0, xi_neg1} {
-p log(xi_pos0) - (1-p) log(xi_neg0) - p log(xi_pos1)
-(1-p) log(xi_neg1)}
s.t.
error_rate <= additive_slack,
xi_pos0 <= ppr0(theta), xi_neg0 <= npr0(theta),
xi_pos1 <= ppr1(theta), xi_neg1 <= npr1(theta),
and formulate the Lagrangian:
max_{lambda's >= 0} min_{xi's} {
-p log(xi_pos0) - (1-p) log(xi_neg0) - p log(xi_pos1)
-(1-p) log(xi_neg1)
+ lambda_pos0 (xi_pos0 - ppr0(theta))
+ lambda_neg0 (xi_neg0 - npr0(theta))
+ lambda_pos1 (xi_pos1 - ppr1(theta))
+ lambda_neg1 (xi_neg1 - npr1(theta))}
s.t.
error_rate <= additive_slack.
We do best response for the slack variables xi:
BR for xi_pos0 = p / lambda_pos0
BR for xi_neg0 = (1 - p) / lambda_neg0
BR for xi_pos1 = p / lambda_pos1
BR for xi_neg1 = (1 - p) / lambda_neg1
We do gradient ascent on the lambda's, where
Gradient w.r.t. lambda_pos0
= BR for xi_pos0 - ppr0(theta)
= p / lambda_pos0 - ppr0(theta)
= Gradient w.r.t. lambda_pos0 of
(p log(lambda_pos0) - lambda_pos0 ppr0(theta))
Gradient w.r.t. lambda_neg0
= Gradient w.r.t. lambda_neg0 of
((1 - p) log(lambda_neg0) - lambda_neg0 npr0(theta))
Gradient w.r.t. lambda_pos1
= Gradient w.r.t. lambda_pos1 of
(p log(lambda_pos1) - lambda_pos1 ppr1(theta))
Gradient w.r.t. lambda_neg1
= Gradient w.r.t. lambda_neg1 of
((1 - p) log(lambda_neg1) - lambda_neg1 npr1(theta)).
We do gradient descent on thetas's, with ppr's and npr's replaced with hinge
surrogates. We use concave lower bounds on ppr's and npr's, so that when they
get negated in the updates, we get convex upper bounds.
See Appendix D.1 in the paper for more details.
Args:
train_set: (features, labels, groups)
additive_slack: float, additive slack on error rate constraint
learning_rate: float, learning rate for model parameters
learning_rate_constraint: float, learning rate for Lagrange multipliers
loops: int, number of iterations
Returns:
stochastic_model containing list of models and probabilities,
deterministic_model.
"""
x_train, y_train, z_train = train_set
dimension = x_train.shape[-1]
tf.reset_default_graph()
# Data tensors.
features_tensor = tf.constant(x_train.astype("float32"), name="features")
labels_tensor = tf.constant(y_train.astype("float32"), name="labels")
# Linear model.
weights = tf.Variable(tf.zeros(dimension, dtype=tf.float32),
name="weights")
threshold = tf.Variable(0, name="threshold", dtype=tf.float32)
predictions_tensor = (tf.tensordot(features_tensor, weights, axes=(1, 0)) +
threshold)
# Group-specific predictions.
predictions_group0 = tf.boolean_mask(predictions_tensor,
mask=(z_train < 1))
num_examples0 = np.sum(z_train < 1)
predictions_group1 = tf.boolean_mask(predictions_tensor,
mask=(z_train > 0))
num_examples1 = np.sum(z_train > 0)
# We use the TF Constrained Optimization (TFCO) library to set up the
# constrained optimization problem. The library doesn't currently support best
# responses for slack variables. So we maintain explicit Lagrange multipliers
# for the slack variables, and let the library deal with the Lagrange
# multipliers for the error rate constraint.
# Since we need to perform a gradient descent update on the model parameters,
# and an ascent update on the Lagrange multipliers on the slack variables, we
# create a single "minimization" objective using stop gradients, where a
# descent gradient update has the effect of minimizing over the model
# parameters and maximizing over the Lagrange multipliers for the slack
# variables. As noted above, the ascent update on the Lagrange multipliers for
# the error rate constraint is done by the library internally.
# Placeholders for Lagrange multipliers for the four slack variables.
lambda_pos0 = tf.Variable(0.5, dtype=tf.float32, name="lambda_pos0")
lambda_neg0 = tf.Variable(0.5, dtype=tf.float32, name="lambda_neg0")
lambda_pos1 = tf.Variable(0.5, dtype=tf.float32, name="lambda_pos1")
lambda_neg1 = tf.Variable(0.5, dtype=tf.float32, name="lambda_neg1")
# Set up prediction rates and surrogate relaxations on them.
p = np.mean(y_train) # Proportion of positives.
# Positive and negative prediction rates for group 0 and group 1.
ppr_group0 = tf.reduce_sum(
tf.cast(
tf.greater(predictions_group0,
tf.zeros(num_examples0, dtype="float32")),
"float32")) / num_examples0
npr_group0 = 1 - ppr_group0
ppr_group1 = tf.reduce_sum(
tf.cast(
tf.greater(predictions_group1,
tf.zeros(num_examples1, dtype="float32")),
"float32")) / num_examples1
npr_group1 = 1 - ppr_group1
# Hinge concave lower bounds on the positive and negative prediction rates.
# In the gradient updates, these get negated and become convex upper bounds.
# For group 0:
ppr_hinge_group0 = tf.reduce_sum(
1 - tf.nn.relu(1 - predictions_group0)) * 1.0 / num_examples0
npr_hinge_group0 = tf.reduce_sum(
1 - tf.nn.relu(1 + predictions_group0)) * 1.0 / num_examples0
# For group 1:
ppr_hinge_group1 = tf.reduce_sum(
1 - tf.nn.relu(1 - predictions_group1)) * 1.0 / num_examples1
npr_hinge_group1 = tf.reduce_sum(
1 - tf.nn.relu(1 + predictions_group1)) * 1.0 / num_examples1
# Set up KL-divergence objective for constrained optimization.
# We use stop gradients to ensure that a single descent gradient update on the
# objective has the effect of minimizing over the model parameters and
# maximizing over the Lagrange multipliers for the slack variables.
# KL-divergence for group 0.
kld_hinge_pos_group0 = (-tf.stop_gradient(lambda_pos0) * ppr_hinge_group0 -
p * tf.log(lambda_pos0) +
lambda_pos0 * tf.stop_gradient(ppr_group0))
kld_hinge_neg_group0 = (-tf.stop_gradient(lambda_neg0) * npr_hinge_group0 -
(1 - p) * tf.log(lambda_neg0) +
lambda_neg0 * tf.stop_gradient(npr_group0))
kld_hinge_group0 = kld_hinge_pos_group0 + kld_hinge_neg_group0
# KL-divergence for group 1.
kld_hinge_pos_group1 = (-tf.stop_gradient(lambda_pos1) * ppr_hinge_group1 -
p * tf.log(lambda_pos1) +
lambda_pos1 * tf.stop_gradient(ppr_group1))
kld_hinge_neg_group1 = (-tf.stop_gradient(lambda_neg1) * npr_hinge_group1 -
(1 - p) * tf.log(lambda_neg1) +
lambda_neg1 * tf.stop_gradient(npr_group1))
kld_hinge_group1 = kld_hinge_pos_group1 + kld_hinge_neg_group1
# Wrap the objective into a rate object.
objective = tfco.wrap_rate(kld_hinge_group0 + kld_hinge_group1)
# Set up error rate constraint for constrained optimization.
context = tfco.rate_context(predictions_tensor, labels_tensor)
error = tfco.error_rate(context)
constraints = [error <= additive_slack]
# Cretae rate minimization problem object.
problem = tfco.RateMinimizationProblem(objective, constraints)
# Set up optimizer.
optimizer = tfco.LagrangianOptimizerV1(
tf.train.AdamOptimizer(learning_rate=learning_rate),
constraint_optimizer=tf.train.AdamOptimizer(
learning_rate=learning_rate_constraint))
train_op = optimizer.minimize(problem)
# Start TF session and initialize variables.
session = tf.Session()
session.run(tf.global_variables_initializer())
# We maintain a list of objectives and model weights during training.
objectives = []
violations = []
models = []
# Perform full gradient updates.
for ii in range(loops):
# Gradient updates.
session.run(train_op)
# Checkpoint once in 10 iterations.
if ii % 10 == 0:
# Model weights.
model = [session.run(weights), session.run(threshold)]
models.append(model)
# Objective.
klds = evaluation.expected_group_klds(x_train, y_train, z_train,
[model], [1.0])
objectives.append(sum(klds))
# Violation.
error = evaluation.expected_error_rate(x_train, y_train, [model],
[1.0])
violations.append([error - additive_slack])
# Use the recorded objectives and constraints to find the best iterate.
best_iterate = tfco.find_best_candidate_index(np.array(objectives),
np.array(violations))
deterministic_model = models[best_iterate]
# Use shrinking to find a sparse distribution over iterates.
probabilities = tfco.find_best_candidate_distribution(
np.array(objectives), np.array(violations))
models_pruned = [
models[i] for i in range(len(models)) if probabilities[i] > 0.0
]
probabilities_pruned = probabilities[probabilities > 0.0]
return (models_pruned, probabilities_pruned), deterministic_model
def model_fn(mode=None, params=None, args=None, inference_features=None):
tf_dtype = tf.float16 if args.dtype == 'float16' else tf.float32
partials_type = np.float16 if args.partials_type == "float16" else np.float32
# Chained convolutional layers with master weights
def conv2d_chain(x, n_filters, name=''):
for i, f in enumerate(n_filters):
# Create variables for master weights (potentially in a higher precision)
w = tf.get_variable("conv2d/kernel" + str(i) + name,
shape=[3, 3, x.shape[3], f],
dtype=args.master_weight_type,
trainable=True,
initializer=tf.initializers.variance_scaling(
scale=2.0,
distribution='uniform',
mode='fan_in'))
b = tf.get_variable("conv2d/bias" + str(i) + name,
shape=[f],
dtype=args.master_weight_type,
trainable=True,
initializer=tf.initializers.constant(0.0))
# Add ops to the graph
x = tf.nn.conv2d(x, filters=tf.cast(w, tf_dtype), padding='SAME')
x = tf.nn.bias_add(x, tf.cast(b, tf_dtype), data_format="NHWC")
x = tf.nn.relu(x)
return x
# Deconvolutional layer with master weights
def conv2d_transpose(x, n_filters, name=''):
# Create variables for master weights (potentially in a higher precision)
w = tf.get_variable("conv2d_transpose/kernel" + name,
shape=[3, 3, n_filters, x.shape[3]],
dtype=args.master_weight_type,
trainable=True,
initializer=tf.initializers.variance_scaling(
scale=2.0,
distribution='uniform',
mode='fan_in'))
b = tf.get_variable("conv2d_transpose/bias" + name,
shape=[n_filters],
dtype=args.master_weight_type,
trainable=True,
initializer=tf.initializers.constant(0.0))
# Add ops to the graph
x = tf.nn.conv2d_transpose(x,
filters=tf.cast(w, tf_dtype),
strides=(2, 2),
output_shape=[
x.shape[0], x.shape[1] * 2,
x.shape[2] * 2, n_filters
],
padding='SAME')
x = tf.nn.bias_add(x, tf.cast(b, tf_dtype), data_format="NHWC")
x = tf.nn.relu(x)
return x
# Pipeline stage: encoder
def pipeline_stage1(global_step, features, labels):
x = features
skip_connections = []
x = conv2d_chain(x, [32, 32], 'encoder0')
skip_connections.append(x)
x = MaxPooling2D((2, 2))(x)
x = conv2d_chain(x, [32, 32], 'encoder1')
skip_connections.append(x)
x = MaxPooling2D((2, 2))(x)
x = conv2d_chain(x, [64, 64], 'encoder2')
skip_connections.append(x)
x = MaxPooling2D((2, 2))(x)
x = conv2d_chain(x, [128, 128], 'encoder3')
skip_connections.append(x)
x = MaxPooling2D((2, 2))(x)
return tuple([global_step, x, labels] + skip_connections)
# Pipeline stage: decoder
def pipeline_stage2(global_step, x, labels, *skip_connections):
skip_connections = [s for s in skip_connections]
x = conv2d_chain(x, [256, 256], 'bottleneck')
x = conv2d_transpose(x, 128, 'decoder1')
x = tf.concat([x, skip_connections.pop()], axis=-1)
x = conv2d_chain(x, [128, 64], 'decoder1')
x = conv2d_transpose(x, 64, 'decoder2')
x = tf.concat([x, skip_connections.pop()], axis=-1)
x = conv2d_chain(x, [64, 32], 'decoder2')
x = conv2d_transpose(x, 32, 'decoder3')
x = tf.concat([x, skip_connections.pop()], axis=-1)
x = conv2d_chain(x, [32, 16], 'decoder3')
x = conv2d_transpose(x, 16, 'decoder4')
x = tf.concat([x, skip_connections.pop()], axis=-1)
x = conv2d_chain(x, [32, 32], 'decoder4')
x = Conv2D(args.output_classes - 1, (1, 1))(x)
y_pred_logits = x
y_pred = tf.math.sigmoid(y_pred_logits)
predictions = tf.math.round(y_pred)
if mode == tf.estimator.ModeKeys.PREDICT:
return predictions
labels_fp32 = tf.cast(labels, tf.float32)
y_pred_fp32 = tf.cast(y_pred, tf.float32)
dice_coef = dice_coef_fn(labels_fp32, y_pred_fp32)
dice_loss = 1 - dice_coef
dice_coef_loss_binary = dice_coef_loss_binary_fn(
labels_fp32, y_pred_fp32)
loss = tf.cond(dice_loss < 0.2,
true_fn=lambda: dice_loss,
false_fn=lambda: dice_coef_loss_binary)
if mode == tf.estimator.ModeKeys.EVAL:
return global_step, loss, predictions, labels
if mode == tf.estimator.ModeKeys.TRAIN:
return global_step, loss
raise NotImplementedError(mode)
def optimizer_function(global_step, loss):
learning_rate = tf.train.exponential_decay(
learning_rate=args.learning_rate,
global_step=global_step,
decay_steps=args.lr_steps,
decay_rate=args.lr_decay_rate,
staircase=True)
optimizer = tf.train.RMSPropOptimizer(learning_rate=learning_rate,
decay=0.9,
momentum=0.8,
epsilon=1e-3,
centered=True)
def map_fn_decay(grad, var):
return grad + (args.weight_decay * var)
if args.weight_decay > 0:
optimizer = ipu.optimizers.MapGradientOptimizer(
optimizer, map_fn_decay)
return ipu.pipelining_ops.OptimizerFunctionOutput(optimizer, loss)
def eval_metrics_fn(global_step, loss, predictions, labels):
return {"loss": loss}
if args.pipeline_schedule == "Interleaved":
pipeline_schedule = ipu.pipelining_ops.PipelineSchedule.Interleaved
elif args.pipeline_schedule == "Grouped":
pipeline_schedule = ipu.pipelining_ops.PipelineSchedule.Grouped
elif args.pipeline_schedule == "Sequential":
pipeline_schedule = ipu.pipelining_ops.PipelineSchedule.Sequential
else:
raise NotImplementedError(args.pipeline_schedule)
if mode == tf.estimator.ModeKeys.PREDICT:
# For inference we do not need pipelining
return pipeline_stage2(*pipeline_stage1(tf.constant(
0), inference_features, tf.constant(0)))
elif mode == tf.estimator.ModeKeys.EVAL or mode == tf.estimator.ModeKeys.TRAIN:
# For training and evaluation we pipeline the graph across several IPUs
global_step_input = tf.cast(tf.train.get_global_step(),
dtype=tf.float32)
return ipu.ipu_pipeline_estimator.IPUPipelineEstimatorSpec(
mode,
computational_stages=[pipeline_stage1, pipeline_stage2],
optimizer_function=optimizer_function,
eval_metrics_fn=eval_metrics_fn,
inputs=[global_step_input],
offload_weight_update_variables=(
args.offload_weight_update_variables == 1),
pipeline_schedule=pipeline_schedule,
gradient_accumulation_count=args.gradient_accumulation_batches)
# 加入L2正则化控制模型复杂度，注意TensorFlow2.0之后完全弃用contrib包！
# tf.add_to_collection('totalLoss', tf.contrib.layers.l2_regularizer(0.5)(layer1W))
# tf.add_to_collection('totalLoss', tf.contrib.layers.l2_regularizer(0.5)(layer2W))
# tf.add_to_collection('totalLoss', tf.contrib.layers.l2_regularizer(0.5)(outputW))
# tf.add_to_collection('totalLoss', mse)
# total_loss = tf.add_n(tf.get_collection('totalLoss'))
# train = tf.train.GradientDescentOptimizer(0.01).minimize(total_loss)
# 使用指数衰减法设置学习率
# global_step = tf.Variable(0)
# learning_rate = tf.train.exponential_decay(0.1, global_step, 100, 0.96, staircase=False)
# train = tf.train.GradientDescentOptimizer(learning_rate)\
# .minimize(mse, global_step=global_step)
# 预测一次
s1 = tf.constant([[1.1], [2.1]])
l1 = tf.matmul(layer1W, s1) + layer1B
l1 = tf.sigmoid(l1)
l2 = tf.matmul(layer2W, l1) + layer2B
l2 = tf.sigmoid(l2)
f = tf.matmul(outputW, l2) + outputB
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
print(sess.run(layer1W))
print(sess.run(layer1B))
print(
sess.run(l1Output,
feed_dict={
x: [[1.0, 2.5], [2.0, 5.0]],
y: [[7.0, 9.0]]
def test_batch_segment_sum_embeddings(self):
# batch_size = 2
# long_max_length = 8
# hidden_size = 2
long_embeddings = tf.constant(
[
[
[0.1, -0.1],
[0.2, -0.2],
[0.3, -0.3],
[0.4, -0.4],
[0.5, -0.5],
[100.0, -100.0], # Padding embeddings may be arbitrary.
[200.0, -200.0],
[300.0, -300.0],
], #
[
[1.1, -1.1],
[1.2, -1.2],
[1.3, -1.3],
[1.4, -1.4],
[1.5, -1.5],
[1.6, -1.6],
[400.0, 400.0], # Padding embeddings may be arbitrary.
[500.0, 500.0],
], #
],
dtype=tf.float32)
long_word_idx = tf.constant(
[
[0, 1, 2, 2, 3, 0, 0, 0], # Padding indices can just be 0.
[0, 0, 0, 1, 2, 2, 0, 0], # Padding indices can just be 0.
],
dtype=tf.int32)
long_input_mask = tf.constant(
[
[1, 1, 1, 1, 1, 0, 0, 0], #
[1, 1, 1, 1, 1, 1, 0, 0], #
],
dtype=tf.int32)
expected = [
[
[0.1, -0.1],
[0.2, -0.2],
[0.7, -0.7],
[0.5, -0.5],
[0.0, 0.0],
[0.0, 0.0],
[0.0, 0.0],
[0.0, 0.0],
], #
[
[3.6, -3.6],
[1.4, -1.4],
[3.1, -3.1],
[0.0, 0.0],
[0.0, 0.0],
[0.0, 0.0],
[0.0, 0.0],
[0.0, 0.0],
], #
]
self.assertAllClose(
expected,
run_finetuning_lib.batch_segment_sum_embeddings(
long_embeddings=long_embeddings,
long_word_idx=long_word_idx,
long_input_mask=long_input_mask))
def test_sum(self):
tf.disable_eager_execution()
nodes = tf.constant([1, 3])
nodes2 = tf.constant([1, 3])
fused = efficientdet_arch.fuse_features([nodes, nodes2], 'sum')
self.assertAllCloseAccordingToType(fused, [2, 6])
nneu = [10] # number of neurons in the 1-st level of ANN
# number of neurons in the last layer == 1
num_of_epochs = 1500
num_to_show = 50
sess = tf.InteractiveSession()
x = tf.placeholder(tf.float32,
shape=[None, num_of_features]) # place for input vectors
y_ = tf.placeholder(tf.float32, shape=[None,
1]) # place for desired output of ANN
IW = tf.Variable(tf.truncated_normal(
[num_of_features, nneu[0]],
stddev=0.1)) # 1-st level weights initialized with normal distribution
b1 = tf.Variable(tf.constant(0.1, shape=[nneu[0]])) # 1-st level biases -||-
h1 = tf.nn.tanh(
tf.matmul(x, IW) + b1
) # output values from 1-st level (using hyperbolic tangent activation func.)
LW21 = tf.Variable(tf.truncated_normal(
[nneu[0], 1], stddev=0.1)) # 2-nd level weights values
b2 = tf.Variable(tf.zeros([1])) # 2-nd level bias values
#LW32 = ... # 3-nd level weights values
#b3 = ... # 3-nd level bias values
# h2 = tf.nn.tanh(tf.matmul(h1, LW21) + b2)
#
# LW32 = tf.Variable(tf.truncated_normal([nneu[0],1], stddev=0.1))
