def resnet_inference(image: flow.typing.Numpy.Placeholder(
image_shape, dtype=flow.float32)) -> flow.typing.Numpy:
input_lbns["image"] = image.logical_blob_name
output = resnet50(image, trainable=False)
output = flow.nn.softmax(output)
output_lbns["output"] = output.logical_blob_name
return output
def __init__(self):
super(Net, self).__init__()
## Modify dimensions based on model
# resnet 18,34
# self.proj0 = nn.Conv2d(512, 8192, kernel_size=1, stride=1)
# self.proj1 = nn.Conv2d(512, 8192, kernel_size=1, stride=1)
# self.proj2 = nn.Conv2d(512, 8192, kernel_size=1, stride=1)
# resnet 50, ...
self.proj0 = nn.Conv2d(2048, 8192, kernel_size=1, stride=1)
self.proj1 = nn.Conv2d(2048, 8192, kernel_size=1, stride=1)
self.proj2 = nn.Conv2d(2048, 8192, kernel_size=1, stride=1)
# fc layer
self.fc_concat = torch.nn.Linear(8192 * 3, 200)
# for m in self.modules():
# if isinstance(m, nn.Conv2d):
# n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
# m.weight.data.normal_(0, math.sqrt(2. / n))
# elif isinstance(m, nn.Linear):
# m.weight.data.normal_(0, 0.01)
# m.bias.data.zero_()
self.softmax = nn.LogSoftmax(dim=1)
self.avgpool = nn.AvgPool2d(kernel_size=14)
## select base-model
# self.features = resnet_model.resnet34(pretrained=True,
# model_root='/data/guijun/HBP_finegrained/pth/resnet34.pth')
self.features = resnet_model.resnet50(
pretrained=True,
model_root='/data/guijun/HBP_finegrained/pth/resnet50.pth')
def __init__(self):
super(Net, self).__init__()
self.proj0 = nn.Conv2d(2048, 1024, kernel_size=1, stride=1)
self.proj1 = nn.Conv2d(2048, 1024, kernel_size=1, stride=1)
self.proj2 = nn.Conv2d(2048, 1024, kernel_size=1, stride=1)
self.fc_concat = torch.nn.Linear(1024 * 196 * 3, 200)
self.softmax = nn.LogSoftmax(dim=1)
self.avgpool = nn.AvgPool2d(kernel_size=14)
self.features = resnet_model.resnet50(
pretrained=True, model_root='D:/Zhen/LMX/LMX/resnet50.pth')
def export_tfhub(model_path, hub_destination):
"""Restores a tf.keras.Model and saves for TF-Hub."""
model = resnet_model.resnet50(
num_classes=imagenet_preprocessing.NUM_CLASSES, rescale_inputs=True)
model.load_weights(model_path)
model.save(os.path.join(hub_destination, "classification"),
include_optimizer=False)
# Extracts a sub-model to use pooling feature vector as model output.
image_input = model.get_layer(index=0).get_output_at(0)
feature_vector_output = model.get_layer(
name="reduce_mean").get_output_at(0)
hub_model = tf.keras.Model(image_input, feature_vector_output)
# Exports a SavedModel.
hub_model.save(os.path.join(hub_destination, "feature-vector"),
include_optimizer=False)
def InferenceNet(images: tp.Numpy.Placeholder((1, 3, 224, 224),
dtype=flow.float)) -> tp.Numpy:
logits = resnet50(images, args, training=False)
predictions = flow.nn.softmax(logits)
return predictions
from load_dataset import VotTrainDataset
#from visualize import Visualizer
import numpy as np
from models import Yolov1_vgg16bn
from resnet_model import resnet18
from resnet_model import resnet50
from resnet_model import resnet101
use_gpu = torch.cuda.is_available()
file_root = '/home/xiec/DLCV/hw2-xiechen0692/hw2_train_val/'
#learning_rate = 0.002
learning_rate = 0.002
num_epochs = 50
batch_size = 8
net = resnet50()
resnet = models.resnet50(pretrained=True)
new_state_dict = resnet.state_dict()
dd = net.state_dict()
for k in new_state_dict.keys():
print(k)
if k in dd.keys() and not k.startswith('fc'):
print('yes')
dd[k] = new_state_dict[k]
net.load_state_dict(dd)
criterion = yoloLoss(7, 2, 5, 0.5)
if use_gpu:
net.cuda()
net.train()
# different learning rate
def __init__(self, flags_obj, time_callback):
standard_runnable.StandardRunnableWithWarmup.__init__(
self,
flags_obj.use_tf_while_loop,
flags_obj.use_tf_function)
self.strategy = tf.distribute.get_strategy()
self.flags_obj = flags_obj
self.dtype = flags_core.get_tf_dtype(flags_obj)
self.time_callback = time_callback
# Input pipeline related
batch_size = flags_obj.batch_size
if batch_size % self.strategy.num_replicas_in_sync != 0:
raise ValueError(
'Batch size must be divisible by number of replicas : {}'.format(
self.strategy.num_replicas_in_sync))
steps_per_epoch, train_epochs = common.get_num_train_iterations(flags_obj)
if train_epochs > 1:
train_epochs = flags_obj.train_epochs
# As auto rebatching is not supported in
# `experimental_distribute_datasets_from_function()` API, which is
# required when cloning dataset to multiple workers in eager mode,
# we use per-replica batch size.
self.batch_size = int(batch_size / self.strategy.num_replicas_in_sync)
self.synthetic_input_fn = common.get_synth_input_fn(
height=imagenet_preprocessing.DEFAULT_IMAGE_SIZE,
width=imagenet_preprocessing.DEFAULT_IMAGE_SIZE,
num_channels=imagenet_preprocessing.NUM_CHANNELS,
num_classes=self.flags_obj.num_classes,
dtype=self.dtype,
drop_remainder=True)
if self.flags_obj.use_synthetic_data:
self.input_fn = self.synthetic_input_fn
else:
self.input_fn = imagenet_preprocessing.input_fn
resnet_model.change_keras_layer(flags_obj.use_tf_keras_layers)
self.model = resnet_model.resnet50(
num_classes=self.flags_obj.num_classes,
batch_size=flags_obj.batch_size,
use_l2_regularizer=not flags_obj.single_l2_loss_op)
self.use_lars_optimizer = False
self.num_accumulation_steps = self.flags_obj.num_accumulation_steps
if self.flags_obj.optimizer == 'LARS':
self.use_lars_optimizer = True
self.optimizer, _ = common.get_optimizer(
flags_obj=flags_obj,
steps_per_epoch=steps_per_epoch,
train_steps=steps_per_epoch * train_epochs)
# Make sure iterations variable is created inside scope.
self.global_step = self.optimizer.iterations
if self.dtype == tf.float16:
loss_scale = flags_core.get_loss_scale(flags_obj, default_for_fp16=128)
self.optimizer = (
tf.keras.mixed_precision.experimental.LossScaleOptimizer(
self.optimizer, loss_scale))
elif flags_obj.fp16_implementation == 'graph_rewrite':
# `dtype` is still float32 in this case. We built the graph in float32
# and let the graph rewrite change parts of it float16.
if not flags_obj.use_tf_function:
raise ValueError('--fp16_implementation=graph_rewrite requires '
'--use_tf_function to be true')
loss_scale = flags_core.get_loss_scale(flags_obj, default_for_fp16=128)
self.optimizer = (
tf.train.experimental.enable_mixed_precision_graph_rewrite(
self.optimizer, loss_scale))
self.one_hot = False
self.label_smoothing = flags_obj.label_smoothing
if self.label_smoothing and self.label_smoothing > 0:
self.one_hot = True
if flags_obj.report_accuracy_metrics:
self.train_loss = tf.keras.metrics.Mean('train_loss', dtype=tf.float32)
if self.one_hot:
self.train_accuracy = tf.keras.metrics.CategoricalAccuracy(
'train_accuracy', dtype=tf.float32)
else:
self.train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(
'train_accuracy', dtype=tf.float32)
self.test_loss = tf.keras.metrics.Mean('test_loss', dtype=tf.float32)
else:
self.train_loss = None
self.train_accuracy = None
self.test_loss = None
if self.one_hot:
self.test_accuracy = tf.keras.metrics.CategoricalAccuracy(
'test_accuracy', dtype=tf.float32)
else:
self.test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(
'test_accuracy', dtype=tf.float32)
# self.test_corrects = tf.keras.metrics.Sum(
# 'test_corrects', dtype=tf.float32)
self.num_eval_steps = common.get_num_eval_steps(flags_obj)
self.checkpoint = tf.train.Checkpoint(
model=self.model, optimizer=self.optimizer)
# Handling epochs.
self.epoch_steps = steps_per_epoch
self.epoch_helper = utils.EpochHelper(steps_per_epoch, self.global_step)
self.steps_per_loop = flags_obj.steps_per_loop
profile_steps = flags_obj.profile_steps
if profile_steps:
profile_steps = [int(i) for i in profile_steps.split(',')]
self.trace_start_step = profile_steps[0] if profile_steps[0] >= 0 else None
self.trace_end_step = profile_steps[1]
else:
self.trace_start_step = None
self.trace_end_step = None
self.epochs_between_evals = flags_obj.epochs_between_evals
self.training_vars = self.model.trainable_variables
self.accum_grads = []
self.accum_grads_dtype = tf.float32
if self.num_accumulation_steps > 1:
for var in self.training_vars:
self.accum_grads.append(self.optimizer.add_weight(
name=var.name + '_accum',
shape=var.shape,
dtype=self.accum_grads_dtype,
initializer='zeros',
trainable=False,
synchronization=tf.VariableSynchronization.ON_READ,
aggregation=tf.VariableAggregation.SUM))
def run(flags_obj):
"""Run ResNet ImageNet training and eval loop using custom training loops.
Args:
flags_obj: An object containing parsed flag values.
Raises:
ValueError: If fp16 is passed as it is not currently supported.
Returns:
Dictionary of training and eval stats.
"""
print('@@@@enable_eager = {}'.format(flags_obj.enable_eager))
keras_utils.set_session_config(
enable_eager=flags_obj.enable_eager,
enable_xla=flags_obj.enable_xla)
dtype = flags_core.get_tf_dtype(flags_obj)
if dtype == tf.float16:
policy = tf.compat.v2.keras.mixed_precision.experimental.Policy(
'mixed_float16')
tf.compat.v2.keras.mixed_precision.experimental.set_policy(policy)
elif dtype == tf.bfloat16:
policy = tf.compat.v2.keras.mixed_precision.experimental.Policy(
'mixed_bfloat16')
tf.compat.v2.keras.mixed_precision.experimental.set_policy(policy)
# This only affects GPU.
common.set_cudnn_batchnorm_mode()
# TODO(anj-s): Set data_format without using Keras.
data_format = flags_obj.data_format
if data_format is None:
data_format = ('channels_first'
if tf.test.is_built_with_cuda() else 'channels_last')
tf.keras.backend.set_image_data_format(data_format)
strategy = distribution_utils.get_distribution_strategy(
distribution_strategy=flags_obj.distribution_strategy,
num_gpus=flags_obj.num_gpus,
num_workers=distribution_utils.configure_cluster(),
all_reduce_alg=flags_obj.all_reduce_alg,
num_packs=flags_obj.num_packs,
tpu_address=flags_obj.tpu)
train_ds, test_ds = get_input_dataset(flags_obj, strategy)
per_epoch_steps, train_epochs, eval_steps = get_num_train_iterations(
flags_obj)
steps_per_loop = min(flags_obj.steps_per_loop, per_epoch_steps)
logging.info("Training %d epochs, each epoch has %d steps, "
"total steps: %d; Eval %d steps",
train_epochs, per_epoch_steps, train_epochs * per_epoch_steps,
eval_steps)
time_callback = keras_utils.TimeHistory(flags_obj.batch_size,
flags_obj.log_steps)
with distribution_utils.get_strategy_scope(strategy):
resnet_model.change_keras_layer(flags_obj.use_tf_keras_layers)
use_l2_regularizer = not flags_obj.single_l2_loss_op
if flags_obj.use_resnet_d:
resnetd = network_tweaks.ResnetD(image_data_format=tf.keras.backend.image_data_format(),
use_l2_regularizer=use_l2_regularizer)
else:
resnetd = None
model = resnet_model.resnet50(
num_classes=imagenet_preprocessing.NUM_CLASSES,
batch_size=flags_obj.batch_size,
zero_gamma=flags_obj.zero_gamma,
last_pool_channel_type=flags_obj.last_pool_channel_type,
use_l2_regularizer=use_l2_regularizer,
resnetd=resnetd)
if flags_obj.learning_rate_decay_type == 'piecewise':
lr_schedule = common.PiecewiseConstantDecayWithWarmup(
batch_size=flags_obj.batch_size,
epoch_size=imagenet_preprocessing.NUM_IMAGES['train'],
warmup_epochs=common.LR_SCHEDULE[0][1],
boundaries=list(p[1] for p in common.LR_SCHEDULE[1:]),
multipliers=list(p[0] for p in common.LR_SCHEDULE),
compute_lr_on_cpu=True)
elif flags_obj.learning_rate_decay_type == 'cosine':
lr_schedule = common.CosineDecayWithWarmup(
base_lr=flags_obj.base_learning_rate,
batch_size=flags_obj.batch_size,
epoch_size=imagenet_preprocessing.NUM_IMAGES['train'],
warmup_epochs=common.LR_SCHEDULE[0][1],
train_epochs=flags_obj.train_epochs,
compute_lr_on_cpu=True)
else:
raise NotImplementedError
optimizer = common.get_optimizer(lr_schedule)
if dtype == tf.float16:
loss_scale = flags_core.get_loss_scale(flags_obj, default_for_fp16=128)
optimizer = tf.keras.mixed_precision.experimental.LossScaleOptimizer(
optimizer, loss_scale)
elif flags_obj.fp16_implementation == 'graph_rewrite':
# `dtype` is still float32 in this case. We built the graph in float32 and
# let the graph rewrite change parts of it float16.
if not flags_obj.use_tf_function:
raise ValueError('--fp16_implementation=graph_rewrite requires '
'--use_tf_function to be true')
loss_scale = flags_core.get_loss_scale(flags_obj, default_for_fp16=128)
optimizer = tf.train.experimental.enable_mixed_precision_graph_rewrite(
optimizer, loss_scale)
current_step = 0
checkpoint = tf.train.Checkpoint(model=model, optimizer=optimizer)
latest_checkpoint = tf.train.latest_checkpoint(flags_obj.model_dir)
if latest_checkpoint:
checkpoint.restore(latest_checkpoint)
logging.info("Load checkpoint %s", latest_checkpoint)
current_step = optimizer.iterations.numpy()
train_loss = tf.keras.metrics.Mean('train_loss', dtype=tf.float32)
test_loss = tf.keras.metrics.Mean('test_loss', dtype=tf.float32)
categorical_cross_entopy_and_acc = losses.CategoricalCrossEntropyAndAcc(
batch_size=flags_obj.batch_size,
num_classes=imagenet_preprocessing.NUM_CLASSES,
label_smoothing=flags_obj.label_smoothing)
trainable_variables = model.trainable_variables
def step_fn(inputs):
"""Per-Replica StepFn."""
images, labels = inputs
with tf.GradientTape() as tape:
logits = model(images, training=True)
loss = categorical_cross_entopy_and_acc.loss_and_update_acc(labels, logits, training=True)
#loss = tf.reduce_sum(prediction_loss) * (1.0/ flags_obj.batch_size)
num_replicas = tf.distribute.get_strategy().num_replicas_in_sync
if flags_obj.single_l2_loss_op:
l2_loss = resnet_model.L2_WEIGHT_DECAY * 2 * tf.add_n([
tf.nn.l2_loss(v)
for v in trainable_variables
if 'bn' not in v.name
])
loss += (l2_loss / num_replicas)
else:
loss += (tf.reduce_sum(model.losses) / num_replicas)
# Scale the loss
if flags_obj.dtype == "fp16":
loss = optimizer.get_scaled_loss(loss)
grads = tape.gradient(loss, trainable_variables)
# Unscale the grads
if flags_obj.dtype == "fp16":
grads = optimizer.get_unscaled_gradients(grads)
optimizer.apply_gradients(zip(grads, trainable_variables))
train_loss.update_state(loss)
@tf.function
def train_steps(iterator, steps):
"""Performs distributed training steps in a loop."""
for _ in tf.range(steps):
strategy.experimental_run_v2(step_fn, args=(next(iterator),))
def train_single_step(iterator):
if strategy:
strategy.experimental_run_v2(step_fn, args=(next(iterator),))
else:
return step_fn(next(iterator))
def test_step(iterator):
"""Evaluation StepFn."""
def step_fn(inputs):
images, labels = inputs
logits = model(images, training=False)
loss = categorical_cross_entopy_and_acc.loss_and_update_acc(labels, logits, training=False)
#loss = tf.reduce_sum(loss) * (1.0/ flags_obj.batch_size)
test_loss.update_state(loss)
if strategy:
strategy.experimental_run_v2(step_fn, args=(next(iterator),))
else:
step_fn(next(iterator))
if flags_obj.use_tf_function:
train_single_step = tf.function(train_single_step)
test_step = tf.function(test_step)
if flags_obj.enable_tensorboard:
summary_writer = tf.summary.create_file_writer(flags_obj.model_dir)
else:
summary_writer = None
train_iter = iter(train_ds)
time_callback.on_train_begin()
for epoch in range(current_step // per_epoch_steps, train_epochs):
train_loss.reset_states()
categorical_cross_entopy_and_acc.training_accuracy.reset_states()
steps_in_current_epoch = 0
while steps_in_current_epoch < per_epoch_steps:
time_callback.on_batch_begin(
steps_in_current_epoch+epoch*per_epoch_steps)
steps = _steps_to_run(steps_in_current_epoch, per_epoch_steps,
steps_per_loop)
if steps == 1:
train_single_step(train_iter)
else:
# Converts steps to a Tensor to avoid tf.function retracing.
train_steps(train_iter, tf.convert_to_tensor(steps, dtype=tf.int32))
time_callback.on_batch_end( steps_in_current_epoch+epoch*per_epoch_steps)
steps_in_current_epoch += steps
#temp_loss = array_ops.identity(categorical_cross_entopy_and_acc.training_loss).numpy()
#temp_loss = categorical_cross_entopy_and_acc.training_loss.numpy()
logging.info('Training loss: %s, accuracy: %s, cross_entropy: %s at epoch %d',
train_loss.result().numpy(),
categorical_cross_entopy_and_acc.training_accuracy.result().numpy(),
0.,
epoch + 1)
if (not flags_obj.skip_eval and
(epoch + 1) % flags_obj.epochs_between_evals == 0):
test_loss.reset_states()
categorical_cross_entopy_and_acc.test_accuracy.reset_states()
test_iter = iter(test_ds)
for _ in range(eval_steps):
test_step(test_iter)
logging.info('Test loss: %s, accuracy: %s%% at epoch: %d',
test_loss.result().numpy(),
categorical_cross_entopy_and_acc.test_accuracy.result().numpy(),
epoch + 1)
if flags_obj.enable_checkpoint_and_export:
checkpoint_name = checkpoint.save(
os.path.join(flags_obj.model_dir,
'model.ckpt-{}'.format(epoch + 1)))
logging.info('Saved checkpoint to %s', checkpoint_name)
if summary_writer:
current_steps = steps_in_current_epoch + (epoch * per_epoch_steps)
with summary_writer.as_default():
#tf.summary.scalar('train_cross_entropy', categorical_cross_entopy_and_acc.training_loss.numpy(), current_steps)
tf.summary.scalar('train_loss', train_loss.result(), current_steps)
tf.summary.scalar('train_accuracy', categorical_cross_entopy_and_acc.training_accuracy.result(),
current_steps)
lr_for_monitor = lr_schedule(current_steps)
if callable(lr_for_monitor):
lr_for_monitor = lr_for_monitor()
tf.summary.scalar('learning_rate', lr_for_monitor, current_steps)
tf.summary.scalar('eval_loss', test_loss.result(), current_steps)
tf.summary.scalar(
'eval_accuracy', categorical_cross_entopy_and_acc.test_accuracy.result(), current_steps)
time_callback.on_train_end()
if summary_writer:
summary_writer.close()
eval_result = None
train_result = None
if not flags_obj.skip_eval:
eval_result = [test_loss.result().numpy(),
categorical_cross_entopy_and_acc.test_accuracy.result().numpy()]
train_result = [train_loss.result().numpy(),
categorical_cross_entopy_and_acc.training_accuracy.result().numpy()]
stats = build_stats(train_result, eval_result, time_callback)
return stats
