def get_training_hooks(mode, model_dir, checkpoint_path=None, skip_checkpoint_variables=None):
assert mode in ('train', 'eval')
training_hooks = [
AutoLoggingHook(
# log_every_n_steps=RUNNING_CONFIG.display_step,
log_every_n_steps=5 if "NGC_JOB_ID" not in os.environ else 100,
# warmup_steps=RUNNING_CONFIG.warmup_steps,
warmup_steps=100,
is_training=True
)
]
if not MPI_is_distributed() or MPI_rank() == 0:
training_hooks.append(PretrainedWeightsLoadingHook(
prefix="resnet50/",
checkpoint_path=checkpoint_path,
skip_variables_regex=skip_checkpoint_variables
))
if MPI_is_distributed() and mode == "train":
training_hooks.append(hvd.BroadcastGlobalVariablesHook(0))
if not MPI_is_distributed() or MPI_rank() == 0:
training_hooks.append(CheckpointSaverHook(
checkpoint_dir=model_dir,
checkpoint_basename="model.ckpt"
))
return training_hooks
def __init__(self, runtime_config, model_fn):
super(EstimatorExecuter, self).__init__(runtime_config, model_fn)
if MPI_is_distributed():
os.environ['HOROVOD_GPU_ALLREDUCE'] = 'NCCL'
os.environ['HOROVOD_NUM_NCCL_STREAMS'] = '1'
# os.environ['HOROVOD_AUTOTUNE'] = '2'
logging.info("SageMaker Distributed Data Parallel successfully initialized ...")
os.environ['TF_GPU_THREAD_MODE'] = 'gpu_private'
os.environ['TF_GPU_THREAD_COUNT'] = '1' if not MPI_is_distributed() else str(hvd.size())
os.environ['TF_SYNC_ON_FINISH'] = '0'
def create_optimizer(learning_rate, params):
"""Creates optimized based on the specified flags."""
optimizer = tf.compat.v1.train.MomentumOptimizer(
learning_rate, momentum=params['momentum'])
if MPI_is_distributed():
optimizer = hvd.DistributedOptimizer(
optimizer,
name=None,
device_dense='/gpu:0',
device_sparse='',
bucket_cap_mb=64,
# compression=hvd.Compression.fp16,
# compression=hvd.Compression.none,
# sparse_as_dense=False
)
if params["amp"]:
loss_scale = tf.train.experimental.DynamicLossScale(
initial_loss_scale=(2**12), increment_period=2000, multiplier=2.0)
optimizer = tf.compat.v1.train.experimental.MixedPrecisionLossScaleOptimizer(
optimizer, loss_scale=loss_scale)
return optimizer
def build_strategy_configuration(self, mode):
"""Retrieves model configuration for running TF Estimator."""
run_config = tf.estimator.RunConfig(
tf_random_seed=(
self._runtime_config.seed
if not MPI_is_distributed() or self._runtime_config.seed is None else
self._runtime_config.seed + MPI_rank()
),
model_dir=self._runtime_config.model_dir,
save_summary_steps=None, # disabled
save_checkpoints_steps=None, # disabled
save_checkpoints_secs=None, # disabled
keep_checkpoint_max=20, # disabled
keep_checkpoint_every_n_hours=None, # disabled
log_step_count_steps=None, # disabled
session_config=self._get_session_config(
mode=mode,
use_xla=self._runtime_config.xla,
use_amp=self._runtime_config.amp,
use_tf_distributed=False,
allow_xla_at_inference=self._runtime_config.allow_xla_at_inference # TODO: Remove when XLA at inference fixed
),
protocol=None,
device_fn=None,
train_distribute=None,
eval_distribute=None,
experimental_distribute=None
)
return run_config
def after_create_session(self, session, coord): # pylint: disable=unused-argument3
"""Called when new TensorFlow session is created.
This is called to signal the hooks that a new session has been created. This
has two essential differences with the situation in which `begin` is called:
* When this is called, the graph is finalized and ops can no longer be added
to the graph.
* This method will also be called as a result of recovering a wrapped
session, not only at the beginning of the overall session.
Args:
session: A TensorFlow Session that has been created.
coord: A Coordinator object which keeps track of all threads.
"""
# ========= Collect the number of GPUs ======== #
if self._is_training:
if MPI_is_distributed():
self._n_gpus = MPI_size()
elif tf.distribute.has_strategy():
self._n_gpus = tf.distribute.get_strategy(
).num_replicas_in_sync
else:
self._n_gpus = 1
else:
self._n_gpus = 1
# =========== TensorFlow Hook Setup =========== #
_global_step, _metrics = setup_tensorflow_hook(
sess=session,
logging_proxy=self._logging_proxy,
is_training=self._is_training,
is_initialized=self._initialized)
if _global_step >= 0:
self._current_step = self._amp_steps_non_skipped = _global_step
self._metrics.update(_metrics)
if not self._is_training:
for metric_name in self._metrics.keys():
self._metrics[metric_name]["aggregator"].reset()
self._initialized = True
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% #
self._session_t0 = time.time()
def __init__(self, runtime_config, model_fn):
super(EstimatorExecuter, self).__init__(runtime_config, model_fn)
# Handle recipe cache. Skip if externally set or empty.
recipe_cache = runtime_config.recipe_cache
if 'TF_RECIPE_CACHE_PATH' not in os.environ.keys() and recipe_cache:
os.environ['TF_RECIPE_CACHE_PATH'] = recipe_cache
# Clear previous recipe cache.
if not MPI_is_distributed() or MPI_rank() == 0:
if os.path.exists(recipe_cache) and os.path.isdir(
recipe_cache):
shutil.rmtree(recipe_cache)
if MPI_is_distributed():
os.environ['HOROVOD_GPU_ALLREDUCE'] = 'NCCL'
os.environ['HOROVOD_NUM_NCCL_STREAMS'] = '1'
# os.environ['HOROVOD_AUTOTUNE'] = '2'
if runtime_config.device == "HPU":
from TensorFlow.common.horovod_helpers import hvd_init, Framework
hvd = hvd_init(framework=Framework.TENSORFLOW)
else:
hvd.init()
# Other ranks should wait for recipe cache to be removed.
# This operation can't be done before hvd_init.
from mpi4py import MPI
MPI.COMM_WORLD.Barrier()
logging.info("Horovod successfully initialized ...")
os.environ['TF_GPU_THREAD_MODE'] = 'gpu_private'
os.environ['TF_GPU_THREAD_COUNT'] = '1' if not MPI_is_distributed(
) else str(hvd.size())
os.environ['TF_SYNC_ON_FINISH'] = '0'
def get_last_checkpoint_path(self):
"""Search for proper checkpoint directory"""
checkpoint_path = Path(self._runtime_config.model_dir)
if MPI_is_distributed():
# Checkpoints are saved only to 'worker_0' directory.
worker_0_path = checkpoint_path.parent.joinpath('worker_0')
if worker_0_path.exists():
checkpoint_path = worker_0_path
else:
# Case when no worker directories are available.
checkpoint_path = checkpoint_path.parent
return tf.train.latest_checkpoint(str(checkpoint_path),
latest_filename=None)
def __init__(self, runtime_config, model_fn):
if MPI_is_distributed():
runtime_config.model_dir = os.path.join(runtime_config.model_dir,
f'worker_{MPI_rank()}')
self._runtime_config = runtime_config
self._model_fn = model_fn
os.environ['CUDA_CACHE_DISABLE'] = '0'
os.environ['TF_USE_CUDNN_BATCHNORM_SPATIAL_PERSISTENT'] = '1'
os.environ['TF_ADJUST_HUE_FUSED'] = '1'
os.environ['TF_ADJUST_SATURATION_FUSED'] = '1'
os.environ['TF_ENABLE_WINOGRAD_NONFUSED'] = '1'
os.environ['TF_AUTOTUNE_THRESHOLD'] = '2'
def __call__(self, params, input_context=None):
batch_size = params['batch_size'] if 'batch_size' in params else 1
try:
seed = params['seed'] if not MPI_is_distributed(
) else params['seed'] * MPI_rank()
except (KeyError, TypeError):
seed = None
if MPI_is_distributed():
n_gpus = MPI_size()
elif input_context is not None:
n_gpus = input_context.num_input_pipelines
else:
n_gpus = 1
##################################################
dataset = tf.data.Dataset.list_files(self._file_pattern, shuffle=False)
if self._mode == tf.estimator.ModeKeys.TRAIN:
if input_context is not None:
logging.info("Using Dataset Sharding with TF Distributed")
_num_shards = input_context.num_input_pipelines
_shard_idx = input_context.input_pipeline_id
elif MPI_is_distributed():
logging.info("Using Dataset Sharding with Horovod")
_shard_idx, _num_shards = MPI_rank_and_size()
try:
dataset = dataset.shard(num_shards=_num_shards,
index=_shard_idx)
dataset = dataset.shuffle(math.ceil(256 / _num_shards))
except NameError: # Not a distributed training setup
pass
def _prefetch_dataset(filename):
return tf.data.TFRecordDataset(filename).prefetch(1)
dataset = dataset.interleave(
map_func=_prefetch_dataset,
cycle_length=32,
block_length=64,
num_parallel_calls=tf.data.experimental.AUTOTUNE,
)
if self._num_examples is not None and self._num_examples > 0:
logging.info("[*] Limiting the amount of sample to: %d" %
self._num_examples)
dataset = dataset.take(self._num_examples)
dataset = dataset.cache()
if self._mode == tf.estimator.ModeKeys.TRAIN:
dataset = dataset.shuffle(buffer_size=4096,
reshuffle_each_iteration=True,
seed=seed)
dataset = dataset.repeat()
# Parse the fetched records to input tensors for model function.
dataset = dataset.map(
map_func=self._create_dataset_parser_fn(params),
num_parallel_calls=16,
)
dataset = dataset.batch(batch_size=batch_size, drop_remainder=True)
if self._use_fake_data:
# Turn this dataset into a semi-fake dataset which always loop at the
# first batch. This reduces variance in performance and is useful in
# testing.
logging.info("Using Fake Dataset Loop...")
dataset = dataset.take(1).cache().repeat()
if self._mode != tf.estimator.ModeKeys.TRAIN:
dataset = dataset.take(int(5000 / batch_size))
dataset = dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE, )
if not tf.distribute.has_strategy():
dataset = dataset.apply(
tf.data.experimental.prefetch_to_device(
'/gpu:0', # With Horovod the local GPU is always 0
buffer_size=1,
))
data_options = tf.data.Options()
data_options.experimental_deterministic = seed is not None
if LooseVersion(tf.__version__) <= LooseVersion("2.0.0"):
data_options.experimental_distribute.auto_shard = False
else:
data_options.experimental_distribute.auto_shard_policy = tf.data.experimental.AutoShardPolicy.OFF
# data_options.experimental_distribute.auto_shard = False
data_options.experimental_slack = True
data_options.experimental_threading.max_intra_op_parallelism = 1
# data_options.experimental_threading.private_threadpool_size = int(multiprocessing.cpu_count() / n_gpus) * 2
# ================= experimental_optimization ================= #
data_options.experimental_optimization.apply_default_optimizations = False
# data_options.experimental_optimization.autotune = True
data_options.experimental_optimization.filter_fusion = True
data_options.experimental_optimization.map_and_batch_fusion = True
data_options.experimental_optimization.map_and_filter_fusion = True
data_options.experimental_optimization.map_fusion = True
data_options.experimental_optimization.map_parallelization = True
map_vectorization_options = tf.data.experimental.MapVectorizationOptions(
)
map_vectorization_options.enabled = True
map_vectorization_options.use_choose_fastest = True
data_options.experimental_optimization.map_vectorization = map_vectorization_options
data_options.experimental_optimization.noop_elimination = True
data_options.experimental_optimization.parallel_batch = True
data_options.experimental_optimization.shuffle_and_repeat_fusion = True
# ========== Stats on TF Data =============
# aggregator = tf.data.experimental.StatsAggregator()
# data_options.experimental_stats.aggregator = aggregator
# data_options.experimental_stats.latency_all_edges = True
dataset = dataset.with_options(data_options)
return dataset
def train_and_eval(self, train_input_fn, eval_input_fn):
"""Run distributed train and eval on Mask RCNN model."""
self._save_config()
output_dir = os.path.join(self._runtime_config.model_dir, 'eval')
tf.io.gfile.makedirs(output_dir)
train_run_config = self.build_strategy_configuration('train')
train_params = self.build_model_parameters('train')
train_estimator = self.build_mask_rcnn_estimator(train_params, train_run_config, 'train')
eval_estimator = None
eval_results = None
num_cycles = math.ceil(self._runtime_config.total_steps / self._runtime_config.num_steps_per_eval)
training_hooks = get_training_hooks(
mode="train",
model_dir=self._runtime_config.model_dir,
checkpoint_path=self._runtime_config.checkpoint,
skip_checkpoint_variables=self._runtime_config.skip_checkpoint_variables
)
for cycle in range(1, num_cycles + 1):
if not MPI_is_distributed() or MPI_rank() == 0:
print() # Visual Spacing
logging.info("=================================")
logging.info(' Start training cycle %02d' % cycle)
logging.info("=================================\n")
max_cycle_step = min(int(cycle * self._runtime_config.num_steps_per_eval), self._runtime_config.total_steps)
PROFILER_ENABLED = False
if (not MPI_is_distributed() or MPI_rank() == 0) and PROFILER_ENABLED:
profiler_context_manager = tf.contrib.tfprof.ProfileContext
else:
from contextlib import suppress
profiler_context_manager = lambda *args, **kwargs: suppress() # No-Op context manager
with profiler_context_manager(
'/workspace/profiling/',
trace_steps=range(100, 200, 3),
dump_steps=[200]
) as pctx:
if (not MPI_is_distributed() or MPI_rank() == 0) and PROFILER_ENABLED:
opts = tf.compat.v1.profiler.ProfileOptionBuilder.time_and_memory()
pctx.add_auto_profiling('op', opts, [150, 200])
train_estimator.train(
input_fn=train_input_fn,
max_steps=max_cycle_step,
hooks=training_hooks,
)
if not MPI_is_distributed() or MPI_rank() == 0:
print() # Visual Spacing
logging.info("=================================")
logging.info(' Start evaluation cycle %02d' % cycle)
logging.info("=================================\n")
if eval_estimator is None:
eval_run_config = self.build_strategy_configuration('eval')
eval_params = self.build_model_parameters('eval')
eval_estimator = self.build_mask_rcnn_estimator(eval_params, eval_run_config, 'eval')
last_ckpt = tf.train.latest_checkpoint(self._runtime_config.model_dir, latest_filename=None)
logging.info("Restoring parameters from %s\n" % last_ckpt)
eval_results, predictions = evaluation.evaluate(
eval_estimator,
eval_input_fn,
self._runtime_config.eval_samples,
self._runtime_config.eval_batch_size,
self._runtime_config.include_mask,
self._runtime_config.val_json_file,
report_frequency=self._runtime_config.report_frequency
)
self._write_summary(output_dir, eval_results, predictions, max_cycle_step)
if MPI_is_distributed():
from mpi4py import MPI
comm = hvd.get_worker_comm()
comm.Barrier() # Waiting for all MPI processes to sync
return eval_results
def _get_session_config(mode, use_xla, use_amp, use_tf_distributed=False, allow_xla_at_inference=False):
assert mode in ('train', 'eval')
rewrite_options = rewriter_config_pb2.RewriterConfig(
# arithmetic_optimization=rewriter_config_pb2.RewriterConfig.OFF,
# arithmetic_optimization=rewriter_config_pb2.RewriterConfig.ON,
# constant_folding=rewriter_config_pb2.RewriterConfig.OFF,
# constant_folding=rewriter_config_pb2.RewriterConfig.ON, # TO TEST
# debug_stripper=rewriter_config_pb2.RewriterConfig.OFF,
# debug_stripper=rewriter_config_pb2.RewriterConfig.ON, # TO TEST
# dependency_optimization=rewriter_config_pb2.RewriterConfig.OFF,
# dependency_optimization=rewriter_config_pb2.RewriterConfig.ON, # TO TEST
# disable_model_pruning=False, # INCOMPATIBLE with AMP
# function_optimization=True,
# implementation_selector=True,
# loop_optimization=rewriter_config_pb2.RewriterConfig.OFF,
# loop_optimization=rewriter_config_pb2.RewriterConfig.ON, # TO TEST
# The default setting (SCHEDULING and SWAPPING HEURISTICS only)
# memory_optimization=rewriter_config_pb2.RewriterConfig.DEFAULT_MEM_OPT,
# Disabled in the meta-optimizer.
# memory_optimization=rewriter_config_pb2.RewriterConfig.NO_MEM_OPT,
# Driven by manual op-level annotations.
# memory_optimization=rewriter_config_pb2.RewriterConfig.MANUAL,
# Swapping heuristic will move a tensor from the GPU to the CPU and move it
# back when needed to reduce peak memory usage..
# memory_optimization=rewriter_config_pb2.RewriterConfig.SWAPPING_HEURISTICS,
# Recomputation heuristics will recompute ops (such as Relu activation)
# during backprop instead of storing them, reducing peak memory usage.
# memory_optimization=rewriter_config_pb2.RewriterConfig.RECOMPUTATION_HEURISTICS,
# Scheduling will split big ops such as AddN and try to enforce a schedule of
# the new computations that decreases peak memory usage.
# memory_optimization=rewriter_config_pb2.RewriterConfig.SCHEDULING_HEURISTICS,
# Use any combination of swapping and recomputation heuristics.
# memory_optimization=rewriter_config_pb2.RewriterConfig.HEURISTICS,
meta_optimizer_iterations=rewriter_config_pb2.RewriterConfig.TWO,
# meta_optimizer_iterations=rewriter_config_pb2.RewriterConfig.ONE,
# meta_optimizer_iterations=rewriter_config_pb2.RewriterConfig.DEFAULT_NUM_ITERS,
# pin_to_host_optimization=rewriter_config_pb2.RewriterConfig.OFF,
# pin_to_host_optimization=rewriter_config_pb2.RewriterConfig.ON, # TO TEST
#
# remapping=rewriter_config_pb2.RewriterConfig.OFF,
# remapping=rewriter_config_pb2.RewriterConfig.ON, # TO TEST
# scoped_allocator_optimization=rewriter_config_pb2.RewriterConfig.OFF,
# scoped_allocator_optimization=rewriter_config_pb2.RewriterConfig.ON, # TO TEST
# shape_optimization=rewriter_config_pb2.RewriterConfig.OFF,
# shape_optimization=rewriter_config_pb2.RewriterConfig.ON, # TO TEST
)
if use_amp:
logging.info("[%s] AMP is activated - Experiment Feature" % mode)
rewrite_options.auto_mixed_precision = True
config = tf.compat.v1.ConfigProto(
allow_soft_placement=True,
log_device_placement=False,
graph_options=tf.compat.v1.GraphOptions(
rewrite_options=rewrite_options,
# infer_shapes=True # Heavily drops throughput by 30%
)
)
if use_tf_distributed:
config.gpu_options.force_gpu_compatible = False
else:
config.gpu_options.force_gpu_compatible = True # Force pinned memory
if MPI_is_distributed():
config.gpu_options.visible_device_list = str(MPI_local_rank())
if use_xla and (mode == "train" or allow_xla_at_inference):
logging.info("[%s] XLA is activated - Experiment Feature" % mode)
config.graph_options.optimizer_options.global_jit_level = tf.compat.v1.OptimizerOptions.ON_1
# config.graph_options.optimizer_options.global_jit_level = tf.OptimizerOptions.ON_2
if mode == 'train':
config.intra_op_parallelism_threads = 1 # Avoid pool of Eigen threads
if MPI_is_distributed():
config.inter_op_parallelism_threads = max(2, multiprocessing.cpu_count() // hvd.local_size())
elif not use_tf_distributed:
config.inter_op_parallelism_threads = 4
return config
def main(argv):
del argv # Unused.
# ============================ Configure parameters ============================ #
RUN_CONFIG = mask_rcnn_params.default_config()
temp_config = FLAGS.flag_values_dict()
if temp_config['device'] == 'HPU':
if not MPI_is_distributed():
from habana_frameworks.tensorflow import load_habana_module
load_habana_module()
temp_config['learning_rate_decay_levels'] = [
float(decay) for decay in temp_config['learning_rate_decay_levels']
]
temp_config['learning_rate_levels'] = [
decay * temp_config['init_learning_rate']
for decay in temp_config['learning_rate_decay_levels']
]
temp_config['learning_rate_steps'] = [
int(step) for step in temp_config['learning_rate_steps']
]
RUN_CONFIG = params_io.override_hparams(RUN_CONFIG, temp_config)
if temp_config['deterministic']:
tf.config.threading.set_inter_op_parallelism_threads(1)
tf.config.threading.set_intra_op_parallelism_threads(1)
if temp_config['seed']:
os.environ['TF_DETERMINISTIC_OPS'] = '1'
tf.compat.v1.reset_default_graph()
tf.random.set_seed(temp_config['seed'])
if temp_config['device'] == "GPU":
os.environ['TF_CUDNN_DETERMINISTIC'] = '1'
else:
raise RuntimeError("Set seed to run in deterministic mode")
# ============================ Configure parameters ============================ #
if RUN_CONFIG.use_tf_distributed and MPI_is_distributed():
raise RuntimeError(
"Incompatible Runtime. Impossible to use `--use_tf_distributed` with MPIRun Horovod"
)
if RUN_CONFIG.mode in ('train', 'train_and_eval'
) and not RUN_CONFIG.training_file_pattern:
raise RuntimeError(
'You must specify `training_file_pattern` for training.')
if RUN_CONFIG.mode in ('eval', 'train_and_eval'):
if not RUN_CONFIG.validation_file_pattern:
raise RuntimeError(
'You must specify `validation_file_pattern` for evaluation.')
if RUN_CONFIG.val_json_file == "" and not RUN_CONFIG.include_groundtruth_in_features:
raise RuntimeError(
'You must specify `val_json_file` or include_groundtruth_in_features=True for evaluation.'
)
if not RUN_CONFIG.include_groundtruth_in_features and not os.path.isfile(
RUN_CONFIG.val_json_file):
raise FileNotFoundError("Validation JSON File not found: %s" %
RUN_CONFIG.val_json_file)
dllogger.init(backends=[
dllogger.JSONStreamBackend(verbosity=dllogger.Verbosity.VERBOSE,
filename=RUN_CONFIG.log_path)
])
if RUN_CONFIG.mode in ('train', 'train_and_eval'):
train_input_fn = dataloader.InputReader(
file_pattern=RUN_CONFIG.training_file_pattern,
mode=tf.estimator.ModeKeys.TRAIN,
num_examples=None,
use_fake_data=RUN_CONFIG.use_fake_data,
use_instance_mask=RUN_CONFIG.include_mask,
seed=RUN_CONFIG.seed)
else:
train_input_fn = None
if RUN_CONFIG.mode in ('eval', 'train_and_eval') or (
RUN_CONFIG.mode == 'train' and RUN_CONFIG.eval_after_training):
eval_input_fn = dataloader.InputReader(
file_pattern=RUN_CONFIG.validation_file_pattern,
mode=tf.estimator.ModeKeys.PREDICT,
num_examples=RUN_CONFIG.eval_samples,
use_fake_data=False,
use_instance_mask=RUN_CONFIG.include_mask,
seed=RUN_CONFIG.seed)
else:
eval_input_fn = None
with tf.profiler.experimental.Profile(
RUN_CONFIG.model_dir) if RUN_CONFIG.profile else suppress():
run_executer(RUN_CONFIG, train_input_fn, eval_input_fn)
def compute_coco_eval_metric(predictor,
num_batches=-1,
include_mask=True,
annotation_json_file="",
eval_batch_size=-1,
report_frequency=None):
"""Compute COCO eval metric given a prediction generator.
Args:
predictor: a generator that iteratively pops a dictionary of predictions
with the format compatible with COCO eval tool.
num_batches: the number of batches to be aggregated in eval. This is how
many times that the predictor gets pulled.
include_mask: a boolean that indicates whether we include the mask eval.
annotation_json_file: the annotation json file of the eval dataset.
Returns:
eval_results: the aggregated COCO metric eval results.
"""
if annotation_json_file == "":
annotation_json_file = None
use_groundtruth_from_json = (annotation_json_file is not None)
predictions = dict()
batch_idx = 0
if use_groundtruth_from_json:
eval_metric = coco_metric.EvaluationMetric(annotation_json_file,
include_mask=include_mask)
else:
eval_metric = coco_metric.EvaluationMetric(filename=None,
include_mask=include_mask)
def evaluation_preds(preds):
# Essential to avoid modifying the source dict
_preds = copy.deepcopy(preds)
for k, v in six.iteritems(_preds):
_preds[k] = np.concatenate(_preds[k], axis=0)
if 'orig_images' in _preds and _preds['orig_images'].shape[0] > 10:
# Only samples a few images for visualization.
_preds['orig_images'] = _preds['orig_images'][:10]
if use_groundtruth_from_json:
eval_results = eval_metric.predict_metric_fn(_preds)
else:
images, annotations = coco_utils.extract_coco_groundtruth(
_preds, include_mask)
coco_dataset = coco_utils.create_coco_format_dataset(
images, annotations)
eval_results = eval_metric.predict_metric_fn(
_preds, groundtruth_data=coco_dataset)
return eval_results
# Take into account cuDNN & Tensorflow warmup
# Drop N first steps for avg throughput calculation
BURNIN_STEPS = 100
model_throughput_list = list()
inference_time_list = list()
if MPI_is_distributed():
eval_batch_size *= MPI_size()
num_batches /= MPI_size()
while num_batches < 0 or batch_idx < num_batches:
try:
step_t0 = time.time()
step_predictions = six.next(predictor)
if MPI_is_distributed():
from mpi4py import MPI
MPI.COMM_WORLD.Barrier(
) # Need to get time for all predictors for given batch_idx
batch_time = time.time() - step_t0
throughput = eval_batch_size / batch_time
model_throughput_list.append(throughput)
inference_time_list.append(batch_time)
logging.info(
'Running inference on batch %03d/%03d... - Step Time: %.4fs - Throughput: %.1f imgs/s'
% (batch_idx + 1, num_batches, batch_time, throughput))
except StopIteration:
logging.info('Get StopIteration at %d batch.' % (batch_idx + 1))
break
step_predictions = process_prediction_for_eval(step_predictions)
for k, v in step_predictions.items():
if k not in predictions:
predictions[k] = [v]
else:
predictions[k].append(v)
batch_idx = batch_idx + 1
# If you want the report to happen each report_frequency to happen each report_frequency batches.
# Thus, each report is of eval_batch_size * report_frequency
if report_frequency and batch_idx % report_frequency == 0:
eval_results = evaluation_preds(preds=predictions)
logging.info('Eval results: %s' %
pprint.pformat(eval_results, indent=4))
if MPI_is_distributed():
from mpi4py import MPI
all_predictions = MPI.COMM_WORLD.gather(predictions, root=0)
MPI.COMM_WORLD.Barrier(
) # FIXME: first gather is calling MPI_FINALIZE causing crash
if MPI_rank() == 0:
predictions.clear()
for pred in all_predictions:
for k in pred.keys():
if k not in predictions:
predictions[k] = pred[k]
else:
predictions[k].extend(pred[k])
inference_time_list.sort()
eval_results = evaluation_preds(preds=predictions)
if not MPI_is_distributed() or MPI_rank() == 0:
average_time = np.mean(inference_time_list)
latency_50 = max(
inference_time_list[:int(len(inference_time_list) * 0.5)])
latency_90 = max(
inference_time_list[:int(len(inference_time_list) * 0.90)])
latency_95 = max(
inference_time_list[:int(len(inference_time_list) * 0.95)])
latency_99 = max(
inference_time_list[:int(len(inference_time_list) * 0.99)])
latency_100 = max(
inference_time_list[:int(len(inference_time_list) * 1)])
print() # Visual Spacing
logging.info("# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ #")
logging.info(" Evaluation Performance Summary ")
logging.info("# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ #")
total_processing_hours, rem = divmod(np.sum(model_throughput_list),
3600)
total_processing_minutes, total_processing_seconds = divmod(rem, 60)
if len(model_throughput_list) > BURNIN_STEPS:
# Take into account cuDNN & Tensorflow warmup
# Drop N first steps for avg throughput calculation
# Also drop last step which may have a different batch size
avg_throughput = np.mean(model_throughput_list[BURNIN_STEPS:-1])
else:
avg_throughput = -1.
print() # Visual Spacing
logging.info("Average throughput: {throughput:.1f} samples/sec".format(
throughput=avg_throughput))
logging.info("Inference Latency Average (s) = {avg:.4f}".format(
avg=average_time))
logging.info(
"Inference Latency 50% (s) = {cf_50:.4f}".format(cf_50=latency_50))
logging.info("Inference Latency 90% (s) = {cf_90:.4f}".format(
cf_90=latency_90))
logging.info("Inference Latency 95% (s) = {cf_95:.4f}".format(
cf_95=latency_95))
logging.info("Inference Latency 99% (s) = {cf_99:.4f}".format(
cf_99=latency_99))
logging.info("Inference Latency 100% (s) = {cf_100:.4f}".format(
cf_100=latency_100))
logging.info("Total processed steps: {total_steps}".format(
total_steps=len(model_throughput_list)))
logging.info(
"Total processing time: {hours}h {minutes:02d}m {seconds:02d}s".
format(hours=total_processing_hours,
minutes=int(total_processing_minutes),
seconds=int(total_processing_seconds)))
dllogger.log(step=(),
data={"avg_inference_throughput": avg_throughput},
verbosity=Verbosity.DEFAULT)
avg_inference_time = float(total_processing_hours * 3600 +
int(total_processing_minutes) * 60 +
int(total_processing_seconds))
dllogger.log(step=(),
data={"avg_inference_time": avg_inference_time},
verbosity=Verbosity.DEFAULT)
logging.info("==================== Metrics ====================")
# logging.info('Eval Epoch results: %s' % pprint.pformat(eval_results, indent=4))
for key, value in sorted(eval_results.items(),
key=operator.itemgetter(0)):
logging.info("%s: %.9f" % (key, value))
print() # Visual Spacing
return eval_results, predictions
def main(argv):
del argv # Unused.
# ============================ Configure parameters ============================ #
RUN_CONFIG = mask_rcnn_params.default_config()
temp_config = FLAGS.flag_values_dict()
temp_config['learning_rate_decay_levels'] = [float(decay) for decay in temp_config['learning_rate_decay_levels']]
temp_config['learning_rate_levels'] = [
decay * temp_config['init_learning_rate'] for decay in temp_config['learning_rate_decay_levels']
]
temp_config['learning_rate_steps'] = [int(step) for step in temp_config['learning_rate_steps']]
RUN_CONFIG = params_io.override_hparams(RUN_CONFIG, temp_config)
# ============================ Configure parameters ============================ #
if RUN_CONFIG.use_tf_distributed and MPI_is_distributed():
raise RuntimeError("Incompatible Runtime. Impossible to use `--use_tf_distributed` with MPIRun Horovod")
if RUN_CONFIG.mode in ('train', 'train_and_eval') and not RUN_CONFIG.training_file_pattern:
raise RuntimeError('You must specify `training_file_pattern` for training.')
if RUN_CONFIG.mode in ('eval', 'train_and_eval'):
if not RUN_CONFIG.validation_file_pattern:
raise RuntimeError('You must specify `validation_file_pattern` for evaluation.')
if RUN_CONFIG.val_json_file == "" and not RUN_CONFIG.include_groundtruth_in_features:
raise RuntimeError(
'You must specify `val_json_file` or include_groundtruth_in_features=True for evaluation.')
if not RUN_CONFIG.include_groundtruth_in_features and not os.path.isfile(RUN_CONFIG.val_json_file):
raise FileNotFoundError("Validation JSON File not found: %s" % RUN_CONFIG.val_json_file)
dllogger.init(backends=[dllogger.JSONStreamBackend(verbosity=dllogger.Verbosity.VERBOSE,
filename=RUN_CONFIG.log_path)])
if RUN_CONFIG.mode in ('train', 'train_and_eval'):
train_input_fn = dataloader.InputReader(
file_pattern=RUN_CONFIG.training_file_pattern,
mode=tf.estimator.ModeKeys.TRAIN,
num_examples=None,
use_fake_data=RUN_CONFIG.use_fake_data,
use_instance_mask=RUN_CONFIG.include_mask,
seed=RUN_CONFIG.seed
)
else:
train_input_fn = None
if RUN_CONFIG.mode in ('eval', 'train_and_eval' or (RUN_CONFIG.mode == 'train' and RUN_CONFIG.eval_after_training)):
eval_input_fn = dataloader.InputReader(
file_pattern=RUN_CONFIG.validation_file_pattern,
mode=tf.estimator.ModeKeys.PREDICT,
num_examples=RUN_CONFIG.eval_samples,
use_fake_data=False,
use_instance_mask=RUN_CONFIG.include_mask,
seed=RUN_CONFIG.seed
)
else:
eval_input_fn = None
run_executer(RUN_CONFIG, train_input_fn, eval_input_fn)