def main():
"""
Starting point of the application
"""
hvd.init()
params = parse_args(PARSER.parse_args())
set_flags(params)
model_dir = prepare_model_dir(params)
params.model_dir = model_dir
logger = get_logger(params)
model = Unet()
dataset = Dataset(data_dir=params.data_dir,
batch_size=params.batch_size,
fold=params.fold,
augment=params.augment,
gpu_id=hvd.rank(),
num_gpus=hvd.size(),
seed=params.seed)
if 'train' in params.exec_mode:
train(params, model, dataset, logger)
if 'evaluate' in params.exec_mode:
if hvd.rank() == 0:
evaluate(params, model, dataset, logger)
if 'predict' in params.exec_mode:
if hvd.rank() == 0:
predict(params, model, dataset, logger)
def main():
params = parse_args()
tf.random.set_seed(params.seed)
tf.get_logger().setLevel(logging.ERROR)
params = setup_horovod(params)
set_flags(params)
model_dir = prepare_model_dir(params)
logger = get_logger(params)
dataset = Dataset(data_dir=params.data_dir,
batch_size=params.batch_size,
fold_idx=params.fold,
n_folds=params.num_folds,
params=params,
seed=params.seed)
estimator = build_estimator(params, model_dir)
if params.tensorboard_logging and (params.worker_id == 0
or params.log_all_workers):
from TensorFlow.common.tb_utils import write_hparams_v1
write_hparams_v1(params.log_dir, vars(params))
if not params.benchmark:
params.max_steps = params.max_steps // params.num_workers
if 'train' in params.exec_mode:
with dump_callback(params.dump_config):
training_hooks = get_hooks(params, logger)
dataset_fn = dataset.synth_train_fn if params.synth_data else dataset.train_fn
estimator.train(input_fn=dataset_fn,
steps=params.max_steps,
hooks=training_hooks)
if 'evaluate' in params.exec_mode:
result = estimator.evaluate(input_fn=dataset.eval_fn,
steps=dataset.eval_size)
data = parse_evaluation_results(result)
if params.worker_id == 0:
logger.log(step=(), data=data)
if 'predict' == params.exec_mode:
inference_hooks = get_hooks(params, logger)
if params.worker_id == 0:
count = 1 if not params.benchmark else 2 * params.warmup_steps * params.batch_size // dataset.test_size
predictions = estimator.predict(input_fn=lambda: dataset.test_fn(
count=count, drop_remainder=params.benchmark),
hooks=inference_hooks)
for idx, p in enumerate(predictions):
volume = p['predictions']
if not params.benchmark:
np.save(
os.path.join(params.model_dir,
"vol_{}.npy".format(idx)), volume)
def main():
"""
Starting point of the application
"""
params = parse_args(description="UNet-medical")
if params.use_horovod:
hvd_init()
set_flags(params)
model_dir = prepare_model_dir(params)
params.model_dir = model_dir
logger = get_logger(params)
tb_logger = None
if params.tensorboard_logging:
log_dir = params.log_dir
if horovod_enabled() and params.log_all_workers:
log_dir = os.path.join(log_dir, f'worker_{hvd_rank()}')
tb_logger = namedtuple('TBSummaryWriters', 'train_writer eval_writer')(
tf.summary.create_file_writer(log_dir),
tf.summary.create_file_writer(os.path.join(log_dir, 'eval')))
model = Unet()
dataset = Dataset(data_dir=params.data_dir,
batch_size=params.batch_size,
fold=params.fold,
augment=params.augment,
hpu_id=hvd_rank() if horovod_enabled() else 0,
num_hpus=hvd_size() if horovod_enabled() else 1,
seed=params.seed)
if 'train' in params.exec_mode:
with dump_callback(params.dump_config):
train(params, model, dataset, logger, tb_logger)
if 'evaluate' in params.exec_mode:
evaluate(params, model, dataset, logger, tb_logger)
if 'predict' in params.exec_mode:
predict(params, model, dataset, logger)
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################
# Copyright (C) 2021 Habana Labs, Ltd. an Intel Company
###############################################################################
# Changes:
# - script migration to Tensorflow 2.x version
# - added seed setting possibility to random operations
import tensorflow as tf
from runtime.arguments import parse_args
params = parse_args()
def apply_transforms(x, y, mean, stdev, transforms):
for _t in transforms:
if _t is not None:
x, y = _t(x, y, mean, stdev)
return x, y
def apply_test_transforms(x, mean, stdev, transforms):
for _t in transforms:
if _t is not None:
x = _t(x, y=None, mean=mean, stdev=stdev)
return x
###############################################################################
# Copyright (C) 2020-2021 Habana Labs, Ltd. an Intel Company
###############################################################################
import os
import sys
from runtime.arguments import parse_args
from central.training_run_config import TrainingRunHWConfig
description = "This script is a distributed launcher for unet2d.py " \
"and accepts the same arguments as orginal unet2d.py script.\n" \
"In case argument --hvd_workers > 1 is passed, " \
"it runs 'unet2d.py [ARGS] --use_horovod' via mpirun with generated HCL config.\n"
params = parse_args(description, distributed_launcher=True)
script_to_run = str(
os.path.abspath(os.path.join(os.path.dirname(__file__), "unet2d.py")))
command_to_run = [sys.executable, script_to_run]
# Prepare mpi command prefix for multinode run
if params.hvd_workers > 1:
hw_config = TrainingRunHWConfig(scaleout=True,
num_workers_per_hls=params.hvd_workers,
kubernetes_run=params.kubernetes_run,
output_filename="demo_unet2d")
mpirun_cmd = hw_config.mpirun_cmd.split(" ")
command_to_run = mpirun_cmd + command_to_run + ["--use_horovod"]
command_to_run += sys.argv[1:]
command_str = ' '.join(command_to_run)