def run_benchmark(args):
# modify the template config file and generate the user defined config file.
cfg_process.generate_cfg(CONFIG_TEMPLATE_DIR, CONFIG_DIR, **vars(args))
config.read(CONFIG_DIR)
# the user defined config file should only have one task
selected_task = config.sections()[0]
metric_patterns = literal_eval(config.get(selected_task, "patterns"))
metric_names = literal_eval(config.get(selected_task, "metrics"))
metric_compute_methods = literal_eval(
config.get(selected_task, "compute_method"))
command_to_execute = config.get(selected_task, "command_to_execute")
num_gpus = int(config.get(selected_task, "num_gpus"))
metrics_manager.benchmark(command_to_execute=command_to_execute,
metric_patterns=metric_patterns,
metric_names=metric_names,
metric_compute_methods=metric_compute_methods,
num_gpus=num_gpus,
task_name=selected_task,
suffix=args.metrics_suffix,
framework=args.framework)
# clean up
os.remove(CONFIG_DIR)
def test_change_num_gpu_and_epochs(self):
args = {
"task_name": "resnet50_cifar10_symbolic",
"num_gpus": 3,
"epochs": 3,
"metrics_postfix": "daily",
}
tmp = os.path.join(tempfile.gettempdir(), "task_config.cfg")
cfg_process.generate_cfg(CONFIG_TEMPLATE_DIR, tmp, **args)
config.read(tmp)
self.assertEqual(int(config.get(args["task_name"], "num_gpus")), args["num_gpus"])
cmd = config.get(args["task_name"], "command_to_execute")
self.assertEqual(cmd.strip(), "python image_classification/image_classification.py "
"--model resnet50_v1 --dataset cifar10"
" --mode symbolic --gpus {0} --epochs {1} "
"--log-interval 50".format(args["num_gpus"], args["epochs"]))
os.remove(tmp)
def test_change_num_gpu_and_epochs(self):
args = {
"task_name": "resnet50_cifar10_symbolic",
"num_gpus": 3,
"epochs": 3,
"metrics_postfix": "daily",
}
tmp = os.path.join(tempfile.gettempdir(), "task_config.cfg")
cfg_process.generate_cfg(CONFIG_TEMPLATE_DIR, tmp, **args)
config.read(tmp)
self.assertEqual(int(config.get(args["task_name"], "num_gpus")),
args["num_gpus"])
cmd = config.get(args["task_name"], "command_to_execute")
self.assertEqual(
cmd.strip(), "python image_classification/image_classification.py "
"--model resnet50_v1 --dataset cifar10"
" --mode symbolic --gpus {0} --epochs {1} "
"--log-interval 50".format(args["num_gpus"], args["epochs"]))
os.remove(tmp)
parser.add_argument('--metrics-suffix',
type=str,
help='Metrics suffix e.g. --metrics-suffix daily')
parser.add_argument('--kvstore',
type=str,
default='device',
help='kvstore to use for trainer/module.')
parser.add_argument('--dtype',
type=str,
default='float32',
help='floating point precision to use')
args = parser.parse_args()
# modify the template config file and generate the user defined config file.
cfg_process.generate_cfg(CONFIG_TEMPLATE_DIR, CONFIG_DIR, **vars(args))
config.read(CONFIG_DIR)
# the user defined config file should only have one task
selected_task = config.sections()[0]
metric_patterns = literal_eval(config.get(selected_task, "patterns"))
metric_names = literal_eval(config.get(selected_task, "metrics"))
metric_compute_methods = literal_eval(
config.get(selected_task, "compute_method"))
command_to_execute = config.get(selected_task, "command_to_execute")
num_gpus = int(config.get(selected_task, "num_gpus"))
benchmark(command_to_execute=command_to_execute,
metric_patterns=metric_patterns,
metric_names=metric_names,
metric_compute_methods=metric_compute_methods,
if __name__ == '__main__':
parser = argparse.ArgumentParser(description="Run a benchmark task.")
parser.add_argument('--framework', type=str, help='Framework name e.g. mxnet')
parser.add_argument('--task-name', type=str, help='Task Name e.g. resnet50_cifar10_symbolic.')
parser.add_argument('--num-gpus', type=int, help='Numbers of gpus. e.g. --num-gpus 8')
parser.add_argument('--epochs', type=int, help='Numbers of epochs for training. e.g. --epochs 20')
parser.add_argument('--metrics-suffix', type=str, help='Metrics suffix e.g. --metrics-suffix daily')
parser.add_argument('--kvstore', type=str, default='device',help='kvstore to use for trainer/module.')
parser.add_argument('--dtype', type=str, default='float32',help='floating point precision to use')
args = parser.parse_args()
# modify the template config file and generate the user defined config file.
cfg_process.generate_cfg(CONFIG_TEMPLATE_DIR, CONFIG_DIR, **vars(args))
config.read(CONFIG_DIR)
# the user defined config file should only have one task
selected_task = config.sections()[0]
metric_patterns = literal_eval(config.get(selected_task, "patterns"))
metric_names = literal_eval(config.get(selected_task, "metrics"))
metric_compute_methods = literal_eval(config.get(selected_task, "compute_method"))
command_to_execute = config.get(selected_task, "command_to_execute")
num_gpus = int(config.get(selected_task, "num_gpus"))
benchmark(
command_to_execute=command_to_execute,
metric_patterns=metric_patterns,
metric_names=metric_names,
metric_compute_methods=metric_compute_methods,