def test_case_evaluate_FID_IS_ddp(self): """ Usage: export PYTHONWARNINGS=ignore export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 export TIME_STR=1 export PYTHONPATH=./exp:./stylegan2-pytorch:./ python -c "from template_lib.v2.tests.test_GAN import TestingTFFIDISScore;\ TestingTFFIDISScore().test_case_evaluate_FID_IS_ddp()" :return: """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '0,1' if 'TIME_STR' not in os.environ: os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0' command, outdir = get_command_and_outdir( self, func_name=sys._getframe().f_code.co_name, file=__file__) argv_str = f""" --tl_config_file none --tl_command none --tl_outdir {outdir} """ args = setup_outdir_and_yaml(argv_str) nproc_per_node = len(os.environ['CUDA_VISIBLE_DEVICES'].split(',')) cmd_str = f""" python -m torch.distributed.launch --nproc_per_node={nproc_per_node} --master_port=8888 template_lib/v2/GAN/evaluation/tf_FID_IS_score.py --run_func TFFIDISScore.test_case_evaluate_FID_IS """ cmd_str += get_append_cmd_str(args) start_cmd_run(cmd_str) pass
def test_test_vae(self): """ Usage: export CUDA_VISIBLE_DEVICES=1 export TIME_STR=1 export PYTHONPATH=./PyTorch_VAE_lib:./ python -c "from exp.tests.test_pytorch_vae import Testing_PyTorch_VAE;\ Testing_PyTorch_VAE().test_test_vae()" :return: """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '1' if 'TIME_STR' not in os.environ: os.environ['TIME_STR'] = '0' if utils.is_debugging() else '1' command, outdir = get_command_and_outdir( self, func_name=sys._getframe().f_code.co_name, file=__file__) argv_str = f""" --tl_config_file exp/configs/pytorch_vae.yaml --tl_command {command} --tl_outdir {outdir} """ args = setup_outdir_and_yaml(argv_str) nproc_per_node = len(os.environ['CUDA_VISIBLE_DEVICES'].split(',')) cmd_str = f""" python exp/scripts/test.py -c PyTorch_VAE_lib/configs/vae.yaml """ cmd_str += get_append_cmd_str(args) start_cmd_run(cmd_str) pass
def test_ddp(self): """ Usage: export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 export TIME_STR=1 export PYTHONPATH=./exp:./stylegan2-pytorch:./ python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\ Testing_stylegan2().test_train_ffhq_128()" :return: """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '0,1' if 'TIME_STR' not in os.environ: os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0' from template_lib.v2.config import get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, \ start_cmd_run command, outdir = get_command_and_outdir(self, func_name=sys._getframe().f_code.co_name, file=__file__) argv_str = f""" --tl_config_file exp/configs/styleganv2.yaml --tl_command {command} --tl_outdir {outdir} """ args = setup_outdir_and_yaml(argv_str) nproc_per_node = len(os.environ['CUDA_VISIBLE_DEVICES'].split(',')) cmd_str = f""" python -m torch.distributed.launch --nproc_per_node={nproc_per_node} --master_port=8888 exp/scripts/train.py {get_append_cmd_str(args)} """ start_cmd_run(cmd_str) pass
def test_nni(self, use_nni=False): """ Usage: export CUDA_VISIBLE_DEVICES=1,2,3,4,5,6,7 export TIME_STR=1 export PYTHONPATH=./ python -c "from exp.tests.test_nni import Testing_nni;\ Testing_nni().test_mnist_pytorch(use_nni=False)" :return: """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '1' if 'TIME_STR' not in os.environ: os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0' from template_lib.v2.config import get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, \ start_cmd_run command, outdir = get_command_and_outdir( self, func_name=sys._getframe().f_code.co_name, file=__file__) if use_nni: outdir = outdir + '_nni' argv_str = f""" --tl_config_file exp/configs/nni.yaml --tl_command {command} --tl_outdir {outdir} """ args = setup_outdir_and_yaml(argv_str) n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(',')) cmd_str = f""" python exp/mnist_pytorch/mnist.py {get_append_cmd_str(args)} """ # use_nni = True if use_nni: # update nni config file nni_config_file = "exp/mnist_pytorch/config.yml" python_command = ' '.join( [s.strip(' ') for s in cmd_str.split('\n')]) + ' --tl_nni' update_nni_cfg_str = f""" logDir: {os.path.abspath(args.tl_outdir)}/nni trialConcurrency: {n_gpus} trial: command: {python_command} codeDir: {os.path.abspath(os.path.curdir)} """ updated_config_file = update_nni_config_file( nni_config_file=nni_config_file, update_nni_cfg_str=update_nni_cfg_str) cmd_str = f""" bash nnictl create --config {updated_config_file} """ start_cmd_run(cmd_str) pass
def test_calculate_fid_stat_CIFAR100(self, debug=True): """ Usage: export LD_LIBRARY_PATH=$HOME/.keras/envs/cuda-10.0/lib64:$HOME/.keras/envs/cudnn-10.0-linux-x64-v7.6.5.32/lib64 export LD_LIBRARY_PATH=/usr/local/cuda-10.0/lib64 export CUDA_VISIBLE_DEVICES=2 export TIME_STR=0 export PYTHONPATH=./ python -c "from template_lib.v2.tests.test_GAN import TestingTFFIDISScore;\ TestingTFFIDISScore().test_calculate_fid_stat_CIFAR100(debug=False)" :return: """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '0' if 'TIME_STR' not in os.environ: os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0' from template_lib.v2.config_cfgnode.argparser import \ (get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run) command, outdir = get_command_and_outdir( self, func_name=sys._getframe().f_code.co_name, file=__file__) argv_str = f""" --tl_config_file template_lib/v2/GAN/configs/TFFIDISScore.yaml --tl_command {command} --tl_outdir {outdir} """ args = setup_outdir_and_yaml(argv_str) os.environ['LD_LIBRARY_PATH'] = '/usr/local/cuda-10.0/lib64' n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(',')) cmd_str = f""" python template_lib/v2/GAN/scripts/calculate_fid_stat_CIFAR.py {get_append_cmd_str(args)} --tl_opts OUTPUT_DIR {args.tl_outdir}/detectron2 """ if debug: cmd_str += f""" """ start_cmd_run(cmd_str) pass
def test_detectron2(self): """ Usage: export ANSI_COLORS_DISABLED=1 export CUDA_VISIBLE_DEVICES=2 export TIME_STR=1 export PYTHONPATH=.:./exp python -c "from exp.tests.test_nas_cgan import TestingPrepareData;\ TestingPrepareData().test_calculate_fid_stat_CIFAR10()" :return: """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '0,1' if 'TIME_STR' not in os.environ: os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0' from template_lib.v2.config import get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, \ start_cmd_run command, outdir = get_command_and_outdir( self, func_name=sys._getframe().f_code.co_name, file=__file__) argv_str = f""" --tl_config_file exp/nas_cgan/config/prepare_data.yaml --tl_command {command} --tl_outdir {outdir} """ args = setup_outdir_and_yaml(argv_str) num_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(',')) cmd_str = f""" python exp/scripts/train_net.py {get_append_cmd_str(args)} --num-gpus {num_gpus} """ start_cmd_run(cmd_str) pass
def test_ddp(self, debug=True): """ Usage: python template_lib/modelarts/scripts/copy_tool.py \ -s s3://bucket-7001/ZhouPeng/pypi/torch1_7_0 -d /cache/pypi -t copytree for filename in /cache/pypi/*.whl; do pip install $filename done proj_root=moco-exp python template_lib/modelarts/scripts/copy_tool.py \ -s s3://bucket-7001/ZhouPeng/codes/$proj_root -d /cache/$proj_root -t copytree -b /cache/$proj_root/code.zip cd /cache/$proj_root pip install -r requirements.txt export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 export TIME_STR=1 export PYTHONPATH=./exp:./stylegan2-pytorch:./ python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\ Testing_stylegan2().test_train_ffhq_128()" :return: """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '0' if 'TIME_STR' not in os.environ: os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0' from template_lib.v2.config_cfgnode.argparser import \ (get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run) tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') + 1:]) if '--tl_opts' in sys.argv else '' print(f'tl_opts:\n {tl_opts}') command, outdir = get_command_and_outdir(self, func_name=sys._getframe().f_code.co_name, file=__file__) argv_str = f""" --tl_config_file template_lib/v2/tests/configs/config.yaml --tl_command {command} --tl_outdir {outdir} """ args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True) n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(',')) cmd_str = f""" python -m torch.distributed.launch --nproc_per_node={n_gpus} --master_port=8888 template_lib/v2/ddp/train.py {get_append_cmd_str(args)} """ if debug: cmd_str += f""" --tl_debug --tl_opts """ else: cmd_str += f""" --tl_opts {tl_opts} """ start_cmd_run(cmd_str) # from template_lib.v2.config_cfgnode import update_parser_defaults_from_yaml, global_cfg # from template_lib.modelarts import modelarts_utils # update_parser_defaults_from_yaml(parser) # modelarts_utils.setup_tl_outdir_obs(global_cfg) # modelarts_utils.modelarts_sync_results_dir(global_cfg, join=True) # modelarts_utils.prepare_dataset(global_cfg.get('modelarts_download', {}), global_cfg=global_cfg) # # modelarts_utils.prepare_dataset(global_cfg.get('modelarts_upload', {}), global_cfg=global_cfg, download=False) # modelarts_utils.modelarts_sync_results_dir(global_cfg, join=True) pass