def test_case_evaluate_FID_IS_ddp(self):
"""
Usage:
export PYTHONWARNINGS=ignore
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
export TIME_STR=1
export PYTHONPATH=./exp:./stylegan2-pytorch:./
python -c "from template_lib.v2.tests.test_GAN import TestingTFFIDISScore;\
TestingTFFIDISScore().test_case_evaluate_FID_IS_ddp()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0,1'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file none
--tl_command none
--tl_outdir {outdir}
"""
args = setup_outdir_and_yaml(argv_str)
nproc_per_node = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
cmd_str = f"""
python -m torch.distributed.launch --nproc_per_node={nproc_per_node} --master_port=8888
template_lib/v2/GAN/evaluation/tf_FID_IS_score.py
--run_func TFFIDISScore.test_case_evaluate_FID_IS
"""
cmd_str += get_append_cmd_str(args)
start_cmd_run(cmd_str)
pass
def test_test_vae(self):
"""
Usage:
export CUDA_VISIBLE_DEVICES=1
export TIME_STR=1
export PYTHONPATH=./PyTorch_VAE_lib:./
python -c "from exp.tests.test_pytorch_vae import Testing_PyTorch_VAE;\
Testing_PyTorch_VAE().test_test_vae()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '1'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '1'
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file exp/configs/pytorch_vae.yaml
--tl_command {command}
--tl_outdir {outdir}
"""
args = setup_outdir_and_yaml(argv_str)
nproc_per_node = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
cmd_str = f"""
python exp/scripts/test.py -c PyTorch_VAE_lib/configs/vae.yaml
"""
cmd_str += get_append_cmd_str(args)
start_cmd_run(cmd_str)
pass
def test_ddp(self):
"""
Usage:
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
export TIME_STR=1
export PYTHONPATH=./exp:./stylegan2-pytorch:./
python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\
Testing_stylegan2().test_train_ffhq_128()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0,1'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config import get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, \
start_cmd_run
command, outdir = get_command_and_outdir(self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file exp/configs/styleganv2.yaml
--tl_command {command}
--tl_outdir {outdir}
"""
args = setup_outdir_and_yaml(argv_str)
nproc_per_node = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
cmd_str = f"""
python -m torch.distributed.launch --nproc_per_node={nproc_per_node} --master_port=8888
exp/scripts/train.py
{get_append_cmd_str(args)}
"""
start_cmd_run(cmd_str)
pass
def test_nni(self, use_nni=False):
"""
Usage:
export CUDA_VISIBLE_DEVICES=1,2,3,4,5,6,7
export TIME_STR=1
export PYTHONPATH=./
python -c "from exp.tests.test_nni import Testing_nni;\
Testing_nni().test_mnist_pytorch(use_nni=False)"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '1'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config import get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, \
start_cmd_run
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
if use_nni: outdir = outdir + '_nni'
argv_str = f"""
--tl_config_file exp/configs/nni.yaml
--tl_command {command}
--tl_outdir {outdir}
"""
args = setup_outdir_and_yaml(argv_str)
n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
cmd_str = f"""
python
exp/mnist_pytorch/mnist.py
{get_append_cmd_str(args)}
"""
# use_nni = True
if use_nni:
# update nni config file
nni_config_file = "exp/mnist_pytorch/config.yml"
python_command = ' '.join(
[s.strip(' ') for s in cmd_str.split('\n')]) + ' --tl_nni'
update_nni_cfg_str = f"""
logDir: {os.path.abspath(args.tl_outdir)}/nni
trialConcurrency: {n_gpus}
trial:
command: {python_command}
codeDir: {os.path.abspath(os.path.curdir)}
"""
updated_config_file = update_nni_config_file(
nni_config_file=nni_config_file,
update_nni_cfg_str=update_nni_cfg_str)
cmd_str = f"""
bash
nnictl create --config {updated_config_file}
"""
start_cmd_run(cmd_str)
pass
def test_calculate_fid_stat_CIFAR100(self, debug=True):
"""
Usage:
export LD_LIBRARY_PATH=$HOME/.keras/envs/cuda-10.0/lib64:$HOME/.keras/envs/cudnn-10.0-linux-x64-v7.6.5.32/lib64
export LD_LIBRARY_PATH=/usr/local/cuda-10.0/lib64
export CUDA_VISIBLE_DEVICES=2
export TIME_STR=0
export PYTHONPATH=./
python -c "from template_lib.v2.tests.test_GAN import TestingTFFIDISScore;\
TestingTFFIDISScore().test_calculate_fid_stat_CIFAR100(debug=False)"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file template_lib/v2/GAN/configs/TFFIDISScore.yaml
--tl_command {command}
--tl_outdir {outdir}
"""
args = setup_outdir_and_yaml(argv_str)
os.environ['LD_LIBRARY_PATH'] = '/usr/local/cuda-10.0/lib64'
n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
cmd_str = f"""
python template_lib/v2/GAN/scripts/calculate_fid_stat_CIFAR.py
{get_append_cmd_str(args)}
--tl_opts OUTPUT_DIR {args.tl_outdir}/detectron2
"""
if debug:
cmd_str += f"""
"""
start_cmd_run(cmd_str)
pass
def test_detectron2(self):
"""
Usage:
export ANSI_COLORS_DISABLED=1
export CUDA_VISIBLE_DEVICES=2
export TIME_STR=1
export PYTHONPATH=.:./exp
python -c "from exp.tests.test_nas_cgan import TestingPrepareData;\
TestingPrepareData().test_calculate_fid_stat_CIFAR10()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0,1'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config import get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, \
start_cmd_run
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file exp/nas_cgan/config/prepare_data.yaml
--tl_command {command}
--tl_outdir {outdir}
"""
args = setup_outdir_and_yaml(argv_str)
num_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
cmd_str = f"""
python exp/scripts/train_net.py
{get_append_cmd_str(args)}
--num-gpus {num_gpus}
"""
start_cmd_run(cmd_str)
pass
def test_ddp(self, debug=True):
"""
Usage:
python template_lib/modelarts/scripts/copy_tool.py \
-s s3://bucket-7001/ZhouPeng/pypi/torch1_7_0 -d /cache/pypi -t copytree
for filename in /cache/pypi/*.whl; do
pip install $filename
done
proj_root=moco-exp
python template_lib/modelarts/scripts/copy_tool.py \
-s s3://bucket-7001/ZhouPeng/codes/$proj_root -d /cache/$proj_root -t copytree -b /cache/$proj_root/code.zip
cd /cache/$proj_root
pip install -r requirements.txt
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
export TIME_STR=1
export PYTHONPATH=./exp:./stylegan2-pytorch:./
python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\
Testing_stylegan2().test_train_ffhq_128()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') + 1:]) if '--tl_opts' in sys.argv else ''
print(f'tl_opts:\n {tl_opts}')
command, outdir = get_command_and_outdir(self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file template_lib/v2/tests/configs/config.yaml
--tl_command {command}
--tl_outdir {outdir}
"""
args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True)
n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
cmd_str = f"""
python -m torch.distributed.launch --nproc_per_node={n_gpus} --master_port=8888
template_lib/v2/ddp/train.py
{get_append_cmd_str(args)}
"""
if debug:
cmd_str += f"""
--tl_debug
--tl_opts
"""
else:
cmd_str += f"""
--tl_opts {tl_opts}
"""
start_cmd_run(cmd_str)
# from template_lib.v2.config_cfgnode import update_parser_defaults_from_yaml, global_cfg
# from template_lib.modelarts import modelarts_utils
# update_parser_defaults_from_yaml(parser)
# modelarts_utils.setup_tl_outdir_obs(global_cfg)
# modelarts_utils.modelarts_sync_results_dir(global_cfg, join=True)
# modelarts_utils.prepare_dataset(global_cfg.get('modelarts_download', {}), global_cfg=global_cfg)
#
# modelarts_utils.prepare_dataset(global_cfg.get('modelarts_upload', {}), global_cfg=global_cfg, download=False)
# modelarts_utils.modelarts_sync_results_dir(global_cfg, join=True)
pass