def test_DDPDataLoaderCIFAR100(self, debug=True):
"""
Usage:
python template_lib/modelarts/scripts/copy_tool.py \
-s s3://bucket-7001/ZhouPeng/pypi/torch1_7_0 -d /cache/pypi -t copytree
for filename in /cache/pypi/*.whl; do
pip install $filename
done
proj_root=moco-exp
python template_lib/modelarts/scripts/copy_tool.py \
-s s3://bucket-7001/ZhouPeng/codes/$proj_root -d /cache/$proj_root -t copytree -b /cache/$proj_root/code.zip
cd /cache/$proj_root
pip install -r requirements.txt
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
export TIME_STR=1
export PYTHONPATH=./exp:./stylegan2-pytorch:./
python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\
Testing_stylegan2().test_train_ffhq_128()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0,1'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file template_lib/v2/GAN/datasets/configs/cifar100.yaml
--tl_command {command}
--tl_outdir {outdir}
"""
args = setup_outdir_and_yaml(argv_str)
tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') +
1:]) if '--tl_opts' in sys.argv else ''
print(f'tl_opts:\n {tl_opts}')
n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
cmd_str = f"""
python -m torch.distributed.launch --nproc_per_node={n_gpus} --master_port=8888
template_lib/v2/GAN/datasets/scripts/cifar10.py
{get_append_cmd_str(args)}
"""
if debug:
cmd_str += f"""
--tl_debug
--tl_opts
"""
else:
cmd_str += f"""
--tl_opts {tl_opts}
"""
start_cmd_run(cmd_str)
def test_run(self, *tmp):
"""
Usage:
export ANSI_COLORS_DISABLED=1
export CUDA_VISIBLE_DEVICES=0
export TIME_STR=1
export PYTHONPATH=./
python template_lib/modelarts/scripts/run.py \
--tl_config_file template_lib/modelarts/tests/configs/run.yaml \
--tl_command run \
--tl_outdir results/Run/run \
--number 1
# default image
/bucket-8280/ZhouPeng/codes/Omni-GAN-ImageNet/template_lib/modelarts/scripts/run.py
number = 3
tl_outdir = results/Run/run
tl_config_file = template_lib/modelarts/tests/configs/run.yaml
tl_opts = root_obs s3://bucket-7001/ZhouPeng/
tl_command = run
# self defined image
bash /home/work/run_train.sh python /home/work/user-job-dir/Omni-GAN-ImageNet/template_lib/modelarts/scripts/run.py --tl_outdir=results/Run/run --tl_config_file=/home/work/user-job-dir/Omni-GAN-ImageNet/template_lib/modelarts/tests/configs/run.yaml --tl_command=run --tl_opts=root_obs s3://bucket-7001/ZhouPeng/ --number=2
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
# if 'TIME_STR' not in os.environ:
# os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
os.environ['TIME_STR'] = '0'
if 'RESULTS_OBS' not in os.environ:
os.environ['RESULTS_OBS'] = 's3://bucket-xx/ZhouPeng/results'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file template_lib/modelarts/tests/configs/run.yaml
--tl_command {command}
--tl_outdir {outdir}
"""
args = setup_outdir_and_yaml(argv_str)
n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
cmd_str = f"""
python template_lib/modelarts/scripts/run.py
{get_append_cmd_str(args)}
--number 1
"""
start_cmd_run(cmd_str)
pass
def test_ddp_calculate_fid_stat_CIFAR100(self, debug=True):
"""
Usage:
export CUDA_VISIBLE_DEVICES=1,4
export master_port=8887
export TIME_STR=0
export PYTHONPATH=./
python -c "from template_lib.v2.GAN.evaluation.tests.test_pytorch_FID_IS_score import TestingTorchFIDISScore;\
TestingTorchFIDISScore().test_ddp_calculate_fid_stat_CIFAR100(debug=False)" \
--tl_opts build_dataloader.shuffle False save_fid_stat False
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file template_lib/v2/GAN/evaluation/configs/TorchFIDISScore.yaml
--tl_command {command}
--tl_outdir {outdir}
"""
args = setup_outdir_and_yaml(argv_str)
tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') +
1:]) if '--tl_opts' in sys.argv else ''
print(f'tl_opts:\n {tl_opts}')
nproc_per_node = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
master_port = os.environ.get('master_port', 8887)
cmd_str = f"""
python -m torch.distributed.launch --nproc_per_node={nproc_per_node} --master_port={master_port}
template_lib/v2/GAN/evaluation/scripts/torch_calculate_fid_stat_CIFAR.py
{get_append_cmd_str(args)}
"""
if debug:
cmd_str += f"""
--tl_debug
--tl_opts
"""
else:
cmd_str += f"""
--tl_opts {tl_opts}
"""
start_cmd_run(cmd_str)
pass
def test_mv_val_dataset(self):
"""
Usage:
export CUDA_VISIBLE_DEVICES=0
export TIME_STR=0
export PYTHONPATH=./
python -c "from template_lib.proj.imagenet.tests.test_imagenet import Testing_PrepareImageNet;\
Testing_PrepareImageNet().test_mv_val_dataset()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file none
--tl_command none
--tl_outdir {outdir}
"""
args = setup_outdir_and_yaml(argv_str)
n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
cmd_str = f"""
bash
ls -la datasets/ImageNet/ &&
cp template_lib/proj/imagenet/imagenet_move_val_images.sh datasets/ImageNet &&
cd datasets/ImageNet/val &&
ls -la ../imagenet_move_val_images.sh &&
bash ../imagenet_move_val_images.sh
"""
start_cmd_run(cmd_str)
# from template_lib.v2.config_cfgnode import update_parser_defaults_from_yaml, global_cfg
# from template_lib.modelarts import modelarts_utils
# update_parser_defaults_from_yaml(parser)
# modelarts_utils.setup_tl_outdir_obs(global_cfg)
# modelarts_utils.modelarts_sync_results_dir(global_cfg, join=True)
# modelarts_utils.prepare_dataset(global_cfg.get('modelarts_download', {}), global_cfg=global_cfg)
#
# modelarts_utils.prepare_dataset(global_cfg.get('modelarts_upload', {}), global_cfg=global_cfg, download=False)
# modelarts_utils.modelarts_sync_results_dir(global_cfg, join=True)
pass
def test_run(self, *tmp):
"""
Usage:
export ANSI_COLORS_DISABLED=1
export CUDA_VISIBLE_DEVICES=0
export TIME_STR=1
export PYTHONPATH=./
python template_lib/modelarts/scripts/run.py \
--tl_config_file template_lib/modelarts/tests/configs/run.yaml \
--tl_command run \
--tl_outdir results/Run/run \
--number 1
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
# if 'TIME_STR' not in os.environ:
# os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
os.environ['TIME_STR'] = '0'
if 'RESULTS_OBS' not in os.environ:
os.environ['RESULTS_OBS'] = 's3://bucket-xx/ZhouPeng/results'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file template_lib/modelarts/tests/configs/run.yaml
--tl_command {command}
--tl_outdir {outdir}
"""
args = setup_outdir_and_yaml(argv_str)
n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
cmd_str = f"""
python template_lib/modelarts/scripts/run.py
{get_append_cmd_str(args)}
--number 1
"""
start_cmd_run(cmd_str)
pass
def test_case_evaluate_FID_IS(self):
"""
Usage:
export PYTHONWARNINGS=ignore
export CUDA_VISIBLE_DEVICES=0,7
export master_port=8887
export TIME_STR=1
export PYTHONPATH=./
python -c "from template_lib.v2.tests.test_GAN import TestingTorchFIDISScore;\
TestingTorchFIDISScore().test_case_evaluate_FID_IS()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0,7'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file none
--tl_command none
--tl_outdir {outdir}
"""
args = setup_outdir_and_yaml(argv_str)
nproc_per_node = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
master_port = os.environ.get('master_port', 8887)
cmd_str = f"""
python -m torch.distributed.launch --nproc_per_node={nproc_per_node} --master_port={master_port}
template_lib/v2/GAN/evaluation/pytorch_FID_IS_score.py
--run_func PyTorchFIDISScore.test_case_evaluate_FID_IS
"""
cmd_str += get_append_cmd_str(args)
start_cmd_run(cmd_str)
pass
def test_hook_for_grad_cam(self, debug=True):
"""
Usage:
python template_lib/modelarts/scripts/copy_tool.py \
-s s3://bucket-7001/ZhouPeng/pypi/torch1_7_0 -d /cache/pypi -t copytree
for filename in /cache/pypi/*.whl; do
pip install $filename
done
proj_root=moco-exp
python template_lib/modelarts/scripts/copy_tool.py \
-s s3://bucket-7001/ZhouPeng/codes/$proj_root -d /cache/$proj_root -t copytree -b /cache/$proj_root/code.zip
cd /cache/$proj_root
pip install -r requirements.txt
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
export TIME_STR=1
export PYTHONPATH=./exp:./stylegan2-pytorch:./
python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\
Testing_stylegan2().test_train_ffhq_128()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') +
1:]) if '--tl_opts' in sys.argv else ''
print(f'tl_opts:\n {tl_opts}')
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file none
--tl_command none
--tl_outdir {outdir}
--tl_opts {tl_opts}
"""
args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True)
n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
cmd_str = f"""
python
template_lib/proj/pytorch/examples/hook_for_grad_cam.py
{get_append_cmd_str(args)}
"""
if debug:
cmd_str += f"""
--tl_debug
--tl_opts
"""
else:
cmd_str += f"""
--tl_opts {tl_opts}
"""
start_cmd_run(cmd_str)
# from template_lib.v2.config_cfgnode import update_parser_defaults_from_yaml, global_cfg
# from template_lib.modelarts import modelarts_utils
# update_parser_defaults_from_yaml(parser)
# modelarts_utils.setup_tl_outdir_obs(global_cfg)
# modelarts_utils.modelarts_sync_results_dir(global_cfg, join=True)
# modelarts_utils.prepare_dataset(global_cfg.get('modelarts_download', {}), global_cfg=global_cfg)
#
# modelarts_utils.prepare_dataset(global_cfg.get('modelarts_upload', {}), global_cfg=global_cfg, download=False)
# modelarts_utils.modelarts_sync_results_dir(global_cfg, join=True)
pass
def test_show_video(self, debug=True):
"""
Usage:
export CUDA_VISIBLE_DEVICES=7
export TIME_STR=1
export PYTHONPATH=./
python -c "from template_lib.proj.streamlit.tests.test_streamlit import Testing_Streamlit;\
Testing_Streamlit().test_show_video(debug=False)" \
--tl_opts port 8530 start_web True show_video.num_video 7
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') +
1:]) if '--tl_opts' in sys.argv else ''
print(f'tl_opts:\n {tl_opts}')
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file template_lib/proj/streamlit/scripts/configs/Streamlit.yaml
--tl_command {command}
--tl_outdir {outdir}
--tl_opts {tl_opts}
"""
args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True)
n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
script = "template_lib/proj/streamlit/scripts/run_web.py"
if debug:
cmd_str = f"""
python
{script}
{get_append_cmd_str(args)}
--tl_debug
--tl_opts
"""
else:
if cfg.start_web:
cmd_str_prefix = f"""
{os.path.dirname(sys.executable)}/streamlit run --server.port {cfg.port}
{script}
--
"""
else:
cmd_str_prefix = f"python {script}"
cmd_str = f"""
{cmd_str_prefix}
{get_append_cmd_str(args)}
--tl_opts {tl_opts}
"""
start_cmd_run(cmd_str)
pass