def test_case_evaluate_FID_IS_ddp(self):
"""
Usage:
export PYTHONWARNINGS=ignore
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
export TIME_STR=1
export PYTHONPATH=./exp:./stylegan2-pytorch:./
python -c "from template_lib.v2.tests.test_GAN import TestingTFFIDISScore;\
TestingTFFIDISScore().test_case_evaluate_FID_IS_ddp()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0,1'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file none
--tl_command none
--tl_outdir {outdir}
"""
args = setup_outdir_and_yaml(argv_str)
nproc_per_node = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
cmd_str = f"""
python -m torch.distributed.launch --nproc_per_node={nproc_per_node} --master_port=8888
template_lib/v2/GAN/evaluation/tf_FID_IS_score.py
--run_func TFFIDISScore.test_case_evaluate_FID_IS
"""
cmd_str += get_append_cmd_str(args)
start_cmd_run(cmd_str)
pass
def test_ddp(self):
"""
Usage:
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
export TIME_STR=1
export PYTHONPATH=./exp:./stylegan2-pytorch:./
python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\
Testing_stylegan2().test_train_ffhq_128()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0,1'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config import get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, \
start_cmd_run
command, outdir = get_command_and_outdir(self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file exp/configs/styleganv2.yaml
--tl_command {command}
--tl_outdir {outdir}
"""
args = setup_outdir_and_yaml(argv_str)
nproc_per_node = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
cmd_str = f"""
python -m torch.distributed.launch --nproc_per_node={nproc_per_node} --master_port=8888
exp/scripts/train.py
{get_append_cmd_str(args)}
"""
start_cmd_run(cmd_str)
pass
def test_test_vae(self):
"""
Usage:
export CUDA_VISIBLE_DEVICES=1
export TIME_STR=1
export PYTHONPATH=./PyTorch_VAE_lib:./
python -c "from exp.tests.test_pytorch_vae import Testing_PyTorch_VAE;\
Testing_PyTorch_VAE().test_test_vae()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '1'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '1'
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file exp/configs/pytorch_vae.yaml
--tl_command {command}
--tl_outdir {outdir}
"""
args = setup_outdir_and_yaml(argv_str)
nproc_per_node = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
cmd_str = f"""
python exp/scripts/test.py -c PyTorch_VAE_lib/configs/vae.yaml
"""
cmd_str += get_append_cmd_str(args)
start_cmd_run(cmd_str)
pass
def test_nni(self, use_nni=False):
"""
Usage:
export CUDA_VISIBLE_DEVICES=1,2,3,4,5,6,7
export TIME_STR=1
export PYTHONPATH=./
python -c "from exp.tests.test_nni import Testing_nni;\
Testing_nni().test_mnist_pytorch(use_nni=False)"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '1'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config import get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, \
start_cmd_run
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
if use_nni: outdir = outdir + '_nni'
argv_str = f"""
--tl_config_file exp/configs/nni.yaml
--tl_command {command}
--tl_outdir {outdir}
"""
args = setup_outdir_and_yaml(argv_str)
n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
cmd_str = f"""
python
exp/mnist_pytorch/mnist.py
{get_append_cmd_str(args)}
"""
# use_nni = True
if use_nni:
# update nni config file
nni_config_file = "exp/mnist_pytorch/config.yml"
python_command = ' '.join(
[s.strip(' ') for s in cmd_str.split('\n')]) + ' --tl_nni'
update_nni_cfg_str = f"""
logDir: {os.path.abspath(args.tl_outdir)}/nni
trialConcurrency: {n_gpus}
trial:
command: {python_command}
codeDir: {os.path.abspath(os.path.curdir)}
"""
updated_config_file = update_nni_config_file(
nni_config_file=nni_config_file,
update_nni_cfg_str=update_nni_cfg_str)
cmd_str = f"""
bash
nnictl create --config {updated_config_file}
"""
start_cmd_run(cmd_str)
pass
def test_calculate_fid_stat_CIFAR100(self, debug=True):
"""
Usage:
export LD_LIBRARY_PATH=$HOME/.keras/envs/cuda-10.0/lib64:$HOME/.keras/envs/cudnn-10.0-linux-x64-v7.6.5.32/lib64
export LD_LIBRARY_PATH=/usr/local/cuda-10.0/lib64
export CUDA_VISIBLE_DEVICES=2
export TIME_STR=0
export PYTHONPATH=./
python -c "from template_lib.v2.tests.test_GAN import TestingTFFIDISScore;\
TestingTFFIDISScore().test_calculate_fid_stat_CIFAR100(debug=False)"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file template_lib/v2/GAN/configs/TFFIDISScore.yaml
--tl_command {command}
--tl_outdir {outdir}
"""
args = setup_outdir_and_yaml(argv_str)
os.environ['LD_LIBRARY_PATH'] = '/usr/local/cuda-10.0/lib64'
n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
cmd_str = f"""
python template_lib/v2/GAN/scripts/calculate_fid_stat_CIFAR.py
{get_append_cmd_str(args)}
--tl_opts OUTPUT_DIR {args.tl_outdir}/detectron2
"""
if debug:
cmd_str += f"""
"""
start_cmd_run(cmd_str)
pass
def test_detectron2(self):
"""
Usage:
export ANSI_COLORS_DISABLED=1
export CUDA_VISIBLE_DEVICES=2
export TIME_STR=1
export PYTHONPATH=.:./exp
python -c "from exp.tests.test_nas_cgan import TestingPrepareData;\
TestingPrepareData().test_calculate_fid_stat_CIFAR10()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0,1'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config import get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, \
start_cmd_run
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file exp/nas_cgan/config/prepare_data.yaml
--tl_command {command}
--tl_outdir {outdir}
"""
args = setup_outdir_and_yaml(argv_str)
num_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
cmd_str = f"""
python exp/scripts/train_net.py
{get_append_cmd_str(args)}
--num-gpus {num_gpus}
"""
start_cmd_run(cmd_str)
pass
def test_save_OmniGAN_ImageNet128_IS(self):
"""
Usage:
export TIME_STR=1
export PYTHONPATH=./exp:./BigGAN_PyTorch_1_lib:./
python -c "from exp.tests.test_BigGAN_v1 import Testing_Figures;\
Testing_Figures().test_save_early_collapse_on_cifar100()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import get_command_and_outdir, setup_outdir_and_yaml, \
get_append_cmd_str, start_cmd_run
from template_lib.v2.matplot import set_font
set_font()
command, outdir = get_command_and_outdir(self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file exp/tests/configs/Figures.yaml
--tl_command {command}
--tl_outdir {outdir}
"""
args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True)
import matplotlib.pyplot as plt
import numpy as np
import pickle
from template_lib.utils import colors_dict
fig, ax = plt.subplots()
# ax.set_xticks(range(0, 600, 100))
ax.tick_params(labelsize=cfg.fontsize.tick_fs)
ax.set_xlabel(cfg.xlabel, fontsize=cfg.fontsize.xylabel_fs)
ax.set_ylabel(cfg.ylabel, fontsize=cfg.fontsize.xylabel_fs)
properties = cfg.get('properties', {})
ax.set(**properties)
with open(cfg.data_pickle, 'rb') as f:
data_list = pickle.load(f)
load_data_dict = data_list[cfg.data_index]
for idx, (_, data_dict) in enumerate(cfg.lines.items()):
data = load_data_dict[data_dict.key]
if 'clip_x' in cfg:
data_xlim = cfg.clip_x[-1]
data = data[data[:, 0] <= data_xlim]
if cfg.get_min_value:
best_index = data[:, 1].argmin()
else:
best_index = data[:, 1].argmax()
best_x = int(data[:, 0][best_index])
best_y = data[:, 1][best_index]
if cfg.get('add_auxi_label', False):
data_dict.properties.label = f'x_{best_x}-y_{best_y:.3f}-' + getattr(data_dict.properties, 'label', '')
ax.plot(data[:, 0], data[:, 1], color=colors_dict[data_dict.color], **data_dict.properties)
pass
ax.legend(prop={'size': cfg.fontsize.legend_size}, ncol=1)
fig.show()
saved_file = os.path.join(args.tl_outdir, cfg.saved_file)
fig.savefig(saved_file, bbox_inches='tight', pad_inches=0.01)
print(f'Save to {saved_file}')
pass
def test_save_early_collapse_on_cifar100(self):
"""
Usage:
export LD_LIBRARY_PATH=~/anaconda3/envs/py36/lib/
export TIME_STR=1
export PYTHONPATH=./exp:./BigGAN_PyTorch_1_lib:./
python -c "from exp.tests.test_BigGAN import TestingCIFAR10_BigGAN_v1;\
TestingCIFAR10_BigGAN_v1().test_save_FID_cbn_index_012_figure()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import get_command_and_outdir, setup_outdir_and_yaml, \
get_append_cmd_str, start_cmd_run
command, outdir = get_command_and_outdir(self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file exp/configs/Figures.yaml
--tl_command {command}
--tl_outdir {outdir}
"""
args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True)
cfg_str = """
xlabel: "Epoch"
ylabel: "FID"
fontsize:
tick_fs: 14
xylabel_fs: 20
legend_size: 12
clip_x:
- 0
- 24500096
# properties:
# xlim:
# - 0
# - 336
# ylim:
# - 0
# - 30
get_min_value: true
add_auxi_label: false
lines:
biggan_pd:
result_dir: "results/CIFAR10/train_cifar100_\
20201002-19_37_05_825"
sub_path: "textdir/evaltf.ma0.FID_tf.log"
properties:
label: "projection discriminator (BigGAN)"
marker: "."
saved_file: "early_collapse_cifar100.pdf"
"""
import matplotlib.pyplot as plt
import numpy as np
from template_lib.utils import colors_dict
fig, ax = plt.subplots()
# ax.set_xticks(range(0, 600, 100))
ax.tick_params(labelsize=cfg.fontsize.tick_fs)
ax.set_xlabel(cfg.xlabel, fontsize=cfg.fontsize.xylabel_fs)
ax.set_ylabel(cfg.ylabel, fontsize=cfg.fontsize.xylabel_fs)
colors = list(colors_dict.values())
# colors = [plt.cm.cool(i / float(num_plot - 1)) for i in range(num_plot)]
properties = cfg.get('properties', {})
ax.set(**properties)
for idx, (_, data_dict) in enumerate(cfg.lines.items()):
log_file = os.path.join(data_dict.result_dir, data_dict.sub_path)
data = np.loadtxt(log_file, delimiter=':')
if 'clip_x' in cfg:
data_xlim = cfg.clip_x[-1]
data = data[data[:, 0] <= data_xlim]
if cfg.get_min_value:
best_index = data[:, 1].argmin()
else:
best_index = data[:, 1].argmax()
best_x = int(data[:, 0][best_index])
best_y = data[:, 1][best_index]
if cfg.add_auxi_label:
data_dict.properties.label = f'x_{best_x}-y_{best_y:.3f}-' + getattr(data_dict.properties, 'label', '')
ax.plot(data[:, 0], data[:, 1], color=colors[idx], **data_dict.properties)
pass
ax.legend(prop={'size': cfg.fontsize.legend_size})
fig.show()
saved_file = os.path.join(args.tl_outdir, cfg.saved_file)
fig.savefig(saved_file, bbox_inches='tight', pad_inches=0.01)
print(f'Save to {saved_file}')
pass
def test_ddp(self, debug=True):
"""
Usage:
python template_lib/modelarts/scripts/copy_tool.py \
-s s3://bucket-7001/ZhouPeng/pypi/torch1_7_0 -d /cache/pypi -t copytree
for filename in /cache/pypi/*.whl; do
pip install $filename
done
proj_root=moco-exp
python template_lib/modelarts/scripts/copy_tool.py \
-s s3://bucket-7001/ZhouPeng/codes/$proj_root -d /cache/$proj_root -t copytree -b /cache/$proj_root/code.zip
cd /cache/$proj_root
pip install -r requirements.txt
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
export TIME_STR=1
export PYTHONPATH=./exp:./stylegan2-pytorch:./
python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\
Testing_stylegan2().test_train_ffhq_128()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') + 1:]) if '--tl_opts' in sys.argv else ''
print(f'tl_opts:\n {tl_opts}')
command, outdir = get_command_and_outdir(self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file template_lib/v2/tests/configs/config.yaml
--tl_command {command}
--tl_outdir {outdir}
"""
args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True)
n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
cmd_str = f"""
python -m torch.distributed.launch --nproc_per_node={n_gpus} --master_port=8888
template_lib/v2/ddp/train.py
{get_append_cmd_str(args)}
"""
if debug:
cmd_str += f"""
--tl_debug
--tl_opts
"""
else:
cmd_str += f"""
--tl_opts {tl_opts}
"""
start_cmd_run(cmd_str)
# from template_lib.v2.config_cfgnode import update_parser_defaults_from_yaml, global_cfg
# from template_lib.modelarts import modelarts_utils
# update_parser_defaults_from_yaml(parser)
# modelarts_utils.setup_tl_outdir_obs(global_cfg)
# modelarts_utils.modelarts_sync_results_dir(global_cfg, join=True)
# modelarts_utils.prepare_dataset(global_cfg.get('modelarts_download', {}), global_cfg=global_cfg)
#
# modelarts_utils.prepare_dataset(global_cfg.get('modelarts_upload', {}), global_cfg=global_cfg, download=False)
# modelarts_utils.modelarts_sync_results_dir(global_cfg, join=True)
pass
def test_plot_lines_figure(self):
"""
Usage:
export LD_LIBRARY_PATH=~/anaconda3/envs/py36/lib/
export TIME_STR=1
export PYTHONPATH=./exp:./BigGAN_PyTorch_1_lib:./
python -c "from exp.tests.test_BigGAN import TestingCIFAR10_BigGAN_v1;\
TestingCIFAR10_BigGAN_v1().test_save_FID_cbn_index_012_figure()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config import get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, \
start_cmd_run
command, outdir = get_command_and_outdir(self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file exp/configs/BigGAN_v1.yaml
--tl_command {command}
--tl_outdir {outdir}
"""
args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True)
import matplotlib.pyplot as plt
import numpy as np
from template_lib.utils import colors_dict
fig, ax = plt.subplots()
ax.set_xticks(range(0, 600, 100))
ax.tick_params(labelsize=cfg.fontsize.tick_fs)
ax.set_xlabel(cfg.xlabel, fontsize=cfg.fontsize.xylabel_fs)
ax.set_ylabel(cfg.ylabel, fontsize=cfg.fontsize.xylabel_fs)
colors = list(colors_dict.values())
# colors = [plt.cm.cool(i / float(num_plot - 1)) for i in range(num_plot)]
ax.set(**cfg.properties)
for idx, (_, data_dict) in enumerate(cfg.lines.items()):
log_file = os.path.join(data_dict.result_dir, data_dict.sub_path)
data = np.loadtxt(log_file, delimiter=':')
if 'xlim' in cfg.properties:
data_xlim = cfg.properties.xlim[-1]
data = data[data[:, 0] <= data_xlim]
if cfg.get_min_value:
best_index = data[:, 1].argmin()
else:
best_index = data[:, 1].argmax()
best_x = int(data[:, 0][best_index])
best_y = data[:, 1][best_index]
if cfg.add_auxi_label:
data_dict.properties.label = f'x_{best_x}-y_{best_y:.3f}-' + getattr(data_dict.properties, 'label', '')
ax.plot(data[:, 0], data[:, 1], color=colors[idx], **data_dict.properties)
pass
ax.legend(prop={'size': cfg.fontsize.legend_size})
fig.show()
saved_file = os.path.join(args.tl_outdir, cfg.saved_file)
fig.savefig(saved_file, bbox_inches='tight', pad_inches=0.01)
print(f'Save to {saved_file}')
pass
def test__select_images(self, debug=True):
"""
Usage:
ssh -o ServerAliveInterval=30 -o ServerAliveCountMax=2 root@localhost -p 2232
export CUDA_VISIBLE_DEVICES=4
export TIME_STR=1
export PYTHONPATH=./:./ada_lib
python -c "from exp.tests.test_ada_ultra import Testing_train_StyleUltraGAN_ADA_Transfer_MixedFaces;\
Testing_train_StyleUltraGAN_ADA_Transfer_MixedFaces().test_projector_web(debug=False)" \
--tl_opts port 8530 start_web True
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') +
1:]) if '--tl_opts' in sys.argv else ''
print(f'tl_opts:\n {tl_opts}')
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file none
--tl_command none
--tl_outdir {outdir}
--tl_opts {tl_opts}
"""
args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True)
from pathlib import Path
import shutil
domain_dict = {}
domain_dict['lso_sm'] = """
x0_lso: results/train_StyleUltraGAN_ADA_Transfer_MixedFaces/double_inversion_web-20210604_093503_613/exp/0024/29218_lso.jpg,
x1_lso: results/train_StyleUltraGAN_ADA_Transfer_MixedFaces/double_inversion_web-20210604_093503_613/exp/0014/29438_lso.jpg,
"""
suffix_str = [''] * 2
for domain, image_list in domain_dict.items():
image_list = image_list.split(',')
for item, suffix in zip(image_list, suffix_str):
layer_name, image_path = item.strip().split(':')
layer_name = layer_name.strip().replace(' ', '_')
image_path = Path(image_path.strip())
select_path = Path(
f"{image_path.parent}/{image_path.stem}{suffix}.jpg")
saved_path = Path(
f"{outdir}/images/{domain}/{layer_name}_{select_path.name}"
)
os.makedirs(saved_path.parent, exist_ok=True)
shutil.copy(select_path, saved_path)
print(outdir)
pass
