def test_Image(self, debug=True):
"""
Usage:
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
export TIME_STR=1
export PYTHONPATH=./exp:./stylegan2-pytorch:./
python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\
Testing_stylegan2().test_train_ffhq_128()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') +
1:]) if '--tl_opts' in sys.argv else ''
print(f'tl_opts:\n {tl_opts}')
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file none
--tl_command none
--tl_outdir {outdir}
--tl_opts {tl_opts}
"""
args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True)
n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
import mmcv
import numpy as np
import matplotlib.pyplot as plt
img_path = "template_lib/datasets/images/zebra_GT_target_origin.png"
img = mmcv.imread(img_path)
img_gray = mmcv.imread(img_path, flag='grayscale')
img_ = mmcv.imread(img) # nothing will happen, img_ = img
mmcv.imwrite(img, f'{args.tl_outdir}/out.png')
# mmcv.imshow(img)
fig, axes = plt.subplots(2, 1)
img = mmcv.bgr2rgb(img)
axes[0].imshow(img)
# plt.imshow(img)
# plt.show()
# ret = mmcv.imresize(img, (1000, 600), return_scale=True)
# ret = mmcv.imrescale(img, (1000, 800))
bboxes = np.array([10, 10, 100, 120])
patch = mmcv.imcrop(img, bboxes)
axes[1].imshow(patch)
fig.show()
pass
def test_make_zip(self):
"""
Usage:
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
export TIME_STR=1
export PYTHONPATH=./exp:./stylegan2-pytorch:./
python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\
Testing_stylegan2().test_train_ffhq_128()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file none
--tl_command none
--tl_outdir {outdir}
"""
args = setup_outdir_and_yaml(argv_str)
from template_lib.utils.utils_func import make_zip, unzip_file
make_zip(source_dir='./template_lib', output_filename='code.zip')
unzip_file(zip_file='code.zip', dst_dir='/tmp/code')
pass
def test_extract_ImageNet100_CMC(self):
"""
Usage:
proj_root=moco-exp
python template_lib/modelarts/scripts/copy_tool.py \
-s s3://bucket-7001/ZhouPeng/codes/$proj_root -d /cache/$proj_root -t copytree
cd /cache/$proj_root
export CUDA_VISIBLE_DEVICES=0
export TIME_STR=0
export PYTHONPATH=./
python -c "from template_lib.proj.imagenet.tests.test_imagenet import Testing_PrepareImageNet;\
Testing_PrepareImageNet().test_extract_ImageNet100_CMC()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
from template_lib.v2.config_cfgnode import update_parser_defaults_from_yaml, global_cfg
from template_lib.modelarts import modelarts_utils
from distutils.dir_util import copy_tree
command, outdir = get_command_and_outdir(self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file template_lib/proj/imagenet/tests/configs/PrepareImageNet.yaml
--tl_command {command}
--tl_outdir {outdir}
"""
args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True)
modelarts_utils.setup_tl_outdir_obs(global_cfg)
modelarts_utils.modelarts_sync_results_dir(global_cfg, join=True)
modelarts_utils.prepare_dataset(global_cfg.get('modelarts_download', {}), global_cfg=global_cfg)
train_dir = f'{cfg.data_dir}/train'
val_dir = f'{cfg.data_dir}/val'
save_train_dir = f'{cfg.saved_dir}/train'
save_val_dir = f'{cfg.saved_dir}/val'
os.makedirs(save_train_dir, exist_ok=True)
os.makedirs(save_val_dir, exist_ok=True)
with open(cfg.class_list_file, 'r') as f:
class_list = f.readlines()
for class_subdir in tqdm.tqdm(class_list):
class_subdir, _ = class_subdir.strip().split()
train_class_dir = f'{train_dir}/{class_subdir}'
save_train_class_dir = f'{save_train_dir}/{class_subdir}'
copy_tree(train_class_dir, save_train_class_dir)
val_class_dir = f'{val_dir}/{class_subdir}'
save_val_class_dir = f'{save_val_dir}/{class_subdir}'
copy_tree(val_class_dir, save_val_class_dir)
modelarts_utils.prepare_dataset(global_cfg.get('modelarts_upload', {}), global_cfg=global_cfg, download=False)
modelarts_utils.modelarts_sync_results_dir(global_cfg, join=True)
pass
def test_draw_text_on_rectangle(self, debug=True):
"""
Usage:
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
export TIME_STR=1
export PYTHONPATH=./exp:./stylegan2-pytorch:./
python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\
Testing_stylegan2().test_train_ffhq_128()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') +
1:]) if '--tl_opts' in sys.argv else ''
print(f'tl_opts:\n {tl_opts}')
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file none
--tl_command none
--tl_outdir {outdir}
--tl_opts {tl_opts}
"""
args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True)
n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
img_path = "template_lib/datasets/images/zebra_GT_target_origin.png"
source_img = Image.open(img_path)
font = ImageFont.truetype(_font)
text = "very loooooooooooooooooong text"
# get text size
text_size = font.getsize(text)
# set button size + 10px margins
button_size = (text_size[0] + 20, text_size[1] + 20)
# create image with correct size and black background
button_img = Image.new('RGB', button_size, "black")
# put text on button with 10px margins
button_draw = ImageDraw.Draw(button_img)
button_draw.text((10, 10), text, font=font)
# put button on source image in position (0, 0)
source_img.paste(button_img, (0, 0))
# save in new file
plt.imshow(source_img)
plt.show()
# source_img.save("output.jpg", "JPEG")
pass
def test_get_subdir2name_dict(self):
"""
Usage:
export CUDA_VISIBLE_DEVICES=0
export TIME_STR=0
export PYTHONPATH=./
python -c "from template_lib.proj.imagenet.tests.test_imagenet import Testing_PrepareImageNet;\
Testing_PrepareImageNet().test_mv_val_dataset()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
command, outdir = get_command_and_outdir(self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file none
--tl_command none
--tl_outdir {outdir}
"""
args = setup_outdir_and_yaml(argv_str)
from template_lib.proj.imagenet.utils import get_subdir2name_dict, subdir2name_dict
subdir2name_d = get_subdir2name_dict()
pass
def test_attr_s(self):
"""
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file none
--tl_command none
--tl_outdir {outdir}
"""
args = setup_outdir_and_yaml(argv_str)
import attr
@attr.s
class SomeClass(object):
a_number = attr.ib(default=42)
list_of_numbers = attr.ib(factory=list)
def hard_math(self, another_number):
return self.a_number + sum(
self.list_of_numbers) * another_number
sc = SomeClass(1, [1, 2, 3])
pass
def test_merge_original_image_and_patches_horizon(self, debug=True):
"""
Usage:
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
export TIME_STR=1
export PYTHONPATH=./exp:./stylegan2-pytorch:./
python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\
Testing_stylegan2().test_train_ffhq_128()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') +
1:]) if '--tl_opts' in sys.argv else ''
print(f'tl_opts:\n {tl_opts}')
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file none
--tl_command none
--tl_outdir {outdir}
--tl_opts {tl_opts}
"""
args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True)
n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
img_path = "template_lib/datasets/images/zebra_GT_target_origin.png"
left = [80, 300]
upper = [120, 140]
w = 50
h = 20
pad = 2
image = Image.open(img_path)
out_image = merge_original_image_and_patches_horizon(image=image,
lefts=left,
uppers=upper,
w=w,
h=h,
pad=pad,
width=3,
patch_width=1)
fig, axes = plt.subplots(1, 1)
axes.imshow(out_image)
# axes[1].imshow(patch)
fig.show()
pass
def test_DDPDataLoaderCIFAR100(self, debug=True):
"""
Usage:
python template_lib/modelarts/scripts/copy_tool.py \
-s s3://bucket-7001/ZhouPeng/pypi/torch1_7_0 -d /cache/pypi -t copytree
for filename in /cache/pypi/*.whl; do
pip install $filename
done
proj_root=moco-exp
python template_lib/modelarts/scripts/copy_tool.py \
-s s3://bucket-7001/ZhouPeng/codes/$proj_root -d /cache/$proj_root -t copytree -b /cache/$proj_root/code.zip
cd /cache/$proj_root
pip install -r requirements.txt
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
export TIME_STR=1
export PYTHONPATH=./exp:./stylegan2-pytorch:./
python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\
Testing_stylegan2().test_train_ffhq_128()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0,1'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file template_lib/v2/GAN/datasets/configs/cifar100.yaml
--tl_command {command}
--tl_outdir {outdir}
"""
args = setup_outdir_and_yaml(argv_str)
tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') +
1:]) if '--tl_opts' in sys.argv else ''
print(f'tl_opts:\n {tl_opts}')
n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
cmd_str = f"""
python -m torch.distributed.launch --nproc_per_node={n_gpus} --master_port=8888
template_lib/v2/GAN/datasets/scripts/cifar10.py
{get_append_cmd_str(args)}
"""
if debug:
cmd_str += f"""
--tl_debug
--tl_opts
"""
else:
cmd_str += f"""
--tl_opts {tl_opts}
"""
start_cmd_run(cmd_str)
def test_plot_figures_template(self):
"""
python -c "from exp.tests.test_styleganv2 import Testing_stylegan2_style_position;\
Testing_stylegan2_style_position().test_plot_FID_cifar10_style_position()"
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '3'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file none
--tl_command none
--tl_outdir {outdir}
"""
args = setup_outdir_and_yaml(argv_str)
outdir = args.tl_outdir
from template_lib.utils.plot_results import PlotResults
import collections
import pickle
outfigure = os.path.join(outdir, 'IS.jpg')
default_dicts = collections.OrderedDict()
show_max = []
IS_GAN_cGANs_CIFAR100 = collections.defaultdict(dict)
title = 'IS_GAN_cGANs_CIFAR100'
log_file = 'textdir/evaltf.ma1.IS_mean_tf.log'
dd = eval(title)
dd['results/stylegan2/train_cifar100-20201030_2234_241'] = \
{'20201030_2234_241-stylegan': "textdir/eval.ma1.IS_mean_tf.log", }
dd['properties'] = {
'title': title,
}
default_dicts[title] = dd
show_max.append(True)
plotobs = PlotResults()
label2datas_list = plotobs.plot_defaultdicts(
outfigure=outfigure,
default_dicts=default_dicts,
show_max=show_max,
figsize_wh=(16, 7.2))
print(f'Save to {outfigure}.')
saved_data = '__'.join(outdir.split('/')[-2:])
saved_data = f"{outdir}/{saved_data}.pkl"
with open(saved_data, 'wb') as f:
pickle.dump(label2datas_list, f)
print(f"Save data to {saved_data}")
pass
def test_save_cifar10_images(self, debug=True):
"""
Usage:
python template_lib/modelarts/scripts/copy_tool.py \
-s s3://bucket-7001/ZhouPeng/pypi/torch1_7_0 -d /cache/pypi -t copytree
for filename in /cache/pypi/*.whl; do
pip install $filename
done
proj_root=deep-generative-prior-exp
python template_lib/modelarts/scripts/copy_tool.py \
-s s3://bucket-7001/ZhouPeng/codes/$proj_root -d /cache/$proj_root -t copytree -b /cache/$proj_root/code.zip
cd /cache/$proj_root
pip install -r requirements.txt
export CUDA_VISIBLE_DEVICES=5
export TIME_STR=1
export PYTHONPATH=./:DGP_lib:BigGAN_Pytorch_lib
python -c "from template_lib.proj.cifar.tests.test_cifar_dataset import Testing_CIFAR10;\
Testing_CIFAR10().test_save_cifar10_images(debug=False)"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '1'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') +
1:]) if '--tl_opts' in sys.argv else ''
print(f'tl_opts:\n {tl_opts}')
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file none
--tl_command none
--tl_outdir {outdir}
--tl_opts {tl_opts}
"""
args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True)
import tqdm
import torchvision
saved_dir = 'datasets/cifar10/cifar10_images'
cifar10 = torchvision.datasets.CIFAR10(root='datasets/cifar10',
train=True)
for idx in tqdm.tqdm(range(len(cifar10))):
img_pil, label_id = cifar10[idx]
subdir = f"{saved_dir}/{cifar10.classes[label_id]}_{label_id:02d}"
os.makedirs(subdir, exist_ok=True)
img_pil.save(f"{subdir}/{idx:05d}.png")
pass
def test_reduce(self, debug=True):
"""
Usage:
python template_lib/modelarts/scripts/copy_tool.py \
-s s3://bucket-7001/ZhouPeng/pypi/torch1_7_0 -d /cache/pypi -t copytree
for filename in /cache/pypi/*.whl; do
pip install $filename
done
proj_root=moco-exp
python template_lib/modelarts/scripts/copy_tool.py \
-s s3://bucket-7001/ZhouPeng/codes/$proj_root -d /cache/$proj_root -t copytree -b /cache/$proj_root/code.zip
cd /cache/$proj_root
pip install -r requirements.txt
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
export TIME_STR=1
export PYTHONPATH=./exp:./stylegan2-pytorch:./
python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\
Testing_stylegan2().test_train_ffhq_128()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') +
1:]) if '--tl_opts' in sys.argv else ''
print(f'tl_opts:\n {tl_opts}')
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file none
--tl_command none
--tl_outdir {outdir}
--tl_opts {tl_opts}
"""
args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True)
import matplotlib.pyplot as plt
import numpy as np
from PIL import Image
from einops import rearrange, reduce
img_path = "template_lib/datasets/images/zebra_GT_target_origin.png"
img_pil = Image.open(img_path)
img_np = np.array(img_pil)
img_np = reduce(img_np, '(h h2) (w w2) c -> h w c', 'min', h2=2, w2=2)
plt.imshow(img_np)
plt.show()
pass
def test_extract_ImageNet_1000x50(self):
"""
Usage:
proj_root=moco-exp
python template_lib/modelarts/scripts/copy_tool.py \
-s s3://bucket-7001/ZhouPeng/codes/$proj_root -d /cache/$proj_root -t copytree
cd /cache/$proj_root
export CUDA_VISIBLE_DEVICES=0
export TIME_STR=0
export PYTHONPATH=./
python -c "from template_lib.proj.imagenet.tests.test_imagenet import Testing_PrepareImageNet;\
Testing_PrepareImageNet().test_extract_ImageNet_1000x50()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
from template_lib.v2.config_cfgnode import update_parser_defaults_from_yaml, global_cfg
from template_lib.modelarts import modelarts_utils
command, outdir = get_command_and_outdir(self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file template_lib/proj/imagenet/tests/configs/PrepareImageNet.yaml
--tl_command {command}
--tl_outdir {outdir}
"""
args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True)
global_cfg.merge_from_dict(cfg)
global_cfg.merge_from_dict(vars(args))
modelarts_utils.setup_tl_outdir_obs(global_cfg)
modelarts_utils.modelarts_sync_results_dir(global_cfg, join=True)
modelarts_utils.prepare_dataset(global_cfg.get('modelarts_download', {}), global_cfg=global_cfg)
train_dir = f'{cfg.data_dir}/train'
counter_cls = 0
for rootdir, subdir, files in os.walk(train_dir):
if len(subdir) == 0:
counter_cls += 1
extracted_files = sorted(files)[:cfg.num_per_class]
for file in tqdm.tqdm(extracted_files, desc=f'class: {counter_cls}'):
img_path = os.path.join(rootdir, file)
img_rel_path = os.path.relpath(img_path, cfg.data_dir)
saved_img_path = f'{cfg.saved_dir}/{os.path.dirname(img_rel_path)}'
os.makedirs(saved_img_path, exist_ok=True)
shutil.copy(img_path, saved_img_path)
pass
modelarts_utils.prepare_dataset(global_cfg.get('modelarts_upload', {}), global_cfg=global_cfg, download=False)
modelarts_utils.modelarts_sync_results_dir(global_cfg, join=True)
pass
def test_zoom_in_video_writer(self, debug=True):
"""
Usage:
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
export TIME_STR=1
export PYTHONPATH=./exp:./stylegan2-pytorch:./
python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\
Testing_stylegan2().test_train_ffhq_128()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') +
1:]) if '--tl_opts' in sys.argv else ''
print(f'tl_opts:\n {tl_opts}')
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file none
--tl_command none
--tl_outdir {outdir}
--tl_opts {tl_opts}
"""
args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True)
n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
import tqdm
from template_lib.proj.pil.pil_utils import get_size
img_path = "template_lib/datasets/images/zebra_GT_target_origin.png"
outvid = f"{args.tl_outdir}/test.mp4"
out_size = 2048
image = Image.open(img_path)
w, h = image.size
max_scale = out_size / min(w, h)
out_w, out_h = get_size(w=w, h=h, dst_size=out_size)
out_video = VideoWriter(outfile=outvid, w=out_w, h=out_h, fps=10)
for scale in tqdm.tqdm(np.arange(1, max_scale, 0.05)):
out_img = Image.new(mode='RGB', size=(out_w, out_h), color='black')
cur_w, cur_h = int(w * scale), int(h * scale)
cur_image = image.resize((cur_w, cur_h), resample=Image.NEAREST)
xy = (out_w - cur_w) // 2, (out_h - cur_h) // 2
out_img.paste(cur_image, xy)
out_video.write(out_img, is_tensor=False, rgb=True)
out_video.release()
pass
def test_plot_figures_template(self):
"""
python -c "from exp.tests.test_styleganv2 import Testing_stylegan2_style_position;\
Testing_stylegan2_style_position().test_plot_FID_cifar10_style_position()"
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '3'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file none
--tl_command none
--tl_outdir {outdir}
"""
args = setup_outdir_and_yaml(argv_str)
outdir = args.tl_outdir
from template_lib.utils.plot_results import PlotResults
import collections
import pickle
outfigure = os.path.join(outdir, 'FID.jpg')
default_dicts = []
show_max = []
top1_randomlabel = collections.defaultdict(dict)
title = 'top1_randomlabel'
log_file = 'textdir/evaltf.ma0.FID_tf.log'
dd = eval(title)
dd['results/CIFAR10/train_R56_cifar10_20200923-23_55_05_422/'] = \
{'23_55_05_422-resnet-R56': log_file, }
dd['properties'] = {'title': title, 'ylim': [0, 80]}
default_dicts.append(dd)
show_max.append(False)
plotobs = PlotResults()
label2datas_list = plotobs.plot_defaultdicts(
outfigure=outfigure,
default_dicts=default_dicts,
show_max=show_max,
figsize_wh=(16, 7.2))
print(f'Save to {outfigure}.')
saved_data = '__'.join(outdir.split('/')[-2:])
saved_data = f"{outdir}/{saved_data}.pkl"
with open(saved_data, 'wb') as f:
pickle.dump(label2datas_list, f)
print(f"Save data to {saved_data}")
pass
def test_run(self, *tmp):
"""
Usage:
export ANSI_COLORS_DISABLED=1
export CUDA_VISIBLE_DEVICES=0
export TIME_STR=1
export PYTHONPATH=./
python template_lib/modelarts/scripts/run.py \
--tl_config_file template_lib/modelarts/tests/configs/run.yaml \
--tl_command run \
--tl_outdir results/Run/run \
--number 1
# default image
/bucket-8280/ZhouPeng/codes/Omni-GAN-ImageNet/template_lib/modelarts/scripts/run.py
number = 3
tl_outdir = results/Run/run
tl_config_file = template_lib/modelarts/tests/configs/run.yaml
tl_opts = root_obs s3://bucket-7001/ZhouPeng/
tl_command = run
# self defined image
bash /home/work/run_train.sh python /home/work/user-job-dir/Omni-GAN-ImageNet/template_lib/modelarts/scripts/run.py --tl_outdir=results/Run/run --tl_config_file=/home/work/user-job-dir/Omni-GAN-ImageNet/template_lib/modelarts/tests/configs/run.yaml --tl_command=run --tl_opts=root_obs s3://bucket-7001/ZhouPeng/ --number=2
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
# if 'TIME_STR' not in os.environ:
# os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
os.environ['TIME_STR'] = '0'
if 'RESULTS_OBS' not in os.environ:
os.environ['RESULTS_OBS'] = 's3://bucket-xx/ZhouPeng/results'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file template_lib/modelarts/tests/configs/run.yaml
--tl_command {command}
--tl_outdir {outdir}
"""
args = setup_outdir_and_yaml(argv_str)
n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
cmd_str = f"""
python template_lib/modelarts/scripts/run.py
{get_append_cmd_str(args)}
--number 1
"""
start_cmd_run(cmd_str)
pass
def test_ddp_calculate_fid_stat_CIFAR100(self, debug=True):
"""
Usage:
export CUDA_VISIBLE_DEVICES=1,4
export master_port=8887
export TIME_STR=0
export PYTHONPATH=./
python -c "from template_lib.v2.GAN.evaluation.tests.test_pytorch_FID_IS_score import TestingTorchFIDISScore;\
TestingTorchFIDISScore().test_ddp_calculate_fid_stat_CIFAR100(debug=False)" \
--tl_opts build_dataloader.shuffle False save_fid_stat False
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file template_lib/v2/GAN/evaluation/configs/TorchFIDISScore.yaml
--tl_command {command}
--tl_outdir {outdir}
"""
args = setup_outdir_and_yaml(argv_str)
tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') +
1:]) if '--tl_opts' in sys.argv else ''
print(f'tl_opts:\n {tl_opts}')
nproc_per_node = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
master_port = os.environ.get('master_port', 8887)
cmd_str = f"""
python -m torch.distributed.launch --nproc_per_node={nproc_per_node} --master_port={master_port}
template_lib/v2/GAN/evaluation/scripts/torch_calculate_fid_stat_CIFAR.py
{get_append_cmd_str(args)}
"""
if debug:
cmd_str += f"""
--tl_debug
--tl_opts
"""
else:
cmd_str += f"""
--tl_opts {tl_opts}
"""
start_cmd_run(cmd_str)
pass
def test_SpacialTransformNet(self, debug=True):
"""
Usage:
python template_lib/modelarts/scripts/copy_tool.py \
-s s3://bucket-7001/ZhouPeng/pypi/torch1_7_0 -d /cache/pypi -t copytree
for filename in /cache/pypi/*.whl; do
pip install $filename
done
proj_root=moco-exp
python template_lib/modelarts/scripts/copy_tool.py \
-s s3://bucket-7001/ZhouPeng/codes/$proj_root -d /cache/$proj_root -t copytree -b /cache/$proj_root/code.zip
cd /cache/$proj_root
pip install -r requirements.txt
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
export TIME_STR=1
export PYTHONPATH=./exp:./stylegan2-pytorch:./
python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\
Testing_stylegan2().test_train_ffhq_128()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') +
1:]) if '--tl_opts' in sys.argv else ''
print(f'tl_opts:\n {tl_opts}')
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file none
--tl_command none
--tl_outdir {outdir}
--tl_opts {tl_opts}
"""
args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True)
from template_lib.proj.pytorch.pytorch_hook import VerboseModel
net = SpacialTransformNet()
net = VerboseModel(model=net)
assert 0
x = torch.rand(1, 32, 16, 16)
out = net(x)
pass
def test_mv_val_dataset(self):
"""
Usage:
export CUDA_VISIBLE_DEVICES=0
export TIME_STR=0
export PYTHONPATH=./
python -c "from template_lib.proj.imagenet.tests.test_imagenet import Testing_PrepareImageNet;\
Testing_PrepareImageNet().test_mv_val_dataset()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file none
--tl_command none
--tl_outdir {outdir}
"""
args = setup_outdir_and_yaml(argv_str)
n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
cmd_str = f"""
bash
ls -la datasets/ImageNet/ &&
cp template_lib/proj/imagenet/imagenet_move_val_images.sh datasets/ImageNet &&
cd datasets/ImageNet/val &&
ls -la ../imagenet_move_val_images.sh &&
bash ../imagenet_move_val_images.sh
"""
start_cmd_run(cmd_str)
# from template_lib.v2.config_cfgnode import update_parser_defaults_from_yaml, global_cfg
# from template_lib.modelarts import modelarts_utils
# update_parser_defaults_from_yaml(parser)
# modelarts_utils.setup_tl_outdir_obs(global_cfg)
# modelarts_utils.modelarts_sync_results_dir(global_cfg, join=True)
# modelarts_utils.prepare_dataset(global_cfg.get('modelarts_download', {}), global_cfg=global_cfg)
#
# modelarts_utils.prepare_dataset(global_cfg.get('modelarts_upload', {}), global_cfg=global_cfg, download=False)
# modelarts_utils.modelarts_sync_results_dir(global_cfg, join=True)
pass
def test_merge_image_pil(self, debug=True):
"""
Usage:
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
export TIME_STR=1
export PYTHONPATH=./exp:./stylegan2-pytorch:./
python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\
Testing_stylegan2().test_train_ffhq_128()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') +
1:]) if '--tl_opts' in sys.argv else ''
print(f'tl_opts:\n {tl_opts}')
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file none
--tl_command none
--tl_outdir {outdir}
--tl_opts {tl_opts}
"""
args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True)
img_path = "template_lib/datasets/images/zebra_GT_target_origin.png"
pad = 2
pad_color = 'white'
img = Image.open(img_path)
image_list = [
img,
] * 5
merged_img = merge_image_pil(image_list=image_list,
nrow=3,
pad=pad,
pad_color=pad_color)
plt.imshow(merged_img)
plt.show()
pass
def test_extract_imagenet_val_for_selection(self):
"""
Usage:
proj_root=moco-exp
python template_lib/modelarts/scripts/copy_tool.py \
-s s3://bucket-7001/ZhouPeng/codes/$proj_root -d /cache/$proj_root -t copytree
cd /cache/$proj_root
export CUDA_VISIBLE_DEVICES=0
export TIME_STR=0
export PYTHONPATH=./
python -c "from template_lib.proj.imagenet.tests.test_imagenet import Testing_PrepareImageNet;\
Testing_PrepareImageNet().test_extract_imagenet_val_for_selection()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
from template_lib.v2.config_cfgnode import update_parser_defaults_from_yaml, global_cfg
from template_lib.modelarts import modelarts_utils
from distutils.dir_util import copy_tree
command, outdir = get_command_and_outdir(self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file none
--tl_command none
--tl_outdir {outdir}
"""
args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True)
import shutil
import tqdm
from template_lib.utils import get_filelist_recursive
val_dir = "datasets/ImageNet/val"
saved_dir = "datasets/ImageNet/imagenet_val_selection"
os.makedirs(saved_dir, exist_ok=True)
image_list = get_filelist_recursive(val_dir, ext='*.JPEG')
for image_path in tqdm.tqdm(image_list):
saved_file = f"{saved_dir}/{image_path.parent.name}_{image_path.name}"
shutil.copyfile(image_path, saved_file)
pass
def test_ImageNet100_CMC_class_file_append_classname(self):
"""
Usage:
proj_root=moco-exp
python template_lib/modelarts/scripts/copy_tool.py \
-s s3://bucket-7001/ZhouPeng/codes/$proj_root -d /cache/$proj_root -t copytree
cd /cache/$proj_root
export CUDA_VISIBLE_DEVICES=0
export TIME_STR=0
export PYTHONPATH=./
python -c "from template_lib.proj.imagenet.tests.test_imagenet import Testing_PrepareImageNet;\
Testing_PrepareImageNet().test_extract_ImageNet_1000x50()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
from template_lib.proj.imagenet.utils import subdir2name_dict
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file template_lib/proj/imagenet/tests/configs/PrepareImageNet.yaml
--tl_command {command}
--tl_outdir {outdir}
"""
args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True)
class_list_file = cfg.class_list_file
saved_class_list_file = cfg.saved_class_list_file
with open(class_list_file, 'r') as f:
class_list = f.readlines()
saved_f = open(saved_class_list_file, 'w')
for class_subdir in tqdm.tqdm(class_list):
class_subdir = class_subdir.strip()
class_name = subdir2name_dict[class_subdir]
saved_f.write(f"{class_subdir} {class_name}\n")
saved_f.close()
pass
def test_imshow(self, debug=True):
"""
Usage:
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
export TIME_STR=1
export PYTHONPATH=./exp:./stylegan2-pytorch:./
python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\
Testing_stylegan2().test_train_ffhq_128()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') +
1:]) if '--tl_opts' in sys.argv else ''
print(f'tl_opts:\n {tl_opts}')
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file none
--tl_command none
--tl_outdir {outdir}
--tl_opts {tl_opts}
"""
args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True)
n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
import mmcv
import numpy as np
img_path = "template_lib/datasets/images/zebra_GT_target_origin.png"
mmcv.imshow(img_path)
# show image with bounding boxes
img = np.random.rand(100, 100, 3)
bboxes = np.array([[0, 0, 50, 50], [20, 20, 60, 60]])
mmcv.imshow_bboxes(img, bboxes)
pass
def test_draw_rectangle(self, debug=True):
"""
Usage:
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
export TIME_STR=1
export PYTHONPATH=./exp:./stylegan2-pytorch:./
python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\
Testing_stylegan2().test_train_ffhq_128()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') +
1:]) if '--tl_opts' in sys.argv else ''
print(f'tl_opts:\n {tl_opts}')
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file none
--tl_command none
--tl_outdir {outdir}
--tl_opts {tl_opts}
"""
args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True)
n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
img_path = "template_lib/datasets/images/zebra_GT_target_origin.png"
im = Image.open(img_path)
draw = ImageDraw.Draw(im)
draw.rectangle(((10, 10), (50, 50)), outline='#ff0000', width=2)
plt.imshow(im)
plt.show()
pass
def test_MaxToKeep(self):
"""
Usage:
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
export TIME_STR=1
export PYTHONPATH=./exp:./stylegan2-pytorch:./
python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\
Testing_stylegan2().test_train_ffhq_128()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file none
--tl_command none
--tl_outdir {outdir}
"""
args = setup_outdir_and_yaml(argv_str)
from template_lib.utils.utils_func import MaxToKeep
max2keep = MaxToKeep(max_to_keep=2)
for i in range(10):
file_path = f"{outdir}/{i:03d}.txt"
with open(file_path, 'w'):
max2keep.step(file_path)
pass
max2keep = MaxToKeep(max_to_keep=2)
for i in range(10):
file_path = f"{outdir}/{i:03d}"
os.makedirs(file_path, exist_ok=True)
max2keep.step(file_path)
pass
def test_run(self, *tmp):
"""
Usage:
export ANSI_COLORS_DISABLED=1
export CUDA_VISIBLE_DEVICES=0
export TIME_STR=1
export PYTHONPATH=./
python template_lib/modelarts/scripts/run.py \
--tl_config_file template_lib/modelarts/tests/configs/run.yaml \
--tl_command run \
--tl_outdir results/Run/run \
--number 1
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
# if 'TIME_STR' not in os.environ:
# os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
os.environ['TIME_STR'] = '0'
if 'RESULTS_OBS' not in os.environ:
os.environ['RESULTS_OBS'] = 's3://bucket-xx/ZhouPeng/results'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file template_lib/modelarts/tests/configs/run.yaml
--tl_command {command}
--tl_outdir {outdir}
"""
args = setup_outdir_and_yaml(argv_str)
n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
cmd_str = f"""
python template_lib/modelarts/scripts/run.py
{get_append_cmd_str(args)}
--number 1
"""
start_cmd_run(cmd_str)
pass
def test_case_evaluate_FID_IS(self):
"""
Usage:
export PYTHONWARNINGS=ignore
export CUDA_VISIBLE_DEVICES=0,7
export master_port=8887
export TIME_STR=1
export PYTHONPATH=./
python -c "from template_lib.v2.tests.test_GAN import TestingTorchFIDISScore;\
TestingTorchFIDISScore().test_case_evaluate_FID_IS()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0,7'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file none
--tl_command none
--tl_outdir {outdir}
"""
args = setup_outdir_and_yaml(argv_str)
nproc_per_node = len(os.environ['CUDA_VISIBLE_DEVICES'].split(','))
master_port = os.environ.get('master_port', 8887)
cmd_str = f"""
python -m torch.distributed.launch --nproc_per_node={nproc_per_node} --master_port={master_port}
template_lib/v2/GAN/evaluation/pytorch_FID_IS_score.py
--run_func PyTorchFIDISScore.test_case_evaluate_FID_IS
"""
cmd_str += get_append_cmd_str(args)
start_cmd_run(cmd_str)
pass
def test_diff_lr(self, debug=True):
"""
Usage:
python template_lib/modelarts/scripts/copy_tool.py \
-s s3://bucket-7001/ZhouPeng/pypi/torch1_7_0 -d /cache/pypi -t copytree
for filename in /cache/pypi/*.whl; do
pip install $filename
done
proj_root=moco-exp
python template_lib/modelarts/scripts/copy_tool.py \
-s s3://bucket-7001/ZhouPeng/codes/$proj_root -d /cache/$proj_root -t copytree -b /cache/$proj_root/code.zip
cd /cache/$proj_root
pip install -r requirements.txt
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
export TIME_STR=1
export PYTHONPATH=./exp:./stylegan2-pytorch:./
python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\
Testing_stylegan2().test_train_ffhq_128()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') +
1:]) if '--tl_opts' in sys.argv else ''
print(f'tl_opts:\n {tl_opts}')
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file none
--tl_command none
--tl_outdir {outdir}
"""
args = setup_outdir_and_yaml(argv_str, return_cfg=True)
from torch.optim import SGD
from torch import nn
class DummyModel(nn.Module):
def __init__(self, class_num=10):
super(DummyModel, self).__init__()
self.base = nn.Sequential(
nn.Conv2d(3, 64, kernel_size=3, padding=1),
nn.ReLU(),
nn.Conv2d(64, 128, kernel_size=3, padding=1),
nn.ReLU(),
)
self.gap = nn.AdaptiveAvgPool2d(1)
self.fc = nn.Linear(128, class_num)
def forward(self, x):
x = self.base(x)
x = self.gap(x)
x = x.view(x.shape[0], -1)
x = self.fc(x)
return x
model = DummyModel().cuda()
optimizer = SGD(
[
{
'params': model.base.parameters()
},
{
'params': model.fc.parameters(),
'lr': 1e-3
} # 对 fc的参数设置不同的学习率
],
lr=1e-2,
momentum=0.9)
pass
def test_step_closure(self, debug=True):
"""
Usage:
python template_lib/modelarts/scripts/copy_tool.py \
-s s3://bucket-7001/ZhouPeng/pypi/torch1_7_0 -d /cache/pypi -t copytree
for filename in /cache/pypi/*.whl; do
pip install $filename
done
proj_root=moco-exp
python template_lib/modelarts/scripts/copy_tool.py \
-s s3://bucket-7001/ZhouPeng/codes/$proj_root -d /cache/$proj_root -t copytree -b /cache/$proj_root/code.zip
cd /cache/$proj_root
pip install -r requirements.txt
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
export TIME_STR=1
export PYTHONPATH=./exp:./stylegan2-pytorch:./
python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\
Testing_stylegan2().test_train_ffhq_128()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') +
1:]) if '--tl_opts' in sys.argv else ''
print(f'tl_opts:\n {tl_opts}')
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file none
--tl_command none
--tl_outdir {outdir}
"""
args = setup_outdir_and_yaml(argv_str, return_cfg=True)
from torch.nn import CrossEntropyLoss
class DummyModel(nn.Module):
def __init__(self, class_num=10):
super(DummyModel, self).__init__()
self.base = nn.Sequential(
nn.Conv2d(3, 64, kernel_size=3, padding=1),
nn.ReLU(),
nn.Conv2d(64, 128, kernel_size=3, padding=1),
nn.ReLU(),
)
self.gap = nn.AdaptiveAvgPool2d(1)
self.fc = nn.Linear(128, class_num)
def forward(self, x):
x = self.base(x)
x = self.gap(x)
x = x.view(x.shape[0], -1)
x = self.fc(x)
return x
dummy_model = DummyModel().cuda()
optimizer = SGD(dummy_model.parameters(),
lr=1e-2,
momentum=0.9,
weight_decay=1e-4)
# 定义loss
loss_fn = CrossEntropyLoss()
# 定义数据
batch_size = 2
data = torch.randn(64, 3, 64,
128).cuda() # 制造假数据shape=64 * 3 * 64 * 128
data_label = torch.randint(0, 10, size=(64, ),
dtype=torch.long).cuda() # 制造假的label
for batch_index in range(10):
batch_data = data[batch_index *
batch_size:batch_index * batch_size + batch_size]
batch_label = data_label[batch_index *
batch_size:batch_index * batch_size +
batch_size]
def closure():
optimizer.zero_grad() # 清空梯度
output = dummy_model(batch_data) # forward
loss = loss_fn(output, batch_label) # 计算loss
loss.backward() # backward
print('No.{: 2d} loss: {:.6f}'.format(batch_index,
loss.item()))
return loss
optimizer.step(closure=closure) # 更新参数
pass
def test_OneCycleLR(self, debug=True):
"""
Usage:
python template_lib/modelarts/scripts/copy_tool.py \
-s s3://bucket-7001/ZhouPeng/pypi/torch1_7_0 -d /cache/pypi -t copytree
for filename in /cache/pypi/*.whl; do
pip install $filename
done
proj_root=moco-exp
python template_lib/modelarts/scripts/copy_tool.py \
-s s3://bucket-7001/ZhouPeng/codes/$proj_root -d /cache/$proj_root -t copytree -b /cache/$proj_root/code.zip
cd /cache/$proj_root
pip install -r requirements.txt
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
export TIME_STR=1
export PYTHONPATH=./exp:./stylegan2-pytorch:./
python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\
Testing_stylegan2().test_train_ffhq_128()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') +
1:]) if '--tl_opts' in sys.argv else ''
print(f'tl_opts:\n {tl_opts}')
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file none
--tl_command none
--tl_outdir {outdir}
"""
args = setup_outdir_and_yaml(argv_str, return_cfg=True)
import torch.nn as nn
from torch.optim import lr_scheduler
from matplotlib import pyplot as plt
model = nn.Linear(3, 64)
def create_optimizer():
return SGD(model.parameters(),
lr=0.1,
momentum=0.9,
weight_decay=1e-4)
def plot_lr(scheduler, title='', labels=['base'], nrof_epoch=100):
lr_li = [[] for _ in range(len(labels))]
epoch_li = list(range(nrof_epoch))
for epoch in epoch_li:
scheduler.step() # 调用step()方法,计算和更新optimizer管理的参数基于当前epoch的学习率
lr = scheduler.get_last_lr() # 获取当前epoch的学习率
for i in range(len(labels)):
lr_li[i].append(lr[i])
for lr, label in zip(lr_li, labels):
plt.plot(epoch_li, lr, label=label)
plt.grid()
plt.xlabel('epoch')
plt.ylabel('lr')
plt.title(title)
plt.legend()
plt.show()
optimizer = create_optimizer()
scheduler = lr_scheduler.OneCycleLR(optimizer, 0.1, total_steps=100)
plot_lr(scheduler, title='OneCycleLR')
pass
def test_base_usage(self, debug=True):
"""
Usage:
python template_lib/modelarts/scripts/copy_tool.py \
-s s3://bucket-7001/ZhouPeng/pypi/torch1_7_0 -d /cache/pypi -t copytree
for filename in /cache/pypi/*.whl; do
pip install $filename
done
proj_root=moco-exp
python template_lib/modelarts/scripts/copy_tool.py \
-s s3://bucket-7001/ZhouPeng/codes/$proj_root -d /cache/$proj_root -t copytree -b /cache/$proj_root/code.zip
cd /cache/$proj_root
pip install -r requirements.txt
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
export TIME_STR=1
export PYTHONPATH=./exp:./stylegan2-pytorch:./
python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\
Testing_stylegan2().test_train_ffhq_128()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') +
1:]) if '--tl_opts' in sys.argv else ''
print(f'tl_opts:\n {tl_opts}')
command, outdir = get_command_and_outdir(
self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file none
--tl_command none
--tl_outdir {outdir}
"""
args = setup_outdir_and_yaml(argv_str, return_cfg=True)
import torch
import numpy as np
import warnings
warnings.filterwarnings('ignore') # ignore warnings
x = torch.linspace(-np.pi, np.pi, 2000)
y = torch.sin(x)
p = torch.tensor([1, 2, 3])
xx = x.unsqueeze(-1).pow(p)
model = torch.nn.Sequential(torch.nn.Linear(3, 1),
torch.nn.Flatten(0, 1))
loss_fn = torch.nn.MSELoss(reduction='sum')
learning_rate = 1e-3
optimizer = torch.optim.RMSprop(model.parameters(), lr=learning_rate)
for t in range(1, 1001):
y_pred = model(xx)
loss = loss_fn(y_pred, y)
if t % 100 == 0:
print('No.{: 5d}, loss: {:.6f}'.format(t, loss.item()))
optimizer.zero_grad() # 梯度清零
loss.backward() # 反向传播计算梯度
optimizer.step() # 梯度下降法更新参数
pass
