def test_Image(self, debug=True): """ Usage: export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 export TIME_STR=1 export PYTHONPATH=./exp:./stylegan2-pytorch:./ python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\ Testing_stylegan2().test_train_ffhq_128()" :return: """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '0' if 'TIME_STR' not in os.environ: os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0' from template_lib.v2.config_cfgnode.argparser import \ (get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run) tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') + 1:]) if '--tl_opts' in sys.argv else '' print(f'tl_opts:\n {tl_opts}') command, outdir = get_command_and_outdir( self, func_name=sys._getframe().f_code.co_name, file=__file__) argv_str = f""" --tl_config_file none --tl_command none --tl_outdir {outdir} --tl_opts {tl_opts} """ args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True) n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(',')) import mmcv import numpy as np import matplotlib.pyplot as plt img_path = "template_lib/datasets/images/zebra_GT_target_origin.png" img = mmcv.imread(img_path) img_gray = mmcv.imread(img_path, flag='grayscale') img_ = mmcv.imread(img) # nothing will happen, img_ = img mmcv.imwrite(img, f'{args.tl_outdir}/out.png') # mmcv.imshow(img) fig, axes = plt.subplots(2, 1) img = mmcv.bgr2rgb(img) axes[0].imshow(img) # plt.imshow(img) # plt.show() # ret = mmcv.imresize(img, (1000, 600), return_scale=True) # ret = mmcv.imrescale(img, (1000, 800)) bboxes = np.array([10, 10, 100, 120]) patch = mmcv.imcrop(img, bboxes) axes[1].imshow(patch) fig.show() pass
def test_make_zip(self): """ Usage: export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 export TIME_STR=1 export PYTHONPATH=./exp:./stylegan2-pytorch:./ python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\ Testing_stylegan2().test_train_ffhq_128()" :return: """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '0' if 'TIME_STR' not in os.environ: os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0' from template_lib.v2.config_cfgnode.argparser import \ (get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run) command, outdir = get_command_and_outdir( self, func_name=sys._getframe().f_code.co_name, file=__file__) argv_str = f""" --tl_config_file none --tl_command none --tl_outdir {outdir} """ args = setup_outdir_and_yaml(argv_str) from template_lib.utils.utils_func import make_zip, unzip_file make_zip(source_dir='./template_lib', output_filename='code.zip') unzip_file(zip_file='code.zip', dst_dir='/tmp/code') pass
def test_extract_ImageNet100_CMC(self): """ Usage: proj_root=moco-exp python template_lib/modelarts/scripts/copy_tool.py \ -s s3://bucket-7001/ZhouPeng/codes/$proj_root -d /cache/$proj_root -t copytree cd /cache/$proj_root export CUDA_VISIBLE_DEVICES=0 export TIME_STR=0 export PYTHONPATH=./ python -c "from template_lib.proj.imagenet.tests.test_imagenet import Testing_PrepareImageNet;\ Testing_PrepareImageNet().test_extract_ImageNet100_CMC()" :return: """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '0' if 'TIME_STR' not in os.environ: os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0' from template_lib.v2.config_cfgnode.argparser import \ (get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run) from template_lib.v2.config_cfgnode import update_parser_defaults_from_yaml, global_cfg from template_lib.modelarts import modelarts_utils from distutils.dir_util import copy_tree command, outdir = get_command_and_outdir(self, func_name=sys._getframe().f_code.co_name, file=__file__) argv_str = f""" --tl_config_file template_lib/proj/imagenet/tests/configs/PrepareImageNet.yaml --tl_command {command} --tl_outdir {outdir} """ args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True) modelarts_utils.setup_tl_outdir_obs(global_cfg) modelarts_utils.modelarts_sync_results_dir(global_cfg, join=True) modelarts_utils.prepare_dataset(global_cfg.get('modelarts_download', {}), global_cfg=global_cfg) train_dir = f'{cfg.data_dir}/train' val_dir = f'{cfg.data_dir}/val' save_train_dir = f'{cfg.saved_dir}/train' save_val_dir = f'{cfg.saved_dir}/val' os.makedirs(save_train_dir, exist_ok=True) os.makedirs(save_val_dir, exist_ok=True) with open(cfg.class_list_file, 'r') as f: class_list = f.readlines() for class_subdir in tqdm.tqdm(class_list): class_subdir, _ = class_subdir.strip().split() train_class_dir = f'{train_dir}/{class_subdir}' save_train_class_dir = f'{save_train_dir}/{class_subdir}' copy_tree(train_class_dir, save_train_class_dir) val_class_dir = f'{val_dir}/{class_subdir}' save_val_class_dir = f'{save_val_dir}/{class_subdir}' copy_tree(val_class_dir, save_val_class_dir) modelarts_utils.prepare_dataset(global_cfg.get('modelarts_upload', {}), global_cfg=global_cfg, download=False) modelarts_utils.modelarts_sync_results_dir(global_cfg, join=True) pass
def test_draw_text_on_rectangle(self, debug=True): """ Usage: export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 export TIME_STR=1 export PYTHONPATH=./exp:./stylegan2-pytorch:./ python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\ Testing_stylegan2().test_train_ffhq_128()" :return: """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '0' if 'TIME_STR' not in os.environ: os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0' from template_lib.v2.config_cfgnode.argparser import \ (get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run) tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') + 1:]) if '--tl_opts' in sys.argv else '' print(f'tl_opts:\n {tl_opts}') command, outdir = get_command_and_outdir( self, func_name=sys._getframe().f_code.co_name, file=__file__) argv_str = f""" --tl_config_file none --tl_command none --tl_outdir {outdir} --tl_opts {tl_opts} """ args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True) n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(',')) img_path = "template_lib/datasets/images/zebra_GT_target_origin.png" source_img = Image.open(img_path) font = ImageFont.truetype(_font) text = "very loooooooooooooooooong text" # get text size text_size = font.getsize(text) # set button size + 10px margins button_size = (text_size[0] + 20, text_size[1] + 20) # create image with correct size and black background button_img = Image.new('RGB', button_size, "black") # put text on button with 10px margins button_draw = ImageDraw.Draw(button_img) button_draw.text((10, 10), text, font=font) # put button on source image in position (0, 0) source_img.paste(button_img, (0, 0)) # save in new file plt.imshow(source_img) plt.show() # source_img.save("output.jpg", "JPEG") pass
def test_get_subdir2name_dict(self): """ Usage: export CUDA_VISIBLE_DEVICES=0 export TIME_STR=0 export PYTHONPATH=./ python -c "from template_lib.proj.imagenet.tests.test_imagenet import Testing_PrepareImageNet;\ Testing_PrepareImageNet().test_mv_val_dataset()" :return: """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '0' if 'TIME_STR' not in os.environ: os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0' from template_lib.v2.config_cfgnode.argparser import \ (get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run) command, outdir = get_command_and_outdir(self, func_name=sys._getframe().f_code.co_name, file=__file__) argv_str = f""" --tl_config_file none --tl_command none --tl_outdir {outdir} """ args = setup_outdir_and_yaml(argv_str) from template_lib.proj.imagenet.utils import get_subdir2name_dict, subdir2name_dict subdir2name_d = get_subdir2name_dict() pass
def test_attr_s(self): """ """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '0' if 'TIME_STR' not in os.environ: os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0' from template_lib.v2.config_cfgnode.argparser import \ (get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run) command, outdir = get_command_and_outdir( self, func_name=sys._getframe().f_code.co_name, file=__file__) argv_str = f""" --tl_config_file none --tl_command none --tl_outdir {outdir} """ args = setup_outdir_and_yaml(argv_str) import attr @attr.s class SomeClass(object): a_number = attr.ib(default=42) list_of_numbers = attr.ib(factory=list) def hard_math(self, another_number): return self.a_number + sum( self.list_of_numbers) * another_number sc = SomeClass(1, [1, 2, 3]) pass
def test_merge_original_image_and_patches_horizon(self, debug=True): """ Usage: export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 export TIME_STR=1 export PYTHONPATH=./exp:./stylegan2-pytorch:./ python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\ Testing_stylegan2().test_train_ffhq_128()" :return: """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '0' if 'TIME_STR' not in os.environ: os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0' from template_lib.v2.config_cfgnode.argparser import \ (get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run) tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') + 1:]) if '--tl_opts' in sys.argv else '' print(f'tl_opts:\n {tl_opts}') command, outdir = get_command_and_outdir( self, func_name=sys._getframe().f_code.co_name, file=__file__) argv_str = f""" --tl_config_file none --tl_command none --tl_outdir {outdir} --tl_opts {tl_opts} """ args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True) n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(',')) img_path = "template_lib/datasets/images/zebra_GT_target_origin.png" left = [80, 300] upper = [120, 140] w = 50 h = 20 pad = 2 image = Image.open(img_path) out_image = merge_original_image_and_patches_horizon(image=image, lefts=left, uppers=upper, w=w, h=h, pad=pad, width=3, patch_width=1) fig, axes = plt.subplots(1, 1) axes.imshow(out_image) # axes[1].imshow(patch) fig.show() pass
def test_DDPDataLoaderCIFAR100(self, debug=True): """ Usage: python template_lib/modelarts/scripts/copy_tool.py \ -s s3://bucket-7001/ZhouPeng/pypi/torch1_7_0 -d /cache/pypi -t copytree for filename in /cache/pypi/*.whl; do pip install $filename done proj_root=moco-exp python template_lib/modelarts/scripts/copy_tool.py \ -s s3://bucket-7001/ZhouPeng/codes/$proj_root -d /cache/$proj_root -t copytree -b /cache/$proj_root/code.zip cd /cache/$proj_root pip install -r requirements.txt export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 export TIME_STR=1 export PYTHONPATH=./exp:./stylegan2-pytorch:./ python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\ Testing_stylegan2().test_train_ffhq_128()" :return: """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '0,1' if 'TIME_STR' not in os.environ: os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0' from template_lib.v2.config_cfgnode.argparser import \ (get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run) command, outdir = get_command_and_outdir( self, func_name=sys._getframe().f_code.co_name, file=__file__) argv_str = f""" --tl_config_file template_lib/v2/GAN/datasets/configs/cifar100.yaml --tl_command {command} --tl_outdir {outdir} """ args = setup_outdir_and_yaml(argv_str) tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') + 1:]) if '--tl_opts' in sys.argv else '' print(f'tl_opts:\n {tl_opts}') n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(',')) cmd_str = f""" python -m torch.distributed.launch --nproc_per_node={n_gpus} --master_port=8888 template_lib/v2/GAN/datasets/scripts/cifar10.py {get_append_cmd_str(args)} """ if debug: cmd_str += f""" --tl_debug --tl_opts """ else: cmd_str += f""" --tl_opts {tl_opts} """ start_cmd_run(cmd_str)
def test_plot_figures_template(self): """ python -c "from exp.tests.test_styleganv2 import Testing_stylegan2_style_position;\ Testing_stylegan2_style_position().test_plot_FID_cifar10_style_position()" """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '3' if 'TIME_STR' not in os.environ: os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0' from template_lib.v2.config_cfgnode.argparser import \ (get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run) command, outdir = get_command_and_outdir( self, func_name=sys._getframe().f_code.co_name, file=__file__) argv_str = f""" --tl_config_file none --tl_command none --tl_outdir {outdir} """ args = setup_outdir_and_yaml(argv_str) outdir = args.tl_outdir from template_lib.utils.plot_results import PlotResults import collections import pickle outfigure = os.path.join(outdir, 'IS.jpg') default_dicts = collections.OrderedDict() show_max = [] IS_GAN_cGANs_CIFAR100 = collections.defaultdict(dict) title = 'IS_GAN_cGANs_CIFAR100' log_file = 'textdir/evaltf.ma1.IS_mean_tf.log' dd = eval(title) dd['results/stylegan2/train_cifar100-20201030_2234_241'] = \ {'20201030_2234_241-stylegan': "textdir/eval.ma1.IS_mean_tf.log", } dd['properties'] = { 'title': title, } default_dicts[title] = dd show_max.append(True) plotobs = PlotResults() label2datas_list = plotobs.plot_defaultdicts( outfigure=outfigure, default_dicts=default_dicts, show_max=show_max, figsize_wh=(16, 7.2)) print(f'Save to {outfigure}.') saved_data = '__'.join(outdir.split('/')[-2:]) saved_data = f"{outdir}/{saved_data}.pkl" with open(saved_data, 'wb') as f: pickle.dump(label2datas_list, f) print(f"Save data to {saved_data}") pass
def test_save_cifar10_images(self, debug=True): """ Usage: python template_lib/modelarts/scripts/copy_tool.py \ -s s3://bucket-7001/ZhouPeng/pypi/torch1_7_0 -d /cache/pypi -t copytree for filename in /cache/pypi/*.whl; do pip install $filename done proj_root=deep-generative-prior-exp python template_lib/modelarts/scripts/copy_tool.py \ -s s3://bucket-7001/ZhouPeng/codes/$proj_root -d /cache/$proj_root -t copytree -b /cache/$proj_root/code.zip cd /cache/$proj_root pip install -r requirements.txt export CUDA_VISIBLE_DEVICES=5 export TIME_STR=1 export PYTHONPATH=./:DGP_lib:BigGAN_Pytorch_lib python -c "from template_lib.proj.cifar.tests.test_cifar_dataset import Testing_CIFAR10;\ Testing_CIFAR10().test_save_cifar10_images(debug=False)" :return: """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '1' if 'TIME_STR' not in os.environ: os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0' from template_lib.v2.config_cfgnode.argparser import \ (get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run) tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') + 1:]) if '--tl_opts' in sys.argv else '' print(f'tl_opts:\n {tl_opts}') command, outdir = get_command_and_outdir( self, func_name=sys._getframe().f_code.co_name, file=__file__) argv_str = f""" --tl_config_file none --tl_command none --tl_outdir {outdir} --tl_opts {tl_opts} """ args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True) import tqdm import torchvision saved_dir = 'datasets/cifar10/cifar10_images' cifar10 = torchvision.datasets.CIFAR10(root='datasets/cifar10', train=True) for idx in tqdm.tqdm(range(len(cifar10))): img_pil, label_id = cifar10[idx] subdir = f"{saved_dir}/{cifar10.classes[label_id]}_{label_id:02d}" os.makedirs(subdir, exist_ok=True) img_pil.save(f"{subdir}/{idx:05d}.png") pass
def test_reduce(self, debug=True): """ Usage: python template_lib/modelarts/scripts/copy_tool.py \ -s s3://bucket-7001/ZhouPeng/pypi/torch1_7_0 -d /cache/pypi -t copytree for filename in /cache/pypi/*.whl; do pip install $filename done proj_root=moco-exp python template_lib/modelarts/scripts/copy_tool.py \ -s s3://bucket-7001/ZhouPeng/codes/$proj_root -d /cache/$proj_root -t copytree -b /cache/$proj_root/code.zip cd /cache/$proj_root pip install -r requirements.txt export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 export TIME_STR=1 export PYTHONPATH=./exp:./stylegan2-pytorch:./ python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\ Testing_stylegan2().test_train_ffhq_128()" :return: """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '0' if 'TIME_STR' not in os.environ: os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0' from template_lib.v2.config_cfgnode.argparser import \ (get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run) tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') + 1:]) if '--tl_opts' in sys.argv else '' print(f'tl_opts:\n {tl_opts}') command, outdir = get_command_and_outdir( self, func_name=sys._getframe().f_code.co_name, file=__file__) argv_str = f""" --tl_config_file none --tl_command none --tl_outdir {outdir} --tl_opts {tl_opts} """ args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True) import matplotlib.pyplot as plt import numpy as np from PIL import Image from einops import rearrange, reduce img_path = "template_lib/datasets/images/zebra_GT_target_origin.png" img_pil = Image.open(img_path) img_np = np.array(img_pil) img_np = reduce(img_np, '(h h2) (w w2) c -> h w c', 'min', h2=2, w2=2) plt.imshow(img_np) plt.show() pass
def test_extract_ImageNet_1000x50(self): """ Usage: proj_root=moco-exp python template_lib/modelarts/scripts/copy_tool.py \ -s s3://bucket-7001/ZhouPeng/codes/$proj_root -d /cache/$proj_root -t copytree cd /cache/$proj_root export CUDA_VISIBLE_DEVICES=0 export TIME_STR=0 export PYTHONPATH=./ python -c "from template_lib.proj.imagenet.tests.test_imagenet import Testing_PrepareImageNet;\ Testing_PrepareImageNet().test_extract_ImageNet_1000x50()" :return: """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '0' if 'TIME_STR' not in os.environ: os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0' from template_lib.v2.config_cfgnode.argparser import \ (get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run) from template_lib.v2.config_cfgnode import update_parser_defaults_from_yaml, global_cfg from template_lib.modelarts import modelarts_utils command, outdir = get_command_and_outdir(self, func_name=sys._getframe().f_code.co_name, file=__file__) argv_str = f""" --tl_config_file template_lib/proj/imagenet/tests/configs/PrepareImageNet.yaml --tl_command {command} --tl_outdir {outdir} """ args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True) global_cfg.merge_from_dict(cfg) global_cfg.merge_from_dict(vars(args)) modelarts_utils.setup_tl_outdir_obs(global_cfg) modelarts_utils.modelarts_sync_results_dir(global_cfg, join=True) modelarts_utils.prepare_dataset(global_cfg.get('modelarts_download', {}), global_cfg=global_cfg) train_dir = f'{cfg.data_dir}/train' counter_cls = 0 for rootdir, subdir, files in os.walk(train_dir): if len(subdir) == 0: counter_cls += 1 extracted_files = sorted(files)[:cfg.num_per_class] for file in tqdm.tqdm(extracted_files, desc=f'class: {counter_cls}'): img_path = os.path.join(rootdir, file) img_rel_path = os.path.relpath(img_path, cfg.data_dir) saved_img_path = f'{cfg.saved_dir}/{os.path.dirname(img_rel_path)}' os.makedirs(saved_img_path, exist_ok=True) shutil.copy(img_path, saved_img_path) pass modelarts_utils.prepare_dataset(global_cfg.get('modelarts_upload', {}), global_cfg=global_cfg, download=False) modelarts_utils.modelarts_sync_results_dir(global_cfg, join=True) pass
def test_zoom_in_video_writer(self, debug=True): """ Usage: export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 export TIME_STR=1 export PYTHONPATH=./exp:./stylegan2-pytorch:./ python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\ Testing_stylegan2().test_train_ffhq_128()" :return: """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '0' if 'TIME_STR' not in os.environ: os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0' from template_lib.v2.config_cfgnode.argparser import \ (get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run) tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') + 1:]) if '--tl_opts' in sys.argv else '' print(f'tl_opts:\n {tl_opts}') command, outdir = get_command_and_outdir( self, func_name=sys._getframe().f_code.co_name, file=__file__) argv_str = f""" --tl_config_file none --tl_command none --tl_outdir {outdir} --tl_opts {tl_opts} """ args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True) n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(',')) import tqdm from template_lib.proj.pil.pil_utils import get_size img_path = "template_lib/datasets/images/zebra_GT_target_origin.png" outvid = f"{args.tl_outdir}/test.mp4" out_size = 2048 image = Image.open(img_path) w, h = image.size max_scale = out_size / min(w, h) out_w, out_h = get_size(w=w, h=h, dst_size=out_size) out_video = VideoWriter(outfile=outvid, w=out_w, h=out_h, fps=10) for scale in tqdm.tqdm(np.arange(1, max_scale, 0.05)): out_img = Image.new(mode='RGB', size=(out_w, out_h), color='black') cur_w, cur_h = int(w * scale), int(h * scale) cur_image = image.resize((cur_w, cur_h), resample=Image.NEAREST) xy = (out_w - cur_w) // 2, (out_h - cur_h) // 2 out_img.paste(cur_image, xy) out_video.write(out_img, is_tensor=False, rgb=True) out_video.release() pass
def test_plot_figures_template(self): """ python -c "from exp.tests.test_styleganv2 import Testing_stylegan2_style_position;\ Testing_stylegan2_style_position().test_plot_FID_cifar10_style_position()" """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '3' if 'TIME_STR' not in os.environ: os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0' from template_lib.v2.config_cfgnode.argparser import \ (get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run) command, outdir = get_command_and_outdir( self, func_name=sys._getframe().f_code.co_name, file=__file__) argv_str = f""" --tl_config_file none --tl_command none --tl_outdir {outdir} """ args = setup_outdir_and_yaml(argv_str) outdir = args.tl_outdir from template_lib.utils.plot_results import PlotResults import collections import pickle outfigure = os.path.join(outdir, 'FID.jpg') default_dicts = [] show_max = [] top1_randomlabel = collections.defaultdict(dict) title = 'top1_randomlabel' log_file = 'textdir/evaltf.ma0.FID_tf.log' dd = eval(title) dd['results/CIFAR10/train_R56_cifar10_20200923-23_55_05_422/'] = \ {'23_55_05_422-resnet-R56': log_file, } dd['properties'] = {'title': title, 'ylim': [0, 80]} default_dicts.append(dd) show_max.append(False) plotobs = PlotResults() label2datas_list = plotobs.plot_defaultdicts( outfigure=outfigure, default_dicts=default_dicts, show_max=show_max, figsize_wh=(16, 7.2)) print(f'Save to {outfigure}.') saved_data = '__'.join(outdir.split('/')[-2:]) saved_data = f"{outdir}/{saved_data}.pkl" with open(saved_data, 'wb') as f: pickle.dump(label2datas_list, f) print(f"Save data to {saved_data}") pass
def test_run(self, *tmp): """ Usage: export ANSI_COLORS_DISABLED=1 export CUDA_VISIBLE_DEVICES=0 export TIME_STR=1 export PYTHONPATH=./ python template_lib/modelarts/scripts/run.py \ --tl_config_file template_lib/modelarts/tests/configs/run.yaml \ --tl_command run \ --tl_outdir results/Run/run \ --number 1 # default image /bucket-8280/ZhouPeng/codes/Omni-GAN-ImageNet/template_lib/modelarts/scripts/run.py number = 3 tl_outdir = results/Run/run tl_config_file = template_lib/modelarts/tests/configs/run.yaml tl_opts = root_obs s3://bucket-7001/ZhouPeng/ tl_command = run # self defined image bash /home/work/run_train.sh python /home/work/user-job-dir/Omni-GAN-ImageNet/template_lib/modelarts/scripts/run.py --tl_outdir=results/Run/run --tl_config_file=/home/work/user-job-dir/Omni-GAN-ImageNet/template_lib/modelarts/tests/configs/run.yaml --tl_command=run --tl_opts=root_obs s3://bucket-7001/ZhouPeng/ --number=2 :return: """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '0' # if 'TIME_STR' not in os.environ: # os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0' os.environ['TIME_STR'] = '0' if 'RESULTS_OBS' not in os.environ: os.environ['RESULTS_OBS'] = 's3://bucket-xx/ZhouPeng/results' from template_lib.v2.config_cfgnode.argparser import \ (get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run) command, outdir = get_command_and_outdir( self, func_name=sys._getframe().f_code.co_name, file=__file__) argv_str = f""" --tl_config_file template_lib/modelarts/tests/configs/run.yaml --tl_command {command} --tl_outdir {outdir} """ args = setup_outdir_and_yaml(argv_str) n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(',')) cmd_str = f""" python template_lib/modelarts/scripts/run.py {get_append_cmd_str(args)} --number 1 """ start_cmd_run(cmd_str) pass
def test_ddp_calculate_fid_stat_CIFAR100(self, debug=True): """ Usage: export CUDA_VISIBLE_DEVICES=1,4 export master_port=8887 export TIME_STR=0 export PYTHONPATH=./ python -c "from template_lib.v2.GAN.evaluation.tests.test_pytorch_FID_IS_score import TestingTorchFIDISScore;\ TestingTorchFIDISScore().test_ddp_calculate_fid_stat_CIFAR100(debug=False)" \ --tl_opts build_dataloader.shuffle False save_fid_stat False :return: """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '0' if 'TIME_STR' not in os.environ: os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0' command, outdir = get_command_and_outdir( self, func_name=sys._getframe().f_code.co_name, file=__file__) argv_str = f""" --tl_config_file template_lib/v2/GAN/evaluation/configs/TorchFIDISScore.yaml --tl_command {command} --tl_outdir {outdir} """ args = setup_outdir_and_yaml(argv_str) tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') + 1:]) if '--tl_opts' in sys.argv else '' print(f'tl_opts:\n {tl_opts}') nproc_per_node = len(os.environ['CUDA_VISIBLE_DEVICES'].split(',')) master_port = os.environ.get('master_port', 8887) cmd_str = f""" python -m torch.distributed.launch --nproc_per_node={nproc_per_node} --master_port={master_port} template_lib/v2/GAN/evaluation/scripts/torch_calculate_fid_stat_CIFAR.py {get_append_cmd_str(args)} """ if debug: cmd_str += f""" --tl_debug --tl_opts """ else: cmd_str += f""" --tl_opts {tl_opts} """ start_cmd_run(cmd_str) pass
def test_SpacialTransformNet(self, debug=True): """ Usage: python template_lib/modelarts/scripts/copy_tool.py \ -s s3://bucket-7001/ZhouPeng/pypi/torch1_7_0 -d /cache/pypi -t copytree for filename in /cache/pypi/*.whl; do pip install $filename done proj_root=moco-exp python template_lib/modelarts/scripts/copy_tool.py \ -s s3://bucket-7001/ZhouPeng/codes/$proj_root -d /cache/$proj_root -t copytree -b /cache/$proj_root/code.zip cd /cache/$proj_root pip install -r requirements.txt export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 export TIME_STR=1 export PYTHONPATH=./exp:./stylegan2-pytorch:./ python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\ Testing_stylegan2().test_train_ffhq_128()" :return: """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '0' if 'TIME_STR' not in os.environ: os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0' from template_lib.v2.config_cfgnode.argparser import \ (get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run) tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') + 1:]) if '--tl_opts' in sys.argv else '' print(f'tl_opts:\n {tl_opts}') command, outdir = get_command_and_outdir( self, func_name=sys._getframe().f_code.co_name, file=__file__) argv_str = f""" --tl_config_file none --tl_command none --tl_outdir {outdir} --tl_opts {tl_opts} """ args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True) from template_lib.proj.pytorch.pytorch_hook import VerboseModel net = SpacialTransformNet() net = VerboseModel(model=net) assert 0 x = torch.rand(1, 32, 16, 16) out = net(x) pass
def test_mv_val_dataset(self): """ Usage: export CUDA_VISIBLE_DEVICES=0 export TIME_STR=0 export PYTHONPATH=./ python -c "from template_lib.proj.imagenet.tests.test_imagenet import Testing_PrepareImageNet;\ Testing_PrepareImageNet().test_mv_val_dataset()" :return: """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '0' if 'TIME_STR' not in os.environ: os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0' from template_lib.v2.config_cfgnode.argparser import \ (get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run) command, outdir = get_command_and_outdir( self, func_name=sys._getframe().f_code.co_name, file=__file__) argv_str = f""" --tl_config_file none --tl_command none --tl_outdir {outdir} """ args = setup_outdir_and_yaml(argv_str) n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(',')) cmd_str = f""" bash ls -la datasets/ImageNet/ && cp template_lib/proj/imagenet/imagenet_move_val_images.sh datasets/ImageNet && cd datasets/ImageNet/val && ls -la ../imagenet_move_val_images.sh && bash ../imagenet_move_val_images.sh """ start_cmd_run(cmd_str) # from template_lib.v2.config_cfgnode import update_parser_defaults_from_yaml, global_cfg # from template_lib.modelarts import modelarts_utils # update_parser_defaults_from_yaml(parser) # modelarts_utils.setup_tl_outdir_obs(global_cfg) # modelarts_utils.modelarts_sync_results_dir(global_cfg, join=True) # modelarts_utils.prepare_dataset(global_cfg.get('modelarts_download', {}), global_cfg=global_cfg) # # modelarts_utils.prepare_dataset(global_cfg.get('modelarts_upload', {}), global_cfg=global_cfg, download=False) # modelarts_utils.modelarts_sync_results_dir(global_cfg, join=True) pass
def test_merge_image_pil(self, debug=True): """ Usage: export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 export TIME_STR=1 export PYTHONPATH=./exp:./stylegan2-pytorch:./ python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\ Testing_stylegan2().test_train_ffhq_128()" :return: """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '0' if 'TIME_STR' not in os.environ: os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0' from template_lib.v2.config_cfgnode.argparser import \ (get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run) tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') + 1:]) if '--tl_opts' in sys.argv else '' print(f'tl_opts:\n {tl_opts}') command, outdir = get_command_and_outdir( self, func_name=sys._getframe().f_code.co_name, file=__file__) argv_str = f""" --tl_config_file none --tl_command none --tl_outdir {outdir} --tl_opts {tl_opts} """ args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True) img_path = "template_lib/datasets/images/zebra_GT_target_origin.png" pad = 2 pad_color = 'white' img = Image.open(img_path) image_list = [ img, ] * 5 merged_img = merge_image_pil(image_list=image_list, nrow=3, pad=pad, pad_color=pad_color) plt.imshow(merged_img) plt.show() pass
def test_extract_imagenet_val_for_selection(self): """ Usage: proj_root=moco-exp python template_lib/modelarts/scripts/copy_tool.py \ -s s3://bucket-7001/ZhouPeng/codes/$proj_root -d /cache/$proj_root -t copytree cd /cache/$proj_root export CUDA_VISIBLE_DEVICES=0 export TIME_STR=0 export PYTHONPATH=./ python -c "from template_lib.proj.imagenet.tests.test_imagenet import Testing_PrepareImageNet;\ Testing_PrepareImageNet().test_extract_imagenet_val_for_selection()" :return: """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '0' if 'TIME_STR' not in os.environ: os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0' from template_lib.v2.config_cfgnode.argparser import \ (get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run) from template_lib.v2.config_cfgnode import update_parser_defaults_from_yaml, global_cfg from template_lib.modelarts import modelarts_utils from distutils.dir_util import copy_tree command, outdir = get_command_and_outdir(self, func_name=sys._getframe().f_code.co_name, file=__file__) argv_str = f""" --tl_config_file none --tl_command none --tl_outdir {outdir} """ args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True) import shutil import tqdm from template_lib.utils import get_filelist_recursive val_dir = "datasets/ImageNet/val" saved_dir = "datasets/ImageNet/imagenet_val_selection" os.makedirs(saved_dir, exist_ok=True) image_list = get_filelist_recursive(val_dir, ext='*.JPEG') for image_path in tqdm.tqdm(image_list): saved_file = f"{saved_dir}/{image_path.parent.name}_{image_path.name}" shutil.copyfile(image_path, saved_file) pass
def test_ImageNet100_CMC_class_file_append_classname(self): """ Usage: proj_root=moco-exp python template_lib/modelarts/scripts/copy_tool.py \ -s s3://bucket-7001/ZhouPeng/codes/$proj_root -d /cache/$proj_root -t copytree cd /cache/$proj_root export CUDA_VISIBLE_DEVICES=0 export TIME_STR=0 export PYTHONPATH=./ python -c "from template_lib.proj.imagenet.tests.test_imagenet import Testing_PrepareImageNet;\ Testing_PrepareImageNet().test_extract_ImageNet_1000x50()" :return: """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '0' if 'TIME_STR' not in os.environ: os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0' from template_lib.v2.config_cfgnode.argparser import \ (get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run) from template_lib.proj.imagenet.utils import subdir2name_dict command, outdir = get_command_and_outdir( self, func_name=sys._getframe().f_code.co_name, file=__file__) argv_str = f""" --tl_config_file template_lib/proj/imagenet/tests/configs/PrepareImageNet.yaml --tl_command {command} --tl_outdir {outdir} """ args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True) class_list_file = cfg.class_list_file saved_class_list_file = cfg.saved_class_list_file with open(class_list_file, 'r') as f: class_list = f.readlines() saved_f = open(saved_class_list_file, 'w') for class_subdir in tqdm.tqdm(class_list): class_subdir = class_subdir.strip() class_name = subdir2name_dict[class_subdir] saved_f.write(f"{class_subdir} {class_name}\n") saved_f.close() pass
def test_imshow(self, debug=True): """ Usage: export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 export TIME_STR=1 export PYTHONPATH=./exp:./stylegan2-pytorch:./ python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\ Testing_stylegan2().test_train_ffhq_128()" :return: """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '0' if 'TIME_STR' not in os.environ: os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0' from template_lib.v2.config_cfgnode.argparser import \ (get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run) tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') + 1:]) if '--tl_opts' in sys.argv else '' print(f'tl_opts:\n {tl_opts}') command, outdir = get_command_and_outdir( self, func_name=sys._getframe().f_code.co_name, file=__file__) argv_str = f""" --tl_config_file none --tl_command none --tl_outdir {outdir} --tl_opts {tl_opts} """ args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True) n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(',')) import mmcv import numpy as np img_path = "template_lib/datasets/images/zebra_GT_target_origin.png" mmcv.imshow(img_path) # show image with bounding boxes img = np.random.rand(100, 100, 3) bboxes = np.array([[0, 0, 50, 50], [20, 20, 60, 60]]) mmcv.imshow_bboxes(img, bboxes) pass
def test_draw_rectangle(self, debug=True): """ Usage: export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 export TIME_STR=1 export PYTHONPATH=./exp:./stylegan2-pytorch:./ python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\ Testing_stylegan2().test_train_ffhq_128()" :return: """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '0' if 'TIME_STR' not in os.environ: os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0' from template_lib.v2.config_cfgnode.argparser import \ (get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run) tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') + 1:]) if '--tl_opts' in sys.argv else '' print(f'tl_opts:\n {tl_opts}') command, outdir = get_command_and_outdir( self, func_name=sys._getframe().f_code.co_name, file=__file__) argv_str = f""" --tl_config_file none --tl_command none --tl_outdir {outdir} --tl_opts {tl_opts} """ args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True) n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(',')) img_path = "template_lib/datasets/images/zebra_GT_target_origin.png" im = Image.open(img_path) draw = ImageDraw.Draw(im) draw.rectangle(((10, 10), (50, 50)), outline='#ff0000', width=2) plt.imshow(im) plt.show() pass
def test_MaxToKeep(self): """ Usage: export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 export TIME_STR=1 export PYTHONPATH=./exp:./stylegan2-pytorch:./ python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\ Testing_stylegan2().test_train_ffhq_128()" :return: """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '0' if 'TIME_STR' not in os.environ: os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0' from template_lib.v2.config_cfgnode.argparser import \ (get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run) command, outdir = get_command_and_outdir( self, func_name=sys._getframe().f_code.co_name, file=__file__) argv_str = f""" --tl_config_file none --tl_command none --tl_outdir {outdir} """ args = setup_outdir_and_yaml(argv_str) from template_lib.utils.utils_func import MaxToKeep max2keep = MaxToKeep(max_to_keep=2) for i in range(10): file_path = f"{outdir}/{i:03d}.txt" with open(file_path, 'w'): max2keep.step(file_path) pass max2keep = MaxToKeep(max_to_keep=2) for i in range(10): file_path = f"{outdir}/{i:03d}" os.makedirs(file_path, exist_ok=True) max2keep.step(file_path) pass
def test_run(self, *tmp): """ Usage: export ANSI_COLORS_DISABLED=1 export CUDA_VISIBLE_DEVICES=0 export TIME_STR=1 export PYTHONPATH=./ python template_lib/modelarts/scripts/run.py \ --tl_config_file template_lib/modelarts/tests/configs/run.yaml \ --tl_command run \ --tl_outdir results/Run/run \ --number 1 :return: """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '0' # if 'TIME_STR' not in os.environ: # os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0' os.environ['TIME_STR'] = '0' if 'RESULTS_OBS' not in os.environ: os.environ['RESULTS_OBS'] = 's3://bucket-xx/ZhouPeng/results' from template_lib.v2.config_cfgnode.argparser import \ (get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run) command, outdir = get_command_and_outdir( self, func_name=sys._getframe().f_code.co_name, file=__file__) argv_str = f""" --tl_config_file template_lib/modelarts/tests/configs/run.yaml --tl_command {command} --tl_outdir {outdir} """ args = setup_outdir_and_yaml(argv_str) n_gpus = len(os.environ['CUDA_VISIBLE_DEVICES'].split(',')) cmd_str = f""" python template_lib/modelarts/scripts/run.py {get_append_cmd_str(args)} --number 1 """ start_cmd_run(cmd_str) pass
def test_case_evaluate_FID_IS(self): """ Usage: export PYTHONWARNINGS=ignore export CUDA_VISIBLE_DEVICES=0,7 export master_port=8887 export TIME_STR=1 export PYTHONPATH=./ python -c "from template_lib.v2.tests.test_GAN import TestingTorchFIDISScore;\ TestingTorchFIDISScore().test_case_evaluate_FID_IS()" :return: """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '0,7' if 'TIME_STR' not in os.environ: os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0' command, outdir = get_command_and_outdir( self, func_name=sys._getframe().f_code.co_name, file=__file__) argv_str = f""" --tl_config_file none --tl_command none --tl_outdir {outdir} """ args = setup_outdir_and_yaml(argv_str) nproc_per_node = len(os.environ['CUDA_VISIBLE_DEVICES'].split(',')) master_port = os.environ.get('master_port', 8887) cmd_str = f""" python -m torch.distributed.launch --nproc_per_node={nproc_per_node} --master_port={master_port} template_lib/v2/GAN/evaluation/pytorch_FID_IS_score.py --run_func PyTorchFIDISScore.test_case_evaluate_FID_IS """ cmd_str += get_append_cmd_str(args) start_cmd_run(cmd_str) pass
def test_diff_lr(self, debug=True): """ Usage: python template_lib/modelarts/scripts/copy_tool.py \ -s s3://bucket-7001/ZhouPeng/pypi/torch1_7_0 -d /cache/pypi -t copytree for filename in /cache/pypi/*.whl; do pip install $filename done proj_root=moco-exp python template_lib/modelarts/scripts/copy_tool.py \ -s s3://bucket-7001/ZhouPeng/codes/$proj_root -d /cache/$proj_root -t copytree -b /cache/$proj_root/code.zip cd /cache/$proj_root pip install -r requirements.txt export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 export TIME_STR=1 export PYTHONPATH=./exp:./stylegan2-pytorch:./ python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\ Testing_stylegan2().test_train_ffhq_128()" :return: """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '0' if 'TIME_STR' not in os.environ: os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0' from template_lib.v2.config_cfgnode.argparser import \ (get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run) tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') + 1:]) if '--tl_opts' in sys.argv else '' print(f'tl_opts:\n {tl_opts}') command, outdir = get_command_and_outdir( self, func_name=sys._getframe().f_code.co_name, file=__file__) argv_str = f""" --tl_config_file none --tl_command none --tl_outdir {outdir} """ args = setup_outdir_and_yaml(argv_str, return_cfg=True) from torch.optim import SGD from torch import nn class DummyModel(nn.Module): def __init__(self, class_num=10): super(DummyModel, self).__init__() self.base = nn.Sequential( nn.Conv2d(3, 64, kernel_size=3, padding=1), nn.ReLU(), nn.Conv2d(64, 128, kernel_size=3, padding=1), nn.ReLU(), ) self.gap = nn.AdaptiveAvgPool2d(1) self.fc = nn.Linear(128, class_num) def forward(self, x): x = self.base(x) x = self.gap(x) x = x.view(x.shape[0], -1) x = self.fc(x) return x model = DummyModel().cuda() optimizer = SGD( [ { 'params': model.base.parameters() }, { 'params': model.fc.parameters(), 'lr': 1e-3 } # 对 fc的参数设置不同的学习率 ], lr=1e-2, momentum=0.9) pass
def test_step_closure(self, debug=True): """ Usage: python template_lib/modelarts/scripts/copy_tool.py \ -s s3://bucket-7001/ZhouPeng/pypi/torch1_7_0 -d /cache/pypi -t copytree for filename in /cache/pypi/*.whl; do pip install $filename done proj_root=moco-exp python template_lib/modelarts/scripts/copy_tool.py \ -s s3://bucket-7001/ZhouPeng/codes/$proj_root -d /cache/$proj_root -t copytree -b /cache/$proj_root/code.zip cd /cache/$proj_root pip install -r requirements.txt export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 export TIME_STR=1 export PYTHONPATH=./exp:./stylegan2-pytorch:./ python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\ Testing_stylegan2().test_train_ffhq_128()" :return: """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '0' if 'TIME_STR' not in os.environ: os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0' from template_lib.v2.config_cfgnode.argparser import \ (get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run) tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') + 1:]) if '--tl_opts' in sys.argv else '' print(f'tl_opts:\n {tl_opts}') command, outdir = get_command_and_outdir( self, func_name=sys._getframe().f_code.co_name, file=__file__) argv_str = f""" --tl_config_file none --tl_command none --tl_outdir {outdir} """ args = setup_outdir_and_yaml(argv_str, return_cfg=True) from torch.nn import CrossEntropyLoss class DummyModel(nn.Module): def __init__(self, class_num=10): super(DummyModel, self).__init__() self.base = nn.Sequential( nn.Conv2d(3, 64, kernel_size=3, padding=1), nn.ReLU(), nn.Conv2d(64, 128, kernel_size=3, padding=1), nn.ReLU(), ) self.gap = nn.AdaptiveAvgPool2d(1) self.fc = nn.Linear(128, class_num) def forward(self, x): x = self.base(x) x = self.gap(x) x = x.view(x.shape[0], -1) x = self.fc(x) return x dummy_model = DummyModel().cuda() optimizer = SGD(dummy_model.parameters(), lr=1e-2, momentum=0.9, weight_decay=1e-4) # 定义loss loss_fn = CrossEntropyLoss() # 定义数据 batch_size = 2 data = torch.randn(64, 3, 64, 128).cuda() # 制造假数据shape=64 * 3 * 64 * 128 data_label = torch.randint(0, 10, size=(64, ), dtype=torch.long).cuda() # 制造假的label for batch_index in range(10): batch_data = data[batch_index * batch_size:batch_index * batch_size + batch_size] batch_label = data_label[batch_index * batch_size:batch_index * batch_size + batch_size] def closure(): optimizer.zero_grad() # 清空梯度 output = dummy_model(batch_data) # forward loss = loss_fn(output, batch_label) # 计算loss loss.backward() # backward print('No.{: 2d} loss: {:.6f}'.format(batch_index, loss.item())) return loss optimizer.step(closure=closure) # 更新参数 pass
def test_OneCycleLR(self, debug=True): """ Usage: python template_lib/modelarts/scripts/copy_tool.py \ -s s3://bucket-7001/ZhouPeng/pypi/torch1_7_0 -d /cache/pypi -t copytree for filename in /cache/pypi/*.whl; do pip install $filename done proj_root=moco-exp python template_lib/modelarts/scripts/copy_tool.py \ -s s3://bucket-7001/ZhouPeng/codes/$proj_root -d /cache/$proj_root -t copytree -b /cache/$proj_root/code.zip cd /cache/$proj_root pip install -r requirements.txt export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 export TIME_STR=1 export PYTHONPATH=./exp:./stylegan2-pytorch:./ python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\ Testing_stylegan2().test_train_ffhq_128()" :return: """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '0' if 'TIME_STR' not in os.environ: os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0' from template_lib.v2.config_cfgnode.argparser import \ (get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run) tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') + 1:]) if '--tl_opts' in sys.argv else '' print(f'tl_opts:\n {tl_opts}') command, outdir = get_command_and_outdir( self, func_name=sys._getframe().f_code.co_name, file=__file__) argv_str = f""" --tl_config_file none --tl_command none --tl_outdir {outdir} """ args = setup_outdir_and_yaml(argv_str, return_cfg=True) import torch.nn as nn from torch.optim import lr_scheduler from matplotlib import pyplot as plt model = nn.Linear(3, 64) def create_optimizer(): return SGD(model.parameters(), lr=0.1, momentum=0.9, weight_decay=1e-4) def plot_lr(scheduler, title='', labels=['base'], nrof_epoch=100): lr_li = [[] for _ in range(len(labels))] epoch_li = list(range(nrof_epoch)) for epoch in epoch_li: scheduler.step() # 调用step()方法,计算和更新optimizer管理的参数基于当前epoch的学习率 lr = scheduler.get_last_lr() # 获取当前epoch的学习率 for i in range(len(labels)): lr_li[i].append(lr[i]) for lr, label in zip(lr_li, labels): plt.plot(epoch_li, lr, label=label) plt.grid() plt.xlabel('epoch') plt.ylabel('lr') plt.title(title) plt.legend() plt.show() optimizer = create_optimizer() scheduler = lr_scheduler.OneCycleLR(optimizer, 0.1, total_steps=100) plot_lr(scheduler, title='OneCycleLR') pass
def test_base_usage(self, debug=True): """ Usage: python template_lib/modelarts/scripts/copy_tool.py \ -s s3://bucket-7001/ZhouPeng/pypi/torch1_7_0 -d /cache/pypi -t copytree for filename in /cache/pypi/*.whl; do pip install $filename done proj_root=moco-exp python template_lib/modelarts/scripts/copy_tool.py \ -s s3://bucket-7001/ZhouPeng/codes/$proj_root -d /cache/$proj_root -t copytree -b /cache/$proj_root/code.zip cd /cache/$proj_root pip install -r requirements.txt export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 export TIME_STR=1 export PYTHONPATH=./exp:./stylegan2-pytorch:./ python -c "from exp.tests.test_styleganv2 import Testing_stylegan2;\ Testing_stylegan2().test_train_ffhq_128()" :return: """ if 'CUDA_VISIBLE_DEVICES' not in os.environ: os.environ['CUDA_VISIBLE_DEVICES'] = '0' if 'TIME_STR' not in os.environ: os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0' from template_lib.v2.config_cfgnode.argparser import \ (get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run) tl_opts = ' '.join(sys.argv[sys.argv.index('--tl_opts') + 1:]) if '--tl_opts' in sys.argv else '' print(f'tl_opts:\n {tl_opts}') command, outdir = get_command_and_outdir( self, func_name=sys._getframe().f_code.co_name, file=__file__) argv_str = f""" --tl_config_file none --tl_command none --tl_outdir {outdir} """ args = setup_outdir_and_yaml(argv_str, return_cfg=True) import torch import numpy as np import warnings warnings.filterwarnings('ignore') # ignore warnings x = torch.linspace(-np.pi, np.pi, 2000) y = torch.sin(x) p = torch.tensor([1, 2, 3]) xx = x.unsqueeze(-1).pow(p) model = torch.nn.Sequential(torch.nn.Linear(3, 1), torch.nn.Flatten(0, 1)) loss_fn = torch.nn.MSELoss(reduction='sum') learning_rate = 1e-3 optimizer = torch.optim.RMSprop(model.parameters(), lr=learning_rate) for t in range(1, 1001): y_pred = model(xx) loss = loss_fn(y_pred, y) if t % 100 == 0: print('No.{: 5d}, loss: {:.6f}'.format(t, loss.item())) optimizer.zero_grad() # 梯度清零 loss.backward() # 反向传播计算梯度 optimizer.step() # 梯度下降法更新参数 pass