def test_extract_ImageNet100_CMC(self):
"""
Usage:
proj_root=moco-exp
python template_lib/modelarts/scripts/copy_tool.py \
-s s3://bucket-7001/ZhouPeng/codes/$proj_root -d /cache/$proj_root -t copytree
cd /cache/$proj_root
export CUDA_VISIBLE_DEVICES=0
export TIME_STR=0
export PYTHONPATH=./
python -c "from template_lib.proj.imagenet.tests.test_imagenet import Testing_PrepareImageNet;\
Testing_PrepareImageNet().test_extract_ImageNet100_CMC()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
from template_lib.v2.config_cfgnode import update_parser_defaults_from_yaml, global_cfg
from template_lib.modelarts import modelarts_utils
from distutils.dir_util import copy_tree
command, outdir = get_command_and_outdir(self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file template_lib/proj/imagenet/tests/configs/PrepareImageNet.yaml
--tl_command {command}
--tl_outdir {outdir}
"""
args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True)
modelarts_utils.setup_tl_outdir_obs(global_cfg)
modelarts_utils.modelarts_sync_results_dir(global_cfg, join=True)
modelarts_utils.prepare_dataset(global_cfg.get('modelarts_download', {}), global_cfg=global_cfg)
train_dir = f'{cfg.data_dir}/train'
val_dir = f'{cfg.data_dir}/val'
save_train_dir = f'{cfg.saved_dir}/train'
save_val_dir = f'{cfg.saved_dir}/val'
os.makedirs(save_train_dir, exist_ok=True)
os.makedirs(save_val_dir, exist_ok=True)
with open(cfg.class_list_file, 'r') as f:
class_list = f.readlines()
for class_subdir in tqdm.tqdm(class_list):
class_subdir, _ = class_subdir.strip().split()
train_class_dir = f'{train_dir}/{class_subdir}'
save_train_class_dir = f'{save_train_dir}/{class_subdir}'
copy_tree(train_class_dir, save_train_class_dir)
val_class_dir = f'{val_dir}/{class_subdir}'
save_val_class_dir = f'{save_val_dir}/{class_subdir}'
copy_tree(val_class_dir, save_val_class_dir)
modelarts_utils.prepare_dataset(global_cfg.get('modelarts_upload', {}), global_cfg=global_cfg, download=False)
modelarts_utils.modelarts_sync_results_dir(global_cfg, join=True)
pass
def test_extract_ImageNet_1000x50(self):
"""
Usage:
proj_root=moco-exp
python template_lib/modelarts/scripts/copy_tool.py \
-s s3://bucket-7001/ZhouPeng/codes/$proj_root -d /cache/$proj_root -t copytree
cd /cache/$proj_root
export CUDA_VISIBLE_DEVICES=0
export TIME_STR=0
export PYTHONPATH=./
python -c "from template_lib.proj.imagenet.tests.test_imagenet import Testing_PrepareImageNet;\
Testing_PrepareImageNet().test_extract_ImageNet_1000x50()"
:return:
"""
if 'CUDA_VISIBLE_DEVICES' not in os.environ:
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
if 'TIME_STR' not in os.environ:
os.environ['TIME_STR'] = '0' if utils.is_debugging() else '0'
from template_lib.v2.config_cfgnode.argparser import \
(get_command_and_outdir, setup_outdir_and_yaml, get_append_cmd_str, start_cmd_run)
from template_lib.v2.config_cfgnode import update_parser_defaults_from_yaml, global_cfg
from template_lib.modelarts import modelarts_utils
command, outdir = get_command_and_outdir(self, func_name=sys._getframe().f_code.co_name, file=__file__)
argv_str = f"""
--tl_config_file template_lib/proj/imagenet/tests/configs/PrepareImageNet.yaml
--tl_command {command}
--tl_outdir {outdir}
"""
args, cfg = setup_outdir_and_yaml(argv_str, return_cfg=True)
global_cfg.merge_from_dict(cfg)
global_cfg.merge_from_dict(vars(args))
modelarts_utils.setup_tl_outdir_obs(global_cfg)
modelarts_utils.modelarts_sync_results_dir(global_cfg, join=True)
modelarts_utils.prepare_dataset(global_cfg.get('modelarts_download', {}), global_cfg=global_cfg)
train_dir = f'{cfg.data_dir}/train'
counter_cls = 0
for rootdir, subdir, files in os.walk(train_dir):
if len(subdir) == 0:
counter_cls += 1
extracted_files = sorted(files)[:cfg.num_per_class]
for file in tqdm.tqdm(extracted_files, desc=f'class: {counter_cls}'):
img_path = os.path.join(rootdir, file)
img_rel_path = os.path.relpath(img_path, cfg.data_dir)
saved_img_path = f'{cfg.saved_dir}/{os.path.dirname(img_rel_path)}'
os.makedirs(saved_img_path, exist_ok=True)
shutil.copy(img_path, saved_img_path)
pass
modelarts_utils.prepare_dataset(global_cfg.get('modelarts_upload', {}), global_cfg=global_cfg, download=False)
modelarts_utils.modelarts_sync_results_dir(global_cfg, join=True)
pass
def main():
parser = build_parser()
args, _ = parser.parse_known_args()
is_main_process = args.local_rank == 0
update_parser_defaults_from_yaml(parser, is_main_process=is_main_process)
if is_main_process:
modelarts_utils.setup_tl_outdir_obs(global_cfg)
modelarts_utils.modelarts_sync_results_dir(global_cfg, join=True)
modelarts_utils.prepare_dataset(global_cfg.get('modelarts_download', {}), global_cfg=global_cfg)
args = parser.parse_args()
setup_runtime(seed=args.seed)
distributed = ddp_utils.is_distributed()
if distributed:
dist_utils.init_dist(args.launcher, backend='nccl')
# important: use different random seed for different process
torch.manual_seed(args.seed + dist.get_rank())
# dataset
dataset = torch_data_utils.ImageListDataset(meta_file=global_cfg.image_list_file, )
if distributed:
sampler = torch.utils.data.distributed.DistributedSampler(dataset, shuffle=False)
else:
sampler = None
train_loader = data_utils.DataLoader(
dataset,
batch_size=1,
shuffle=False,
sampler=sampler,
num_workers=args.num_workers,
pin_memory=False)
# test
data_iter = iter(train_loader)
data = next(data_iter)
if is_main_process:
modelarts_utils.prepare_dataset(global_cfg.get('modelarts_upload', {}), global_cfg=global_cfg, download=False)
modelarts_utils.modelarts_sync_results_dir(global_cfg, join=True)
if distributed:
dist.barrier()
pass
def main():
update_parser_defaults_from_yaml(parser)
args = parser.parse_args()
global_cfg.merge_from_dict(vars(args))
modelarts_utils.setup_tl_outdir_obs(global_cfg)
modelarts_utils.modelarts_sync_results_dir(global_cfg, join=True)
modelarts_utils.prepare_dataset(global_cfg.get('modelarts_download', {}),
global_cfg=global_cfg)
if args.seed is not None:
random.seed(args.seed)
torch.manual_seed(args.seed)
cudnn.deterministic = True
warnings.warn('You have chosen to seed training. '
'This will turn on the CUDNN deterministic setting, '
'which can slow down your training considerably! '
'You may see unexpected behavior when restarting '
'from checkpoints.')
if args.gpu is not None:
warnings.warn('You have chosen a specific GPU. This will completely '
'disable data parallelism.')
if args.dist_url == "env://" and args.world_size == -1:
args.world_size = int(os.environ["WORLD_SIZE"])
args.distributed = args.world_size > 1 or args.multiprocessing_distributed
ngpus_per_node = torch.cuda.device_count()
if args.multiprocessing_distributed:
# Since we have ngpus_per_node processes per node, the total world_size
# needs to be adjusted accordingly
args.world_size = ngpus_per_node * args.world_size
# Use torch.multiprocessing.spawn to launch distributed processes: the
# main_worker process function
mp.spawn(main_worker,
nprocs=ngpus_per_node,
args=(ngpus_per_node, args))
else:
# Simply call main_worker function
main_worker(args.gpu, ngpus_per_node, args)