def convert_py(self, runstate, mode=0):
# manuvision config
mv_config_file = "ainnovision_train.yaml"
mv_config_path = os.path.join(self.py_dir, mv_config_file)
mvcfg = Config.fromfile(mv_config_path)
# mmseg config
# mm_config_path = '../configs/pspnet/pspnet_r50-d8_yantai_st12.py'
mm_config_file = "mm_det.py"
mm_config_path = os.path.join(self.py_dir, mm_config_file)
mmcfg = Config.fromfile(mm_config_path)
cfg = merge_to_mmcfg_from_mvcfg(mmcfg, mvcfg)
checkpath = osp.join(cfg.work_dir, 'F1_best_model.pth.tar')
checkpoint = torch.load(checkpath, map_location='cpu')
model_cfg = checkpoint['meta']["config"].model
# build the model and load checkpoint
model_cfg.pretrained = None
segmentor = build_segmentor(model_cfg,
train_cfg=None,
test_cfg=cfg.test_cfg)
# convert SyncBN to BN
segmentor = _convert_batchnorm(segmentor)
checkpoint = load_checkpoint(segmentor, checkpath, map_location='cpu')
# conver model to onnx file
input_shape = (1, 3, cfg.convert_size[0], cfg.convert_size[1])
output_file = osp.join(cfg.work_dir, 'F1_best_model.onnx')
pytorch2onnx(segmentor,
input_shape,
opset_version=10,
show=True,
output_file=output_file,
verify=True)
def get_model(args, classes=12, train=True):
repo_root = args.repo_root
if args.network[-1] == 's':
cfg = Config.fromfile(
os.path.join(
repo_root,
'swin/configs/swin/upernet_swin_small_patch4_window7_512x512_160k_ade20k.py'
))
ck_path = os.path.join(
repo_root,
'pretrained/upernet_swin_small_patch4_window7_512x512.pth')
elif args.network[-1] == 'b':
cfg = Config.fromfile(
os.path.join(
repo_root,
'swin/configs/swin/upernet_swin_base_patch4_window7_512x512_160k_ade20k.py'
))
ck_path = os.path.join(
repo_root,
'pretrained/upernet_swin_base_patch4_window7_512x512.pth')
else:
cfg = Config.fromfile(
os.path.join(
repo_root,
'swin/configs/swin/upernet_swin_tiny_patch4_window7_512x512_160k_ade20k.py'
))
ck_path = os.path.join(
repo_root,
'pretrained/upernet_swin_tiny_patch4_window7_512x512.pth')
if train:
return load_swin(cfg, ck_path)
else:
return make_swin_model(cfg)
def inference_py(self, runstate):
# manuvision config
mv_config_file = "ainnovision_train.yaml"
mv_config_path = os.path.join(self.py_dir, mv_config_file)
mvcfg = Config.fromfile(mv_config_path)
# mmseg config
# mm_config_path = '../configs/pspnet/pspnet_r50-d8_yantai_st12.py'
mm_config_file = "mm_det.py"
mm_config_path = os.path.join(self.py_dir, mm_config_file)
mmcfg = Config.fromfile(mm_config_path)
cfg = merge_to_mmcfg_from_mvcfg(mmcfg, mvcfg)
# set cudnn_benchmark
if cfg.get('cudnn_benchmark', False):
torch.backends.cudnn.benchmark = True
cfg.model.pretrained = None
cfg.data.test.test_mode = True
# init distributed env first, since logger depends on the dist info.
if cfg.get('launcher', 'none') == 'none' or len(cfg.gpu_ids) == 1:
distributed = False
else:
distributed = True
init_dist(cfg.launcher, **cfg.dist_params)
# build the dataloader
# TODO: support multiple images per gpu (only minor changes are needed)
dataset = build_dataset(cfg.data.test)
data_loader = build_dataloader(dataset,
samples_per_gpu=1,
workers_per_gpu=0,
dist=distributed,
shuffle=False)
# build the model and load checkpoint
model = build_segmentor(cfg.model,
train_cfg=None,
test_cfg=cfg.test_cfg)
model_path = osp.join(cfg.work_dir, 'F1_best_model.pth.tar')
checkpoint = load_checkpoint(model, model_path, map_location='cpu')
config = checkpoint['meta']['config']
if not distributed:
model = MMDataParallel(model, device_ids=[0])
mv_single_gpu_test(model,
data_loader,
runstate=runstate,
draw_contours=True,
out_dir=cfg.data_root)
def __init__(self,
# num_stages,
backbone,
decode_head,
neck=None,
auxiliary_head=None,
train_cfg=None,
test_cfg=None,
pretrained=None,
with_aux_kd=None,
teacher_config=None):
# self.num_stages = num_stages
super(KDEncoderDecoder, self).__init__(
backbone=backbone,
decode_head=decode_head,
neck=neck,
auxiliary_head=auxiliary_head,
train_cfg=train_cfg,
test_cfg=test_cfg,
pretrained=pretrained)
# self.with_aux_kd = False
self.with_aux_kd = with_aux_kd
if teacher_config:
T_cfg = Config.fromfile(teacher_config.config_file)
self.teacher = build_segmentor(T_cfg.model, train_cfg=None, test_cfg=None)
checkpoint = load_checkpoint(self.teacher, teacher_config.checkpoint)
self.teacher.eval()
def __call__(self, config, gpu_num, out, update_config, tensorboard_dir):
logging.basicConfig(level=logging.INFO)
logging.info(f'Commandline:\n{" ".join(sys.argv)}')
cfg = Config.fromfile(config)
# This is required to skip the parameters that were not set in the template
# (e.g. base_learning_rate or epochs) -- they will have default value None in
# the parser
update_config = {
k: v
for k, v in update_config.items() if v is not None
}
self.work_dir = update_config.get(self.parameter_work_dir)
update_config = ' '.join(
[f'{k}={v}' for k, v in update_config.items()])
update_config = f' --update_config {update_config}' if update_config else ''
update_config = self._add_extra_args(cfg, config, update_config)
logging.info('Training started ...')
training_info = self._train_internal(config, gpu_num, update_config,
tensorboard_dir)
logging.info('... training completed.')
with open(out, 'a+') as dst_file:
yaml.dump(training_info, dst_file)
def main():
""" Main function. """
logging.basicConfig(level=logging.INFO)
args = parse_args()
logging.info(f'Commandline:\n{" ".join(sys.argv)}')
cfg = Config.fromfile(args.config)
update_config = f' --update_config {args.update_config}' if args.update_config else ''
if is_clustering_needed(cfg):
update_config = cluster(cfg, args.config, update_config)
logging.info('Training started ...')
training_info = train(args.config, args.gpu_num, update_config)
logging.info('... training completed.')
work_dir = get_work_dir(cfg, args.update_config)
logging.info('Evaluation started ...')
evaluate(os.path.join(work_dir, "config.py"),
os.path.join(work_dir, "latest.pth"), args.out, '', args.show_dir)
logging.info('... evaluation completed.')
with open(args.out, 'a+') as dst_file:
yaml.dump(training_info, dst_file)
def _export_to_openvino(self, args, tools_dir):
run_through_shell(f'python {os.path.join(tools_dir, "export.py")} '
f'{args["config"]} '
f'{args["load_weights"]} '
f'{args["save_model_to"]} '
f'openvino '
f'--input_format {args["openvino_input_format"]}')
# FIXME(ikrylov): remove alt_ssd_export block as soon as it becomes useless.
# (LeonidBeynenson): Please, note that alt_ssd_export appoach may be applied only
# to SSD models only that were not compressed by NNCF.
config = Config.fromfile(args["config"])
should_run_alt_ssd_export = (hasattr(config.model, 'bbox_head') and
config.model.bbox_head.type == 'SSDHead'
and not config.get('nncf_config'))
if is_checkpoint_nncf and is_checkpoint_nncf(args['load_weights']):
# If the config does not contain NNCF part,
# but the checkpoint was trained with NNCF compression,
# the NNCF config will be read from checkpoint.
# Since alt_ssd_export is incompatible with NNCF compression,
# alt_ssd_export should not be run in this case.
should_run_alt_ssd_export = False
if should_run_alt_ssd_export:
run_through_shell(
f'python {os.path.join(tools_dir, "export.py")} '
f'{args["config"]} '
f'{args["load_weights"]} '
f'{os.path.join(args["save_model_to"], "alt_ssd_export")} '
f'openvino '
f'--input_format {args["openvino_input_format"]} '
f'--alt_ssd_export ')
def __init__(
self,
image_size,
crop_size,
split,
config_path,
normalization,
**kwargs,
):
super().__init__()
self.image_size = image_size
self.crop_size = crop_size
self.split = split
self.normalization = STATS[normalization].copy()
self.ignore_label = None
for k, v in self.normalization.items():
v = np.round(255 * np.array(v), 2)
self.normalization[k] = tuple(v)
print(f"Use normalization: {self.normalization}")
config = Config.fromfile(config_path)
self.ratio = config.max_ratio
self.dataset = None
self.config = self.update_default_config(config)
self.dataset = build_dataset(getattr(self.config.data,
f"{self.split}"))
def _export_to_openvino(self, args, tools_dir):
update_config = self._get_update_config(args)
run_through_shell(f'python3 {os.path.join(tools_dir, "export.py")} '
f'{args["config"]} '
f'{args["load_weights"]} '
f'{args["save_model_to"]} '
f'{update_config} '
f'--opset={self.opset} '
f'openvino '
f'--input_format {args["openvino_input_format"]}')
# FIXME(ikrylov): remove alt_ssd_export block as soon as it becomes useless.
config = Config.fromfile(args["config"])
should_run_alt_ssd_export = (hasattr(config.model, 'bbox_head') and
config.model.bbox_head.type == 'SSDHead')
if should_run_alt_ssd_export:
run_through_shell(
f'python3 {os.path.join(tools_dir, "export.py")} '
f'{args["config"]} '
f'{args["load_weights"]} '
f'{os.path.join(args["save_model_to"], "alt_ssd_export")} '
f'{update_config} '
f'--opset={self.opset} '
f'openvino '
f'--input_format {args["openvino_input_format"]} '
f'--alt_ssd_export ')
def _export_to_openvino(self, args, tools_dir):
config = Config.fromfile(args["config"])
height, width = self._get_input_shape(config)
run(f'python {os.path.join(tools_dir, "export.py")} '
f'{args["config"]} '
f'{args["load_weights"]} '
f'{args["save_model_to"]} '
f'openvino '
f'--input_format {args["openvino_input_format"]} '
f'--input_shape {height} {width}',
shell=True,
check=True)
def train(config, gpu_num, out, update_config, tensorboard_dir):
""" Main function. """
logging.basicConfig(level=logging.INFO)
logging.info(f'Commandline:\n{" ".join(sys.argv)}')
cfg = Config.fromfile(config)
update_config = ' '.join([f'{k}={v}' for k, v in update_config.items()])
update_config = f' --update_config {update_config}' if update_config else ''
if is_clustering_needed(cfg):
update_config = cluster(cfg, config, update_config)
logging.info('Training started ...')
train_internal(config, gpu_num, update_config, tensorboard_dir)
logging.info('... training completed.')
def updated_config_file(WORK_DIR, expr_name, base_config_file, save_path,
para_file, network_setting_ini, gpu_num):
'''
update mmseg config file and save to a new file in local working folder
:param WORK_DIR:
:param expr_name:
:param base_config_file:
:param save_path:
:param para_file:
:param network_setting_ini:
:return:
'''
# read the config, then modifi some, them dump to disk
cfg = Config.fromfile(base_config_file)
# 4 basic component: dataset, model, schedule, default_runtime
# change model (no need): when we choose a base_config_file, we already choose a model, including backbone)
# change dataset
modify_dataset(cfg, para_file, network_setting_ini, gpu_num)
# change num_classes
# NUM_CLASSES_noBG = parameters.get_digit_parameters(para_file,'NUM_CLASSES_noBG','int')
# cfg.model.decode_head['num_classes'] = NUM_CLASSES_noBG + 1
# cfg.model.auxiliary_head['num_classes'] = NUM_CLASSES_noBG + 1
# change schedule
iteration_num = get_iteration_num(WORK_DIR, para_file, network_setting_ini)
cfg.runner['max_iters'] = iteration_num
checkpoint_interval = parameters.get_digit_parameters(
network_setting_ini, 'checkpoint_interval', 'int')
cfg.checkpoint_config['interval'] = checkpoint_interval
evaluation_interval = parameters.get_digit_parameters(
network_setting_ini, 'evaluation_interval', 'int')
cfg.evaluation['interval'] = evaluation_interval
# change runtime (log level, resume_from or load_from)
cfg.work_dir = os.path.join(WORK_DIR, expr_name)
# update parameters for testing (used in later step of prediction)
updated_config_file_for_test(cfg)
# dump config
cfg.dump(save_path)
return True
def _export_to_openvino(self, args, tools_dir):
super()._export_to_openvino(args, tools_dir)
# FIXME(ikrylov): remove alt_ssd_export block as soon as it becomes useless.
# (LeonidBeynenson): Please, note that alt_ssd_export appoach may be applied only
# to SSD models only that were not compressed by NNCF.
config = Config.fromfile(args["config"])
should_run_alt_ssd_export = (hasattr(config.model, 'bbox_head')
and config.model.bbox_head.type == 'SSDHead'
and not config.get('nncf_config'))
if should_run_alt_ssd_export:
run_through_shell(f'python {os.path.join(tools_dir, "export.py")} '
f'{args["config"]} '
f'{args["load_weights"]} '
f'{os.path.join(args["save_model_to"], "alt_ssd_export")} '
f'openvino '
f'--input_format {args["openvino_input_format"]} '
f'--alt_ssd_export ')
def main():
args = parse_args()
cfg = Config.fromfile(args.config)
if args.cfg_options is not None:
cfg.merge_from_dict(args.cfg_options)
# set cudnn_benchmark
if cfg.get('cudnn_benchmark', False):
torch.backends.cudnn.benchmark = True
cfg.model.pretrained = None # set pretrained to None
# init distributed env first
if args.launcher == 'none':
distributed = False
else:
distributed = True
init_dist(args.launcher, **cfg.dist_params)
# build the data loader
dataset = build_dataset(cfg.data.test)
data_loader = build_dataloader(dataset,
samples_per_gpu=cfg.data.samples_per_gpu,
workers_per_gpu=cfg.data.workers_per_gpu,
dist=distributed,
shuffle=False,
sampler=None)
# build the model and load checkpoint
model = build_reid(cfg.model)
load_checkpoint(model, args.checkpoint, map_location='cpu')
if not distributed:
model = MMDataParallel(model, device_ids=[0])
else:
model = MMDistributedDataParallel(
model.cuda(),
device_ids=[torch.cuda.current_device()],
broadcast_buffers=False)
eval_cfg = cfg.get('evaluation', {})
eval_cfg['progress'] = True # always show ProgressBar
evaluator = Evaluator(**eval_cfg)
evaluator.eval(model, data_loader)
def get_cfg():
# OK args_config = "C:/_koray/korhun/mmsegmentation/configs/deeplabv3plus/deeplabv3plus_r50-d8_512x1024_80k_cityscapes_koray.py"
# args_config = "C:/_koray/korhun/mmsegmentation/configs/deeplabv3plus/deeplabv3plus_r50-d8_512x1024_80k_cityscapes_koray_3.py"
# args_config = "C:/_koray/korhun/mmsegmentation/configs/pspnet/pspnet_r50-d8_512x1024_40k_cityscapes.py"
# args_config = "C:/_koray/korhun/mmsegmentation/configs/pspnet/pspnet_koray.py"
# args_config = "C:/_koray/korhun/mmsegmentation/configs/ocrnet/ocrnet_hr18_512x1024_160k_cityscapes_koray.py"
# args_config = "C:/_koray/korhun/mmsegmentation/configs/ocrnet/ocrnet_hr18_512x1024_160k_cityscapes_koray.py"
args_config = "C:/_koray/korhun/mmsegmentation/configs/ocrnet/ocrnet_hr48_512x1024_160k_custom_koray.py"
if not os.path.isfile(args_config):
print("File does not exists: " + args_config)
exit(1)
cfg = Config.fromfile(args_config)
print(f'Config:\n{cfg.pretty_text}')
return cfg, args_config
def _evaluate_internal(self, config_path, snapshot, out, update_config,
metrics_functions, **kwargs):
assert isinstance(update_config, dict)
cfg = Config.fromfile(config_path)
work_dir = tempfile.mkdtemp()
print('results are stored in:', work_dir)
if os.path.islink(snapshot):
snapshot = os.path.join(os.path.dirname(snapshot),
os.readlink(snapshot))
metrics = []
metrics.extend(
self._get_complexity_and_size(cfg, config_path, work_dir,
update_config))
metric_args = {
'config_path': config_path,
'work_dir': work_dir,
'snapshot': snapshot,
'update_config': update_config
}
metric_args.update(kwargs)
for func in metrics_functions:
metrics.extend(func(**metric_args))
outputs = {
'files': [get_file_size_and_sha256(snapshot)],
'metrics': self._round_metrics(metrics)
}
if os.path.exists(out):
with open(out) as read_file:
content = yaml.load(read_file, Loader=yaml.SafeLoader)
content.update(outputs)
outputs = content
with open(out, 'w') as write_file:
yaml.dump(outputs, write_file)
def main():
""" Main function. """
args = parse_args()
if args.wider_dir:
wider_val_zip = os.path.join(args.wider_dir, 'WIDER_val.zip')
assert os.path.exists(
wider_val_zip
), f'failed to find WIDER_val.zip here: {wider_val_zip}'
mmdetection_tools = f'{os.path.dirname(__file__)}/../../../../external/mmdetection/tools'
subprocess.run(f'{mmdetection_tools}/dist_train.sh'
f' {args.config}'
f' {args.gpu_num}'.split(' '),
check=True)
cfg = Config.fromfile(args.config)
eval(args.config, os.path.join(cfg.work_dir, "latest.pth"), args.wider_dir,
args.out)
def init_twodimestimator(config, checkpoint=None, device='cpu'):
if isinstance(config, str):
config = Config.fromfile(config)
config = config.processor_cfg
elif isinstance(config, OrderedDict):
config = config
else:
raise ValueError(
'Input config type is: {}, expect "str" or "Orderdict"'.format(
type(config)))
model_cfg = config.model_cfg
if isinstance(model_cfg, list):
model = [call_obj(**c) for c in model_cfg]
model = torch.nn.Sequential(*model)
else:
model = call_obj(**model_cfg)
load_checkpoint(model, checkpoint, map_location=device)
model.to(device)
model = model.eval()
return model
def eval(config_path, snapshot, wider_dir, out):
""" Main evaluation procedure. """
cfg = Config.fromfile(config_path)
work_dir = tempfile.mkdtemp()
print('results are stored in:', work_dir)
if os.path.islink(snapshot):
snapshot = os.path.join(os.path.dirname(snapshot), os.readlink(snapshot))
files = get_file_size_and_sha256(snapshot)
metrics = []
res_pkl = os.path.join(work_dir, "res.pkl")
metrics = coco_ap_eval(config_path, work_dir, snapshot, res_pkl, metrics)
metrics = custom_ap_eval(config_path, work_dir, res_pkl, metrics)
if wider_dir:
metrics = compute_wider_metrics(config_path, snapshot, work_dir, wider_dir, metrics)
metrics = get_complexity_and_size(cfg, config_path, work_dir, metrics)
for metric in metrics:
metric['value'] = round(metric['value'], 3)
outputs = {
'files': [files],
'metrics': metrics
}
if os.path.exists(out):
with open(out) as read_file:
content = yaml.load(read_file)
content.update(outputs)
outputs = content
with open(out, 'w') as write_file:
yaml.dump(outputs, write_file)
def _update_configuration_file(self, config_path, update_config):
cfg = Config.fromfile(config_path)
if 'classes' in update_config:
classes = update_config['classes']
update_config.pop('classes')
else:
annotation_file = cfg.data.train.dataset.ann_file
ann_file_key = 'data.train.dataset.ann_file'
if ann_file_key in update_config:
annotation_file = update_config[ann_file_key]
annotation_file = annotation_file.split(',')
if isinstance(annotation_file, (list, tuple)):
annotation_file = annotation_file[0]
with open(annotation_file) as read_file:
categories = sorted(json.load(read_file)['categories'],
key=lambda x: x['id'])
classes = [category_dict['name'] for category_dict in categories]
update_config_dict = self.classes_list_to_update_config_dict(
cfg, classes)
cfg.merge_from_dict(update_config_dict)
cfg.dump(
config_path) # TODO(lbeynens): check this, it may be dangerous
return update_config
from mmcv.utils import Config
cfg = Config.fromfile('config_b.py')
print(cfg)
def main():
args = parse_args()
cfg = Config.fromfile(args.config)
if args.options is not None:
cfg.merge_from_dict(args.options)
# set cudnn_benchmark
if cfg.get('cudnn_benchmark', False):
torch.backends.cudnn.benchmark = True
# work_dir is determined in this priority: CLI > segment in file > filename
if args.work_dir is not None:
# update configs according to CLI args if args.work_dir is not None
cfg.work_dir = args.work_dir
elif cfg.get('work_dir', None) is None:
# use config filename as default work_dir if cfg.work_dir is None
cfg.work_dir = osp.join('./work_dirs',
osp.splitext(osp.basename(args.config))[0])
if args.load_from is not None:
cfg.load_from = args.load_from
if args.resume_from is not None:
cfg.resume_from = args.resume_from
if args.gpu_ids is not None:
cfg.gpu_ids = args.gpu_ids
else:
cfg.gpu_ids = range(1) if args.gpus is None else range(args.gpus)
# init distributed env first, since logger depends on the dist info.
if args.launcher == 'none':
distributed = False
else:
distributed = True
init_dist(args.launcher, **cfg.dist_params)
# create work_dir
mmcv.mkdir_or_exist(osp.abspath(cfg.work_dir))
# dump config
cfg.dump(osp.join(cfg.work_dir, osp.basename(args.config)))
# init the logger before other steps
timestamp = time.strftime('%Y%m%d_%H%M%S', time.localtime())
log_file = osp.join(cfg.work_dir, f'{timestamp}.log')
logger = get_root_logger(log_file=log_file, log_level=cfg.log_level)
# init the meta dict to record some important information such as
# environment info and seed, which will be logged
meta = dict()
# log env info
env_info_dict = collect_env()
env_info = '\n'.join([f'{k}: {v}' for k, v in env_info_dict.items()])
dash_line = '-' * 60 + '\n'
logger.info('Environment info:\n' + dash_line + env_info + '\n' +
dash_line)
meta['env_info'] = env_info
# log some basic info
logger.info(f'Distributed training: {distributed}')
logger.info(f'Config:\n{cfg.pretty_text}')
# set random seeds
if args.seed is not None:
logger.info(f'Set random seed to {args.seed}, deterministic: '
f'{args.deterministic}')
set_random_seed(args.seed, deterministic=args.deterministic)
cfg.seed = args.seed
meta['seed'] = args.seed
meta['exp_name'] = osp.basename(args.config)
model = build_segmentor(cfg.model,
train_cfg=cfg.get('train_cfg'),
test_cfg=cfg.get('test_cfg'))
logger.info(model)
datasets = [build_dataset(cfg.data.train)]
if len(cfg.workflow) == 2:
val_dataset = copy.deepcopy(cfg.data.val)
val_dataset.pipeline = cfg.data.train.pipeline
datasets.append(build_dataset(val_dataset))
if cfg.checkpoint_config is not None:
# save mmseg version, config file content and class names in
# checkpoints as meta data
cfg.checkpoint_config.meta = dict(
mmseg_version=f'{__version__}+{get_git_hash()[:7]}',
config=cfg.pretty_text,
CLASSES=datasets[0].CLASSES,
PALETTE=datasets[0].PALETTE)
# add an attribute for visualization convenience
model.CLASSES = datasets[0].CLASSES
train_segmentor(model,
datasets,
cfg,
distributed=distributed,
validate=(not args.no_validate),
timestamp=timestamp,
meta=meta)
def __init__(self, args):
cudnn.enabled = True
self.args = args
self.mode_name = 'kd_mmseg'
device = args.device
######## skd
# self.S_device = 'cuda:0'
# self.T_device = 'cuda:1'
# student = Res_pspnet(BasicBlock, [2, 2, 2, 2], num_classes = args.classes_num, deep_base=False)
# load_S_model(args, student, False)
# # print(student)
#
# # print(student.device)
# print_model_parm_nums(student, 'student_model')
# self.student = self.DataParallelModelProcess(student, 2, 'train', device=self.S_device)
# self.student = student
# # self.student.cuda()
# # self.student.to('cuda:0')
# # self.student.train()
# teacher = Res_pspnet(Bottleneck, [3, 4, 23, 3], num_classes = args.classes_num)
# load_T_model(teacher, args.T_ckpt_path)
# print_model_parm_nums(teacher, 'teacher_model')
# self.teacher = self.DataParallelModelProcess(teacher, 2, 'eval', device=self.T_device)
# self.teacher = teacher
# # self.teacher.to('cuda:1')
# # self.teacher.eval()
########################## mmseg
self.S_device = 'cuda:0'
S_config = 'configs/pspnet/pspnet_r18-d8_512x512_40k_cityscapes_1gpu.py'
S_cfg = Config.fromfile(S_config)
# print(S_cfg)
# S_cfg.model.pretrained = args.student_pretrain_model_imgnet
self.student = build_segmentor(S_cfg.model, train_cfg=S_cfg.train_cfg, test_cfg=S_cfg.test_cfg)
# self.student = build_segmentor(S_cfg.model, train_cfg=None, test_cfg=None)
# checkpoint = args.student_pretrain_model_imgnet
# print(checkpoint)
# checkpoint = load_checkpoint(self.student, checkpoint)
# load_S_model(args, self.student, False)
# self.student = self.DataParallelModelProcess(self.student, 2, 'train', device=self.S_device)
self.student.train()
self.student.to(self.S_device)
# print(self.student)
# for name, parameters in self.student.named_parameters():
# print(name, ':', parameters.size(), parameters.requires_grad)
#
# # print(self.student.parameters())
#
# # for parameters in self.student.parameters():
# # print(parameters)
if self.args.pi or self.args.pa or self.args.ho:
self.T_device = 'cuda:1'
T_config = 'configs/pspnet/pspnet_r101-d8_512x512_80k_cityscapes_1gpu.py'
T_cfg = Config.fromfile(T_config)
# print(T_cfg)
# self.teacher = build_segmentor(T_cfg.model, train_cfg=T_cfg.train_cfg, test_cfg=T_cfg.test_cfg)
self.teacher = build_segmentor(T_cfg.model, train_cfg=None, test_cfg=None)
checkpoint = 'work_dirs/models_zoo/pspnet_r101-d8_512x512_80k_cityscapes.2.2_iter_80000.pth'
checkpoint = load_checkpoint(self.teacher, checkpoint)
self.teacher = self.DataParallelModelProcess(self.teacher, 2, 'eval', device=self.T_device)
####################################################
D_model = Discriminator(args.preprocess_GAN_mode, args.classes_num, args.batch_size, args.imsize_for_adv,
args.adv_conv_dim)
load_D_model(args, D_model, False)
print_model_parm_nums(D_model, 'D_model')
# self.parallel_D = self.DataParallelModelProcess(D_model, 2, 'train', device)
self.parallel_D = self.DataParallelModelProcess(D_model, 2, 'train', device='cuda:0')
self.D_model = D_model
self.G_solver = optim.SGD(
[{'params': filter(lambda p: p.requires_grad, self.student.parameters()), 'initial_lr': args.lr_g}],
lr=args.lr_g, momentum=args.momentum, weight_decay=args.weight_decay)
# self.G_solver = optim.SGD(self.student.parameters(),
# lr=args.lr_g, momentum=args.momentum, weight_decay=args.weight_decay)
self.D_solver = optim.SGD(
[{'params': filter(lambda p: p.requires_grad, D_model.parameters()), 'initial_lr': args.lr_d}],
lr=args.lr_d, momentum=args.momentum, weight_decay=args.weight_decay)
self.best_mean_IU = args.best_mean_IU
self.criterion = self.DataParallelCriterionProcess(CriterionDSN()) # CriterionCrossEntropy()
self.criterion_ce = self.DataParallelCriterionProcess(CriterionCE()) # CriterionCrossEntropy()
self.criterion_pixel_wise = self.DataParallelCriterionProcess(CriterionPixelWise())
# self.criterion_pair_wise_for_interfeat = [self.DataParallelCriterionProcess(CriterionPairWiseforWholeFeatAfterPool(scale=args.pool_scale[ind], feat_ind=-(ind+1))) for ind in range(len(args.lambda_pa))]
self.criterion_pair_wise_for_interfeat = self.DataParallelCriterionProcess(
CriterionPairWiseforWholeFeatAfterPool(scale=args.pool_scale, feat_ind=-5))
self.criterion_adv = self.DataParallelCriterionProcess(CriterionAdv(args.adv_loss_type))
if args.adv_loss_type == 'wgan-gp':
self.criterion_AdditionalGP = self.DataParallelCriterionProcess(
CriterionAdditionalGP(self.parallel_D, args.lambda_gp))
self.criterion_adv_for_G = self.DataParallelCriterionProcess(CriterionAdvForG(args.adv_loss_type))
self.mc_G_loss = 0.0
self.pi_G_loss = 0.0
self.pa_G_loss = 0.0
self.D_loss = 0.0
self.criterion_AT = self.DataParallelCriterionProcess(AT(p=2))
cudnn.benchmark = True
if not os.path.exists(args.snapshot_dir):
os.makedirs(args.snapshot_dir)
print('init finish')
coeff = (1 - float(iter) / max_iter)**power
return (base_lr - min_lr) * coeff + min_lr
def adjust_learning_rate(base_lr, optimizer, i_iter):
lr = lr_poly(base_lr, i_iter, cfg.total_iters, cfg.lr_config.power,
cfg.lr_config.min_lr)
optimizer.param_groups[0]['lr'] = lr
return lr
args = TrainOptions().initialize()
# config='configs/kd/kd_pspnet_r18-d8_512x512_40k_cityscapes_8x1gpu.py'
config = args.config
cfg = Config.fromfile(config)
cfg.gpu_ids = range(1)
cfg.seed = None
if args.work_dir:
cfg.work_dir = args.work_dir
# print(cfg)
# print(cfg.pretty_text)
# create work_dir
mmcv.mkdir_or_exist(osp.abspath(cfg.work_dir))
# dump config
# cfg.dump(osp.join(cfg.work_dir, osp.basename(args.config)))
# init the logger before other steps
timestamp = time.strftime('%Y%m%d_%H%M%S', time.localtime())
log_file = osp.join(cfg.work_dir, f'{timestamp}.log')
def __next__(self):
try:
data = next(self.iter_loader)
except StopIteration:
self._epoch += 1
if hasattr(self._dataloader.sampler, 'set_epoch'):
self._dataloader.sampler.set_epoch(self._epoch)
self.iter_loader = iter(self._dataloader)
data = next(self.iter_loader)
return data
config = 'configs/pspnet/pspnet_r18-d8_512x512_40k_cityscapes_8x1gpu.py'
cfg = Config.fromfile(config)
cfg.gpu_ids = range(1)
cfg.seed = None
# print(cfg)
# print(cfg.pretty_text)
def lr_poly(base_lr, iter, max_iter, power=0.9, min_lr=0):
# return base_lr*((1-float(iter)/max_iter)**(power))
coeff = (1 - float(iter) / max_iter)**power
return (base_lr - min_lr) * coeff + min_lr
def adjust_learning_rate(base_lr, optimizer, i_iter):
lr = lr_poly(base_lr, i_iter, cfg.total_iters, cfg.lr_config.power,
cfg.lr_config.min_lr)
def main():
args = parse_args()
cfg = Config.fromfile(args.config)
if args.cfg_options is not None:
cfg.merge_from_dict(args.cfg_options)
# set cudnn_benchmark
if cfg.get('cudnn_benchmark', False):
torch.backends.cudnn.benchmark = True
# work_dir is determined in this priority: CLI > segment in file > filename
if args.work_dir is not None:
# update configs according to CLI args if args.work_dir is not None
cfg.work_dir = args.work_dir
elif cfg.get('work_dir', None) is None:
# use config filename as default work_dir if cfg.work_dir is None
cfg.work_dir = osp.join('./work_dirs',
osp.splitext(osp.basename(args.config))[0])
if args.resume_from is not None:
cfg.resume_from = args.resume_from
if args.gpu_ids is not None:
cfg.gpu_ids = args.gpu_ids
else:
cfg.gpu_ids = range(1) if args.gpus is None else range(args.gpus)
# init distributed env first, since logger depends on the dist info.
if args.launcher == 'none':
distributed = False
else:
distributed = True
init_dist(args.launcher, **cfg.dist_params)
# re-set gpu_ids with distributed training mode
_, world_size = get_dist_info()
cfg.gpu_ids = range(world_size)
# create work_dir
mmcv.mkdir_or_exist(osp.abspath(cfg.work_dir))
# dump config
cfg.dump(osp.join(cfg.work_dir, osp.basename(args.config)))
# init the logger before other steps
timestamp = time.strftime('%Y%m%d_%H%M%S', time.localtime())
log_file = osp.join(cfg.work_dir, f'{timestamp}.log')
logger = get_root_logger(log_file=log_file, log_level=cfg.log_level)
# init the meta dict to record some important information such as
# environment info and seed, which will be logged
meta = dict()
# log env info
env_info_dict = collect_env()
env_info = '\n'.join([(f'{k}: {v}') for k, v in env_info_dict.items()])
dash_line = '-' * 60 + '\n'
logger.info('Environment info:\n' + dash_line + env_info + '\n' +
dash_line)
meta['env_info'] = env_info
meta['config'] = cfg.text
# log some basic info
logger.info(f'Distributed training: {distributed}')
logger.info(f'Config:\n{cfg.text}')
# set random seeds
if args.seed is not None:
logger.info(f'Set random seed to {args.seed}, '
f'deterministic: {args.deterministic}')
set_random_seed(args.seed, deterministic=args.deterministic)
cfg.seed = args.seed
meta['seed'] = args.seed
meta['exp_name'] = osp.basename(args.config)
model = build_reid(cfg.model)
datasets = [build_dataset(cfg.data.train)]
if cfg.checkpoint_config is not None:
# save project version in checkpoints as meta
cfg.checkpoint_config.meta = dict(reid_version=__version__)
train_reid(model,
datasets,
cfg,
distributed=distributed,
validate=(not args.no_validate),
timestamp=timestamp,
meta=meta)
def test_calculate_metric(iter_nums):
if args.net == 'unet':
# timm-efficientnet performs slightly worse.
if not args.vis_mode:
backbone_type = re.sub("^eff", "efficientnet", args.backbone_type)
net = smp.Unet(backbone_type, classes=args.num_classes, encoder_weights='imagenet')
else:
net = VanillaUNet(n_channels=3, num_classes=args.num_classes)
elif args.net == 'unet-scratch':
# net = UNet(num_classes=args.num_classes)
net = VanillaUNet(n_channels=3, num_classes=args.num_classes,
use_polyformer=args.polyformer_mode,
num_modes=args.num_modes)
elif args.net == 'nestedunet':
net = NestedUNet(num_classes=args.num_classes)
elif args.net == 'unet3plus':
net = UNet_3Plus(num_classes=args.num_classes)
elif args.net == 'pranet':
net = PraNet(num_classes=args.num_classes - 1)
elif args.net == 'attunet':
net = AttU_Net(output_ch=args.num_classes)
elif args.net == 'attr2unet':
net = R2AttU_Net(output_ch=args.num_classes)
elif args.net == 'dunet':
net = DeformableUNet(n_channels=3, n_classes=args.num_classes)
elif args.net == 'setr':
# Install mmcv first:
# pip install mmcv-full==1.2.2 -f https://download.openmmlab.com/mmcv/dist/cu{Your CUDA Version}/torch{Your Pytorch Version}/index.html
# E.g.: pip install mmcv-full==1.2.2 -f https://download.openmmlab.com/mmcv/dist/cu102/torch1.7.1/index.html
from mmcv.utils import Config
sys.path.append("networks/setr")
from mmseg.models import build_segmentor
task2config = { 'refuge': 'SETR_PUP_288x288_10k_refuge_context_bs_4.py',
'polyp': 'SETR_PUP_320x320_10k_polyp_context_bs_4.py' }
setr_cfg = Config.fromfile("networks/setr/configs/SETR/{}".format(task2config[args.task_name]))
net = build_segmentor(setr_cfg.model, train_cfg=setr_cfg.train_cfg, test_cfg=setr_cfg.test_cfg)
# By default, net() calls forward_train(), which receives extra parameters, and returns losses.
# net.forward_dummy() receives/returns the traditional input/output.
# Relevant file: mmseg/models/segmentors/encoder_decoder.py
net.forward = net.forward_dummy
elif args.net == 'transunet':
transunet_config = TransUNet_CONFIGS[args.backbone_type]
transunet_config.n_classes = args.num_classes
if args.backbone_type.find('R50') != -1:
# The "patch" in TransUNet means grid-like patches of the input image.
# The "patch" in our code means the whole input image after cropping/resizing (part of the augmentation)
transunet_config.patches.grid = (int(args.patch_size[0] / transunet_config.patches.size[0]),
int(args.patch_size[1] / transunet_config.patches.size[1]))
net = TransUNet(transunet_config, img_size=args.patch_size, num_classes=args.num_classes)
elif args.net.startswith('deeplab'):
use_smp_deeplab = args.net.endswith('smp')
if use_smp_deeplab:
backbone_type = re.sub("^eff", "efficientnet", args.backbone_type)
net = smp.DeepLabV3Plus(backbone_type, classes=args.num_classes, encoder_weights='imagenet')
else:
model_name = args.net + "_" + args.backbone_type
model_map = {
'deeplabv3_resnet50': deeplab.deeplabv3_resnet50,
'deeplabv3plus_resnet50': deeplab.deeplabv3plus_resnet50,
'deeplabv3_resnet101': deeplab.deeplabv3_resnet101,
'deeplabv3plus_resnet101': deeplab.deeplabv3plus_resnet101,
'deeplabv3_mobilenet': deeplab.deeplabv3_mobilenet,
'deeplabv3plus_mobilenet': deeplab.deeplabv3plus_mobilenet
}
net = model_map[model_name](num_classes=args.num_classes, output_stride=8)
elif args.net == 'nnunet':
from nnunet.network_architecture.initialization import InitWeights_He
from nnunet.network_architecture.generic_UNet import Generic_UNet
net = Generic_UNet(
input_channels=3,
base_num_features=32,
num_classes=args.num_classes,
num_pool=7,
num_conv_per_stage=2,
feat_map_mul_on_downscale=2,
norm_op=nn.InstanceNorm2d,
norm_op_kwargs={'eps': 1e-05, 'affine': True},
dropout_op_kwargs={'p': 0, 'inplace': True},
nonlin_kwargs={'negative_slope': 0.01, 'inplace': True},
final_nonlin=(lambda x: x),
weightInitializer=InitWeights_He(1e-2),
pool_op_kernel_sizes=[[2, 2], [2, 2], [2, 2], [2, 2], [2, 2], [2, 2], [2, 2]],
conv_kernel_sizes=([[3, 3], [3, 3], [3, 3], [3, 3], [3, 3], [3, 3], [3, 3], [3, 3]]),
upscale_logits=False,
convolutional_pooling=True,
convolutional_upsampling=True,
)
net.inference_apply_nonlin = (lambda x: F.softmax(x, 1))
elif args.net == 'segtran':
get_default(args, 'num_modes', default_settings, -1, [args.net, 'num_modes', args.in_fpn_layers])
set_segtran2d_config(args)
print(args)
net = Segtran2d(config)
else:
breakpoint()
net.cuda()
net.eval()
if args.robust_ref_cp_path:
refnet = copy.deepcopy(net)
print("Reference network created")
load_model(refnet, args, args.robust_ref_cp_path)
else:
refnet = None
# Currently colormap is used only for OCT task.
colormap = get_seg_colormap(args.num_classes, return_torch=True).cuda()
# prepred: pre-prediction. postpred: post-prediction.
task2mask_prepred = { 'refuge': refuge_map_mask, 'polyp': polyp_map_mask,
'oct': partial(index_to_onehot, num_classes=args.num_classes) }
task2mask_postpred = { 'refuge': refuge_inv_map_mask, 'polyp': polyp_inv_map_mask,
'oct': partial(onehot_inv_map, colormap=colormap) }
mask_prepred_mapping_func = task2mask_prepred[args.task_name]
mask_postpred_mapping_funcs = [ task2mask_postpred[args.task_name] ]
if args.do_remove_frag:
remove_frag = lambda segmap: remove_fragmentary_segs(segmap, 255)
mask_postpred_mapping_funcs.append(remove_frag)
if not args.checkpoint_dir:
if args.vis_mode is not None:
visualize_model(net, args.vis_mode, args.vis_layers, args.patch_size, db_test)
return
if args.eval_robustness:
eval_robustness(args, net, refnet, testloader, mask_prepred_mapping_func)
return
allcls_avg_metric = None
all_results = np.zeros((args.num_classes, len(iter_nums)))
for iter_idx, iter_num in enumerate(iter_nums):
if args.checkpoint_dir:
checkpoint_path = os.path.join(args.checkpoint_dir, 'iter_' + str(iter_num) + '.pth')
load_model(net, args, checkpoint_path)
if args.vis_mode is not None:
visualize_model(net, args.vis_mode, args.vis_layers, args.patch_size, db_test)
continue
if args.eval_robustness:
eval_robustness(args, net, refnet, testloader, mask_prepred_mapping_func)
continue
save_results = args.save_results and (not args.test_interp)
if save_results:
test_save_paths = []
test_save_dirs = []
test_save_dir_tmpl = "%s-%s-%s-%d" %(args.net, args.job_name, timestamp, iter_num)
for suffix in ("-soft", "-%.1f" %args.mask_thres):
test_save_dir = test_save_dir_tmpl + suffix
test_save_path = "../prediction/%s" %(test_save_dir)
if not os.path.exists(test_save_path):
os.makedirs(test_save_path)
test_save_dirs.append(test_save_dir)
test_save_paths.append(test_save_path)
else:
test_save_paths = None
test_save_dirs = None
if args.save_features_img_count > 0:
args.save_features_file_path = "%s-%s-feat-%s.pth" %(args.net, args.job_name, timestamp)
else:
args.save_features_file_path = None
allcls_avg_metric, allcls_metric_count = \
test_all_cases(net, testloader, task_name=args.task_name,
num_classes=args.num_classes,
mask_thres=args.mask_thres,
model_type=args.net,
orig_input_size=args.orig_input_size,
patch_size=args.patch_size,
stride=(args.orig_input_size[0] // 2, args.orig_input_size[1] // 2),
test_save_paths=test_save_paths,
out_origsize=args.out_origsize,
mask_prepred_mapping_func=mask_prepred_mapping_func,
mask_postpred_mapping_funcs=mask_postpred_mapping_funcs,
reload_mask=args.reload_mask,
test_interp=args.test_interp,
save_features_img_count=args.save_features_img_count,
save_features_file_path=args.save_features_file_path,
verbose=args.verbose_output)
print("Iter-%d scores on %d images:" %(iter_num, allcls_metric_count[0]))
dice_sum = 0
for cls in range(1, args.num_classes):
dice = allcls_avg_metric[cls-1]
print('class %d: dice = %.3f' %(cls, dice))
dice_sum += dice
all_results[cls, iter_idx] = dice
avg_dice = dice_sum / (args.num_classes - 1)
print("Average dice: %.3f" %avg_dice)
if args.net == 'segtran':
max_attn, avg_attn, clamp_count, call_count = \
[ segtran_shared.__dict__[v] for v in ('max_attn', 'avg_attn', 'clamp_count', 'call_count') ]
print("max_attn={:.2f}, avg_attn={:.2f}, clamp_count={}, call_count={}".format(
max_attn, avg_attn, clamp_count, call_count))
if save_results:
FNULL = open(os.devnull, 'w')
for pred_type, test_save_dir, test_save_path in zip(('soft', 'hard'), test_save_dirs, test_save_paths):
do_tar = subprocess.run(["tar", "cvf", "%s.tar" %test_save_dir, test_save_dir], cwd="../prediction",
stdout=FNULL, stderr=subprocess.STDOUT)
# print(do_tar)
print("{} tarball:\n{}.tar".format(pred_type, os.path.abspath(test_save_path)))
np.set_printoptions(precision=3, suppress=True)
print(all_results[1:])
return allcls_avg_metric
def __init__(self, cfgfile):
super().__init__()
cfg = Config.fromfile(cfgfile)
temp = build_segmentor(cfg.model)
self.backbone = temp.backbone
self.decode_head = temp.decode_head
def main():
args = parse_args()
cfg = Config.fromfile(args.config)
if args.cfg_options is not None:
cfg.merge_from_dict(args.cfg_options)
# set cudnn_benchmark
if cfg.get('cudnn_benchmark', False):
torch.backends.cudnn.benchmark = True
# work_dir is determined in this priority: CLI > segment in file > filename
if args.work_dir is not None:
# update configs according to CLI args if args.work_dir is not None
cfg.work_dir = args.work_dir
elif cfg.get('work_dir', None) is None:
# use config filename as default work_dir if cfg.work_dir is None
cfg.work_dir = osp.join('./work_dirs',
osp.splitext(osp.basename(args.config))[0])
if args.load_from is not None:
cfg.load_from = args.load_from
if args.resume_from is not None:
cfg.resume_from = args.resume_from
if args.gpus is not None:
cfg.gpu_ids = range(1)
warnings.warn('`--gpus` is deprecated because we only support '
'single GPU mode in non-distributed training. '
'Use `gpus=1` now.')
if args.gpu_ids is not None:
cfg.gpu_ids = args.gpu_ids[0:1]
warnings.warn('`--gpu-ids` is deprecated, please use `--gpu-id`. '
'Because we only support single GPU mode in '
'non-distributed training. Use the first GPU '
'in `gpu_ids` now.')
if args.gpus is None and args.gpu_ids is None:
cfg.gpu_ids = [args.gpu_id]
cfg.auto_resume = args.auto_resume
# init distributed env first, since logger depends on the dist info.
if args.launcher == 'none':
distributed = False
else:
distributed = True
init_dist(args.launcher, **cfg.dist_params)
# gpu_ids is used to calculate iter when resuming checkpoint
_, world_size = get_dist_info()
cfg.gpu_ids = range(world_size)
# create work_dir
mmcv.mkdir_or_exist(osp.abspath(cfg.work_dir))
# dump config
cfg.dump(osp.join(cfg.work_dir, osp.basename(args.config)))
# init the logger before other steps
timestamp = time.strftime('%Y%m%d_%H%M%S', time.localtime())
log_file = osp.join(cfg.work_dir, f'{timestamp}.log')
logger = get_root_logger(log_file=log_file, log_level=cfg.log_level)
# set multi-process settings
setup_multi_processes(cfg)
# init the meta dict to record some important information such as
# environment info and seed, which will be logged
meta = dict()
# log env info
env_info_dict = collect_env()
env_info = '\n'.join([f'{k}: {v}' for k, v in env_info_dict.items()])
dash_line = '-' * 60 + '\n'
logger.info('Environment info:\n' + dash_line + env_info + '\n' +
dash_line)
meta['env_info'] = env_info
# log some basic info
logger.info(f'Distributed training: {distributed}')
logger.info(f'Config:\n{cfg.pretty_text}')
# set random seeds
seed = init_random_seed(args.seed)
seed = seed + dist.get_rank() if args.diff_seed else seed
logger.info(f'Set random seed to {seed}, '
f'deterministic: {args.deterministic}')
set_random_seed(seed, deterministic=args.deterministic)
cfg.seed = seed
meta['seed'] = seed
meta['exp_name'] = osp.basename(args.config)
model = build_segmentor(
cfg.model,
train_cfg=cfg.get('train_cfg'),
test_cfg=cfg.get('test_cfg'))
model.init_weights()
# SyncBN is not support for DP
if not distributed:
warnings.warn(
'SyncBN is only supported with DDP. To be compatible with DP, '
'we convert SyncBN to BN. Please use dist_train.sh which can '
'avoid this error.')
model = revert_sync_batchnorm(model)
logger.info(model)
datasets = [build_dataset(cfg.data.train)]
if len(cfg.workflow) == 2:
val_dataset = copy.deepcopy(cfg.data.val)
val_dataset.pipeline = cfg.data.train.pipeline
datasets.append(build_dataset(val_dataset))
if cfg.checkpoint_config is not None:
# save mmseg version, config file content and class names in
# checkpoints as meta data
cfg.checkpoint_config.meta = dict(
mmseg_version=f'{__version__}+{get_git_hash()[:7]}',
config=cfg.pretty_text,
CLASSES=datasets[0].CLASSES,
PALETTE=datasets[0].PALETTE)
# add an attribute for visualization convenience
model.CLASSES = datasets[0].CLASSES
# passing checkpoint meta for saving best checkpoint
meta.update(cfg.checkpoint_config.meta)
train_segmentor(
model,
datasets,
cfg,
distributed=distributed,
validate=(not args.no_validate),
timestamp=timestamp,
meta=meta)
from mmcv.utils import Config
import os
import pandas as pd
def pseudo_label(csv_original_train, csv_pseudo_train, csv_pseudo_test):
df_train_original = pd.read_csv(csv_original_train)
df_test_pseudo = pd.read_csv(csv_pseudo_test)
df_train_pseudo = pd.DataFrame()
df_train_pseudo['filename'] = pd.Series(list(df_train_original['filename'])+(list(df_test_pseudo['image_raw'])))
df_train_pseudo['label'] = pd.Series(list(df_train_original['label'])+(list(df_test_pseudo['label_raw'])))
df_train_pseudo.to_csv(csv_pseudo_train, index=False)
if __name__ == '__main__':
cfg = Config.fromfile('config_data_utils_test.py')
csv_pseudo_test = '/home/muyun99/data/dataset/competition_data/xunfei_face_reco/test_pseudo.csv'
for fold_idx in range(cfg.num_KFold):
csv_original_train = os.path.join(cfg.path_save_trainval_csv, f'train_fold{fold_idx}.csv')
csv_pseudo_train = os.path.join(cfg.path_save_trainval_csv, f'train_fold{fold_idx}_pseudo.csv')
pseudo_label(csv_original_train=csv_original_train, csv_pseudo_train=csv_pseudo_train, csv_pseudo_test=csv_pseudo_test)