def from_args(cls, args, options=''):
"""
Initialize this class from some cli arguments. Used in train, test.
"""
for opt in options:
args.add_argument(*opt.flags, default=None, type=opt.type)
if not isinstance(args, tuple):
args = args.parse_args()
if args.device is not None:
os.environ["CUDA_VISIBLE_DEVICES"] = args.device
if args.resume is not None:
resume = Path(args.resume)
cfg_fname = resume.parent / 'config.json'
else:
msg_no_cfg = "Configuration file need to be specified. Add '-c config.json', for example."
assert args.config is not None, msg_no_cfg
resume = None
cfg_fname = Path(args.config)
config = read_json(cfg_fname)
if args.config and resume:
# update new config for fine-tuning
config.update(read_json(args.config))
# parse custom cli options into dictionary
modification = {
opt.target: getattr(args, _get_opt_name(opt.flags))
for opt in options
}
return cls(config, resume, modification)
def __init__(self, config):
super().__init__()
print("Initializing AudioFeatureExtracor...")
self.feature_dim = config["audio"]["feature_dim"]
self.feature_file = config["audio"]["feature_file"]
self.data = read_json(config["data_file"])
self.features = read_json(self.feature_file)
self.missing_tensor = torch.zeros((self.feature_dim))
def main(args_parsed):
o_path = Path(args_parsed.output)
o_path.mkdir(parents=True, exist_ok=True)
res = dict()
res["name"] = args_parsed.name
res["createTime"] = int(time.time())
if args_parsed.describe is not None:
res["description"] = args_parsed.describe
mods = list()
for l_mod in args_parsed.model:
mod_cont = dict()
conf = l_mod[0]
path = l_mod[1]
# if not ensure_exist(conf) or not ensure_exist(path):
if not ensure_exist(conf):
continue
try:
cf_content = read_json(conf)
for sec in ("name", "arch", "metrics", "tester"):
mod_cont[sec] = cf_content[sec]
mod_cont["path"] = os.path.basename(path)
# copy model & rename
mods.append(mod_cont)
except Exception as e:
print(str(e))
continue
res["models"] = mods
write_json(res, o_path / (args_parsed.name + ".json"))
def from_args(cls, args, options=''):
"""
Initialize this class from some cli arguments. Used in train, test.
:param args: Arguments from which to initialize the configuration.
:param options: additional options to add to the config.
:return: ConfigParser object configured with the arguments.
"""
# Update the arguments with the options.
for opt in options:
args.add_argument(*opt.flags, default=None, type=opt.type)
if not isinstance(args, tuple):
args = args.parse_args()
# Set cuda device.
if args.device is not None:
os.environ["CUDA_VISIBLE_DEVICES"] = args.device
# Set training resume file.
if args.resume is not None:
resume = Path(args.resume)
cfg_fname = resume.parent / 'config.json'
else:
msg_no_cfg = "Configuration file need to be specified. Add '-c config.json', for example."
assert args.config is not None, msg_no_cfg
resume = None
cfg_fname = Path(args.config)
config = read_json(cfg_fname)
if args.config and resume:
# update new config for fine-tuning
config.update(read_json(args.config))
config['debug'] = args.debug
# parse custom cli options into dictionary
modification = {
opt.target: getattr(args, _get_opt_name(opt.flags))
for opt in options
}
ConfigParser.resolve_constants(config)
return cls(config, resume, modification)
def find_config(self, data_path):
if self.sensor_resolution is None:
config = os.path.join(data_path, "dataset_config.json")
if os.path.exists(config):
self.config = read_json(config)
self.data_source = self.config['data_source']
self.sensor_resolution = self.config["sensor_resolution"]
else:
data_source = 'unknown'
self.sensor_resolution = self.infer_resolution()
def load_model_for_eval(checkpoint):
config_file = Path(checkpoint).parent / 'config.json'
config = read_json(config_file)
model = get_instance(module_arch, 'arch', config)
model.summary()
checkpoint = torch.load(checkpoint, map_location='cpu')
state_dict = checkpoint['state_dict']
model.load_state_dict(clean_state_dict(state_dict))
model.eval()
return model
def get_result(self, target):
if self.f_augment:
self.pred = self.f_augment(self.pred)
# load previous result
if os.path.isfile(target+".json"):
res = read_json(target+".json")
if not "meta" in res:
print("Warning: no meta in exist json file.")
elif self.meta != res["meta"]:
print("Warning: meta for the current image is not same as previous result, may result in wrong align.")
else:
res = dict()
if self.type == "Building":
try:
cnts = self.get_contours(self.pred)
bboxs = self.get_bboxs(cnts[0])
building = self.encoding(bboxs, self.meta, fun_prop=random_height)
except (RuntimeError, TypeError):
building = list()
res["meta"] = self.meta
res["buildings"] = building
with open(target+".json", 'w') as f:
f.write(json.dumps(res))
elif self.type == "Road":
res["meta"] = self.meta
try:
skel = self.pred
for i in range(2):
skel = self.get_skeleton(skel)
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3))
for i in range(4):
skel = cv2.dilate(skel, kernel, iterations=5)
skel = SegmentOutputUtil.connect_line(skel, 10, 1)
mid_line = self.get_skeleton(skel)
img = np.zeros(skel.shape).astype(np.uint8)
img[skel == 255] = [50]
img[mid_line == 255] = [255]
alpha = Image.fromarray(skel)
img = Image.merge('LA', [Image.fromarray(img), alpha])
img_path = target+"_road.png"
img.save(img_path)
res["roadImg"] = img_path
except (RuntimeError, TypeError):
res["roadImg"] = ""
with open(target+".json", 'w') as f:
f.write(json.dumps(res))
return target+".json"
def test_draw(self):
j = read_json("test_res.json")
img = np.zeros((j["meta"]["h"], j["meta"]["w"], 3), np.uint8)
for building in j["buildings"]:
rect = list()
for coor in building["coordinates"]:
rect.append([coor["x"], coor["y"]])
narray = np.array(rect)
narray *= 10
narray = narray.astype(np.int)
cv2.drawContours(img, [narray], 0, (0, 255, 0), 2) # green
cv2.imshow("img", img)
cv2.waitKey(0)
cv2.destroyAllWindows()
def __init__(self, args, options='', timestamp=True):
# parse default and custom cli options
for opt in options:
args.add_argument(*opt.flags, default=None, type=opt.type)
args = args.parse_args()
self.resume = None
if args.device:
os.environ["CUDA_VISIBLE_DEVICES"] = args.device
if args.resume:
self.resume = Path(args.resume)
self.cfg_fname = self.resume.parent / 'config.json'
if args.config:
self.cfg_fname = Path(args.config)
msg_no_cfg = ("Configuration file need to be specified. "
"Add '-c config.json', for example.")
assert self.cfg_fname is not None, msg_no_cfg
# load config file and apply custom cli options
config = read_json(self.cfg_fname)
self.__config = _update_config(config, options, args)
self.__raw = copy.deepcopy(self.__config)
# set save_dir where trained model and log will be saved.
save_dir = Path(
parse_value(self.config['trainer']['extra_args']['save_dir']))
timestamp = datetime.now().strftime(
r'%m%d_%H%M%S') if timestamp else ''
exper_name = self.config['name']
self.__save_dir = save_dir / 'models' / exper_name / timestamp
self.__log_dir = save_dir / 'log' / exper_name / timestamp
self.save_dir.mkdir(parents=True, exist_ok=True)
self.log_dir.mkdir(parents=True, exist_ok=True)
# save updated config file to the checkpoint dir
if get_global_rank() == 0:
write_json(self.config, self.save_dir / 'config.json')
# configure logging module
setup_logging(self.log_dir)
self.log_levels = {
0: logging.WARNING,
1: logging.INFO,
2: logging.DEBUG
}
logger = self.get_logger('config')
logger.info(f"Experiment name: {exper_name}")
def summarise(group_id, log_dir="data/saved/log", model_dir="data/saved/models"):
seeded_runs = sorted(list(Path(log_dir).glob(f"**/{group_id}/seed-*")))
print(f"Found a total of {len(seeded_runs)} seed runs in {group_id}")
msg = f"Found no seeded runs for group_id: {group_id} in {log_dir}"
assert len(seeded_runs) > 0, msg
info_logs = OrderedDict()
for seeded_run in seeded_runs:
info_log_matches = list(Path(seeded_run).glob("**/info.log"))
msg = f"expected to find a single info.log file, found {len(info_log_matches)}"
assert len(info_log_matches) == 1, msg
info_logs[seeded_run.stem] = info_log_matches[0]
summary_log = []
for seeded_run, info_log_path in info_logs.items():
with open(info_log_path, "r") as f:
log = f.read().splitlines()
summary_log.extend(log)
first_info_log = list(info_logs.values())[0]
summary_log_name = f"summary-{'_'.join(list(info_logs.keys()))}.json"
summary_log_path = first_info_log.parent / summary_log_name
with open(summary_log_path, "w") as f:
f.write("\n".join(summary_log))
print(f"Wrote concatenated logs to {summary_log_path}")
# retrieve the config from the first run
rel_path = first_info_log.relative_to(log_dir).parent
config_path = Path(model_dir) / rel_path / "config.json"
assert config_path.exists(), f"Could not find config at {config_path}"
config = read_json(config_path)
logger = logging.getLogger("summary")
# some care is required with logging to avoid sending all experiment logs
# to the same file. We avoid this by essentially resetting the logging utility
# Remove all handlers associated with the root logger object
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=summary_log_path, level=logging.INFO)
if not logger.handlers:
logger.addHandler(logging.StreamHandler())
log_summary(
logger=logger,
log_path=summary_log_path,
eval_mode=config["eval_mode"],
fixed_num_epochs=config["trainer"]["epochs"],
)
def load_conf(self, config):
self.conf_path = Path(config)
self.config = read_json(config)
if "models" not in self.config:
return False
for m in self.config["models"]:
# check
if not os.path.exists(m["path"]):
if not (self.conf_path.parents[0] / m["path"]).exists():
print("Model", m["name"], "path", m["path"],
"is not exist.")
# continue
m["path"] = str(self.conf_path.parents[0] / m["path"])
self.models[m["name"]] = m
def setup_logging(save_dir, log_config='logger/logger_config.json', default_level=logging.INFO):
"""
Setup logging configuration
"""
log_config = Path(log_config)
if log_config.is_file():
config = read_json(log_config)
# modify logging paths based on run config
for _, handler in config['handlers'].items():
if 'filename' in handler:
handler['filename'] = str(save_dir / handler['filename'])
logging.config.dictConfig(config)
else:
print("Warning: logging configuration file is not found in {}.".format(log_config))
logging.basicConfig(level=default_level)
def summarise(group_id,
log_dir="data/saved/log",
model_dir="data/saved/models"):
seeded_runs = sorted(list(Path(log_dir).glob(f"**/{group_id}/seed-*")))
print(f"Found a total of {len(seeded_runs)} seed runs in {group_id}")
info_logs = OrderedDict()
for seeded_run in seeded_runs:
info_log_matches = list(Path(seeded_run).glob("**/info.log"))
msg = f"expected to find a single info.log file, found {len(info_log_matches)}"
assert len(info_log_matches) == 1, msg
info_logs[seeded_run.stem] = info_log_matches[0]
summary_log = []
for seeded_run, info_log_path in info_logs.items():
with open(info_log_path, "r") as f:
log = f.read().splitlines()
summary_log.extend(log)
first_info_log = list(info_logs.values())[0]
summary_log_name = f"summary-{'_'.join(list(info_logs.keys()))}.json"
summary_log_path = first_info_log.parent / summary_log_name
with open(summary_log_path, "w") as f:
f.write("\n".join(summary_log))
print(f"Wrote summary log to {summary_log_path}")
# retrieve the config from the first run
rel_path = first_info_log.relative_to(log_dir).parent
config_path = Path(model_dir) / rel_path / "config.json"
assert config_path.exists(), f"Could not find config at {config_path}"
config = read_json(config_path)
logger = logging.getLogger("summary")
logging.basicConfig(filename=summary_log_path, level=logging.INFO)
logger.addHandler(logging.StreamHandler())
log_summary(
logger=logger,
log_path=summary_log_path,
eval_mode=config["eval_mode"],
fixed_num_epochs=config["trainer"]["epochs"],
)
def init_params(cls, config_fname=None):
"""
intialize and setup params from config file
Parameters
----------
config_fname
Returns
-------
"""
if config_fname is None:
config_fname = os.path.join(Path(__file__).parent.parent, 'config.json')
logging.info('Using default configuration file: ', config_fname)
# parse the config file
_config = read_json(config_fname)
# specify the saving directory
proj_name = _config['project_name']
save_root_dir = Path(_config['resparams']['save_dir'])
run_id = datetime.now().strftime(r'%m%d_%H%M%S')
save_model_dir = save_root_dir / proj_name / 'models'
save_log_dir = save_root_dir / proj_name / 'logs'
if not os.path.exists(save_model_dir):
save_model_dir.mkdir(parents=True, exist_ok=True)
if not os.path.exists(save_log_dir):
save_log_dir.mkdir(parents=True, exist_ok=True)
_config['resparams']['run_id'] = run_id
_config['resparams']['save_model_dir'] = str(save_model_dir)
_config['resparams']['save_log_dir'] = str(save_log_dir)
# setup logging
setup_logging(save_log_dir, run_id=run_id)
return cls(_config)
def setup_logging(save_dir,
log_config='logger/logger_config.json',
default_level=logging.INFO):
"""
设置日志配置
Setup logging configuration
:param save_dir: 日志保存目录
:param log_config: 日志配置文件路径
:param default_level: 默认日志等级
:return:
"""
# 解析配置文件路径
log_config = Path(log_config)
# 是个文件
if log_config.is_file():
# json.load解析
config = read_json(log_config)
# 找到并设置日志保存路径
for _, handler in config['handlers'].items():
if 'filename' in handler:
# 保存路径，文件名
handler['filename'] = str(save_dir / handler['filename'])
# config给logger
logging.config.dictConfig(config)
# 没找到
else:
print("Warning: logging configuration file is not found in {}.".format(
log_config))
# 用默认配置，并info等级
logging.basicConfig(level=default_level)
def __init__(self, args, options='', timestamp=True, slave_mode=False):
# slave_mode - when calling the config parser form an existing process, we
# avoid reinitialising the logger and ignore sys.argv when argparsing.
# parse default and custom cli options
for opt in options:
args.add_argument(*opt.flags, default=None, type=opt.type)
if slave_mode:
args = args.parse_args(args=[])
else:
args = args.parse_args()
if args.device:
os.environ["CUDA_VISIBLE_DEVICES"] = args.device
if args.resume and not slave_mode:
self.resume = Path(args.resume)
# self.cfg_fname = self.resume.parent / 'config.json'
else:
msg_no_cfg = "Config file must be specified"
assert args.config is not None, msg_no_cfg
self.resume = None
self.cfg_fname = Path(args.config)
# load config file and apply custom cli options
config = read_json(self.cfg_fname)
self._config = _update_config(config, options, args)
if self._config.get("eval_config", False):
# validate path to evaluation file
eval_cfg_path = self._config.get("eval_config")
msg = f"eval_config was specified, but `{eval_cfg_path}` does not exist"
assert Path(self._config.get("eval_config")).exists(), msg
# set save_dir where trained model and log will be saved.
if "trainer" in self.config:
save_dir = Path(self.config['trainer']['save_dir'])
else:
save_dir = Path(self.config['tester']['save_dir'])
timestamp = datetime.now().strftime(
r"%Y-%m-%d_%H-%M-%S") if timestamp else ""
if slave_mode:
timestamp = f"{timestamp}-eval-worker"
# We assume that the config files are organised into directories such that
# each directory has the name of the dataset.
dataset_name = self.cfg_fname.parent.stem
exper_name = f"{dataset_name}-{self.cfg_fname.stem}"
self._save_dir = save_dir / 'models' / exper_name / timestamp
self._log_dir = save_dir / 'log' / exper_name / timestamp
self._web_log_dir = save_dir / 'web' / exper_name / timestamp
self._exper_name = exper_name
self._args = args
# if set, remove all previous experiments with the current config
if vars(args).get("purge_exp_dir", False):
for dirpath in (self._save_dir, self._log_dir, self._web_log_dir):
config_dir = dirpath.parent
existing = list(config_dir.glob("*"))
print(
f"purging {len(existing)} directories from config_dir...")
tic = time.time()
os.system(f"rm -rf {config_dir}")
print(f"Finished purge in {time.time() - tic:.3f}s")
self.save_dir.mkdir(parents=True, exist_ok=True)
self.log_dir.mkdir(parents=True, exist_ok=True)
# save updated config file to the checkpoint dir
write_json(self.config, self.save_dir / 'config.json')
# configure logging module
if not slave_mode:
self.log_path = setup_logging(self.log_dir)
self.log_levels = {
0: logging.WARNING,
1: logging.INFO,
2: logging.DEBUG
}
def from_args(cls, args, options=''):
"""
用命令行参数，初始化这个类（训练、测试时）
Initialize this class from some cli arguments. Used in train, test.
"""
# 处理options(value)到args(key)
# 遍历options
for opt in options:
# opt，添加到args
# opt.flags，为单个命令list
args.add_argument(*opt.flags, default=None, type=opt.type)
# 若args不是元组
if not isinstance(args, tuple):
# parse_args()，转元组
args = args.parse_args()
# 用args初始化
# 初始化计算代理
if args.device is not None:
os.environ["CUDA_VISIBLE_DEVICES"] = args.device
# 初始化config配置文件（重启方式、原始配置方式、命令更新方式）
# 重启方式（args.resume存在）
if args.resume is not None:
# Path
resume = Path(args.resume)
# 配置文件路径为：重启点的配置文件
# 'config.json'
cfg_fname = resume.parent / 'config.json'
else:
# msg：'配置文件需要被指定，请添加'-c rnn_config.json'的命令
msg_no_cfg = "Configuration file need to be specified. Add '-c rnn_config.json', for example."
# 保证args.config存在
assert args.config is not None, msg_no_cfg
# resume设为none
resume = None
# 配置文件路径为：原始配置文件
cfg_fname = Path(args.config)
# 读取，配置文件路径（cfg_fname），为json.load
config = read_json(cfg_fname)
# 计算代理
config['device_id'] = args.device
# 配置文件
config['config_file_name'] = args.config
# 命令更新方式（若args.config和resume都存在）（可能这里不是这个意思）
if args.config and resume:
# 更新新的配置，来fine-tuning
# update new config for fine-tuning
config.update(read_json(args.config))
# 解析命令行参数进字典
"""
parse custom cli options into dictionary
1、遍历options，为opt，
2、获取opt.flags的所有参数名(key)；从args(keys)中，寻找对应，
3、opt.target(k): opt.flag(v)
"""
modification = {
opt.target: getattr(args, _get_opt_name(opt.flags))
for opt in options
}
# 返回这个类对象本身，
# 可能是保持这个对象，后面继续执行解析新的命令行
return cls(config, resume, modification)
