def setup(args):
cfg = get_cfg()
cfg.merge_from_file(args.config_file)
cfg.SOLVER.BASE_LR = 0.001 # Avoid NaNs. Not useful in this script anyway.
cfg.merge_from_list(args.opts)
cfg.freeze()
setup_logger(distributed_rank=comm.get_rank())
return cfg
def __init__(self, cfg, model_build_func):
"""
Args:
cfg (BaseConfig):
"""
logger = logging.getLogger("cvpods")
if not logger.isEnabledFor(
logging.INFO): # setup_logger is not called for d2
setup_logger()
self.start_iter = 0
data_loader = self.build_train_loader(cfg)
epoch_iters = adjust_epoch_and_iter(cfg, data_loader)
self.max_iter = cfg.SOLVER.LR_SCHEDULER.MAX_ITER
self.max_epoch = cfg.SOLVER.LR_SCHEDULER.MAX_EPOCH
model = model_build_func(cfg)
model = maybe_convert_module(model)
logger.info(f"Model structure: {model}")
# Assume these objects must be constructed in this order.
optimizer = self.build_optimizer(cfg, model)
# For training, wrap with DDP. But don't need this for inference.
if comm.get_world_size() > 1:
model = DistributedDataParallel(model,
device_ids=[comm.get_local_rank()],
broadcast_buffers=False,
find_unused_parameters=True)
# TODO: @wangfeng02, `batch_subdivisions`
super().__init__(model, data_loader, optimizer,
cfg.SOLVER.BATCH_SUBDIVISIONS)
if not cfg.SOLVER.LR_SCHEDULER.get("EPOCH_WISE", False):
epoch_iters = -1
self.scheduler = self.build_lr_scheduler(cfg,
optimizer,
epoch_iters=epoch_iters)
# Assume no other objects need to be checkpointed.
# We can later make it checkpoint the stateful hooks
self.checkpointer = DetectionCheckpointer(
# Assume you want to save checkpoints together with logs/statistics
model,
cfg.OUTPUT_DIR,
optimizer=optimizer,
scheduler=self.scheduler,
)
self.cfg = cfg
self.register_hooks(self.build_hooks())
def default_setup(cfg, args):
"""
Perform some basic common setups at the beginning of a job, including:
1. Set up the cvpods logger
2. Log basic information about environment, cmdline arguments, and config
3. Backup the config to the output directory
Args:
cfg (BaseConfig): the full config to be used
args (argparse.NameSpace): the command line arguments to be logged
"""
output_dir = cfg.OUTPUT_DIR
if comm.is_main_process() and output_dir:
ensure_dir(output_dir)
rank = comm.get_rank()
# setup_logger(output_dir, distributed_rank=rank, name="cvpods")
setup_logger(output_dir, distributed_rank=rank)
logger.info("Rank of current process: {}. World size: {}".format(
rank, comm.get_world_size()))
logger.info("Environment info:\n" + collect_env_info())
logger.info("Command line arguments: " + str(args))
if hasattr(args, "config_file") and args.config_file != "":
logger.info("Contents of args.config_file={}:\n{}".format(
args.config_file,
megfile.smart_open(args.config_file, "r").read()))
adjust_config(cfg)
# make sure each worker has a different, yet deterministic seed if specified
seed = seed_all_rng(None if cfg.SEED < 0 else cfg.SEED + rank)
# save seed to config for dump
cfg.SEED = seed
# cudnn benchmark has large overhead. It shouldn't be used considering the small size of
# typical validation set.
if not (hasattr(args, "eval_only") and args.eval_only):
torch.backends.cudnn.benchmark = cfg.CUDNN_BENCHMARK
return cfg
def default_setup(cfg, args):
"""
Perform some basic common setups at the beginning of a job, including:
1. Set up the cvpods logger
2. Log basic information about environment, cmdline arguments, and config
3. Backup the config to the output directory
Args:
cfg (BaseConfig): the full config to be used
args (argparse.NameSpace): the command line arguments to be logged
"""
output_dir = cfg.OUTPUT_DIR
if comm.is_main_process() and output_dir:
PathManager.mkdirs(output_dir)
rank = comm.get_rank()
# setup_logger(output_dir, distributed_rank=rank, name="cvpods")
logger = setup_logger(output_dir, distributed_rank=rank)
logger.info("Rank of current process: {}. World size: {}".format(
rank, comm.get_world_size()))
logger.info("Environment info:\n" + collect_env_info())
logger.info("Command line arguments: " + str(args))
if hasattr(args, "config_file") and args.config_file != "":
logger.info("Contents of args.config_file={}:\n{}".format(
args.config_file,
PathManager.open(args.config_file, "r").read()))
adjust_config(cfg)
logger.info("Running with full config:\n{}".format(cfg))
base_config = cfg.__class__.__base__()
logger.info("different config with base class:\n{}".format(
cfg.diff(base_config)))
# if comm.is_main_process() and output_dir:
# # Note: some of our scripts may expect the existence of
# # config.yaml in output directory
# path = os.path.join(output_dir, "config.yaml")
# with PathManager.open(path, "w") as f:
# f.write(cfg.dump())
# logger.info("Full config saved to {}".format(os.path.abspath(path)))
# make sure each worker has a different, yet deterministic seed if specified
seed = seed_all_rng(None if cfg.SEED < 0 else cfg.SEED + rank)
# save seed to config for dump
cfg.SEED = seed
# cudnn benchmark has large overhead. It shouldn't be used considering the small size of
# typical validation set.
if not (hasattr(args, "eval_only") and args.eval_only):
torch.backends.cudnn.benchmark = cfg.CUDNN_BENCHMARK
return cfg, logger
def setUp(self):
setup_logger()
help="path to output directory")
parser.add_argument("--show",
action="store_true",
help="show output in a window")
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER,
)
return parser.parse_args(in_args)
if __name__ == "__main__":
args = parse_args()
logger = setup_logger()
logger.info("Arguments: " + str(args))
cfg = setup(args)
dirname = args.output_dir
os.makedirs(dirname, exist_ok=True)
metadata = MetadataCatalog.get(cfg.DATASETS.TRAIN[0])
def output(vis, fname):
if args.show:
print(fname)
cv2.imshow("window", vis.get_image()[:, :, ::-1])
cv2.waitKey()
else:
filepath = os.path.join(dirname, fname)
print("Saving to {} ...".format(filepath))
def default_setup(cfg, args):
"""
Perform some basic common setups at the beginning of a job, including:
1. Set up the cvpods logger
2. Log basic information about environment, cmdline arguments, and config
3. Backup the config to the output directory
Args:
cfg (BaseConfig): the full config to be used
args (argparse.NameSpace): the command line arguments to be logged
"""
output_dir = cfg.OUTPUT_DIR
if comm.is_main_process() and output_dir:
PathManager.mkdirs(output_dir)
rank = comm.get_rank()
# setup_logger(output_dir, distributed_rank=rank, name="cvpods")
logger = setup_logger(output_dir, distributed_rank=rank)
logger.info("Rank of current process: {}. World size: {}".format(
rank, comm.get_world_size()))
logger.info("Environment info:\n" + collect_env_info())
logger.info("Command line arguments: " + str(args))
if hasattr(args, "config_file") and args.config_file != "":
logger.info("Contents of args.config_file={}:\n{}".format(
args.config_file,
PathManager.open(args.config_file, "r").read()))
logger.info("Running with full config:\n{}".format(cfg))
base_config = cfg.__class__.__base__()
logger.info("different config with base class:\n{}".format(
cfg.show_diff(base_config)))
# if comm.is_main_process() and output_dir:
# # Note: some of our scripts may expect the existence of
# # config.yaml in output directory
# path = os.path.join(output_dir, "config.yaml")
# with PathManager.open(path, "w") as f:
# f.write(cfg.dump())
# logger.info("Full config saved to {}".format(os.path.abspath(path)))
# make sure each worker has a different, yet deterministic seed if specified
seed_all_rng(None if cfg.SEED < 0 else cfg.SEED + rank)
# cudnn benchmark has large overhead. It shouldn't be used considering the small size of
# typical validation set.
if not (hasattr(args, "eval_only") and args.eval_only):
torch.backends.cudnn.benchmark = cfg.CUDNN_BENCHMARK
# dynamic adjust batch_size, steps according to world size
base_world_size = int(cfg.SOLVER.IMS_PER_BATCH / cfg.SOLVER.IMS_PER_DEVICE)
world_size = comm.get_world_size()
ratio = world_size / base_world_size
cfg.SOLVER.IMS_PER_BATCH = int(ratio * cfg.SOLVER.IMS_PER_BATCH)
cfg.SOLVER.LR_SCHEDULER.MAX_ITER = int(cfg.SOLVER.LR_SCHEDULER.MAX_ITER /
ratio)
# Divided by scale ratio when using iterations rather than epochs
if cfg.SOLVER.LR_SCHEDULER.MAX_EPOCH is None:
cfg.SOLVER.LR_SCHEDULER.STEPS = list(
(int(step / ratio) for step in cfg.SOLVER.LR_SCHEDULER.STEPS))
cfg.SOLVER.CHECKPOINT_PERIOD = int(cfg.SOLVER.CHECKPOINT_PERIOD /
ratio)
cfg.TEST.EVAL_PERIOD = int(cfg.TEST.EVAL_PERIOD / ratio)
cfg.SOLVER.OPTIMIZER.BASE_LR = ratio * cfg.SOLVER.OPTIMIZER.BASE_LR
assert cfg.SOLVER.IMS_PER_BATCH / cfg.SOLVER.IMS_PER_DEVICE == world_size
return cfg, logger
def __init__(self, cfg, model_build_func):
"""
Args:
cfg (BaseConfig):
"""
logger = logging.getLogger("cvpods")
if not logger.isEnabledFor(
logging.INFO): # setup_logger is not called for d2
setup_logger()
# For simulate large batch training
images_per_batch = cfg.SOLVER.IMS_PER_BATCH
batch_subdivisions = cfg.SOLVER.BATCH_SUBDIVISIONS
assert (
batch_subdivisions > 0
), "cfg.SOLVER.BATCH_SUBDIVISIONS ({}) must be greater than or equal to 1.".format(
batch_subdivisions
)
if batch_subdivisions > 1:
# if batch_subdivisions is equal to 1, the following check is redundant
assert (
images_per_batch % batch_subdivisions == 0
), "SOLVER.IMS_PER_BATCH ({}) must be divisible by the " \
"cfg.SOLVER.BATCH_SUBDIVISIONS ({}).".format(images_per_batch, batch_subdivisions)
images_per_mini_batch = images_per_batch // batch_subdivisions
num_workers = comm.get_world_size()
assert (
images_per_mini_batch % num_workers == 0
), "images per mini batch ({}, is calculated from cfg.SOLVER.IMS_PER_BATCH // " \
"cfg.SOLVER.BATCH_SUBDIVISIONS) must be divisible by the number of workers " \
"({}).".format(images_per_mini_batch, num_workers)
assert (
images_per_mini_batch >= num_workers
), "images per mini batch ({}, is calculated from cfg.SOLVER.IMS_PER_BATCH // " \
"cfg.SOLVER.BATCH_SUBDIVISIONS) must be larger than the number of workers " \
"({}).".format(images_per_mini_batch, num_workers)
self.batch_subdivisions = batch_subdivisions
data_loader = self.build_train_loader(cfg)
self.start_iter = 0
self.max_iter = cfg.SOLVER.LR_SCHEDULER.MAX_ITER
self.max_epoch = cfg.SOLVER.LR_SCHEDULER.MAX_EPOCH
self.cfg = cfg
if self.max_epoch is not None:
epoch_iter = len(data_loader.dataset) // (
comm.get_world_size() * data_loader.batch_size * cfg.SOLVER.BATCH_SUBDIVISIONS
) + 1
cfg.SOLVER.LR_SCHEDULER.MAX_ITER = self.max_iter = self.max_epoch * epoch_iter
cfg.SOLVER.LR_SCHEDULER.STEPS = list(
(x * epoch_iter for x in cfg.SOLVER.LR_SCHEDULER.STEPS))
cfg.SOLVER.LR_SCHEDULER.WARMUP_ITERS = int(
cfg.SOLVER.LR_SCHEDULER.WARMUP_ITERS * epoch_iter)
cfg.SOLVER.CHECKPOINT_PERIOD = epoch_iter * cfg.SOLVER.CHECKPOINT_PERIOD
cfg.TEST.EVAL_PERIOD = epoch_iter * cfg.TEST.EVAL_PERIOD
else:
epoch_iter = -1
model = model_build_func(cfg)
logger.info(f"Model structure: {model}")
# Assume these objects must be constructed in this order.
optimizer = self.build_optimizer(cfg, model)
# For training, wrap with DDP. But don't need this for inference.
if comm.get_world_size() > 1:
model = DistributedDataParallel(model,
device_ids=[comm.get_local_rank()],
broadcast_buffers=False,
find_unused_parameters=True)
super().__init__(model, data_loader, optimizer)
self.scheduler = self.build_lr_scheduler(cfg, optimizer, epoch_iters=epoch_iter)
# Assume no other objects need to be checkpointed.
# We can later make it checkpoint the stateful hooks
self.checkpointer = DetectionCheckpointer(
# Assume you want to save checkpoints together with logs/statistics
model,
cfg.OUTPUT_DIR,
optimizer=optimizer,
scheduler=self.scheduler,
)
self.register_hooks(self.build_hooks())
def __init__(self, cfg, build_model):
"""
Args:
cfg (config dict):
"""
logger = logging.getLogger("cvpods")
if not logger.isEnabledFor(
logging.INFO): # setup_logger is not called for cvpods
setup_logger()
self.logger = logger
self.data_loader = self.build_train_loader(cfg)
# Assume these objects must be constructed in this order.
model = build_model(cfg)
self.model = maybe_convert_module(model)
self.logger.info(f"Model: \n{self.model}")
# Assume these objects must be constructed in this order.
self.optimizer = self.build_optimizer(cfg, self.model)
# For training, wrap with DDP. But don't need this for inference.
if comm.get_world_size() > 1:
if cfg.TRAINER.FP16.ENABLED:
self.mixed_precision = True
if cfg.TRAINER.FP16.TYPE == "APEX":
from apex import amp
self.model, self.optimizer = amp.initialize(
self.model,
self.optimizer,
opt_level=cfg.TRAINER.FP16.OPTS.OPT_LEVEL)
else:
self.mixed_precision = False
torch.cuda.set_device(comm.get_local_rank())
self.model = DistributedDataParallel(
self.model,
device_ids=[comm.get_local_rank()],
broadcast_buffers=False,
find_unused_parameters=True)
super().__init__(
self.model,
self.data_loader,
self.optimizer,
)
if not cfg.SOLVER.LR_SCHEDULER.get("EPOCH_WISE", False):
epoch_iters = -1
else:
epoch_iters = cfg.SOLVER.LR_SCHEDULER.get("EPOCH_ITERS")
self.logger.warning(
f"Setup LR Scheduler in EPOCH mode: {epoch_iters}")
auto_scale_config(cfg, self.data_loader)
self.scheduler = self.build_lr_scheduler(cfg,
self.optimizer,
epoch_iters=epoch_iters)
# Assume no other objects need to be checkpointed.
# We can later make it checkpoint the stateful hooks
self.checkpointer = DefaultCheckpointer(
# Assume you want to save checkpoints together with logs/statistics
self.model,
cfg.OUTPUT_DIR,
optimizer=self.optimizer,
scheduler=self.scheduler,
)
self.start_iter = 0
self.start_epoch = 0
self.max_iter = cfg.SOLVER.LR_SCHEDULER.MAX_ITER
self.max_epoch = cfg.SOLVER.LR_SCHEDULER.MAX_EPOCH
self.window_size = cfg.TRAINER.WINDOW_SIZE
self.cfg = cfg
self.register_hooks(self.build_hooks())
with open(md_file, "w") as f:
f.write("# {} ".format(title))
for dump_info_per_task in dump_infos:
task_name = dump_info_per_task["task"]
tables = dump_info_per_task["tables"]
tables = [table.replace("\n", " \n") for table in tables]
f.write(
"\n\n## Evaluation results for {}: \n\n".format(task_name))
f.write(tables[0])
f.write("\n")
if __name__ == "__main__":
import argparse
from cvpods.utils import setup_logger
setup_logger()
parser = argparse.ArgumentParser()
parser.add_argument("--gt-json")
parser.add_argument("--gt-dir")
parser.add_argument("--pred-json")
parser.add_argument("--pred-dir")
args = parser.parse_args()
from panopticapi.evaluation import pq_compute
with contextlib.redirect_stdout(io.StringIO()):
pq_res = pq_compute(args.gt_json,
args.pred_json,
gt_folder=args.gt_dir,
pred_folder=args.pred_dir)
