def main():
parser = argparse.ArgumentParser(description="PyTorch Object Detection Training")
parser.add_argument(
"--config-file",
default="",
metavar="FILE",
help="path to config file",
type=str,
)
parser.add_argument("--local_rank", type=int, default=0)
parser.add_argument(
"--skip-test",
dest="skip_test",
help="Do not test the final model",
action="store_true",
)
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER,
)
args = parser.parse_args()
num_gpus = int(os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
args.distributed = num_gpus > 1
if args.distributed:
torch.cuda.set_device(args.local_rank)
torch.distributed.deprecated.init_process_group(
backend="nccl", init_method="env://"
)
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
cfg.freeze()
output_dir = cfg.OUTPUT_DIR
if output_dir:
mkdir(output_dir)
logger = setup_logger("maskrcnn_benchmark", output_dir, get_rank())
logger.info("Using {} GPUs".format(num_gpus))
logger.info(args)
logger.info("Collecting env info (might take some time)")
logger.info("\n" + collect_env_info())
logger.info("Loaded configuration file {}".format(args.config_file))
with open(args.config_file, "r") as cf:
config_str = "\n" + cf.read()
logger.info(config_str)
logger.info("Running with config:\n{}".format(cfg))
model = train(cfg, args.local_rank, args.distributed)
if not args.skip_test:
test(cfg, model, args.distributed)
def main():
torch.cuda.set_device(0)
parser = argparse.ArgumentParser(description="PyTorch Object Detection Training")
parser.add_argument(
"--config-file",
default="/home/SelfDriving/maskrcnn/maskrcnn-benchmark/configs/e2e_faster_rcnn_R_50_C4_1x.yaml",
metavar="FILE",
help="path to config file",
type=str,
)
parser.add_argument("--local_rank", type=int, default=0)
parser.add_argument(
"--skip-test",
dest="skip_test",
help="Do not test the final model",
action="store_true",
)
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER,
)
args = parser.parse_args()
num_gpus = int(os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
args.distributed = num_gpus > 1
if args.distributed:
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(
backend="nccl", init_method="env://"
)
synchronize()
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
cfg.freeze()
output_dir = cfg.OUTPUT_DIR
if output_dir:
mkdir(output_dir)
logger = setup_logger("maskrcnn_benchmark", output_dir, get_rank())
logger.info("Using {} GPUs".format(num_gpus))
logger.info(args)
logger.info("Collecting env info (might take some time)")
logger.info("\n" + collect_env_info())
logger.info("Loaded configuration file {}".format(args.config_file))
with open(args.config_file, "r") as cf:
config_str = "\n" + cf.read()
logger.info(config_str)
logger.info("Running with config:\n{}".format(cfg))
output_config_path = os.path.join(cfg.OUTPUT_DIR, 'config.yml')
logger.info("Saving config into: {}".format(output_config_path))
# save overloaded model config in the output directory
save_config(cfg, output_config_path)
model = train(cfg, args.local_rank, args.distributed)
def main():
parser = argparse.ArgumentParser(description="PyTorch Object Detection Inference")
parser.add_argument(
"--config-file",
default="/private/home/fmassa/github/detectron.pytorch_v2/configs/e2e_faster_rcnn_R_50_C4_1x_caffe2.yaml",
metavar="FILE",
help="path to config file",
)
parser.add_argument("--local_rank", type=int, default=0)
parser.add_argument("--no-eval", default=False, action="store_true")
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER,
)
args = parser.parse_args()
num_gpus = int(os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
distributed = num_gpus > 1
if distributed:
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(
backend="nccl", init_method="env://"
)
synchronize()
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
cfg.freeze()
save_dir = ""
logger = setup_logger("maskrcnn_benchmark", save_dir, get_rank())
logger.info("Using {} GPUs".format(num_gpus))
logger.info("\n" + str(cfg))
logger.info("Collecting env info (might take some time)")
logger.info("\n" + collect_env_info())
seed = cfg.MODEL.SEED
if seed >= 0:
if distributed:
seed += args.local_rank
torch.manual_seed(seed)
if torch.cuda.is_available():
torch.cuda.manual_seed(seed)
logger.info("Using seed: {}".format(seed))
model = build_detection_model(cfg)
model.to(cfg.MODEL.DEVICE)
output_dir = cfg.OUTPUT_DIR
checkpointer = DetectronCheckpointer(cfg, model, save_dir=output_dir, force_weight=cfg.MODEL.FORCE_WEIGHT)
iou_types = ("bbox",)
if cfg.MODEL.MASK_ON:
iou_types = iou_types + ("segm",)
if cfg.MODEL.KEYPOINT_ON:
iou_types = iou_types + ("keypoints",)
output_folders = [None] * len(cfg.DATASETS.TEST)
dataset_names = cfg.DATASETS.TEST
if cfg.OUTPUT_DIR:
for idx, dataset_name in enumerate(dataset_names):
output_folder = os.path.join(cfg.OUTPUT_DIR, "inference", dataset_name)
mkdir(output_folder)
output_folders[idx] = output_folder
patt = re.compile(r'^model_(\d+).pth$')
names = list(os.listdir(os.path.join(cfg.OUTPUT_DIR)))
names = filter(lambda x: patt.match(x), names)
names = list(map(lambda x: (patt.match(x).group(1), x), names))
names.sort(key=lambda x: x[0])
# names.reverse()
for iter, name in names:
iter = int(iter)
_ = checkpointer.load(os.path.join(cfg.OUTPUT_DIR, name))
run_test(
iter=iter,
model=model,
args=args,
iou_types=iou_types,
output_folders=output_folders,
dataset_names=dataset_names,
distributed=distributed,
)
def main():
parser = argparse.ArgumentParser(
description="PyTorch Object Detection Training")
parser.add_argument(
"--config-file",
default="",
metavar="FILE",
help="path to config file",
type=str,
)
parser.add_argument("--local_rank", type=int, default=0)
parser.add_argument(
"--skip-test",
dest="skip_test",
help="Do not test the final model",
action="store_true",
)
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER,
)
args = parser.parse_args()
num_gpus = int(
os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
args.distributed = num_gpus > 1
if args.distributed:
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(backend="nccl",
init_method="env://")
synchronize()
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
cfg.freeze()
output_dir = cfg.OUTPUT_DIR
if output_dir:
mkdir(output_dir)
from datetime import datetime
if is_main_process():
current_time = datetime.now().strftime('%b%d_%H-%M-%S')
print(cfg.OUTPUT_DIR)
log_dir = os.path.join(cfg.OUTPUT_DIR, current_time)
tb_writer = tensorboardX.SummaryWriter(log_dir=log_dir)
print(log_dir)
else:
tb_writer = None
logger = setup_logger("maskrcnn_benchmark", output_dir, get_rank())
logger.info("Using {} GPUs".format(num_gpus))
logger.info(args)
logger.info("Collecting env info (might take some time)")
logger.info("\n" + collect_env_info())
logger.info("Loaded configuration file {}".format(args.config_file))
with open(args.config_file, "r") as cf:
config_str = "\n" + cf.read()
logger.info(config_str)
logger.info("Running with config:\n{}".format(cfg))
model = train(cfg, args.local_rank, args.distributed, tb_writer)
if not args.skip_test:
run_test(cfg,
model,
args.distributed,
iter=cfg.SOLVER.MAX_ITER,
valid=False)
def main():
args = parse_args()
num_gpus = int(
os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
distributed = num_gpus > 1
if distributed:
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(backend="nccl",
init_method="env://")
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
output_dir = os.path.dirname(cfg.MODEL.WEIGHT)
cfg.OUTPUT_DIR = output_dir
if not os.path.exists(output_dir):
os.makedirs(output_dir)
cfg.freeze()
save_dir = ""
logger = setup_logger("maskrcnn_benchmark", save_dir, get_rank())
logger.info("Using {} GPUs".format(num_gpus))
logger.info(cfg)
logger.info("Collecting env info (might take some time)")
logger.info("\n" + collect_env_info())
model = build_detection_model(cfg)
model.to(cfg.MODEL.DEVICE)
checkpointer = DetectronCheckpointer(cfg, model, save_dir=cfg.MODEL.WEIGHT)
_ = checkpointer.load(cfg.MODEL.WEIGHT)
iou_types = ("bbox", )
if cfg.MODEL.MASK_ON:
iou_types = iou_types + ("segm", )
output_folders = [None] * len(cfg.DATASETS.TEST)
dataset_names = cfg.DATASETS.TEST
if cfg.OUTPUT_DIR:
for idx, dataset_name in enumerate(dataset_names):
output_folder = os.path.join(cfg.OUTPUT_DIR, "inference",
dataset_name)
mkdir(output_folder)
output_folders[idx] = output_folder
data_loaders_val = make_data_loader(cfg,
is_train=False,
is_distributed=distributed)
for output_folder, dataset_name, data_loader_val in zip(
output_folders, dataset_names, data_loaders_val):
inference(
model,
data_loader_val,
dataset_name=dataset_name,
iou_types=iou_types,
box_only=cfg.MODEL.RPN_ONLY,
device=cfg.MODEL.DEVICE,
expected_results=cfg.TEST.EXPECTED_RESULTS,
expected_results_sigma_tol=cfg.TEST.EXPECTED_RESULTS_SIGMA_TOL,
output_folder=output_folder,
)
synchronize()
def test_maskscoring_rcnn(config_file):
import sys
sys.path.append('./detection_model/maskscoring_rcnn')
import argparse
import os
os.environ['CUDA_VISIBLE_DEVICES'] = '2'
import torch
#from maskrcnn_benchmark.config import cfg
from maskrcnn_benchmark.data import make_data_loader
from maskrcnn_benchmark.engine.inference import inference
from maskrcnn_benchmark.modeling.detector import build_detection_model
from maskrcnn_benchmark.utils.checkpoint import DetectronCheckpointer
from maskrcnn_benchmark.utils.collect_env import collect_env_info
from maskrcnn_benchmark.utils.comm import synchronize, get_rank
from maskrcnn_benchmark.utils.logger import setup_logger
from maskrcnn_benchmark.utils.miscellaneous import mkdir
from yacs.config import CfgNode as CN
def read_config_file(config_file):
"""
read config information form yaml file
"""
f = open(config_file)
opt = CN.load_cfg(f)
return opt
opt = read_config_file(config_file)
num_gpus = int(os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
distributed = num_gpus > 1
if distributed:
torch.cuda.set_device(opt.local_rank)
torch.distributed.deprecated.init_process_group(
backend="nccl", init_method="env://"
)
save_dir = ""
logger = setup_logger("maskrcnn_benchmark", save_dir, get_rank())
logger.info("Using {} GPUs".format(num_gpus))
logger.info(opt)
logger.info("Collecting env info (might take some time)")
logger.info("\n" + collect_env_info())
model = build_detection_model(opt)
model.to(opt.MODEL.DEVICE)
output_dir = opt.OUTPUT_DIR
checkpointer = DetectronCheckpointer(opt, model, save_dir=output_dir)
_ = checkpointer.load(opt.MODEL.WEIGHT)
iou_types = ("bbox",)
if opt.MODEL.MASK_ON:
iou_types = iou_types + ("segm",)
output_folders = [None] * len(opt.DATASETS.TEST)
if opt.OUTPUT_DIR:
dataset_names = opt.DATASETS.TEST
for idx, dataset_name in enumerate(dataset_names):
output_folder = os.path.join(opt.OUTPUT_DIR, "inference", dataset_name)
mkdir(output_folder)
output_folders[idx] = output_folder
data_loaders_val = make_data_loader(opt, is_train=False, is_distributed=distributed)
for output_folder, data_loader_val in zip(output_folders, data_loaders_val):
inference(
model,
data_loader_val,
iou_types=iou_types,
box_only=opt.MODEL.RPN_ONLY,
device=opt.MODEL.DEVICE,
expected_results=opt.TEST.EXPECTED_RESULTS,
expected_results_sigma_tol=opt.TEST.EXPECTED_RESULTS_SIGMA_TOL,
output_folder=output_folder,
maskiou_on=opt.MODEL.MASKIOU_ON
)
synchronize()
def main():
# apply_prior prior_mask
# 0 - -
# 1 Y -
# 2 - Y
# 3 Y Y
parser = argparse.ArgumentParser(
description="PyTorch Object Detection Inference")
parser.add_argument(
"--config-file",
default="configs/e2e_faster_rcnn_R_50_FPN_1x_HICO_sp_only.yaml",
metavar="FILE",
help="path to config file",
)
parser.add_argument("--local_rank", type=int, default=0)
parser.add_argument(
"--ckpt",
help=
"The path to the checkpoint for test, default is the latest checkpoint.",
default=None,
)
parser.add_argument('--num_iteration',
dest='num_iteration',
help='Specify which weight to load',
default=-1,
type=int)
parser.add_argument('--object_thres',
dest='object_thres',
help='Object threshold',
default=0.4,
type=float) # used to be 0.4 or 0.05
parser.add_argument('--human_thres',
dest='human_thres',
help='Human threshold',
default=0.6,
type=float)
parser.add_argument('--prior_flag',
dest='prior_flag',
help='whether use prior_flag',
default=1,
type=int)
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER,
)
args = parser.parse_args()
num_gpus = int(
os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
distributed = num_gpus > 1 and torch.cuda.is_available()
if distributed:
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(backend="nccl",
init_method="env://")
synchronize()
print('prior flag: {}'.format(args.prior_flag))
ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
DATA_DIR = os.path.abspath(os.path.join(ROOT_DIR, 'Data'))
args.config_file = os.path.join(ROOT_DIR, args.config_file)
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
cfg.freeze()
save_dir = ""
logger = setup_logger("DRG", save_dir, get_rank())
logger.info("Using {} GPUs".format(num_gpus))
logger.info(cfg)
logger.info("Collecting env info (might take some time)")
logger.info("\n" + collect_env_info())
model = build_detection_model(cfg)
# model.to(cfg.MODEL.DEVICE)
device = torch.device(
"cuda") if torch.cuda.is_available() else torch.device("cpu")
model.to(device)
# # Initialize mixed-precision if necessary
use_mixed_precision = cfg.DTYPE == 'float16'
amp_handle = amp.init(enabled=use_mixed_precision, verbose=cfg.AMP_VERBOSE)
output_dir = cfg.OUTPUT_DIR
checkpointer = DetectronCheckpointer(cfg, model, save_dir=output_dir)
if args.num_iteration != -1:
args.ckpt = os.path.join(cfg.OUTPUT_DIR,
'model_%07d.pth' % args.num_iteration)
ckpt = cfg.MODEL.WEIGHT if args.ckpt is None else args.ckpt
logger.info("Testing checkpoint {}".format(ckpt))
_ = checkpointer.load(ckpt, use_latest=args.ckpt is None)
# iou_types = ("bbox",)
output_folders = [None] * len(cfg.DATASETS.TEST)
dataset_names = cfg.DATASETS.TEST
if cfg.OUTPUT_DIR:
for idx, dataset_name in enumerate(dataset_names):
if args.num_iteration != -1:
output_folder = os.path.join(cfg.OUTPUT_DIR, "inference_sp",
dataset_name,
"model_%07d" % args.num_iteration)
else:
output_folder = os.path.join(cfg.OUTPUT_DIR, "inference_sp",
dataset_name)
mkdir(output_folder)
output_folders[idx] = output_folder
opt = {}
opt['word_dim'] = 300
opt['use_thres_dic'] = 1
for output_folder, dataset_name in zip(output_folders, dataset_names):
data = DatasetCatalog.get(dataset_name)
data_args = data["args"]
test_detection = pickle.load(open(data_args['test_detection_file'],
"rb"),
encoding='latin1')
word_embeddings = pickle.load(open(data_args['word_embedding_file'],
"rb"),
encoding='latin1')
opt['thres_dic'] = pickle.load(open(data_args['threshold_dic'], "rb"),
encoding='latin1')
output_file = os.path.join(output_folder, 'detection.pkl')
# hico_folder = os.path.join(output_folder, 'HICO')
output_map_folder = os.path.join(output_folder, 'map')
logger.info("Output will be saved in {}".format(output_file))
logger.info("Start evaluation on {} dataset.".format(dataset_name))
run_test(model,
dataset_name=dataset_name,
test_detection=test_detection,
word_embeddings=word_embeddings,
output_file=output_file,
object_thres=args.object_thres,
human_thres=args.human_thres,
device=device,
cfg=cfg,
opt=opt)
# Generate_HICO_detection(output_file, hico_folder)
compute_hico_map(output_map_folder, output_file, 'test')
def main():
parser = argparse.ArgumentParser(description="PyTorch Object Detection Training")
parser.add_argument(
"--config-file",
default="../configs/kidney/e2e_mask_rcnn_X_101_32x8d_FPN_1x_liver_augmentation_test.yaml",
metavar="FILE",
help="path to config file",
type=str,
)
parser.add_argument("--local_rank", type=int, default=0)
parser.add_argument(
"--skip-test",
dest="skip_test",
help="Do not test the final model",
action="store_true",
)
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER,
)
args = parser.parse_args()
num_gpus = int(os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
args.distributed = num_gpus > 1
if args.distributed:
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(
backend="nccl", init_method="env://"
)
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
if cfg.OUTPUT_SUB_DIR:
output_dir = os.path.join(cfg.OUTPUT_DIR, cfg.OUTPUT_SUB_DIR)
else:
now = time.localtime()
time_dir_name = "%04d%02d%02d-%02d%02d%02d" % (
now.tm_year, now.tm_mon, now.tm_mday, now.tm_hour, now.tm_min, now.tm_sec)
output_dir = os.path.join(cfg.OUTPUT_DIR, time_dir_name)
cfg.merge_from_list(["OUTPUT_DIR", output_dir])
cfg.freeze()
if output_dir:
mkdir(output_dir)
logger = setup_logger("maskrcnn_benchmark", output_dir, get_rank())
logger.info("Using {} GPUs".format(num_gpus))
logger.info(args)
logger.info("Collecting env info (might take some time)")
logger.info("\n" + collect_env_info())
logger.info("Loaded configuration file {}".format(args.config_file))
with open(args.config_file, "r") as cf:
config_str = "\n" + cf.read()
logger.info(config_str)
logger.info("Running with config:\n{}".format(cfg))
model = train(cfg, args.local_rank, args.distributed)
def main():
parser = argparse.ArgumentParser(
description="PyTorch Object Detection Training")
parser.add_argument(
"--config-file",
default="configs/e2e_mask_rcnn_R_50_FPN_1x.yaml",
metavar="FILE",
help="path to config file",
type=str,
)
parser.add_argument("--local_rank", type=int, default=0)
parser.add_argument(
"--skip-test",
dest="skip_test",
help="Do not test the final model",
action="store_true",
)
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER, # 所有剩余的参数，均转化为一个列表赋值给此项
)
args = parser.parse_args()
# 获取 gpu 的数量
num_gpus = int(
os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
# 是否进行多GPU的分布式训练
args.distributed = num_gpus > 1
# args.distributed = 0
# 设置分布式训练时的一些初始化信息
if args.distributed:
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(backend="nccl",
init_method="env://")
synchronize()
# 将 config_file 指定的配置项覆盖到默认配置项当中
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
cfg.freeze()
output_dir = cfg.OUTPUT_DIR
if output_dir:
mkdir(output_dir)
# 下面的多行代码都是一些向屏幕上输出相关信息的日志代码
# 同时也会保存在 output_dir 文件夹下的 log.txt 文件内
# 输出的信息来源于自身系统以及配置文件中的信息
logger = setup_logger("maskrcnn_benchmark", output_dir, get_rank())
logger.info("Using {} GPUs".format(num_gpus))
logger.info(args)
logger.info("Collecting env info (might take some time)")
logger.info("\n" + collect_env_info())
logger.info("Loaded configuration file {}".format(args.config_file))
with open(args.config_file, "r") as cf:
config_str = "\n" + cf.read()
logger.info(config_str)
logger.info("Running with config:\n{}".format(cfg))
# 调用 train 函数, 该函数会执行模型训练的代码逻辑
model = train(cfg, args.local_rank, args.distributed)
if not args.skip_test:
run_test(cfg, model, args.distributed)
def main():
parser = argparse.ArgumentParser(
description="PyTorch Object Detection Training")
parser.add_argument(
"--config-file",
default="",
metavar="FILE",
help="path to config file",
type=str,
)
parser.add_argument("--local_rank", type=int, default=0)
parser.add_argument(
"--data-dir",
default=".",
metavar="DIR",
help="data dir for training",
type=str,
)
parser.add_argument(
"--out-dir",
default=".",
metavar="DIR",
help="output dir for model",
type=str,
)
parser.add_argument(
"--gpu_ids",
default="-1",
help="gpu id",
type=str,
)
parser.add_argument(
"--skip-test",
dest="skip_test",
help="Do not test the final model",
action="store_true",
)
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER,
)
args = parser.parse_args()
if args.gpu_ids != '-1':
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu_ids
num_gpus = int(
os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
args.distributed = num_gpus > 1
if args.distributed:
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(backend="nccl",
init_method="env://")
synchronize()
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
cfg.MODEL.WEIGHT = args.data_dir + cfg.MODEL.WEIGHT[1:]
cfg.DATA_DIR = args.data_dir + cfg.DATA_DIR[1:]
cfg.OUTPUT_DIR = args.out_dir + cfg.OUTPUT_DIR[1:]
print("cfg.OUTPUT_DIR: ", cfg.OUTPUT_DIR)
print("cfg.MODEL.WEIGHT: ", cfg.MODEL.WEIGHT)
print("cfg.DATA_DIR: ", cfg.DATA_DIR)
print("cfg.MODEL.ATTRIBUTE_ON: ", cfg.MODEL.ATTRIBUTE_ON)
cfg.freeze()
output_dir = cfg.OUTPUT_DIR
if output_dir:
mkdir(output_dir)
logger = setup_logger("maskrcnn_benchmark", output_dir, get_rank())
logger.info("Using {} GPUs".format(num_gpus))
logger.info(args)
logger.info("Collecting env info (might take some time)")
logger.info("\n" + collect_env_info())
logger.info("Loaded configuration file {}".format(args.config_file))
with open(args.config_file, "r") as cf:
config_str = "\n" + cf.read()
logger.info(config_str)
logger.info("Running with config:\n{}".format(cfg))
model = train(cfg, args.local_rank, args.distributed)
if not args.skip_test:
run_test(cfg, model, args.distributed)
def main():
configure_logger(constants.MASKRCNN)
log_start(key=constants.INIT_START)
parser = argparse.ArgumentParser(
description="PyTorch Object Detection Training")
parser.add_argument(
"--config-file",
default="",
metavar="FILE",
help="path to config file",
type=str,
)
parser.add_argument("--local_rank", type=int, default=0)
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER,
)
args = parser.parse_args()
num_gpus = int(
os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
args.distributed = num_gpus > 1
if args.distributed:
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(backend="nccl",
init_method="env://")
synchronize()
# setting seeds - needs to be timed, so after RUN_START
if is_main_process():
master_seed = random.SystemRandom().randint(0, 2**32 - 1)
seed_tensor = torch.tensor(master_seed,
dtype=torch.float32,
device=torch.device("cuda"))
else:
seed_tensor = torch.tensor(0,
dtype=torch.float32,
device=torch.device("cuda"))
torch.distributed.broadcast(seed_tensor, 0)
master_seed = int(seed_tensor.item())
else:
# random master seed, random.SystemRandom() uses /dev/urandom on Unix
master_seed = random.SystemRandom().randint(0, 2**32 - 1)
# actually use the random seed
args.seed = master_seed
# random number generator with seed set to master_seed
random_number_generator = random.Random(master_seed)
log_event(key=constants.SEED, value=master_seed)
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
cfg.freeze()
output_dir = cfg.OUTPUT_DIR
if output_dir:
mkdir(output_dir)
logger = setup_logger("maskrcnn_benchmark", output_dir, get_rank())
logger.info("Using {} GPUs".format(num_gpus))
logger.info(args)
# generate worker seeds, one seed for every distributed worker
worker_seeds = generate_seeds(
random_number_generator,
torch.distributed.get_world_size()
if torch.distributed.is_initialized() else 1)
# todo sharath what if CPU
# broadcast seeds from rank=0 to other workers
worker_seeds = broadcast_seeds(worker_seeds, device='cuda')
# Setting worker seeds
logger.info("Worker {}: Setting seed {}".format(
args.local_rank, worker_seeds[args.local_rank]))
torch.manual_seed(worker_seeds[args.local_rank])
logger.info("Collecting env info (might take some time)")
logger.info("\n" + collect_env_info())
logger.info("Loaded configuration file {}".format(args.config_file))
with open(args.config_file, "r") as cf:
config_str = "\n" + cf.read()
logger.info(config_str)
logger.info("Running with config:\n{}".format(cfg))
model, success = train(cfg, args.local_rank, args.distributed,
args.disable_allreduce_for_logging,
random_number_generator)
if success is not None:
if success:
log_end(key=constants.RUN_STOP, metadata={"status": "success"})
else:
log_end(key=constants.RUN_STOP, metadata={"status": "aborted"})
def main():
parser = argparse.ArgumentParser(
description="PyTorch Object Detection Inference")
parser.add_argument(
"--config-file",
default=
"/private/home/fmassa/github/detectron.pytorch_v2/configs/e2e_faster_rcnn_R_50_C4_1x_caffe2.yaml",
metavar="FILE",
help="path to config file",
)
parser.add_argument("--local_rank", type=int, default=0)
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER,
)
args = parser.parse_args()
import random
import torch.backends.cudnn as cudnn
import numpy as np
seed = 1
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed + 1)
random.seed(seed + 2)
np.random.seed(seed + 3)
print('use seed')
cudnn.deterministic = True
num_gpus = int(
os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
distributed = num_gpus > 1
if distributed:
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(backend="nccl",
init_method="env://")
synchronize()
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
cfg.freeze()
save_dir = ""
logger = setup_logger("maskrcnn_benchmark", save_dir, get_rank())
logger.info("Using {} GPUs".format(num_gpus))
logger.info(cfg)
logger.info("Collecting env info (might take some time)")
logger.info("\n" + collect_env_info())
model = build_detection_model(cfg)
model.to(cfg.MODEL.DEVICE)
# load a reid model
reid_model = build_reid_model(cfg)
reid_model.to(cfg.MODEL.DEVICE)
print('#######loading reid model from {}#######'.format(
cfg.REID.TEST.WEIGHT))
f = torch.load(
cfg.REID.TEST.WEIGHT,
map_location=torch.device("cpu"),
)
if 'model' in f:
load_state_dict(reid_model, f['model'])
else:
reid_model.load_state_dict(f, strict=False)
output_dir = os.path.join(cfg.OUTPUT_DIR, cfg.SUBDIR)
print('Output dir: {}'.format(output_dir))
iou_types = ("bbox", )
if cfg.MODEL.MASK_ON:
iou_types = iou_types + ("segm", )
if cfg.MODEL.KEYPOINT_ON:
iou_types = iou_types + ("keypoints", )
dataset_names = cfg.DATASETS.TEST
if cfg.OUTPUT_DIR:
output_folder = os.path.join(output_dir, "inference",
dataset_names[0].split('_')[0])
mkdir(output_folder)
data_loaders_val = make_data_loader(cfg,
is_train=False,
is_distributed=distributed)
checkpointer = DetectronCheckpointer(cfg, model, save_dir=output_dir)
_ = checkpointer.load(cfg.MODEL.WEIGHT)
print('#######loading detector from {}#######'.format(cfg.MODEL.WEIGHT))
inference(
reid_model,
model,
data_loaders_val,
dataset_name=dataset_names[0].split('_')[0],
iou_types=iou_types,
box_only=False if cfg.MODEL.RETINANET_ON else cfg.MODEL.RPN_ONLY,
device=cfg.MODEL.DEVICE,
expected_results=cfg.TEST.EXPECTED_RESULTS,
expected_results_sigma_tol=cfg.TEST.EXPECTED_RESULTS_SIGMA_TOL,
output_folder=output_folder,
)
synchronize()
def main():
parser = argparse.ArgumentParser(
description="PyTorch Object Detection Inference")
parser.add_argument(
"--student-file",
default=
"/private/home/fmassa/github/detectron.pytorch_v2/configs/e2e_faster_rcnn_R_50_C4_1x_caffe2.yaml",
metavar="FILE",
help="path to config file",
)
parser.add_argument(
"--teacher-file",
default=
"/private/home/fmassa/github/detectron.pytorch_v2/configs/e2e_faster_rcnn_R_50_C4_1x_caffe2.yaml",
metavar="FILE",
help="path to config file",
)
parser.add_argument("--local_rank", type=int, default=0)
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER,
)
parser.add_argument(
'--teacher-pretrained',
type=str,
default='True',
help=
'Load weights from previously saved parameters. You can specify parameter file name.'
)
parser.add_argument(
'--student-pretrained',
type=str,
default='True',
help=
'Load weights from previously saved parameters. You can specify parameter file name.'
)
args = parser.parse_args()
num_gpus = int(
os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
distributed = num_gpus > 1
if distributed:
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(backend="nccl",
init_method="env://")
synchronize()
save_dir = ""
logger = setup_logger("maskrcnn_benchmark", save_dir, get_rank())
logger.info("Using {} GPUs".format(num_gpus))
logger.info(cfg)
logger.info("Collecting env info (might take some time)")
logger.info("\n" + collect_env_info())
#########===================student net=================#############
cfg.merge_from_file(args.student_file)
cfg.merge_from_list(args.opts)
output_dir = cfg.OUTPUT_DIR
Student_G, _, _ = build_detection_model_GAN_Disting(cfg)
Student_G.to(cfg.MODEL.DEVICE)
Student_G_checkpointer = DetectronCheckpointer(cfg,
Student_G,
save_dir=output_dir)
_ = Student_G_checkpointer.load(args.student_pretrained)
####teacher net##########
cfg.merge_from_file(args.teacher_file)
cfg.merge_from_list(args.opts)
cfg.freeze()
output_dir = cfg.OUTPUT_DIR
_, Teacher_Head, _ = build_detection_model_GAN_Disting(cfg)
Teacher_Head.to(cfg.MODEL.DEVICE)
Teacher_Head_checkpointer = DetectronCheckpointer(cfg,
Teacher_Head,
save_dir=output_dir)
_ = Teacher_Head_checkpointer.load(args.teacher_pretrained)
model = torch.nn.Sequential(*[Student_G, Teacher_Head])
iou_types = ("bbox", )
if cfg.MODEL.MASK_ON:
iou_types = iou_types + ("segm", )
if cfg.MODEL.KEYPOINT_ON:
iou_types = iou_types + ("keypoints", )
output_folders = [None] * len(cfg.DATASETS.TEST)
dataset_names = cfg.DATASETS.TEST
if cfg.OUTPUT_DIR:
for idx, dataset_name in enumerate(dataset_names):
output_folder = os.path.join(cfg.OUTPUT_DIR, "inference",
dataset_name)
mkdir(output_folder)
output_folders[idx] = output_folder
data_loaders_val = make_data_loader(cfg,
is_train=False,
is_distributed=distributed)
for output_folder, dataset_name, data_loader_val in zip(
output_folders, dataset_names, data_loaders_val):
inference_G_HEAD(
model,
data_loader_val,
dataset_name=dataset_name,
iou_types=iou_types,
box_only=False if cfg.MODEL.FCOS_ON or cfg.MODEL.RETINANET_ON else
cfg.MODEL.RPN_ONLY,
device=cfg.MODEL.DEVICE,
expected_results=cfg.TEST.EXPECTED_RESULTS,
expected_results_sigma_tol=cfg.TEST.EXPECTED_RESULTS_SIGMA_TOL,
output_folder=output_folder,
)
synchronize()
import numpy as np
import torch
from torchvision import transforms as T
from matplotlib.pyplot import clf, close, figure, imshow, title, show
from maskrcnn_benchmark.config import cfg
from maskrcnn_benchmark.utils.visualization import (COCO_CATEGORIES,
DetectionVisualizer)
from maskrcnn_benchmark.modeling.detector import build_detection_model
from maskrcnn_benchmark.modeling.roi_heads.mask_head.inference import Masker
from maskrcnn_benchmark.structures.image_list import to_image_list
from maskrcnn_benchmark.utils.logger import setup_logger
from maskrcnn_benchmark.utils.model_serialization import load_state_dict
LOG = setup_logger(__name__, None, 0)
def build_transforms(cfg):
transforms = [T.ToTensor()] # [0-1]
if cfg.INPUT.TO_BGR255:
transforms.append(T.Lambda(lambda x: x[[2, 1, 0]] * 255.))
else:
transforms.append(T.Lambda(lambda x: x * 255.))
mean, std = cfg.INPUT.PIXEL_MEAN, cfg.INPUT.PIXEL_STD
transforms.append(T.Normalize(mean, std))
return T.Compose(transforms)
def main():
parser = argparse.ArgumentParser(
description="PyTorch Object Detection Inference")
parser.add_argument(
"--config-file",
default=
"/private/home/fmassa/github/detectron.pytorch_v2/configs/e2e_faster_rcnn_R_50_C4_1x_caffe2.yaml",
metavar="FILE",
help="path to config file",
)
parser.add_argument("--local_rank", type=int, default=0)
parser.add_argument(
"--ckpt",
help=
"The path to the checkpoint for test, default is the latest checkpoint.",
default=None,
)
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER,
)
args = parser.parse_args()
num_gpus = int(
os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
distributed = num_gpus > 1
if distributed:
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(backend="nccl",
init_method="env://")
synchronize()
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
cfg.freeze()
save_dir = ""
logger = setup_logger("maskrcnn_benchmark", save_dir, get_rank())
logger.info("Using {} GPUs".format(num_gpus))
logger.info(cfg)
logger.info("Collecting env info (might take some time)")
logger.info("\n" + collect_env_info())
model = build_face_trainer(cfg)
model.to(cfg.MODEL.DEVICE)
# Initialize mixed-precision if necessary
use_mixed_precision = cfg.DTYPE == 'float16'
amp_handle = amp.init(enabled=use_mixed_precision, verbose=cfg.AMP_VERBOSE)
output_dir = cfg.OUTPUT_DIR
checkpointer = DetectronCheckpointer(cfg, model, save_dir=output_dir)
ckpt = cfg.MODEL.WEIGHT if args.ckpt is None else args.ckpt
_ = checkpointer.load(ckpt, use_latest=args.ckpt is None)
TEST_PATH_LIST = cfg.DATASETS.TEST_PATH_LIST
for TEST_PATH in TEST_PATH_LIST:
if cfg.OUTPUT_DIR:
output_folder = os.path.join(cfg.OUTPUT_DIR, "inference",
os.path.basename(TEST_PATH))
mkdir(output_folder)
total_time, acc = rank_1(model, TEST_PATH, True, cfg.MODEL.DEVICE,
cfg.INPUT.SIZE_TRAIN,
cfg.MODEL.BACKBONE.BACKBONE_OUT_CHANNELS,
get_transform(cfg), cfg.SOLVER.IMS_PER_BATCH,
'test')
TEST_PATH = os.path.basename(TEST_PATH)
logger.info(
"the {:s} rank-1 program consume time {:.4f} seconds and the accuracy is {:.4f}"
.format(TEST_PATH, total_time, acc))
def main():
parser = argparse.ArgumentParser(
description="PyTorch Object Detection Training")
parser.add_argument(
"--config-file",
default="../configs/e2e_mask_rcnn_R_50_C4_1x.yaml",
metavar="FILE",
help="path to config file",
type=str,
)
parser.add_argument("--local_rank", type=int, default=0)
parser.add_argument(
"--skip-test",
dest="skip_test",
help="Do not test the final model",
action="store_true",
)
parser.add_argument(
"--opts",
help="Modify config options using the command-line",
default=[
"SOLVER.IMS_PER_BATCH", 4, "SOLVER.BASE_LR", 0.005,
"SOLVER.MAX_ITER", 360000, "SOLVER.STEPS", "(240000, 320000)",
"TEST.IMS_PER_BATCH", 1, "MODEL.RPN.FPN_POST_NMS_TOP_N_TRAIN", 2000
],
# default=["MODEL.RPN.FPN_POST_NMS_TOP_N_TRAIN", 4000],
type=list,
nargs=argparse.REMAINDER,
)
args = parser.parse_args()
num_gpus = int(
os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
# args.distributed = num_gpus > 1
args.distributed = False
env_dist = os.environ
print(env_dist)
if args.distributed:
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(backend="nccl",
init_method="env://")
synchronize()
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
cfg.freeze()
output_dir = cfg.OUTPUT_DIR
if output_dir:
mkdir(output_dir)
print("get_rank(): ", get_rank())
logger = setup_logger("maskrcnn_benchmark", output_dir, get_rank())
logger.info("Using {} GPUs".format(num_gpus))
logger.info(args)
logger.info("Collecting env info (might take some time)")
logger.info("\n" + collect_env_info())
logger.info("Loaded configuration file {}".format(args.config_file))
with open(args.config_file, "r") as cf:
config_str = "\n" + cf.read()
logger.info(config_str)
logger.info("Running with config:\n{}".format(cfg))
model = train(cfg, args.local_rank, args.distributed)
if not args.skip_test:
run_test(cfg, model, args.distributed)
num_gpus = int(os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
distributed = num_gpus > 1
if distributed:
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(
backend="nccl", init_method="env://"
)
synchronize()
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
cfg.freeze()
save_dir = ""
logger = setup_logger("maskrcnn_benchmark", save_dir, get_rank())
if is_main_process():
dllogger.init(backends=[dllogger.JSONStreamBackend(verbosity=dllogger.Verbosity.VERBOSE,
filename=args.json_summary),
dllogger.StdOutBackend(verbosity=dllogger.Verbosity.VERBOSE, step_format=format_step)])
else:
dllogger.init(backends=[])
save_dir = ""
dllogger.log(step="PARAMETER", data={"config":cfg})
dllogger.log(step="PARAMETER", data={"gpu_count": num_gpus})
# dllogger.log(step="PARAMETER", data={"env_info": collect_env_info()})
model = build_detection_model(cfg)
model.to(cfg.MODEL.DEVICE)
def main():
mlperf_log.ROOT_DIR_MASKRCNN = os.path.dirname(os.path.abspath(__file__))
parser = argparse.ArgumentParser(
description="PyTorch Object Detection Training")
parser.add_argument(
"--config-file",
default="",
metavar="FILE",
help="path to config file",
type=str,
)
parser.add_argument("--local_rank", type=int, default=0)
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER,
)
args = parser.parse_args()
num_gpus = int(
os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
args.distributed = num_gpus > 1
if is_main_process:
# Setting logging file parameters for compliance logging
os.environ["COMPLIANCE_FILE"] = '/MASKRCNN_complVv0.5.0_' + str(
datetime.datetime.now())
mlperf_log.LOG_FILE = os.getenv("COMPLIANCE_FILE")
mlperf_log._FILE_HANDLER = logging.FileHandler(mlperf_log.LOG_FILE)
mlperf_log._FILE_HANDLER.setLevel(logging.DEBUG)
mlperf_log.LOGGER.addHandler(mlperf_log._FILE_HANDLER)
if args.distributed:
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(backend="nccl",
init_method="env://")
synchronize()
print_mlperf(key=mlperf_log.RUN_START)
# setting seeds - needs to be timed, so after RUN_START
if is_main_process():
master_seed = random.SystemRandom().randint(0, 2**32 - 1)
seed_tensor = torch.tensor(master_seed,
dtype=torch.float32,
device=torch.device("cuda"))
else:
seed_tensor = torch.tensor(0,
dtype=torch.float32,
device=torch.device("cuda"))
torch.distributed.broadcast(seed_tensor, 0)
master_seed = int(seed_tensor.item())
else:
print_mlperf(key=mlperf_log.RUN_START)
# random master seed, random.SystemRandom() uses /dev/urandom on Unix
master_seed = random.SystemRandom().randint(0, 2**32 - 1)
# actually use the random seed
args.seed = master_seed
# random number generator with seed set to master_seed
random_number_generator = random.Random(master_seed)
print_mlperf(key=mlperf_log.RUN_SET_RANDOM_SEED, value=master_seed)
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
cfg.freeze()
output_dir = cfg.OUTPUT_DIR
if output_dir:
mkdir(output_dir)
logger = setup_logger("maskrcnn_benchmark", output_dir, get_rank())
logger.info("Using {} GPUs".format(num_gpus))
logger.info(args)
# generate worker seeds, one seed for every distributed worker
worker_seeds = generate_seeds(
random_number_generator,
torch.distributed.get_world_size()
if torch.distributed.is_initialized() else 1)
# todo sharath what if CPU
# broadcast seeds from rank=0 to other workers
worker_seeds = broadcast_seeds(worker_seeds, device='cuda')
# Setting worker seeds
logger.info("Worker {}: Setting seed {}".format(
args.local_rank, worker_seeds[args.local_rank]))
torch.manual_seed(worker_seeds[args.local_rank])
logger.info("Collecting env info (might take some time)")
logger.info("\n" + collect_env_info())
logger.info("Loaded configuration file {}".format(args.config_file))
with open(args.config_file, "r") as cf:
config_str = "\n" + cf.read()
logger.info(config_str)
logger.info("Running with config:\n{}".format(cfg))
model = train(cfg, args.local_rank, args.distributed)
print_mlperf(key=mlperf_log.RUN_FINAL)
def pred_with_weight(args):
num_gpus = int(
os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
distributed = num_gpus > 1
if distributed:
torch.cuda.set_device(args.local_rank)
torch.distributed.deprecated.init_process_group(backend="nccl",
init_method="env://")
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
cfg.freeze()
save_dir = ""
logger = setup_logger("maskrcnn_benchmark", save_dir, get_rank())
logger.info("Using {} GPUs".format(num_gpus))
logger.info(cfg)
logger.info("Collecting env info (might take some time)")
logger.info("\n" + collect_env_info())
model = build_detection_model(cfg)
model.to(cfg.MODEL.DEVICE)
for weight in args.weights:
# skipping evaluations already performed
out_json = "{}/detections/{}.json".format(
cfg.OUTPUT_DIR,
weight.split('/')[-1].split('_')[-1].split('.')[0])
if os.path.exists(out_json):
print('skipping', out_json)
continue
checkpointer = DetectronCheckpointer(cfg, model)
_ = checkpointer.load(weight)
iou_types = ("bbox", )
if cfg.MODEL.MASK_ON:
iou_types = iou_types + ("segm", )
output_folders = [None] * len(cfg.DATASETS.TEST)
if cfg.OUTPUT_DIR:
dataset_names = cfg.DATASETS.TEST
for idx, dataset_name in enumerate(dataset_names):
output_folder = cfg.OUTPUT_DIR
mkdir(output_folder)
output_folders[idx] = output_folder
data_loaders_val = make_data_loader(cfg,
is_train=False,
is_distributed=distributed)
for output_folder, data_loader_val in zip(output_folders,
data_loaders_val):
inference(
model,
data_loader_val,
iou_types=iou_types,
box_only=cfg.MODEL.RPN_ONLY,
device=cfg.MODEL.DEVICE,
expected_results=cfg.TEST.EXPECTED_RESULTS,
expected_results_sigma_tol=cfg.TEST.EXPECTED_RESULTS_SIGMA_TOL,
output_folder=output_folder,
name=weight.split('_')[-1].split('.')[0])
synchronize()
def main():
parser = argparse.ArgumentParser(
description="PyTorch Object Detection Inference")
parser.add_argument(
"--config-file",
default=
"configs/e2e_faster_rcnn_R_50_FPN_1x_sp_only_object_centric.yaml",
metavar="FILE",
help="path to config file",
)
parser.add_argument("--local_rank", type=int, default=0)
parser.add_argument(
"--ckpt",
help=
"The path to the checkpoint for test, default is the latest checkpoint.",
default=None,
)
parser.add_argument(
"--dataset_name",
help="vcoco_val_test_object_centric or vcoco_test_object_centric",
default=None,
)
parser.add_argument('--num_iteration',
dest='num_iteration',
help='Specify which weight to load',
default=-1,
type=int)
parser.add_argument('--object_thres',
dest='object_thres',
help='Object threshold',
default=0.1,
type=float) # used to be 0.4 or 0.05
parser.add_argument('--human_thres',
dest='human_thres',
help='Human threshold',
default=0.8,
type=float)
parser.add_argument('--prior_flag',
dest='prior_flag',
help='whether use prior_flag',
default=1,
type=int)
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER,
)
args = parser.parse_args()
num_gpus = int(
os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
distributed = num_gpus > 1 and torch.cuda.is_available()
if distributed:
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(backend="nccl",
init_method="env://")
synchronize()
ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
# DATA_DIR = os.path.abspath(os.path.join(ROOT_DIR, 'Data'))
args.config_file = os.path.join(ROOT_DIR, args.config_file)
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
cfg.freeze()
save_dir = ""
logger = setup_logger("DRG", save_dir, get_rank())
logger.info("Using {} GPUs".format(num_gpus))
logger.info(cfg)
logger.info("Collecting env info (might take some time)")
logger.info("\n" + collect_env_info())
model = build_detection_model(cfg)
# model.to(cfg.MODEL.DEVICE)
device = torch.device(
"cuda") if torch.cuda.is_available() else torch.device("cpu")
model.to(device)
# Initialize mixed-precision if necessary
use_mixed_precision = cfg.DTYPE == 'float16'
amp_handle = amp.init(enabled=use_mixed_precision, verbose=cfg.AMP_VERBOSE)
output_dir = cfg.OUTPUT_DIR
checkpointer = DetectronCheckpointer(cfg, model, save_dir=output_dir)
if args.num_iteration != -1:
args.ckpt = os.path.join(cfg.OUTPUT_DIR,
'model_%07d.pth' % args.num_iteration)
ckpt = cfg.MODEL.WEIGHT if args.ckpt is None else args.ckpt
logger.info("Testing checkpoint {}".format(ckpt))
_ = checkpointer.load(ckpt, use_latest=args.ckpt is None)
# iou_types = ("bbox",)
output_folders = [None] * len(cfg.DATASETS.TEST)
# dataset_names = cfg.DATASETS.TEST
dataset_names = (args.dataset_name, )
if cfg.OUTPUT_DIR:
for idx, dataset_name in enumerate(dataset_names):
if args.num_iteration != -1:
output_folder = os.path.join(cfg.OUTPUT_DIR, "inference_sp",
dataset_name,
"model_%07d" % args.num_iteration)
else:
output_folder = os.path.join(cfg.OUTPUT_DIR, "inference_sp",
dataset_name)
mkdir(output_folder)
output_folders[idx] = output_folder
opt = {}
# opt['word_dim'] = 300
for output_folder, dataset_name in zip(output_folders, dataset_names):
data = DatasetCatalog.get(dataset_name)
data_args = data["args"]
im_dir = data_args['im_dir']
test_detection = pickle.load(open(data_args['test_detection_file'],
"rb"),
encoding='latin1')
prior_mask = pickle.load(open(data_args['prior_mask'], "rb"),
encoding='latin1')
action_dic = json.load(open(data_args['action_index']))
action_dic_inv = {y: x for x, y in action_dic.items()}
vcoco_test_ids = open(data_args['vcoco_test_ids_file'], 'r')
test_image_id_list = [int(line.rstrip()) for line in vcoco_test_ids]
vcocoeval = VCOCOeval(data_args['vcoco_test_file'],
data_args['ann_file'],
data_args['vcoco_test_ids_file'])
word_embeddings = pickle.load(open(data_args['word_embedding_file'],
"rb"),
encoding='latin1')
output_file = os.path.join(output_folder,
'detection_object_centric.pkl')
output_dict_file = os.path.join(
output_folder,
'detection_object_centric_all_pairs_{}_new.pkl'.format(
dataset_name))
logger.info("Output will be saved in {}".format(output_file))
logger.info("Start evaluation on {} dataset({} images).".format(
dataset_name, len(test_image_id_list)))
run_test(model,
dataset_name=dataset_name,
im_dir=im_dir,
test_detection=test_detection,
word_embeddings=word_embeddings,
test_image_id_list=test_image_id_list,
prior_mask=prior_mask,
action_dic_inv=action_dic_inv,
output_file=output_file,
output_dict_file=output_dict_file,
object_thres=args.object_thres,
human_thres=args.human_thres,
prior_flag=args.prior_flag,
device=device,
cfg=cfg)
synchronize()
vcocoeval._do_eval(output_file, ovr_thresh=0.5)
def main():
parser = argparse.ArgumentParser(
description="PyTorch Object Detection Inference")
parser.add_argument(
"--config-file",
default=
"/private/home/fmassa/github/detectron.pytorch_v2/configs/e2e_faster_rcnn_R_50_C4_1x_caffe2.yaml",
metavar="FILE",
help="path to config file",
)
parser.add_argument("--local_rank", type=int, default=0)
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER,
)
args = parser.parse_args()
num_gpus = int(
os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
distributed = num_gpus > 1
if distributed:
torch.cuda.set_device(args.local_rank)
torch.distributed.deprecated.init_process_group(backend="nccl",
init_method="env://")
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
cfg.freeze()
save_dir = "./eval"
logger = setup_logger("maskrcnn_benchmark", save_dir, get_rank())
# logger.info("Using {} GPUs".format(num_gpus))
# logger.info(cfg)
# logger.info("Collecting env info (might take some time)")
# logger.info("\n" + collect_env_info())
logger.info("\n############## Starting New Eval")
model = build_detection_model(cfg)
model.to(cfg.MODEL.DEVICE)
logger.info("Load weight path -- " + str(cfg.MODEL.WEIGHT))
checkpointer = DetectronCheckpointer(cfg, model)
_ = checkpointer.load(cfg.MODEL.WEIGHT)
iou_types = ("bbox", )
if cfg.MODEL.MASK_ON:
iou_types = iou_types + ("segm", )
output_folders = [None] * len(cfg.DATASETS.TEST)
if cfg.OUTPUT_DIR:
dataset_names = cfg.DATASETS.TEST
for idx, dataset_name in enumerate(dataset_names):
output_folder = os.path.join(cfg.OUTPUT_DIR, "inference",
dataset_name)
mkdir(output_folder)
output_folders[idx] = output_folder
data_loaders_val = make_data_loader(cfg,
is_train=False,
is_distributed=distributed)
for output_folder, data_loader_val in zip(output_folders,
data_loaders_val):
inference(
model,
data_loader_val,
iou_types=iou_types,
#box_only=cfg.MODEL.RPN_ONLY,
box_only=
False, # if cfg.RETINANET.RETINANET_ON else cfg.MODEL.RPN_ONLY,
device=cfg.MODEL.DEVICE,
expected_results=cfg.TEST.EXPECTED_RESULTS,
expected_results_sigma_tol=cfg.TEST.EXPECTED_RESULTS_SIGMA_TOL,
output_folder=output_folder,
)
synchronize()
def main(
config_file="/home/nprasad/Documents/github/maskrcnn-benchmark/configs/heads.yaml",
save_dir="model"):
local_rank = 0
opts = None
num_gpus = int(
os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
distributed = num_gpus > 1
if distributed:
torch.cuda.set_device(local_rank)
torch.distributed.init_process_group(backend="nccl",
init_method="env://")
cfg.merge_from_file(config_file)
cfg.merge_from_list(opts)
cfg.freeze()
logger = setup_logger("maskrcnn_benchmark", save_dir, get_rank())
logger.info("Using {} GPUs".format(num_gpus))
logger.info(cfg)
logger.info("Collecting env info (might take some time)")
logger.info("\n" + collect_env_info())
model = build_detection_model(cfg)
model.to(cfg.MODEL.DEVICE)
output_dir = cfg.OUTPUT_DIR
checkpointer = DetectronCheckpointer(cfg, model, save_dir=output_dir)
_ = checkpointer.load(cfg.MODEL.WEIGHT)
iou_types = ("bbox", )
if cfg.MODEL.MASK_ON:
iou_types = iou_types + ("segm", )
output_folders = [None] * len(cfg.DATASETS.TEST)
dataset_names = cfg.DATASETS.TEST
if cfg.OUTPUT_DIR:
for idx, dataset_name in enumerate(dataset_names):
output_folder = os.path.join(cfg.OUTPUT_DIR, "inference",
dataset_name)
mkdir(output_folder)
output_folders[idx] = output_folder
data_loaders_val = make_data_loader(cfg,
is_train=False,
is_distributed=distributed)
output_tuple = []
for output_folder, dataset_name, data_loader_val in zip(
output_folders, dataset_names, data_loaders_val):
if "test" in dataset_name:
continue
result = inference(
model,
data_loader_val,
dataset_name=dataset_name,
iou_types=iou_types,
box_only=cfg.MODEL.RPN_ONLY,
device=cfg.MODEL.DEVICE,
expected_results=cfg.TEST.EXPECTED_RESULTS,
expected_results_sigma_tol=cfg.TEST.EXPECTED_RESULTS_SIGMA_TOL,
output_folder=output_folder,
)[0].results['bbox']
output_tuple.append(
(result['AP'].item(), result['AP50'].item(), dataset_name))
synchronize()
return output_tuple
def main():
parser = argparse.ArgumentParser(description="PyTorch Object Detection Inference")
parser.add_argument(
"--config-file",
default="/private/home/fmassa/github/detectron.pytorch_v2/configs/e2e_faster_rcnn_R_50_C4_1x_caffe2.yaml",
metavar="FILE",
help="path to config file",
)
parser.add_argument("--local_rank", type=int, default=0)
parser.add_argument("--seq_test", action='store_true')
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER,
)
args = parser.parse_args()
num_gpus = int(os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
distributed = num_gpus > 1
if distributed:
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(
backend="nccl", init_method="env://"
)
synchronize()
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
cfg.freeze()
save_dir = cfg.OUTPUT_DIR
logger = setup_logger("maskrcnn_benchmark", save_dir, get_rank(), filename='test_log.txt')
logger.info("Using {} GPUs".format(num_gpus))
# logger.info(cfg)
logger.info("Collecting env info (might take some time)")
# logger.info("\n" + collect_env_info())
model = build_detection_model(cfg)
model.to(cfg.MODEL.DEVICE)
ori_output_dir = cfg.OUTPUT_DIR
if args.seq_test:
load_dir = cfg.TEST.LOAD_DIR
model_files = glob.glob(load_dir+'/*.pth')
model_files.sort()
min_iter = cfg.TEST.MIN_ITER
max_iter = cfg.TEST.MAX_ITER
# print(model_files)
model_files = [model_file for model_file in model_files if 'final' not in model_file and int(model_file[-11:-4])>=min_iter and int(model_file[-11:-4])<=max_iter]
else:
model_files = [cfg.MODEL.WEIGHT]
for model_file in model_files:
cfg.defrost()
cfg.MODEL.WEIGHT = model_file
logger.info('testing from {} '.format(model_file))
cfg.OUTPUT_DIR = os.path.join(ori_output_dir, model_file[-11:-4])
cfg.freeze()
output_dir = cfg.OUTPUT_DIR
checkpointer = DetectronCheckpointer(cfg, model, save_dir=output_dir)
_ = checkpointer.load(cfg.MODEL.WEIGHT)
iou_types = ("bbox",)
if cfg.MODEL.MASK_ON:
iou_types = iou_types + ("segm",)
if cfg.MODEL.KEYPOINT_ON:
iou_types = iou_types + ("keypoints",)
output_folders = [None] * len(cfg.DATASETS.TEST)
dataset_names = cfg.DATASETS.TEST
if cfg.OUTPUT_DIR:
for idx, dataset_name in enumerate(dataset_names):
output_folder = os.path.join(cfg.OUTPUT_DIR, "inference", dataset_name)
mkdir(output_folder)
output_folders[idx] = output_folder
data_loaders_val = make_data_loader(cfg, is_train=False, is_distributed=distributed)
for output_folder, dataset_name, data_loader_val in zip(output_folders, dataset_names, data_loaders_val):
inference(
model,
data_loader_val,
dataset_name=dataset_name,
iou_types=iou_types,
box_only=False, #False if cfg.MODEL.FCOS_ON or cfg.MODEL.RETINANET_ON else cfg.MODEL.RPN_ONLY,
device=cfg.MODEL.DEVICE,
expected_results=cfg.TEST.EXPECTED_RESULTS,
expected_results_sigma_tol=cfg.TEST.EXPECTED_RESULTS_SIGMA_TOL,
output_folder=output_folder,
stop_iter=cfg.FEW_SHOT.STOP_ITER
)
synchronize()
def main():
configure_logger(constants.MASKRCNN)
log_start(key=constants.INIT_START)
parser = argparse.ArgumentParser(
description="PyTorch Object Detection Training")
parser.add_argument(
"--config-file",
default="",
metavar="FILE",
help="path to config file",
type=str,
)
parser.add_argument("--local_rank",
type=int,
default=herring.get_local_rank())
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER,
)
args = parser.parse_args()
num_gpus = herring.get_world_size()
args.distributed = num_gpus > 1
args.local_rank = herring.get_local_rank()
# if is_main_process:
# # Setting logging file parameters for compliance logging
# os.environ["COMPLIANCE_FILE"] = '/MASKRCNN_complVv0.5.0_' + str(datetime.datetime.now())
# constants.LOG_FILE = os.getenv("COMPLIANCE_FILE")
# constants._FILE_HANDLER = logging.FileHandler(constants.LOG_FILE)
# constants._FILE_HANDLER.setLevel(logging.DEBUG)
# constants.LOGGER.addHandler(constants._FILE_HANDLER)
if args.distributed:
torch.cuda.set_device(args.local_rank)
# setting seeds - needs to be timed, so after RUN_START
if is_main_process():
master_seed = random.SystemRandom().randint(0, 2**32 - 1)
seed_tensor = torch.tensor(master_seed,
dtype=torch.float32,
device=torch.device("cuda"))
else:
seed_tensor = torch.tensor(0,
dtype=torch.float32,
device=torch.device("cuda"))
herring.broadcast(seed_tensor, 0)
master_seed = int(seed_tensor.item())
else:
# random master seed, random.SystemRandom() uses /dev/urandom on Unix
master_seed = random.SystemRandom().randint(0, 2**32 - 1)
# actually use the random seed
args.seed = master_seed
# random number generator with seed set to master_seed
random_number_generator = random.Random(master_seed)
log_event(key=constants.SEED, value=master_seed)
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
cfg.freeze()
output_dir = cfg.OUTPUT_DIR
if output_dir:
mkdir(output_dir)
logger = setup_logger("maskrcnn_benchmark", output_dir, get_rank())
logger.info("Using {} GPUs".format(num_gpus))
logger.info(args)
# generate worker seeds, one seed for every distributed worker
worker_seeds = generate_seeds(random_number_generator,
herring.get_world_size())
# todo sharath what if CPU
# broadcast seeds from rank=0 to other workers
worker_seeds = broadcast_seeds(worker_seeds, device='cuda')
# Setting worker seeds
logger.info("Worker {}: Setting seed {}".format(
args.local_rank, worker_seeds[args.local_rank]))
torch.manual_seed(worker_seeds[args.local_rank])
logger.info("Collecting env info (might take some time)")
logger.info("\n" + collect_env_info())
logger.info("Loaded configuration file {}".format(args.config_file))
with open(args.config_file, "r") as cf:
config_str = "\n" + cf.read()
logger.info(config_str)
logger.info("Running with config:\n{}".format(cfg))
# Initialise async eval
init()
model, success = train(cfg, args.local_rank, args.distributed,
random_number_generator)
if success is not None:
if success:
log_end(key=constants.RUN_STOP, metadata={"status": "success"})
else:
log_end(key=constants.RUN_STOP, metadata={"status": "aborted"})
def main():
parser = argparse.ArgumentParser(
description="PyTorch Object Detection Training")
parser.add_argument(
"--config-file",
default="",
metavar="FILE",
help="path to config file",
type=str,
)
parser.add_argument("--local_rank", type=int, default=0)
parser.add_argument(
"--skip-test",
dest="skip_test",
help="Do not test the final model",
action="store_true",
)
parser.add_argument("--benchmark",
help='enable `torch.backends.cudnn.benchmark`',
action="store_true")
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER,
)
args = parser.parse_args()
num_gpus = int(
os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
args.distributed = num_gpus > 1
if args.distributed:
torch.backends.cudnn.benchmark = args.benchmark
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(backend="nccl",
init_method="env://")
synchronize()
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
cfg.freeze()
output_dir = cfg.OUTPUT_DIR
if output_dir:
mkdir(output_dir)
logger = setup_logger("maskrcnn_benchmark", output_dir, get_rank())
logger.info("Using {} GPUs".format(num_gpus))
logger.info(args)
logger.info("Collecting env info (might take some time)")
logger.info("\n" + collect_env_info())
logger.info("Loaded configuration file {}".format(args.config_file))
with open(args.config_file, "r") as cf:
config_str = "\n" + cf.read()
logger.info(config_str)
logger.info("Running with config:\n{}".format(cfg))
model = train(cfg, args.local_rank, args.distributed)
if not args.skip_test:
run_test(cfg, model, args.distributed)
def main():
parser = argparse.ArgumentParser(description="PyTorch Object Detection Inference")
parser.add_argument(
"--config-file",
default="/private/home/fmassa/github/detectron.pytorch_v2/configs/e2e_faster_rcnn_R_50_C4_1x_caffe2.yaml",
metavar="FILE",
help="path to config file",
)
parser.add_argument("--local_rank", type=int, default=0)
parser.add_argument("--no-eval", default=False, action="store_true")
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER,
)
args = parser.parse_args()
num_gpus = int(os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
distributed = num_gpus > 1
if distributed:
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(
backend="nccl", init_method="env://"
)
synchronize()
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
cfg.freeze()
save_dir = ""
logger = setup_logger("maskrcnn_benchmark", save_dir, get_rank())
logger.info("Using {} GPUs".format(num_gpus))
logger.info("\n" + str(cfg))
logger.info("Collecting env info (might take some time)")
logger.info("\n" + collect_env_info())
seed = cfg.SEED
if seed >= 0:
if distributed:
seed += args.local_rank
import numpy as np
np.random.seed(seed)
torch.manual_seed(seed)
if torch.cuda.is_available():
torch.cuda.manual_seed(seed)
logger.info("Using seed: {}".format(seed))
model = build_detection_model(cfg)
model.to(cfg.MODEL.DEVICE)
output_dir = cfg.OUTPUT_DIR
checkpointer = DetectronCheckpointer(cfg, model, save_dir=output_dir, force_weight=cfg.MODEL.FORCE_WEIGHT)
_ = checkpointer.load(cfg.MODEL.WEIGHT)
iou_types = ("bbox",)
if cfg.MODEL.MASK_ON:
iou_types = iou_types + ("segm",)
if cfg.MODEL.KEYPOINT_ON:
iou_types = iou_types + ("keypoints",)
output_folders = [None] * len(cfg.DATASETS.TEST)
dataset_names = cfg.DATASETS.TEST
if cfg.OUTPUT_DIR:
for idx, dataset_name in enumerate(dataset_names):
output_folder = os.path.join(cfg.OUTPUT_DIR, "inference", dataset_name)
mkdir(output_folder)
output_folders[idx] = output_folder
data_loaders_val = make_data_loader(cfg, is_train=False, is_distributed=distributed)
predictions_file_name = cfg.TEST.PREDICTIONS_FILE_NAME
for output_folder, dataset_name, data_loader_val in zip(output_folders, dataset_names, data_loaders_val):
inference(
model,
data_loader_val,
dataset_name=dataset_name,
iou_types=iou_types,
box_only=False if cfg.MODEL.RETINANET_ON else cfg.MODEL.RPN_ONLY,
device=cfg.MODEL.DEVICE,
expected_results=cfg.TEST.EXPECTED_RESULTS,
expected_results_sigma_tol=cfg.TEST.EXPECTED_RESULTS_SIGMA_TOL,
output_folder=output_folder,
no_eval=args.no_eval,
predictions_file_name=predictions_file_name,
)
synchronize()
def main():
parser = argparse.ArgumentParser(
description="PyTorch Object Detection Inference")
parser.add_argument(
"--config-file",
default=
"/private/home/fmassa/github/detectron.pytorch_v2/configs/e2e_faster_rcnn_R_50_C4_1x_caffe2.yaml",
metavar="FILE",
help="path to config file",
)
parser.add_argument("--local_rank", type=int, default=0)
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER,
)
args = parser.parse_args()
num_gpus = int(
os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
distributed = num_gpus > 1
if distributed:
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(backend="nccl",
init_method="env://")
synchronize()
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
cfg.freeze()
save_dir = ""
logger = setup_logger("maskrcnn_benchmark", save_dir, get_rank())
logger.info("Using {} GPUs".format(num_gpus))
logger.info(cfg)
logger.info("Collecting env info (might take some time)")
logger.info("\n" + collect_env_info())
init()
tag = 17
set_epoch_tag(tag)
model = build_detection_model(cfg)
model.to(cfg.MODEL.DEVICE)
is_fp16 = (cfg.DTYPE == "float16")
if is_fp16:
# convert model to FP16
model.half()
output_dir = cfg.OUTPUT_DIR
checkpointer = DetectronCheckpointer(cfg, model, save_dir=output_dir)
_ = checkpointer.load(cfg.MODEL.WEIGHT)
iou_types = ("bbox", )
if cfg.MODEL.MASK_ON:
iou_types = iou_types + ("segm", )
if cfg.MODEL.KEYPOINT_ON:
iou_types = iou_types + ("keypoints", )
output_folders = [None] * len(cfg.DATASETS.TEST)
dataset_names = cfg.DATASETS.TEST
if cfg.OUTPUT_DIR:
for idx, dataset_name in enumerate(dataset_names):
output_folder = os.path.join(cfg.OUTPUT_DIR, "inference",
dataset_name)
mkdir(output_folder)
output_folders[idx] = output_folder
data_loaders_val = make_data_loader(cfg,
is_train=False,
is_distributed=distributed)
# evaluator = get_evaluator()
start_test_time = time.time()
results = []
for output_folder, dataset_name, data_loader_val in zip(
output_folders, dataset_names, data_loaders_val):
result = inference(
model,
data_loader_val,
dataset_name=dataset_name,
iou_types=iou_types,
box_only=False if cfg.MODEL.RETINANET_ON else cfg.MODEL.RPN_ONLY,
device=cfg.MODEL.DEVICE,
expected_results=cfg.TEST.EXPECTED_RESULTS,
expected_results_sigma_tol=cfg.TEST.EXPECTED_RESULTS_SIGMA_TOL,
output_folder=output_folder,
)
results.append(result)
end_test_time = time.time()
total_testing_time = end_test_time - start_test_time
if is_main_process():
map_results, raw_results = results[0]
bbox_map = map_results.results["bbox"]['AP']
segm_map = map_results.results["segm"]['AP']
print("BBOX_mAP: ", bbox_map, " MASK_mAP: ", segm_map)
print("Inference time: ", total_testing_time)
def main():
parser = argparse.ArgumentParser(
description="PyTorch Object Detection Inference")
parser.add_argument(
"--config-file",
# default="configs/muscima/e2e_faster_rcnn_R_50_FPN_1x_muscima_pretrained_imagenet.yaml",
default="configs/muscima/e2e_faster_rcnn_R_50_FPN_1x_muscima.yaml",
metavar="FILE",
help="path to config file",
)
parser.add_argument("--local_rank", type=int, default=0)
parser.add_argument("--output_dirs",
type=str,
default=None,
help="multiple dirs, will be compared")
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER,
)
args = parser.parse_args()
num_gpus = int(
os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
distributed = num_gpus > 1
if distributed:
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(backend="nccl",
init_method="env://")
synchronize()
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
if args.output_dirs is None:
if args.config_file.startswith("configs/"):
# se il file di configurazione specificato sta nella cartella configs, allora determino automaticamente la cartella di output
variant_folders = (
"models/" + args.config_file[len("configs/"):-len(".yaml")], )
cfg.update({"OUTPUT_DIR": variant_folders[0]})
else:
# se il file di configurazione non sta nella cartella configs, allora l'output deve essere specificato mediante OUTPUT_DIR
variant_folders = (cfg.OUTPUT_DIR, )
else:
# se ho specificato --output_dirs, questo parametro ha maggiore priorità rispetto ad OUTPUT_DIR e al --config-file
# --output_dirs serve per confrontare varianti diverse
variant_folders = args.output_dirs.split(",")
cfg.freeze()
save_dir = ""
logger = setup_logger("maskrcnn_benchmark", save_dir, get_rank())
logger.info("Using {} GPUs".format(num_gpus))
logger.info(cfg)
logger.info("Collecting env info (might take some time)")
logger.info("\n" + collect_env_info())
metrics_comparison_datasets = {}
metrics_comparison_variants = {}
for jdx, variant_folder in enumerate(variant_folders):
cfg['OUTPUT_DIR'] = variant_folder
pthFiles = [
f for f in os.listdir(variant_folder)
if os.path.isfile(os.path.join(variant_folder, f))
and f.endswith(".pth")
]
assert all(pthFile.endswith(".pth") for pthFile in pthFiles)
for pthFile in pthFiles:
pthPrefix = pthFile[:-4]
cfg['MODEL'].update(
{"WEIGHT": os.path.join(variant_folder, pthFile)})
model = build_detection_model(cfg)
model.to(cfg.MODEL.DEVICE)
output_dir = cfg.OUTPUT_DIR
checkpointer = DetectronCheckpointer(cfg,
model,
save_dir=output_dir)
_ = checkpointer.load(cfg.MODEL.WEIGHT)
iou_types = ("bbox", )
if cfg.MODEL.MASK_ON:
iou_types = iou_types + ("segm", )
output_folders = [None] * len(cfg.DATASETS.TEST)
dataset_names = cfg.DATASETS.TEST
if cfg.OUTPUT_DIR:
for idx, dataset_name in enumerate(dataset_names):
output_folder = os.path.join(cfg.OUTPUT_DIR, "inference",
dataset_name, pthPrefix)
mkdir(output_folder)
output_folders[idx] = output_folder
data_loaders_val = make_data_loader(cfg,
is_train=False,
is_distributed=distributed)
for output_folder, dataset_name, data_loader_val in zip(
output_folders, dataset_names, data_loaders_val):
if not os.path.exists(
os.path.join(output_folder, "coco_results.pth")):
inference(
model,
data_loader_val,
dataset_name=dataset_name,
iou_types=iou_types,
box_only=cfg.MODEL.RPN_ONLY,
device=cfg.MODEL.DEVICE,
expected_results=cfg.TEST.EXPECTED_RESULTS,
expected_results_sigma_tol=cfg.TEST.
EXPECTED_RESULTS_SIGMA_TOL,
output_folder=output_folder,
)
synchronize()
for output_folder, dataset_name, data_loader_val in zip(
output_folders, dataset_names, data_loaders_val):
avgDistFilePath = os.path.join(output_folder,
"average_distance.pth")
if not os.path.exists(avgDistFilePath):
AD = 0
truths = loadPredictionsFromJsonData(
data_loader_val.dataset.coco.dataset['annotations'])
preds = loadPredictionsFromJson(
os.path.join(output_folder, "bbox.json"), 0.7)
allkeys = set(preds.keys()).union(set(truths.keys()))
for image in allkeys:
if image not in truths or image not in preds:
AD += 1
continue
trueSeq = getLabelsSequence(*sortAnnotations(
truths[image]["bboxes"], truths[image]["labels"]))
predSeq = getLabelsSequence(*sortAnnotations(
preds[image]["bboxes"], preds[image]["labels"]))
AD += normalizedSequencesDistance(trueSeq, predSeq)
AD /= len(allkeys)
torch.save(AD, avgDistFilePath)
AD = torch.load(avgDistFilePath)
results = torch.load(
os.path.join(output_folder,
"coco_results.pth")).results['bbox']
results["AD"] = AD
for metric in results:
if metric not in metrics_comparison_datasets:
metrics_comparison_datasets[metric] = {}
if dataset_name not in metrics_comparison_datasets[metric]:
metrics_comparison_datasets[metric][dataset_name] = {}
metrics_comparison_datasets[metric][dataset_name][
pthPrefix] = results[metric]
if dataset_name not in metrics_comparison_variants:
metrics_comparison_variants[dataset_name] = {}
if metric not in metrics_comparison_variants[dataset_name]:
metrics_comparison_variants[dataset_name][metric] = {}
if variant_folder not in metrics_comparison_variants[
dataset_name][metric]:
metrics_comparison_variants[dataset_name][metric][
variant_folder] = {}
metrics_comparison_variants[dataset_name][metric][
variant_folder][pthPrefix] = results[metric]
plot_comparison_datasets(metrics_comparison_datasets,
os.path.join(cfg.OUTPUT_DIR, "inference"))
plot_comparison_variants(metrics_comparison_variants, "inference")
def main():
parser = argparse.ArgumentParser(
description="PyTorch Object Detection Inference")
parser.add_argument(
"--config-file",
default=
"/private/home/fmassa/github/detectron.pytorch_v2/configs/e2e_faster_rcnn_R_50_C4_1x_caffe2.yaml",
metavar="FILE",
help="path to config file",
)
parser.add_argument("--local_rank", type=int, default=0)
parser.add_argument(
"--ckpt",
help=
"The path to the checkpoint for test, default is the latest checkpoint.",
default=None,
)
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER,
)
args = parser.parse_args()
num_gpus = int(
os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
distributed = num_gpus > 1
if distributed:
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(backend="nccl",
init_method="env://")
synchronize()
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
cfg.freeze()
save_dir = ""
logger = setup_logger("maskrcnn_benchmark", save_dir, get_rank())
logger.info("Using {} GPUs".format(num_gpus))
logger.info(cfg)
logger.info("Collecting env info (might take some time)")
logger.info("\n" + collect_env_info())
model = build_detection_model(cfg)
model.to(cfg.MODEL.DEVICE)
# Initialize mixed-precision if necessary
# use_mixed_precision = False # cfg.DTYPE == 'float16'
# amp_handle = amp.init(enabled=use_mixed_precision, verbose=cfg.AMP_VERBOSE)
output_dir = cfg.OUTPUT_DIR
checkpointer = DetectronCheckpointer(cfg, model, save_dir=output_dir)
ckpt = cfg.MODEL.WEIGHT if args.ckpt is None else args.ckpt
_ = checkpointer.load(ckpt, convert=True)
iou_types = ("bbox", )
if cfg.MODEL.MASK_ON:
iou_types = iou_types + ("segm", )
if cfg.MODEL.KEYPOINT_ON:
iou_types = iou_types + ("keypoints", )
output_folders = [None] * len(cfg.DATASETS.TEST)
dataset_names = cfg.DATASETS.TEST
if cfg.OUTPUT_DIR:
for idx, dataset_name in enumerate(dataset_names):
output_folder = os.path.join(cfg.OUTPUT_DIR, "inference",
dataset_name)
mkdir(output_folder)
output_folders[idx] = output_folder
data_loaders_val = make_data_loader(cfg,
is_train=False,
is_distributed=distributed)
for output_folder, dataset_name, data_loader_val in zip(
output_folders, dataset_names, data_loaders_val):
inference(
model,
data_loader_val,
dataset_name=dataset_name,
iou_types=iou_types,
box_only=False if cfg.MODEL.RETINANET_ON else cfg.MODEL.RPN_ONLY,
bbox_aug=False,
device=cfg.MODEL.DEVICE,
expected_results=cfg.TEST.EXPECTED_RESULTS,
expected_results_sigma_tol=cfg.TEST.EXPECTED_RESULTS_SIGMA_TOL,
output_folder=output_folder,
)
synchronize()
def main():
parser = argparse.ArgumentParser(description="PyTorch Object Detection Training")
parser.add_argument(
"--config-file",
default="",
metavar="FILE",
help="path to config file",
type=str,
)
parser.add_argument("--local_rank", type=int, default=dist.get_local_rank())
parser.add_argument(
"--seed",
help="manually set random seed for torch",
type=int,
default=99
)
parser.add_argument(
"--skip-test",
dest="skip_test",
help="Do not test the final model",
action="store_true",
)
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER,
)
parser.add_argument(
"--bucket-cap-mb",
dest="bucket_cap_mb",
help="specify bucket size for SMDataParallel",
default=25,
type=int,
)
parser.add_argument(
"--data-dir",
dest="data_dir",
help="Absolute path of dataset ",
type=str,
default=None
)
parser.add_argument(
"--dtype",
dest="dtype"
)
parser.add_argument(
"--spot_ckpt",
default=None
)
args = parser.parse_args()
keys = list(os.environ.keys())
args.data_dir = os.environ['SM_CHANNEL_TRAIN'] if 'SM_CHANNEL_TRAIN' in keys else args.data_dir
print("dataset dir: ", args.data_dir)
# Set seed to reduce randomness
random.seed(args.seed + dist.get_local_rank())
np.random.seed(args.seed + dist.get_local_rank())
torch.manual_seed(args.seed + dist.get_local_rank())
torch.cuda.manual_seed(args.seed + dist.get_local_rank())
# num_gpus = int(os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
num_gpus = dist.get_world_size()
args.distributed = num_gpus > 1
if args.distributed:
# SMDataParallel: Pin each GPU to a single SMDataParallel process.
torch.cuda.set_device(args.local_rank)
# torch.distributed.init_process_group(
# backend="nccl", init_method="env://"
# )
#synchronize()
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
cfg.DTYPE=args.dtype
# grab checkpoint file to start from
os.system(f"aws s3 cp {args.spot_ckpt} /opt/ml/checkpoints/{args.spot_ckpt.split('/')[-1]}")
cfg.MODEL.WEIGHT = f"/opt/ml/checkpoints/{args.spot_ckpt.split('/')[-1]}"
cfg.freeze()
print ("CONFIG")
print (cfg)
output_dir = cfg.OUTPUT_DIR
if output_dir:
mkdir(output_dir)
logger = setup_logger("maskrcnn_benchmark", output_dir, dist.get_rank())
logger.info("Using {} GPUs".format(num_gpus))
logger.info(args)
logger.info("Collecting env info (might take some time)")
logger.info("\n" + collect_env_info())
logger.info("Loaded configuration file {}".format(args.config_file))
with open(args.config_file, "r") as cf:
config_str = "\n" + cf.read()
logger.info(config_str)
logger.info("Running with config:\n{}".format(cfg))
model = train(cfg, args)
if not args.skip_test:
if not cfg.PER_EPOCH_EVAL:
test_model(cfg, model, args)
def main():
parser = argparse.ArgumentParser(description="PyTorch Object Detection Inference")
parser.add_argument(
"--config-file",
default="/private/home/fmassa/github/detectron.pytorch_v2/configs/e2e_faster_rcnn_R_50_C4_1x_caffe2.yaml",
metavar="FILE",
help="path to config file",
)
parser.add_argument("--local_rank", type=int, default=0)
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER,
)
args = parser.parse_args()
num_gpus = int(os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
distributed = num_gpus > 1
if distributed:
torch.cuda.set_device(args.local_rank)
torch.distributed.deprecated.init_process_group(
backend="nccl", init_method="env://"
)
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
cfg.freeze()
save_dir = ""
logger = setup_logger("maskrcnn_benchmark", save_dir, get_rank())
logger.info("Using {} GPUs".format(num_gpus))
logger.info(cfg)
logger.info("Collecting env info (might take some time)")
logger.info("\n" + collect_env_info())
model = build_detection_model(cfg)
model.to(cfg.MODEL.DEVICE)
output_dir = cfg.OUTPUT_DIR
checkpointer = DetectronCheckpointer(cfg, model, save_dir=output_dir)
_ = checkpointer.load(cfg.MODEL.WEIGHT)
iou_types = ("bbox",)
if cfg.MODEL.MASK_ON:
iou_types = iou_types + ("segm",)
output_folders = [None] * len(cfg.DATASETS.TEST)
dataset_names = cfg.DATASETS.TEST
if cfg.OUTPUT_DIR:
for idx, dataset_name in enumerate(dataset_names):
output_folder = os.path.join(cfg.OUTPUT_DIR, "inference", dataset_name)
mkdir(output_folder)
output_folders[idx] = output_folder
data_loaders_val = make_data_loader(cfg, is_train=False, is_distributed=distributed)
for output_folder, dataset_name, data_loader_val in zip(output_folders, dataset_names, data_loaders_val):
inference(
model,
data_loader_val,
dataset_name=dataset_name,
iou_types=iou_types,
box_only=cfg.MODEL.RPN_ONLY,
device=cfg.MODEL.DEVICE,
expected_results=cfg.TEST.EXPECTED_RESULTS,
expected_results_sigma_tol=cfg.TEST.EXPECTED_RESULTS_SIGMA_TOL,
output_folder=output_folder,
)
synchronize()
def main():
parser = argparse.ArgumentParser(
description="PyTorch Object Detection Inference")
parser.add_argument(
"--config-file",
default=
"/private/home/fmassa/github/detectron.pytorch_v2/configs/e2e_faster_rcnn_R_50_C4_1x_caffe2.yaml",
metavar="FILE",
help="path to config file",
)
parser.add_argument("--local_rank", type=int, default=0)
parser.add_argument(
"--data-dir",
default=".",
metavar="DIR",
help="data dir for training",
type=str,
)
parser.add_argument(
"--out-dir",
default=".",
metavar="DIR",
help="output dir for model",
type=str,
)
parser.add_argument(
"--gpu_ids",
default="-1",
help="gpu id",
type=str,
)
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER,
)
args = parser.parse_args()
if args.gpu_ids != '-1':
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu_ids
num_gpus = int(
os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
distributed = num_gpus > 1
if distributed:
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(backend="nccl",
init_method="env://")
synchronize()
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
cfg.MODEL.WEIGHT = args.data_dir + cfg.MODEL.WEIGHT[1:]
cfg.DATA_DIR = args.data_dir + cfg.DATA_DIR[1:]
cfg.OUTPUT_DIR = args.out_dir + cfg.OUTPUT_DIR[1:]
print("cfg.OUTPUT_DIR: ", cfg.OUTPUT_DIR)
print("cfg.MODEL.WEIGHT: ", cfg.MODEL.WEIGHT)
print("cfg.DATA_DIR: ", cfg.DATA_DIR)
print("cfg.MODEL.ATTRIBUTE_ON: ", cfg.MODEL.ATTRIBUTE_ON)
cfg.freeze()
save_dir = ""
logger = setup_logger("maskrcnn_benchmark", save_dir, get_rank())
logger.info("Using {} GPUs".format(num_gpus))
logger.info(cfg)
logger.info("Collecting env info (might take some time)")
logger.info("\n" + collect_env_info())
model = build_detection_model(cfg)
model.to(cfg.MODEL.DEVICE)
output_dir = cfg.OUTPUT_DIR
checkpointer = DetectronCheckpointer(cfg, model, save_dir=output_dir)
_ = checkpointer.load(cfg.MODEL.WEIGHT)
iou_types = ("bbox", )
if cfg.MODEL.MASK_ON:
iou_types = iou_types + ("segm", )
if cfg.MODEL.KEYPOINT_ON:
iou_types = iou_types + ("keypoints", )
output_folders = [None] * len(cfg.DATASETS.TEST)
dataset_names = cfg.DATASETS.TEST
if cfg.OUTPUT_DIR:
for idx, dataset_name in enumerate(dataset_names):
output_folder = os.path.join(cfg.OUTPUT_DIR, "inference",
dataset_name)
mkdir(output_folder)
output_folders[idx] = output_folder
# evaluate object detection
data_loaders_val = make_data_loader(cfg,
is_train=False,
is_distributed=distributed)
for output_folder, dataset_name, data_loader_val in zip(
output_folders, dataset_names, data_loaders_val):
result_obj = inference(
model,
data_loader_val,
dataset_name=dataset_name,
iou_types=iou_types,
box_only=False if cfg.MODEL.RETINANET_ON else cfg.MODEL.RPN_ONLY,
device=cfg.MODEL.DEVICE,
expected_results=cfg.TEST.EXPECTED_RESULTS,
expected_results_sigma_tol=cfg.TEST.EXPECTED_RESULTS_SIGMA_TOL,
output_folder=output_folder,
eval_attributes=False,
)
synchronize()
# evaluate attribute detection
data_loaders_val = make_data_loader(cfg,
is_train=False,
is_distributed=distributed)
for output_folder, dataset_name, data_loader_val in zip(
output_folders, dataset_names, data_loaders_val):
result_attr = inference(
model,
data_loader_val,
dataset_name=dataset_name,
iou_types=iou_types,
box_only=False if cfg.MODEL.RETINANET_ON else cfg.MODEL.RPN_ONLY,
device=cfg.MODEL.DEVICE,
expected_results=cfg.TEST.EXPECTED_RESULTS,
expected_results_sigma_tol=cfg.TEST.EXPECTED_RESULTS_SIGMA_TOL,
output_folder=output_folder,
eval_attributes=True,
)
synchronize()
# evaluate RPN
cfg.defrost()
cfg.MODEL.RPN_ONLY = True
cfg.freeze()
logger.info(cfg)
# pdb.set_trace()
model = build_detection_model(cfg)
model.to(cfg.MODEL.DEVICE)
output_dir = cfg.OUTPUT_DIR
checkpointer = DetectronCheckpointer(cfg, model, save_dir=output_dir)
_ = checkpointer.load(cfg.MODEL.WEIGHT)
data_loaders_val = make_data_loader(cfg,
is_train=False,
is_distributed=distributed)
for output_folder, dataset_name, data_loader_val in zip(
output_folders, dataset_names, data_loaders_val):
result_rpn = inference(
model,
data_loader_val,
dataset_name=dataset_name,
iou_types=iou_types,
box_only=False if cfg.MODEL.RETINANET_ON else cfg.MODEL.RPN_ONLY,
device=cfg.MODEL.DEVICE,
expected_results=cfg.TEST.EXPECTED_RESULTS,
expected_results_sigma_tol=cfg.TEST.EXPECTED_RESULTS_SIGMA_TOL,
output_folder=output_folder,
eval_attributes=False,
)
synchronize()
if is_main_process():
results = {**result_rpn, **result_obj, **result_attr}
print(results)