def main():
parser = argparse.ArgumentParser(
description='SSD Evaluation on VOC and COCO dataset.')
parser.add_argument(
"--config-file",
default="",
metavar="FILE",
help="path to config file",
type=str,
)
parser.add_argument("--local_rank", type=int, default=0)
parser.add_argument(
"--ckpt",
help=
"The path to the checkpoint for test, default is the latest checkpoint.",
default=None,
type=str,
)
parser.add_argument("--output_dir",
default="eval_results",
type=str,
help="The directory to store evaluation results.")
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER,
)
args = parser.parse_args()
num_gpus = int(
os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
distributed = num_gpus > 1
if torch.cuda.is_available():
# This flag allows you to enable the inbuilt cudnn auto-tuner to
# find the best algorithm to use for your hardware.
torch.backends.cudnn.benchmark = True
if distributed:
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(backend="nccl",
init_method="env://")
synchronize()
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
cfg.freeze()
logger = setup_logger("SSD", dist_util.get_rank(), cfg.OUTPUT_DIR)
logger.info("Using {} GPUs".format(num_gpus))
logger.info(args)
logger.info("Loaded configuration file {}".format(args.config_file))
with open(args.config_file, "r") as cf:
config_str = "\n" + cf.read()
logger.info(config_str)
logger.info("Running with config:\n{}".format(cfg))
evaluation(cfg, ckpt=args.ckpt, distributed=distributed)
def inference(model,
data_loader,
dataset_name,
device,
output_folder=None,
use_cached=False,
**kwargs):
dataset = data_loader.dataset
logger = logging.getLogger("SSD.inference")
logger.info("Evaluating {} dataset({} images):".format(
dataset_name, len(dataset)))
predictions_path = os.path.join(output_folder, 'predictions.pth')
if use_cached and os.path.exists(predictions_path):
predictions = torch.load(predictions_path, map_location='cpu')
else:
predictions = compute_on_dataset(model, data_loader, device)
synchronize()
predictions = _accumulate_predictions_from_multiple_gpus(predictions)
if not is_main_process():
return
if output_folder:
torch.save(predictions, predictions_path)
return evaluate(dataset=dataset,
predictions=predictions,
output_dir=output_folder,
**kwargs)
def cache_url(url, model_dir=None, progress=True):
r"""Loads the Torch serialized object at the given URL.
If the object is already present in `model_dir`, it's deserialized and
returned. The filename part of the URL should follow the naming convention
``filename-<sha256>.ext`` where ``<sha256>`` is the first eight or more
digits of the SHA256 hash of the contents of the file. The hash is used to
ensure unique names and to verify the contents of the file.
The default value of `model_dir` is ``$TORCH_HOME/models`` where
``$TORCH_HOME`` defaults to ``~/.torch``. The default directory can be
overridden with the ``$TORCH_MODEL_ZOO`` environment variable.
Args:
url (string): URL of the object to download
model_dir (string, optional): directory in which to save the object
progress (bool, optional): whether or not to display a progress bar to stderr
Example:
>>> cached_file = maskrcnn_benchmark.utils.model_zoo.cache_url('https://s3.amazonaws.com/pytorch/models/resnet18-5c106cde.pth')
"""
if model_dir is None:
torch_home = os.path.expanduser(os.getenv("TORCH_HOME", "~/.torch"))
model_dir = os.getenv("TORCH_MODEL_ZOO",
os.path.join(torch_home, "models"))
if not os.path.exists(model_dir):
os.makedirs(model_dir)
parts = urlparse(url)
filename = os.path.basename(parts.path)
if filename == "model_final.pkl":
# workaround as pre-trained Caffe2 models from Detectron have all the same filename
# so make the full path the filename by replacing / with _
filename = parts.path.replace("/", "_")
cached_file = os.path.join(model_dir, filename)
if not os.path.exists(cached_file) and is_main_process():
sys.stderr.write('Downloading: "{}" to {}\n'.format(url, cached_file))
hash_prefix = HASH_REGEX.search(filename)
if hash_prefix is not None:
hash_prefix = hash_prefix.group(1)
# workaround: Caffe2 models don't have a hash, but follow the R-50 convention,
# which matches the hash PyTorch uses. So we skip the hash matching
# if the hash_prefix is less than 6 characters
if len(hash_prefix) < 6:
hash_prefix = None
download_url_to_file(url, cached_file, hash_prefix, progress=progress)
synchronize()
return cached_file
def main():
parser = argparse.ArgumentParser(
description='Single Shot MultiBox Detector Training With PyTorch')
parser.add_argument(
"--config-file",
default="",
metavar="FILE",
help="path to config file",
type=str,
)
parser.add_argument("--local_rank", type=int, default=0)
parser.add_argument('--log_step',
default=10,
type=int,
help='Print logs every log_step')
parser.add_argument('--save_step',
default=2500,
type=int,
help='Save checkpoint every save_step')
parser.add_argument(
'--eval_step',
default=2500,
type=int,
help='Evaluate dataset every eval_step, disabled when eval_step < 0')
parser.add_argument('--use_tensorboard', default=True, type=str2bool)
parser.add_argument(
"--skip-test",
dest="skip_test",
help="Do not test the final model",
action="store_true",
)
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER,
)
args = parser.parse_args()
num_gpus = int(
os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
args.distributed = num_gpus > 1
args.num_gpus = num_gpus
if torch.cuda.is_available():
# This flag allows you to enable the inbuilt cudnn auto-tuner to
# find the best algorithm to use for your hardware.
torch.backends.cudnn.benchmark = True
if args.distributed:
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(backend="nccl",
init_method="env://")
synchronize()
# Train distance regression network
train_distance_regr()
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
cfg.freeze()
if cfg.OUTPUT_DIR:
mkdir(cfg.OUTPUT_DIR)
logger = setup_logger("SSD", dist_util.get_rank(), cfg.OUTPUT_DIR)
logger.info("Using {} GPUs".format(num_gpus))
logger.info(args)
logger.info("Loaded configuration file {}".format(args.config_file))
with open(args.config_file, "r") as cf:
config_str = "\n" + cf.read()
logger.info(config_str)
logger.info("Running with config:\n{}".format(cfg))
model = train(cfg, args)
if not args.skip_test:
logger.info('Start evaluating...')
torch.cuda.empty_cache() # speed up evaluating after training finished
do_evaluation(cfg, model, distributed=args.distributed)
def main():
"""
python train.py --config-file ../SSD/configs/mobilenet_v2_ssd320_voc0712.yaml \
--log_step 10 \
--init_size 500 \
--query_size 100 \
--query_step 2 \
--train_step_per_query 50 \
--strategy uncertainty_aldod_sampling
nohup python train.py --config-file ../SSD/configs/mobilenet_v2_ssd320_voc0712.yaml \
--log_step 10 \
--init_size 1000 \
--query_size 300 \
--query_step 10 \
--train_step_per_query 1000 \
--strategy uncertainty_aldod_sampling &
"""
parser = argparse.ArgumentParser(
description='Single Shot MultiBox Detector Training With PyTorch')
parser.add_argument(
"--config-file",
default="",
metavar="FILE",
help="path to config file",
type=str,
)
parser.add_argument("--local_rank", type=int, default=0)
parser.add_argument('--log_step',
default=10,
type=int,
help='Print logs every log_step')
parser.add_argument('--init_size',
default=1000,
type=int,
help='Number of initial labeled samples')
parser.add_argument('--query_step',
default=10,
type=int,
help='Number of queries')
parser.add_argument('--query_size',
default=300,
type=int,
help='Number of assets to query each time')
parser.add_argument('--strategy',
default='random_sampling',
type=str,
help='Strategy to use to sample assets')
parser.add_argument('--train_step_per_query',
default=500,
type=int,
help='Number of training steps after each query')
parser.add_argument('--previous_queries',
default=None,
type=str,
help='Path to previous queries to use')
parser.add_argument('--use_tensorboard', default=True, type=str2bool)
parser.add_argument(
"--skip-test",
dest="skip_test",
help="Do not test the final model",
action="store_true",
)
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER,
)
args = parser.parse_args()
args.save_step = 10000000
args.eval_step = 10000000
np.random.seed(42)
num_gpus = int(
os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
args.distributed = num_gpus > 1
args.num_gpus = num_gpus
if torch.cuda.is_available():
# This flag allows you to enable the inbuilt cudnn auto-tuner to
# find the best algorithm to use for your hardware.
torch.backends.cudnn.benchmark = True
if args.distributed:
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(backend="nccl",
init_method="env://")
synchronize()
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
cfg.freeze()
time = datetime.now().strftime("%Y%m%d%H%M%S")
experiment_dir = os.path.join(
cfg.OUTPUT_DIR, f'results/{args.strategy}/experiment-{time}')
args.result_dir = experiment_dir
filename = os.path.join(experiment_dir, f'csv.txt')
argspath = os.path.join(experiment_dir, f'args.pickle')
querypath = os.path.join(experiment_dir, f'queries.txt')
model_dir = os.path.join(experiment_dir, 'model')
mkdir(experiment_dir)
mkdir(model_dir)
args.filename = filename
args.querypath = querypath
args.model_dir = model_dir
fields = [
'strategy', 'args', 'step', 'mAP', 'train_time', 'active_time',
'total_time', 'total_samples', 'bboxes'
]
with open(filename, 'w') as f:
writer = csv.writer(f)
writer.writerow(fields)
with open(querypath, 'w') as f:
writer = csv.writer(f)
writer.writerow(['step', 'indices'])
with open(argspath, 'wb') as f:
pickle.dump(args, f)
logger = setup_logger("SSD", dist_util.get_rank(), experiment_dir)
logger.info("Using {} GPUs".format(num_gpus))
logger.info(args)
logger.info("Loaded configuration file {}".format(args.config_file))
with open(args.config_file, "r") as cf:
config_str = "\n" + cf.read()
logger.info(config_str)
logger.info("Running with config:\n{}".format(cfg))
active_train(cfg, args)
def main():
# 解析命令行 读取配置文件
'''
规定了模型的基本参数，训练的类，一共是20类加上背景所以是21
模型的输入大小，为了不对原图造成影响，一般是填充为300*300的图像
训练的文件夹路径2007和2012，测试的文件夹路径2007
最大迭代次数为120000.学习率还有gamma的值，总之就是一系列的超参数
输出的文件目录
MODEL:
NUM_CLASSES: 21
INPUT:
IMAGE_SIZE: 300
DATASETS:
TRAIN: ("voc_2007_trainval", "voc_2012_trainval")
TEST: ("voc_2007_test", )
SOLVER:
MAX_ITER: 120000
LR_STEPS: [80000, 100000]
GAMMA: 0.1
BATCH_SIZE: 32
LR: 1e-3
OUTPUT_DIR: 'outputs/vgg_ssd300_voc0712'
Returns:
'''
parser = argparse.ArgumentParser(description='Single Shot MultiBox Detector Training With PyTorch')
parser.add_argument(
"--config-file",
default="configs/vgg_ssd300_voc0712.yaml",
# default="configs/vgg_ssd300_visdrone0413.yaml",
metavar="FILE",
help="path to config file",
type=str,
)
# 每2500步保存一次文件，并且验证一次文件，记录是每10次记录一次，然后如果不想看tensor的记录的话，可以关闭，使用的是tensorboardX
parser.add_argument("--local_rank", type=int, default=0)
parser.add_argument('--log_step', default=10, type=int, help='Print logs every log_step')
parser.add_argument('--save_step', default=2500, type=int, help='Save checkpoint every save_step')
parser.add_argument('--eval_step', default=2500, type=int, help='Evaluate dataset every eval_step, disabled when eval_step < 0')
parser.add_argument('--use_tensorboard', default=True, type=str2bool)
parser.add_argument(
"--skip-test",
dest="skip_test",
help="Do not test the final model",
action="store_true",
)
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER,
)
# 参数解析，可以使用多GPU进行训练
args = parser.parse_args()
num_gpus = int(os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
args.distributed = num_gpus > 1
args.num_gpus = num_gpus
# 做一些启动前必要的检查
if torch.cuda.is_available():
# This flag allows you to enable the inbuilt cudnn auto-tuner to
# find the best algorithm to use for your hardware.
torch.backends.cudnn.benchmark = True
if args.distributed:
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(backend="nccl", init_method="env://")
synchronize()
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
cfg.freeze()
# 创建模型输出文件夹
if cfg.OUTPUT_DIR:
mkdir(cfg.OUTPUT_DIR)
# 使用logger来进行记录
logger = setup_logger("SSD", dist_util.get_rank(), cfg.OUTPUT_DIR)
logger.info("Using {} GPUs".format(num_gpus))
logger.info(args)
# 加载配置文件
logger.info("Loaded configuration file {}".format(args.config_file))
with open(args.config_file, "r") as cf:
config_str = "\n" + cf.read()
logger.info(config_str)
logger.info("Running with config:\n{}".format(cfg))
# 模型训练
# model = train(cfg, args)
model = train(cfg, args)
# 开始进行验证
if not args.skip_test:
logger.info('Start evaluating...')
torch.cuda.empty_cache() # speed up evaluating after training finished
do_evaluation(cfg, model, distributed=args.distributed)
def main():
parser = ArgumentParser(
description="Single Shot MultiBox Detector Training With PyTorch")
parser.add_argument(
"--config-file",
default="",
metavar="FILE",
help="config file name or path (relative to the configs/ folder) ",
type=str,
)
parser.add_argument("--local_rank", type=int, default=0)
parser.add_argument("--log_step",
default=50,
type=int,
help="Print logs every log_step")
parser.add_argument("--save_step",
default=5000,
type=int,
help="Save checkpoint every save_step")
parser.add_argument(
"--eval_step",
default=5000,
type=int,
help="Evaluate dataset every eval_step, disabled when eval_step < 0",
)
parser.add_argument("--use_tensorboard", default=True, type=str2bool)
parser.add_argument(
"--skip-test",
dest="skip_test",
help="Do not test the final model",
action="store_true",
)
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=REMAINDER,
)
parser.add_argument(
"--resume_experiment",
default="None",
dest="resume",
type=str,
help="Checkpoint state_dict file to resume training from",
)
args = parser.parse_args()
num_gpus = int(
os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
args.distributed = num_gpus > 1
args.num_gpus = num_gpus
if torch.cuda.is_available():
# This flag allows you to enable the inbuilt cudnn auto-tuner to
# find the best algorithm to use for your hardware.
torch.backends.cudnn.benchmark = True
else:
cfg.MODEL.DEVICE = "cpu"
if args.distributed:
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(backend="nccl",
init_method="env://")
synchronize()
eman = ExperimentManager("ssd")
output_dir = eman.get_output_dir()
args.config_file = str(
Path(__file__).parent / "configs" / args.config_file)
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
cfg.OUTPUT_DIR = str(output_dir)
cfg.freeze()
eman.start({"cfg": cfg, "args": vars(args)})
# We use our own output dir, set by ExperimentManager:
# if cfg.OUTPUT_DIR:
# mkdir(cfg.OUTPUT_DIR)
logger = setup_logger("SSD", dist_util.get_rank(), output_dir / "logs")
logger.info("Using {} GPUs".format(num_gpus))
logger.info(args)
logger.info("Loaded configuration file {}".format(args.config_file))
with open(args.config_file, "r") as cf:
config_str = "\n" + cf.read()
logger.info(config_str)
logger.info("Running with config:\n{}".format(cfg))
logger.info(f"Output dir: {output_dir}")
model_manager = {"best": None, "new": None}
model = train(cfg, args, output_dir, model_manager)
if not args.skip_test:
logger.info("Start evaluating...")
torch.cuda.empty_cache() # speed up evaluating after training finished
eval_results = do_evaluation(
cfg,
model,
distributed=args.distributed,
)
do_best_model_checkpointing(
cfg,
output_dir / "model_final.pth",
eval_results,
model_manager,
logger,
is_final=True,
)
eman.mark_dir_if_complete()
def main():
parser = ArgumentParser(
description="Single Shot MultiBox Detector Training With PyTorch"
)
parser.add_argument(
"--config_file",
default="vgg_ssd300_coco_simple_sigma_in_backbone.yaml",
metavar="FILE",
help="config file name or path (relative to the configs/ folder) ",
type=str,
)
parser.add_argument("--local_rank", type=int, default=0)
parser.add_argument(
"--log_step", default=50, type=int, help="Print logs every log_step"
)
parser.add_argument(
"--save_step", default=5000, type=int, help="Save checkpoint every save_step"
)
parser.add_argument(
"--eval_step",
default=5000,
type=int,
help="Evaluate dataset every eval_step, disabled when eval_step < 0",
)
parser.add_argument("--use_tensorboard", default=True, type=str2bool)
parser.add_argument(
"--skip-test",
dest="skip_test",
help="Do not test the final model",
action="store_true",
)
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=REMAINDER,
)
parser.add_argument(
"--resume_experiment",
default="None",
dest="resume",
type=str,
help="Checkpoint state_dict file to resume training from",
)
args = parser.parse_args()
num_gpus = int(os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
args.distributed = num_gpus > 1
args.num_gpus = num_gpus
if torch.cuda.is_available():
# This flag allows you to enable the inbuilt cudnn auto-tuner to
# find the best algorithm to use for your hardware.
torch.backends.cudnn.benchmark = True
print("Using cuda")
else:
cfg.MODEL.DEVICE = "cpu"
if args.distributed:
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(backend="nccl", init_method="env://")
synchronize()
args.config_file = str(Path(__file__).parent / "configs" / args.config_file)
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
cfg.freeze()
get_model_info(args, cfg)