def __init__(self, cfg): """ Args: cfg (CfgNode): """ super().__init__() logger = logging.getLogger("fastreid") if not logger.isEnabledFor( logging.INFO): # setup_logger is not called for fastreid setup_logger() # Assume these objects must be constructed in this order. data_loader = self.build_train_loader(cfg) cfg = self.auto_scale_hyperparams(cfg, data_loader.dataset.num_classes) model = self.build_model(cfg) optimizer = self.build_optimizer(cfg, model) optimizer_ckpt = dict(optimizer=optimizer) if cfg.SOLVER.FP16_ENABLED: model, optimizer = amp.initialize(model, optimizer, opt_level="O1") optimizer_ckpt.update(dict(amp=amp)) # For training, wrap with DDP. But don't need this for inference. if comm.get_world_size() > 1: # ref to https://github.com/pytorch/pytorch/issues/22049 to set `find_unused_parameters=True` # for part of the parameters is not updated. # model = DistributedDataParallel( # model, device_ids=[comm.get_local_rank()], broadcast_buffers=False # ) model = DistributedDataParallel(model, delay_allreduce=True) self._trainer = (AMPTrainer if cfg.SOLVER.FP16_ENABLED else SimpleTrainer)(model, data_loader, optimizer) self.iters_per_epoch = len( data_loader.dataset) // cfg.SOLVER.IMS_PER_BATCH self.scheduler = self.build_lr_scheduler(cfg, optimizer, self.iters_per_epoch) # Assume no other objects need to be checkpointed. # We can later make it checkpoint the stateful hooks self.checkpointer = Checkpointer( # Assume you want to save checkpoints together with logs/statistics model, cfg.OUTPUT_DIR, save_to_disk=comm.is_main_process(), **optimizer_ckpt, **self.scheduler, ) self.start_epoch = 0 self.max_epoch = cfg.SOLVER.MAX_EPOCH self.max_iter = self.max_epoch * self.iters_per_epoch self.warmup_iters = cfg.SOLVER.WARMUP_ITERS self.delay_epochs = cfg.SOLVER.DELAY_EPOCHS self.cfg = cfg self.register_hooks(self.build_hooks())
def __init__(self, cfg): """ Args: cfg (CfgNode): """ logger = logging.getLogger("fastreid") if not logger.isEnabledFor( logging.INFO): # setup_logger is not called for fastreid setup_logger() # Assume these objects must be constructed in this order. data_loader = self.build_train_loader(cfg) cfg = self.auto_scale_hyperparams(cfg, data_loader) model = self.build_model(cfg) optimizer = self.build_optimizer(cfg, model) # For training, wrap with DDP. But don't need this for inference. if comm.get_world_size() > 1: # ref to https://github.com/pytorch/pytorch/issues/22049 to set `find_unused_parameters=True` # for part of the parameters is not updated. model = DistributedDataParallel(model, device_ids=[comm.get_local_rank()], broadcast_buffers=False) super().__init__(model, data_loader, optimizer, cfg.SOLVER.BASE_LR, cfg.MODEL.LOSSES.CENTER.LR, cfg.MODEL.LOSSES.CENTER.SCALE, cfg.SOLVER.AMP_ENABLED) self.scheduler = self.build_lr_scheduler(cfg, optimizer) # Assume no other objects need to be checkpointed. # We can later make it checkpoint the stateful hooks self.checkpointer = Checkpointer( # Assume you want to save checkpoints together with logs/statistics model, cfg.OUTPUT_DIR, save_to_disk=comm.is_main_process(), optimizer=optimizer, scheduler=self.scheduler, ) self.start_iter = 0 if cfg.SOLVER.SWA.ENABLED: self.max_iter = cfg.SOLVER.MAX_ITER + cfg.SOLVER.SWA.ITER else: self.max_iter = cfg.SOLVER.MAX_ITER self.cfg = cfg self.register_hooks(self.build_hooks())
def default_setup(cfg, args): """ Perform some basic common setups at the beginning of a job, including: 1. Set up the detectron2 logger 2. Log basic information about environment, cmdline arguments, and config 3. Backup the config to the output directory Args: cfg (CfgNode): the full config to be used args (argparse.NameSpace): the command line arguments to be logged """ output_dir = cfg.OUTPUT_DIR if comm.is_main_process() and output_dir: PathManager.mkdirs(output_dir) rank = comm.get_rank() setup_logger(output_dir, distributed_rank=rank, name="fvcore") logger = setup_logger(output_dir, distributed_rank=rank) logger.info("Rank of current process: {}. World size: {}".format(rank, comm.get_world_size())) logger.info("Environment info:\n" + collect_env_info()) logger.info("Command line arguments: " + str(args)) if hasattr(args, "config_file") and args.config_file != "": logger.info( "Contents of args.config_file={}:\n{}".format( args.config_file, PathManager.open(args.config_file, "r").read() ) ) logger.info("Running with full config:\n{}".format(cfg)) if comm.is_main_process() and output_dir: # Note: some of our scripts may expect the existence of # config.yaml in output directory path = os.path.join(output_dir, "config.yaml") with PathManager.open(path, "w") as f: f.write(cfg.dump()) logger.info("Full config saved to {}".format(os.path.abspath(path))) # make sure each worker has a different, yet deterministic seed if specified seed_all_rng() # cudnn benchmark has large overhead. It shouldn't be used considering the small size of # typical validation set. if not (hasattr(args, "eval_only") and args.eval_only): torch.backends.cudnn.benchmark = cfg.CUDNN_BENCHMARK
def __init__(self, cfg): """ Args: cfg (CfgNode): """ self.cfg = cfg logger = logging.getLogger(__name__) if not logger.isEnabledFor( logging.INFO): # setup_logger is not called for fastreid setup_logger() # Assume these objects must be constructed in this order. model = self.build_model(cfg) optimizer = self.build_optimizer(cfg, model) logger.info('Prepare training set') data_loader = self.build_train_loader(cfg) # For training, wrap with DP. But don't need this for inference. model = DataParallel(model) if cfg.MODEL.BACKBONE.NORM == "syncBN": # Monkey-patching with syncBN patch_replication_callback(model) model = model.cuda() super().__init__(model, data_loader, optimizer) self.scheduler = self.build_lr_scheduler(cfg, optimizer) # Assume no other objects need to be checkpointed. # We can later make it checkpoint the stateful hooks self.checkpointer = Checkpointer( # Assume you want to save checkpoints together with logs/statistics model, self.data_loader.loader.dataset, cfg.OUTPUT_DIR, optimizer=optimizer, scheduler=self.scheduler, ) self.start_iter = 0 if cfg.SOLVER.SWA.ENABLED: self.max_iter = cfg.SOLVER.MAX_ITER + cfg.SOLVER.SWA.ITER else: self.max_iter = cfg.SOLVER.MAX_ITER self.cfg = cfg self.register_hooks(self.build_hooks())
self.predictors = [] for cfg in cfgs: self.predictors.append(DefaultPredictor(cfg)) def run_on_loader(self, data_loader): for batch in data_loader: predictions = [] for predictor in self.predictors: predictions.append(predictor(batch["images"])) yield torch.cat(predictions, dim=-1), batch if __name__ == "__main__": args = get_parser().parse_args() logger = setup_logger() cfgs = [] for config_file in args.config_file: cfg = setup_cfg(config_file, args.opts) cfgs.append(cfg) results = OrderedDict() for dataset_name in cfgs[0].DATASETS.TESTS: test_loader, num_query = build_reid_test_loader(cfgs[0], dataset_name) evaluator = ReidEvaluator(cfgs[0], num_query) feat_extract = FeatureExtraction(cfgs) for (feat, batch) in tqdm.tqdm(feat_extract.run_on_loader(test_loader), total=len(test_loader)): evaluator.process(batch, feat) result = evaluator.evaluate() results[dataset_name] = result print_csv_format(results)
import torch sys.path.append('../../') import pytorch_to_caffe from fastreid.config import get_cfg from fastreid.modeling.meta_arch import build_model from fastreid.utils.file_io import PathManager from fastreid.utils.checkpoint import Checkpointer from fastreid.utils.logger import setup_logger # import some modules added in project like this below # sys.path.append('../projects/FastCls') # from fastcls import * setup_logger(name='fastreid') logger = logging.getLogger("fastreid.caffe_export") def setup_cfg(args): cfg = get_cfg() # add_cls_config(cfg) cfg.merge_from_file(args.config_file) cfg.merge_from_list(args.opts) cfg.freeze() return cfg def get_parser(): parser = argparse.ArgumentParser( description="Convert Pytorch to Caffe model")
""" import argparse import torch import sys sys.path.append('../../') import pytorch_to_caffe from fastreid.config import get_cfg from fastreid.modeling.meta_arch import build_model from fastreid.utils.file_io import PathManager from fastreid.utils.checkpoint import Checkpointer from fastreid.utils.logger import setup_logger logger = setup_logger(name='caffe_export') def setup_cfg(args): cfg = get_cfg() cfg.merge_from_file(args.config_file) cfg.merge_from_list(args.opts) cfg.freeze() return cfg def get_parser(): parser = argparse.ArgumentParser(description="Convert Pytorch to Caffe model") parser.add_argument( "--config-file",
def __init__(self, cfg): TrainerBase.__init__(self) logger = logging.getLogger('fastreid.partial-fc.trainer') if not logger.isEnabledFor( logging.INFO): # setup_logger is not called for fastreid setup_logger() # Assume these objects must be constructed in this order. data_loader = self.build_train_loader(cfg) cfg = self.auto_scale_hyperparams(cfg, data_loader.dataset.num_classes) model = self.build_model(cfg) optimizer = self.build_optimizer(cfg, model) if cfg.MODEL.HEADS.PFC.ENABLED: # fmt: off feat_dim = cfg.MODEL.BACKBONE.FEAT_DIM embedding_dim = cfg.MODEL.HEADS.EMBEDDING_DIM num_classes = cfg.MODEL.HEADS.NUM_CLASSES sample_rate = cfg.MODEL.HEADS.PFC.SAMPLE_RATE cls_type = cfg.MODEL.HEADS.CLS_LAYER scale = cfg.MODEL.HEADS.SCALE margin = cfg.MODEL.HEADS.MARGIN # fmt: on # Partial-FC module embedding_size = embedding_dim if embedding_dim > 0 else feat_dim self.pfc_module = PartialFC(embedding_size, num_classes, sample_rate, cls_type, scale, margin) self.pfc_optimizer = self.build_optimizer(cfg, self.pfc_module) # For training, wrap with DDP. But don't need this for inference. if comm.get_world_size() > 1: # ref to https://github.com/pytorch/pytorch/issues/22049 to set `find_unused_parameters=True` # for part of the parameters is not updated. model = DistributedDataParallel(model, device_ids=[comm.get_local_rank()], broadcast_buffers=False, find_unused_parameters=True) self._trainer = PFCTrainer(model, data_loader, optimizer, self.pfc_module, self.pfc_optimizer) \ if cfg.MODEL.HEADS.PFC.ENABLED else SimpleTrainer(model, data_loader, optimizer) self.iters_per_epoch = len( data_loader.dataset) // cfg.SOLVER.IMS_PER_BATCH self.scheduler = self.build_lr_scheduler(cfg, optimizer, self.iters_per_epoch) if cfg.MODEL.HEADS.PFC.ENABLED: self.pfc_scheduler = self.build_lr_scheduler( cfg, self.pfc_optimizer, self.iters_per_epoch) self.checkpointer = Checkpointer( # Assume you want to save checkpoints together with logs/statistics model, cfg.OUTPUT_DIR, save_to_disk=comm.is_main_process(), optimizer=optimizer, **self.scheduler, ) if cfg.MODEL.HEADS.PFC.ENABLED: self.pfc_checkpointer = Checkpointer( self.pfc_module, cfg.OUTPUT_DIR, optimizer=self.pfc_optimizer, **self.pfc_scheduler, ) self.start_epoch = 0 self.max_epoch = cfg.SOLVER.MAX_EPOCH self.max_iter = self.max_epoch * self.iters_per_epoch self.warmup_iters = cfg.SOLVER.WARMUP_ITERS self.delay_epochs = cfg.SOLVER.DELAY_EPOCHS self.cfg = cfg self.register_hooks(self.build_hooks())
""" import argparse import os import sys import tensorrt as trt from trt_calibrator import FeatEntropyCalibrator sys.path.append('../../') from fastreid.utils.logger import setup_logger from fastreid.utils.file_io import PathManager logger = setup_logger(name='trt_export') def get_parser(): parser = argparse.ArgumentParser(description="Convert ONNX to TRT model") parser.add_argument('--name', default='baseline', help="name for converted model") parser.add_argument('--output', default='outputs/trt_model', help="path to save converted trt model") parser.add_argument( '--mode', default='fp32', help=
from onnxsim import simplify from torch.onnx import OperatorExportTypes sys.path.append('../../') from fastreid.config import get_cfg from fastreid.modeling.meta_arch import build_model from fastreid.utils.file_io import PathManager from fastreid.utils.checkpoint import Checkpointer from fastreid.utils.logger import setup_logger # import some modules added in project like this below # sys.path.append('../../projects/FastDistill') # from fastdistill import * logger = setup_logger(name='onnx_export') def setup_cfg(args): cfg = get_cfg() cfg.merge_from_file(args.config_file) cfg.merge_from_list(args.opts) cfg.freeze() return cfg def get_parser(): parser = argparse.ArgumentParser( description="Convert Pytorch to ONNX model") parser.add_argument(
def __init__(self, cfg): super().__init__() logger = logging.getLogger("fastreid") if not logger.isEnabledFor( logging.INFO): # if setup_logger is not called for fastreid setup_logger() logger.info("==> Load target-domain dataset") self.tgt = tgt = self.load_dataset(cfg.DATASETS.TGT) self.tgt_nums = len(tgt.train) cfg = self.auto_scale_hyperparams(cfg, self.tgt_nums) # Create model self.model = self.build_model(cfg, load_model=cfg.MODEL.PRETRAIN, show_model=True, use_dsbn=False) # Optimizer self.optimizer, self.param_wrapper = self.build_optimizer( cfg, self.model) # For training, wrap with DDP. But don't need this for inference. if comm.get_world_size() > 1: # ref to https://github.com/pytorch/pytorch/issues/22049 to set `find_unused_parameters=True` # for part of the parameters is not updated. self.model = DistributedDataParallel( self.model, device_ids=[comm.get_local_rank()], broadcast_buffers=False, find_unused_parameters=True) # Learning rate scheduler self.iters_per_epoch = cfg.SOLVER.ITERS self.scheduler = self.build_lr_scheduler(cfg, self.optimizer, self.iters_per_epoch) # Assume no other objects need to be checkpointed. # We can later make it checkpoint the stateful hooks self.checkpointer = Checkpointer( # Assume you want to save checkpoints together with logs/statistics self.model, cfg.OUTPUT_DIR, save_to_disk=comm.is_main_process(), optimizer=self.optimizer, **self.scheduler, ) self.start_epoch = 0 self.max_epoch = cfg.SOLVER.MAX_EPOCH self.max_iter = self.max_epoch * self.iters_per_epoch self.warmup_iters = cfg.SOLVER.WARMUP_ITERS self.delay_epochs = cfg.SOLVER.DELAY_EPOCHS self.cfg = cfg self.register_hooks(self.build_hooks()) if cfg.SOLVER.AMP.ENABLED: unsupported = "AMPTrainer does not support single-process multi-device training!" if isinstance(self.model, DistributedDataParallel): assert not (self.model.device_ids and len(self.model.device_ids) > 1), unsupported from torch.cuda.amp.grad_scaler import GradScaler self.grad_scaler = GradScaler() else: self.grad_scaler = None
@contact: [email protected] """ import argparse import os import sys import tensorrt as trt from trt_calibrator import FeatEntropyCalibrator sys.path.append('.') from fastreid.utils.logger import setup_logger, PathManager logger = setup_logger(name="trt_export") def get_parser(): parser = argparse.ArgumentParser(description="Convert ONNX to TRT model") parser.add_argument( '--name', default='baseline', help="name for converted model" ) parser.add_argument( '--output', default='outputs/trt_model', help="path to save converted trt model" )
sys.path.append('.') from fastreid.evaluation import evaluate_rank from fastreid.config import get_cfg from fastreid.utils.logger import setup_logger from fastreid.data import build_reid_test_loader from predictor import FeatureExtractionDemo from fastreid.utils.visualizer import Visualizer # import some modules added in project # for example, add partial reid like this below # sys.path.append("projects/PartialReID") # from partialreid import * cudnn.benchmark = True setup_logger(name="fastreid") logger = logging.getLogger('fastreid.visualize_result') def setup_cfg(args): # load config from file and command-line arguments cfg = get_cfg() # add_partialreid_config(cfg) cfg.merge_from_file(args.config_file) cfg.merge_from_list(args.opts) cfg.freeze() return cfg def get_parser():
def __init__(self, cfg): super().__init__() logger = logging.getLogger("fastreid") if not logger.isEnabledFor( logging.INFO): # if setup_logger is not called for fastreid setup_logger() # Create datasets logger.info("==> Load source-domain dataset") self.src = src = self.load_dataset(cfg.DATASETS.SRC) self.src_pid_nums = src.get_num_pids(src.train) logger.info("==> Load target-domain dataset") self.tgt = tgt = self.load_dataset(cfg.DATASETS.TGT) self.tgt_nums = len(tgt.train) # Create model self.model = self.build_model(cfg, load_model=False, show_model=False, use_dsbn=True) # Create hybrid memorys self.hm = HybridMemory(num_features=cfg.MODEL.BACKBONE.FEAT_DIM, num_samples=self.src_pid_nums + self.tgt_nums, temp=cfg.MEMORY.TEMP, momentum=cfg.MEMORY.MOMENTUM, use_half=cfg.SOLVER.AMP.ENABLED).cuda() # Initialize source-domain class centroids logger.info( "==> Initialize source-domain class centroids in the hybrid memory" ) with inference_context(self.model), torch.no_grad(): src_train = self.build_dataset(cfg, src.train, is_train=False, relabel=False, with_mem_idx=False) src_init_feat_loader = self.build_test_loader(cfg, src_train) src_fname_feat_dict, _ = extract_features(self.model, src_init_feat_loader) src_feat_dict = collections.defaultdict(list) for f, pid, _ in sorted(src.train): src_feat_dict[pid].append(src_fname_feat_dict[f].unsqueeze(0)) src_centers = [ torch.cat(src_feat_dict[pid], 0).mean(0) for pid in sorted(src_feat_dict.keys()) ] src_centers = torch.stack(src_centers, 0) src_centers = F.normalize(src_centers, dim=1) # Initialize target-domain instance features logger.info( "==> Initialize target-domain instance features in the hybrid memory" ) with inference_context(self.model), torch.no_grad(): tgt_train = self.build_dataset(cfg, tgt.train, is_train=False, relabel=False, with_mem_idx=False) tgt_init_feat_loader = self.build_test_loader(cfg, tgt_train) tgt_fname_feat_dict, _ = extract_features(self.model, tgt_init_feat_loader) tgt_features = torch.cat([ tgt_fname_feat_dict[f].unsqueeze(0) for f, _, _ in sorted(self.tgt.train) ], 0) tgt_features = F.normalize(tgt_features, dim=1) self.hm.features = torch.cat((src_centers, tgt_features), dim=0).cuda() del (src_train, src_init_feat_loader, src_fname_feat_dict, src_feat_dict, src_centers, tgt_train, tgt_init_feat_loader, tgt_fname_feat_dict, tgt_features) # Optimizer self.optimizer, self.param_wrapper = self.build_optimizer( cfg, self.model) # For training, wrap with DDP. But don't need this for inference. if comm.get_world_size() > 1: # ref to https://github.com/pytorch/pytorch/issues/22049 to set `find_unused_parameters=True` # for part of the parameters is not updated. self.model = DistributedDataParallel( self.model, device_ids=[comm.get_local_rank()], broadcast_buffers=False, find_unused_parameters=True) # Learning rate scheduler self.iters_per_epoch = cfg.SOLVER.ITERS self.scheduler = self.build_lr_scheduler(cfg, self.optimizer, self.iters_per_epoch) # Assume no other objects need to be checkpointed. # We can later make it checkpoint the stateful hooks self.checkpointer = Checkpointer( # Assume you want to save checkpoints together with logs/statistics self.model, cfg.OUTPUT_DIR, save_to_disk=comm.is_main_process(), optimizer=self.optimizer, **self.scheduler, ) self.start_epoch = 0 self.max_epoch = cfg.SOLVER.MAX_EPOCH self.max_iter = self.max_epoch * self.iters_per_epoch self.warmup_iters = cfg.SOLVER.WARMUP_ITERS self.delay_epochs = cfg.SOLVER.DELAY_EPOCHS self.cfg = cfg self.register_hooks(self.build_hooks()) if cfg.SOLVER.AMP.ENABLED: unsupported = "AMPTrainer does not support single-process multi-device training!" if isinstance(self.model, DistributedDataParallel): assert not (self.model.device_ids and len(self.model.device_ids) > 1), unsupported from torch.cuda.amp.grad_scaler import GradScaler self.grad_scaler = GradScaler() else: self.grad_scaler = None
def __init__(self, cfg): """ Args: cfg (CfgNode): """ super().__init__() logger = logging.getLogger("fastreid") if not logger.isEnabledFor( logging.INFO): # setup_logger is not called for fastreid setup_logger() # Assume these objects must be constructed in this order. data_loader = self.build_train_loader(cfg) cfg = self.auto_scale_hyperparams(cfg, data_loader.dataset.num_classes) self.model = self.build_model(cfg) self.optimizer, self.param_wrapper = self.build_optimizer( cfg, self.model) # For training, wrap with DDP. But don't need this for inference. if comm.get_world_size() > 1: # ref to https://github.com/pytorch/pytorch/issues/22049 to set `find_unused_parameters=True` # for part of the parameters is not updated. self.model = DistributedDataParallel( self.model, device_ids=[comm.get_local_rank()], broadcast_buffers=False, ) self._data_loader_iter = iter(data_loader) self.iters_per_epoch = len( data_loader.dataset) // cfg.SOLVER.IMS_PER_BATCH self.scheduler = self.build_lr_scheduler(cfg, self.optimizer, self.iters_per_epoch) # Assume no other objects need to be checkpointed. # We can later make it checkpoint the stateful hooks self.checkpointer = Checkpointer( # Assume you want to save checkpoints together with logs/statistics self.model, cfg.OUTPUT_DIR, save_to_disk=comm.is_main_process(), optimizer=self.optimizer, **self.scheduler, ) self.start_epoch = 0 self.max_epoch = cfg.SOLVER.MAX_EPOCH self.max_iter = self.max_epoch * self.iters_per_epoch self.warmup_iters = cfg.SOLVER.WARMUP_ITERS self.delay_epochs = cfg.SOLVER.DELAY_EPOCHS self.cfg = cfg self.register_hooks(self.build_hooks()) if cfg.SOLVER.AMP.ENABLED: unsupported = f"[{self.__class__.__name__}] does not support single-process multi-device training!" if isinstance(self.model, DistributedDataParallel): assert not (self.model.device_ids and len(self.model.device_ids) > 1), unsupported from torch.cuda.amp.grad_scaler import GradScaler self.grad_scaler = GradScaler() else: self.grad_scaler = None