def save_checkpoint(path_to_checkpoint, model, optimizer, epoch, cfg): """ save a checkpoint :param path_to_checkpoint: :param mdoel: :param optimizer: :param epoch: :param cfg: :return: """ logger.info("save checkpoint in epoch {}".format(epoch)) PathManager.mkdirs(get_checkpoint_dir(path_to_checkpoint)) sd = model.state_dict() #Recode the state checkpoint = { "epoch": epoch, "model_state": sd, "optimizer_state": optimizer.state_dict(), "cfg": cfg.dump() } checkpoint_path = get_path_to_checkpoints(path_to_checkpoint, epoch + 1) # if (epoch+1)%10==0 or (epoch+1)==cfg.SOLVER.MAX_EPOCH: with PathManager.open(checkpoint_path, "wb") as f: torch.save(checkpoint, f) return checkpoint_path
def make_checkpoint_dir(path_to_checkpoint): checkpoint_dir = os.path.join(path_to_checkpoint, "checkpoints") if not PathManager.exists(checkpoint_dir): PathManager.mkdirs(checkpoint_dir) return checkpoint_dir
def setup_logger( output=None, distributed_rank=0, *, color=True, name="segmentation", abbrev_name=None ): """ Initialize the segmentation logger and set its verbosity level to "INFO". Args: output (str): a file name or a directory to save log. If None, will not save log file. If ends with ".txt" or ".log", assumed to be a file name. Otherwise, logs will be saved to `output/log.txt`. name (str): the root module name of this logger abbrev_name (str): an abbreviation of the module, to avoid long names in logs. Set to "" to not log the root module in logs. By default, will abbreviate "segmentation" to "seg" and leave other modules unchanged. Returns: logging.Logger: a logger """ logger = logging.getLogger(name) logger.setLevel(logging.DEBUG) logger.propagate = False if abbrev_name is None: abbrev_name = "seg" if name == "segmentation" else name plain_formatter = logging.Formatter( "[%(asctime)s] %(name)s %(levelname)s: %(message)s", datefmt="%m/%d %H:%M:%S" ) # stdout logging: master only if distributed_rank == 0: ch = logging.StreamHandler(stream=sys.stdout) ch.setLevel(logging.DEBUG) if color: formatter = _ColorfulFormatter( colored("[%(asctime)s %(name)s]: ", "green") + "%(message)s", datefmt="%m/%d %H:%M:%S", root_name=name, abbrev_name=str(abbrev_name), ) else: formatter = plain_formatter ch.setFormatter(formatter) logger.addHandler(ch) # file logging: all workers if output is not None: if output.endswith(".txt") or output.endswith(".log"): filename = output else: filename = os.path.join(output, "log.txt") if distributed_rank > 0: filename = filename + ".rank{}".format(distributed_rank) # os.makedirs(os.path.dirname(filename)) PathManager.mkdirs(os.path.dirname(filename)) fh = logging.StreamHandler(_cached_log_stream(filename)) fh.setLevel(logging.DEBUG) fh.setFormatter(plain_formatter) logger.addHandler(fh) return logger
def do_feature_extraction(cfg, model, dataset_name): with inference_context(model): dump_folder = os.path.join(cfg.OUTPUT_DIR, "features", dataset_to_folder_mapper[dataset_name]) PathManager.mkdirs(dump_folder) # data_loader = build_detection_test_loader_with_attributes(cfg, dataset_name) extract_grid_feature_on_local(model, dump_folder, 'data/train_images')
def evaluate(self): if self._distributed: comm.synchronize() self._predictions = comm.gather(self._predictions, dst=0) self._predictions = list(itertools.chain(*self._predictions)) if not comm.is_main_process(): return {} if len(self._predictions) == 0: self._logger.warning( "[COCOEvaluator] Did not receive valid predictions.") return {} if self._output_dir: PathManager.mkdirs(self._output_dir) file_path = os.path.join(self._output_dir, "instances_predictions.pth") with PathManager.open(file_path, "wb") as f: torch.save(self._predictions, f) self._results = OrderedDict() if "instances" in self._predictions[0]: self._eval_predictions() # Copy so the caller can do whatever with results return copy.deepcopy(self._results)
def process(self, inputs, outputs): for input, output in zip(inputs, outputs): samples = output[ 'samples'] # list of size num_samples with el: # nc, t, h, w v_idx = input['video_idx'] for sample_idx in range(len(samples)): sample = samples[sample_idx].squeeze( 0) # t, h, w if nc == 1 or nc, t, h, w if sample.dim() == 4: sample = sample.transpose(0, 1) # t, nc, h, w self.vqvae.to(sample.device) code = sample.detach().cpu().numpy() sample = self.vqvae.decode(sample) # T, 3, H, W sample = self.vqvae.back_normalizer(sample) # T, 3, H, W if self.scale_to_zeroone: sample = sample * 255 sample.clamp_(0.0, 255.0) sample = sample.permute(0, 2, 3, 1).contiguous() # T, H, W, 3 sample = sample.detach().cpu().numpy().astype(np.uint8) video_dir = os.path.join(self._output_dir, "samples", self._dataset_name, f'video_{sample_idx}_{v_idx}') PathManager.mkdirs(video_dir) np.save(os.path.join(video_dir, f'codes.npy'), code) for frame_idx in range(len(sample)): frame_path = os.path.join(video_dir, f'{frame_idx}.png') for i in range(10): try: save_image(sample[frame_idx], frame_path) break except OSError: print(f'sleep 3 sec and try again #{i}') time.sleep(3) continue
def default_setup(cfg, args): output_dir = cfg.OUTPUT_DIR if comm.is_main_process() and output_dir: PathManager.mkdirs(output_dir) rank = comm.get_rank() setup_logger(output_dir, distributed_rank=rank, name="fvcore") logger = setup_logger(output_dir, distributed_rank=rank) logger.info("Rank of current process: {}. World size: {}".format( rank, comm.get_world_size())) logger.info("Environment info:\n" + collect_env_info()) logger.info("Command line arguments: " + str(args)) if hasattr(args, "config") and args.config != "": logger.info("Contents of args.config={}:\n{}".format( args.config, PathManager.open(args.config, "r").read())) logger.info("Running with full config:\n{}".format(cfg)) seed_all_rng(None if cfg.SEED < 0 else cfg.SEED + rank) if not (hasattr(args, "eval_only") and args.eval_only): torch.backends.cudnn.benchmark = cfg.CUDNN_BENCHMARK
def evaluate(self): if self._distributed: synchronize() endpoint_errors = all_gather(self._endpoint_errors) endpoint_errors = [per_image for per_gpu in endpoint_errors for per_image in per_gpu] self._predictions = all_gather(self._predictions) if not is_main_process(): return if self._output_dir: PathManager.mkdirs(self._output_dir) file_path = os.path.join(self._output_dir, "flow_predictions.json") with PathManager.open(file_path, "w") as f: f.write(json.dumps(self._predictions)) ave_epe = sum(endpoint_errors) / len(endpoint_errors) res = {"ave_epe": ave_epe} if self._output_dir: file_path = os.path.join(self._output_dir, "flow_evaluation.pth") with PathManager.open(file_path, "wb") as f: torch.save(res, f) results = OrderedDict({"flow": res}) small_table = create_small_table(res) self._logger.info("Evaluation results for flow: \n" + small_table) dump_info_one_task = { "task": "flow", "tables": [small_table], } _dump_to_markdown([dump_info_one_task]) return results
def convert_to_coco_json(dataset_name, output_file, allow_cached=True): """ Converts dataset into COCO format and saves it to a json file. dataset_name must be registered in DatasetCatalog and in detectron2's standard format. Args: dataset_name: reference from the config file to the catalogs must be registered in DatasetCatalog and in detectron2's standard format output_file: path of json file that will be saved to allow_cached: if json file is already present then skip conversion """ # TODO: The dataset or the conversion script *may* change, # a checksum would be useful for validating the cached data PathManager.mkdirs(os.path.dirname(output_file)) with file_lock(output_file): if os.path.exists(output_file) and allow_cached: logger.info( f"Cached annotations in COCO format already exist: {output_file}" ) else: logger.info( f"Converting dataset annotations in '{dataset_name}' to COCO format ...)" ) coco_dict = convert_to_coco_dict(dataset_name) with PathManager.open(output_file, "w") as json_file: logger.info( f"Caching annotations in COCO format: {output_file}") json.dump(coco_dict, json_file)
def evaluate(self): if self._distributed: comm.synchronize() predictions = comm.gather(self._predictions, dst=0) predictions = list(itertools.chain(*predictions)) if not comm.is_main_process(): return {} else: predictions = self._predictions # preictions: list of dict [{'image_id', 'instances'(list of dict [{'image_id', 'category_id', bbox, score}])}] if len(predictions) == 0: self._logger.warning( "[SMDEvaluator] Did not receive valid predictions.") return {} if self._output_dir: PathManager.mkdirs(self._output_dir) file_path = os.path.join(self._output_dir, "instances_predictions.pth") with PathManager.open(file_path, "wb") as f: torch.save(predictions, f) self._results = OrderedDict() if "proposals" in predictions[0]: self._eval_box_proposals(predictions) if "instances" in predictions[0]: self._eval_predictions(set(self._tasks), predictions) self._eval_predictions_others(self._coco_api, predictions) # Copy so the caller can do whatever with results return copy.deepcopy(self._results)
def convert_to_coco_json(dataset_name, output_file, allow_cached=True): """ Converts dataset into COCO format and saves it to a json file. dataset_name must be registered in DatasetCatalog and in detectron2's standard format. Args: dataset_name: reference from the config file to the catalogs must be registered in DatasetCatalog and in detectron2's standard format output_file: path of json file that will be saved to allow_cached: if json file is already present then skip conversion """ # TODO: The dataset or the conversion script *may* change, # a checksum would be useful for validating the cached data PathManager.mkdirs(os.path.dirname(output_file)) with file_lock(output_file): if PathManager.exists(output_file) and allow_cached: logger.warning( f"Using previously cached COCO format annotations at '{output_file}'. " "You need to clear the cache file if your dataset has been modified." ) else: logger.info( f"Converting annotations of dataset '{dataset_name}' to COCO format ...)" ) coco_dict = convert_to_coco_dict(dataset_name) logger.info( f"Caching COCO format annotations at '{output_file}' ...") with PathManager.open(output_file, "w") as f: json.dump(coco_dict, f)
def evaluate(self): if self._distributed: comm.synchronize() predictions = comm.gather(self._predictions, dst=0) predictions = list(itertools.chain(*predictions)) if not comm.is_main_process(): return else: predictions = self._predictions if len(predictions) == 0: self._logger.warning( "[LVISEvaluator] Did not receive valid predictions.") return {} if self._output_dir: PathManager.mkdirs(self._output_dir) file_path = os.path.join(self._output_dir, "instances_predictions.pth") with PathManager.open(file_path, "wb") as f: torch.save(predictions, f) self._results = OrderedDict() if "proposals" in predictions[0]: self._eval_box_proposals(predictions) if "instances" in predictions[0]: self._eval_predictions(set(self._tasks), predictions) return copy.deepcopy(self._results)
def trace_and_save_torchscript( model: nn.Module, inputs: Tuple[torch.Tensor], output_path: str, _extra_files: Optional[Dict[str, bytes]] = None, ): logger.info("Tracing and saving TorchScript to {} ...".format(output_path)) # TODO: patch_builtin_len depends on D2, we should either copy the function or # dynamically registering the D2's version. from detectron2.export.torchscript_patch import patch_builtin_len with torch.no_grad(), patch_builtin_len(): script_model = torch.jit.trace(model, inputs) if _extra_files is None: _extra_files = {} model_file = os.path.join(output_path, "model.jit") PathManager.mkdirs(output_path) with PathManager.open(model_file, "wb") as f: torch.jit.save(script_model, f, _extra_files=_extra_files) data_file = os.path.join(output_path, "data.pth") with PathManager.open(data_file, "wb") as f: torch.save(inputs, f) # NOTE: new API doesn't require return return model_file
def simple_default_setup(cfg): """ Perform some basic common setups at the beginning of a job, including: 1. Set up the detectron2 logger 2. Log basic information about environment, and config 3. Backup the config to the output directory Args: cfg (CfgNode): the full config to be used """ output_dir = cfg.OUTPUT_DIR if comm.is_main_process() and output_dir: PathManager.mkdirs(output_dir) rank = comm.get_rank() setup_logger(output_dir, distributed_rank=rank, name="fvcore") logger = setup_logger(output_dir, distributed_rank=rank) logger.info("Rank of current process: {}. World size: {}".format( rank, comm.get_world_size())) logger.info("Environment info:\n" + collect_env_info()) logger.info("Running with full config:\n{}".format(cfg)) if comm.is_main_process() and output_dir: # Note: some of our scripts may expect the existence of # config.yaml in output directory path = os.path.join(output_dir, "config.yaml") with PathManager.open(path, "w") as f: f.write(cfg.dump()) logger.info("Full config saved to {}".format(os.path.abspath(path))) # make sure each worker has a different, yet deterministic seed if specified seed_all_rng(None if cfg.SEED < 0 else cfg.SEED + rank)
def evaluate(self, img_ids=None): """ Args: img_ids: a list of image IDs to evaluate on. Default to None for the whole dataset """ if self._distributed: comm.synchronize() predictions = comm.gather(self._predictions, dst=0) predictions = list(itertools.chain(*predictions)) if not comm.is_main_process(): return {} else: predictions = self._predictions if len(predictions) == 0: self._logger.warning("[COCOEvaluator] Did not receive valid predictions.") return {} if self._output_dir: PathManager.mkdirs(self._output_dir) file_path = os.path.join(self._output_dir, "instances_predictions.pth") with PathManager.open(file_path, "wb") as f: torch.save(predictions, f) self._results = OrderedDict() self._TTA_results = OrderedDict() self._score_df = OrderedDict() if "proposals" in predictions[0]: self._eval_box_proposals(predictions) if "instances" in predictions[0]: self._eval_predictions(set(self._tasks), predictions, img_ids=img_ids) # Copy so the caller can do whatever with results return copy.deepcopy(self._results)
def save_checkpoint(path_to_job, model, model_name, optimizer, epoch, cfg): """ Save a checkpoint. Args: model (model): model to save the weight to the checkpoint. optimizer (optim): optimizer to save the historical state. epoch (int): current number of epoch of the model. cfg (CfgNode): configs to save. """ # Save checkpoints only from the master process. # Ensure that the checkpoint dir exists. PathManager.mkdirs(get_checkpoint_dir(path_to_job)) # Omit the DDP wrapper in the multi-gpu setting. sd = model.module.state_dict() if cfg.NUM_GPUS > 1 else model.state_dict() # Record the state. checkpoint = { "epoch": epoch, "model_state": sd, "optimizer_state": optimizer.state_dict(), "cfg": cfg.dump(), } # Write the checkpoint. path_to_checkpoint = get_path_to_checkpoint(path_to_job, model_name, epoch + 1) # if (epoch+1)%10==0 or(epoch+1)==cfg.SOLVER.MAX_EPOCH : with PathManager.open(path_to_checkpoint, "wb") as f: torch.save(checkpoint, f) return path_to_checkpoint
def merge(self): """merge all clip features of a video into one/several fix-size matrix(es) """ if not PathManager.exists(self.merge_dir): PathManager.mkdirs(self.merge_dir) for video_name in PathManager.ls(self.save_dir): video_dir = os.path.join(self.save_dir, video_name) num_feats = len(PathManager.ls(video_dir)) if self.min_length <= num_feats <= self.max_length: merged_feat = torch.zeros((num_feats, self.dim), dtype=torch.float32) for clip_idx in range(num_feats): feat = torch.load( os.path.join(video_dir, f'{clip_idx}.pth')) merged_feat[clip_idx, :] = torch.from_numpy(feat) torch.save(merged_feat, os.path.join(self.merge_dir, f'{video_name}.pth')) else: # TODO print(video_name)
def _eval_predictions(self, predictions, imgIds=None): """ Evaluate predictions on densepose. Return results with the metrics of the tasks. """ self._logger.info("Preparing results for COCO format ...") if self._output_dir: PathManager.mkdirs(self._output_dir) file_path = os.path.join(self._output_dir, "coco_densepose_predictions.pth") with PathManager.open(file_path, "wb") as f: torch.save(predictions, f) self._logger.info("Evaluating predictions ...") res = OrderedDict() results_gps, results_gpsm, results_segm = _evaluate_predictions_on_coco( self._coco_api, predictions, min_threshold=self._min_threshold, imgIds=imgIds) res["densepose_gps"] = results_gps res["densepose_gpsm"] = results_gpsm res["densepose_segm"] = results_segm return res
def __init__(self, unified_label_file, dataset_name, cfg, distributed, output_dir=None): super().__init__(dataset_name, cfg, distributed, output_dir=output_dir) meta_dataset_name = dataset_name[:dataset_name.find('_')] print('meta_dataset_name', meta_dataset_name) self.unified_novel_classes_eval = cfg.MULTI_DATASET.UNIFIED_NOVEL_CLASSES_EVAL if self.unified_novel_classes_eval: match_novel_classes_file = cfg.MULTI_DATASET.MATCH_NOVEL_CLASSES_FILE print('Loading map back from', match_novel_classes_file) novel_classes_map = json.load(open(match_novel_classes_file, 'r'))[meta_dataset_name] self.map_back = {} for c, match in enumerate(novel_classes_map): for m in match: self.map_back[m] = c else: unified_label_data = json.load(open(unified_label_file, 'r')) label_map = unified_label_data['label_map'] label_map = label_map[meta_dataset_name] self.map_back = {int(v): i for i, v in enumerate(label_map)} self._logger.info("saving outputs to {}".format(self._output_dir)) self._temp_dir = self._output_dir + '/cityscapes_style_eval_tmp/' self._logger.info( "Writing cityscapes results to temporary directory {} ...".format( self._temp_dir)) PathManager.mkdirs(self._temp_dir)
def test_bad_args(self) -> None: with self.assertRaises(NotImplementedError): PathManager.copy( self._remote_uri, self._remote_uri, foo="foo" # type: ignore ) with self.assertRaises(NotImplementedError): PathManager.exists(self._remote_uri, foo="foo") # type: ignore with self.assertRaises(ValueError): PathManager.get_local_path( self._remote_uri, foo="foo" # type: ignore ) with self.assertRaises(NotImplementedError): PathManager.isdir(self._remote_uri, foo="foo") # type: ignore with self.assertRaises(NotImplementedError): PathManager.isfile(self._remote_uri, foo="foo") # type: ignore with self.assertRaises(NotImplementedError): PathManager.ls(self._remote_uri, foo="foo") # type: ignore with self.assertRaises(NotImplementedError): PathManager.mkdirs(self._remote_uri, foo="foo") # type: ignore with self.assertRaises(ValueError): PathManager.open(self._remote_uri, foo="foo") # type: ignore with self.assertRaises(NotImplementedError): PathManager.rm(self._remote_uri, foo="foo") # type: ignore PathManager.set_strict_kwargs_checking(False) PathManager.get_local_path(self._remote_uri, foo="foo") # type: ignore f = PathManager.open(self._remote_uri, foo="foo") # type: ignore f.close() PathManager.set_strict_kwargs_checking(True)
def save_checkpoint(checkpoint_folder, state, checkpoint_file=CHECKPOINT_FILE): """ Saves a state variable to the specified checkpoint folder. Returns the filename of the checkpoint if successful. Raises an exception otherwise. """ # make sure that we have a checkpoint folder: if not PathManager.isdir(checkpoint_folder): try: PathManager.mkdirs(checkpoint_folder) except BaseException: logging.warning("Could not create folder %s." % checkpoint_folder) raise # write checkpoint atomically: try: full_filename = f"{checkpoint_folder}/{checkpoint_file}" with PathManager.open(full_filename, "wb") as f: torch.save(state, f) return full_filename except BaseException: logging.warning("Unable to write checkpoint to %s." % checkpoint_folder, exc_info=True) raise
def convert_to_coco_json(dataset_name, output_folder="", allow_cached=True): """ Converts dataset into COCO format and saves it to a json file. dataset_name must be registered in DatastCatalog and in detectron2's standard format. Args: dataset_name: reference from the config file to the catalogs must be registered in DatastCatalog and in detectron2's standard format output_folder: where json file will be saved and loaded from allow_cached: if json file is already present then skip conversion Returns: cache_path: path to the COCO-format json file """ # TODO: The dataset or the conversion script *may* change, # a checksum would be useful for validating the cached data cache_path = os.path.join(output_folder, f"{dataset_name}_coco_format.json") PathManager.mkdirs(output_folder) if os.path.exists(cache_path) and allow_cached: logger.info( f"Reading cached annotations in COCO format from:{cache_path} ...") else: logger.info( f"Converting dataset annotations in '{dataset_name}' to COCO format ...)" ) coco_dict = convert_to_coco_dict(dataset_name) with PathManager.open(cache_path, "w") as json_file: logger.info(f"Caching annotations in COCO format: {cache_path}") json.dump(coco_dict, json_file) return cache_path
def save_checkpoint(path_to_job, model, optimizer, epoch, cfg): """ Save a checkpoint. Args: model (model): model to save the weight to the checkpoint. optimizer (optim): optimizer to save the historical state. epoch (int): current number of epoch of the model. cfg (CfgNode): configs to save. """ # Save checkpoints only from the master process. if not du.is_master_proc(cfg.NUM_GPUS * cfg.NUM_SHARDS): return # Ensure that the checkpoint dir exists. PathManager.mkdirs(get_checkpoint_dir(path_to_job)) # Omit the DDP wrapper in the multi-gpu setting. sd = model.module.state_dict() if cfg.NUM_GPUS > 1 else model.state_dict() normalized_sd = sub_to_normal_bn(sd) # Record the state. checkpoint = { "epoch": epoch, "model_state": normalized_sd, "optimizer_state": optimizer.state_dict(), "cfg": cfg.dump(), } # Write the checkpoint. path_to_checkpoint = get_path_to_checkpoint(path_to_job, epoch + 1) with PathManager.open(path_to_checkpoint, "wb") as f: torch.save(checkpoint, f) return path_to_checkpoint
def do_feature_extraction(cfg, model, image_dir, image_list): with inference_context(model): # dump_folder = os.path.join(cfg.OUTPUT_DIR, "features") dump_folder = 'gridfeats' PathManager.mkdirs(dump_folder) # data_loader = build_detection_test_loader_with_attributes(cfg, dataset_name) extract_grid_feature_on_dataset(model, image_dir, image_list, dump_folder)
def evaluate(self): """ Evaluates standard semantic segmentation metrics (http://cocodataset.org/#stuff-eval): * Mean intersection-over-union averaged across classes (mIoU) * Frequency Weighted IoU (fwIoU) * Mean pixel accuracy averaged across classes (mACC) * Pixel Accuracy (pACC) """ if self._distributed: synchronize() conf_matrix_list = all_gather(self._conf_matrix) self._predictions = all_gather(self._predictions) self._predictions = list(itertools.chain(*self._predictions)) if not is_main_process(): return self._conf_matrix = np.zeros_like(self._conf_matrix) for conf_matrix in conf_matrix_list: self._conf_matrix += conf_matrix if self._output_dir: PathManager.mkdirs(self._output_dir) file_path = os.path.join(self._output_dir, "sem_seg_predictions.json") with PathManager.open(file_path, "w") as f: f.write(json.dumps(self._predictions)) acc = np.zeros(self._num_classes, dtype=np.float) iou = np.zeros(self._num_classes, dtype=np.float) tp = self._conf_matrix.diagonal()[:-1].astype(np.float) pos_gt = np.sum(self._conf_matrix[:-1, :-1], axis=0).astype(np.float) class_weights = pos_gt / np.sum(pos_gt) pos_pred = np.sum(self._conf_matrix[:-1, :-1], axis=1).astype(np.float) acc_valid = pos_gt > 0 acc[acc_valid] = tp[acc_valid] / pos_gt[acc_valid] iou_valid = (pos_gt + pos_pred) > 0 union = pos_gt + pos_pred - tp iou[acc_valid] = tp[acc_valid] / union[acc_valid] macc = np.sum(acc) / np.sum(acc_valid) miou = np.sum(iou) / np.sum(iou_valid) fiou = np.sum(iou * class_weights) pacc = np.sum(tp) / np.sum(pos_gt) res = {} res["mIoU"] = 100 * miou res["fwIoU"] = 100 * fiou res["mACC"] = 100 * macc res["pACC"] = 100 * pacc if self._output_dir: file_path = os.path.join(self._output_dir, "sem_seg_evaluation.pth") with PathManager.open(file_path, "wb") as f: torch.save(res, f) results = OrderedDict({"sem_seg": res}) self._logger.info(results) return results
def evaluate(self): if self._distributed: comm.synchronize() self._predictions = comm.gather(self._predictions, dst=0) self._predictions = list(itertools.chain(*self._predictions)) if not comm.is_main_process(): return if len(self._predictions) == 0: self._logger.warning( "[LVISEvaluator] Did not receive valid predictions.") return {} self._logger.info("Preparing results in the OID format ...") _unified_results = list( itertools.chain(*[x["instances"] for x in self._predictions])) if self._output_dir: PathManager.mkdirs(self._output_dir) file_path = os.path.join(self._output_dir, "unified_instances_results.json") self._logger.info("Saving results to {}".format(file_path)) with PathManager.open(file_path, "w") as f: f.write(json.dumps(_unified_results)) f.flush() self._oid_results = map_back_unified_id(_unified_results, self.map_back) # unmap the category ids for LVIS (from 0-indexed to 1-indexed) for result in self._oid_results: result["category_id"] += 1 PathManager.mkdirs(self._output_dir) file_path = os.path.join(self._output_dir, "oid_instances_results.json") self._logger.info("Saving results to {}".format(file_path)) with PathManager.open(file_path, "w") as f: f.write(json.dumps(self._oid_results)) f.flush() if not self._do_evaluation: self._logger.info("Annotations are not available for evaluation.") return self._logger.info("Evaluating predictions ...") self._results = OrderedDict() res = _evaluate_predictions_on_oid(self._oid_api, file_path, eval_seg=self._mask_on) self._results['bbox'] = res return copy.deepcopy(self._results)
def evaluate( self, max_table_size: int = 25) -> "OrderedDict[str, Dict[str, Any]]": # if distributed, gather and sum confusion matrices cm: torch.Tensor if self._distributed: comm.synchronize() cms = comm.gather(self._cm, dst=0) if not comm.is_main_process(): return OrderedDict() cm = torch.stack(cms, dim=0).sum(dim=0) else: cm = self._cm # saving confusion matrix if self._output_dir: PathManager.mkdirs(self._output_dir) file_path = os.path.join(self._output_dir, self._output_name + ".pth") with PathManager.open(file_path, "wb") as f: torch.save(cm, f) file_path = os.path.join(self._output_dir, self._output_name + ".json") with PathManager.open(file_path, "w") as f: json_dict = {"confusion_matrix": cm.to("cpu").tolist()} json.dump(json_dict, f) # calculating accuracy accuracy = self.accuracy(cm) # displaying confusion matrix as table (if it isn't too huge) if self.num_classes <= max_table_size: headers, showindex = (), "default" if self._metadata is not None: headers = self._metadata.get("classes", default=headers) showindex = self._metadata.get("classes", default=showindex) table = tabulate( cm, headers=headers, showindex=showindex, tablefmt="pipe", floatfmt=".0f", numalign="left", ) self._logger.info(table) # collect and return results results: OrderedDict[str, Dict[str, float]] = OrderedDict([(self._task_name, { "top1": accuracy })]) self._logger.info(results) return results
def save_video(video, output_dir): """ Save video Args: video: shape [T, H, W, C] video_dir: save video under video_dir/sample/ """ PathManager.mkdirs(output_dir) for frame_idx in range(len(video)): frame_path = os.path.join(output_dir, f'{frame_idx}.png') save_image(video[frame_idx], frame_path)
def evaluate(self): if self._distributed: synchronize() conf_matrix_list = all_gather(self._conf_matrix) self._predictions = all_gather(self._predictions) self._predictions = list(itertools.chain(*self._predictions)) if not is_main_process(): return self._conf_matrix = np.zeros_like(self._conf_matrix) for conf_matrix in conf_matrix_list: self._conf_matrix += conf_matrix if self._output_dir: PathManager.mkdirs(self._output_dir) file_path = os.path.join(self._output_dir, "sem_seg_predictions.json") with PathManager.open(file_path, "w") as f: f.write(json.dumps(self._predictions)) acc = np.full(self._num_classes, np.nan, dtype=np.float) iou = np.full(self._num_classes, np.nan, dtype=np.float) tp = self._conf_matrix.diagonal()[:-1].astype(np.float) pos_gt = np.sum(self._conf_matrix[:-1, :-1], axis=0).astype(np.float) class_weights = pos_gt / np.sum(pos_gt) pos_pred = np.sum(self._conf_matrix[:-1, :-1], axis=1).astype(np.float) acc_valid = pos_gt > 0 acc[acc_valid] = tp[acc_valid] / pos_gt[acc_valid] iou_valid = (pos_gt + pos_pred) > 0 union = pos_gt + pos_pred - tp iou[acc_valid] = tp[acc_valid] / union[acc_valid] macc = np.sum(acc[acc_valid]) / np.sum(acc_valid) miou = np.sum(iou[acc_valid]) / np.sum(iou_valid) fiou = np.sum(iou[acc_valid] * class_weights[acc_valid]) pacc = np.sum(tp) / np.sum(pos_gt) res = {} res["mIoU"] = 100 * miou res["fwIoU"] = 100 * fiou for i, name in enumerate(self._class_names): res["IoU-{}".format(name)] = 100 * iou[i] res["mACC"] = 100 * macc res["pACC"] = 100 * pacc for i, name in enumerate(self._class_names): res["ACC-{}".format(name)] = 100 * acc[i] if self._output_dir: file_path = os.path.join(self._output_dir, "sem_seg_evaluation.pth") with PathManager.open(file_path, "wb") as f: torch.save(res, f) results = OrderedDict({"sem_seg": res}) self._logger.info(results) return results
def makedir(dir_path): """ Create the directory if it does not exist. """ is_success = False try: if not PathManager.exists(dir_path): PathManager.mkdirs(dir_path) is_success = True except BaseException: logging.info(f"Error creating directory: {dir_path}") return is_success