def get_valid_files(args, cfg, logger):
if "MODEL.WEIGHTS" in args.opts:
model_weights = cfg.MODEL.WEIGHTS
assert PathManager.exists(model_weights), "{} not exist!!!".format(
model_weights)
return [model_weights]
file_list = glob.glob(os.path.join(cfg.OUTPUT_DIR, "model_*.pth"))
if len(file_list) == 0: # local file invalid, get it from oss
model_prefix = cfg.OUTPUT_DIR.split("cvpods_playground")[-1][1:]
remote_file_path = os.path.join(cfg.OSS.DUMP_PREFIX, model_prefix)
logger.warning(
"No checkpoint file was found locally, try to "
f"load the corresponding dump file on OSS site: {remote_file_path}."
)
file_list = [
str(filename) for filename in PathManager.ls(remote_file_path)
if re.match(r"model_.+\.pth", filename.name) is not None
]
assert len(file_list) != 0, "No valid file found on OSS"
file_list = filter_by_iters(file_list, args.start_iter, args.end_iter)
assert file_list, "No checkpoint valid in {}.".format(cfg.OUTPUT_DIR)
logger.info("All files below will be tested in order:\n{}".format(
pformat(file_list)))
return file_list
def convert_to_coco_json(dataset_name, output_file, allow_cached=True):
"""
Converts dataset into COCO format and saves it to a json file.
dataset_name must be registered in DatasetCatalog and in cvpods's standard format.
Args:
dataset_name:
reference from the config file to the catalogs
must be registered in DatasetCatalog and in cvpods's standard format
output_file: path of json file that will be saved to
allow_cached: if json file is already present then skip conversion
"""
# TODO: The dataset or the conversion script *may* change,
# a checksum would be useful for validating the cached data
PathManager.mkdirs(os.path.dirname(output_file))
with file_lock(output_file):
if PathManager.exists(output_file) and allow_cached:
logger.info(
f"Cached annotations in COCO format already exist: {output_file}"
)
else:
logger.info(
f"Converting dataset annotations in '{dataset_name}' to COCO format ...)"
)
coco_dict = convert_to_coco_dict(dataset_name)
with PathManager.open(output_file, "w") as json_file:
logger.info(
f"Caching annotations in COCO format: {output_file}")
json.dump(coco_dict, json_file)
def _load_file(self, filename):
"""
Args:
filename (str): load checkpoint file from local or oss. checkpoint can be of type
pkl, pth
"""
if filename.endswith(".pkl"):
with PathManager.open(filename, "rb") as f:
data = pickle.load(f, encoding="latin1")
if "model" in data and "__author__" in data:
# file is in cvpods model zoo format
self.logger.info("Reading a file from '{}'".format(data["__author__"]))
return data
else:
# assume file is from Caffe2 / Detectron1 model zoo
if "blobs" in data:
# Detection models have "blobs", but ImageNet models don't
data = data["blobs"]
data = {k: v for k, v in data.items() if not k.endswith("_momentum")}
return {"model": data, "__author__": "Caffe2", "matching_heuristics": True}
elif filename.endswith(".pth"):
if filename.startswith("s3://"):
with PathManager.open(filename, "rb") as f:
loaded = torch.load(f, map_location=torch.device("cpu"))
else:
loaded = super()._load_file(filename) # load native pth checkpoint
if "model" not in loaded:
loaded = {"model": loaded}
return loaded
def evaluate(self):
if self._distributed:
comm.synchronize()
self._predictions = comm.gather(self._predictions, dst=0)
self._predictions = list(itertools.chain(*self._predictions))
self._targets = comm.gather(self._targets, dst=0)
self._targets = list(itertools.chain(*self._targets))
if not comm.is_main_process():
return {}
if len(self._predictions) == 0:
self._logger.warning("[ClassificationEvaluator] Did not receive valid predictions.")
return {}
if self._output_dir:
PathManager.mkdirs(self._output_dir)
file_path = os.path.join(self._output_dir, "instances_predictions.pth")
with PathManager.open(file_path, "wb") as f:
torch.save(self._predictions, f)
self._results = OrderedDict()
assert len(self._predictions) == len(self._targets)
if self._predictions[0] is not None:
self._eval_classification_accuracy()
if self._dump:
_dump_to_markdown(self._dump_infos)
# Copy so the caller can do whatever with results
return copy.deepcopy(self._results)
def evaluate(self):
if self._distributed:
comm.synchronize()
self._predictions = comm.gather(self._predictions, dst=0)
self._predictions = list(itertools.chain(*self._predictions))
if not comm.is_main_process():
return {}
if len(self._predictions) == 0:
self._logger.warning(
"[COCOEvaluator] Did not receive valid predictions.")
return {}
if self._output_dir:
PathManager.mkdirs(self._output_dir)
file_path = os.path.join(self._output_dir,
"instances_predictions.pth")
with PathManager.open(file_path, "wb") as f:
torch.save(self._predictions, f)
self._results = OrderedDict()
self._eval_predictions(set(self._tasks))
# Copy so the caller can do whatever with results
return copy.deepcopy(self._results)
def evaluate(self):
comm.synchronize()
self._predictions = comm.gather(self._predictions)
self._predictions = list(itertools.chain(*self._predictions))
if not comm.is_main_process():
return
gt_json = PathManager.get_local_path(self._metadata.panoptic_json)
gt_folder = self._metadata.panoptic_root
with tempfile.TemporaryDirectory(prefix="panoptic_eval") as pred_dir:
logger.info(
"Writing all panoptic predictions to {} ...".format(pred_dir))
for p in self._predictions:
with open(os.path.join(pred_dir, p["file_name"]), "wb") as f:
f.write(p.pop("png_string"))
with open(gt_json, "r") as f:
json_data = json.load(f)
json_data["annotations"] = self._predictions
with PathManager.open(self._predictions_json, "w") as f:
f.write(json.dumps(json_data))
from panopticapi.evaluation import pq_compute
with contextlib.redirect_stdout(io.StringIO()):
pq_res = pq_compute(
gt_json,
PathManager.get_local_path(self._predictions_json),
gt_folder=gt_folder,
pred_folder=pred_dir,
)
res = {}
res["PQ"] = 100 * pq_res["All"]["pq"]
res["SQ"] = 100 * pq_res["All"]["sq"]
res["RQ"] = 100 * pq_res["All"]["rq"]
res["PQ_th"] = 100 * pq_res["Things"]["pq"]
res["SQ_th"] = 100 * pq_res["Things"]["sq"]
res["RQ_th"] = 100 * pq_res["Things"]["rq"]
res["PQ_st"] = 100 * pq_res["Stuff"]["pq"]
res["SQ_st"] = 100 * pq_res["Stuff"]["sq"]
res["RQ_st"] = 100 * pq_res["Stuff"]["rq"]
results = OrderedDict({"panoptic_seg": res})
table = _print_panoptic_results(pq_res)
if self._dump:
dump_info_one_task = {
"task": "panoptic_seg",
"tables": [table],
}
_dump_to_markdown([dump_info_one_task])
return results
def default_setup(cfg, args):
"""
Perform some basic common setups at the beginning of a job, including:
1. Set up the cvpods logger
2. Log basic information about environment, cmdline arguments, and config
3. Backup the config to the output directory
Args:
cfg (BaseConfig): the full config to be used
args (argparse.NameSpace): the command line arguments to be logged
"""
output_dir = cfg.OUTPUT_DIR
if comm.is_main_process() and output_dir:
PathManager.mkdirs(output_dir)
rank = comm.get_rank()
# setup_logger(output_dir, distributed_rank=rank, name="cvpods")
logger = setup_logger(output_dir, distributed_rank=rank)
logger.info("Rank of current process: {}. World size: {}".format(
rank, comm.get_world_size()))
logger.info("Environment info:\n" + collect_env_info())
logger.info("Command line arguments: " + str(args))
if hasattr(args, "config_file") and args.config_file != "":
logger.info("Contents of args.config_file={}:\n{}".format(
args.config_file,
PathManager.open(args.config_file, "r").read()))
adjust_config(cfg)
logger.info("Running with full config:\n{}".format(cfg))
base_config = cfg.__class__.__base__()
logger.info("different config with base class:\n{}".format(
cfg.diff(base_config)))
# if comm.is_main_process() and output_dir:
# # Note: some of our scripts may expect the existence of
# # config.yaml in output directory
# path = os.path.join(output_dir, "config.yaml")
# with PathManager.open(path, "w") as f:
# f.write(cfg.dump())
# logger.info("Full config saved to {}".format(os.path.abspath(path)))
# make sure each worker has a different, yet deterministic seed if specified
seed = seed_all_rng(None if cfg.SEED < 0 else cfg.SEED + rank)
# save seed to config for dump
cfg.SEED = seed
# cudnn benchmark has large overhead. It shouldn't be used considering the small size of
# typical validation set.
if not (hasattr(args, "eval_only") and args.eval_only):
torch.backends.cudnn.benchmark = cfg.CUDNN_BENCHMARK
return cfg, logger
def get_all_checkpoint_files(self):
"""
Returns:
list: All available checkpoint files (.pth files) in target
directory.
"""
all_model_checkpoints = [
os.path.join(self.save_dir, file)
for file in PathManager.ls(self.save_dir)
if PathManager.isfile(os.path.join(self.save_dir, file)) and file.endswith(".pth")
]
return all_model_checkpoints
def save(self, name: str, tag_checkpoint: bool = True, **kwargs: dict):
"""
Dump model and checkpointables to a file.
Args:
name (str): name of the file.
kwargs (dict): extra arbitrary data to save.
"""
if not self.save_dir or not self.save_to_disk:
return
data = {}
data["model"] = self.model.state_dict()
for key, obj in self.checkpointables.items():
data[key] = obj.state_dict()
data.update(kwargs)
basename = "{}.pth".format(name)
save_file = os.path.join(self.save_dir, basename)
assert os.path.basename(save_file) == basename, basename
self.logger.info("Saving checkpoint to {}".format(save_file))
with PathManager.open(save_file, "wb") as f:
torch.save(data, f)
if tag_checkpoint:
self.tag_last_checkpoint(basename)
def process(self, inputs, outputs):
"""
Args:
inputs: the inputs to a model.
It is a list of dicts. Each dict corresponds to an image and
contains keys like "height", "width", "file_name".
outputs: the outputs of a model. It is either list of semantic segmentation predictions
(Tensor [H, W]) or list of dicts with key "sem_seg" that contains semantic
segmentation prediction in the same format.
"""
for input, output in zip(inputs, outputs):
output = output["sem_seg"].argmax(dim=0).to(self._cpu_device)
pred = np.array(output, dtype=np.int)
with PathManager.open(
self.input_file_to_gt_file[input["file_name"]], "rb") as f:
gt = np.array(Image.open(f), dtype=np.int)
gt[gt == self._ignore_label] = self._num_classes
self._conf_matrix += np.bincount(
self._N * pred.reshape(-1) + gt.reshape(-1),
minlength=self._N**2).reshape(self._N, self._N)
self._predictions.extend(
self.encode_json_sem_seg(pred, input["file_name"]))
def _eval_longtail_subgroup_accuracy(self, preds, target):
# category_frequency_file = os.path.join(dataset_path,'category_frequency.json')
with PathManager.open(self._longtail_json, 'r') as f:
category_frequency = json.load(f)
many_cats = category_frequency['many_cats']
medium_cats = category_frequency['medium_cats']
low_cats = category_frequency['low_cats']
cat_indicator = torch.zeros(len(self._metadata.thing_classes))
cat_indicator[many_cats] = 1
cat_indicator[medium_cats] = 2
cat_indicator[low_cats] = 3
labels_group_ids = cat_indicator[target]
labels_many = target[labels_group_ids == 1]
labels_medium = target[labels_group_ids == 2]
labels_low = target[labels_group_ids == 3]
preds_many = preds[:, labels_group_ids == 1]
preds_medium = preds[:, labels_group_ids == 2]
preds_low = preds[:, labels_group_ids == 3]
many_topks_correct = self._accuracy(preds_many, labels_many)
medium_topks_correct = self._accuracy(preds_medium, labels_medium)
low_topks_correct = self._accuracy(preds_low, labels_low)
top_acc_subgroups = [
many_topks_correct, medium_topks_correct, low_topks_correct
]
return top_acc_subgroups
def _load_semantic_annotations(self, image_dir, gt_dir):
"""
Args:
image_dir (str): path to the raw dataset. e.g., "~/cityscapes/leftImg8bit/train".
gt_dir (str): path to the raw annotations. e.g., "~/cityscapes/gtFine/train".
Returns:
list[dict]: a list of dict, each has "file_name" and
"sem_seg_file_name".
"""
ret = []
for image_file in glob.glob(os.path.join(image_dir, "**/*.png")):
suffix = "leftImg8bit.png"
assert image_file.endswith(suffix)
prefix = image_dir
label_file = (gt_dir + image_file[len(prefix):-len(suffix)] +
"gtFine_labelTrainIds.png")
assert os.path.isfile(
label_file
), "Please generate labelTrainIds.png with cityscapesscripts/preparation/createTrainIdLabelImgs.py" # noqa
json_file = gt_dir + image_file[
len(prefix):-len(suffix)] + "gtFine_polygons.json"
with PathManager.open(json_file, "r") as f:
jsonobj = json.load(f)
ret.append({
"file_name": image_file,
"sem_seg_file_name": label_file,
"height": jsonobj["imgHeight"],
"width": jsonobj["imgWidth"],
})
return ret
def filter_by_iters(file_list, start_iter, end_iter):
# sort file_list by modified time
if file_list[0].startswith("s3://"):
file_list.sort(key=lambda x: PathManager.stat(x).m_date)
else:
file_list.sort(key=os.path.getmtime)
if start_iter is None:
if end_iter is None:
# use latest ckpt if start_iter and end_iter are not given
return [file_list[-1]]
else:
start_iter = 0
elif end_iter is None:
end_iter = float("inf")
iter_infos = [re.split(r"model_|\.pth", f)[-2] for f in file_list]
keep_list = [0] * len(iter_infos)
start_index = 0
if "final" in iter_infos and iter_infos[-1] != "final":
start_index = iter_infos.index("final")
for i in range(len(iter_infos) - 1, start_index, -1):
if iter_infos[i] == "final":
if end_iter == float("inf"):
keep_list[i] = 1
elif float(start_iter) < float(iter_infos[i]) < float(end_iter):
keep_list[i] = 1
if float(iter_infos[i - 1]) > float(iter_infos[i]):
break
return [
filename for keep, filename in zip(keep_list, file_list) if keep == 1
]
def has_checkpoint(self):
"""
Returns:
bool: whether a checkpoint exists in the target directory.
"""
save_file = os.path.join(self.save_dir, "last_checkpoint")
return PathManager.exists(save_file)
def _eval_predictions(self, tasks):
"""
Evaluate self._predictions on the given tasks.
Fill self._results with the metrics of the tasks.
"""
self._logger.info("Preparing results for CrowdHuman format ...")
self._coco_results = self._predictions
if self._output_dir:
file_path = os.path.join(self._output_dir,
"coco_instances_results.json")
self._logger.info("Saving results to {}".format(file_path))
with PathManager.open(file_path, "w") as f:
for db in self._coco_results:
line = json.dumps(db) + '\n'
f.write(line)
self._logger.info("Evaluating predictions ...")
for task in sorted(tasks):
coco_eval = (
_evaluate_predictions_on_crowdhuman(self._metadata.json_file,
file_path)
if len(self._coco_results) > 0 else
None # cocoapi does not handle empty results very well
)
res = self._derive_coco_results(coco_eval, task)
self._results[task] = res
def __init__(self,
dataset_name,
meta,
cfg,
distributed,
output_dir=None,
dump=False):
"""
Args:
dataset_name (str): name of the dataset to be evaluated.
It must have either the following corresponding metadata:
"json_file": the path to the COCO format annotation
Or it must be in cvpods's standard dataset format
so it can be converted to COCO format automatically.
meta (SimpleNamespace): dataset metadata.
cfg (config dict): cvpods Config instance.
distributed (True): if True, will collect results from all ranks for evaluation.
Otherwise, will evaluate the results in the current process.
output_dir (str): optional, an output directory to dump all
results predicted on the dataset. The dump contains two files:
1. "instance_predictions.pth" a file in torch serialization
format that contains all the raw original predictions.
2. "coco_instances_results.json" a json file in COCO's result
format.
dump (bool): If True, after the evaluation is completed, a Markdown file
that records the model evaluation metrics and corresponding scores
will be generated in the working directory.
"""
self._dump = dump
self.cfg = cfg
self._tasks = self._tasks_from_config(cfg)
self._distributed = distributed
self._output_dir = output_dir
self._cpu_device = torch.device("cpu")
self._logger = logging.getLogger(__name__)
self._metadata = meta
if not hasattr(self._metadata, "json_file"):
self._logger.warning(
f"json_file was not found in MetaDataCatalog for '{dataset_name}'."
" Trying to convert it to COCO format ...")
cache_path = os.path.join(output_dir,
f"{dataset_name}_coco_format.json")
self._metadata.json_file = cache_path
convert_to_coco_json(dataset_name, cache_path)
json_file = PathManager.get_local_path(self._metadata.json_file)
with contextlib.redirect_stdout(io.StringIO()):
self._coco_api = COCO(json_file)
self._kpt_oks_sigmas = cfg.TEST.KEYPOINT_OKS_SIGMAS
# Test set json files do not contain annotations (evaluation must be
# performed using the COCO evaluation server).
self._do_evaluation = "annotations" in self._coco_api.dataset
def _load_annotations(self):
"""
Load Pascal VOC detection annotations to cvpods format.
Args:
dirname: Contain "Annotations", "ImageSets", "JPEGImages"
split (str): one of "train", "test", "val", "trainval"
"""
dirname = self.image_root
split = self.split
with PathManager.open(
os.path.join(dirname, "ImageSets", "Main",
split + ".txt")) as f:
fileids = np.loadtxt(f, dtype=np.str)
dicts = []
for fileid in fileids:
anno_file = os.path.join(dirname, "Annotations", fileid + ".xml")
jpeg_file = os.path.join(dirname, "JPEGImages", fileid + ".jpg")
tree = ET.parse(anno_file)
r = {
"file_name": jpeg_file,
"image_id": fileid,
"height": int(tree.findall("./size/height")[0].text),
"width": int(tree.findall("./size/width")[0].text),
}
instances = []
for obj in tree.findall("object"):
cls = obj.find("name").text
# We include "difficult" samples in training.
# Based on limited experiments, they don't hurt accuracy.
# difficult = int(obj.find("difficult").text)
# if difficult == 1:
# continue
bbox = obj.find("bndbox")
bbox = [
float(bbox.find(x).text)
for x in ["xmin", "ymin", "xmax", "ymax"]
]
# Original annotations are integers in the range [1, W or H]
# Assuming they mean 1-based pixel indices (inclusive),
# a box with annotation (xmin=1, xmax=W) covers the whole image.
# In coordinate space this is represented by (xmin=0, xmax=W)
bbox[0] -= 1.0
bbox[1] -= 1.0
instances.append({
"category_id": CLASS_NAMES.index(cls),
"bbox": bbox,
"bbox_mode": BoxMode.XYXY_ABS
})
r["annotations"] = instances
dicts.append(r)
return dicts
def __getitem__(self, index):
"""Load data, apply transforms, converto to Instances.
"""
dataset_dict = copy.deepcopy(self.dataset_dicts[index])
# read image
image = read_image(dataset_dict["file_name"], format=self.data_format)
check_image_size(dataset_dict, image)
if "annotations" in dataset_dict:
annotations = dataset_dict.pop("annotations")
annotations = [
ann for ann in annotations if ann.get("iscrowd", 0) == 0
]
else:
annotations = None
if "sem_seg_file_name" in dataset_dict:
assert annotations is None
annotations = []
with PathManager.open(dataset_dict.get("sem_seg_file_name"),
"rb") as f:
sem_seg_gt = Image.open(f)
sem_seg_gt = np.asarray(sem_seg_gt, dtype="uint8")
annotations.append({"sem_seg": sem_seg_gt})
# apply transfrom
image, annotations = self._apply_transforms(image, annotations)
if "sem_seg_file_name" in dataset_dict:
dataset_dict.pop("sem_seg_file_name")
sem_seg_gt = annotations[0].pop("sem_seg")
sem_seg_gt = torch.as_tensor(sem_seg_gt.astype("long"))
dataset_dict["sem_seg"] = sem_seg_gt
annotations = None
if annotations is not None:
image_shape = image.shape[:2] # h, w
instances = annotations_to_instances(annotations,
image_shape,
mask_format=self.mask_format)
# # Create a tight bounding box from masks, useful when image is cropped
# if self.crop_gen and instances.has("gt_masks"):
# instances.gt_boxes = instances.gt_masks.get_bounding_boxes()
dataset_dict["instances"] = filter_empty_instances(instances)
# convert to Instance type
# Pytorch's dataloader is efficient on torch.Tensor due to shared-memory,
# but not efficient on large generic data structures due to the use of pickle & mp.Queue.
# Therefore it's important to use torch.Tensor.
# h, w, c -> c, h, w
dataset_dict["image"] = torch.as_tensor(
np.ascontiguousarray(image.transpose(2, 0, 1)))
return dataset_dict
def __init__(self, json_file, window_size=20):
"""
Args:
json_file (str): path to the json file. New data will be appended if the file exists.
window_size (int): the window size of median smoothing for the scalars whose
`smoothing_hint` are True.
"""
self._file_handle = PathManager.open(json_file, "a")
self._window_size = window_size
def main(args):
config.merge_from_list(args.opts)
cfg, logger = default_setup(config, args)
if args.debug:
batches = int(cfg.SOLVER.IMS_PER_BATCH / 8 * args.num_gpus)
if cfg.SOLVER.IMS_PER_BATCH != batches:
cfg.SOLVER.IMS_PER_BATCH = batches
logger.warning(
"SOLVER.IMS_PER_BATCH is changed to {}".format(batches))
if "MODEL.WEIGHTS" in args.opts:
if cfg.MODEL.WEIGHTS.endswith(".pth") and not PathManager.exists(
cfg.MODEL.WEIGHTS):
ckpt_name = cfg.MODEL.WEIGHTS.split("/")[-1]
model_prefix = cfg.OUTPUT_DIR.split("cvpods_playground")[1][1:]
remote_file_path = os.path.join(cfg.OSS.DUMP_PREFIX, model_prefix,
ckpt_name)
logger.warning(
f"The specified ckpt file ({cfg.MODEL.WEIGHTS}) was not found locally,"
f" try to load the corresponding dump file on OSS ({remote_file_path})."
)
cfg.MODEL.WEIGHTS = remote_file_path
valid_files = [cfg.MODEL.WEIGHTS]
else:
list_of_files = glob.glob(os.path.join(cfg.OUTPUT_DIR, '*.pth'))
assert list_of_files, "No checkpoint file found in {}.".format(
cfg.OUTPUT_DIR)
list_of_files.sort(key=os.path.getctime)
latest_file = list_of_files[-1]
if not args.end_iter:
valid_files = [latest_file]
else:
files = [f for f in list_of_files if str(f) <= str(latest_file)]
valid_files = []
for f in files:
try:
model_iter = int(re.split(r'(model_|\.pth)', f)[-3])
except Exception:
logger.warning("remove {}".format(f))
continue
if args.start_iter <= model_iter <= args.end_iter:
valid_files.append(f)
assert valid_files, "No .pth files satisfy your requirement"
# * means all if need specific format then *.csv
for current_file in valid_files:
cfg.MODEL.WEIGHTS = current_file
model = build_model(cfg)
DetectionCheckpointer(model, save_dir=cfg.OUTPUT_DIR).resume_or_load(
cfg.MODEL.WEIGHTS, resume=args.resume)
res = Trainer.test(cfg, model)
if comm.is_main_process():
verify_results(cfg, res)
if cfg.TEST.AUG.ENABLED:
res.update(Trainer.test_with_TTA(cfg, model))
def load_proposals_into_dataset(dataset_dicts, proposal_file):
r"""
Load precomputed object proposals into the dataset.
The proposal file should be a pickled dict with the following keys:
- "ids": list[int] or list[str], the image ids
- "boxes": list[np.ndarray], each is an Nx4 array of boxes corresponding to the image id
- "objectness_logits": list[np.ndarray], each is an N sized array of objectness scores
corresponding to the boxes.
- "bbox_mode": the BoxMode of the boxes array. Defaults to ``BoxMode.XYXY_ABS``.
Args:
dataset_dicts (list[dict]): annotations in cvpods Dataset format.
proposal_file (str): file path of pre-computed proposals, in pkl format.
Returns:
list[dict]: the same format as dataset_dicts, but added proposal field.
"""
logger = logging.getLogger(__name__)
logger.info("Loading proposals from: {}".format(proposal_file))
with PathManager.open(proposal_file, "rb") as f:
proposals = pickle.load(f, encoding="latin1")
# Rename the key names in D1 proposal files
rename_keys = {"indexes": "ids", "scores": "objectness_logits"}
for key in rename_keys:
if key in proposals:
proposals[rename_keys[key]] = proposals.pop(key)
# Fetch the indexes of all proposals that are in the dataset
# Convert image_id to str since they could be int.
img_ids = set({str(record["image_id"]) for record in dataset_dicts})
id_to_index = {
str(id): i
for i, id in enumerate(proposals["ids"]) if str(id) in img_ids
}
# Assuming default bbox_mode of precomputed proposals are 'XYXY_ABS'
bbox_mode = BoxMode(proposals["bbox_mode"]
) if "bbox_mode" in proposals else BoxMode.XYXY_ABS
for record in dataset_dicts:
# Get the index of the proposal
i = id_to_index[str(record["image_id"])]
boxes = proposals["boxes"][i]
objectness_logits = proposals["objectness_logits"][i]
# Sort the proposals in descending order of the scores
inds = objectness_logits.argsort()[::-1]
record["proposal_boxes"] = boxes[inds]
record["proposal_objectness_logits"] = objectness_logits[inds]
record["proposal_bbox_mode"] = bbox_mode
return dataset_dicts
def after_step(self):
if self._profiler is None:
return
self._profiler.__exit__(None, None, None)
PathManager.mkdirs(self._output_dir)
out_file = os.path.join(
self._output_dir,
"profiler-trace-iter{}.json".format(self.trainer.iter))
if "://" not in out_file:
self._profiler.export_chrome_trace(out_file)
else:
# Support non-posix filesystems
with tempfile.TemporaryDirectory(prefix="cvpods_profiler") as d:
tmp_file = os.path.join(d, "tmp.json")
self._profiler.export_chrome_trace(tmp_file)
with open(tmp_file) as f:
content = f.read()
with PathManager.open(out_file, "w") as f:
f.write(content)
def tag_last_checkpoint(self, last_filename_basename: str):
"""
Tag the last checkpoint.
Args:
last_filename_basename (str): the basename of the last filename.
"""
save_file = os.path.join(self.save_dir, "last_checkpoint")
with PathManager.open(save_file, "w") as f:
f.write(last_filename_basename)
def _eval_predictions(self, tasks):
"""
Evaluate self._predictions on the given tasks.
Fill self._results with the metrics of the tasks.
"""
self._logger.info("Preparing results for COCO format ...")
self._coco_results = list(
itertools.chain(*[x["instances"] for x in self._predictions]))
# unmap the category ids for COCO
if hasattr(self._metadata, "thing_dataset_id_to_contiguous_id"):
reverse_id_mapping = {
v: k
for k, v in
self._metadata.thing_dataset_id_to_contiguous_id.items()
}
for result in self._coco_results:
category_id = result["category_id"]
assert (
category_id in reverse_id_mapping
), "A prediction has category_id={}, which is not available in the dataset.".format(
category_id)
result["category_id"] = reverse_id_mapping[category_id]
if self._output_dir:
file_path = os.path.join(self._output_dir,
"coco_instances_results.json")
self._logger.info("Saving results to {}".format(file_path))
with PathManager.open(file_path, "w") as f:
f.write(json.dumps(self._coco_results))
f.flush()
if not self._do_evaluation:
self._logger.info("Annotations are not available for evaluation.")
return
self._logger.info("Evaluating predictions ...")
for task in sorted(tasks):
coco_eval, summary = (
_evaluate_predictions_on_coco(
self._coco_api,
self._coco_results,
task,
kpt_oks_sigmas=self._kpt_oks_sigmas)
if len(self._coco_results) > 0 else
None # cocoapi does not handle empty results very well
)
self._logger.info("\n" + summary.getvalue())
res = self._derive_coco_results(
coco_eval,
task,
summary,
class_names=self._metadata.thing_classes)
self._results[task] = res
def build_darknet_backbone(cfg, input_shape):
depth = cfg.MODEL.DARKNET.DEPTH
stem_channels = cfg.MODEL.DARKNET.STEM_OUT_CHANNELS
output_features = cfg.MODEL.DARKNET.OUT_FEATURES
model = Darknet(depth, input_shape.channels, stem_channels,
output_features)
filename = cfg.MODEL.DARKNET.WEIGHTS
with PathManager.open(filename, "rb") as f:
state_dict = torch.load(f, map_location='cpu')
model.load_state_dict(state_dict)
return model
def _eval_predictions(self, tasks):
"""
Evaluate self._predictions on the given tasks.
Fill self._results with the metrics of the tasks.
"""
self._logger.info("Preparing results in the LVIS format ...")
self._lvis_results = list(
itertools.chain(*[x["instances"] for x in self._predictions]))
# LVIS evaluator can be used to evaluate results for COCO dataset categories.
# In this case `_metadata` variable will have a field with COCO-specific category mapping.
if hasattr(self._metadata, "thing_dataset_id_to_contiguous_id"):
reverse_id_mapping = {
v: k
for k, v in
self._metadata.thing_dataset_id_to_contiguous_id.items()
}
for result in self._lvis_results:
result["category_id"] = reverse_id_mapping[
result["category_id"]]
else:
# unmap the category ids for LVIS (from 0-indexed to 1-indexed)
for result in self._lvis_results:
result["category_id"] += 1
if self._output_dir:
file_path = os.path.join(self._output_dir,
"lvis_instances_results.json")
self._logger.info("Saving results to {}".format(file_path))
with PathManager.open(file_path, "w") as f:
f.write(json.dumps(self._lvis_results))
f.flush()
if not self._do_evaluation:
self._logger.info("Annotations are not available for evaluation.")
return
self._logger.info(
"Evaluating predictions with use_fast_impl={} ...".format(
self._use_fast_impl))
for task in sorted(tasks):
lvis_eval, summary = (_evaluate_predictions_on_lvis(
self._lvis_api,
self._lvis_results,
task,
use_fast_impl=self._use_fast_impl,
max_dets=self._max_dets)
if len(self._lvis_results) > 0 else None)
self._logger.info("\n" + summary.getvalue())
res = self._derive_lvis_results(lvis_eval, task, summary)
self._results[task] = res
def get_checkpoint_file(self):
"""
Returns:
str: The latest checkpoint file in target directory.
"""
save_file = os.path.join(self.save_dir, "last_checkpoint")
try:
with PathManager.open(save_file, "r") as f:
last_saved = f.read().strip()
except IOError:
# if file doesn't exist, maybe because it has just been
# deleted by a separate process
return ""
return os.path.join(self.save_dir, last_saved)
def load(self, path: str):
"""
Load from the given checkpoint. When path points to network file, this
function has to be called on all ranks.
Args:
path (str): path or url to the checkpoint. If empty, will not load
anything.
Returns:
dict:
extra data loaded from the checkpoint that has not been
processed. For example, those saved with
:meth:`.save(**extra_data)`.
"""
if not path:
# no checkpoint provided
self.logger.info(
"No checkpoint found. Initializing model from scratch")
return {}
self.logger.info("Loading checkpoint from {}".format(path))
if not os.path.isfile(path):
path = PathManager.get_local_path(path)
assert PathManager.isfile(path), "Checkpoint {} not found!".format(
path)
checkpoint = self._load_file(path)
self._load_model(checkpoint)
if self.resume:
for key, obj in self.checkpointables.items():
if key in checkpoint:
self.logger.info("Loading {} from {}".format(key, path))
obj.load_state_dict(checkpoint.pop(key))
# return any further checkpoint data
return checkpoint
else:
return {}
def evaluate(self):
if self._distributed:
comm.synchronize()
self._predictions = comm.gather(self._predictions, dst=0)
self._predictions = list(itertools.chain(*self._predictions))
if not comm.is_main_process():
return {}
if len(self._predictions) == 0:
self._logger.warning(
"[COCOEvaluator] Did not receive valid predictions.")
return {}
if self._output_dir:
PathManager.mkdirs(self._output_dir)
file_path = os.path.join(self._output_dir,
"instances_predictions.pth")
with PathManager.open(file_path, "wb") as f:
torch.save(self._predictions, f)
self._results = OrderedDict()
if "proposals" in self._predictions[0]:
self._eval_box_proposals()
if "instances" in self._predictions[0]:
self._eval_predictions(set(self._tasks))
if self._dump:
extra_infos = {
"title": os.path.basename(os.getcwd()),
"seed": self.cfg.SEED,
}
_dump_to_markdown(extra_infos, self._dump_infos)
# Copy so the caller can do whatever with results
return copy.deepcopy(self._results)
def setup_cfg(args):
# load config from file and command-line arguments
cfg = get_config(args.config, None)
cfg.merge_from_list(args.opts)
if cfg.MODEL.WEIGHTS == "":
oss_prefix = os.path.join(cfg.OSS.MODEL_PREFIX, "model_zoo")
file_path = os.path.join(oss_prefix, args.config, "model_final.pth")
logger.warning(
f"No checkpoint file specified, "
f"trying to get it from Model Zoo (OSS URI: {file_path}).")
assert PathManager.isfile(
file_path), f"No checkpoint file found in {file_path}."
cfg.MODEL.WEIGHTS = file_path
return cfg
