def process_video(
filepath: str,
fps: float,
start_time: int,
max_frames: int,
out_dir: str,
quiet: bool = False,
) -> None:
"""Break one video into a folder of images."""
if not check_video_format(filepath):
logger.warning("Ignore invalid file %s", filepath)
return
if not os.path.exists(filepath):
logger.warning("%s does not exist", filepath)
return
cmd = ["ffmpeg", "-i", filepath, "-qscale:v", "2"]
if start_time > 0:
cmd.extend(["-ss", str(start_time)])
if max_frames > 0:
cmd.extend(["-frames:v", str(max_frames)])
if fps > 0:
cmd.extend(["-r", str(fps)])
video_name = os.path.splitext(os.path.split(filepath)[1])[0]
out_dir = join(out_dir, video_name)
os.makedirs(out_dir, exist_ok=True)
cmd.append("{}/{}-%07d.jpg".format(out_dir, video_name))
if not quiet:
logger.info("RUNNING %s", cmd)
pipe = DEVNULL if quiet else None
check_call(cmd, stdout=pipe, stderr=pipe)
def main() -> None:
"""Run main function."""
args = parse_arguments()
if args.out_dir:
if args.scratch and os.path.exists(args.out_dir):
logger.info("Remove existing target directory")
shutil.rmtree(args.out_dir)
os.makedirs(args.out_dir, exist_ok=True)
prepare_data(args)
def write_output(filename: str, labels: List[LabelObject]) -> None:
"""Write output file."""
ext = splitext(filename)[1]
logger.info("Writing %s", filename)
with open(filename, "w") as fp:
if ext == ".json":
json.dump(labels, fp)
elif ext in [".yml", ".yaml"]:
yaml.dump(labels, fp)
else:
raise ValueError("Unrecognized file extension {}".format(ext))
def read_input(filename: str) -> List[LabelObject]:
"""Read one input label file."""
labels: List[LabelObject]
ext = splitext(filename)[1]
logger.info("Reading %s", filename)
with open(filename, "r") as fp:
if ext == ".json":
labels = json.load(fp)
elif ext in [".yml", ".yaml"]:
labels = yaml.load(fp)
else:
raise ValueError("Unrecognized file extension {}".format(ext))
return labels
def copy_images(in_path: str, out_path: str) -> None:
"""Copy images to the output folder with proper video name."""
file_list = glob.glob(join(in_path, "**/*.jpg"), recursive=True)
in_path_parts = os.path.split(in_path)
if len(in_path_parts) > 0 and len(in_path_parts[-1]) > 0:
video_name = in_path_parts[-1]
else:
video_name = "default"
out_dir = join(out_path, video_name)
os.makedirs(out_dir)
logger.info("Copying %s to %s", in_path, out_dir)
for image_path in tqdm(file_list):
image_name = os.path.split(image_path)[-1]
shutil.copyfile(image_path, join(out_dir, image_name))
def create_image_list(out_dir: str, url_root: str) -> str:
"""Create image list from the output directory."""
file_list = sorted(glob.glob(join(out_dir, "*/*.jpg")))
yaml_items = [{
"url": (os.path.abspath(img) if not url_root else join(
url_root,
img.split("/")[-2], os.path.basename(img))),
"videoName":
img.split("/")[-2],
} for img in file_list]
list_path = join(out_dir, "image_list.yml")
with open(list_path, "w") as f:
yaml.dump(yaml_items, f)
logger.info("The configuration file saved at %s", list_path)
return list_path
def prepare_data(args: argparse.Namespace) -> None:
"""Break one or a list of videos into frames."""
url_root = args.url_root
if args.s3:
url_root = s3_setup(args.s3)
inputs = parse_input_list(args)
logger.info("processing %d video(s) ...", len(inputs))
num_videos = len(inputs)
video_range = range(len(inputs))
quiet = num_videos > 1
if num_videos > 1:
video_range = tqdm(video_range)
jobs = args.jobs
if num_videos >= jobs > 0:
Parallel(n_jobs=jobs,
backend="multiprocessing")(delayed(process_input)(
inputs[i],
args.fps,
args.start_time,
args.max_frames,
args.out_dir,
quiet,
) for i in video_range)
else:
for i in video_range:
process_input(
inputs[i],
args.fps,
args.start_time,
args.max_frames,
args.out_dir,
quiet,
)
# upload to s3 if needed
if args.s3:
upload_files_to_s3(args.s3, args.out_dir)
# create the yaml file
if not args.no_list:
create_image_list(args.out_dir, url_root)
def upload_files_to_s3(s3_path: str, out_dir: str) -> None:
"""Send the files to s3."""
s3 = boto3.resource("s3")
s3_param = parse_s3_path(s3_path)
file_list = glob.glob(join(out_dir, "**/*.jpg"), recursive=True)
logger.info(
"Uploading %d files to s3 %s:%s",
len(file_list),
s3_param.bucket,
s3_param.folder,
)
for f in tqdm(file_list):
try:
# pylint is added here because it thinks boto3.resource is a string
s3.Bucket(s3_param.bucket).upload_file(
f,
join(s3_param.folder, f[len(out_dir) + 1:]),
ExtraArgs={"ACL": "public-read"},
)
except boto3.exceptions.S3UploadFailedError as e:
logger.error("s3 bucket is not properly configured %s", e)
break
def evaluate_pan_seg(
ann_frames: List[Frame],
pred_frames: List[Frame],
config: Config,
ignore_unknown_cats: bool = False,
nproc: int = NPROC,
) -> PanSegResult:
"""Evaluate panoptic segmentation with Scalabel format.
Args:
ann_frames: the ground truth frames.
pred_frames: the prediction frames.
config: Metadata config.
ignore_unknown_cats: ignore unknown categories.
nproc: the number of process.
Returns:
PanSegResult: evaluation results.
"""
categories = get_leaf_categories(config.categories)
assert all(
category.isThing is not None for category in categories
), "isThing should be defined for all categories for PanSeg."
categories_stuff = [
category for category in categories if not category.isThing
]
categories_thing = [
category for category in categories if category.isThing
]
category_names = [category.name for category in categories]
pred_frames = reorder_preds(ann_frames, pred_frames)
label_ids_to_int(ann_frames)
# check overlap of masks
logger.info("checking for overlap of masks...")
if check_overlap(pred_frames, config, nproc):
logger.critical(
"Found overlap in prediction bitmasks, but panoptic segmentation "
"evaluation does not allow overlaps. Removing such predictions."
)
logger.info("evaluating...")
if nproc > 1:
with Pool(nproc) as pool:
pq_stats = pool.starmap(
partial(
pq_per_image,
categories=categories,
ignore_unknown_cats=ignore_unknown_cats,
image_size=config.imageSize,
),
tqdm(zip(ann_frames, pred_frames), total=len(ann_frames)),
)
else:
pq_stats = [
pq_per_image(
ann_frame,
pred_frame,
categories=categories,
ignore_unknown_cats=ignore_unknown_cats,
image_size=config.imageSize,
)
for ann_frame, pred_frame in tqdm(
zip(ann_frames, pred_frames), total=len(ann_frames)
)
]
pq_stat = PQStat(categories)
for pq_stat_ in pq_stats:
pq_stat += pq_stat_
logger.info("accumulating...")
res_dict: Dict[str, ScoresList] = {}
for category_name, category in zip(category_names, categories):
result = pq_stat.pq_average([category])
for metric, score in result.items():
if metric not in res_dict:
res_dict[metric] = [{}, {}, {}]
res_dict[metric][0][category_name] = score
result = pq_stat.pq_average(categories_stuff)
for metric, score in result.items():
res_dict[metric][1][STUFF] = score
result = pq_stat.pq_average(categories_thing)
for metric, score in result.items():
res_dict[metric][1][THING] = score
result = pq_stat.pq_average(categories)
for metric, score in result.items():
res_dict[metric][2][OVERALL] = score
return PanSegResult(**res_dict)
"--ignore-unknown-cats",
action="store_true",
help="ignore unknown categories for panseg evaluation",
)
parser.add_argument(
"--nproc",
"-p",
type=int,
default=NPROC,
help="number of processes for panseg evaluation",
)
return parser.parse_args()
if __name__ == "__main__":
args = parse_arguments()
dataset = load(args.gt, args.nproc)
gts, cfg = dataset.frames, dataset.config
preds = load(args.result).frames
if args.config is not None:
cfg = load_label_config(args.config)
assert cfg is not None
eval_result = evaluate_pan_seg(
gts, preds, cfg, args.ignore_unknown_cats, args.nproc
)
logger.info(eval_result)
logger.info(eval_result.summary())
if args.out_file:
with open_write_text(args.out_file) as fp:
json.dump(eval_result.dict(), fp, indent=2)
def evaluate_sem_seg(
ann_frames: List[Frame],
pred_frames: List[Frame],
config: Config,
nproc: int = NPROC,
) -> SegResult:
"""Evaluate segmentation with Scalabel format.
Args:
ann_frames: the ground truth frames.
pred_frames: the prediction frames.
config: Metadata config.
nproc: the number of process.
Returns:
SegResult: evaluation results.
"""
categories = {
cat.name: id
for id, cat in enumerate(get_leaf_categories(config.categories))
}
ignore_label = 255
pred_frames = reorder_preds(ann_frames, pred_frames)
logger.info("evaluating...")
if nproc > 1:
with Pool(nproc) as pool:
hist_and_gt_id_sets = pool.starmap(
partial(
per_image_hist,
categories=categories,
image_size=config.imageSize,
ignore_label=ignore_label,
),
tqdm(zip(ann_frames, pred_frames), total=len(ann_frames)),
)
else:
hist_and_gt_id_sets = [
per_image_hist(
ann_frame,
pred_frame,
categories=categories,
image_size=config.imageSize,
ignore_label=ignore_label,
)
for ann_frame, pred_frame in tqdm(
zip(ann_frames, pred_frames), total=len(ann_frames)
)
]
logger.info("accumulating...")
num_classes = len(categories) + 1
hist: NDArrayI32 = np.zeros((num_classes, num_classes), dtype=np.int32)
gt_id_set = set()
for (hist_, gt_id_set_) in hist_and_gt_id_sets:
hist += hist_
gt_id_set.update(gt_id_set_)
ious = per_class_iou(hist)
accs = per_class_acc(hist)
IoUs = [ # pylint: disable=invalid-name
{cat_name: 100 * score for cat_name, score in zip(categories, ious)},
{AVERAGE: np.multiply(np.mean(ious[list(gt_id_set)]), 100)},
]
Accs = [ # pylint: disable=invalid-name
{cat_name: 100 * score for cat_name, score in zip(categories, accs)},
{AVERAGE: np.multiply(np.mean(accs[list(gt_id_set)]), 100)},
]
res_dict: Dict[str, Union[float, ScoresList]] = dict(
IoU=IoUs,
Acc=Accs,
fIoU=np.multiply(freq_iou(hist), 100), # pylint: disable=invalid-name
pAcc=np.multiply(whole_acc(hist), 100), # pylint: disable=invalid-name
)
logger.info("GT id set [%s]", ",".join(str(s) for s in gt_id_set))
return SegResult(**res_dict)
def evaluate_seg_track(
acc_single_video: VidFunc,
gts: List[List[Frame]],
results: List[List[Frame]],
config: Config,
iou_thr: float = 0.5,
ignore_iof_thr: float = 0.5,
ignore_unknown_cats: bool = False,
nproc: int = NPROC,
) -> TrackResult:
"""Evaluate CLEAR MOT metrics for a Scalabel format dataset.
Args:
acc_single_video: Function for calculating metrics over a single video.
gts: the ground truth annotations in Scalabel format
results: the prediction results in Scalabel format.
config: Config object
iou_thr: Minimum IoU for a mask to be considered a positive.
ignore_iof_thr: Min. Intersection over foreground with ignore regions.
ignore_unknown_cats: if False, raise KeyError when trying to evaluate
unknown categories.
nproc: processes number for loading files
Returns:
TrackResult: evaluation results.
"""
logger.info("Tracking evaluation with CLEAR MOT metrics.")
t = time.time()
assert len(gts) == len(results)
# check overlap of masks
logger.info("checking for overlap of masks...")
if check_overlap([frame for res in results for frame in res], config,
nproc):
logger.critical(
"Found overlap in prediction bitmasks, but segmentation tracking "
"evaluation does not allow overlaps. Removing such predictions.")
classes = get_leaf_categories(config.categories)
super_classes = get_parent_categories(config.categories)
logger.info("evaluating...")
image_size = config.imageSize
if nproc > 1:
with Pool(nproc) as pool:
video_accs = pool.starmap(
partial(
acc_single_video,
classes=classes,
ignore_iof_thr=ignore_iof_thr,
ignore_unknown_cats=ignore_unknown_cats,
image_size=image_size,
),
zip(gts, results),
)
else:
video_accs = [
acc_single_video(
gt,
result,
classes,
iou_thr,
ignore_iof_thr,
ignore_unknown_cats,
image_size,
) for gt, result in zip(gts, results)
]
class_names, metric_names, class_accs = aggregate_accs(
video_accs, classes, super_classes)
logger.info("accumulating...")
if nproc > 1:
with Pool(nproc) as pool:
flat_dicts = pool.starmap(evaluate_single_class,
zip(metric_names, class_accs))
else:
flat_dicts = [
evaluate_single_class(names, accs)
for names, accs in zip(metric_names, class_accs)
]
metrics = list(METRIC_MAPS.values())
result = generate_results(flat_dicts, class_names, metrics, classes,
super_classes)
t = time.time() - t
logger.info("evaluation finishes with %.1f s.", t)
return result