def _write_generic_sample_dataset(dataset_exporter, samples, num_samples):
with fou.ProgressBar(total=num_samples) as pb:
with dataset_exporter:
if isinstance(samples, foc.SampleCollection):
dataset_exporter.log_collection(samples)
for sample in pb(samples):
dataset_exporter.export_sample(sample)
def compute_metadata(self, overwrite=False):
"""Populates the ``metadata`` field of all samples in the collection.
Any samples with existing metadata are skipped, unless
``overwrite == True``.
Args:
overwrite (False): whether to overwrite existing metadata
"""
with fou.ProgressBar() as pb:
for sample in pb(self):
if sample.metadata is None or overwrite:
sample.compute_metadata()
def save_tp_fp_fn_counts(samples, pred_field, gt_field, iou):
"""Saves the true positive (TP), false positive (FP), and false negative
(FN) counts at the given IoU level in top-level fields of each sample.
The counts are stored in the following fields::
TP: sample.tp_iou_<iou>
FP: sample.fp_iou_<iou>
FN: sample.fn_iou_<iou>
where ``<iou> = str(iou).replace(".", "_")``.
The samples must have been previously evaluated by passing them to
:meth:`evaluate_detections`.
Args:
samples: an iterable of :class:`fiftyone.core.sample.Sample` instances.
For example, this may be a :class:`fiftyone.core.dataset.Dataset`
or a :class:`fiftyone.core.view.DatasetView`
pred_field: the name of the field containing the predicted
:class:`fiftyone.core.labels.Detections` that were evaluated
gt_field: the name of the field containing the ground truth
:class:`fiftyone.core.labels.Detections`
iou: the IoU value for which to save the TP/FP/FN counts
"""
pred_key = "%s_eval" % gt_field
save_iou_str = str(iou).replace(".", "_")
try:
iou_ind = IOU_THRESHOLDS.index(iou)
iou_str = _IOU_THRESHOLD_STRS[iou_ind]
except ValueError:
logger.info(
"IoU %f is not an available IoU threshold: %s", iou, IOU_THRESHOLDS
)
return
logger.info("Saving TP/FP/FN counts for IoU %f...", iou)
with fou.ProgressBar() as pb:
for sample in pb(samples):
result_dict = sample[pred_field][pred_key]
true_positives = result_dict["true_positives"][iou_str]
false_positives = result_dict["false_positives"][iou_str]
false_negatives = result_dict["false_negatives"][iou_str]
sample["tp_iou_%s" % save_iou_str] = true_positives
sample["fp_iou_%s" % save_iou_str] = false_positives
sample["fn_iou_%s" % save_iou_str] = false_negatives
sample.save()
def condense_image_labels_field(
dataset,
label_field,
prefix=None,
labels_dict=None,
keep_label_fields=False,
):
"""Condenses multiple :class:`fiftyone.core.labels.Label`` fields into a
single :class:`fiftyone.core.labels.ImageLabels` field.
Provide either ``prefix`` or ``labels_dict`` to customize the fields that
are condensed. If you provide neither, all
:class:`fiftyone.core.labels.Label`` fields are condensed.
Args:
dataset: a :class:`fiftyone.core.dataset.Dataset`
label_field: the name of the :class:`fiftyone.core.labels.ImageLabels`
field to create
prefix (None): a label field prefix; all
:class:`fiftyone.core.labels.Label` fields matching this prefix are
merged into ``label_field``, with the prefix removed from the names
of the labels
labels_dict (None): a dictionary mapping names of
:class:`fiftyone.core.labels.Label` fields to names to give them in
the condensed :class:`fiftyone.core.labels.ImageLabels`
keep_label_fields (False): whether to keep the input label fields after
``label_field`` is created. By default, the fields are deleted
"""
if prefix is None:
prefix = ""
if labels_dict is None:
labels_dict = _get_label_dict_for_prefix(dataset, prefix)
logger.info("Condensing image labels into field '%s'", label_field)
with fou.ProgressBar() as pb:
for sample in pb(dataset):
image_labels = etai.ImageLabels()
for field_name, name in labels_dict.items():
image_labels.merge_labels(
sample[field_name].to_image_labels(name=name))
if not keep_label_fields:
sample.clear_field(field_name)
sample[label_field] = fol.ImageLabels(labels=image_labels)
sample.save()
if not keep_label_fields:
for field_name in labels_dict:
dataset.delete_sample_field(field_name)
def draw_labeled_images(samples,
label_fields,
anno_dir,
annotation_config=None):
"""Renders annotated versions of the image samples with label field(s)
overlaid to the given directory.
The filenames of the sample images are maintained, unless a name conflict
would occur in ``anno_dir``, in which case an index of the form
``"-%d" % count`` is appended to the base filename.
The images are written in format ``fo.config.default_image_ext``.
Args:
samples: an iterable of :class:`fiftyone.core.sample.Sample` instances
label_fields: the list of :class:`fiftyone.core.labels.ImageLabel`
fields to render
anno_dir: the directory to write the annotated images
annotation_config (None): an :class:`AnnotationConfig` specifying how
to render the annotations
Returns:
the list of paths to the labeled images
"""
if annotation_config is None:
annotation_config = _DEFAULT_ANNOTATION_CONFIG
filename_maker = fou.UniqueFilenameMaker(output_dir=anno_dir)
output_ext = fo.config.default_image_ext
outpaths = []
with fou.ProgressBar() as pb:
for sample in pb(samples):
outpath = filename_maker.get_output_path(sample.filepath,
output_ext=output_ext)
draw_labeled_image(
sample,
label_fields,
outpath,
annotation_config=annotation_config,
)
outpaths.append(outpath)
return outpaths
def to_dict(self, rel_dir=None):
"""Returns a JSON dictionary representation of the collection.
Args:
rel_dir (None): a relative directory to remove from the
``filepath`` of each sample, if possible. The path is converted
to an absolute path (if necessary) via
``os.path.abspath(os.path.expanduser(rel_dir))``. The typical
use case for this argument is that your source data lives in
a single directory and you wish to serialize relative, rather
than absolute, paths to the data within that directory
Returns:
a JSON dict
"""
# @todo support serializing video datasets?
# That would be a lot of labels to store in one JSON......
if self.media_type == fom.VIDEO:
raise ValueError("Serializing video datasets is not supported")
if rel_dir is not None:
rel_dir = (os.path.abspath(os.path.expanduser(rel_dir)) +
os.path.sep)
len_rel_dir = len(rel_dir)
# Serialize samples
samples = []
with fou.ProgressBar() as pb:
for sample in pb(self):
d = sample.to_dict()
if rel_dir and d["filepath"].startswith(rel_dir):
d["filepath"] = d["filepath"][len_rel_dir:]
samples.append(d)
return {
"name": self.name,
"num_samples": len(self),
"tags": self.get_tags(),
"info": self.info,
"sample_fields": self._serialize_field_schema(),
"samples": samples,
}
def _make_images_list(images_dir):
logger.info("Computing image metadata for '%s'", images_dir)
image_paths = foud.parse_images_dir(images_dir)
images = []
with fou.ProgressBar() as pb:
for idx, image_path in pb(enumerate(image_paths)):
metadata = fom.ImageMetadata.build_for(image_path)
images.append({
"id": idx,
"file_name": os.path.basename(image_path),
"height": metadata.height,
"width": metadata.width,
"license": None,
"coco_url": None,
})
return images
def add_samples(self, samples, expand_schema=True, num_samples=None):
"""Adds the given samples to the dataset.
Any sample instances that do not belong to a dataset are updated
in-place to reflect membership in this dataset. Any sample instances
that belong to other datasets are not modified.
Args:
samples: an iterable of :class:`fiftyone.core.sample.Sample`
instances. For example, ``samples`` may be a :class:`Dataset`
or a :class:`fiftyone.core.views.DatasetView`
expand_schema (True): whether to dynamically add new sample fields
encountered to the dataset schema. If False, an error is raised
if a sample's schema is not a subset of the dataset schema
num_samples (None): the number of samples in ``samples``. If not
provided, this is computed via ``len(samples)``, if possible.
This value is optional and is used only for optimization and
progress tracking
Returns:
a list of IDs of the samples in the dataset
Raises:
:class:`mongoengine.errors.ValidationError` if a field of a sample
has a type that is inconsistent with the dataset schema, or if
``expand_schema == False`` and a new field is encountered
"""
if num_samples is None:
try:
num_samples = len(samples)
except:
pass
sample_ids = []
with fou.ProgressBar(total=num_samples) as pb:
for batch in fou.iter_batches(samples, self._BATCH_SIZE):
sample_ids.extend(
self._add_samples_batch(batch, expand_schema)
)
pb.update(count=len(batch))
return sample_ids
def to_dict(self, rel_dir=None):
"""Returns a JSON dictionary representation of the collection.
Args:
rel_dir (None): a relative directory to remove from the
``filepath`` of each sample, if possible. The path is converted
to an absolute path (if necessary) via
``os.path.abspath(os.path.expanduser(rel_dir))``. The typical
use case for this argument is that your source data lives in
a single directory and you wish to serialize relative, rather
than absolute, paths to the data within that directory
Returns:
a JSON dict
"""
if rel_dir is not None:
rel_dir = (os.path.abspath(os.path.expanduser(rel_dir)) +
os.path.sep)
len_rel_dir = len(rel_dir)
# Get field schema
fields = self.get_field_schema()
# Serialize samples
samples = []
with fou.ProgressBar() as pb:
for sample in pb(self):
d = sample.to_dict()
if rel_dir and d["filepath"].startswith(rel_dir):
d["filepath"] = d["filepath"][len_rel_dir:]
samples.append(d)
return {
"name": self.name,
"num_samples": len(self),
"tags": self.get_tags(),
"sample_fields":
{field_name: str(field)
for field_name, field in fields.items()},
"samples": samples,
}
def _write_video_dataset(
dataset_exporter, samples, sample_parser, num_samples
):
labeled_videos = isinstance(dataset_exporter, LabeledVideoDatasetExporter)
with fou.ProgressBar(total=num_samples) as pb:
with dataset_exporter:
if isinstance(samples, foc.SampleCollection):
dataset_exporter.log_collection(samples)
for sample in pb(samples):
sample_parser.with_sample(sample)
# Parse video
video_path = sample_parser.get_video_path()
# Parse metadata
if dataset_exporter.requires_video_metadata:
if sample_parser.has_video_metadata:
metadata = sample_parser.get_video_metadata()
else:
metadata = None
if metadata is None:
metadata = fom.VideoMetadata.build_for(video_path)
else:
metadata = None
if labeled_videos:
# Parse labels
frames = sample_parser.get_frame_labels()
# Export sample
dataset_exporter.export_sample(
video_path, frames, metadata=metadata
)
else:
# Export sample
dataset_exporter.export_sample(
video_path, metadata=metadata
)
def clone_field(self, field_name, new_field_name, samples=None):
"""Clones the field values of the samples into a new field of this
dataset.
Any samples in ``samples`` that are not in this dataset (i.e., their
sample ID does not match any samples in this dataset) are skipped.
The fields of the input samples are **deep copied**.
Args:
field_name: the field name to clone
new_field_name: the new field name to populate
samples (None): an iterable of :class:`fiftyone.core.sample.Sample`
instances whose fields to clone. For example, ``samples`` may
be a :class:`fiftyone.core.views.DatasetView`. By default, this
dataset itself is used
Returns:
tuple of
- num_cloned: the number of samples that were cloned
- num_skipped: the number of samples that were skipped
"""
if samples is None:
samples = self
num_cloned = 0
num_skipped = 0
with fou.ProgressBar() as pb:
for sample in pb(samples):
try:
_sample = self[sample.id]
except KeyError:
num_skipped += 1
continue
_sample[new_field_name] = deepcopy(sample[field_name])
_sample.save()
num_cloned += 1
return num_cloned, num_skipped
def add_open_images_predictions(
dataset,
predictions_path,
class_descriptions_path=None,
prediction_field_name="predicted_detections",
):
"""Adds TF Object Detection API format predictions to a
:class:`fiftyone.core.dataset.Dataset`.
Args:
dataset: the :class:`fiftyone.core.dataset.Dataset` instance to add
the predictions to
predictions_path: path to a TF Object Detection API format
predictions CSV
class_descriptions_path: optional metadata file. if provided, the
MID labels are mapped to descriptive labels
prediction_field_name: the name of the field to save the predictions
under
"""
all_predictions = pd.read_csv(predictions_path)
all_predictions.rename(columns={"Score": "Confidence"}, inplace=True)
# map label MID to descriptive label
if class_descriptions_path is not None:
class_descriptions = pd.read_csv(
class_descriptions_path, header=None, index_col=0
)
temp = class_descriptions.loc[all_predictions["LabelName"], 1]
temp.index = all_predictions.index
all_predictions["LabelName"] = temp
with fou.ProgressBar(dataset) as pb:
for sample in pb(dataset):
# parse prediction bounding boxes
cur_preds = all_predictions.loc[
all_predictions["ImageID"] == sample[OPEN_IMAGES_ID]
]
if not cur_preds.empty:
sample[prediction_field_name] = df2detections(cur_preds)
sample.save()
def convert_classification_field_to_detections(
dataset,
classification_field,
detections_field=None,
keep_classification_field=False,
):
"""Converts the :class:`fiftyone.core.labels.Classification` field of the
dataset into a :class:`fiftyone.core.labels.Detections` field containing
the classification label.
The detections are given bounding boxes that span the entire image.
Args:
dataset: a :class:`fiftyone.core.dataset.Dataset`
classification_field: the name of the
:class:`fiftyone.core.labels.Classification` field to convert to
detections
detections_field (None): the name of the
:class:`fiftyone.core.labels.Detections` field to create. By
default, ``classification_field`` is overwritten
keep_classification_field (False): whether to keep
``classification_field`` after the conversion is completed. By
default, the field is deleted from the dataset. If
``classification_field`` is being overwritten, this flag has no
effect
"""
dataset.validate_field_type(
classification_field,
fof.EmbeddedDocumentField,
embedded_doc_type=fol.Classification,
)
if detections_field is None:
detections_field = classification_field
overwrite = detections_field == classification_field
if overwrite:
logger.info(
"Converting Classification field '%s' to Detections format",
classification_field,
)
keep_classification_field = False
detections_field = dataset.make_unique_field_name(
root=classification_field)
else:
logger.info(
"Converting Classification field '%s' to Detections format in "
"field '%s'",
classification_field,
detections_field,
)
with fou.ProgressBar() as pb:
for sample in pb(dataset):
label = sample[classification_field]
if label is None:
continue
detection = fol.Detection(
label=label.label,
bounding_box=[0, 0, 1, 1], # entire image
confidence=label.confidence,
)
sample[detections_field] = fol.Detections(detections=[detection])
if not keep_classification_field:
sample.clear_field(classification_field)
sample.save()
if not keep_classification_field:
dataset.delete_sample_field(classification_field)
if overwrite:
dataset.rename_field(detections_field, classification_field)
def _write_image_dataset(dataset_exporter, samples, sample_parser,
num_samples):
labeled_images = isinstance(dataset_exporter, LabeledImageDatasetExporter)
with fou.ProgressBar(total=num_samples) as pb:
with dataset_exporter:
if isinstance(samples, foc.SampleCollection):
dataset_exporter.log_collection(samples)
for sample in pb(samples):
sample_parser.with_sample(sample)
# Parse image
if sample_parser.has_image_path:
try:
image_or_path = sample_parser.get_image_path()
except:
image_or_path = sample_parser.get_image()
else:
image_or_path = sample_parser.get_image()
# Parse metadata
if dataset_exporter.requires_image_metadata:
if sample_parser.has_image_metadata:
metadata = sample_parser.get_image_metadata()
else:
metadata = None
if metadata is None:
metadata = fom.ImageMetadata.build_for(image_or_path)
else:
metadata = None
if labeled_images:
# Parse label
label = sample_parser.get_label()
#
# SPECIAL CASE
#
# Convert `Classification` labels to `Detections` format,
# if necessary
#
if (dataset_exporter.label_cls is fol.Detections
and isinstance(label, fol.Classification)):
msg = ("Dataset exporter expects labels in %s format, "
"but found %s. Converting labels to detections "
"whose bounding boxes span the entire image" %
(fol.Detections, label.__class__))
warnings.warn(msg)
label = fol.Detections(detections=[
fol.Detection(
label=label.label,
bounding_box=[0, 0, 1, 1], # entire image
confidence=label.confidence,
)
])
# Export sample
dataset_exporter.export_sample(image_or_path,
label,
metadata=metadata)
else:
# Export sample
dataset_exporter.export_sample(image_or_path,
metadata=metadata)
def evaluate_detections(
samples, pred_field, gt_field="ground_truth", save_iou=0.75
):
"""Evaluates the predicted detections in the given samples with respect to
the specified ground truth detections for each of the following
Intersection over Union (IoU) thresholds::
[0.50, 0.55, 0.60, 0.65, 0.70, 0.75, 0.80, 0.85, 0.90, 0.95]
It should be noted that if a :class:`fiftyone.core.labels.Detection` in the
ground truth field has a boolean attribute called `iscrowd`, then this
detection will be matched to multiple predictions and result in them all
being true positives, as per the evaluation strategy used by the COCO
authors.
Dictionaries are added to each predicted/ground truth
:class:`fiftyone.core.labels.Detections` instance in the fields listed
below; these fields tabulate the true positive (TP), false positive (FP),
and false negative (FN) counts for the sample at each IoU::
Ground truth: detections.<pred_field>_eval
Predictions: detections.<gt_field>_eval
Dictionaries are also added to each individual
:class:`fiftyone.core.labels.Detection` instance in the fields listed
below; these fields tabulate the IDs of the matching ground
truth/prediction for the detection at each IoU::
Ground truth: detection.<pred_field>_eval
Predictions: detection.<gt_field>_eval
In addition, true positive (TP), false positive (FP), and false negative
(FN) counts at the specified ``save_iou`` are saved in the following
top-level fields of each sample::
TP: sample.tp_iou_<save_iou>
FP: sample.fp_iou_<save_iou>
FN: sample.fn_iou_<save_iou>
where ``<save_iou> = str(save_iou).replace(".", "_")``.
Args:
samples: an iterable of :class:`fiftyone.core.sample.Sample` instances.
For example, this may be a :class:`fiftyone.core.dataset.Dataset`
or a :class:`fiftyone.core.view.DatasetView`
pred_field: the name of the field containing the predicted
:class:`fiftyone.core.labels.Detections` to evaluate
gt_field ("ground_truth"): the name of the field containing the ground
truth :class:`fiftyone.core.labels.Detections`
save_iou (0.75): an IoU value for which to save per-sample TP/FP/FN
counts as top-level sample fields
"""
gt_key = "%s_eval" % pred_field
pred_key = "%s_eval" % gt_field
eval_id = 0
try:
save_iou_ind = IOU_THRESHOLDS.index(save_iou)
save_iou_str = _IOU_THRESHOLD_STRS[save_iou_ind]
except ValueError:
logger.info(
"IoU %f is not in the list of available IoU thresholds: %s",
save_iou,
IOU_THRESHOLDS,
)
save_iou_str = None
logger.info("Evaluating detections...")
with fou.ProgressBar() as pb:
for sample in pb(samples):
preds = sample[pred_field]
gts = sample[gt_field]
# Sort preds and gt detections by category label
sample_cats = {}
for det in preds.detections:
det[pred_key] = {}
det[pred_key]["ious"] = {}
det[pred_key]["matches"] = {
iou_str: {"gt_id": -1, "iou": -1}
for iou_str in _IOU_THRESHOLD_STRS
}
det[pred_key]["pred_id"] = eval_id
eval_id += 1
if det.label not in sample_cats:
sample_cats[det.label] = {}
sample_cats[det.label]["preds"] = []
sample_cats[det.label]["gts"] = []
sample_cats[det.label]["preds"].append(det)
for det in gts.detections:
det[gt_key] = {}
det[gt_key]["matches"] = {
iou_str: {"pred_id": -1, "iou": -1}
for iou_str in _IOU_THRESHOLD_STRS
}
det[gt_key]["gt_id"] = eval_id
eval_id += 1
if det.label not in sample_cats:
sample_cats[det.label] = {}
sample_cats[det.label]["preds"] = []
sample_cats[det.label]["gts"] = []
sample_cats[det.label]["gts"].append(det)
# Compute IoU for every detection and gt
for cat, dets in sample_cats.items():
gts = dets["gts"]
preds = dets["preds"]
inds = np.argsort(
[-(p.confidence or 0.0) for p in preds], kind="mergesort"
)
preds = [preds[i] for i in inds]
sample_cats[cat]["preds"] = preds
gt_ids = [g[gt_key]["gt_id"] for g in gts]
gt_boxes = [list(g.bounding_box) for g in gts]
pred_boxes = [list(p.bounding_box) for p in preds]
iscrowd = [False] * len(gt_boxes)
for gind, g in enumerate(gts):
if "iscrowd" in g.attributes:
iscrowd[gind] = bool(g.attributes["iscrowd"].value)
# Get the IoU of every prediction with every ground truth
# shape = [num_preds, num_gts]
ious = _compute_iou(pred_boxes, gt_boxes, iscrowd)
for pind, gt_ious in enumerate(ious):
preds[pind][pred_key]["ious"][cat] = list(
zip(gt_ids, gt_ious)
)
#
# Starting with highest confidence prediction, match all with gts
# Store true and false positives
#
# Reference implementation:
# https://github.com/cocodataset/cocoapi/blob/8c9bcc3cf640524c4c20a9c40e89cb6a2f2fa0e9/PythonAPI/pycocotools/cocoeval.py#L273
#
result_dict = {
"true_positives": {},
"false_positives": {},
"false_negatives": {},
}
for iou_ind, iou_thresh in enumerate(IOU_THRESHOLDS):
iou_str = _IOU_THRESHOLD_STRS[iou_ind]
true_positives = 0
false_positives = 0
for cat, dets in sample_cats.items():
gt_by_id = {g[gt_key]["gt_id"]: g for g in dets["gts"]}
# Note: predictions were sorted by confidence in the
# previous step
preds = dets["preds"]
# Match each prediction to the highest IoU ground truth
# available
for pred in preds:
if cat in pred[pred_key]["ious"]:
best_match = -1
best_match_iou = min([iou_thresh, 1 - 1e-10])
for gt_id, iou in pred[pred_key]["ious"][cat]:
gt = gt_by_id[gt_id]
curr_gt_match = gt[gt_key]["matches"][iou_str][
"pred_id"
]
if "iscrowd" in gt.attributes:
iscrowd = bool(
gt.attributes["iscrowd"].value
)
else:
iscrowd = False
# Cannot match two preds to the same gt unless
# the gt is a crowd
if curr_gt_match > -1 and not iscrowd:
continue
# Ignore gts with an IoU lower than what was
# already found
if iou < best_match_iou:
continue
best_match_iou = iou
best_match = gt_id
if best_match > -1:
# If the prediction was matched, store the eval
# id of the pred in the gt and of the gt in the
# pred
gt_to_store = gt_by_id[best_match][gt_key]
gt_to_store["matches"][iou_str] = {
"pred_id": pred[pred_key]["pred_id"],
"iou": best_match_iou,
}
pred[pred_key]["matches"][iou_str] = {
"gt_id": best_match,
"iou": best_match_iou,
}
true_positives += 1
else:
false_positives += 1
elif pred.label == cat:
false_positives += 1
result_dict["true_positives"][iou_str] = true_positives
result_dict["false_positives"][iou_str] = false_positives
false_negatives = len(
[
g
for g in dets["gts"]
if g[gt_key]["matches"][iou_str]["pred_id"] == -1
]
)
result_dict["false_negatives"][iou_str] = false_negatives
if iou_str == save_iou_str:
sample["tp_iou_%s" % save_iou_str] = true_positives
sample["fp_iou_%s" % save_iou_str] = false_positives
sample["fn_iou_%s" % save_iou_str] = false_negatives
sample[pred_field][pred_key] = result_dict
# @todo compute sample-wise AP
sample.save()
def write_dataset(samples,
sample_parser,
dataset_dir=None,
dataset_type=None,
dataset_exporter=None,
num_samples=None,
**kwargs):
"""Writes the samples to disk as a dataset in the specified format.
Provide either ``dataset_dir`` and ``dataset_type`` or ``dataset_exporter``
to perform the write.
Args:
samples: an iterable of samples
sample_parser: a :class:`fiftyone.utils.data.parsers.SampleParser` to
use to parse the samples
dataset_dir (None): the directory to which to write the dataset in
format ``dataset_type``
dataset_type (None): the :class:`fiftyone.types.dataset_types.Dataset`
type to write
dataset_exporter (None): a
:class:`fiftyone.utils.data.exporters.DatasetExporter` to use to
write the dataset
num_samples (None): the number of samples in ``samples``. If omitted,
this is computed (if possible) via ``len(samples)``
**kwargs: optional keyword arguments to pass to
``dataset_type.get_dataset_exporter_cls(dataset_dir, **kwargs)``
"""
if dataset_type is not None:
if inspect.isclass(dataset_type):
dataset_type = dataset_type()
if not isinstance(
dataset_type,
(fot.UnlabeledImageDataset, fot.LabeledImageDataset),
):
raise ValueError("Unsupported `dataset_type` %s" %
type(dataset_type))
if dataset_exporter is None:
dataset_exporter_cls = dataset_type.get_dataset_exporter_cls()
dataset_exporter = dataset_exporter_cls(dataset_dir, **kwargs)
if num_samples is None:
try:
num_samples = len(samples)
except:
pass
raw_samples = False
labeled_images = False
if isinstance(dataset_exporter, GenericSampleDatasetExporter):
raw_samples = True
elif isinstance(dataset_exporter, UnlabeledImageDatasetExporter):
labeled_images = False
elif isinstance(dataset_exporter, LabeledImageDatasetExporter):
labeled_images = True
else:
raise ValueError("Unsupported DatasetExporter %s" %
type(dataset_exporter))
with fou.ProgressBar(total=num_samples) as pb:
with dataset_exporter:
for sample in pb(samples):
# GenericSampleDatasetExporter
if raw_samples:
dataset_exporter.export_sample(sample)
continue
# UnlabeledImageDatasetExporter and LabeledImageDatasetExporter
sample_parser.with_sample(sample)
if sample_parser.has_image_path:
try:
image_or_path = sample_parser.get_image_path()
except:
image_or_path = sample_parser.get_image()
else:
image_or_path = sample_parser.get_image()
if dataset_exporter.requires_image_metadata:
if sample_parser.has_image_metadata:
metadata = sample_parser.get_image_metadata()
else:
metadata = None
if metadata is None:
metadata = fom.ImageMetadata.build_for(image_or_path)
else:
metadata = None
if labeled_images:
label = sample_parser.get_label()
dataset_exporter.export_sample(image_or_path,
label,
metadata=metadata)
else:
dataset_exporter.export_sample(image_or_path,
metadata=metadata)
def evaluate_detections(
samples,
pred_field,
gt_field="ground_truth",
iou=0.75,
):
"""Evaluates the predicted detections in the given samples with respect to
the specified ground truth detections using the specified Intersection over
Union (IoU) threshold to determine matches.
This method uses COCO-style evaluation. In particular, this means that if a
:class:`fiftyone.core.labels.Detection` in the ground truth field has a
boolean attribute called ``iscrowd``, then this detection can have multiple
true positive predictions matched to it.
Dictionaries are added to each predicted/ground truth
:class:`fiftyone.core.labels.Detections` instance in the fields listed
below; these fields tabulate the true positive (TP), false positive (FP),
and false negative (FN) counts for the sample at the specified IoU::
Ground truth: detections.<pred_field>_eval
Predictions: detections.<gt_field>_eval
Dictionaries are also added to each individual
:class:`fiftyone.core.labels.Detection` instance in the fields listed
below; these fields tabulate the IDs of the matching ground
truth/prediction for the detection at the specified IoU::
Ground truth: detection.<pred_field>_eval
Predictions: detection.<gt_field>_eval
In addition, true positive (TP), false positive (FP), and false negative
(FN) counts at the specified IoU are saved in the following top-level
fields of each sample::
TP: sample.tp_iou_<iou>
FP: sample.fp_iou_<iou>
FN: sample.fn_iou_<iou>
where ``<iou> = str(iou).replace(".", "_")``.
Args:
samples: an iterable of :class:`fiftyone.core.sample.Sample` instances.
For example, this may be a :class:`fiftyone.core.dataset.Dataset`
or a :class:`fiftyone.core.view.DatasetView`
pred_field: the name of the field containing the predicted
:class:`fiftyone.core.labels.Detections` to evaluate
gt_field ("ground_truth"): the name of the field containing the ground
truth :class:`fiftyone.core.labels.Detections`
iou (0.75): an IoU value for which to compute
per-detection and per-image TP/FP/FN
"""
gt_key = "%s_eval" % pred_field
pred_key = "%s_eval" % gt_field
eval_id = 0
iou_str = str(iou).replace(".", "_")
logger.info("Evaluating detections...")
with fou.ProgressBar() as pb:
for sample in pb(samples):
# Get image(s) to process
if sample.media_type == fom.VIDEO:
images = sample.frames.values()
has_frames = True
else:
images = [sample]
has_frames = False
# Initialize sample result dict tp, fp, fn = 0 for each IoU
sample_result_dict = {
"true_positives": 0,
"false_positives": 0,
"false_negatives": 0,
}
for image in images:
preds = image[pred_field]
gts = image[gt_field]
# Sort preds and gt detections by category label
image_cats = {}
for det in preds.detections:
if pred_key not in det:
det[pred_key] = {}
if "matches" not in det[pred_key]:
det[pred_key]["matches"] = {
iou_str: {
"gt_id": -1,
"iou": -1
}
}
else:
matches = dict(det[pred_key]["matches"])
matches[iou_str] = {"gt_id": -1, "iou": -1}
det[pred_key]["matches"] = matches
if det.label not in image_cats:
image_cats[det.label] = {}
image_cats[det.label]["preds"] = []
image_cats[det.label]["gts"] = []
image_cats[det.label]["preds"].append(det)
for det in gts.detections:
if gt_key not in det:
det[gt_key] = {}
if "matches" not in det[gt_key]:
det[gt_key]["matches"] = {
iou_str: {
"pred_id": -1,
"iou": -1
}
}
else:
matches = dict(det[gt_key]["matches"])
matches[iou_str] = {"pred_id": -1, "iou": -1}
det[gt_key]["matches"] = matches
if det.label not in image_cats:
image_cats[det.label] = {}
image_cats[det.label]["preds"] = []
image_cats[det.label]["gts"] = []
image_cats[det.label]["gts"].append(det)
# Compute IoU for every detection and gt
pred_ious = {}
for cat, dets in image_cats.items():
gts = dets["gts"]
preds = dets["preds"]
inds = np.argsort(
[-(p.confidence or 0.0) for p in preds],
kind="mergesort",
)
preds = [preds[i] for i in inds]
image_cats[cat]["preds"] = preds
gt_ids = [g.id for g in gts]
gt_boxes = [list(g.bounding_box) for g in gts]
pred_boxes = [list(p.bounding_box) for p in preds]
iscrowd = [False] * len(gt_boxes)
for gind, g in enumerate(gts):
if "iscrowd" in g.attributes:
iscrowd[gind] = bool(g.attributes["iscrowd"].value)
# Get the IoU of every prediction with every ground truth
# shape = [num_preds, num_gts]
ious = _compute_iou(pred_boxes, gt_boxes, iscrowd)
for pind, gt_ious in enumerate(ious):
pred_ious[preds[pind].id] = list(zip(gt_ids, gt_ious))
#
# Starting with highest confidence prediction, match all with
# GTs. Store true and false positives
#
# Reference implementation:
# https://github.com/cocodataset/cocoapi/blob/8c9bcc3cf640524c4c20a9c40e89cb6a2f2fa0e9/PythonAPI/pycocotools/cocoeval.py#L273
#
true_pos = 0
false_pos = 0
for cat, dets in image_cats.items():
gt_by_id = {g.id: g for g in dets["gts"]}
# Note: predictions were sorted by confidence in the
# previous step
preds = dets["preds"]
# Match each prediction to the highest IoU ground truth
# available
for pred in preds:
if pred.id in pred_ious:
best_match = -1
best_match_iou = min([iou, 1 - 1e-10])
for gt_id, det_iou in pred_ious[pred.id]:
gt = gt_by_id[gt_id]
curr_gt_match = gt[gt_key]["matches"][iou_str][
"pred_id"]
if "iscrowd" in gt.attributes:
iscrowd = bool(
gt.attributes["iscrowd"].value)
else:
iscrowd = False
# Cannot match two preds to the same gt unless
# the gt is a crowd
if curr_gt_match != -1 and not iscrowd:
continue
# Ignore gts with an IoU lower than what was
# already found
if det_iou < best_match_iou:
continue
best_match_iou = det_iou
best_match = gt_id
if best_match != -1:
# If the prediction was matched, store the eval
# id of the pred in the gt and of the gt in the
# pred
gt_to_store = gt_by_id[best_match][gt_key]
gt_to_store["matches"][iou_str] = {
"pred_id": pred.id,
"iou": best_match_iou,
}
pred[pred_key]["matches"][iou_str] = {
"gt_id": best_match,
"iou": best_match_iou,
}
true_pos += 1
else:
false_pos += 1
elif pred.label == cat:
false_pos += 1
sample_result_dict["true_positives"] += true_pos
sample_result_dict["false_positives"] += false_pos
false_neg = len([
g for g in dets["gts"]
if g[gt_key]["matches"][iou_str]["pred_id"] == -1
])
sample_result_dict["false_negatives"] += false_neg
image["tp_iou_%s" % iou_str] = true_pos
image["fp_iou_%s" % iou_str] = false_pos
image["fn_iou_%s" % iou_str] = false_neg
if has_frames:
sample["tp_iou_%s" %
iou_str] = sample_result_dict["true_positives"]
sample["fp_iou_%s" %
iou_str] = sample_result_dict["false_positives"]
sample["fn_iou_%s" %
iou_str] = sample_result_dict["false_negatives"]
sample.save()
def evaluate_dataset(
dataset,
label_map_path,
groundtruth_loc_field_name="groundtruth_detections",
groundtruth_img_labels_field_name="groundtruth_image_labels",
prediction_field_name="predicted_detections",
iou_threshold=0.5,
):
"""Evaluates a FiftyOne dataset that contains all necessary fields for
evaluation via Tensorflow Object Detection API on a per-image granularity.
Args:
dataset: the :class:`fiftyone.core.dataset.Dataset` to evaluate
label_map_path: path to the label map .pbtxt file
groundtruth_loc_field_name: the name of the groundtruth
:class:`fiftyone.core.labels.Detections` field
groundtruth_img_labels_field_name: the name of the groundtruth
:class:`fiftyone.core.labels.Classifications` field
prediction_field_name: the name of the predicted
:class:`fiftyone.core.labels.Detections` field
iou_threshold: the intersection-over-union bounding box matching
threshold
"""
_, categories = TensorflowObjectDetectionAPIEvaluator.load_labelmap(
label_map_path)
display2name_map = {d["display_name"]: d["name"] for d in categories}
name2display_map = {d["name"]: d["display_name"] for d in categories}
evaluator = TensorflowObjectDetectionAPIEvaluator(
label_map_path, iou_threshold=iou_threshold)
with fou.ProgressBar(dataset) as pb:
for sample in pb(dataset):
# convert groundtruth to dataframe
loc_annos = detections2df(
sample.open_images_id,
sample[groundtruth_loc_field_name],
display2name_map=display2name_map,
is_groundtruth=True,
)
label_annos = classifications2df(
sample.open_images_id,
sample[groundtruth_img_labels_field_name],
display2name_map=display2name_map,
)
groundtruth = pd.concat([loc_annos, label_annos])
# convert predictions to dataframe
predictions = detections2df(
sample.open_images_id,
sample[prediction_field_name],
display2name_map=display2name_map,
)
# evaluate
result = evaluator.evaluate_image(sample.open_images_id,
groundtruth, predictions)
# store mAP
mAP = result["mAP"]
if not np.isnan(mAP):
sample["mAP"] = mAP
# store AP per class
sample["AP_per_class"] = {
name2display_map[k]: v
for k, v in result["AP_per_class"].items() if not np.isnan(v)
}
# store false positives
for idx in result["false_positive_indexes"]:
det = sample[prediction_field_name].detections[idx]
det["eval"] = "false_positive"
# store true positives
for idx in result["true_positive_indexes"]:
det = sample[prediction_field_name].detections[idx]
det["eval"] = "true_positive"
sample.save()
def load_open_images_dataset(
dataset_name,
images_dir,
bounding_boxes_path=None,
image_labels_path=None,
predictions_path=None,
prediction_field_name="predicted_detections",
class_descriptions_path=None,
load_images_with_preds=False,
max_num_images=-1,
):
"""Loads an Open Images format dataset into FiftyOne.
**Note** If this takes a long time it is highly recommended to save the
dataset via:
dataset.persistent = True
such that this function only needs to be called once!
Args:
dataset_name: the name of the dataset to create in FiftyOne.
images_dir: directory where images are stored. Images should be in
<open-images-id>.jpg format
bounding_boxes_path: path to the expanded-hierarchy annotation bounding
boxes CSV
image_labels_path: path to the expanded-hierarchy annotation image
labels CSV
predictions_path: path to the predicted bounding boxes CSV
prediction_field_name: the name of the field to save the predictions
under. Useful if other predictions may be added later
class_descriptions_path: optional metadata file. if provided, the
MID labels are mapped to descriptive labels
load_images_with_preds: if True, skip any images that do not have
predictions
Returns:
a :class:`fiftyone.core.dataset.Dataset` instance
"""
# pylint: disable=unsubscriptable-object
# read data from disk
all_location_annotations = (pd.read_csv(bounding_boxes_path)
if bounding_boxes_path else None)
all_label_annotations = (pd.read_csv(image_labels_path)
if image_labels_path else None)
if predictions_path:
all_predictions = pd.read_csv(predictions_path)
all_predictions.rename(columns={"Score": "Confidence"}, inplace=True)
else:
all_predictions = None
class_descriptions = (pd.read_csv(
class_descriptions_path, header=None, index_col=0)
if class_descriptions_path else None)
# map label MID to descriptive label
if class_descriptions is not None:
for df in [
all_location_annotations,
all_label_annotations,
all_predictions,
]:
if df is None:
continue
temp = class_descriptions.loc[df["LabelName"], 1]
temp.index = df.index
df["LabelName"] = temp
if load_images_with_preds:
img_paths = [
os.path.join(images_dir, image_id + ".jpg")
for image_id in set(all_predictions["ImageID"])
]
else:
img_paths = glob.glob(os.path.join(images_dir, "*.jpg"))
if max_num_images != -1:
img_paths = img_paths[:max_num_images]
print("Parsing CSV labels...")
_samples = []
with fou.ProgressBar(img_paths) as pb:
for image_path in pb(img_paths):
image_id = os.path.splitext(os.path.basename(image_path))[0]
kwargs = {"filepath": image_path, OPEN_IMAGES_ID: image_id}
# parse ground truth image labels
if all_label_annotations is not None:
cur_lab_anns = all_label_annotations.query("ImageID == '%s'" %
image_id)
if not cur_lab_anns.empty:
kwargs[GT_IMAGE_LABELS] = df2classifications(cur_lab_anns)
# parse ground truth bounding boxes
if all_location_annotations is not None:
cur_loc_anns = all_location_annotations.query(
"ImageID == '%s'" % image_id)
if not cur_loc_anns.empty:
kwargs[GT_DETECTIONS] = df2detections(cur_loc_anns)
# parse prediction bounding boxes
if all_predictions is not None:
cur_preds = all_predictions.query("ImageID == '%s'" % image_id)
if not cur_preds.empty:
kwargs[prediction_field_name] = df2detections(cur_preds)
_samples.append(fos.Sample(**kwargs))
print("Creating FiftyOne Dataset...")
dataset = fod.Dataset(dataset_name)
dataset.add_samples(_samples)
return dataset
def expand_image_labels_field(
dataset,
label_field,
prefix=None,
labels_dict=None,
multilabel=False,
skip_non_categorical=False,
keep_label_field=False,
):
"""Expands the :class:`fiftyone.core.labels.ImageLabels` field of the
dataset into per-label fields.
Provide ``labels_dict`` if you want to customize which components of the
labels are expanded. Otherwise, all objects/attributes are expanded as
explained below.
If ``multilabel`` is False, frame attributes will be stored in separate
:class:`fiftyone.core.labels.Classification` fields with names
``prefix + attr.name``.
If ``multilabel`` if True, all frame attributes will be stored in a
:class:`fiftyone.core.labels.Classifications` field called
``prefix + "attrs"``.
Objects are stored in :class:`fiftyone.core.labels.Detections` fields whose
names are ``prefix + obj.name``, or ``prefix + "objs"`` for objects that
do not have their ``name`` field populated.
Args:
dataset: a :class:`fiftyone.core.dataset.Dataset`
label_field: the name of the :class:`fiftyone.core.labels.ImageLabels`
field to expand
prefix (None): a string prefix to prepend to each expanded field name
labels_dict (None): a dictionary mapping names of attributes/objects
in ``label_field`` to field names into which to expand them
multilabel (False): whether to store frame attributes in a single
:class:`fiftyone.core.labels.Classifications` field
skip_non_categorical (False): whether to skip non-categorical frame
attributes (True) or cast them to strings (False)
keep_label_field (False): whether to keep ``label_field`` after the
expansion is completed. By default, the field is deleted from the
dataset
"""
logger.info("Expanding image labels field '%s'", label_field)
with fou.ProgressBar() as pb:
for sample in pb(dataset):
labels = sample[label_field]
if labels is None:
continue
sample.update_fields(
labels.expand(
prefix=prefix,
labels_dict=labels_dict,
multilabel=multilabel,
skip_non_categorical=skip_non_categorical,
))
if not keep_label_field:
sample.clear_field(label_field)
sample.save()
if not keep_label_field:
dataset.delete_sample_field(label_field)
