def _download_and_prepare(
dataset_dir,
scratch_dir,
download_fcn,
get_class_labels_fcn,
get_num_samples_fcn,
sample_parser,
):
#
# Download the TFDS dataset, if necessary
#
# Prevents ResourceExhaustedError that can arise...
# https://github.com/tensorflow/datasets/issues/1441#issuecomment-581660890
#
with fou.ResourceLimit("RLIMIT_NOFILE", soft=4096, warn_on_failure=True):
dataset, info = download_fcn(scratch_dir)
classes = get_class_labels_fcn(info)
num_samples = get_num_samples_fcn(info)
sample_parser.classes = classes
if isinstance(sample_parser, foud.ImageClassificationSampleParser):
dataset_type = fot.FiftyOneImageClassificationDataset()
dataset_exporter = foud.FiftyOneImageClassificationDatasetExporter(
dataset_dir, classes=classes)
elif isinstance(sample_parser, foud.ImageDetectionSampleParser):
dataset_type = fot.FiftyOneImageDetectionDataset()
dataset_exporter = foud.FiftyOneImageDetectionDatasetExporter(
dataset_dir, classes=classes)
elif isinstance(sample_parser, foud.ImageLabelsSampleParser):
dataset_type = fot.FiftyOneImageLabelsDataset()
dataset_exporter = foud.FiftyOneImageLabelsDatasetExporter(dataset_dir)
else:
raise ValueError("Unsupported SampleParser %s" % type(sample_parser))
try:
samples = dataset.as_numpy_iterator()
except AttributeError:
# Must be tensorflow < 2.1
samples = tfds.as_numpy(dataset)
# Write the formatted dataset to `dataset_dir`
foud.write_dataset(
samples,
sample_parser,
dataset_exporter=dataset_exporter,
num_samples=num_samples,
)
return dataset_type, num_samples, classes
def _get_default_dataset_type(sample_collection, label_field):
if label_field is None:
return fot.ImageDirectory()
sample = next(iter(sample_collection))
label = sample[label_field]
if isinstance(label, fol.Classification):
return fot.FiftyOneImageClassificationDataset()
if isinstance(label, fol.Detections):
return fot.FiftyOneImageDetectionDataset()
if isinstance(label, fol.ImageLabels):
return fot.FiftyOneImageLabelsDataset()
raise ValueError("Unsupported label type %s" % type(label))
def _download_and_prepare(
dataset_dir,
scratch_dir,
download_fcn,
get_class_labels_fcn,
sample_parser,
):
# Download the torchvision dataset, if necessary
dataset = download_fcn(scratch_dir)
classes = get_class_labels_fcn(dataset)
num_samples = len(dataset)
sample_parser.classes = classes
if isinstance(sample_parser, foud.ImageClassificationSampleParser):
dataset_type = fot.FiftyOneImageClassificationDataset()
dataset_exporter = foud.FiftyOneImageClassificationDatasetExporter(
dataset_dir, classes=classes)
elif isinstance(sample_parser, foud.ImageDetectionSampleParser):
dataset_type = fot.FiftyOneImageDetectionDataset()
dataset_exporter = foud.FiftyOneImageDetectionDatasetExporter(
dataset_dir, classes=classes)
elif isinstance(sample_parser, foud.ImageLabelsSampleParser):
dataset_type = fot.FiftyOneImageLabelsDataset()
dataset_exporter = foud.FiftyOneImageLabelsDatasetExporter(dataset_dir)
else:
raise ValueError("Unsupported SampleParser %s" % type(sample_parser))
# Write the formatted dataset to `dataset_dir`
foud.write_dataset(
dataset,
sample_parser,
dataset_exporter=dataset_exporter,
num_samples=num_samples,
)
return dataset_type, num_samples, classes