def class_label_to_hub(tf_dt, max_shape=None): if hasattr(tf_dt, "_num_classes"): return ClassLabel( num_classes=tf_dt.num_classes, ) else: return ClassLabel(names=tf_dt.names)
def test_class_label_value(): ds = Dataset( "./data/tests/test_check_label", mode="w", shape=(5, ), schema={ "label": ClassLabel(names=["name1", "name2", "name3"]), "label/b": ClassLabel(num_classes=5), }, ) ds["label", 0:7] = 2 ds["label", 0:2] = np.array([0, 1]) ds["label", 0:3] = ["name1", "name2", "name3"] ds[0:3]["label"] = [0, "name2", 2] try: ds["label/b", 0] = 6 except Exception as ex: assert isinstance(ex, ClassLabelValueError) try: ds[0:4]["label/b"] = np.array([0, 1, 2, 3, 7]) except Exception as ex: assert isinstance(ex, ClassLabelValueError) try: ds["label", 4] = "name4" except Exception as ex: assert isinstance(ex, ClassLabelValueError) try: ds[0]["label/b"] = ["name"] except Exception as ex: assert isinstance(ex, ValueError)
def test_check_label_name(): my_schema = {"label": ClassLabel(names=["red", "green", "blue"])} ds = Dataset("./data/test/dataset2", shape=(5,), mode="w", schema=my_schema) ds["label", 0] = 1 ds["label", 1] = 2 ds["label", 0] = 1 ds["label", 1] = 2 ds["label", 2] = 0 assert ds.compute(label_name=True).tolist() == [ {"label": "green"}, {"label": "blue"}, {"label": "red"}, {"label": "red"}, {"label": "red"}, ] assert ds.compute().tolist() == [ {"label": 1}, {"label": 2}, {"label": 0}, {"label": 0}, {"label": 0}, ] assert ds[1].compute(label_name=True) == {"label": "blue"} assert ds[1].compute() == {"label": 2} assert ds[1:3].compute(label_name=True).tolist() == [ {"label": "blue"}, {"label": "red"}, ] assert ds[1:3].compute().tolist() == [{"label": 2}, {"label": 0}]
def test_class_label_value(): ds = Dataset( "./data/tests/test_check_label", mode="w", shape=(5, ), schema={ "label": ClassLabel(names=["name1", "name2", "name3"]), "label/b": ClassLabel(num_classes=5), "label_mult": ClassLabel(shape=(None, ), max_shape=(3, ), names=["name1", "name2", "name3"]), }, ) ds["label", 0:2] = np.array([0, 1]) ds["label", 0:3] = ["name1", "name2", "name3"] ds[0:3]["label"] = [0, "name2", 2] ds[0]["label_mult"] = np.array(["name1", "name3"]) ds["label_mult", 1] = "name2" ds["label_mult", 2:4] = [np.array(["name2", "name3"]), np.array(["name1"])] ds["label_mult", 3] = np.array([1, 0, 2]) ds["label_mult", 4] = [1] ds["label_mult", 3:5] = [[2, 2], [0]] try: ds["label", 0:7] = 2 except Exception as ex: assert isinstance(ex, hub.exceptions.ValueShapeError) try: ds["label/b", 0] = 6 except Exception as ex: assert isinstance(ex, ClassLabelValueError) try: ds[0:4]["label/b"] = np.array([0, 1, 2, 3, 7]) except Exception as ex: assert isinstance(ex, ClassLabelValueError) try: ds["label", 4] = "name4" except Exception as ex: assert isinstance(ex, ClassLabelValueError) try: ds[0]["label/b"] = ["name"] except Exception as ex: assert isinstance(ex, ValueError)
def test_dataset_filter_4(): schema = { "img": Image((None, None, 3), max_shape=(100, 100, 3)), "cl": ClassLabel(names=["cat", "dog", "horse"]), } ds = Dataset("./data/tests/filtering_4", shape=(100,), schema=schema, mode="w") for i in range(100): ds["cl", i] = 0 if i < 10 else 1 ds["img", i] = i * np.ones((5, 6, 3)) ds_filtered = ds.filter(lambda x: x["cl"].compute() == 0) assert (ds_filtered[3:8, "cl"].compute() == np.zeros((5,))).all()
def make_schema(path_to_dir, labels, dtype): """| make_schema internal function to generate the schema internally.""" max_shape = get_max_shape(path_to_dir) image_shape = (None, None, None) if labels is None: labels = ClassLabel(names=os.listdir(path_to_dir)) else: labels = ClassLabel(labels) schema = { "label": labels, "image": Tensor( shape=image_shape, max_shape=max_shape, dtype=dtype, ), } return schema
def test_dataset_filter_3(): schema = { "img": Image((None, None, 3), max_shape=(100, 100, 3)), "cl": ClassLabel(names=["cat", "dog", "horse"]), } ds = Dataset("./data/tests/filtering_3", shape=(100,), schema=schema, mode="w") for i in range(100): ds["cl", i] = 0 if i % 5 == 0 else 1 ds["img", i] = i * np.ones((5, 6, 3)) ds["cl", 4] = 2 ds_filtered = ds.filter(lambda x: x["cl"].compute() == 0) assert ds_filtered.indexes == [5 * i for i in range(20)] ds_filtered_2 = ds.filter(lambda x: x["cl"].compute() == 2) assert (ds_filtered_2["img"].compute() == 4 * np.ones((1, 5, 6, 3))).all() for item in ds_filtered_2: assert (item["img"].compute() == 4 * np.ones((5, 6, 3))).all() assert item["cl"].compute() == 2
def main(): schema = { "image": Image(shape=(None, None), max_shape=(28, 28)), "label": ClassLabel(num_classes=10), } path = "./data/examples/new_api_intro2" ds = Dataset(path, shape=(10, ), mode="w", schema=schema) print(len(ds)) for i in range(len(ds)): with Timer("writing single element"): ds["image", i] = np.ones((28, 28), dtype="uint8") ds["label", i] = 3 ds.resize_shape(200) print(ds.shape) print(ds["label", 100:110].numpy()) with Timer("Committing"): ds.flush() ds = Dataset(path) print(ds.schema) print(ds["image", 0].compute())
import numpy as np import hub from hub.schema import Image, ClassLabel from hub.utils import Timer schema = { "image": Image((28, 28), chunks=(1000, 28, 28)), "label": ClassLabel(num_classes=10), } def main(): sample_count = 70000 step = 10 with Timer("Time"): ds = hub.Dataset( "./data/examples/mnist_upload_speed_benchmark", mode="w", schema=schema, shape=(sample_count, ), cache=2**26, ) arr = (np.random.rand(step, 28, 28) * 100).astype("uint8") for i in range(0, sample_count, step): # with Timer(f"Sample {i}"): ds["image", i:i + step] = arr
""" import glob import os import numpy as np import PIL.Image import hub from hub.schema import ClassLabel, Image # Create a new dataset schema = { "image": Image(shape=(None, None, None), max_shape=(3000, 3000, 3), dtype="uint8"), "label": ClassLabel(num_classes=2), } tag = "/tmp/chest_xray/train" len_ds = 5216 ds = hub.Dataset(tag, mode="w+", shape=(len_ds, ), schema=schema) # Transform function @hub.transform(schema=schema, scheduler="threaded", workers=8) def fill_ds(filename): if os.path.basename(os.path.dirname(filename)) == "NORMAL": label = 0 else: label = 1 image = np.array(PIL.Image.open(filename)) if len(image.shape) == 2:
If a copy of the MPL was not distributed with this file, You can obtain one at https://mozilla.org/MPL/2.0/. """ from hub import Dataset from hub.api.datasetview import TensorView from hub.exceptions import NoneValueException from hub.schema import Tensor, ClassLabel import numpy as np import pytest my_schema = { "image": Tensor((None, None, None, None), "uint8", max_shape=(10, 1920, 1080, 4)), "label": ClassLabel(num_classes=3), } my_schema2 = { "image": Tensor((None, None, None, None), "uint8", max_shape=(10, 1920, 1080, 4)), "label": ClassLabel(names=["red", "green", "blue"]), } ds = Dataset("./data/test/dataset", shape=(100, ), mode="w", schema=my_schema) ds2 = Dataset("./data/test/dataset2", shape=(5, ), mode="w", schema=my_schema2) ds["label", 0] = 1 ds["label", 1] = 2 ds["label", 2] = 0 ds2["label", 0] = 1
def _from_supervisely(project, scheduler: str = "single", workers: int = 1): try: import supervisely_lib as sly from supervisely_lib.project import project as sly_image_project from supervisely_lib.project import video_project as sly_video_project from skvideo.io import FFmpegReader, vread except ModuleNotFoundError: raise ModuleNotInstalledException("supervisely") if isinstance(project, str): with open(project + "meta.json") as meta_file: project_meta_dict = json.load(meta_file) instantiated = False else: project_meta_dict = project.meta.to_json() instantiated = True project_type = project_meta_dict["projectType"] mode = sly.OpenMode.READ def infer_image(paths): bboxes, masks = [], [] classes_bb, classes_mask = [], [] item_path, item_ann_path = paths ann = sly.Annotation.load_json_file(item_ann_path, project.meta) ann_dict = ann.to_json() sizes = (ann_dict["size"]["height"], ann_dict["size"]["width"]) for obj in ann_dict["objects"]: if obj["geometryType"] == "rectangle": bboxes.append([ item for sublist in obj["points"]["exterior"] for item in sublist ]) classes_bb.append(obj["classTitle"]) elif obj["geometryType"] == "polygon": img = PIL.Image.new("L", (sizes[1], sizes[0]), 0) PIL.ImageDraw.Draw(img).polygon( [tuple(obj) for obj in obj["points"]["exterior"]], outline=1, fill=1, ) masks.append(np.array(img)) classes_mask.append(obj["classTitle"]) return sizes, bboxes, masks, classes_bb, classes_mask def infer_video(paths): item_path, item_ann_path = paths vreader = FFmpegReader(item_path) return (vreader.getShape(), ) def infer_project(project, project_type, read_mode): if project_type == "images": if not instantiated: project = sly_image_project.Project(project, mode) max_shape = (0, 0) return ( project, Image, infer_image, max_shape, ) elif project_type == "videos": if not instantiated: project = sly_video_project.VideoProject(project, mode) max_shape = (0, 0, 0, 0) return ( project, Video, infer_video, max_shape, ) project, main_blob, infer_ds, max_shape = infer_project( project, project_type, mode) image_paths = [] label_names = [] max_num_bboxes = 0 max_num_polys = 0 masks = False datasets = project.datasets.items() uniform = True for ds in datasets: for i, item in enumerate(ds): path = ds.get_item_paths(item) image_paths.append(path) inf = infer_ds(path) if len(inf) > 1: if inf[3]: label_names.extend(inf[3]) if len(inf[3]) > max_num_bboxes: max_num_bboxes = len(inf[3]) if inf[4]: label_names.extend(inf[4]) if len(inf[4]) > max_num_polys: max_num_polys = len(inf[4]) if inf[2]: masks = True shape = inf[0] max_shape = np.maximum(shape, max_shape) if uniform and max_shape.any() and (shape != max_shape).any(): uniform = False label_names = list(np.unique(label_names)) items = chain(*datasets) idatasets = iter(datasets) ds, i = next(idatasets), 0 key = "shape" if uniform else "max_shape" if project_type == "images": read = sly.imaging.image.read blob_shape = {key: (*max_shape.tolist(), 3)} elif project_type == "videos": read = vread blob_shape = {key: max_shape.tolist()} if key == "max_shape": blob_shape["shape"] = (None, None, None, 3) schema = { project_type: main_blob(**blob_shape), } if max_num_bboxes: schema["bbox"] = BBox(shape=(None, 4), max_shape=(max_num_bboxes, 4)) if label_names: schema["label"] = ClassLabel( shape=(None, ), max_shape=(max(max_num_bboxes, max_num_polys), ), names=label_names, ) if masks: schema["mask"] = Mask(shape=(None, None, None), max_shape=(*max_shape.tolist(), 1)) @hub.transform(schema=schema, scheduler=scheduler, workers=workers) def transformation(item): nonlocal i, ds sample = {} if i >= len(ds): ds, i = next(idatasets), 0 item_path, item_ann_path = ds.get_item_paths(item) i += 1 _, bboxes, masks, classes_bbox, classes_mask = infer_ds( (item_path, item_ann_path)) sample[project_type] = read(item_path) if bboxes: sample["bbox"] = np.array(bboxes) sample["label"] = [label_names.index(i) for i in classes_bbox] if masks: sample["mask"] = np.expand_dims(masks[0], -1) sample["label"] = [label_names.index(i) for i in classes_mask] return sample return transformation(list(items))