def __getitem__(self, slice_): """| Gets a slice or slices from DatasetView | Usage: >>> ds_view = ds[5:15] >>> return ds_view["image", 7, 0:1920, 0:1080, 0:3].compute() # returns numpy array of 12th image """ if not isinstance(slice_, abc.Iterable) or isinstance(slice_, str): slice_ = [slice_] slice_ = list(slice_) subpath, slice_list = slice_split(slice_) slice_list = [0] + slice_list if self.squeeze_dim else slice_list if not subpath: if len(slice_list) > 1: raise ValueError( "Can't slice a dataset with multiple slices without subpath" ) num, ofs = slice_extract_info(slice_list[0], self.num_samples) return DatasetView( dataset=self.dataset, num_samples=num, offset=ofs + self.offset, squeeze_dim=isinstance(slice_list[0], int), ) elif not slice_list: slice_ = slice(self.offset, self.offset + self.num_samples) if subpath in self.dataset._tensors.keys(): return TensorView( dataset=self.dataset, subpath=subpath, slice_=slice_, squeeze_dims=[True] if self.squeeze_dim else [], ) return self._get_dictionary(self.dataset, subpath, slice=slice_) else: num, ofs = slice_extract_info(slice_list[0], self.num_samples) slice_list[0] = ( ofs + self.offset if num == 1 else slice(ofs + self.offset, ofs + self.offset + num) ) if subpath in self.dataset._tensors.keys(): return TensorView( dataset=self.dataset, subpath=subpath, slice_=slice_list, squeeze_dims=[True] if self.squeeze_dim else [], ) if len(slice_list) > 1: raise ValueError("You can't slice a dictionary of Tensors") return self._get_dictionary(subpath, slice_list[0])
def __getitem__(self, slice_): """| Gets a slice or slices from dataset | Usage: >>> return ds["image", 5, 0:1920, 0:1080, 0:3].numpy() # returns numpy array >>> images = ds["image"] >>> return images[5].numpy() # returns numpy array >>> images = ds["image"] >>> image = images[5] >>> return image[0:1920, 0:1080, 0:3].numpy() """ if not isinstance(slice_, abc.Iterable) or isinstance(slice_, str): slice_ = [slice_] slice_ = list(slice_) subpath, slice_list = slice_split(slice_) if not subpath: if len(slice_list) > 1: raise ValueError( "Can't slice a dataset with multiple slices without subpath" ) num, ofs = slice_extract_info(slice_list[0], self.shape[0]) return DatasetView( dataset=self, num_samples=num, offset=ofs, squeeze_dim=isinstance(slice_list[0], int), ) elif not slice_list: if subpath in self._tensors.keys(): return TensorView(dataset=self, subpath=subpath, slice_=slice(0, self.shape[0])) return self._get_dictionary(subpath) else: num, ofs = slice_extract_info(slice_list[0], self.shape[0]) if subpath in self._tensors.keys(): return TensorView(dataset=self, subpath=subpath, slice_=slice_list) if len(slice_list) > 1: raise ValueError("You can't slice a dictionary of Tensors") return self._get_dictionary(subpath, slice_list[0])
def _get_dictionary(self, subpath, slice_=None): """Gets dictionary from dataset given incomplete subpath""" tensor_dict = {} subpath = subpath if subpath.endswith("/") else subpath + "/" for key in self.keys: if key.startswith(subpath): suffix_key = key[len(subpath) :] split_key = suffix_key.split("/") cur = tensor_dict for i in range(len(split_key) - 1): if split_key[i] not in cur.keys(): cur[split_key[i]] = {} cur = cur[split_key[i]] slice_ = slice_ or slice(0, self._shape[0]) tensorview = TensorView( dataset=self, subpath=key, slice_=slice_, lazy=self.lazy ) cur[split_key[-1]] = tensorview if self.lazy else tensorview.compute() if not tensor_dict: raise KeyError(f"Key {subpath} was not found in dataset") return tensor_dict
def _get_dictionary(self, subpath, slice_): """Gets dictionary from dataset given incomplete subpath""" tensor_dict = {} subpath = subpath if subpath.endswith("/") else subpath + "/" for key in self.keys: if key.startswith(subpath): suffix_key = key[len(subpath) :] split_key = suffix_key.split("/") cur = tensor_dict for sub_key in split_key[:-1]: if sub_key not in cur.keys(): cur[sub_key] = {} cur = cur[sub_key] tensorview = TensorView( dataset=self.dataset, subpath=key, slice_=slice_, lazy=self.lazy, ) cur[split_key[-1]] = tensorview if self.lazy else tensorview.compute() if not tensor_dict: raise KeyError(f"Key {subpath} was not found in dataset") return tensor_dict
def _get_dictionary(self, subpath, slice_=None): """Gets dictionary from dataset given incomplete subpath""" tensor_dict = {} subpath = subpath if subpath.endswith("/") else subpath + "/" for key in self.dataset._tensors.keys(): if key.startswith(subpath): suffix_key = key[len(subpath) :] split_key = suffix_key.split("/") cur = tensor_dict for i in range(len(split_key) - 1): if split_key[i] not in cur.keys(): cur[split_key[i]] = {} cur = cur[split_key[i]] slice_ = slice_ if slice_ else slice(0, self.dataset.shape[0]) cur[split_key[-1]] = TensorView( dataset=self.dataset, subpath=key, slice_=slice_, squeeze_dims=[True] if self.squeeze_dim else [], ) if len(tensor_dict) == 0: raise KeyError(f"Key {subpath} was not found in dataset") return tensor_dict
def __getitem__(self, slice_): """| Gets a slice or slices from DatasetView | Usage: >>> ds_view = ds[5:15] >>> return ds_view["image", 7, 0:1920, 0:1080, 0:3].compute() # returns numpy array of 12th image """ if not isinstance(slice_, abc.Iterable) or isinstance(slice_, str): slice_ = [slice_] slice_ = list(slice_) subpath, slice_list = slice_split(slice_) slice_list = [0] + slice_list if isinstance(self.indexes, int) else slice_list if not subpath: if len(slice_list) > 1: raise ValueError( "Can't slice dataset with multiple slices without key") indexes = self.indexes[slice_list[0]] return DatasetView(dataset=self.dataset, lazy=self.lazy, indexes=indexes) elif not slice_list: slice_ = ([slice(self.indexes[0], self.indexes[-1] + 1)] if self.is_contiguous else [self.indexes]) if subpath in self.keys: tensorview = TensorView( dataset=self.dataset, subpath=subpath, slice_=slice_, lazy=self.lazy, ) return tensorview if self.lazy else tensorview.compute() for key in self.keys: if subpath.startswith(key): objectview = ObjectView( dataset=self.dataset, subpath=subpath, slice_=slice_, lazy=self.lazy, ) return objectview if self.lazy else objectview.compute() return self._get_dictionary(subpath, slice_) else: if isinstance(self.indexes, list): indexes = self.indexes[slice_list[0]] if self.is_contiguous and isinstance(indexes, list) and indexes: indexes = slice(indexes[0], indexes[-1] + 1) else: indexes = self.indexes slice_list[0] = indexes schema_obj = self.dataset.schema.dict_[subpath.split("/")[1]] if subpath in self.keys and (not isinstance(schema_obj, Sequence) or len(slice_list) <= 1): tensorview = TensorView( dataset=self.dataset, subpath=subpath, slice_=slice_list, lazy=self.lazy, ) return tensorview if self.lazy else tensorview.compute() for key in self.keys: if subpath.startswith(key): objectview = ObjectView( dataset=self.dataset, subpath=subpath, slice_=slice_list, lazy=self.lazy, ) return objectview if self.lazy else objectview.compute() if len(slice_list) > 1: raise ValueError("You can't slice a dictionary of Tensors") return self._get_dictionary(subpath, slice_list[0])
def __getitem__(self, slice_): """| Gets a slice or slices from dataset | Usage: >>> return ds["image", 5, 0:1920, 0:1080, 0:3].compute() # returns numpy array >>> images = ds["image"] >>> return images[5].compute() # returns numpy array >>> images = ds["image"] >>> image = images[5] >>> return image[0:1920, 0:1080, 0:3].compute() """ if not isinstance(slice_, abc.Iterable) or isinstance(slice_, str): slice_ = [slice_] slice_ = list(slice_) subpath, slice_list = slice_split(slice_) if not subpath: if len(slice_list) > 1: raise ValueError( "Can't slice a dataset with multiple slices without key") indexes = self.indexes[slice_list[0]] return DatasetView( dataset=self, indexes=indexes, lazy=self.lazy, ) elif not slice_list: if subpath in self.keys: tensorview = TensorView( dataset=self, subpath=subpath, slice_=slice(0, self._shape[0]), lazy=self.lazy, ) return tensorview if self.lazy else tensorview.compute() for key in self.keys: if subpath.startswith(key): objectview = ObjectView( dataset=self, subpath=subpath, lazy=self.lazy, slice_=[slice(0, self._shape[0])], ) return objectview if self.lazy else objectview.compute() return self._get_dictionary(subpath) else: schema_obj = self.schema.dict_[subpath.split("/")[1]] if subpath in self.keys and (not isinstance(schema_obj, Sequence) or len(slice_list) <= 1): tensorview = TensorView(dataset=self, subpath=subpath, slice_=slice_list, lazy=self.lazy) return tensorview if self.lazy else tensorview.compute() for key in self.keys: if subpath.startswith(key): objectview = ObjectView( dataset=self, subpath=subpath, slice_=slice_list, lazy=self.lazy, ) return objectview if self.lazy else objectview.compute() if len(slice_list) > 1: raise ValueError("You can't slice a dictionary of Tensors") return self._get_dictionary(subpath, slice_list[0])
def __getitem__(self, slice_): """| Gets a slice or slices from dataset | Usage: >>> return ds["image", 5, 0:1920, 0:1080, 0:3].compute() # returns numpy array >>> images = ds["image"] >>> return images[5].compute() # returns numpy array >>> images = ds["image"] >>> image = images[5] >>> return image[0:1920, 0:1080, 0:3].compute() """ if not isinstance(slice_, abc.Iterable) or isinstance(slice_, str): slice_ = [slice_] slice_ = list(slice_) subpath, slice_list = slice_split(slice_) if not subpath: if len(slice_list) > 1: raise ValueError( "Can't slice a dataset with multiple slices without subpath" ) num, ofs = slice_extract_info(slice_list[0], self.shape[0]) return DatasetView( dataset=self, num_samples=num, offset=ofs, squeeze_dim=isinstance(slice_list[0], int), lazy=self.lazy, ) elif not slice_list: if subpath in self._tensors.keys(): tensorview = TensorView( dataset=self, subpath=subpath, slice_=slice(0, self.shape[0]), lazy=self.lazy, ) if self.lazy: return tensorview else: return tensorview.compute() for key in self._tensors.keys(): if subpath.startswith(key): objectview = ObjectView(dataset=self, subpath=subpath, lazy=self.lazy) if self.lazy: return objectview else: return objectview.compute() return self._get_dictionary(subpath) else: num, ofs = slice_extract_info(slice_list[0], self.shape[0]) schema_obj = self.schema.dict_[subpath.split("/")[1]] if subpath in self._tensors.keys() and (not isinstance( schema_obj, Sequence) or len(slice_list) <= 1): tensorview = TensorView(dataset=self, subpath=subpath, slice_=slice_list, lazy=self.lazy) if self.lazy: return tensorview else: return tensorview.compute() for key in self._tensors.keys(): if subpath.startswith(key): objectview = ObjectView( dataset=self, subpath=subpath, slice_list=slice_list, lazy=self.lazy, ) if self.lazy: return objectview else: return objectview.compute() if len(slice_list) > 1: raise ValueError("You can't slice a dictionary of Tensors") return self._get_dictionary(subpath, slice_list[0])
def test_tensorview_init(): with pytest.raises(NoneValueException): tensorview_object = TensorView(ds, subpath=None) with pytest.raises(NoneValueException): tensorview_object_2 = TensorView(dataset=None, subpath="image")
def __getitem__(self, slice_): """| Gets a slice or slices from DatasetView | Usage: >>> ds_view = ds[5:15] >>> return ds_view["image", 7, 0:1920, 0:1080, 0:3].compute() # returns numpy array of 12th image """ if not isinstance(slice_, abc.Iterable) or isinstance(slice_, str): slice_ = [slice_] slice_ = list(slice_) subpath, slice_list = slice_split(slice_) slice_list = [0] + slice_list if self.squeeze_dim else slice_list if not subpath: if len(slice_list) > 1: raise ValueError( "Can't slice a dataset with multiple slices without subpath" ) num, ofs = slice_extract_info(slice_list[0], self.num_samples) return DatasetView( dataset=self.dataset, num_samples=num, offset=ofs + self.offset, squeeze_dim=isinstance(slice_list[0], int), lazy=self.lazy, ) elif not slice_list: slice_ = (slice(self.offset, self.offset + self.num_samples) if not self.squeeze_dim else self.offset) if subpath in self.dataset._tensors.keys(): tensorview = TensorView( dataset=self.dataset, subpath=subpath, slice_=slice_, lazy=self.lazy, ) return tensorview if self.lazy else tensorview.compute() for key in self.dataset._tensors.keys(): if subpath.startswith(key): objectview = objv.ObjectView( dataset=self.dataset, subpath=subpath, slice_list=[slice_], lazy=self.lazy, ) return objectview if self.lazy else objectview.compute() return self._get_dictionary(self.dataset, subpath, slice=slice_) else: num, ofs = slice_extract_info(slice_list[0], self.num_samples) slice_list[0] = (ofs + self.offset if isinstance( slice_list[0], int) else slice(ofs + self.offset, ofs + self.offset + num)) schema_obj = self.dataset.schema.dict_[subpath.split("/")[1]] if subpath in self.dataset._tensors.keys() and (not isinstance( schema_obj, objv.Sequence) or len(slice_list) <= 1): tensorview = TensorView( dataset=self.dataset, subpath=subpath, slice_=slice_list, lazy=self.lazy, ) return tensorview if self.lazy else tensorview.compute() for key in self.dataset._tensors.keys(): if subpath.startswith(key): objectview = objv.ObjectView( dataset=self.dataset, subpath=subpath, slice_list=slice_list, lazy=self.lazy, ) return objectview if self.lazy else objectview.compute() if len(slice_list) > 1: raise ValueError("You can't slice a dictionary of Tensors") return self._get_dictionary(subpath, slice_list[0])