def __setitem__(self, slice_, value):
"""| Sets a slice or slices with a value
| Usage:
>>> ds["image", 5, 0:1920, 0:1080, 0:3] = np.zeros((1920, 1080, 3), "uint8")
>>> images = ds["image"]
>>> image = images[5]
>>> image[0:1920, 0:1080, 0:3] = np.zeros((1920, 1080, 3), "uint8")
"""
assign_value = get_value(value)
# handling strings and bytes
assign_value = str_to_int(assign_value, self.tokenizer)
if not isinstance(slice_, abc.Iterable) or isinstance(slice_, str):
slice_ = [slice_]
slice_ = list(slice_)
subpath, slice_list = slice_split(slice_)
if not subpath:
raise ValueError("Can't assign to dataset sliced without subpath")
elif not slice_list:
if subpath in self._tensors.keys():
self._tensors[subpath][:] = assign_value # Add path check
else:
ObjectView(dataset=self, subpath=subpath)[:] = assign_value
else:
if subpath in self._tensors.keys():
self._tensors[subpath][slice_list] = assign_value
else:
ObjectView(dataset=self,
subpath=subpath,
slice_list=slice_list)[:] = assign_value
def __getitem__(self, slice_):
"""| Gets a slice or slices from DatasetView
| Usage:
>>> ds_view = ds[5:15]
>>> return ds_view["image", 7, 0:1920, 0:1080, 0:3].compute() # returns numpy array of 12th image
"""
if not isinstance(slice_, abc.Iterable) or isinstance(slice_, str):
slice_ = [slice_]
slice_ = list(slice_)
subpath, slice_list = slice_split(slice_)
slice_list = [0] + slice_list if isinstance(self.indexes,
int) else slice_list
if not subpath:
if len(slice_list) > 1:
raise ValueError(
"Can't slice dataset with multiple slices without key")
indexes = self.indexes[slice_list[0]]
return DatasetView(dataset=self.dataset,
lazy=self.lazy,
indexes=indexes)
elif not slice_list:
slice_ = ([slice(self.indexes[0], self.indexes[-1] +
1)] if self.is_contiguous else [self.indexes])
if subpath in self.keys:
tensorview = TensorView(
dataset=self.dataset,
subpath=subpath,
slice_=slice_,
lazy=self.lazy,
)
return tensorview if self.lazy else tensorview.compute()
for key in self.keys:
if subpath.startswith(key):
objectview = ObjectView(
dataset=self.dataset,
subpath=subpath,
slice_=slice_,
lazy=self.lazy,
)
return objectview if self.lazy else objectview.compute()
return self._get_dictionary(subpath, slice_)
else:
if isinstance(self.indexes, list):
indexes = self.indexes[slice_list[0]]
if self.is_contiguous and isinstance(indexes,
list) and indexes:
indexes = slice(indexes[0], indexes[-1] + 1)
else:
indexes = self.indexes
slice_list[0] = indexes
schema_obj = self.dataset.schema.dict_[subpath.split("/")[1]]
if subpath in self.keys and (not isinstance(schema_obj, Sequence)
or len(slice_list) <= 1):
tensorview = TensorView(
dataset=self.dataset,
subpath=subpath,
slice_=slice_list,
lazy=self.lazy,
)
return tensorview if self.lazy else tensorview.compute()
for key in self.keys:
if subpath.startswith(key):
objectview = ObjectView(
dataset=self.dataset,
subpath=subpath,
slice_=slice_list,
lazy=self.lazy,
)
return objectview if self.lazy else objectview.compute()
if len(slice_list) > 1:
raise ValueError("You can't slice a dictionary of Tensors")
return self._get_dictionary(subpath, slice_list[0])
def __getitem__(self, slice_):
"""| Gets a slice or slices from dataset
| Usage:
>>> return ds["image", 5, 0:1920, 0:1080, 0:3].compute() # returns numpy array
>>> images = ds["image"]
>>> return images[5].compute() # returns numpy array
>>> images = ds["image"]
>>> image = images[5]
>>> return image[0:1920, 0:1080, 0:3].compute()
"""
if not isinstance(slice_, abc.Iterable) or isinstance(slice_, str):
slice_ = [slice_]
slice_ = list(slice_)
subpath, slice_list = slice_split(slice_)
if not subpath:
if len(slice_list) > 1:
raise ValueError(
"Can't slice a dataset with multiple slices without key")
indexes = self.indexes[slice_list[0]]
return DatasetView(
dataset=self,
indexes=indexes,
lazy=self.lazy,
)
elif not slice_list:
if subpath in self.keys:
tensorview = TensorView(
dataset=self,
subpath=subpath,
slice_=slice(0, self._shape[0]),
lazy=self.lazy,
)
return tensorview if self.lazy else tensorview.compute()
for key in self.keys:
if subpath.startswith(key):
objectview = ObjectView(
dataset=self,
subpath=subpath,
lazy=self.lazy,
slice_=[slice(0, self._shape[0])],
)
return objectview if self.lazy else objectview.compute()
return self._get_dictionary(subpath)
else:
schema_obj = self.schema.dict_[subpath.split("/")[1]]
if subpath in self.keys and (not isinstance(schema_obj, Sequence)
or len(slice_list) <= 1):
tensorview = TensorView(dataset=self,
subpath=subpath,
slice_=slice_list,
lazy=self.lazy)
return tensorview if self.lazy else tensorview.compute()
for key in self.keys:
if subpath.startswith(key):
objectview = ObjectView(
dataset=self,
subpath=subpath,
slice_=slice_list,
lazy=self.lazy,
)
return objectview if self.lazy else objectview.compute()
if len(slice_list) > 1:
raise ValueError("You can't slice a dictionary of Tensors")
return self._get_dictionary(subpath, slice_list[0])
def __getitem__(self, slice_):
"""| Gets a slice or slices from dataset
| Usage:
>>> return ds["image", 5, 0:1920, 0:1080, 0:3].compute() # returns numpy array
>>> images = ds["image"]
>>> return images[5].compute() # returns numpy array
>>> images = ds["image"]
>>> image = images[5]
>>> return image[0:1920, 0:1080, 0:3].compute()
"""
if not isinstance(slice_, abc.Iterable) or isinstance(slice_, str):
slice_ = [slice_]
slice_ = list(slice_)
subpath, slice_list = slice_split(slice_)
if not subpath:
if len(slice_list) > 1:
raise ValueError(
"Can't slice a dataset with multiple slices without subpath"
)
num, ofs = slice_extract_info(slice_list[0], self.shape[0])
return DatasetView(
dataset=self,
num_samples=num,
offset=ofs,
squeeze_dim=isinstance(slice_list[0], int),
lazy=self.lazy,
)
elif not slice_list:
if subpath in self._tensors.keys():
tensorview = TensorView(
dataset=self,
subpath=subpath,
slice_=slice(0, self.shape[0]),
lazy=self.lazy,
)
if self.lazy:
return tensorview
else:
return tensorview.compute()
for key in self._tensors.keys():
if subpath.startswith(key):
objectview = ObjectView(dataset=self,
subpath=subpath,
lazy=self.lazy)
if self.lazy:
return objectview
else:
return objectview.compute()
return self._get_dictionary(subpath)
else:
num, ofs = slice_extract_info(slice_list[0], self.shape[0])
schema_obj = self.schema.dict_[subpath.split("/")[1]]
if subpath in self._tensors.keys() and (not isinstance(
schema_obj, Sequence) or len(slice_list) <= 1):
tensorview = TensorView(dataset=self,
subpath=subpath,
slice_=slice_list,
lazy=self.lazy)
if self.lazy:
return tensorview
else:
return tensorview.compute()
for key in self._tensors.keys():
if subpath.startswith(key):
objectview = ObjectView(
dataset=self,
subpath=subpath,
slice_list=slice_list,
lazy=self.lazy,
)
if self.lazy:
return objectview
else:
return objectview.compute()
if len(slice_list) > 1:
raise ValueError("You can't slice a dictionary of Tensors")
return self._get_dictionary(subpath, slice_list[0])