def extend_dataset(dataset: h5py.Dataset, data: Union[np.array, List]):
if type(data) is list:
data = np.array(data)
if data.shape[0] == 0:
return
dataset.resize((dataset.shape[0] + data.shape[0]), axis=0)
dataset[-data.shape[0]:] = data
def extend_dataset(dataset: h5py.Dataset, data: np.ndarray,) -> h5py.Dataset:
"""Ectend a dataset in the input HDF5 group.
Used to update the images in a split.
"""
shape = dataset.shape
newshape = (dataset.shape[0] + data.shape[0], *dataset.shape[1:])
dataset.resize(newshape)
dataset[shape[0] :] = data
return dataset
def append_buffer_to_dataset(dset: h5py.Dataset, buffer: ListBuffer):
"""Append values to resizable h5py dataset."""
if len(buffer): # Buffer is not empty
logging.info("")
values = np.array(buffer)
new_shape = (buffer.end,) + values.shape[1:]
dset.resize(new_shape)
dset[buffer.start:] = values
else:
logging.warning("Buffer is empty.")
def maybe_resize(examples: h5py.Dataset, index: int, resize_chunk: int):
if index >= examples.shape[0]:
current_shape = list(examples.shape)
current_shape[0] += resize_chunk
examples.resize(current_shape)
def _resize_prop_dset(dset: h5py.Dataset) -> None:
"""Ensure that **dset** is as long as its dimensional scale."""
scale = dset.dims[0][0]
n = len(scale)
if n > len(dset):
dset.resize(n, axis=0)
