def __init__(self,
datapipe: Iterable[Tuple[str, BufferedIOBase]],
length: int = -1):
super().__init__()
self.datapipe: Iterable[Tuple[str, BufferedIOBase]] = datapipe
self.length: int = length
deprecation_warning_torchdata(type(self).__name__)
def __new__(cls,
datapipe: Iterable[str],
mode: str = 'b',
length: int = -1):
deprecation_warning_torchdata(type(cls).__name__)
return FileOpenerIterDataPipe(datapipe=datapipe,
mode=mode,
length=length)
def __init__(self,
datapipe: Iterable[Tuple[str, BufferedIOBase]],
*handlers: Callable,
key_fn: Callable = extension_extract_fn) -> None:
super().__init__()
self.datapipe: Iterable[Tuple[str, BufferedIOBase]] = datapipe
if not handlers:
handlers = (decoder_basichandlers, decoder_imagehandler('torch'))
self.decoder = Decoder(*handlers, key_fn=key_fn)
deprecation_warning_torchdata(type(self).__name__)
def __init__(self,
datapipe: IterDataPipe[T_co],
batch_size: int,
drop_last: bool = False,
batch_num: int = 100,
bucket_num: int = 1,
sort_key: Optional[Callable] = None,
in_batch_shuffle: bool = True) -> None:
assert batch_size > 0, "Batch size is required to be larger than 0!"
assert batch_num > 0, "Number of batches is required to be larger than 0!"
assert bucket_num > 0, "Number of buckets is required to be larger than 0!"
deprecation_warning_torchdata(type(self).__name__)
super().__init__()
# TODO: Verify _datapippe is not going to be serialized twice
# and be able to reconstruct
self._datapipe = datapipe
self.batch_size = batch_size
self.drop_last = drop_last
self.batch_num = batch_num
self.bucket_num = bucket_num
self.sort_key = sort_key
self.in_batch_shuffle = in_batch_shuffle
self.bucket_size = batch_size * batch_num
self.pool_size = self.bucket_size * bucket_num
if bucket_num > 1 or sort_key is None:
if in_batch_shuffle:
datapipe = datapipe.batch(
batch_size=self.pool_size,
drop_last=False).map(fn=_in_batch_shuffle_fn).unbatch()
else:
datapipe = datapipe.shuffle(buffer_size=self.pool_size)
if sort_key is not None:
datapipe = datapipe.batch(
self.bucket_size).map(fn=sort_key).unbatch()
datapipe = datapipe.batch(batch_size, drop_last=drop_last)
if sort_key is not None:
# In-batch shuffle each bucket seems not that useful
if in_batch_shuffle:
datapipe = datapipe.batch(
batch_size=bucket_num,
drop_last=False).map(fn=_in_batch_shuffle_fn).unbatch()
else:
datapipe = datapipe.shuffle(buffer_size=self.bucket_size)
self.datapipe = datapipe
self.length = None
def __init__(self, datapipe, timeout=None):
self.datapipe = datapipe
self.timeout = timeout
deprecation_warning_torchdata(type(self).__name__)
def __init__(self, datapipe):
self.datapipe = datapipe
deprecation_warning_torchdata(type(self).__name__)
