def _get_shards(self, input_files, shard_size):
"""Defines a generator which returns data for each shard
Parameters
----------
input_files: list[str]
List of filenames to process
shard_size: int
The size of a shard of data to process at a time.
"""
return load_csv_files(input_files, shard_size)
def _get_shards(self, input_files: List[str],
shard_size: Optional[int]) -> Iterator[pd.DataFrame]:
"""Defines a generator which returns data for each shard
Parameters
----------
input_files: List[str]
List of filenames to process
shard_size: int, optional
The size of a shard of data to process at a time.
Returns
-------
Iterator[pd.DataFrame]
Iterator over shards
"""
return load_csv_files(input_files, shard_size)
def _get_shards(self, input_files, shard_size):
"""Defines a generator which returns data for each shard"""
return load_csv_files(input_files, shard_size)
def get_shards(self, input_files, shard_size): """Defines a generator which returns data for each shard""" return load_csv_files(input_files, shard_size)
