def _get_shards(self, input_files: List[str],
shard_size: Optional[int]) -> Iterator[pd.DataFrame]:
"""Defines a generator which returns data for each shard
Parameters
----------
input_files: List[str]
List of filenames to process
shard_size: int, optional
The size of a shard of data to process at a time.
Returns
-------
Iterator[pd.DataFrame]
Iterator over shards
"""
return load_sdf_files(input_files=input_files,
clean_mols=self.sanitize,
tasks=self.tasks,
shard_size=shard_size)
def _get_shards(self, input_files, shard_size):
"""Defines a generator which returns data for each shard"""
return load_sdf_files(input_files, self.sanitize, tasks=self.tasks)
def get_shards(self, input_files, shard_size):
"""Defines a generator which returns data for each shard"""
return load_sdf_files(input_files, self.clean_mols)
def get_shards(self, input_files, shard_size): """Defines a generator which returns data for each shard""" return load_sdf_files(input_files, self.clean_mols)
