def _get_shards(self, input_files: List[str], shard_size: Optional[int]) -> Iterator[pd.DataFrame]: """Defines a generator which returns data for each shard Parameters ---------- input_files: List[str] List of filenames to process shard_size: int, optional The size of a shard of data to process at a time. Returns ------- Iterator[pd.DataFrame] Iterator over shards """ return load_sdf_files(input_files=input_files, clean_mols=self.sanitize, tasks=self.tasks, shard_size=shard_size)
def _get_shards(self, input_files, shard_size): """Defines a generator which returns data for each shard""" return load_sdf_files(input_files, self.sanitize, tasks=self.tasks)
def get_shards(self, input_files, shard_size): """Defines a generator which returns data for each shard""" return load_sdf_files(input_files, self.clean_mols)