def _process(self, chunk: List[DataPointIDSet]) -> \ Iterable[Tuple[str, Any]]: """Create the batches by creating the set of data points for each :class:`.DataPointIDSet` instance. When the subordinate stash dumps the batch (specifically a subclass of :class:`.Batch`), the overrided pickle logic is used to *detatch* the batch by encoded all data in to :class:`.FeatureContext` instances. """ if logger.isEnabledFor(logging.INFO): logger.info(f'{self.name}: processing: {len(chunk)} data points') if logger.isEnabledFor(logging.DEBUG): logger.debug(f'chunk data points: {chunk}') tseed = chunk[0].torch_seed_context dpcls: Type[DataPoint] = self.data_point_type cont = self.split_stash_container if tseed is not None: TorchConfig.set_random_seed( tseed['seed'], tseed['disable_cudnn'], False) dset: DataPointIDSet for dset in chunk: batch_id: str = dset.batch_id points: Tuple[DataPoint] = tuple( map(lambda dpid: dpcls(dpid, self, cont[dpid]), dset.data_point_ids)) batch: Batch = self.create_batch(points, dset.split_name, batch_id) if logger.isEnabledFor(logging.DEBUG): logger.debug(f'created batch: {batch}') yield (batch_id, batch)
def main(): print() TorchConfig.set_random_seed() ProtoModelFacade.configure_default_cli_logging() facade = create_facade() # test mem deallocation on feature changes runs = [4, 5, 0, 4, 5, 7, 8, 9, 10] runs = [3] for run in runs: res = {-1: tmp, 0: facade.tmp, 1: facade.print_sample, 2: lambda: facade.batch_metadata.write(), 3: lambda: facade.debug(3), 4: facade.train, 5: facade.test, 6: facade.clear, 7: facade.write_result, 8: facade.persist_result, 9: facade.deallocate, 10: end_dealloc}[run]() return res