def _process(self, chunk: List[DataPointIDSet]) -> \
Iterable[Tuple[str, Any]]:
"""Create the batches by creating the set of data points for each
:class:`.DataPointIDSet` instance. When the subordinate stash dumps
the batch (specifically a subclass of :class:`.Batch`), the overrided
pickle logic is used to *detatch* the batch by encoded all data in to
:class:`.FeatureContext` instances.
"""
if logger.isEnabledFor(logging.INFO):
logger.info(f'{self.name}: processing: {len(chunk)} data points')
if logger.isEnabledFor(logging.DEBUG):
logger.debug(f'chunk data points: {chunk}')
tseed = chunk[0].torch_seed_context
dpcls: Type[DataPoint] = self.data_point_type
cont = self.split_stash_container
if tseed is not None:
TorchConfig.set_random_seed(
tseed['seed'], tseed['disable_cudnn'], False)
dset: DataPointIDSet
for dset in chunk:
batch_id: str = dset.batch_id
points: Tuple[DataPoint] = tuple(
map(lambda dpid: dpcls(dpid, self, cont[dpid]),
dset.data_point_ids))
batch: Batch = self.create_batch(points, dset.split_name, batch_id)
if logger.isEnabledFor(logging.DEBUG):
logger.debug(f'created batch: {batch}')
yield (batch_id, batch)
def main():
print()
TorchConfig.set_random_seed()
ProtoModelFacade.configure_default_cli_logging()
facade = create_facade()
# test mem deallocation on feature changes
runs = [4, 5, 0, 4, 5, 7, 8, 9, 10]
runs = [3]
for run in runs:
res = {-1: tmp,
0: facade.tmp,
1: facade.print_sample,
2: lambda: facade.batch_metadata.write(),
3: lambda: facade.debug(3),
4: facade.train,
5: facade.test,
6: facade.clear,
7: facade.write_result,
8: facade.persist_result,
9: facade.deallocate,
10: end_dealloc}[run]()
return res