def test_sparse_create(self): conf = TorchConfig(False, data_type=torch.float16) arr = conf.sparse([[ 7, 22, 22, 42, 60, 62, 70, 76, 112, 124, 124, 128, 135, 141, 153 ], [3, 2, 5, 0, 4, 6, 1, 5, 6, 2, 5, 4, 3, 0, 1 ]], [1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.], (174, 30)) self.assertTrue((174, 30), arr.shape) self.assertEqual(0., arr[7, 2].item()) self.assertEqual(1., arr[7, 3].item())
def test_datasets(self): tc = TorchConfig(False) fac = self.fac stash = fac('dataloader_stash') dataset = fac('mnist_batch_stash') dataset.delegate_attr = True ds_name = 'train val test'.split() batch_size = dataset.delegate.batch_size name: str ds: Tuple[Tuple[torch.Tensor, torch.Tensor]] for name, ds in zip(ds_name, stash.get_data_by_split()): ds_start = 0 ds_stash = dataset.splits[name] ds_data = torch.cat(tuple(map(lambda x: x[0], ds))) ds_labels = torch.cat(tuple(map(lambda x: x[1], ds))) dpts = sum(map(lambda b: len(b.data_point_ids), ds_stash.values())) logger.info(f'name: stash size: {len(ds_stash)}, ' + f'data set size: {len(ds)}, ' + f'stash X batch_size: {len(ds_stash) * batch_size}, ' + f'data/label shapes: {ds_data.shape}/{ds_labels.shape}, ' + f'data points: {dpts}') assert len(ds) == len(ds_stash) assert dpts == ds_labels.shape[0] assert ds_labels.shape[0] == ds_data.shape[0] for id, batch in ds_stash: ds_end = ds_start + len(batch) dsb_labels = ds_labels[ds_start:ds_end] dsb_data = ds_data[ds_start:ds_end] ds_start = ds_end blabels = batch.get_labels() bdata = batch.get_data() if logger.isEnabledFor(logging.DEBUG): logger.debug(f'data point ids: {batch.data_point_ids}') logger.debug(f'ds/batch labels: {dsb_labels}/{blabels}') assert (tc.equal(dsb_labels, blabels)) assert (tc.equal(dsb_data, bdata))
def test_create_empty(self): conf = TorchConfig(False, data_type=torch.float16) tensor = conf.empty((3, 10)) self.assertEqual(torch.float16, tensor.dtype) self.assertEqual(3, tensor.shape[0]) self.assertEqual(10, tensor.shape[1])
def test_create_tensor(self): conf = TorchConfig(False) tensor = conf.from_iterable(it.islice(it.count(), 5)) self.assertEqual(torch.float32, tensor.dtype) should = torch.FloatTensor([0, 1, 2, 3, 4]) self.assertTrue(torch.all(should.eq(tensor)))
def test_config_type(self): conf = TorchConfig(False) self.assertEqual(torch.float32, conf.data_type) self.assertEqual(torch.FloatTensor, conf.tensor_class)
def test_cuda_config_cpu(self): conf = TorchConfig(False) self.assertEqual(TorchConfig.cpu_device_name(), conf.device.type)
def test_cuda_config_write(self): writer = StringIO() conf = TorchConfig() conf.write(writer=writer) logger.debug(writer.getvalue()) self.assertTrue(len(writer.getvalue()) > 0)
def test_cuda_config(self): conf = TorchConfig() self.assertNotEqual(None, conf.info)
def test_rand(self): conf = self.conf size = (10, 20) self.rand_assert(50, size, conf) conf = TorchConfig(True, data_type=torch.float64) self.rand_assert(50, size, conf)
def setUp(self): super().setUp() self.conf = TorchConfig(False, data_type=torch.float64)
def setUp(self): tc = TorchConfig(False) self.de = NonUniformDimensionEncoder(tc)