def test_givenAWindowsOS_whenFasttextModelCollateFnInDataLoaderNumWorkers2_thenWorkProperly(
self,
):
model = FastTextEmbeddingsModel(self.a_fasttext_model_path, verbose=self.verbose)
data_transform = MockedDataTransform(model)
data_loader = DataLoader(
self.training_container,
collate_fn=data_transform.collate_fn,
batch_size=32,
num_workers=2,
)
dataset = []
for data in data_loader:
dataset.append(data)
self.assertGreater(len(dataset), 0)
def test_givenAWindowsOS_whenFasttextModelCollateFnInDataLoaderForNotWindows_thenRaiseError(self, platform_mock):
platform_mock.system().__eq__.return_value = True
with platform_mock:
model = FastTextEmbeddingsModel(self.a_fasttext_model_path, verbose=self.verbose)
data_transform = MockedDataTransform(model)
data_loader = DataLoader(
self.training_container,
collate_fn=data_transform.collate_fn,
batch_size=32,
num_workers=0,
)
with self.assertRaises(TypeError):
for _ in data_loader:
pass
def test_givenAWindowsOS_whenBPEmbModelCollateFnInDataLoaderNumWorkers2_thenWorkProperly(
self, ):
# we setup a smaller model for simplicity
model = BPEmbEmbeddingsModel(verbose=self.verbose)
data_transform = MockedDataTransform(model)
data_loader = DataLoader(
self.training_container,
collate_fn=data_transform.collate_fn,
batch_size=32,
num_workers=2,
)
dataset = []
for data in data_loader:
dataset.append(data)
self.assertGreater(len(dataset), 0)
def test_givenAWindowsOS_whenFasttextModelCollateFnInDataLoaderEvenWithWindowsSetup_thenWorkProperly(
self, platform_mock
):
platform_mock.system().__eq__.return_value = True
with platform_mock:
model = FastTextEmbeddingsModel(self.a_fasttext_model_path, verbose=self.verbose)
data_transform = MockedDataTransform(model)
data_loader = DataLoader(
self.training_container,
collate_fn=data_transform.collate_fn,
batch_size=32,
num_workers=0,
)
dataset = []
for data in data_loader:
dataset.append(data)
self.assertGreater(len(dataset), 0)