def test_loadfiles(self, mock_isfile): mock_isfile.return_value = True # without size cardinality = Cardinality(3, "", "") read_data = pickle.dumps({ 'word2vec': -1, 'counter_patterns': { "1": 1, "10": 10, "100": 100, "1000": 1000, "10000": 10000, "100000": 100000 } }) mockOpen = mock_open(read_data=read_data) tf = tempfile.NamedTemporaryFile() f = h5py.File(tf, 'w') f.create_dataset("list_classes", data=[1, 2, 3, 4, 5]) f.close() cardinality.path_list_classes = tf.name with patch('builtins.open', mockOpen): cardinality.load_files() tf.close() self.assertEqual(len(cardinality.counter), 6) self.assertEqual(len(cardinality.list_classes), 5) # with size cardinality = Cardinality(3, "", "", size=2) read_data = pickle.dumps({ 'word2vec': -1, 'counter_patterns': { "1": 1, "10": 10, "100": 100, "1000": 1000, "10000": 10000, "100000": 100000 } }) mockOpen = mock_open(read_data=read_data) tf = tempfile.NamedTemporaryFile() f = h5py.File(tf, 'w') f.create_dataset("list_classes", data=[1, 2, 3, 4, 5]) f.close() cardinality.path_list_classes = tf.name with patch('builtins.open', mockOpen): cardinality.load_files() tf.close() self.assertEqual(len(cardinality.counter), 6) self.assertEqual(len(cardinality.list_classes), 2)
def test_train_model(self, mock_isfile): mock_isfile.return_value = True cardinality = Cardinality(3, "", "") read_data = pickle.dumps({ 'word2vec': { "1": [1] * 20, "2": [2] * 20, "3": [3] * 20, "4": [4] * 20, "5": [5] * 20, "6": [6] * 20, "7": [7] * 20 }, 'counter_patterns': { 1: 10, 2: 100, 3: 100, 4: 100, 6: 1000, 5: 1000 }, "LSTM": { 3: self.model.state_dict() } }) mockOpen = mock_open(read_data=read_data) tf = tempfile.NamedTemporaryFile() f = h5py.File(tf, 'w') f.create_dataset("list_classes", data=[ 1, 1, 1, 2, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 4, 5, 6 ]) f.close() cardinality.path_list_classes = tf.name cardinality.counter = {1: 10, 2: 100, 3: 100, 4: 100, 6: 1000, 5: 1000} with patch('builtins.open', mockOpen): worker_single = Worker_single(batch_result=1, cardinality=cardinality, lock=self.lock, batch_size=1) dataloader = worker_single.create_dataloader() worker_single.train(resuming=False) worker_single.train(resuming=True)
def test_without_file(self, mock_isfile): mock_isfile.return_value = True cardinality = Cardinality(3, "", "") read_data = pickle.dumps({ 'word2vec': { "1": [1] * 20, "2": [2] * 20, "3": [3] * 20, "4": [4] * 20, "5": [5] * 20, "6": [6] * 20, "7": [7] * 20 }, 'counter_patterns': { 1: 10, 2: 100, 3: 100, 4: 100, 6: 1000, 5: 1000 }, "LSTM": { 3: self.model.state_dict() } }) mockOpen = mock_open(read_data=read_data) tf = tempfile.NamedTemporaryFile() f = h5py.File(tf, 'w') f.create_dataset("list_classes", data=[ 1, 1, 1, 2, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 4, 5, 6 ]) f.close() cardinality.path_list_classes = tf.name cardinality.counter = {1: 10, 2: 100, 3: 100, 4: 100, 6: 1000, 5: 1000} # No data with self.assertRaises(Exception): worker_single = Worker_single(cardinality=cardinality, lock=self.lock, batch_size=1) worker_single.load_model()
def test_creation(self): cardinality = Cardinality(3, "", "", size=2) read_data = pickle.dumps({ 'word2vec': -1, 'counter_patterns': { "1": 1, "10": 10, "100": 100, "1000": 1000, "10000": 10000, "100000": 100000 } }) mockOpen = mock_open(read_data=read_data) tf = tempfile.NamedTemporaryFile() f = h5py.File(tf, 'w') f.create_dataset("list_classes", data=[1, 2, 3, 4, 5]) f.close() cardinality.path_list_classes = tf.name with patch('builtins.open', mockOpen): cardinality.load_files() self.cardinality = cardinality worker_single = Worker_single(cardinality=self.cardinality, lock=self.lock) worker_single = Worker_single(cardinality=self.cardinality, lock=self.lock, stoppingcondition="timer") worker_single = Worker_single(cardinality=self.cardinality, lock=self.lock, stoppingcondition="earlystopping") worker_single = Worker_single(cardinality=self.cardinality, lock=self.lock, stoppingcondition="epoch") with self.assertRaises(Exception): worker_single = Worker_single(cardinality=self.cardinality, lock=self.lock, stoppingcondition="unimplemented") tf.close()
def test_create(self, mock_isfile): mock_isfile.return_value = True cardinality = Cardinality(3, "", "") read_data = pickle.dumps({ 'word2vec': { "1": [1] * 20, "2": [2] * 20, "3": [3] * 20, "4": [4] * 20, "5": [5] * 20, "6": [6] * 20, "7": [7] * 20 }, 'counter_patterns': { 1: 10, 2: 100, 3: 100, 4: 100, 6: 1000, 5: 1000 } }) mockOpen = mock_open(read_data=read_data) tf = tempfile.NamedTemporaryFile() f = h5py.File(tf, 'w') f.create_dataset("list_classes", data=[ 1, 1, 1, 2, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 4, 5, 6 ]) f.close() cardinality.path_list_classes = tf.name cardinality.counter = {1: 10, 2: 100, 3: 100, 4: 100, 6: 1000, 5: 1000} # Test with patch('builtins.open', mockOpen): #cardinality.load_files() worker_single = Worker_single(cardinality=cardinality, lock=self.lock, batch_size=1) dataloader = worker_single.create_dataloader() self.assertEqual(len(dataloader), 7) # 60% of 12 self.assertTrue(worker_single.dataset.loaded) # Train # Avoid to create a new object worker_single.dataset.loaded = False with patch('builtins.open', mockOpen): #cardinality.load_files() worker_single = Worker_single(cardinality=cardinality, lock=self.lock, batch_size=1) dataloader = worker_single.create_dataloader(condition="Train") self.assertEqual(len(dataloader), 5) # 40% of 12 worker_single.dataset.loaded = False with patch('builtins.open', mockOpen): #cardinality.load_files() worker_single = Worker_single(cardinality=cardinality, lock=self.lock, batch_size=1) dataloader = worker_single.create_dataloader(condition="Train", subsample=True, subsample_split=0.25) self.assertEqual(len(dataloader), 1) # 25% of 40% of 12 tf.close()