def train_and_infer(self, idx=0): results = [] set_determinism(0) losses, best_metric, best_metric_epoch = run_training_test(self.data_dir, device=self.device, cachedataset=idx) infer_metric = run_inference_test(self.data_dir, device=self.device) # check training properties print("losses", losses) print("best metric", best_metric) print("infer metric", infer_metric) self.assertTrue(test_integration_value(TASK, key="losses", data=losses, rtol=1e-3)) self.assertTrue(test_integration_value(TASK, key="best_metric", data=best_metric, rtol=1e-2)) self.assertTrue(len(glob(os.path.join(self.data_dir, "runs"))) > 0) model_file = os.path.join(self.data_dir, "best_metric_model.pth") self.assertTrue(os.path.exists(model_file)) # check inference properties self.assertTrue(test_integration_value(TASK, key="infer_metric", data=infer_metric, rtol=1e-2)) output_files = sorted(glob(os.path.join(self.data_dir, "output", "img*", "*.nii.gz"))) print([np.mean(nib.load(output).get_fdata()) for output in output_files]) results.extend(losses) results.append(best_metric) results.append(infer_metric) for output in output_files: ave = np.mean(nib.load(output).get_fdata()) results.append(ave) self.assertTrue(test_integration_value(TASK, key="output_sums", data=results[8:], rtol=1e-2)) return results
def train_and_infer(self, idx=0): results = [] if not os.path.exists(os.path.join(self.data_dir, "MedNIST")): # skip test if no MedNIST dataset return results set_determinism(seed=0) losses, best_metric, best_metric_epoch = run_training_test( self.data_dir, self.train_x, self.train_y, self.val_x, self.val_y, device=self.device ) infer_metric = run_inference_test(self.data_dir, self.test_x, self.test_y, device=self.device) print(f"integration_classification_2d {losses}") print("best metric", best_metric) print("infer metric", infer_metric) # check training properties self.assertTrue(test_integration_value(TASK, key="losses", data=losses, rtol=1e-2)) self.assertTrue(test_integration_value(TASK, key="best_metric", data=best_metric, rtol=1e-4)) np.testing.assert_allclose(best_metric_epoch, 4) model_file = os.path.join(self.data_dir, "best_metric_model.pth") self.assertTrue(os.path.exists(model_file)) # check inference properties self.assertTrue(test_integration_value(TASK, key="infer_prop", data=np.asarray(infer_metric), rtol=1)) results.extend(losses) results.append(best_metric) results.extend(infer_metric) return results
def test_training(self): repeated = [] for i in range(3): set_determinism(0) repeated.append([]) losses, best_metric, best_metric_epoch = run_training_test( self.data_dir, device=self.device, cachedataset=(i == 2) ) # check training properties print("losses", losses) self.assertTrue(test_integration_value(TASK, key="losses", data=losses, rtol=1e-3)) repeated[i].extend(losses) print("best metric", best_metric) self.assertTrue(test_integration_value(TASK, key="best_metric", data=best_metric, rtol=1e-2)) repeated[i].append(best_metric) self.assertTrue(len(glob(os.path.join(self.data_dir, "runs"))) > 0) model_file = os.path.join(self.data_dir, "best_metric_model.pth") self.assertTrue(os.path.exists(model_file)) infer_metric = run_inference_test(self.data_dir, device=self.device) # check inference properties print("infer metric", infer_metric) self.assertTrue(test_integration_value(TASK, key="infer_metric", data=infer_metric, rtol=1e-2)) repeated[i].append(infer_metric) output_files = sorted(glob(os.path.join(self.data_dir, "output", "img*", "*.nii.gz"))) print([np.mean(nib.load(output).get_fdata()) for output in output_files]) for output in output_files: ave = np.mean(nib.load(output).get_fdata()) repeated[i].append(ave) self.assertTrue(test_integration_value(TASK, key="output_sums", data=repeated[i][8:], rtol=1e-2)) np.testing.assert_allclose(repeated[0], repeated[1]) np.testing.assert_allclose(repeated[0], repeated[2])
def _test_saved_files(postfix): output_files = sorted(glob(os.path.join(self.data_dir, "img*", f"*{postfix}.nii.gz"))) values = [] for output in output_files: ave = np.mean(nib.load(output).get_fdata()) values.append(ave) if idx == 2: self.assertTrue(test_integration_value(TASK, key="output_sums_2", data=values, rtol=1e-2)) else: self.assertTrue(test_integration_value(TASK, key="output_sums", data=values, rtol=1e-2))
def train_and_infer(self, idx=0): results = [] set_determinism(seed=0) best_metric = run_training_test(self.data_dir, device=self.device, amp=(idx == 2)) model_file = sorted(glob(os.path.join(self.data_dir, "net_key_metric*.pt")))[-1] infer_metric = run_inference_test(self.data_dir, model_file, device=self.device, amp=(idx == 2)) print("best metric", best_metric) print("infer metric", infer_metric) if idx == 2: self.assertTrue(test_integration_value(TASK, key="best_metric_2", data=best_metric, rtol=1e-2)) else: self.assertTrue(test_integration_value(TASK, key="best_metric", data=best_metric, rtol=1e-2)) # check inference properties if idx == 2: self.assertTrue(test_integration_value(TASK, key="infer_metric_2", data=infer_metric, rtol=1e-2)) else: self.assertTrue(test_integration_value(TASK, key="infer_metric", data=infer_metric, rtol=1e-2)) results.append(best_metric) results.append(infer_metric) def _test_saved_files(postfix): output_files = sorted(glob(os.path.join(self.data_dir, "img*", f"*{postfix}.nii.gz"))) values = [] for output in output_files: ave = np.mean(nib.load(output).get_fdata()) values.append(ave) if idx == 2: self.assertTrue(test_integration_value(TASK, key="output_sums_2", data=values, rtol=1e-2)) else: self.assertTrue(test_integration_value(TASK, key="output_sums", data=values, rtol=1e-2)) _test_saved_files(postfix="seg_handler") _test_saved_files(postfix="seg_transform") try: os.remove(model_file) except Exception as e: warnings.warn(f"Fail to remove {model_file}: {e}.") if torch.cuda.is_available(): try: torch.cuda.empty_cache() except Exception: pass return results
def test_training(self): repeated = [] test_rounds = 3 if monai.config.get_torch_version_tuple() >= (1, 6) else 2 for i in range(test_rounds): set_determinism(seed=0) repeated.append([]) best_metric = run_training_test(self.data_dir, device=self.device, amp=(i == 2)) print("best metric", best_metric) if i == 2: self.assertTrue( test_integration_value(TASK, key="best_metric_2", data=best_metric, rtol=1e-2)) else: self.assertTrue( test_integration_value(TASK, key="best_metric", data=best_metric, rtol=1e-2)) repeated[i].append(best_metric) model_file = sorted( glob(os.path.join(self.data_dir, "net_key_metric*.pt")))[-1] infer_metric = run_inference_test(self.data_dir, model_file, device=self.device, amp=(i == 2)) print("infer metric", infer_metric) # check inference properties if i == 2: self.assertTrue( test_integration_value(TASK, key="infer_metric_2", data=infer_metric, rtol=1e-2)) else: self.assertTrue( test_integration_value(TASK, key="infer_metric", data=infer_metric, rtol=1e-2)) repeated[i].append(infer_metric) output_files = sorted( glob(os.path.join(self.data_dir, "img*", "*.nii.gz"))) for output in output_files: ave = np.mean(nib.load(output).get_fdata()) repeated[i].append(ave) if i == 2: self.assertTrue( test_integration_value(TASK, key="output_sums_2", data=repeated[i][2:], rtol=1e-2)) else: self.assertTrue( test_integration_value(TASK, key="output_sums", data=repeated[i][2:], rtol=1e-2)) np.testing.assert_allclose(repeated[0], repeated[1])