def test_actions(self): tempdir = tempfile.mkdtemp() url = "https://www.dropbox.com/s/5wwskxctvcxiuea/MedNIST.tar.gz?dl=1" filepath = os.path.join(tempdir, "MedNIST.tar.gz") output_dir = tempdir md5_value = "0bc7306e7427e00ad1c5526a6677552d" try: download_and_extract(url, filepath, output_dir, md5_value) download_and_extract(url, filepath, output_dir, md5_value) except (ContentTooShortError, HTTPError): pass # ignore remote errors in this test wrong_md5 = "0" try: download_url(url, filepath, wrong_md5) except RuntimeError as e: self.assertTrue(str(e).startswith("MD5 check")) shutil.rmtree(os.path.join(tempdir, "MedNIST")) try: extractall(filepath, output_dir, wrong_md5) except RuntimeError as e: self.assertTrue(str(e).startswith("MD5 check")) shutil.rmtree(tempdir)
def test_actions(self): testing_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "testing_data") url = "https://drive.google.com/uc?id=13MhoPsNgI6qboJfLicFf_jNvsFUbIYsd" filepath = os.path.join(testing_dir, "MedNIST.tar.gz") output_dir = testing_dir md5_value = "0bc7306e7427e00ad1c5526a6677552d" try: download_and_extract(url, filepath, output_dir, md5_value) download_and_extract(url, filepath, output_dir, md5_value) except (ContentTooShortError, HTTPError, RuntimeError) as e: print(str(e)) if isinstance(e, RuntimeError): # FIXME: skip MD5 check as current downloading method may fail self.assertTrue(str(e).startswith("md5 check")) return # skipping this test due the network connection errors wrong_md5 = "0" try: download_url(url, filepath, wrong_md5) except (ContentTooShortError, HTTPError, RuntimeError) as e: print(str(e)) if isinstance(e, RuntimeError): # FIXME: skip MD5 check as current downloading method may fail self.assertTrue(str(e).startswith("md5 check")) return # skipping this test due the network connection errors try: extractall(filepath, output_dir, wrong_md5) except RuntimeError as e: self.assertTrue(str(e).startswith("md5 check"))
def get_data(keys): """Get the example data to be used. Use MarsAtlas as it only contains 1 image for quick download and that image is parcellated. """ cache_dir = os.environ.get("MONAI_DATA_DIRECTORY") or tempfile.mkdtemp() fname = "MarsAtlas-MNI-Colin27.zip" url = "https://www.dropbox.com/s/ndz8qtqblkciole/" + fname + "?dl=1" out_path = os.path.join(cache_dir, "MarsAtlas-MNI-Colin27") zip_path = os.path.join(cache_dir, fname) download_and_extract(url, zip_path, out_path) image, label = sorted(glob(os.path.join(out_path, "*.nii"))) data = {CommonKeys.IMAGE: image, CommonKeys.LABEL: label} transforms = Compose([ LoadImaged(keys), AddChanneld(keys), ScaleIntensityd(CommonKeys.IMAGE), Rotate90d(keys, spatial_axes=[0, 2]) ]) data = transforms(data) max_size = max(data[keys[0]].shape) padder = SpatialPadd(keys, (max_size, max_size, max_size)) return padder(data)
def _download_dcmqi_tools(self): target = os.path.join(self.app_dir, "bin") os.makedirs(target, exist_ok=True) dcmqi_tools = [ "segimage2itkimage", "itkimage2segimage", "segimage2itkimage.exe", "itkimage2segimage.exe" ] existing = [ tool for tool in dcmqi_tools if shutil.which(tool) or os.path.exists(os.path.join(target, tool)) ] logger.debug(f"Existing Tools: {existing}") if len(existing) in [len(dcmqi_tools), len(dcmqi_tools) // 2]: logger.debug("No need to download dcmqi tools") return target_os = "win64.zip" if any( platform.win32_ver()) else "linux.tar.gz" with tempfile.TemporaryDirectory() as tmp: download_and_extract( url= f"https://github.com/QIICR/dcmqi/releases/download/v1.2.4/dcmqi-1.2.4-{target_os}", output_dir=tmp) for root, _, files in os.walk(tmp): for f in files: if f in dcmqi_tools: shutil.copy(os.path.join(root, f), target)
def test_actions(self): testing_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "testing_data") config_dict = testing_data_config("images", "mednist") url = config_dict["url"] filepath = Path(testing_dir) / "MedNIST.tar.gz" output_dir = Path(testing_dir) hash_val, hash_type = config_dict["hash_val"], config_dict["hash_type"] with skip_if_downloading_fails(): download_and_extract(url, filepath, output_dir, hash_val=hash_val, hash_type=hash_type) download_and_extract(url, filepath, output_dir, hash_val=hash_val, hash_type=hash_type) wrong_md5 = "0" with self.assertLogs(logger="monai.apps", level="ERROR"): try: download_url(url, filepath, wrong_md5) except (ContentTooShortError, HTTPError, RuntimeError) as e: if isinstance(e, RuntimeError): # FIXME: skip MD5 check as current downloading method may fail self.assertTrue(str(e).startswith("md5 check")) return # skipping this test due the network connection errors try: extractall(filepath, output_dir, wrong_md5) except RuntimeError as e: self.assertTrue(str(e).startswith("md5 check"))
def test_default(self, key, file_type): with tempfile.TemporaryDirectory() as tmp_dir: with skip_if_downloading_fails(): img_spec = testing_data_config("images", key) download_and_extract( img_spec["url"], output_dir=tmp_dir, hash_val=img_spec["hash_val"], hash_type=img_spec["hash_type"], file_type=file_type, )
def setUp(self): set_determinism(seed=0) self.data_dir = os.path.join( os.path.dirname(os.path.realpath(__file__)), "testing_data") data_dir = os.path.join(self.data_dir, "MedNIST") dataset_file = os.path.join(self.data_dir, "MedNIST.tar.gz") if not os.path.exists(data_dir): try: download_and_extract(TEST_DATA_URL, dataset_file, self.data_dir, MD5_VALUE) except (ContentTooShortError, HTTPError, RuntimeError) as e: print(str(e)) if isinstance(e, RuntimeError): # FIXME: skip MD5 check as current downloading method may fail self.assertTrue(str(e).startswith("MD5 check")) return # skipping this test due the network connection errors assert os.path.exists(data_dir) class_names = sorted((x for x in os.listdir(data_dir) if os.path.isdir(os.path.join(data_dir, x)))) image_files = [[ os.path.join(data_dir, class_name, x) for x in sorted(os.listdir(os.path.join(data_dir, class_name))) ] for class_name in class_names] image_file_list, image_classes = [], [] for i, _ in enumerate(class_names): image_file_list.extend(image_files[i]) image_classes.extend([i] * len(image_files[i])) # split train, val, test valid_frac, test_frac = 0.1, 0.1 self.train_x, self.train_y = [], [] self.val_x, self.val_y = [], [] self.test_x, self.test_y = [], [] for i in range(len(image_classes)): rann = np.random.random() if rann < valid_frac: self.val_x.append(image_file_list[i]) self.val_y.append(image_classes[i]) elif rann < test_frac + valid_frac: self.test_x.append(image_file_list[i]) self.test_y.append(image_classes[i]) else: self.train_x.append(image_file_list[i]) self.train_y.append(image_classes[i]) self.device = torch.device( "cuda:0" if torch.cuda.is_available() else "cpu:0")
def setUp(self): set_determinism(seed=0) self.data_dir = os.path.join( os.path.dirname(os.path.realpath(__file__)), "testing_data") data_dir = os.path.join(self.data_dir, "MedNIST") dataset_file = os.path.join(self.data_dir, "MedNIST.tar.gz") if not os.path.exists(data_dir): with skip_if_downloading_fails(): data_spec = testing_data_config("images", "mednist") download_and_extract( data_spec["url"], dataset_file, self.data_dir, hash_val=data_spec["hash_val"], hash_type=data_spec["hash_type"], ) assert os.path.exists(data_dir) class_names = sorted(x for x in os.listdir(data_dir) if os.path.isdir(os.path.join(data_dir, x))) image_files = [[ os.path.join(data_dir, class_name, x) for x in sorted(os.listdir(os.path.join(data_dir, class_name))) ] for class_name in class_names] image_file_list, image_classes = [], [] for i, _ in enumerate(class_names): image_file_list.extend(image_files[i]) image_classes.extend([i] * len(image_files[i])) # split train, val, test valid_frac, test_frac = 0.1, 0.1 self.train_x, self.train_y = [], [] self.val_x, self.val_y = [], [] self.test_x, self.test_y = [], [] for i in range(len(image_classes)): rann = np.random.random() if rann < valid_frac: self.val_x.append(image_file_list[i]) self.val_y.append(image_classes[i]) elif rann < test_frac + valid_frac: self.test_x.append(image_file_list[i]) self.test_y.append(image_classes[i]) else: self.train_x.append(image_file_list[i]) self.train_y.append(image_classes[i]) self.device = "cuda:0" if torch.cuda.is_available() else "cpu:0"
def main(): parser = argparse.ArgumentParser(description="training") parser.add_argument( "--msd_task", action="store", default="Task07_Pancreas", help="msd task", ) parser.add_argument( "--root", action="store", default="./data_msd", help="data root", ) args = parser.parse_args() resource = "https://msd-for-monai.s3-us-west-2.amazonaws.com/" + args.msd_task + ".tar" compressed_file = os.path.join(args.root, args.msd_task + ".tar") if not os.path.exists(args.root): download_and_extract(resource, compressed_file, args.root)
def test_default(self): with tempfile.TemporaryDirectory() as tmp_dir: try: # icon.tar.gz https://drive.google.com/file/d/1aHztVYbynCCq-byTf6cRI8ViETpBWHyC/view?usp=sharing download_and_extract( "https://drive.google.com/uc?id=1aHztVYbynCCq-byTf6cRI8ViETpBWHyC", output_dir=tmp_dir, hash_val="a55d11ad26ed9eb7277905d796205531", file_type="tar", ) # favicon.ico.zip https://drive.google.com/file/d/1YGa8N3LBddOkxBnnLndGicoW0klsAH6D/view?usp=sharing download_and_extract( "https://drive.google.com/uc?id=1YGa8N3LBddOkxBnnLndGicoW0klsAH6D", output_dir=tmp_dir, hash_val="ac6e167ee40803577d98237f2b0241e5", file_type="zip", ) except (ContentTooShortError, HTTPError, RuntimeError) as e: print(str(e)) if isinstance(e, RuntimeError): # FIXME: skip MD5 check as current downloading method may fail self.assertTrue(str(e).startswith("md5 check")) return # skipping this test due the network connection errors
def test_default(self): with tempfile.TemporaryDirectory() as tmp_dir: try: # icon.tar.gz https://drive.google.com/file/d/1HrQd-AKPbts9jkTNN4pT8vLZyhM5irVn/view?usp=sharing download_and_extract( "https://drive.google.com/uc?id=1HrQd-AKPbts9jkTNN4pT8vLZyhM5irVn", output_dir=tmp_dir, hash_val="a55d11ad26ed9eb7277905d796205531", file_type="tar", ) # favicon.ico.zip https://drive.google.com/file/d/1TqBTJap621NO9arzXRrYi04lr9NTVF8H/view?usp=sharing download_and_extract( "https://drive.google.com/uc?id=1TqBTJap621NO9arzXRrYi04lr9NTVF8H", output_dir=tmp_dir, hash_val="ac6e167ee40803577d98237f2b0241e5", file_type="zip", ) except (ContentTooShortError, HTTPError, RuntimeError) as e: print(str(e)) if isinstance(e, RuntimeError): # FIXME: skip MD5 check as current downloading method may fail self.assertTrue(str(e).startswith("md5 check")) return # skipping this test due the network connection errors
from monai.utils.nvtx import Range nvidia_dlprof_pytorch_nvtx.init() # set directories random.seed(0) root_dir = "/tmp/tmp" + ''.join(random.choice(string.ascii_lowercase) for i in range(16)) print(f"root dir is: {root_dir}") resource = "https://msd-for-monai.s3-us-west-2.amazonaws.com/Task09_Spleen.tar" md5 = "410d4a301da4e5b2f6f86ec3ddba524e" compressed_file = os.path.join(root_dir, "Task09_Spleen.tar") data_root = os.path.join(root_dir, "Task09_Spleen") if not os.path.exists(data_root): download_and_extract(resource, compressed_file, root_dir, md5) out_dir = "./outputs_fast" train_images = sorted(glob.glob(os.path.join(data_root, "imagesTr", "*.nii.gz"))) train_labels = sorted(glob.glob(os.path.join(data_root, "labelsTr", "*.nii.gz"))) data_dicts = [ {"image": image_name, "label": label_name} for image_name, label_name in zip(train_images, train_labels) ] train_files, val_files = data_dicts[:-9], data_dicts[-9:] set_determinism(seed=0) train_transforms = Compose( [