def setUp(self): """ Setup test. """ data = fetch_cifar(datasetdir="/tmp/cifar") self.manager = DataManager(input_path=data.input_path, labels=["label"], metadata_path=data.metadata_path, number_of_folds=10, batch_size=10, stratify_label="category", test_size=0.1, sample_size=0.01) class Net(nn.Module): def __init__(self): super(Net, self).__init__() self.conv1 = nn.Conv2d(3, 6, 5) self.pool = nn.MaxPool2d(2, 2) self.conv2 = nn.Conv2d(6, 16, 5) self.fc1 = nn.Linear(16 * 5 * 5, 120) self.fc2 = nn.Linear(120, 84) self.fc3 = nn.Linear(84, 10) def forward(self, x): x = self.pool(func.relu(self.conv1(x))) x = self.pool(func.relu(self.conv2(x))) x = x.view(-1, 16 * 5 * 5) x = func.relu(self.fc1(x)) x = func.relu(self.fc2(x)) x = self.fc3(x) return x self.cl = DeepLearningInterface(model=Net(), optimizer_name="SGD", momentum=0.9, learning_rate=0.001, loss_name="CrossEntropyLoss", metrics=["accuracy"])
from pynet.history import History from pynet.losses import MSELoss, NCCLoss, RCNetLoss, PCCLoss from pynet.plotting import Board, update_board import matplotlib.pyplot as plt setup_logging(level="debug") logger = logging.getLogger("pynet") losses = pynet.get_tools(tool_name="losses") outdir = "/neurospin/nsap/tmp/registration" data = fetch_registration(datasetdir=outdir) manager = DataManager(input_path=data.input_path, metadata_path=data.metadata_path, number_of_folds=2, batch_size=8, sampler="random", stratify_label="studies", projection_labels={"studies": ["abide"]}, test_size=0.1, add_input=True, sample_size=0.1) ############################################################################# # Training # -------- # # From the available models load the VoxelMorphRegister, VTNetRegister or # ADDNet and start the training. # Note that the two first estimate a non linear deformation and require # the input data to be afinely registered. The ADDNet estimate an affine # transform. We will see in the next section how to combine them in an # efficient way.
# neural network is evaluated on the validation set, but not trained on it. # If the validation loss starts to grow, it means that the network is # overfitting the training set, and that it is time to stop the training. # # The following cell create stratified test, train, and validation loaders. from pynet.datasets import fetch_orientation from pynet.datasets import DataManager data = fetch_orientation( datasetdir="/tmp/orientation", flatten=True) manager = DataManager( input_path=data.input_path, labels=["label"], metadata_path=data.metadata_path, number_of_folds=10, batch_size=1000, stratify_label="label", test_size=0.1) ############################################################################# # Displaying some images of the test dataset. from pynet.plotting import plot_data dataset = manager["test"] sample = dataset.inputs.reshape(-1, data.height, data.width) sample = np.expand_dims(sample, axis=1) plot_data(sample, nb_samples=5)
from pynet.plotting import Board, update_board ############################################################################# # The model will be trained on MNIST - handwritten digits dataset. The input # is an image in R(28×28). def flatten(arr): return arr.flatten() data = fetch_minst(datasetdir="/neurospin/nsap/datasets/minst") manager = DataManager( input_path=data.input_path, metadata_path=data.metadata_path, stratify_label="label", number_of_folds=10, batch_size=64, test_size=0, input_transforms=[flatten], add_input=True, sample_size=0.05) ############################################################################# # The Model # --------- # # The model is composed of two sub-networks: # # 1. Given x (image), encode it into a distribution over the latent space - # referred to as Q(z|x). # 2. Given z in latent space (code representation of an image), decode it into
the activation map of the last convolutional layer in our model. Load the data ------------- Load some images and apply the ImageNet transformation. You may need to change the 'datasetdir' parameter. """ from pynet.datasets import DataManager, fetch_gradcam from pynet.plotting import plot_data data = fetch_gradcam(datasetdir="/tmp/gradcam") manager = DataManager(input_path=data.input_path, metadata_path=data.metadata_path, number_of_folds=2, batch_size=5, test_size=1) dataset = manager["test"] print(dataset.inputs.shape) plot_data(dataset.inputs, nb_samples=5, random=False, rgb=True) ############################################################################# # Explore different architectures # ------------------------------- # # Let's automate this procedure for different networks. # We need to reload the data for the inception network. # You may need to change the 'datasetdir' parameter. import os
if not isinstance(imgtype, list): imgtype = [imgtype] imgtype = [typemap[key] for key in imgtype] transformed_data = [] for channel_id in range(len(data)): if channel_id not in imgtype: continue arr = data[channel_id] transformed_data.append(downsample(arr, scale=3)) return np.asarray(transformed_data) manager = DataManager(input_path=data.input_path, metadata_path=data.metadata_path, stratify_label="grade", number_of_folds=10, batch_size=batch_size, test_size=0, input_transforms=[transformer], sample_size=0.2) ######################## # Loss # ---- def calc_gradient_penalty(model, x, x_gen, w=10): """ WGAN-GP gradient penalty. """ assert (x.size() == x_gen.size()), "Real and sampled sizes do not match." alpha_size = tuple((len(x), *(1, ) * (x.dim() - 1))) alpha_t = torch.cuda.FloatTensor if x.is_cuda else torch.Tensor
# # Use the fetcher of the pynet package. from pynet.datasets import DataManager, fetch_brats from pynet.plotting import plot_data from pynet.transforms import RandomFlipDimensions, Offset data = fetch_brats(datasetdir="/neurospin/nsap/datasets/brats") manager = DataManager( input_path=data.input_path, metadata_path=data.metadata_path, output_path=data.output_path, projection_labels=None, number_of_folds=10, batch_size=1, stratify_label="grade", #input_transforms=[ # RandomFlipDimensions(ndims=3, proba=0.5, with_channels=True), # Offset(nb_channels=4, factor=0.1)], sampler="random", add_input=True, test_size=0.1, pin_memory=True) dataset = manager["test"][:1] print(dataset.inputs.shape, dataset.outputs.shape) plot_data(dataset.inputs, channel=1, nb_samples=5) plot_data(dataset.outputs, channel=1, nb_samples=5) ############################################################################# # Training # --------
# You may need to change the 'datasetdir' parameter. import os import numpy as np from pynet.datasets import DataManager, fetch_echocardiography from pynet.plotting import plot_data from pynet.utils import setup_logging setup_logging(level="info") data = fetch_echocardiography(datasetdir="/tmp/echocardiography") manager = DataManager(input_path=data.input_path, metadata_path=data.metadata_path, output_path=data.output_path, number_of_folds=2, stratify_label="label", sampler="random", batch_size=10, test_size=0.1, sample_size=0.2) dataset = manager["test"] print(dataset.inputs.shape, dataset.outputs.shape) data = np.concatenate((dataset.inputs, dataset.outputs), axis=1) plot_data(data, nb_samples=5) ############################################################################# # Optimisation # ------------ # # From the available models load the UNet, and start the training. # You may need to change the 'outdir' parameter.
from pynet.plotting import plot_history from pynet.history import History from pynet.losses import MSELoss, NCCLoss, RCNetLoss import matplotlib.pyplot as plt setup_logging(level="debug") logger = logging.getLogger("pynet") outdir = "/neurospin/nsap/tmp/registration" data = fetch_registration( datasetdir=outdir) manager = DataManager( input_path=data.input_path, metadata_path=data.metadata_path, number_of_folds=10, batch_size=1, sampler="random", #stratify_label="centers", test_size=0.1, add_input=True, sample_size=1) ############################################################################# # Training # -------- # # From the available models load the VoxelMorphRegister, VTNetRegister or # ADDNet and start the training. # Note that the two first estimate a non linear deformation and require # the input data to be afinely registered. The ADDNet estimate an affine # transform. We will see in the next section how to combine them in an # efficient way.
Load some data. You may need to change the 'datasetdir' parameter. """ import os from pynet.datasets import DataManager, fetch_genomic_pred from pynet.utils import setup_logging setup_logging(level="info") data = fetch_genomic_pred(datasetdir="/tmp/genomic_pred") manager = DataManager(input_path=data.input_path, labels=["env0"], metadata_path=data.metadata_path, number_of_folds=2, batch_size=5, test_size=0.2, continuous_labels=True) ############################################################################# # Basic inspection import numpy as np from sklearn.decomposition import PCA import matplotlib.pyplot as plt train_dataset = manager["train"][0] X_train = train_dataset.inputs[train_dataset.indices] y_train = train_dataset.labels[train_dataset.indices] test_dataset = manager["test"]
import os import numpy as np from pynet.datasets import DataManager, fetch_echocardiography from pynet.plotting import plot_data from pynet.utils import setup_logging setup_logging(level="info") data = fetch_echocardiography( datasetdir="/tmp/echocardiography") manager = DataManager( input_path=data.input_path, metadata_path=data.metadata_path, output_path=data.output_path, number_of_folds=2, stratify_label="label", sampler="weighted_random", batch_size=10, test_size=0.1, sample_size=(1 if "CI_MODE" not in os.environ else 0.05)) dataset = manager["test"] print(dataset.inputs.shape, dataset.outputs.shape) data = np.concatenate((dataset.inputs, dataset.outputs), axis=1) plot_data(data, nb_samples=5) ############################################################################# # Optimisation # ------------ # # From the available models load the UNet, and start the training.
# # A validation step is a useful way to avoid overfitting. At each epoch, the # neural network is evaluated on the validation set, but not trained on it. # If the validation loss starts to grow, it means that the network is # overfitting the training set, and that it is time to stop the training. # # The following cell create stratified test, train, and validation loaders. from pynet.datasets import fetch_orientation from pynet.datasets import DataManager data = fetch_orientation(datasetdir="/tmp/orientation", flatten=True) manager = DataManager(input_path=data.input_path, labels=["label"], metadata_path=data.metadata_path, number_of_folds=10, batch_size=1000, stratify_label="label", test_size=0.1, sample_size=(1 if "CI_MODE" not in os.environ else 0.1)) ############################################################################# # Displaying some images of the test dataset. from pynet.plotting import plot_data dataset = manager["test"] sample = dataset.inputs.reshape(-1, data.height, data.width) sample = np.expand_dims(sample, axis=1) plot_data(sample, nb_samples=5) #############################################################################
import matplotlib.pyplot as plt setup_logging(level="info") logger = logging.getLogger("pynet") use_toy = False dtype = "all" data = fetch_impac(datasetdir="/neurospin/nsap/datasets/impac", mode="train", dtype=dtype) nb_features = data.nb_features manager = DataManager(input_path=data.input_path, labels=["participants_asd"], metadata_path=data.metadata_path, number_of_folds=3, batch_size=128, sampler="random", test_size=2, sample_size=1) if use_toy: toy_data = {} nb_features = 50 for name, nb_samples in (("train", 1000), ("test", 2)): x1 = torch.randn(nb_samples, 50) x2 = torch.randn(nb_samples, 50) + 1.5 x = torch.cat([x1, x2], dim=0) y1 = torch.zeros(nb_samples, 1) y2 = torch.ones(nb_samples, 1) y = torch.cat([y1, y2], dim=0) toy_data[name] = (x, y)
""" ############################################################################# # Import the dataset # ------------------ # # You may need to change the 'datasetdir' parameter. import numpy as np from pynet.datasets import DataManager, fetch_echocardiography from pynet.plotting import plot_data data = fetch_echocardiography(datasetdir="/tmp/echocardiography") manager = DataManager(input_path=data.input_path, metadata_path=data.metadata_path, output_path=data.output_path, number_of_folds=10, batch_size=10, test_size=0.1) dataset = manager["test"] data = np.concatenate((dataset.inputs, dataset.outputs), axis=1) plot_data(data, nb_samples=5) ############################################################################# # Optimisation # ------------ # # From the available models load the UNet, and start the training. # You may need to change the 'outdir' parameter. import os import torch
board = Board(port=8097, host="http://localhost", env="data-augmentation") compose_transforms = Transformer() compose_transforms.register(flip, probability=0.5, axis=0, apply_to=["input", "output"]) compose_transforms.register(add_blur, probability=1, sigma=4, apply_to=["input"]) manager = DataManager(input_path=data.input_path, metadata_path=data.metadata_path, output_path=data.output_path, number_of_folds=2, batch_size=2, test_size=0.1, sample_size=0.1, sampler=None, add_input=True, data_augmentation_transforms=[compose_transforms]) loaders = manager.get_dataloader(train=True, validation=False, fold_index=0) for dataitem in loaders.train: print("-" * 50) print(dataitem.inputs.shape, dataitem.outputs.shape, dataitem.labels) images = [ dataitem.inputs[0, 0].numpy(), dataitem.inputs[0, 1].numpy(), dataitem.outputs[0, 0].numpy(), dataitem.outputs[0, 1].numpy(), dataitem.outputs[0, 4].numpy(), dataitem.outputs[0, 5].numpy() ] images = np.asarray(images) images = np.expand_dims(images, axis=1)
import os import sys from pynet.datasets import DataManager, fetch_height_biobank from pynet.utils import setup_logging # This example cannot run in CI : it accesses NS intra filesystems if "CI_MODE" in os.environ: sys.exit(0) setup_logging(level="info") data = fetch_height_biobank(datasetdir="/neurospin/tmp/height_bb") manager = DataManager(input_path=data.input_path, labels=["Height"], metadata_path=data.metadata_path, number_of_folds=2, batch_size=5, test_size=0.2, continuous_labels=True) ############################################################################# # Basic inspection import numpy as np import matplotlib.pyplot as plt train_dataset = manager["train"][0] X_train = train_dataset.inputs[train_dataset.indices] y_train = train_dataset.labels[train_dataset.indices] test_dataset = manager["test"] X_test = test_dataset.inputs[test_dataset.indices]