Example #1
0
def test_dti_xval():
    """
    Test k-fold cross-validation
    """
    data = nib.load(fdata).get_data()
    gtab = gt.gradient_table(fbval, fbvec)
    dm = dti.TensorModel(gtab, 'LS')
    # The data has 102 directions, so will not divide neatly into 10 bits
    npt.assert_raises(ValueError, xval.kfold_xval, dm, data, 10)

    # In simulation with no noise, COD should be perfect:
    psphere = dpd.get_sphere('symmetric362')
    bvecs = np.concatenate(([[0, 0, 0]], psphere.vertices))
    bvals = np.zeros(len(bvecs)) + 1000
    bvals[0] = 0
    gtab = gt.gradient_table(bvals, bvecs)
    mevals = np.array(([0.0015, 0.0003, 0.0001], [0.0015, 0.0003, 0.0003]))
    mevecs = [
        np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]]),
        np.array([[0, 0, 1], [0, 1, 0], [1, 0, 0]])
    ]
    S = sims.single_tensor(gtab, 100, mevals[0], mevecs[0], snr=None)

    dm = dti.TensorModel(gtab, 'LS')
    kf_xval = xval.kfold_xval(dm, S, 2)
    cod = xval.coeff_of_determination(S, kf_xval)
    npt.assert_array_almost_equal(cod, np.ones(kf_xval.shape[:-1]) * 100)

    # Test with 2D data for use of a mask
    S = np.array([[S, S], [S, S]])
    mask = np.ones(S.shape[:-1], dtype=bool)
    mask[1, 1] = 0
    kf_xval = xval.kfold_xval(dm, S, 2, mask=mask)
    cod2d = xval.coeff_of_determination(S, kf_xval)
    npt.assert_array_almost_equal(np.round(cod2d[0, 0]), cod)
Example #2
0
def test_dti_xval():
    """
    Test k-fold cross-validation
    """
    data = nib.load(fdata).get_data()
    gtab = gt.gradient_table(fbval, fbvec)
    dm = dti.TensorModel(gtab, "LS")
    # The data has 102 directions, so will not divide neatly into 10 bits
    npt.assert_raises(ValueError, xval.kfold_xval, dm, data, 10)

    # But we can do this with 2 folds:
    kf_xval = xval.kfold_xval(dm, data, 2)

    # In simulation with no noise, COD should be perfect:
    psphere = dpd.get_sphere("symmetric362")
    bvecs = np.concatenate(([[0, 0, 0]], psphere.vertices))
    bvals = np.zeros(len(bvecs)) + 1000
    bvals[0] = 0
    gtab = gt.gradient_table(bvals, bvecs)
    mevals = np.array(([0.0015, 0.0003, 0.0001], [0.0015, 0.0003, 0.0003]))
    mevecs = [np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]]), np.array([[0, 0, 1], [0, 1, 0], [1, 0, 0]])]
    S = sims.single_tensor(gtab, 100, mevals[0], mevecs[0], snr=None)

    dm = dti.TensorModel(gtab, "LS")
    kf_xval = xval.kfold_xval(dm, S, 2)
    cod = xval.coeff_of_determination(S, kf_xval)
    npt.assert_array_almost_equal(cod, np.ones(kf_xval.shape[:-1]) * 100)

    # Test with 2D data for use of a mask
    S = np.array([[S, S], [S, S]])
    mask = np.ones(S.shape[:-1], dtype=bool)
    mask[1, 1] = 0
    kf_xval = xval.kfold_xval(dm, S, 2, mask=mask)
    cod2d = xval.coeff_of_determination(S, kf_xval)
    npt.assert_array_almost_equal(np.round(cod2d[0, 0]), cod)
Example #3
0
def test_predict():
    SNR = 1000
    S0 = 100
    _, fbvals, fbvecs = dpd.get_data('small_64D')
    bvals = np.load(fbvals)
    bvecs = np.load(fbvecs)
    gtab = grad.gradient_table(bvals, bvecs)
    mevals = np.array(([0.0015, 0.0003, 0.0003], [0.0015, 0.0003, 0.0003]))
    angles = [(0, 0), (60, 0)]
    S, sticks = sims.multi_tensor(gtab,
                                  mevals,
                                  S0,
                                  angles=angles,
                                  fractions=[10, 90],
                                  snr=SNR)

    sfmodel = sfm.SparseFascicleModel(gtab, response=[0.0015, 0.0003, 0.0003])
    sffit = sfmodel.fit(S)
    pred = sffit.predict()
    npt.assert_(xval.coeff_of_determination(pred, S) > 97)

    # Should be possible to predict using a different gtab:
    new_gtab = grad.gradient_table(bvals[::2], bvecs[::2])
    new_pred = sffit.predict(new_gtab)
    npt.assert_(xval.coeff_of_determination(new_pred, S[::2]) > 97)
Example #4
0
def test_csd_xval():
    # First, let's see that it works with some data:
    data = load_nifti_data(fdata)[1:3, 1:3, 1:3]  # Make it *small*
    gtab = gt.gradient_table(fbval, fbvec)
    S0 = np.mean(data[..., gtab.b0s_mask])
    response = ([0.0015, 0.0003, 0.0001], S0)

    # In simulation, it should work rather well (high COD):
    psphere = dpd.get_sphere('symmetric362')
    bvecs = np.concatenate(([[0, 0, 0]], psphere.vertices))
    bvals = np.zeros(len(bvecs)) + 1000
    bvals[0] = 0
    gtab = gt.gradient_table(bvals, bvecs)
    mevals = np.array(([0.0015, 0.0003, 0.0001], [0.0015, 0.0003, 0.0003]))
    mevecs = [
        np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]]),
        np.array([[0, 0, 1], [0, 1, 0], [1, 0, 0]])
    ]
    S0 = 100
    S = sims.single_tensor(gtab, S0, mevals[0], mevecs[0], snr=None)
    with warnings.catch_warnings():
        warnings.filterwarnings("ignore",
                                message=descoteaux07_legacy_msg,
                                category=PendingDeprecationWarning)
        sm = csd.ConstrainedSphericalDeconvModel(gtab, response)
    np.random.seed(12345)
    response = ([0.0015, 0.0003, 0.0001], S0)
    with warnings.catch_warnings():
        warnings.filterwarnings("ignore",
                                message=descoteaux07_legacy_msg,
                                category=PendingDeprecationWarning)
        kf_xval = xval.kfold_xval(sm, S, 2, response, sh_order=2)
    # Because of the regularization, COD is not going to be perfect here:
    cod = xval.coeff_of_determination(S, kf_xval)
    # We'll just test for regressions:
    csd_cod = 97  # pre-computed by hand for this random seed

    # We're going to be really lenient here:
    npt.assert_array_almost_equal(np.round(cod), csd_cod)
    # Test for sD data with more than one voxel for use of a mask:
    S = np.array([[S, S], [S, S]])
    mask = np.ones(S.shape[:-1], dtype=bool)
    mask[1, 1] = 0
    with warnings.catch_warnings():
        warnings.filterwarnings("ignore",
                                message=descoteaux07_legacy_msg,
                                category=PendingDeprecationWarning)
        kf_xval = xval.kfold_xval(sm, S, 2, response, sh_order=2, mask=mask)

    cod = xval.coeff_of_determination(S, kf_xval)
    npt.assert_array_almost_equal(np.round(cod[0]), csd_cod)
Example #5
0
def test_csd_xval():
    # First, let's see that it works with some data:
    data = nib.load(fdata).get_data()[1:3, 1:3, 1:3]  # Make it *small*
    gtab = gt.gradient_table(fbval, fbvec)
    S0 = np.mean(data[..., gtab.b0s_mask])
    response = ([0.0015, 0.0003, 0.0001], S0)
    csdm = csd.ConstrainedSphericalDeconvModel(gtab, response)
    kf_xval = xval.kfold_xval(csdm, data, 2, response, sh_order=2)

    # In simulation, it should work rather well (high COD):
    psphere = dpd.get_sphere('symmetric362')
    bvecs = np.concatenate(([[0, 0, 0]], psphere.vertices))
    bvals = np.zeros(len(bvecs)) + 1000
    bvals[0] = 0
    gtab = gt.gradient_table(bvals, bvecs)
    mevals = np.array(([0.0015, 0.0003, 0.0001], [0.0015, 0.0003, 0.0003]))
    mevecs = [ np.array( [ [1, 0, 0], [0, 1, 0], [0, 0, 1] ] ),
               np.array( [ [0, 0, 1], [0, 1, 0], [1, 0, 0] ] ) ]
    S0 = 100
    S = sims.single_tensor( gtab, S0, mevals[0], mevecs[0], snr=None )
    sm = csd.ConstrainedSphericalDeconvModel(gtab, response)
    smfit = sm.fit(S)
    np.random.seed(12345)
    response = ([0.0015, 0.0003, 0.0001], S0)
    kf_xval = xval.kfold_xval(sm, S, 2, response, sh_order=2)
    # Because of the regularization, COD is not going to be perfect here:
    cod = xval.coeff_of_determination(S, kf_xval)
    # We'll just test for regressions:
    csd_cod = 97 # pre-computed by hand for this random seed

    # We're going to be really lenient here:
    npt.assert_array_almost_equal(np.round(cod), csd_cod)
Example #6
0
def test_sfm_stick():
    fdata, fbvals, fbvecs = dpd.get_fnames()
    data = load_nifti_data(fdata)
    gtab = grad.gradient_table(fbvals, fbvecs)
    sfmodel = sfm.SparseFascicleModel(gtab,
                                      solver='NNLS',
                                      response=[0.001, 0, 0])
    sffit1 = sfmodel.fit(data[0, 0, 0])
    sphere = dpd.get_sphere()
    sffit1.odf(sphere)
    sffit1.predict(gtab)

    SNR = 1000
    S0 = 100
    mevals = np.array(([0.001, 0, 0], [0.001, 0, 0]))
    angles = [(0, 0), (60, 0)]
    S, sticks = sims.multi_tensor(gtab,
                                  mevals,
                                  S0,
                                  angles=angles,
                                  fractions=[50, 50],
                                  snr=SNR)

    sfmodel = sfm.SparseFascicleModel(gtab,
                                      solver='NNLS',
                                      response=[0.001, 0, 0])
    sffit = sfmodel.fit(S)
    pred = sffit.predict()
    npt.assert_(xval.coeff_of_determination(pred, S) > 96)
Example #7
0
def test_exponential_iso():
    fdata, fbvals, fbvecs = dpd.get_fnames()
    data_dti = nib.load(fdata).get_data()
    gtab_dti = grad.gradient_table(fbvals, fbvecs)
    data_multi, gtab_multi = dpd.dsi_deconv_voxels()

    for data, gtab in zip([data_dti, data_multi], [gtab_dti, gtab_multi]):
        sfmodel = sfm.SparseFascicleModel(
                  gtab, isotropic=sfm.ExponentialIsotropicModel)

        sffit1 = sfmodel.fit(data[0, 0, 0])
        sphere = dpd.get_sphere()
        sffit1.odf(sphere)
        sffit1.predict(gtab)

        SNR = 1000
        S0 = 100
        mevals = np.array(([0.0015, 0.0005, 0.0005],
                           [0.0015, 0.0005, 0.0005]))
        angles = [(0, 0), (60, 0)]
        S, sticks = sims.multi_tensor(gtab, mevals, S0, angles=angles,
                                      fractions=[50, 50], snr=SNR)
        sffit = sfmodel.fit(S)
        pred = sffit.predict()
        npt.assert_(xval.coeff_of_determination(pred, S) > 96)
Example #8
0
def test_csd_xval():
    # First, let's see that it works with some data:
    data = nib.load(fdata).get_data()[1:3, 1:3, 1:3]  # Make it *small*
    gtab = gt.gradient_table(fbval, fbvec)
    S0 = np.mean(data[..., gtab.b0s_mask])
    response = ([0.0015, 0.0003, 0.0001], S0)
    csdm = csd.ConstrainedSphericalDeconvModel(gtab, response)
    kf_xval = xval.kfold_xval(csdm, data, 2, response, sh_order=2)

    # In simulation, it should work rather well (high COD):
    psphere = dpd.get_sphere('symmetric362')
    bvecs = np.concatenate(([[0, 0, 0]], psphere.vertices))
    bvals = np.zeros(len(bvecs)) + 1000
    bvals[0] = 0
    gtab = gt.gradient_table(bvals, bvecs)
    mevals = np.array(([0.0015, 0.0003, 0.0001], [0.0015, 0.0003, 0.0003]))
    mevecs = [
        np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]]),
        np.array([[0, 0, 1], [0, 1, 0], [1, 0, 0]])
    ]
    S0 = 100
    S = sims.single_tensor(gtab, S0, mevals[0], mevecs[0], snr=None)
    sm = csd.ConstrainedSphericalDeconvModel(gtab, response)
    smfit = sm.fit(S)
    np.random.seed(12345)
    response = ([0.0015, 0.0003, 0.0001], S0)
    kf_xval = xval.kfold_xval(sm, S, 2, response, sh_order=2)
    # Because of the regularization, COD is not going to be perfect here:
    cod = xval.coeff_of_determination(S, kf_xval)
    # We'll just test for regressions:
    csd_cod = 97  # pre-computed by hand for this random seed

    # We're going to be really lenient here:
    npt.assert_array_almost_equal(np.round(cod), csd_cod)
Example #9
0
def test_exponential_iso():
    fdata, fbvals, fbvecs = dpd.get_data()
    data_dti = nib.load(fdata).get_data()
    gtab_dti = grad.gradient_table(fbvals, fbvecs)
    data_multi, gtab_multi = dpd.dsi_deconv_voxels()

    for data, gtab in zip([data_dti, data_multi], [gtab_dti, gtab_multi]):
        sfmodel = sfm.SparseFascicleModel(
                  gtab, isotropic=sfm.ExponentialIsotropicModel)

        sffit1 = sfmodel.fit(data[0, 0, 0])
        sphere = dpd.get_sphere()
        odf1 = sffit1.odf(sphere)
        pred1 = sffit1.predict(gtab)

        SNR = 1000
        S0 = 100
        mevals = np.array(([0.0015, 0.0005, 0.0005],
                           [0.0015, 0.0005, 0.0005]))
        angles = [(0, 0), (60, 0)]
        S, sticks = sims.multi_tensor(gtab, mevals, S0, angles=angles,
                                      fractions=[50, 50], snr=SNR)
        sffit = sfmodel.fit(S)
        pred = sffit.predict()
        npt.assert_(xval.coeff_of_determination(pred, S) > 96)
Example #10
0
def test_coeff_of_determination():
    """
    Test the calculation of the coefficient of determination
    """

    model = np.random.randn(10, 10, 10, 150)
    data = np.copy(model)
    # If the model predicts the data perfectly, the COD is all 100s:
    cod = xval.coeff_of_determination(data, model)
    npt.assert_array_equal(100, cod)
Example #11
0
def test_coeff_of_determination():
    """
    Test the calculation of the coefficient of determination
    """

    model = np.random.randn(10, 10, 10, 150)
    data = np.copy(model)
    # If the model predicts the data perfectly, the COD is all 100s:
    cod = xval.coeff_of_determination(data, model)
    npt.assert_array_equal(100, cod)
Example #12
0
def test_predict():
    SNR = 1000
    S0 = 100
    _, fbvals, fbvecs = dpd.get_fnames('small_64D')
    bvals, bvecs = read_bvals_bvecs(fbvals, fbvecs)
    gtab = grad.gradient_table(bvals, bvecs)
    mevals = np.array(([0.0015, 0.0003, 0.0003],
                       [0.0015, 0.0003, 0.0003]))
    angles = [(0, 0), (60, 0)]
    S, sticks = sims.multi_tensor(gtab, mevals, S0, angles=angles,
                                  fractions=[10, 90], snr=SNR)

    sfmodel = sfm.SparseFascicleModel(gtab, response=[0.0015, 0.0003, 0.0003])
    sffit = sfmodel.fit(S)
    pred = sffit.predict()
    npt.assert_(xval.coeff_of_determination(pred, S) > 97)

    # Should be possible to predict using a different gtab:
    new_gtab = grad.gradient_table(bvals[::2], bvecs[::2])
    new_pred = sffit.predict(new_gtab)
    npt.assert_(xval.coeff_of_determination(new_pred, S[::2]) > 97)
Example #13
0
def test_exponential_iso():
    fdata, fbvals, fbvecs = dpd.get_fnames()
    data_dti = load_nifti_data(fdata)
    gtab_dti = grad.gradient_table(fbvals, fbvecs)
    data_multi, gtab_multi = dpd.dsi_deconv_voxels()

    for data, gtab in zip([data_dti, data_multi], [gtab_dti, gtab_multi]):
        sfmodel = sfm.SparseFascicleModel(
            gtab, isotropic=sfm.ExponentialIsotropicModel)

        sffit1 = sfmodel.fit(data[0, 0, 0])
        sphere = dpd.get_sphere()
        odf = sffit1.odf(sphere)
        pred = sffit1.predict(gtab)
        npt.assert_equal(pred.shape, data[0, 0, 0].shape)
        npt.assert_equal(odf.shape,
                         data[0, 0, 0].shape[:-1] + (sphere.x.shape[0], ))

        sffit2 = sfmodel.fit(data)
        sphere = dpd.get_sphere()
        odf = sffit2.odf(sphere)
        pred = sffit2.predict(gtab)
        npt.assert_equal(pred.shape, data.shape)
        npt.assert_equal(odf.shape, data.shape[:-1] + (sphere.x.shape[0], ))

        mask = np.zeros(data.shape[:3])
        mask[2:5, 2:5, :] = 1
        sffit3 = sfmodel.fit(data, mask=mask)
        sphere = dpd.get_sphere()
        odf = sffit3.odf(sphere)
        pred = sffit3.predict(gtab)
        npt.assert_equal(pred.shape, data.shape)
        npt.assert_equal(odf.shape, data.shape[:-1] + (sphere.x.shape[0], ))

        SNR = 1000
        S0 = 100
        mevals = np.array(([0.0015, 0.0005, 0.0005], [0.0015, 0.0005, 0.0005]))
        angles = [(0, 0), (60, 0)]
        S, sticks = sims.multi_tensor(gtab,
                                      mevals,
                                      S0,
                                      angles=angles,
                                      fractions=[50, 50],
                                      snr=SNR)
        sffit = sfmodel.fit(S)
        pred = sffit.predict()
        npt.assert_(xval.coeff_of_determination(pred, S) > 96)
Example #14
0
def test_sfm_stick():
    fdata, fbvals, fbvecs = dpd.get_data()
    data = nib.load(fdata).get_data()
    gtab = grad.gradient_table(fbvals, fbvecs)
    sfmodel = sfm.SparseFascicleModel(gtab, solver='NNLS',
                                      response=[0.001, 0, 0])
    sffit1 = sfmodel.fit(data[0, 0, 0])
    sphere = dpd.get_sphere()
    odf1 = sffit1.odf(sphere)
    pred1 = sffit1.predict(gtab)

    SNR = 1000
    S0 = 100
    mevals = np.array(([0.001, 0, 0],
                       [0.001, 0, 0]))
    angles = [(0, 0), (60, 0)]
    S, sticks = sims.multi_tensor(gtab, mevals, S0, angles=angles,
                                  fractions=[50, 50], snr=SNR)

    sfmodel = sfm.SparseFascicleModel(gtab, solver='NNLS',
                                      response=[0.001, 0, 0])
    sffit = sfmodel.fit(S)
    pred = sffit.predict()
    npt.assert_(xval.coeff_of_determination(pred, S) > 96)
Example #15
0
def calc_cod1000(subject):
    s3 = boto3.resource('s3')
    boto3.setup_default_session(profile_name='cirrus')
    bucket = s3.Bucket('hcp-dki')
    path_dti = '%s/%s_cod_dti_1000.nii.gz' % (subject, subject)
    path_dki = '%s/%s_cod_dki_1000.nii.gz' % (subject, subject)
    if not (exists(path_dti, bucket.name) and exists(path_dki, bucket.name)):
        print("File doesn't exist - going ahead")
        with tempfile.TemporaryDirectory() as tempdir:
            try:
                bucket = setup_boto()
                dwi_file = op.join(tempdir, 'data.nii.gz')
                bvec_file = op.join(tempdir, 'data.bvec')
                bval_file = op.join(tempdir, 'data.bval')

                data_files = {}

                data_files[dwi_file] = \
                    'HCP_900/%s/T1w/Diffusion/data.nii.gz' % subject
                data_files[bvec_file] = \
                    'HCP_900/%s/T1w/Diffusion/bvecs' % subject
                data_files[bval_file] = \
                    'HCP_900/%s/T1w/Diffusion/bvals' % subject
                for k in data_files.keys():
                    if not op.exists(k):
                        bucket.download_file(data_files[k], k)

                wm_file = op.join(tempdir, 'wm.nii.gz')
                boto3.setup_default_session(profile_name='cirrus')
                s3 = boto3.resource('s3')
                bucket = s3.Bucket('hcp-dki')

                s3.meta.client.download_file(
                    'hcp-dki',
                    '%s/%s_white_matter_mask.nii.gz' % (subject, subject),
                    wm_file)
                wm_mask = nib.load(wm_file).get_data().astype(bool)
                dwi_img = nib.load(dwi_file)
                data = dwi_img.get_data()
                bvals = np.loadtxt(bval_file)
                bvecs = np.loadtxt(bvec_file)
                idx = bvals < 1985

                if not exists(path_dki, bucket.name):
                    gtab = dpg.gradient_table(bvals, bvecs, b0_threshold=10)
                    dki_model = dki.DiffusionKurtosisModel(gtab)
                    # Use all the data to calculate the mode
                    pred = xval.kfold_xval(dki_model, data, 5, mask=wm_mask)
                    # But compare only on the b=1000 shell (same as DTI):
                    cod = xval.coeff_of_determination(pred[..., idx],
                                                      data[..., idx])
                    cod_file = op.join(tempdir, 'cod_dki_1000.nii.gz')
                    nib.save(nib.Nifti1Image(cod, dwi_img.affine),
                             cod_file)
                    s3.meta.client.upload_file(
                        cod_file,
                        'hcp-dki',
                        path_dki)

                if not exists(path_dti, bucket.name):
                    data = data[..., idx]
                    gtab = dpg.gradient_table(bvals[idx],
                                              bvecs[:, idx].squeeze(),
                                              b0_threshold=10)

                    model = dti.TensorModel(gtab)
                    pred = xval.kfold_xval(model, data, 5, mask=wm_mask)
                    cod = xval.coeff_of_determination(pred, data)
                    cod_file = op.join(tempdir, 'cod_dti_1000.nii.gz')
                    nib.save(nib.Nifti1Image(cod, dwi_img.affine),
                             cod_file)
                    s3.meta.client.upload_file(
                        cod_file,
                        'hcp-dki',
                        path_dti)

                return subject, True
            except Exception as err:
                return subject, err.args
    else:
        return subject, True
Example #16
0
def compare_models(subject):
    s3 = boto3.resource('s3')
    boto3.setup_default_session(profile_name='cirrus')
    bucket = s3.Bucket('hcp-dki')
    path_dki = '%s/%s_cod_dki.nii.gz' % (subject, subject)
    path_dti = '%s/%s_cod_dti.nii.gz' % (subject, subject)
    if not (exists(path_dki, bucket.name) and exists(path_dti, bucket.name)):
        print("Files don't exist - going ahead")
        with tempfile.TemporaryDirectory() as tempdir:
            try:
                bucket = setup_boto()
                dwi_file = op.join(tempdir, 'data.nii.gz')
                bvec_file = op.join(tempdir, 'data.bvec')
                bval_file = op.join(tempdir, 'data.bval')

                data_files = {}

                data_files[dwi_file] = \
                    'HCP_900/%s/T1w/Diffusion/data.nii.gz' % subject
                data_files[bvec_file] = \
                    'HCP_900/%s/T1w/Diffusion/bvecs' % subject
                data_files[bval_file] = \
                    'HCP_900/%s/T1w/Diffusion/bvals' % subject
                for k in data_files.keys():
                    if not op.exists(k):
                        bucket.download_file(data_files[k], k)

                wm_file = op.join(tempdir, 'wm.nii.gz')
                boto3.setup_default_session(profile_name='cirrus')
                s3 = boto3.resource('s3')
                s3.meta.client.download_file(
                    'hcp-dki',
                    '%s/%s_white_matter_mask.nii.gz' % (subject, subject),
                    wm_file)
                wm_mask = nib.load(wm_file).get_data().astype(bool)
                dwi_img = nib.load(dwi_file)
                data = dwi_img.get_data()
                bvals = np.loadtxt(bval_file)
                bvecs = np.loadtxt(bvec_file)
                gtab = dpg.gradient_table(bvals, bvecs, b0_threshold=10)
                s3 = boto3.resource('s3')
                boto3.setup_default_session(profile_name='cirrus')
                bucket = s3.Bucket('hcp-dki')
                for model_object, method in zip(
                    [dti.TensorModel, dki.DiffusionKurtosisModel],
                    ['dti', 'dki']):
                    path_method = '%s/%s_cod_%s.nii.gz' % (subject, subject,
                                                           method)
                    if not (exists(path_method, bucket.name)):
                        print("No %s file - fitting" % method)

                        print("1")
                        model = model_object(gtab)
                        print("2")
                        pred = xval.kfold_xval(model, data, 5, mask=wm_mask)
                        print("3")
                        cod = xval.coeff_of_determination(pred, data)
                        cod_file = op.join(tempdir, 'cod_%s.nii.gz' % method)
                        print("4")
                        nib.save(nib.Nifti1Image(cod, dwi_img.affine),
                                 cod_file)
                        print("5")
                        s3.meta.client.upload_file(cod_file, 'hcp-dki',
                                                   path_method)
                return subject, True
            except Exception as err:
                return subject, err.args
    else:
        return subject, True
Example #17
0
def compare_models(subject):
    s3 = boto3.resource('s3')
    boto3.setup_default_session(profile_name='cirrus')
    bucket = s3.Bucket('hcp-dki')
    path_dki = '%s/%s_cod_dki.nii.gz' % (subject, subject)
    path_dti = '%s/%s_cod_dti.nii.gz' % (subject, subject)
    if not (exists(path_dki, bucket.name) and exists(path_dti, bucket.name)):
        print("Files don't exist - going ahead")
        with tempfile.TemporaryDirectory() as tempdir:
            try:
                bucket = setup_boto()
                dwi_file = op.join(tempdir, 'data.nii.gz')
                bvec_file = op.join(tempdir, 'data.bvec')
                bval_file = op.join(tempdir, 'data.bval')

                data_files = {}

                data_files[dwi_file] = \
                    'HCP_900/%s/T1w/Diffusion/data.nii.gz' % subject
                data_files[bvec_file] = \
                    'HCP_900/%s/T1w/Diffusion/bvecs' % subject
                data_files[bval_file] = \
                    'HCP_900/%s/T1w/Diffusion/bvals' % subject
                for k in data_files.keys():
                    if not op.exists(k):
                        bucket.download_file(data_files[k], k)

                wm_file = op.join(tempdir, 'wm.nii.gz')
                boto3.setup_default_session(profile_name='cirrus')
                s3 = boto3.resource('s3')
                s3.meta.client.download_file(
                    'hcp-dki',
                    '%s/%s_white_matter_mask.nii.gz' % (subject, subject),
                    wm_file)
                wm_mask = nib.load(wm_file).get_data().astype(bool)
                dwi_img = nib.load(dwi_file)
                data = dwi_img.get_data()
                bvals = np.loadtxt(bval_file)
                bvecs = np.loadtxt(bvec_file)
                gtab = dpg.gradient_table(bvals, bvecs,
                                          b0_threshold=10)
                s3 = boto3.resource('s3')
                boto3.setup_default_session(profile_name='cirrus')
                bucket = s3.Bucket('hcp-dki')
                for model_object, method in zip([dti.TensorModel,
                                                 dki.DiffusionKurtosisModel],
                                                ['dti', 'dki']):
                    path_method = '%s/%s_cod_%s.nii.gz' % (subject,
                                                           subject,
                                                           method)
                    if not (exists(path_method, bucket.name)):
                        print("No %s file - fitting" % method)

                        print("1")
                        model = model_object(gtab)
                        print("2")
                        pred = xval.kfold_xval(model, data, 5, mask=wm_mask)
                        print("3")
                        cod = xval.coeff_of_determination(pred, data)
                        cod_file = op.join(tempdir, 'cod_%s.nii.gz' % method)
                        print("4")
                        nib.save(nib.Nifti1Image(cod, dwi_img.affine),
                                 cod_file)
                        print("5")
                        s3.meta.client.upload_file(
                            cod_file,
                            'hcp-dki',
                            path_method)
                return subject, True
            except Exception as err:
                return subject, err.args
    else:
        return subject, True
Example #18
0
def test_predict():
    SNR = 1000
    S0 = 100
    _, fbvals, fbvecs = dpd.get_fnames('small_64D')
    bvals, bvecs = read_bvals_bvecs(fbvals, fbvecs)
    gtab = grad.gradient_table(bvals, bvecs)
    mevals = np.array(([0.0015, 0.0003, 0.0003], [0.0015, 0.0003, 0.0003]))
    angles = [(0, 0), (60, 0)]
    S, sticks = sims.multi_tensor(gtab,
                                  mevals,
                                  S0,
                                  angles=angles,
                                  fractions=[10, 90],
                                  snr=SNR)

    sfmodel = sfm.SparseFascicleModel(gtab, response=[0.0015, 0.0003, 0.0003])
    sffit = sfmodel.fit(S)
    pred = sffit.predict()
    npt.assert_(xval.coeff_of_determination(pred, S) > 97)

    # Should be possible to predict using a different gtab:
    new_gtab = grad.gradient_table(bvals[::2], bvecs[::2])
    new_pred = sffit.predict(new_gtab)
    npt.assert_(xval.coeff_of_determination(new_pred, S[::2]) > 97)

    # Should be possible to predict for a single direction:
    with warnings.catch_warnings():
        warnings.simplefilter("ignore", category=UserWarning)
        new_gtab = grad.gradient_table(bvals[1][None], bvecs[1][None, :])
    new_pred = sffit.predict(new_gtab)

    # Fitting and predicting with a volume of data:
    fdata, fbval, fbvec = dpd.get_fnames('small_25')
    gtab = grad.gradient_table(fbval, fbvec)
    data = load_nifti_data(fdata)
    sfmodel = sfm.SparseFascicleModel(gtab, response=[0.0015, 0.0003, 0.0003])
    sffit = sfmodel.fit(data)
    pred = sffit.predict()

    # Should be possible to predict using a different gtab:
    new_gtab = grad.gradient_table(bvals[::2], bvecs[::2])
    new_pred = sffit.predict(new_gtab)
    npt.assert_equal(
        new_pred.shape,
        data.shape[:-1] + bvals[::2].shape,
    )

    # Should be possible to predict for a single direction:
    with warnings.catch_warnings():
        warnings.simplefilter("ignore", category=UserWarning)
        new_gtab = grad.gradient_table(bvals[1][None], bvecs[1][None, :])
    new_pred = sffit.predict(new_gtab)
    npt.assert_equal(new_pred.shape, data.shape[:-1])

    # Fitting and predicting with masked data:
    mask = np.zeros(data.shape[:3])
    mask[2:5, 2:5, :] = 1
    sffit = sfmodel.fit(data, mask=mask)
    pred = sffit.predict()
    npt.assert_equal(pred.shape, data.shape)

    # Should be possible to predict using a different gtab:
    new_gtab = grad.gradient_table(bvals[::2], bvecs[::2])
    new_pred = sffit.predict(new_gtab)
    npt.assert_equal(
        new_pred.shape,
        data.shape[:-1] + bvals[::2].shape,
    )
    npt.assert_equal(new_pred[0, 0, 0], 0)

    # Should be possible to predict for a single direction:
    with warnings.catch_warnings():
        warnings.simplefilter("ignore", category=UserWarning)
        new_gtab = grad.gradient_table(bvals[1][None], bvecs[1][None, :])
    new_pred = sffit.predict(new_gtab)
    npt.assert_equal(new_pred.shape, data.shape[:-1])
    npt.assert_equal(new_pred[0, 0, 0], 0)
Example #19
0
def calc_cod1000(subject):
    s3 = boto3.resource('s3')
    boto3.setup_default_session(profile_name='cirrus')
    bucket = s3.Bucket('hcp-dki')
    path_dti = '%s/%s_cod_dti_1000.nii.gz' % (subject, subject)
    path_dki = '%s/%s_cod_dki_1000.nii.gz' % (subject, subject)
    if not (exists(path_dti, bucket.name) and exists(path_dki, bucket.name)):
        print("File doesn't exist - going ahead")
        with tempfile.TemporaryDirectory() as tempdir:
            try:
                bucket = setup_boto()
                dwi_file = op.join(tempdir, 'data.nii.gz')
                bvec_file = op.join(tempdir, 'data.bvec')
                bval_file = op.join(tempdir, 'data.bval')

                data_files = {}

                data_files[dwi_file] = \
                    'HCP_900/%s/T1w/Diffusion/data.nii.gz' % subject
                data_files[bvec_file] = \
                    'HCP_900/%s/T1w/Diffusion/bvecs' % subject
                data_files[bval_file] = \
                    'HCP_900/%s/T1w/Diffusion/bvals' % subject
                for k in data_files.keys():
                    if not op.exists(k):
                        bucket.download_file(data_files[k], k)

                wm_file = op.join(tempdir, 'wm.nii.gz')
                boto3.setup_default_session(profile_name='cirrus')
                s3 = boto3.resource('s3')
                bucket = s3.Bucket('hcp-dki')

                s3.meta.client.download_file(
                    'hcp-dki',
                    '%s/%s_white_matter_mask.nii.gz' % (subject, subject),
                    wm_file)
                wm_mask = nib.load(wm_file).get_data().astype(bool)
                dwi_img = nib.load(dwi_file)
                data = dwi_img.get_data()
                bvals = np.loadtxt(bval_file)
                bvecs = np.loadtxt(bvec_file)
                idx = bvals < 1985

                if not exists(path_dki, bucket.name):
                    gtab = dpg.gradient_table(bvals, bvecs, b0_threshold=10)
                    dki_model = dki.DiffusionKurtosisModel(gtab)
                    # Use all the data to calculate the mode
                    pred = xval.kfold_xval(dki_model, data, 5, mask=wm_mask)
                    # But compare only on the b=1000 shell (same as DTI):
                    cod = xval.coeff_of_determination(pred[..., idx],
                                                      data[..., idx])
                    cod_file = op.join(tempdir, 'cod_dki_1000.nii.gz')
                    nib.save(nib.Nifti1Image(cod, dwi_img.affine), cod_file)
                    s3.meta.client.upload_file(cod_file, 'hcp-dki', path_dki)

                if not exists(path_dti, bucket.name):
                    data = data[..., idx]
                    gtab = dpg.gradient_table(bvals[idx],
                                              bvecs[:, idx].squeeze(),
                                              b0_threshold=10)

                    model = dti.TensorModel(gtab)
                    pred = xval.kfold_xval(model, data, 5, mask=wm_mask)
                    cod = xval.coeff_of_determination(pred, data)
                    cod_file = op.join(tempdir, 'cod_dti_1000.nii.gz')
                    nib.save(nib.Nifti1Image(cod, dwi_img.affine), cod_file)
                    s3.meta.client.upload_file(cod_file, 'hcp-dki', path_dti)

                return subject, True
            except Exception as err:
                return subject, err.args
    else:
        return subject, True