def test_simulated_correlations():
# Get standard brain mask
mr_directory = get_data_directory()
standard = "%s/MNI152_T1_2mm_brain_mask.nii.gz" %(mr_directory)
thresholds = [0.0,0.5,1.0,1.5,1.96,2.0]
# Generate random data inside brain mask, run 10 iterations
standard = nibabel.load(standard)
number_values = len(numpy.where(standard.get_data()!=0)[0])
numpy.random.seed(9191986)
for x in range(0,10):
data1 = norm.rvs(size=number_values)
data2 = norm.rvs(size=number_values)
corr = pearsonr(data1,data2)[0]
# Put into faux nifti images
mr1 = numpy.zeros(standard.shape)
mr1[standard.get_data()!=0] = data1
mr1 = nibabel.nifti1.Nifti1Image(mr1,affine=standard.get_affine(),header=standard.get_header())
mr2 = numpy.zeros(standard.shape)
mr2[standard.get_data()!=0] = data2
mr2 = nibabel.nifti1.Nifti1Image(mr2,affine=standard.get_affine(),header=standard.get_header())
pdmask = make_binary_deletion_mask([mr1,mr2])
pdmask = nibabel.Nifti1Image(pdmask,header=mr1.get_header(),affine=mr1.get_affine())
score = calculate_correlation(images = [mr1,mr2],mask=pdmask)
assert_almost_equal(corr,score,decimal=5)
def test_simulated_correlations():
# Get standard brain mask
mr_directory = get_data_directory()
standard = "%s/MNI152_T1_2mm_brain_mask.nii.gz" % (mr_directory)
thresholds = [0.0, 0.5, 1.0, 1.5, 1.96, 2.0]
# Generate random data inside brain mask, run 10 iterations
standard = nibabel.load(standard)
number_values = len(numpy.where(standard.get_data() != 0)[0])
numpy.random.seed(9191986)
for x in range(0, 10):
data1 = norm.rvs(size=number_values)
data2 = norm.rvs(size=number_values)
corr = pearsonr(data1, data2)[0]
# Put into faux nifti images
mr1 = numpy.zeros(standard.shape)
mr1[standard.get_data() != 0] = data1
mr1 = nibabel.nifti1.Nifti1Image(mr1,
affine=standard.get_affine(),
header=standard.get_header())
mr2 = numpy.zeros(standard.shape)
mr2[standard.get_data() != 0] = data2
mr2 = nibabel.nifti1.Nifti1Image(mr2,
affine=standard.get_affine(),
header=standard.get_header())
pdmask = make_binary_deletion_mask([mr1, mr2])
pdmask = nibabel.Nifti1Image(pdmask,
header=mr1.get_header(),
affine=mr1.get_affine())
score = calculate_correlation(images=[mr1, mr2], mask=pdmask)
assert_almost_equal(corr, score, decimal=5)
def test_binary_deletion_vector():
mr_directory = get_data_directory()
# We will generate data with the following overlap percentages
overlap_percents = [0.0,0.25,0.5,0.75,1.0]
for overlap in overlap_percents:
vector_length = 10000
image_vector1 = numpy.zeros((vector_length))
image_vector2 = numpy.zeros((vector_length))
number_overlap_voxels = int(numpy.floor(overlap*vector_length))
remaining_voxels = int(vector_length - number_overlap_voxels)
idx = range(0,vector_length)
# We break the remaining voxels into 4 groups:
# - nans that will overlap
# - zeros that will overlap (no change to images here, already zeros)
# - nans in image1, random sample of values in image2
# - zeros in image2, random sample of values in image1
group_size = remaining_voxels/4
if overlap != 0.0:
# Here are the overlapping voxels for each image
overlap_idx = range(0,number_overlap_voxels)
image_vector1[overlap_idx] = 1
image_vector2[overlap_idx] = 1
if overlap != 1.0:
# Nans that will overlap
nans_overlap_idx = range(number_overlap_voxels,(number_overlap_voxels+group_size))
image_vector1[nans_overlap_idx] = numpy.nan
image_vector2[nans_overlap_idx] = numpy.nan
# Nans in image1, random sample of values in image 2
start = number_overlap_voxels+group_size
end = number_overlap_voxels+2*group_size
nans_image1 = idx[start:end]
values_image2 = range(nans_image1[-1],(nans_image1[-1] + int(group_size/2)))
image_vector1[nans_image1] = numpy.nan
image_vector2[values_image2] = 0.5
# Zeros in image2, random sample of values in image 1
start = number_overlap_voxels+2*group_size
end = number_overlap_voxels+3*group_size
zeros_image2 = idx[start:end]
values_image1 = range(zeros_image2[-1],(zeros_image2[-1] + int(group_size/2)))
image_vector1[values_image1] = 0.75
# Create nifti images and pdmask
pdmask = make_binary_deletion_vector([image_vector1,image_vector2])
actual_overlap = len(numpy.where(pdmask!=0)[0])
print "Overlap %s percent: should have %s, actual %s" %(overlap,number_overlap_voxels,actual_overlap)
assert_equal(actual_overlap,number_overlap_voxels)
# Also check that is binary
if overlap != 0 and overlap != 1:
assert_equal(numpy.unique(pdmask)[0],0)
assert_equal(numpy.unique(pdmask)[1],1)
if overlap == 0:
assert_equal(numpy.unique(pdmask)[0],0)
if overlap == 1:
assert_equal(numpy.unique(pdmask)[0],1)
def test_binary_deletion_vector():
mr_directory = get_data_directory()
# We will generate data with the following overlap percentages
overlap_percents = [0.0,0.25,0.5,0.75,1.0]
for overlap in overlap_percents:
vector_length = 10000
image_vector1 = numpy.zeros((vector_length))
image_vector2 = numpy.zeros((vector_length))
number_overlap_voxels = int(numpy.floor(overlap*vector_length))
remaining_voxels = int(vector_length - number_overlap_voxels)
idx = list(range(0,vector_length))
# We break the remaining voxels into 4 groups:
# - nans that will overlap
# - zeros that will overlap (no change to images here, already zeros)
# - nans in image1, random sample of values in image2
# - zeros in image2, random sample of values in image1
group_size = old_div(remaining_voxels,4)
if overlap != 0.0:
# Here are the overlapping voxels for each image
overlap_idx = list(range(0,number_overlap_voxels))
image_vector1[overlap_idx] = 1
image_vector2[overlap_idx] = 1
if overlap != 1.0:
# Nans that will overlap
nans_overlap_idx = list(range(number_overlap_voxels,(number_overlap_voxels+group_size)))
image_vector1[nans_overlap_idx] = numpy.nan
image_vector2[nans_overlap_idx] = numpy.nan
# Nans in image1, random sample of values in image 2
start = number_overlap_voxels+group_size
end = number_overlap_voxels+2*group_size
nans_image1 = idx[start:end]
values_image2 = list(range(nans_image1[-1],(nans_image1[-1] + int(old_div(group_size,2)))))
image_vector1[nans_image1] = numpy.nan
image_vector2[values_image2] = 0.5
# Zeros in image2, random sample of values in image 1
start = number_overlap_voxels+2*group_size
end = number_overlap_voxels+3*group_size
zeros_image2 = idx[start:end]
values_image1 = list(range(zeros_image2[-1],(zeros_image2[-1] + int(old_div(group_size,2)))))
image_vector1[values_image1] = 0.75
# Create nifti images and pdmask
pdmask = make_binary_deletion_vector([image_vector1,image_vector2])
actual_overlap = len(numpy.where(pdmask!=0)[0])
print("Overlap %s percent: should have %s, actual %s" %(overlap,number_overlap_voxels,actual_overlap))
assert_equal(actual_overlap,number_overlap_voxels)
# Also check that is binary
if overlap != 0 and overlap != 1:
assert_equal(numpy.unique(pdmask)[0],0)
assert_equal(numpy.unique(pdmask)[1],1)
if overlap == 0:
assert_equal(numpy.unique(pdmask)[0],0)
if overlap == 1:
assert_equal(numpy.unique(pdmask)[0],1)
def test_binary_deletion_mask():
mr_directory = get_data_directory()
standard = "%s/MNI152_T1_8mm_brain_mask.nii.gz" %(mr_directory)
brain_mask = nibabel.load(standard)
unzip = lambda l:tuple(zip(*l))
# We will generate data with the following overlap percentages
overlap_percents = [0.0,0.25,0.5,0.75,1.0]
for overlap in overlap_percents:
image1 = numpy.zeros(brain_mask.shape)
image2 = numpy.zeros(brain_mask.shape)
x,y,z = numpy.where(brain_mask.get_data()==1)
idx = zip(x,y,z)
numpy.random.shuffle(idx)
number_voxels = len(idx)
number_overlap_voxels = int(numpy.floor(overlap*number_voxels))
remaining_voxels = int(number_voxels - number_overlap_voxels)
# We break the remaining voxels into 4 groups:
# - nans that will overlap
# - zeros that will overlap (no change to images here, already zeros)
# - nans in image1, random sample of values in image2
# - zeros in image2, random sample of values in image1
group_size = remaining_voxels/4
if overlap != 0.0:
# Here are the overlapping voxels for each image
overlap_idx = unzip(idx[0:number_overlap_voxels])
image1[overlap_idx] = 1
image2[overlap_idx] = 1
if overlap != 1.0:
# Nans that will overlap
nans_overlap_idx = unzip(idx[number_overlap_voxels:(number_overlap_voxels+group_size)])
image1[nans_overlap_idx] = numpy.nan
image2[nans_overlap_idx] = numpy.nan
# Nans in image1, random sample of values in image 2
start = number_overlap_voxels+group_size
end = number_overlap_voxels+2*group_size
nans_image1 = idx[start:end]
values_image2 = unzip(random.sample(nans_image1,int(group_size/2)))
image1[unzip(nans_image1)] = numpy.nan
image2[values_image2] = 0.5
# Zeros in image2, random sample of values in image 1
start = number_overlap_voxels+2*group_size
end = number_overlap_voxels+3*group_size
zeros_image2 = idx[start:end]
values_image1 = unzip(random.sample(zeros_image2,int(group_size/2)))
image1[values_image1] = 0.75
# Create nifti images and pdmask
nii1 = nibabel.Nifti1Image(image1,affine=brain_mask.get_affine(),header=brain_mask.get_header())
nii2 = nibabel.Nifti1Image(image2,affine=brain_mask.get_affine(),header=brain_mask.get_header())
pdmask = make_binary_deletion_mask([nii1,nii2])
actual_overlap = len(numpy.where(pdmask!=0)[0])
print "Overlap %s percent: should have %s, actual %s" %(overlap,number_overlap_voxels,actual_overlap)
assert_equal(actual_overlap,number_overlap_voxels)
def test_binary_deletion_mask():
mr_directory = get_data_directory()
standard = "%s/MNI152_T1_8mm_brain_mask.nii.gz" %(mr_directory)
brain_mask = nibabel.load(standard)
unzip = lambda l:tuple(zip(*l))
# We will generate data with the following overlap percentages
overlap_percents = [0.0,0.25,0.5,0.75,1.0]
for overlap in overlap_percents:
image1 = numpy.zeros(brain_mask.shape)
image2 = numpy.zeros(brain_mask.shape)
x,y,z = numpy.where(brain_mask.get_data()==1)
idx = list(zip(x,y,z))
numpy.random.shuffle(idx)
number_voxels = len(idx)
number_overlap_voxels = int(numpy.floor(overlap*number_voxels))
remaining_voxels = int(number_voxels - number_overlap_voxels)
# We break the remaining voxels into 4 groups:
# - nans that will overlap
# - zeros that will overlap (no change to images here, already zeros)
# - nans in image1, random sample of values in image2
# - zeros in image2, random sample of values in image1
group_size = old_div(remaining_voxels,4)
if overlap != 0.0:
# Here are the overlapping voxels for each image
overlap_idx = unzip(idx[0:number_overlap_voxels])
image1[overlap_idx] = 1
image2[overlap_idx] = 1
if overlap != 1.0:
# Nans that will overlap
nans_overlap_idx = unzip(idx[number_overlap_voxels:(number_overlap_voxels+group_size)])
image1[nans_overlap_idx] = numpy.nan
image2[nans_overlap_idx] = numpy.nan
# Nans in image1, random sample of values in image 2
start = number_overlap_voxels+group_size
end = number_overlap_voxels+2*group_size
nans_image1 = idx[start:end]
values_image2 = unzip(random.sample(nans_image1,int(old_div(group_size,2))))
image1[unzip(nans_image1)] = numpy.nan
image2[values_image2] = 0.5
# Zeros in image2, random sample of values in image 1
start = number_overlap_voxels+2*group_size
end = number_overlap_voxels+3*group_size
zeros_image2 = idx[start:end]
values_image1 = unzip(random.sample(zeros_image2,int(old_div(group_size,2))))
image1[values_image1] = 0.75
# Create nifti images and pdmask
nii1 = nibabel.Nifti1Image(image1,affine=brain_mask.get_affine(),header=brain_mask.get_header())
nii2 = nibabel.Nifti1Image(image2,affine=brain_mask.get_affine(),header=brain_mask.get_header())
pdmask = make_binary_deletion_mask([nii1,nii2])
actual_overlap = len(numpy.where(pdmask!=0)[0])
print("Overlap %s percent: should have %s, actual %s" %(overlap,number_overlap_voxels,actual_overlap))
assert_equal(actual_overlap,number_overlap_voxels)
def save_voxelwise_pearson_similarity_resample(pk1,
pk2,
resample_dim=[4, 4, 4]):
from neurovault.apps.statmaps.models import Similarity, Comparison
# We will always calculate Comparison 1 vs 2, never 2 vs 1
if pk1 != pk2:
sorted_images = get_images_by_ordered_id(pk1, pk2)
image1 = sorted_images[0]
image2 = sorted_images[1]
pearson_metric = Similarity.objects.get(
similarity_metric="pearson product-moment correlation coefficient",
transformation="voxelwise")
# Get standard space brain
mr_directory = get_data_directory()
reference = "%s/MNI152_T1_2mm_brain_mask.nii.gz" % (mr_directory)
image_paths = [image.file.path for image in [image1, image2]]
images_resamp, _ = resample_images_ref(images=image_paths,
reference=reference,
interpolation="continuous",
resample_dim=resample_dim)
# resample_images_ref will "squeeze" images, but we should keep error here for now
for image_nii, image_obj in zip(images_resamp, [image1, image2]):
if len(numpy.squeeze(image_nii.get_data()).shape) != 3:
raise Exception(
"Image %s (id=%d) has incorrect number of dimensions %s" %
(image_obj.name, image_obj.id,
str(image_nii.get_data().shape)))
# Calculate correlation only on voxels that are in both maps (not zero, and not nan)
image1_res = images_resamp[0]
image2_res = images_resamp[1]
binary_mask = make_binary_deletion_mask(images_resamp)
binary_mask = nib.Nifti1Image(binary_mask,
header=image1_res.get_header(),
affine=image1_res.get_affine())
# Will return nan if comparison is not possible
pearson_score = calculate_correlation([image1_res, image2_res],
mask=binary_mask,
corr_type="pearson")
# Only save comparison if is not nan
if not numpy.isnan(pearson_score):
Comparison.objects.update_or_create(
image1=image1,
image2=image2,
similarity_metric=pearson_metric,
similarity_score=pearson_score)
return image1.pk, image2.pk, pearson_score
else:
raise Exception("You are trying to compare an image with itself!")
def save_voxelwise_pearson_similarity_resample(pk1, pk2,resample_dim=[4,4,4]):
from neurovault.apps.statmaps.models import Similarity, Comparison
# We will always calculate Comparison 1 vs 2, never 2 vs 1
if pk1 != pk2:
try:
sorted_images = get_images_by_ordered_id(pk1, pk2)
except Http404:
# files have been deleted in the meantime
return
image1 = sorted_images[0]
image2 = sorted_images[1]
pearson_metric = Similarity.objects.get(
similarity_metric="pearson product-moment correlation coefficient",
transformation="voxelwise")
# Get standard space brain
mr_directory = get_data_directory()
reference = "%s/MNI152_T1_2mm_brain_mask.nii.gz" %(mr_directory)
image_paths = [image.file.path for image in [image1, image2]]
images_resamp, _ = resample_images_ref(images=image_paths,
reference=reference,
interpolation="continuous",
resample_dim=resample_dim)
# resample_images_ref will "squeeze" images, but we should keep error here for now
for image_nii, image_obj in zip(images_resamp, [image1, image2]):
if len(numpy.squeeze(image_nii.get_data()).shape) != 3:
raise Exception("Image %s (id=%d) has incorrect number of dimensions %s"%(image_obj.name,
image_obj.id,
str(image_nii.get_data().shape)))
# Calculate correlation only on voxels that are in both maps (not zero, and not nan)
image1_res = images_resamp[0]
image2_res = images_resamp[1]
binary_mask = make_binary_deletion_mask(images_resamp)
binary_mask = nib.Nifti1Image(binary_mask,header=image1_res.get_header(),affine=image1_res.get_affine())
# Will return nan if comparison is not possible
pearson_score = calculate_correlation([image1_res,image2_res],mask=binary_mask,corr_type="pearson")
# Only save comparison if is not nan
if not numpy.isnan(pearson_score):
Comparison.objects.update_or_create(image1=image1, image2=image2,
similarity_metric=pearson_metric,
similarity_score=pearson_score)
return image1.pk,image2.pk,pearson_score
else:
raise Exception("You are trying to compare an image with itself!")
def test_scatterplot_error_message():
# Get standard brain mask
mr_directory = get_data_directory()
standard = "%s/MNI152_T1_8mm_brain_mask.nii.gz" %(mr_directory)
standard = nibabel.load(standard)
unzip = lambda l:tuple(zip(*l))
# This is the error message we should see
error = re.compile('Scatterplot Comparison Correlations Not Possible')
# Case 1: provided pdmask masks all voxels (eg, no overlap in images)
data1 = norm.rvs(size=500)
data2 = norm.rvs(size=500)
image1 = numpy.zeros(standard.shape)
image2 = numpy.zeros(standard.shape)
x,y,z = numpy.where(standard.get_data()==1)
idx = list(zip(x,y,z))
image1_voxels = unzip(idx[0:500])
image2_voxels = unzip(idx[1500:2000])
image1[image1_voxels] = data1
image2[image2_voxels] = data2
image1 = nibabel.nifti1.Nifti1Image(image1,affine=standard.get_affine(),header=standard.get_header())
image2 = nibabel.nifti1.Nifti1Image(image2,affine=standard.get_affine(),header=standard.get_header())
html_snippet,data_table = scatterplot_compare(images=[image1,image2],
reference = standard,
image_names=["image 1","image 2"],
corr_type="pearson")
html_snippet = " ".join(html_snippet)
assert_true(bool(error.search(html_snippet)))
# Case 2: fewer than 3 voxels overlapping
data1 = norm.rvs(size=2)
data2 = norm.rvs(size=2)
image1 = numpy.zeros(standard.shape)
image2 = numpy.zeros(standard.shape)
x,y,z = numpy.where(standard.get_data()==1)
idx = list(zip(x,y,z))
idx = unzip(idx[10:12])
image1[idx] = data1
image2[idx] = data2
image1 = nibabel.nifti1.Nifti1Image(image1,affine=standard.get_affine(),header=standard.get_header())
image2 = nibabel.nifti1.Nifti1Image(image2,affine=standard.get_affine(),header=standard.get_header())
html_snippet,data_table = scatterplot_compare(images=[image1,image2],
reference = standard,
image_names=["image 1","image 2"],
corr_type="pearson")
html_snippet = " ".join(html_snippet)
assert_true(bool(error.search(html_snippet)))
# Case 2: But 3 should work
data1 = norm.rvs(size=3)
data2 = norm.rvs(size=3)
image1 = numpy.zeros(standard.shape)
image2 = numpy.zeros(standard.shape)
x,y,z = numpy.where(standard.get_data()==1)
idx = list(zip(x,y,z))
idx = unzip(idx[10:13])
image1[idx] = data1
image2[idx] = data2
image1 = nibabel.nifti1.Nifti1Image(image1,affine=standard.get_affine(),header=standard.get_header())
image2 = nibabel.nifti1.Nifti1Image(image2,affine=standard.get_affine(),header=standard.get_header())
html_snippet,data_table = scatterplot_compare(images=[image1,image2],
reference = standard,
image_names=["image 1","image 2"],
corr_type="pearson")
html_snippet = " ".join(html_snippet)
assert_false(bool(error.search(html_snippet)))