def test_simulated_correlations(): # Get standard brain mask mr_directory = get_data_directory() standard = "%s/MNI152_T1_2mm_brain_mask.nii.gz" %(mr_directory) thresholds = [0.0,0.5,1.0,1.5,1.96,2.0] # Generate random data inside brain mask, run 10 iterations standard = nibabel.load(standard) number_values = len(numpy.where(standard.get_data()!=0)[0]) numpy.random.seed(9191986) for x in range(0,10): data1 = norm.rvs(size=number_values) data2 = norm.rvs(size=number_values) corr = pearsonr(data1,data2)[0] # Put into faux nifti images mr1 = numpy.zeros(standard.shape) mr1[standard.get_data()!=0] = data1 mr1 = nibabel.nifti1.Nifti1Image(mr1,affine=standard.get_affine(),header=standard.get_header()) mr2 = numpy.zeros(standard.shape) mr2[standard.get_data()!=0] = data2 mr2 = nibabel.nifti1.Nifti1Image(mr2,affine=standard.get_affine(),header=standard.get_header()) pdmask = make_binary_deletion_mask([mr1,mr2]) pdmask = nibabel.Nifti1Image(pdmask,header=mr1.get_header(),affine=mr1.get_affine()) score = calculate_correlation(images = [mr1,mr2],mask=pdmask) assert_almost_equal(corr,score,decimal=5)
def test_simulated_correlations(): # Get standard brain mask mr_directory = get_data_directory() standard = "%s/MNI152_T1_2mm_brain_mask.nii.gz" % (mr_directory) thresholds = [0.0, 0.5, 1.0, 1.5, 1.96, 2.0] # Generate random data inside brain mask, run 10 iterations standard = nibabel.load(standard) number_values = len(numpy.where(standard.get_data() != 0)[0]) numpy.random.seed(9191986) for x in range(0, 10): data1 = norm.rvs(size=number_values) data2 = norm.rvs(size=number_values) corr = pearsonr(data1, data2)[0] # Put into faux nifti images mr1 = numpy.zeros(standard.shape) mr1[standard.get_data() != 0] = data1 mr1 = nibabel.nifti1.Nifti1Image(mr1, affine=standard.get_affine(), header=standard.get_header()) mr2 = numpy.zeros(standard.shape) mr2[standard.get_data() != 0] = data2 mr2 = nibabel.nifti1.Nifti1Image(mr2, affine=standard.get_affine(), header=standard.get_header()) pdmask = make_binary_deletion_mask([mr1, mr2]) pdmask = nibabel.Nifti1Image(pdmask, header=mr1.get_header(), affine=mr1.get_affine()) score = calculate_correlation(images=[mr1, mr2], mask=pdmask) assert_almost_equal(corr, score, decimal=5)
def test_binary_deletion_vector(): mr_directory = get_data_directory() # We will generate data with the following overlap percentages overlap_percents = [0.0,0.25,0.5,0.75,1.0] for overlap in overlap_percents: vector_length = 10000 image_vector1 = numpy.zeros((vector_length)) image_vector2 = numpy.zeros((vector_length)) number_overlap_voxels = int(numpy.floor(overlap*vector_length)) remaining_voxels = int(vector_length - number_overlap_voxels) idx = range(0,vector_length) # We break the remaining voxels into 4 groups: # - nans that will overlap # - zeros that will overlap (no change to images here, already zeros) # - nans in image1, random sample of values in image2 # - zeros in image2, random sample of values in image1 group_size = remaining_voxels/4 if overlap != 0.0: # Here are the overlapping voxels for each image overlap_idx = range(0,number_overlap_voxels) image_vector1[overlap_idx] = 1 image_vector2[overlap_idx] = 1 if overlap != 1.0: # Nans that will overlap nans_overlap_idx = range(number_overlap_voxels,(number_overlap_voxels+group_size)) image_vector1[nans_overlap_idx] = numpy.nan image_vector2[nans_overlap_idx] = numpy.nan # Nans in image1, random sample of values in image 2 start = number_overlap_voxels+group_size end = number_overlap_voxels+2*group_size nans_image1 = idx[start:end] values_image2 = range(nans_image1[-1],(nans_image1[-1] + int(group_size/2))) image_vector1[nans_image1] = numpy.nan image_vector2[values_image2] = 0.5 # Zeros in image2, random sample of values in image 1 start = number_overlap_voxels+2*group_size end = number_overlap_voxels+3*group_size zeros_image2 = idx[start:end] values_image1 = range(zeros_image2[-1],(zeros_image2[-1] + int(group_size/2))) image_vector1[values_image1] = 0.75 # Create nifti images and pdmask pdmask = make_binary_deletion_vector([image_vector1,image_vector2]) actual_overlap = len(numpy.where(pdmask!=0)[0]) print "Overlap %s percent: should have %s, actual %s" %(overlap,number_overlap_voxels,actual_overlap) assert_equal(actual_overlap,number_overlap_voxels) # Also check that is binary if overlap != 0 and overlap != 1: assert_equal(numpy.unique(pdmask)[0],0) assert_equal(numpy.unique(pdmask)[1],1) if overlap == 0: assert_equal(numpy.unique(pdmask)[0],0) if overlap == 1: assert_equal(numpy.unique(pdmask)[0],1)
def test_binary_deletion_vector(): mr_directory = get_data_directory() # We will generate data with the following overlap percentages overlap_percents = [0.0,0.25,0.5,0.75,1.0] for overlap in overlap_percents: vector_length = 10000 image_vector1 = numpy.zeros((vector_length)) image_vector2 = numpy.zeros((vector_length)) number_overlap_voxels = int(numpy.floor(overlap*vector_length)) remaining_voxels = int(vector_length - number_overlap_voxels) idx = list(range(0,vector_length)) # We break the remaining voxels into 4 groups: # - nans that will overlap # - zeros that will overlap (no change to images here, already zeros) # - nans in image1, random sample of values in image2 # - zeros in image2, random sample of values in image1 group_size = old_div(remaining_voxels,4) if overlap != 0.0: # Here are the overlapping voxels for each image overlap_idx = list(range(0,number_overlap_voxels)) image_vector1[overlap_idx] = 1 image_vector2[overlap_idx] = 1 if overlap != 1.0: # Nans that will overlap nans_overlap_idx = list(range(number_overlap_voxels,(number_overlap_voxels+group_size))) image_vector1[nans_overlap_idx] = numpy.nan image_vector2[nans_overlap_idx] = numpy.nan # Nans in image1, random sample of values in image 2 start = number_overlap_voxels+group_size end = number_overlap_voxels+2*group_size nans_image1 = idx[start:end] values_image2 = list(range(nans_image1[-1],(nans_image1[-1] + int(old_div(group_size,2))))) image_vector1[nans_image1] = numpy.nan image_vector2[values_image2] = 0.5 # Zeros in image2, random sample of values in image 1 start = number_overlap_voxels+2*group_size end = number_overlap_voxels+3*group_size zeros_image2 = idx[start:end] values_image1 = list(range(zeros_image2[-1],(zeros_image2[-1] + int(old_div(group_size,2))))) image_vector1[values_image1] = 0.75 # Create nifti images and pdmask pdmask = make_binary_deletion_vector([image_vector1,image_vector2]) actual_overlap = len(numpy.where(pdmask!=0)[0]) print("Overlap %s percent: should have %s, actual %s" %(overlap,number_overlap_voxels,actual_overlap)) assert_equal(actual_overlap,number_overlap_voxels) # Also check that is binary if overlap != 0 and overlap != 1: assert_equal(numpy.unique(pdmask)[0],0) assert_equal(numpy.unique(pdmask)[1],1) if overlap == 0: assert_equal(numpy.unique(pdmask)[0],0) if overlap == 1: assert_equal(numpy.unique(pdmask)[0],1)
def test_binary_deletion_mask(): mr_directory = get_data_directory() standard = "%s/MNI152_T1_8mm_brain_mask.nii.gz" %(mr_directory) brain_mask = nibabel.load(standard) unzip = lambda l:tuple(zip(*l)) # We will generate data with the following overlap percentages overlap_percents = [0.0,0.25,0.5,0.75,1.0] for overlap in overlap_percents: image1 = numpy.zeros(brain_mask.shape) image2 = numpy.zeros(brain_mask.shape) x,y,z = numpy.where(brain_mask.get_data()==1) idx = zip(x,y,z) numpy.random.shuffle(idx) number_voxels = len(idx) number_overlap_voxels = int(numpy.floor(overlap*number_voxels)) remaining_voxels = int(number_voxels - number_overlap_voxels) # We break the remaining voxels into 4 groups: # - nans that will overlap # - zeros that will overlap (no change to images here, already zeros) # - nans in image1, random sample of values in image2 # - zeros in image2, random sample of values in image1 group_size = remaining_voxels/4 if overlap != 0.0: # Here are the overlapping voxels for each image overlap_idx = unzip(idx[0:number_overlap_voxels]) image1[overlap_idx] = 1 image2[overlap_idx] = 1 if overlap != 1.0: # Nans that will overlap nans_overlap_idx = unzip(idx[number_overlap_voxels:(number_overlap_voxels+group_size)]) image1[nans_overlap_idx] = numpy.nan image2[nans_overlap_idx] = numpy.nan # Nans in image1, random sample of values in image 2 start = number_overlap_voxels+group_size end = number_overlap_voxels+2*group_size nans_image1 = idx[start:end] values_image2 = unzip(random.sample(nans_image1,int(group_size/2))) image1[unzip(nans_image1)] = numpy.nan image2[values_image2] = 0.5 # Zeros in image2, random sample of values in image 1 start = number_overlap_voxels+2*group_size end = number_overlap_voxels+3*group_size zeros_image2 = idx[start:end] values_image1 = unzip(random.sample(zeros_image2,int(group_size/2))) image1[values_image1] = 0.75 # Create nifti images and pdmask nii1 = nibabel.Nifti1Image(image1,affine=brain_mask.get_affine(),header=brain_mask.get_header()) nii2 = nibabel.Nifti1Image(image2,affine=brain_mask.get_affine(),header=brain_mask.get_header()) pdmask = make_binary_deletion_mask([nii1,nii2]) actual_overlap = len(numpy.where(pdmask!=0)[0]) print "Overlap %s percent: should have %s, actual %s" %(overlap,number_overlap_voxels,actual_overlap) assert_equal(actual_overlap,number_overlap_voxels)
def test_binary_deletion_mask(): mr_directory = get_data_directory() standard = "%s/MNI152_T1_8mm_brain_mask.nii.gz" %(mr_directory) brain_mask = nibabel.load(standard) unzip = lambda l:tuple(zip(*l)) # We will generate data with the following overlap percentages overlap_percents = [0.0,0.25,0.5,0.75,1.0] for overlap in overlap_percents: image1 = numpy.zeros(brain_mask.shape) image2 = numpy.zeros(brain_mask.shape) x,y,z = numpy.where(brain_mask.get_data()==1) idx = list(zip(x,y,z)) numpy.random.shuffle(idx) number_voxels = len(idx) number_overlap_voxels = int(numpy.floor(overlap*number_voxels)) remaining_voxels = int(number_voxels - number_overlap_voxels) # We break the remaining voxels into 4 groups: # - nans that will overlap # - zeros that will overlap (no change to images here, already zeros) # - nans in image1, random sample of values in image2 # - zeros in image2, random sample of values in image1 group_size = old_div(remaining_voxels,4) if overlap != 0.0: # Here are the overlapping voxels for each image overlap_idx = unzip(idx[0:number_overlap_voxels]) image1[overlap_idx] = 1 image2[overlap_idx] = 1 if overlap != 1.0: # Nans that will overlap nans_overlap_idx = unzip(idx[number_overlap_voxels:(number_overlap_voxels+group_size)]) image1[nans_overlap_idx] = numpy.nan image2[nans_overlap_idx] = numpy.nan # Nans in image1, random sample of values in image 2 start = number_overlap_voxels+group_size end = number_overlap_voxels+2*group_size nans_image1 = idx[start:end] values_image2 = unzip(random.sample(nans_image1,int(old_div(group_size,2)))) image1[unzip(nans_image1)] = numpy.nan image2[values_image2] = 0.5 # Zeros in image2, random sample of values in image 1 start = number_overlap_voxels+2*group_size end = number_overlap_voxels+3*group_size zeros_image2 = idx[start:end] values_image1 = unzip(random.sample(zeros_image2,int(old_div(group_size,2)))) image1[values_image1] = 0.75 # Create nifti images and pdmask nii1 = nibabel.Nifti1Image(image1,affine=brain_mask.get_affine(),header=brain_mask.get_header()) nii2 = nibabel.Nifti1Image(image2,affine=brain_mask.get_affine(),header=brain_mask.get_header()) pdmask = make_binary_deletion_mask([nii1,nii2]) actual_overlap = len(numpy.where(pdmask!=0)[0]) print("Overlap %s percent: should have %s, actual %s" %(overlap,number_overlap_voxels,actual_overlap)) assert_equal(actual_overlap,number_overlap_voxels)
def save_voxelwise_pearson_similarity_resample(pk1, pk2, resample_dim=[4, 4, 4]): from neurovault.apps.statmaps.models import Similarity, Comparison # We will always calculate Comparison 1 vs 2, never 2 vs 1 if pk1 != pk2: sorted_images = get_images_by_ordered_id(pk1, pk2) image1 = sorted_images[0] image2 = sorted_images[1] pearson_metric = Similarity.objects.get( similarity_metric="pearson product-moment correlation coefficient", transformation="voxelwise") # Get standard space brain mr_directory = get_data_directory() reference = "%s/MNI152_T1_2mm_brain_mask.nii.gz" % (mr_directory) image_paths = [image.file.path for image in [image1, image2]] images_resamp, _ = resample_images_ref(images=image_paths, reference=reference, interpolation="continuous", resample_dim=resample_dim) # resample_images_ref will "squeeze" images, but we should keep error here for now for image_nii, image_obj in zip(images_resamp, [image1, image2]): if len(numpy.squeeze(image_nii.get_data()).shape) != 3: raise Exception( "Image %s (id=%d) has incorrect number of dimensions %s" % (image_obj.name, image_obj.id, str(image_nii.get_data().shape))) # Calculate correlation only on voxels that are in both maps (not zero, and not nan) image1_res = images_resamp[0] image2_res = images_resamp[1] binary_mask = make_binary_deletion_mask(images_resamp) binary_mask = nib.Nifti1Image(binary_mask, header=image1_res.get_header(), affine=image1_res.get_affine()) # Will return nan if comparison is not possible pearson_score = calculate_correlation([image1_res, image2_res], mask=binary_mask, corr_type="pearson") # Only save comparison if is not nan if not numpy.isnan(pearson_score): Comparison.objects.update_or_create( image1=image1, image2=image2, similarity_metric=pearson_metric, similarity_score=pearson_score) return image1.pk, image2.pk, pearson_score else: raise Exception("You are trying to compare an image with itself!")
def save_voxelwise_pearson_similarity_resample(pk1, pk2,resample_dim=[4,4,4]): from neurovault.apps.statmaps.models import Similarity, Comparison # We will always calculate Comparison 1 vs 2, never 2 vs 1 if pk1 != pk2: try: sorted_images = get_images_by_ordered_id(pk1, pk2) except Http404: # files have been deleted in the meantime return image1 = sorted_images[0] image2 = sorted_images[1] pearson_metric = Similarity.objects.get( similarity_metric="pearson product-moment correlation coefficient", transformation="voxelwise") # Get standard space brain mr_directory = get_data_directory() reference = "%s/MNI152_T1_2mm_brain_mask.nii.gz" %(mr_directory) image_paths = [image.file.path for image in [image1, image2]] images_resamp, _ = resample_images_ref(images=image_paths, reference=reference, interpolation="continuous", resample_dim=resample_dim) # resample_images_ref will "squeeze" images, but we should keep error here for now for image_nii, image_obj in zip(images_resamp, [image1, image2]): if len(numpy.squeeze(image_nii.get_data()).shape) != 3: raise Exception("Image %s (id=%d) has incorrect number of dimensions %s"%(image_obj.name, image_obj.id, str(image_nii.get_data().shape))) # Calculate correlation only on voxels that are in both maps (not zero, and not nan) image1_res = images_resamp[0] image2_res = images_resamp[1] binary_mask = make_binary_deletion_mask(images_resamp) binary_mask = nib.Nifti1Image(binary_mask,header=image1_res.get_header(),affine=image1_res.get_affine()) # Will return nan if comparison is not possible pearson_score = calculate_correlation([image1_res,image2_res],mask=binary_mask,corr_type="pearson") # Only save comparison if is not nan if not numpy.isnan(pearson_score): Comparison.objects.update_or_create(image1=image1, image2=image2, similarity_metric=pearson_metric, similarity_score=pearson_score) return image1.pk,image2.pk,pearson_score else: raise Exception("You are trying to compare an image with itself!")
def test_scatterplot_error_message(): # Get standard brain mask mr_directory = get_data_directory() standard = "%s/MNI152_T1_8mm_brain_mask.nii.gz" %(mr_directory) standard = nibabel.load(standard) unzip = lambda l:tuple(zip(*l)) # This is the error message we should see error = re.compile('Scatterplot Comparison Correlations Not Possible') # Case 1: provided pdmask masks all voxels (eg, no overlap in images) data1 = norm.rvs(size=500) data2 = norm.rvs(size=500) image1 = numpy.zeros(standard.shape) image2 = numpy.zeros(standard.shape) x,y,z = numpy.where(standard.get_data()==1) idx = list(zip(x,y,z)) image1_voxels = unzip(idx[0:500]) image2_voxels = unzip(idx[1500:2000]) image1[image1_voxels] = data1 image2[image2_voxels] = data2 image1 = nibabel.nifti1.Nifti1Image(image1,affine=standard.get_affine(),header=standard.get_header()) image2 = nibabel.nifti1.Nifti1Image(image2,affine=standard.get_affine(),header=standard.get_header()) html_snippet,data_table = scatterplot_compare(images=[image1,image2], reference = standard, image_names=["image 1","image 2"], corr_type="pearson") html_snippet = " ".join(html_snippet) assert_true(bool(error.search(html_snippet))) # Case 2: fewer than 3 voxels overlapping data1 = norm.rvs(size=2) data2 = norm.rvs(size=2) image1 = numpy.zeros(standard.shape) image2 = numpy.zeros(standard.shape) x,y,z = numpy.where(standard.get_data()==1) idx = list(zip(x,y,z)) idx = unzip(idx[10:12]) image1[idx] = data1 image2[idx] = data2 image1 = nibabel.nifti1.Nifti1Image(image1,affine=standard.get_affine(),header=standard.get_header()) image2 = nibabel.nifti1.Nifti1Image(image2,affine=standard.get_affine(),header=standard.get_header()) html_snippet,data_table = scatterplot_compare(images=[image1,image2], reference = standard, image_names=["image 1","image 2"], corr_type="pearson") html_snippet = " ".join(html_snippet) assert_true(bool(error.search(html_snippet))) # Case 2: But 3 should work data1 = norm.rvs(size=3) data2 = norm.rvs(size=3) image1 = numpy.zeros(standard.shape) image2 = numpy.zeros(standard.shape) x,y,z = numpy.where(standard.get_data()==1) idx = list(zip(x,y,z)) idx = unzip(idx[10:13]) image1[idx] = data1 image2[idx] = data2 image1 = nibabel.nifti1.Nifti1Image(image1,affine=standard.get_affine(),header=standard.get_header()) image2 = nibabel.nifti1.Nifti1Image(image2,affine=standard.get_affine(),header=standard.get_header()) html_snippet,data_table = scatterplot_compare(images=[image1,image2], reference = standard, image_names=["image 1","image 2"], corr_type="pearson") html_snippet = " ".join(html_snippet) assert_false(bool(error.search(html_snippet)))