def test_searchlight_with_diamond(): sl = Searchlight(sl_rad=3, shape=Diamond) comm = MPI.COMM_WORLD rank = comm.rank size = comm.size dim0, dim1, dim2 = (50, 50, 50) ntr = 30 nsubj = 3 mask = np.zeros((dim0, dim1, dim2), dtype=np.bool) data = [np.empty((dim0, dim1, dim2, ntr), dtype=np.object) if i % size == rank else None for i in range(0, nsubj)] # Put a spot in the mask mask[10:17, 10:17, 10:17] = Diamond(3).mask_ sl.distribute(data, mask) global_outputs = sl.run_searchlight(diamond_sfn) if rank == 0: assert global_outputs[13, 13, 13] == 1.0 global_outputs[13, 13, 13] = None for i in range(global_outputs.shape[0]): for j in range(global_outputs.shape[1]): for k in range(global_outputs.shape[2]): assert global_outputs[i, j, k] is None
def voxel_test(data, mask, max_blk_edge, rad): comm = MPI.COMM_WORLD rank = comm.rank (dim0, dim1, dim2) = mask.shape # Initialize dataset with known pattern for subj in data: if subj is not None: for tr in range(subj.shape[3]): for d1 in range(dim0): for d2 in range(dim1): for d3 in range(dim2): subj[d1, d2, d3, tr] = np.array( [d1, d2, d3, tr]) sl = Searchlight(sl_rad=rad, max_blk_edge=max_blk_edge) sl.distribute(data, mask) sl.broadcast(MaskRadBcast(mask, rad)) global_outputs = sl.run_searchlight(voxel_test_sfn) if rank == 0: for d0 in range(rad, global_outputs.shape[0]-rad): for d1 in range(rad, global_outputs.shape[1]-rad): for d2 in range(rad, global_outputs.shape[2]-rad): if mask[d0, d1, d2]: assert np.array_equal( np.array(global_outputs[d0, d1, d2]), np.array([d0, d1, d2]))
def run_anal(mask_path: str, bold_path: str, sl_rad: float, max_blk_edge: int, pool_size: int, nn_paths: list, part: int, save_dir: str, rsa) -> bool: """ DOCS: TODO: """ # extract the subject string from the bold path sub = bold_path.split("/")[-1].split(".")[0].split("_")[-1] print("starting run {}".format(sub)) # this is to account for if we're using part 1 of the movie clip or part 2 if part == 1: tr_start_idx = 0 tr_end_idx = 946 elif part == 2: tr_start_idx = 946 tr_end_idx = 1976 # get the correct correlation matrices for alexnet nncor_files = nnpart_files(part, nn_paths) # nibabel load, this loads an object but not the numpy arrays of the data mask_obj = nib.load(mask_path) bold_obj = nib.load(bold_path) # converting to numpy arrays data = np.array(bold_obj.dataobj)[:, :, :, tr_start_idx:tr_end_idx] mask = np.array(mask_obj.dataobj).astype(int) sl = Searchlight(sl_rad=sl_rad, max_blk_edge=max_blk_edge) sl.distribute([data], mask) counter = 0 for nn_path in nncor_files: # this is alexnet bcvar = np.load(nn_path) save_name = nn_path.split("/")[-1].split(".")[0] # broadcast the NN matrix sl.broadcast(bcvar) # now the rsa part sl_result = sl.run_searchlight(rsa, pool_size=pool_size) sl_result[sl_result == None] = 0 sl_data = np.array(sl_result, dtype=float) sl_img = nib.Nifti1Image(sl_data, affine=mask_obj.affine) # save the result to file as a nifti image save_path = os.path.join(save_dir, save_name + "_" + sub) sl_img.to_filename(save_path + ".nii.gz") counter += 1 if counter == len(nncor_files): print("\nCOMPLETE SUCCESSFULLY\n") return True print("\nSOMETHING FAILED\n") return False
def predict(sample, models, mask): test_searchlight = Searchlight(sl_rad=2, shape=Diamond) test_searchlight.distribute([sample, models[..., np.newaxis]], mask) test_classes = test_searchlight.run_searchlight(test_models) test_display = np.empty(test_classes.shape, dtype=float) test_display[test_classes == "face"] = 0.5 test_display[test_classes == "house"] = 1 test_display[test_classes == None] = np.nan return test_display
def voxel_test(data, mask, max_blk_edge, rad): comm = MPI.COMM_WORLD rank = comm.rank size = comm.size nsubj = len(data) (dim0, dim1, dim2) = mask.shape # Initialize dataset with known pattern for subj in data: if subj is not None: for tr in range(subj.shape[3]): for d1 in range(dim0): for d2 in range(dim1): for d3 in range(dim2): subj[d1,d2,d3,tr] = np.array([d1, d2, d3, tr]) def sfn(l,msk,myrad,bcast_var): # Check each point for subj in l: for _tr in range(subj.shape[3]): tr = subj[:,:,:,_tr] midpt = tr[rad,rad,rad] for d0 in range(tr.shape[0]): for d1 in range(tr.shape[1]): for d2 in range(tr.shape[2]): assert np.array_equal(tr[d0,d1,d2] - midpt, np.array([d0-rad,d1-rad,d2-rad,0])) # Determine midpoint midpt = l[0][rad,rad,rad,0] midpt = (midpt[0], midpt[1], midpt[2]) for d0 in range(msk.shape[0]): for d1 in range(msk.shape[1]): for d2 in range(msk.shape[2]): pt = (midpt[0] - rad + d0, midpt[1] - rad + d1, midpt[2] - rad + d2) assert bcast_var[pt] == msk[d0,d1,d2] # Return midpoint return midpt sl = Searchlight(sl_rad=rad, max_blk_edge=max_blk_edge) sl.distribute(data, mask) sl.broadcast(mask) global_outputs = sl.run_searchlight(sfn) if rank == 0: for d0 in range(rad, global_outputs.shape[0]-rad): for d1 in range(rad, global_outputs.shape[1]-rad): for d2 in range(rad, global_outputs.shape[2]-rad): if mask[d0, d1, d2]: assert np.array_equal(np.array(global_outputs[d0,d1,d2]), np.array([d0,d1,d2]))
# Load functional data and mask data data1 = load_img(datadir + 'subjects/' + subjs[i] + '/data/avg_reorder1.nii') data2 = load_img(datadir + 'subjects/' + subjs[i] + '/data/avg_reorder2.nii') data1 = data1.get_data() data2 = data2.get_data() np.seterr(divide='ignore', invalid='ignore') # Create and run searchlight sl = Searchlight(sl_rad=1, max_blk_edge=5) sl.distribute([data1, data2], mask_img) sl.broadcast(None) print('Running Searchlight...') global_outputs = sl.run_searchlight(corr2_coeff) global_outputs_all[:, :, :, i] = global_outputs # Plot and save searchlight results global_outputs_avg = np.mean(global_outputs_all, 3) maxval = np.max(global_outputs_avg[np.not_equal(global_outputs_avg, None)]) minval = np.min(global_outputs_avg[np.not_equal(global_outputs_avg, None)]) global_outputs = np.array(global_outputs_avg, dtype=np.float) global_nonans = global_outputs[np.not_equal(global_outputs, None)] global_nonans = np.reshape(global_nonans, (91, 109, 91)) min1 = np.min(global_nonans[~np.isnan(global_nonans)]) max1 = np.max(global_nonans[~np.isnan(global_nonans)]) img = nib.Nifti1Image(global_nonans, np.eye(4)) img.header['cal_min'] = min1 img.header['cal_max'] = max1 nib.save(img, 'classical_within_permuted.nii.gz')
subjectName = brain_file[sub_idx:] output_name = searchlights_path + subjectName # Make the mask mask = node_brain != 0 mask = mask[:, :, :, 0] # Create searchlight object sl = Searchlight(sl_rad=1, max_blk_edge=5) # Distribute data to processes sl.distribute([node_brain], mask) sl.broadcast(None) # Run clusters sl_outputs = sl.run_searchlight(rsa_sl, pool_size=1) if rank == 0: # Convert the output into what can be used sl_outputs = sl_outputs.astype('double') sl_outputs[np.isnan(sl_outputs)] = 0 # Save the volume sl_nii = nibabel.Nifti1Image(sl_outputs, nii.affine) hdr = sl_nii.header hdr.set_zooms((dimsize[0], dimsize[1], dimsize[2])) nibabel.save(sl_nii, output_name) # Save
# say some things about the mask. print('mask dimensions: {}'.format(mask.shape)) print('number of voxels in mask: {}'.format(np.sum(mask))) sl_rad = radius max_blk_edge = 5 pool_size = 1 # Create the searchlight object sl = Searchlight(sl_rad=sl_rad, max_blk_edge=max_blk_edge) # Distribute the information to the searchlights (preparing it to run) sl.distribute(data, mask) sl.broadcast(bcvar) slstart = time.time() sl_result = sl.run_searchlight(Class) #result = Class(data, np.zeros((5,5,5)), 2, bcvar) SL = time.time() - slstart tot = time.time() - starttime print('total time: {}, searchlight time: {}'.format(tot, SL)) ''' # Only save the data if this is the first core if rank == 0: output = ('{}/{}_r{}.npy'.format(outloc, subject, radius)) #np.save(output, sl_result) sl_result = sl_result.astype('double') sl_result[np.isnan(sl_result)] = 0 # If there are nans we want this
labels_train = np.array(labels_train) # Train classifier clf = SVC(kernel='linear', C=1) clf.fit(bolddata_sl_train, labels_train) # Test classifier score = clf.score(bolddata_sl_test, labels_test) accuracy.append(score) return accuracy # Run the searchlight analysis print("Begin SearchLight in rank %s\n" % rank) all_sl_result = sl.run_searchlight(calc_svm, pool_size=pool_size) print("End SearchLight in rank %s\n" % rank) # Only save the data if this is the first core if rank == 0: all_sl_result = all_sl_result[mask == 1] all_sl_result = [num_subj * [0] if not n else n for n in all_sl_result] # replace all None # The average result avg_vol = np.zeros((mask.shape[0], mask.shape[1], mask.shape[2])) # Loop over subjects for sub_id in range(1, num_subj + 1): sl_result = [r[sub_id - 1] for r in all_sl_result] # reshape result_vol = np.zeros((mask.shape[0], mask.shape[1], mask.shape[2]))
srm = SRM(bcast_var[0], bcast_var[1]) srm.fit(train_data) # transform test data shared_data = srm.transform(test_data) for s in range(len(l)): shared_data[s] = np.nan_to_num( stats.zscore(shared_data[s], axis=1, ddof=1)) # experiment accu = timesegmentmatching_accuracy(shared_data, 6) return np.mean(accu), stats.sem( accu) # multiple outputs will be saved as tuples # Run searchlight global_outputs = sl.run_searchlight(sfn) # output is in shape (dim1,dim2,dim3) # Unpack and save result if rank == 0: acc = np.zeros((dim1, dim2, dim3)) se = np.zeros((dim1, dim2, dim3)) for i in range(dim1): for j in range(dim2): for k in range(dim3): if global_outputs[i][j][k] is not None: acc[i][j][k] = global_outputs[i][j][k][0] se[i][j][k] = global_outputs[i][j][k][1] print(acc) np.savez_compressed('data/sherlock/searchlight_srm_tsm_acc.npz', acc=acc, se=se)
min_active_voxels_proportion=0) # Distribute data to processes sl.distribute([data], mask) sl.broadcast(labels) # Define voxel function def sfn(l, msk, myrad, bcast_var): import sklearn.svm import sklearn.model_selection classifier = sklearn.svm.SVC() data = l[0][msk,:].T return np.mean(sklearn.model_selection.cross_val_score(classifier, data, bcast_var,n_jobs=1)) # Run searchlight global_outputs = sl.run_searchlight(sfn) # Visualize result if rank == 0: print(global_outputs) global_outputs = np.array(global_outputs, dtype=np.float) import matplotlib.pyplot as plt for (cnt, img) in enumerate(global_outputs): plt.imshow(img,cmap='hot',vmin=0,vmax=1) plt.savefig('img' + str(cnt) + '.png') plt.clf()
for i in range(len(subjs)): data_run2 = np.nan_to_num( load_img(datadir + 'subjects/' + subjs[i] + '/data/avg_reorder2.nii').get_data()) runs.append(data_run2) print("All Subjects Loaded") #np.seterr(divide='ignore',invalid='ignore') # Create and run searchlight sl = Searchlight(sl_rad=5, max_blk_edge=5) sl.distribute(runs, mask_img) sl.broadcast([nfeature, niter, loo_idx, exclude_songs]) print('Running Searchlight...') global_outputs = sl.run_searchlight(corr2_coeff, pool_size=1) global_outputs_all[:, :, :, i] = global_outputs # Plot and save searchlight results global_outputs_avg = np.mean(global_outputs_all, 3) #maxval = np.max(global_outputs_avg[np.not_equal(global_outputs_avg,None)]) #minval = np.min(global_outputs_avg[np.not_equal(global_outputs_avg,None)]) global_outputs_avg = np.array(global_outputs_avg, dtype=np.float) #global_nonans = global_outputs_avg[np.not_equal(global_outputs_avg,None)] #global_nonans = np.reshape(global_nonans,(91,109,91)) #img = nib.Nifti1Image(global_nonans, np.eye(4)) #img.header['cal_min'] = minval #img.header['cal_max'] = maxval #nib.save(img,datadir + 'prototype/link/scripts/data/searchlight_output/janice_srm_results/loo_' + subjs[loo_idx]) np.save( datadir +
classJazz_between = np.mean(corrAB[8:16,0:8]) jazzClass_between = np.mean(corrAB[0:8,8:16]) within_genre = np.mean([classical_within,jazz_within]) between_genre = np.mean([classJazz_between,jazzClass_between]) diff = within_genre - between_genre return diff comm.Barrier() begin_time = time.time() comm.Barrier() # Create and run searchlight sl = Searchlight(sl_rad=1,max_blk_edge=5) sl.distribute([data1,data2],mask_img) sl.broadcast(None) global_outputs = sl.run_searchlight(corr2_coeff) comm.Barrier() end_time = time.time() comm.Barrier() # Plot searchlight results if rank == 0: print('Searchlight Done: ', end_time - begin_time) maxval = np.max(global_outputs[np.not_equal(global_outputs,None)]) minval = np.min(global_outputs[np.not_equal(global_outputs,None)]) global_outputs = np.array(global_outputs, dtype=np.float) print(global_outputs) # Save searchlight images out_dir = "searchlight_images"
'/idata/cdl/data/fMRI/andy/sherlock/data/sherlock_movie_s%s_10000.nii.gz' % str(subid)).get_data() mask = data[:, :, :, 0] != 10000 model = np.load('/idata/cdl/data/fMRI/andy/sherlock/data/movie_corrmat.npy') params = dict(sl_rad=5) # Create searchlight object sl = Searchlight(**params) # Distribute data to processes sl.distribute([data], mask) sl.broadcast(model) # Define voxel function def sfn(l, msk, myrad, bcast_var): from scipy.spatial.distance import cdist from scipy.stats import pearsonr b = l[0][msk, :].T c = 1 - cdist(b, b, 'correlation').ravel() return pearsonr(c, bcast_var.ravel())[0] # Run searchlight result = sl.run_searchlight(sfn) np.save( '/idata/cdl/data/fMRI/andy/sherlock/analyses/searchlight_movie/s%s' % str(subid), result)
# Distribute data to processes sl.distribute([data], mask) sl.broadcast(labels) # Define voxel function def sfn(l, msk, myrad, bcast_var): import sklearn.svm import sklearn.model_selection classifier = sklearn.svm.SVC() data = l[0][msk, :].T return np.mean( sklearn.model_selection.cross_val_score(classifier, data, bcast_var, n_jobs=1)) # Run searchlight global_outputs = sl.run_searchlight(sfn) # Visualize result if rank == 0: print(global_outputs) global_outputs = np.array(global_outputs, dtype=np.float) import matplotlib.pyplot as plt for (cnt, img) in enumerate(global_outputs): plt.imshow(img, cmap='hot', vmin=0, vmax=1) plt.savefig('img' + str(cnt) + '.png') plt.clf()
d1,d2,d3,ntr = l[0].shape nvx = d1*d2*d3 for s in l: train_data.append(np.reshape(s[:,:,:,:int(ntr/2)],(nvx,int(ntr/2)))) test_data.append(np.reshape(s[:,:,:,int(ntr/2):],(nvx,ntr-int(ntr/2)))) # train an srm model srm = SRM(bcast_var[0],bcast_var[1]) srm.fit(train_data) # transform test data shared_data = srm.transform(test_data) for s in range(len(l)): shared_data[s] = np.nan_to_num(stats.zscore(shared_data[s],axis=1,ddof=1)) # run experiment accu = time_segment_matching_accuracy(shared_data) # return: can also return several values. In that case, the final output will be # a 3D array of tuples return np.mean(accu) # Run searchlight acc = sl.run_searchlight(sfn) # output is a 3D array in shape (dim1,dim2,dim3) # save result if rank == 0: print (acc) np.savez_compressed('data/searchlight_srm_tsm_acc.npz',acc=acc)