def transform_points(src, dst, transformfiles, resample_points=False, param_dictionary_for_reorientation=False): """ Inputs --------- src = numpy file consiting of nx3 (ZYX points) dst = folder location to write points transformfiles = list of all elastix transform files used, and in order of the original transform**** resample_points = [original_dims, resample_dims] if there was resampling done, use this here param_dictionary_for_reorientation = param_dictionary for lightsheet package to use for reorientation """ #load src = np.load(src) if src[-3:] == "npy" else loadmat( src)["cell_centers_orig_coord"] #reorient - test to make sure this works if param_dictionary_for_reorientation: from tools.imageprocessing.orientation import fix_contour_orientation kwargs = load_dictionary(param_dictionary_for_reorientation) if resample_points: from tools.imageprocessing.orientation import fix_dimension_orientation resample_points = [ fix_dimension_orientation(resample_points[0], **kwargs), resample_points[1] ] #fix dims src = fix_contour_orientation(src, axes=None, verbose=True, **kwargs) #optionally resample points if resample_points: original_dims, resample_dims = resample_points src = points_resample(src, original_dims, resample_dims) #generate text file pretransform_text_file = create_text_file_for_elastix(src, dst) #copy over elastix files transformfiles = modify_transform_files(transformfiles, dst) #run transformix on points points_file = point_transformix(pretransform_text_file, transformfiles[-1], dst) #convert registered points into structure counts unpack_pnts(points_file, dst) return
def generate_transformed_cellcount(dataframe, dst, transformfiles, lightsheet_parameter_dictionary, verbose=False): '''Function to take a csv file and generate an input to transformix Inputs ---------------- dataframe = preloaded pandas dataframe dst = destination to save files transformfiles = list of all elastix transform files used, and in order of the original transform**** lightsheet_parameter_file = .p file generated from lightsheet package ''' #set up locations transformed_dst = os.path.join(dst, 'transformed_points') makedir(transformed_dst) #make zyx numpy arry zyx = dataframe[['z', 'y', 'x']].values #adjust for reorientation THEN rescaling, remember full size data needs dimension change releative to resample kwargs = load_dictionary(lightsheet_parameter_dictionary) vol = [xx for xx in kwargs['volumes'] if xx.ch_type == 'cellch'][0] fullsizedimensions = get_fullsizedims_from_kwargs( kwargs ) #don't get from kwargs['volumes'][0].fullsizedimensions it's bad! use this instead zyx = fix_contour_orientation(zyx, verbose=verbose, **kwargs) #now in orientation of resample zyx = points_resample( zyx, original_dims=fix_dimension_orientation(fullsizedimensions, **kwargs), resample_dims=tifffile.imread(vol.resampled_for_elastix_vol).shape, verbose=verbose)[:, :3] #make into transformix-friendly text file pretransform_text_file = create_text_file_for_elastix(zyx, transformed_dst) #copy over elastix files transformfiles = modify_transform_files(transformfiles, transformed_dst) change_transform_parameter_initial_transform(transformfiles[0], 'NoInitialTransform') #run transformix on points points_file = point_transformix(pretransform_text_file, transformfiles[-1], transformed_dst) #convert registered points into structure counts converted_points = unpack_pnts(points_file, transformed_dst) return converted_points
def generate_transformed_cellcount(dataframe, dst, transformfiles, lightsheet_parameter_dictionary, verbose=False): """Function to take a csv file and generate an input to transformix Inputs ---------------- dataframe = preloaded pandas dataframe dst = destination to save files transformfiles = list of all elastix transform files used, and in order of the original transform**** lightsheet_parameter_file = .p file generated from lightsheet package """ #set up locations transformed_dst = os.path.join(dst, "transformed_points"); makedir(transformed_dst) #make zyx numpy arry zyx = dataframe[["z","y","x"]].values #adjust for reorientation THEN rescaling, remember full size data needs dimension change releative to resample fullsizedimensions = get_fullsizedimensions(lightsheet_parameter_dictionary) kwargs = load_kwargs(lightsheet_parameter_dictionary) zyx = fix_contour_orientation(zyx, verbose=verbose, **kwargs) #now in orientation of resample resampled_dims, resampled_vol = get_resampledvol_n_dimensions(lightsheet_parameter_dictionary) zyx = points_resample(zyx, original_dims = fix_dimension_orientation(fullsizedimensions, **kwargs), resample_dims = resampled_dims, verbose = verbose)[:, :3] #make into transformix-friendly text file pretransform_text_file = create_text_file_for_elastix(zyx, transformed_dst) #copy over elastix files transformfiles = modify_transform_files(transformfiles, transformed_dst) change_transform_parameter_initial_transform(transformfiles[0], "NoInitialTransform") #run transformix on points points_file = point_transformix(pretransform_text_file, transformfiles[-1], transformed_dst) #convert registered points into structure counts converted_points = unpack_pnts(points_file, transformed_dst) return converted_points
#####generate a downsized version###### if generate_downsized_overlay: #load centers zyx, dataframe = load_cell_centers(cell_centers_path) cellvolloaded = tifffile.imread(cellvol.resampled_for_elastix_vol) cnn_cellvolloaded = np.zeros_like(cellvolloaded) #adjust for reorientation THEN rescaling, remember full size data needs dimension change releative to resample fullsizedimensions = get_fullsizedims_from_kwargs( kwargs ) #don't get from kwargs['volumes'][0].fullsizedimensions it's bad! use this instead zyx = fix_contour_orientation( zyx, verbose=verbose, **kwargs) #now in orientation of resample zyx = points_resample(zyx, original_dims=fix_dimension_orientation( fullsizedimensions, **kwargs), resample_dims=tifffile.imread( cellvol.resampled_for_elastix_vol).shape, verbose=verbose)[:, :3] zyx = np.asarray( [str((int(xx[0]), int(xx[1]), int(xx[2]))) for xx in zyx]) zyx_cnt = Counter(zyx) #now overlay for zyx, v in zyx_cnt.items(): z, y, x = [ int(xx) for xx in zyx.replace( '(', '', ).replace(')', '').split(',') ] try:
def overlay_qc(args): #unpacking this way for multiprocessing fld, folder_suffix, output_folder, verbose, doubletransform, make_volumes = args try: #get 3dunet cell dataframe csv file input_csv = listdirfull(os.path.join(fld, folder_suffix), ".csv") assert len(input_csv) == 1, "multiple csv files" dataframe = pd.read_csv(input_csv[0]) #location to save out dst = os.path.join(output_folder, os.path.basename(fld)); makedir(dst) #EXAMPLE USING LIGHTSHEET - assumes marking centers in the "raw" full sized cell channel. This will transform those #centers into "atlas" space (in this case the moving image) #in this case the "inverse transform has the atlas as the moving image in the first step, #and the autofluorescence channel as the moving image in the second step #NOTE - it seems that the registration of cell to auto is failing on occasion....thus get new files... ################################ cell_inverse_folder = listdirfull(os.path.join(fld, "elastix_inverse_transform"), "cellch")[0] a2r = listall(cell_inverse_folder, "atlas2reg_TransformParameters"); a2r.sort() r2s = listall(cell_inverse_folder, "reg2sig_TransformParameters"); r2s.sort() #possibly remove #IMPORTANT. the idea is to apply cfos->auto->atlas transformfiles = r2s + a2r if doubletransform else a2r #might get rid of r2s lightsheet_parameter_dictionary = os.path.join(fld, "param_dict.p") converted_points = generate_transformed_cellcount(dataframe, dst, transformfiles, lightsheet_parameter_dictionary, verbose=verbose) #load and convert to single voxel loc zyx = np.asarray([str((int(xx[0]), int(xx[1]), int(xx[2]))) for xx in np.nan_to_num(np.load(converted_points))]) from collections import Counter zyx_cnt = Counter(zyx) #check... if make_volumes: #manually call transformix kwargs = load_dictionary(lightsheet_parameter_dictionary) vol = [xx for xx in kwargs["volumes"] if xx.ch_type == "cellch"][0].resampled_for_elastix_vol transformed_vol = os.path.join(dst, "transformed_volume"); makedir(transformed_vol) if not doubletransform: transformfiles = [os.path.join(fld, "elastix/TransformParameters.0.txt"), os.path.join(fld, "elastix/TransformParameters.1.txt")] transformfiles = modify_transform_files(transformfiles, transformed_vol) #copy over elastix files transformix_command_line_call(vol, transformed_vol, transformfiles[-1]) else: v=[xx for xx in kwargs["volumes"] if xx.ch_type == "cellch"][0] #sig to reg tps = [listall(os.path.dirname(v.ch_to_reg_to_atlas), "/TransformParameters.0")[0], listall(os.path.dirname(v.ch_to_reg_to_atlas), "/TransformParameters.1")[0]] #reg to atlas transformfiles = tps+[os.path.join(fld, "elastix/TransformParameters.0.txt"), os.path.join(fld, "elastix/TransformParameters.1.txt")] transformfiles = modify_transform_files(transformfiles, transformed_vol) #copy over elastix files transformix_command_line_call(vol, transformed_vol, transformfiles[-1]) #cell_registered channel cell_reg = tifffile.imread(os.path.join(transformed_vol, "result.tif")) tifffile.imsave(os.path.join(transformed_vol, "result.tif"), cell_reg, compress=1) cell_cnn = np.zeros_like(cell_reg) tarr = []; badlist=[] for zyx,v in zyx_cnt.items(): z,y,x = [int(xx) for xx in zyx.replace("(","",).replace(")","").split(",")] tarr.append([z,y,x]) try: cell_cnn[z,y,x] = v*100 except: badlist.append([z,y,x]) #apply x y dilation r = 2 selem = ball(r)[int(r/2)] cell_cnn = cell_cnn.astype("uint8") cell_cnn = np.asarray([cv2.dilate(cell_cnn[i], selem, iterations = 1) for i in range(cell_cnn.shape[0])]) tarr=np.asarray(tarr) if len(badlist)>0: print("{} errors in mapping with cell_cnn shape {}, each max dim {}, \npossibly due to a registration overshoot \ or not using double transform\n\n{}".format(len(badlist), cell_cnn.shape, np.max(tarr,0), badlist)) merged = np.stack([cell_cnn, cell_reg, np.zeros_like(cell_reg)], -1) tifffile.imsave(os.path.join(transformed_vol, "merged.tif"), merged)#, compress=1) #out = np.concatenate([cell_cnn, cell_reg, ], 0) #####check at the resampled for elastix phase before transform...this mapping looks good... if make_volumes: #make zyx numpy arry zyx = dataframe[["z","y","x"]].values kwargs = load_dictionary(lightsheet_parameter_dictionary) vol = [xx for xx in kwargs["volumes"] if xx.ch_type =="cellch"][0] fullsizedimensions = get_fullsizedims_from_kwargs(kwargs) #don"t get from kwargs["volumes"][0].fullsizedimensions it"s bad! use this instead zyx = fix_contour_orientation(zyx, verbose=verbose, **kwargs) #now in orientation of resample zyx = points_resample(zyx, original_dims = fix_dimension_orientation(fullsizedimensions, **kwargs), resample_dims = tifffile.imread(vol.resampled_for_elastix_vol).shape, verbose = verbose)[:, :3] #cell channel cell_ch = tifffile.imread(vol.resampled_for_elastix_vol) cell_cnn = np.zeros_like(cell_ch) tarr = []; badlist=[] for _zyx in zyx: z,y,x = [int(xx) for xx in _zyx] tarr.append([z,y,x]) try: cell_cnn[z,y,x] = 100 except: badlist.append([z,y,x]) tarr=np.asarray(tarr) merged = np.stack([cell_cnn, cell_ch, np.zeros_like(cell_ch)], -1) tifffile.imsave(os.path.join(transformed_vol, "resampled_merged.tif"), merged)#, compress=1) except Exception as e: print(e) with open(error_file, "a") as err_fl: err_fl.write("\n\n{} {}\n\n".format(fld, e))
dst0 = os.path.join(dst, os.path.basename(fld)); makedir(dst0) dst1 = os.path.join(dst0, 'elastix'); makedir(dst1) #####check cell detection (modeled from lightsheet/tools/registration/transform_cell_counts) #3dunet cell dataframe dataframe = pd.read_csv(listdirfull(os.path.join(fld, '3dunet_output/pooled_cell_measures'), '.csv')[0]) #####generate a downsized version###### if generate_downsized_overlay: cellvolloaded = tifffile.imread(cellvol.resampled_for_elastix_vol) cnn_cellvolloaded = np.zeros_like(cellvolloaded) zyx = dataframe[['z','y','x']].values #adjust for reorientation THEN rescaling, remember full size data needs dimension change releative to resample fullsizedimensions = get_fullsizedims_from_kwargs(kwargs) #don't get from kwargs['volumes'][0].fullsizedimensions it's bad! use this instead zyx = fix_contour_orientation(zyx, verbose=verbose, **kwargs) #now in orientation of resample zyx = points_resample(zyx, original_dims = fix_dimension_orientation(fullsizedimensions, **kwargs), resample_dims = tifffile.imread(cellvol.resampled_for_elastix_vol).shape, verbose = verbose)[:, :3] zyx = np.asarray([str((int(xx[0]), int(xx[1]), int(xx[2]))) for xx in load_np(zyx)]) from collections import Counter zyx_cnt = Counter(zyx) #now overlay for zyx,v in zyx_cnt.iteritems(): z,y,x = [int(xx) for xx in zyx.replace('(','',).replace(')','').split(',')] try: cnn_cellvolloaded[z,y,x] = v*100 except Exception, e: print e merged = np.stack([cnn_cellvolloaded, cellvolloaded, np.zeros_like(cellvolloaded)], -1) merged = np.swapaxes(merged, 0,2)#reorient to horizontal tifffile.imsave(os.path.join(dst, '{}_points_merged_resampled_for_elastix.tif'.format(os.path.basename(fld))), merged) #EXAMPLE USING LIGHTSHEET - assumes marking centers in the 'raw' full sized cell channel. This will transform those centers into "atlas" space (in this case the moving image)
def find_location(src, dst=False, correspondence_type='post_elastix', verbose=False): ''' Function to transform an excel sheet (e.g.: lightsheet/supp_files/sample_coordinate_to_location.xlsx) and output transformed locations. Suggestion is to use imagej to find XYZ coordinates to input into excel sheet. Inputs ---------------- src = excelsheet correspondence_type = 'post_elastix': your coordinates are the corresponding post-registered elastix file (outputfolder/elastix/..../result....tif) 'full_size_data': you coordinates are from the "full_sizedatafld" where: Z== #### in 'file_name_Z####.tif' X,Y are the pixel of that tif file Returns ---------------- dst = (optional) output excelfile. Ensure path ends with '.xlsx' ''' #from __future__ import division #import shutil, os, tifffile, cv2, numpy as np, pandas as pd, sys, SimpleITK as sitk #from tools.utils.io import listdirfull, load_kwargs, writer, makedir #from tools.conv_net.read_roi import read_roi, read_roi_zip from tools.registration.register import transformed_pnts_to_allen_helper_func from tools.registration.transform import structure_lister from tools.utils.io import load_kwargs, listdirfull, listall import SimpleITK as sitk import pandas as pd, numpy as np, os from skimage.external import tifffile if correspondence_type == 'post_elastix': print( 'This function assumes coordinates are from the corresponding post-registered elastix file. \nMake sure the excel file has number,<space>number,<space>number and not number,number,number' ) #inputs df = pd.read_excel(src) for brain in df.columns[1:]: print(brain) #load and find files kwargs = load_kwargs( df[brain][df['Inputs'] == 'Path to folder'][0]) ann = sitk.GetArrayFromImage( sitk.ReadImage(kwargs['annotationfile'])) #Look up coordinates to pixel value xyz_points = np.asarray([(int(xx.split(',')[0]), int(xx.split(',')[1]), int(xx.split(',')[2])) for xx in df[brain][3:].tolist()]) xyz_points = transformed_pnts_to_allen_helper_func(xyz_points, ann=ann, order='XYZ') #pixel id to transform if 'allen_id_table' in kwargs: structures = structure_lister( pd.read_excel(kwargs['allen_id_table']), *xyz_points) else: structures = structure_lister( pd.read_excel(kwargs['volumes'][0].allen_id_table), *xyz_points) #update dataframe df[brain + ' point transform'] = df[brain][:3].tolist() + [ str(s.tolist()[0]) for s in structures ] if not dst: dst = src[:-5] + '_output.xlsx' df.to_excel(dst) print('Saved as {}'.format(dst)) if correspondence_type == 'full_size_data': from tools.imageprocessing.orientation import fix_dimension_orientation, fix_contour_orientation from tools.utils.directorydeterminer import pth_update from tools.registration.register import collect_points_post_transformix from tools.registration.transform import points_resample, points_transform print( 'This function assumes coordinates are from the corresponding "full_sizedatafld". \nMake sure the excel file has number,<space>number,<space>number and not number,number,number' ) #inputs df = pd.read_excel(src) for brain in df.columns[1:]: #load and find files kwargs = load_kwargs( df[brain][df['Inputs'] == 'Path to folder'][0]) ann = sitk.GetArrayFromImage( sitk.ReadImage(kwargs['annotationfile'])) ch_type = str( df[brain][df['Inputs'] == 'Channel Type'].tolist()[0]) vol = [xx for xx in kwargs['volumes'] if xx.ch_type == ch_type][0] #Look up coordinates to pixel value zyx_points = np.asarray([(int(xx.split(',')[2]), int(xx.split(',')[1]), int(xx.split(',')[0])) for xx in df[brain][3:].tolist()]) #Fix orientation zyx_points = fix_contour_orientation(np.asarray(zyx_points), verbose=verbose, **kwargs) #Fix Scaling trnsfmdpnts = points_resample( zyx_points, original_dims=fix_dimension_orientation( vol.fullsizedimensions, **kwargs), resample_dims=tifffile.imread( pth_update(vol.resampled_for_elastix_vol)).shape, verbose=verbose) #write out points for transformix transformfile = [ xx for xx in listall(os.path.join(vol.inverse_elastixfld)) if os.path.basename(vol.full_sizedatafld_vol)[:-5] in xx and 'atlas2reg2sig' in xx and 'reg2sig_TransformParameters.1.txt' in xx ][0] tmpdst = os.path.join(os.path.dirname(src), 'coordinate_to_location_tmp') output = points_transform(src=trnsfmdpnts[:, :3], dst=tmpdst, transformfile=transformfile, verbose=True) #collect from transformix xyz_points = collect_points_post_transformix(output) #now ID: pix_ids = transformed_pnts_to_allen_helper_func(xyz_points, ann=ann, order='XYZ') #pixel id to transform aid = kwargs[ 'allen_id_table'] if 'allen_id_table' in kwargs else kwargs[ 'volumes'][0].allen_id_table structures = structure_lister(pd.read_excel(aid), *pix_ids) #update dataframe df[brain + ' xyz points atlas space'] = df[brain][:3].tolist() + [ str(s.tolist()[0]) for zyx in xyz_points ] df[brain + ' structures'] = df[brain][:3].tolist() + [ str(s.tolist()[0]) for s in structures ] if not dst: dst = src[:-5] + '_output.xlsx' df.to_excel(dst) print('Saved as {}'.format(dst)) return
# #zyx_list=[(1920, 326,322), (1920, 305,314), (1920, 1333, 1020), (1920, 1367, 1020), (1920, 1350, 1000), (1920, 1350, 1030)] zyx_list = [(1920, 1350, 1020)] #extract ROIs, fill contours, find nonzero pixels im = tifffile.imread(fl) #rois = read_roi_zip(roipth) #blnk = np.zeros((im.shape))#.astype('uint32') #for roi in rois: # cv2.fillPoly(blnk, [np.int32(roi)], 255) #plt.ion(); plt.figure(); plt.imshow(blnk[0]) #fix orientations from tools.imageprocessing.orientation import fix_contour_orientation, fix_dimension_orientation zyx_list = fix_contour_orientation(np.asarray(zyx_list), **kwargs) z, y, x = fix_dimension_orientation( kwargs['volumes'][0].fullsizedimensions, **kwargs) #account for point transform due to resizing nx4centers = np.ones((len(zyx_list), 4)) nx4centers[:, :-1] = zyx_list #inv zr, yr, xr = tifffile.imread( pth_update(kwargs['volumes'][0].resampled_for_elastix_vol)).shape trnsfrmmatrix = np.identity(4) * ( zr / z, yr / y, xr / x, 1) ###downscale to "resampledforelastix size" #nx4 * 4x4 to give transform trnsfmdpnts = nx4centers.dot(trnsfrmmatrix) ##z,y,x #write out points for transformix txtflnm = 'rois_xyz' os.remove(pth + '/' + txtflnm)