def transform_points(src,
dst,
transformfiles,
resample_points=False,
param_dictionary_for_reorientation=False):
"""
Inputs
---------
src = numpy file consiting of nx3 (ZYX points)
dst = folder location to write points
transformfiles =
list of all elastix transform files used, and in order of the original transform****
resample_points = [original_dims, resample_dims] if there was resampling done, use this here
param_dictionary_for_reorientation = param_dictionary for lightsheet package to use for reorientation
"""
#load
src = np.load(src) if src[-3:] == "npy" else loadmat(
src)["cell_centers_orig_coord"]
#reorient - test to make sure this works
if param_dictionary_for_reorientation:
from tools.imageprocessing.orientation import fix_contour_orientation
kwargs = load_dictionary(param_dictionary_for_reorientation)
if resample_points:
from tools.imageprocessing.orientation import fix_dimension_orientation
resample_points = [
fix_dimension_orientation(resample_points[0], **kwargs),
resample_points[1]
] #fix dims
src = fix_contour_orientation(src, axes=None, verbose=True, **kwargs)
#optionally resample points
if resample_points:
original_dims, resample_dims = resample_points
src = points_resample(src, original_dims, resample_dims)
#generate text file
pretransform_text_file = create_text_file_for_elastix(src, dst)
#copy over elastix files
transformfiles = modify_transform_files(transformfiles, dst)
#run transformix on points
points_file = point_transformix(pretransform_text_file, transformfiles[-1],
dst)
#convert registered points into structure counts
unpack_pnts(points_file, dst)
return
def generate_transformed_cellcount(dataframe,
dst,
transformfiles,
lightsheet_parameter_dictionary,
verbose=False):
'''Function to take a csv file and generate an input to transformix
Inputs
----------------
dataframe = preloaded pandas dataframe
dst = destination to save files
transformfiles = list of all elastix transform files used, and in order of the original transform****
lightsheet_parameter_file = .p file generated from lightsheet package
'''
#set up locations
transformed_dst = os.path.join(dst, 'transformed_points')
makedir(transformed_dst)
#make zyx numpy arry
zyx = dataframe[['z', 'y', 'x']].values
#adjust for reorientation THEN rescaling, remember full size data needs dimension change releative to resample
kwargs = load_dictionary(lightsheet_parameter_dictionary)
vol = [xx for xx in kwargs['volumes'] if xx.ch_type == 'cellch'][0]
fullsizedimensions = get_fullsizedims_from_kwargs(
kwargs
) #don't get from kwargs['volumes'][0].fullsizedimensions it's bad! use this instead
zyx = fix_contour_orientation(zyx, verbose=verbose,
**kwargs) #now in orientation of resample
zyx = points_resample(
zyx,
original_dims=fix_dimension_orientation(fullsizedimensions, **kwargs),
resample_dims=tifffile.imread(vol.resampled_for_elastix_vol).shape,
verbose=verbose)[:, :3]
#make into transformix-friendly text file
pretransform_text_file = create_text_file_for_elastix(zyx, transformed_dst)
#copy over elastix files
transformfiles = modify_transform_files(transformfiles, transformed_dst)
change_transform_parameter_initial_transform(transformfiles[0],
'NoInitialTransform')
#run transformix on points
points_file = point_transformix(pretransform_text_file, transformfiles[-1],
transformed_dst)
#convert registered points into structure counts
converted_points = unpack_pnts(points_file, transformed_dst)
return converted_points
def generate_transformed_cellcount(dataframe, dst, transformfiles, lightsheet_parameter_dictionary, verbose=False):
"""Function to take a csv file and generate an input to transformix
Inputs
----------------
dataframe = preloaded pandas dataframe
dst = destination to save files
transformfiles = list of all elastix transform files used, and in order of the original transform****
lightsheet_parameter_file = .p file generated from lightsheet package
"""
#set up locations
transformed_dst = os.path.join(dst, "transformed_points"); makedir(transformed_dst)
#make zyx numpy arry
zyx = dataframe[["z","y","x"]].values
#adjust for reorientation THEN rescaling, remember full size data needs dimension change releative to resample
fullsizedimensions = get_fullsizedimensions(lightsheet_parameter_dictionary)
kwargs = load_kwargs(lightsheet_parameter_dictionary)
zyx = fix_contour_orientation(zyx, verbose=verbose, **kwargs) #now in orientation of resample
resampled_dims, resampled_vol = get_resampledvol_n_dimensions(lightsheet_parameter_dictionary)
zyx = points_resample(zyx, original_dims = fix_dimension_orientation(fullsizedimensions,
**kwargs), resample_dims = resampled_dims, verbose = verbose)[:, :3]
#make into transformix-friendly text file
pretransform_text_file = create_text_file_for_elastix(zyx, transformed_dst)
#copy over elastix files
transformfiles = modify_transform_files(transformfiles, transformed_dst)
change_transform_parameter_initial_transform(transformfiles[0], "NoInitialTransform")
#run transformix on points
points_file = point_transformix(pretransform_text_file, transformfiles[-1], transformed_dst)
#convert registered points into structure counts
converted_points = unpack_pnts(points_file, transformed_dst)
return converted_points
#####generate a downsized version######
if generate_downsized_overlay:
#load centers
zyx, dataframe = load_cell_centers(cell_centers_path)
cellvolloaded = tifffile.imread(cellvol.resampled_for_elastix_vol)
cnn_cellvolloaded = np.zeros_like(cellvolloaded)
#adjust for reorientation THEN rescaling, remember full size data needs dimension change releative to resample
fullsizedimensions = get_fullsizedims_from_kwargs(
kwargs
) #don't get from kwargs['volumes'][0].fullsizedimensions it's bad! use this instead
zyx = fix_contour_orientation(
zyx, verbose=verbose,
**kwargs) #now in orientation of resample
zyx = points_resample(zyx,
original_dims=fix_dimension_orientation(
fullsizedimensions, **kwargs),
resample_dims=tifffile.imread(
cellvol.resampled_for_elastix_vol).shape,
verbose=verbose)[:, :3]
zyx = np.asarray(
[str((int(xx[0]), int(xx[1]), int(xx[2]))) for xx in zyx])
zyx_cnt = Counter(zyx)
#now overlay
for zyx, v in zyx_cnt.items():
z, y, x = [
int(xx) for xx in zyx.replace(
'(',
'',
).replace(')', '').split(',')
]
try:
def overlay_qc(args):
#unpacking this way for multiprocessing
fld, folder_suffix, output_folder, verbose, doubletransform, make_volumes = args
try:
#get 3dunet cell dataframe csv file
input_csv = listdirfull(os.path.join(fld, folder_suffix), ".csv")
assert len(input_csv) == 1, "multiple csv files"
dataframe = pd.read_csv(input_csv[0])
#location to save out
dst = os.path.join(output_folder, os.path.basename(fld)); makedir(dst)
#EXAMPLE USING LIGHTSHEET - assumes marking centers in the "raw" full sized cell channel. This will transform those
#centers into "atlas" space (in this case the moving image)
#in this case the "inverse transform has the atlas as the moving image in the first step,
#and the autofluorescence channel as the moving image in the second step
#NOTE - it seems that the registration of cell to auto is failing on occasion....thus get new files...
################################
cell_inverse_folder = listdirfull(os.path.join(fld, "elastix_inverse_transform"), "cellch")[0]
a2r = listall(cell_inverse_folder, "atlas2reg_TransformParameters"); a2r.sort()
r2s = listall(cell_inverse_folder, "reg2sig_TransformParameters"); r2s.sort() #possibly remove
#IMPORTANT. the idea is to apply cfos->auto->atlas
transformfiles = r2s + a2r if doubletransform else a2r #might get rid of r2s
lightsheet_parameter_dictionary = os.path.join(fld, "param_dict.p")
converted_points = generate_transformed_cellcount(dataframe, dst, transformfiles,
lightsheet_parameter_dictionary, verbose=verbose)
#load and convert to single voxel loc
zyx = np.asarray([str((int(xx[0]), int(xx[1]), int(xx[2]))) for xx in np.nan_to_num(np.load(converted_points))])
from collections import Counter
zyx_cnt = Counter(zyx)
#check...
if make_volumes:
#manually call transformix
kwargs = load_dictionary(lightsheet_parameter_dictionary)
vol = [xx for xx in kwargs["volumes"] if xx.ch_type == "cellch"][0].resampled_for_elastix_vol
transformed_vol = os.path.join(dst, "transformed_volume"); makedir(transformed_vol)
if not doubletransform:
transformfiles = [os.path.join(fld, "elastix/TransformParameters.0.txt"), os.path.join(fld,
"elastix/TransformParameters.1.txt")]
transformfiles = modify_transform_files(transformfiles, transformed_vol) #copy over elastix files
transformix_command_line_call(vol, transformed_vol, transformfiles[-1])
else:
v=[xx for xx in kwargs["volumes"] if xx.ch_type == "cellch"][0]
#sig to reg
tps = [listall(os.path.dirname(v.ch_to_reg_to_atlas), "/TransformParameters.0")[0],
listall(os.path.dirname(v.ch_to_reg_to_atlas), "/TransformParameters.1")[0]]
#reg to atlas
transformfiles = tps+[os.path.join(fld, "elastix/TransformParameters.0.txt"),
os.path.join(fld, "elastix/TransformParameters.1.txt")]
transformfiles = modify_transform_files(transformfiles, transformed_vol) #copy over elastix files
transformix_command_line_call(vol, transformed_vol, transformfiles[-1])
#cell_registered channel
cell_reg = tifffile.imread(os.path.join(transformed_vol, "result.tif"))
tifffile.imsave(os.path.join(transformed_vol, "result.tif"), cell_reg, compress=1)
cell_cnn = np.zeros_like(cell_reg)
tarr = []; badlist=[]
for zyx,v in zyx_cnt.items():
z,y,x = [int(xx) for xx in zyx.replace("(","",).replace(")","").split(",")]
tarr.append([z,y,x])
try:
cell_cnn[z,y,x] = v*100
except:
badlist.append([z,y,x])
#apply x y dilation
r = 2
selem = ball(r)[int(r/2)]
cell_cnn = cell_cnn.astype("uint8")
cell_cnn = np.asarray([cv2.dilate(cell_cnn[i], selem, iterations = 1) for i in range(cell_cnn.shape[0])])
tarr=np.asarray(tarr)
if len(badlist)>0:
print("{} errors in mapping with cell_cnn shape {}, each max dim {}, \npossibly due to a registration overshoot \
or not using double transform\n\n{}".format(len(badlist), cell_cnn.shape, np.max(tarr,0), badlist))
merged = np.stack([cell_cnn, cell_reg, np.zeros_like(cell_reg)], -1)
tifffile.imsave(os.path.join(transformed_vol, "merged.tif"), merged)#, compress=1)
#out = np.concatenate([cell_cnn, cell_reg, ], 0)
#####check at the resampled for elastix phase before transform...this mapping looks good...
if make_volumes:
#make zyx numpy arry
zyx = dataframe[["z","y","x"]].values
kwargs = load_dictionary(lightsheet_parameter_dictionary)
vol = [xx for xx in kwargs["volumes"] if xx.ch_type =="cellch"][0]
fullsizedimensions = get_fullsizedims_from_kwargs(kwargs) #don"t get from kwargs["volumes"][0].fullsizedimensions it"s bad! use this instead
zyx = fix_contour_orientation(zyx, verbose=verbose, **kwargs) #now in orientation of resample
zyx = points_resample(zyx, original_dims = fix_dimension_orientation(fullsizedimensions, **kwargs),
resample_dims = tifffile.imread(vol.resampled_for_elastix_vol).shape, verbose = verbose)[:, :3]
#cell channel
cell_ch = tifffile.imread(vol.resampled_for_elastix_vol)
cell_cnn = np.zeros_like(cell_ch)
tarr = []; badlist=[]
for _zyx in zyx:
z,y,x = [int(xx) for xx in _zyx]
tarr.append([z,y,x])
try:
cell_cnn[z,y,x] = 100
except:
badlist.append([z,y,x])
tarr=np.asarray(tarr)
merged = np.stack([cell_cnn, cell_ch, np.zeros_like(cell_ch)], -1)
tifffile.imsave(os.path.join(transformed_vol, "resampled_merged.tif"), merged)#, compress=1)
except Exception as e:
print(e)
with open(error_file, "a") as err_fl:
err_fl.write("\n\n{} {}\n\n".format(fld, e))
dst0 = os.path.join(dst, os.path.basename(fld)); makedir(dst0)
dst1 = os.path.join(dst0, 'elastix'); makedir(dst1)
#####check cell detection (modeled from lightsheet/tools/registration/transform_cell_counts)
#3dunet cell dataframe
dataframe = pd.read_csv(listdirfull(os.path.join(fld, '3dunet_output/pooled_cell_measures'), '.csv')[0])
#####generate a downsized version######
if generate_downsized_overlay:
cellvolloaded = tifffile.imread(cellvol.resampled_for_elastix_vol)
cnn_cellvolloaded = np.zeros_like(cellvolloaded)
zyx = dataframe[['z','y','x']].values
#adjust for reorientation THEN rescaling, remember full size data needs dimension change releative to resample
fullsizedimensions = get_fullsizedims_from_kwargs(kwargs) #don't get from kwargs['volumes'][0].fullsizedimensions it's bad! use this instead
zyx = fix_contour_orientation(zyx, verbose=verbose, **kwargs) #now in orientation of resample
zyx = points_resample(zyx, original_dims = fix_dimension_orientation(fullsizedimensions, **kwargs), resample_dims = tifffile.imread(cellvol.resampled_for_elastix_vol).shape, verbose = verbose)[:, :3]
zyx = np.asarray([str((int(xx[0]), int(xx[1]), int(xx[2]))) for xx in load_np(zyx)])
from collections import Counter
zyx_cnt = Counter(zyx)
#now overlay
for zyx,v in zyx_cnt.iteritems():
z,y,x = [int(xx) for xx in zyx.replace('(','',).replace(')','').split(',')]
try:
cnn_cellvolloaded[z,y,x] = v*100
except Exception, e:
print e
merged = np.stack([cnn_cellvolloaded, cellvolloaded, np.zeros_like(cellvolloaded)], -1)
merged = np.swapaxes(merged, 0,2)#reorient to horizontal
tifffile.imsave(os.path.join(dst, '{}_points_merged_resampled_for_elastix.tif'.format(os.path.basename(fld))), merged)
#EXAMPLE USING LIGHTSHEET - assumes marking centers in the 'raw' full sized cell channel. This will transform those centers into "atlas" space (in this case the moving image)
def find_location(src,
dst=False,
correspondence_type='post_elastix',
verbose=False):
'''
Function to transform an excel sheet (e.g.: lightsheet/supp_files/sample_coordinate_to_location.xlsx) and output transformed locations.
Suggestion is to use imagej to find XYZ coordinates to input into excel sheet.
Inputs
----------------
src = excelsheet
correspondence_type =
'post_elastix': your coordinates are the corresponding post-registered elastix file (outputfolder/elastix/..../result....tif)
'full_size_data': you coordinates are from the "full_sizedatafld" where:
Z== #### in 'file_name_Z####.tif'
X,Y are the pixel of that tif file
Returns
----------------
dst = (optional) output excelfile. Ensure path ends with '.xlsx'
'''
#from __future__ import division
#import shutil, os, tifffile, cv2, numpy as np, pandas as pd, sys, SimpleITK as sitk
#from tools.utils.io import listdirfull, load_kwargs, writer, makedir
#from tools.conv_net.read_roi import read_roi, read_roi_zip
from tools.registration.register import transformed_pnts_to_allen_helper_func
from tools.registration.transform import structure_lister
from tools.utils.io import load_kwargs, listdirfull, listall
import SimpleITK as sitk
import pandas as pd, numpy as np, os
from skimage.external import tifffile
if correspondence_type == 'post_elastix':
print(
'This function assumes coordinates are from the corresponding post-registered elastix file. \nMake sure the excel file has number,<space>number,<space>number and not number,number,number'
)
#inputs
df = pd.read_excel(src)
for brain in df.columns[1:]:
print(brain)
#load and find files
kwargs = load_kwargs(
df[brain][df['Inputs'] == 'Path to folder'][0])
ann = sitk.GetArrayFromImage(
sitk.ReadImage(kwargs['annotationfile']))
#Look up coordinates to pixel value
xyz_points = np.asarray([(int(xx.split(',')[0]),
int(xx.split(',')[1]),
int(xx.split(',')[2]))
for xx in df[brain][3:].tolist()])
xyz_points = transformed_pnts_to_allen_helper_func(xyz_points,
ann=ann,
order='XYZ')
#pixel id to transform
if 'allen_id_table' in kwargs:
structures = structure_lister(
pd.read_excel(kwargs['allen_id_table']), *xyz_points)
else:
structures = structure_lister(
pd.read_excel(kwargs['volumes'][0].allen_id_table),
*xyz_points)
#update dataframe
df[brain + ' point transform'] = df[brain][:3].tolist() + [
str(s.tolist()[0]) for s in structures
]
if not dst: dst = src[:-5] + '_output.xlsx'
df.to_excel(dst)
print('Saved as {}'.format(dst))
if correspondence_type == 'full_size_data':
from tools.imageprocessing.orientation import fix_dimension_orientation, fix_contour_orientation
from tools.utils.directorydeterminer import pth_update
from tools.registration.register import collect_points_post_transformix
from tools.registration.transform import points_resample, points_transform
print(
'This function assumes coordinates are from the corresponding "full_sizedatafld". \nMake sure the excel file has number,<space>number,<space>number and not number,number,number'
)
#inputs
df = pd.read_excel(src)
for brain in df.columns[1:]:
#load and find files
kwargs = load_kwargs(
df[brain][df['Inputs'] == 'Path to folder'][0])
ann = sitk.GetArrayFromImage(
sitk.ReadImage(kwargs['annotationfile']))
ch_type = str(
df[brain][df['Inputs'] == 'Channel Type'].tolist()[0])
vol = [xx for xx in kwargs['volumes'] if xx.ch_type == ch_type][0]
#Look up coordinates to pixel value
zyx_points = np.asarray([(int(xx.split(',')[2]),
int(xx.split(',')[1]),
int(xx.split(',')[0]))
for xx in df[brain][3:].tolist()])
#Fix orientation
zyx_points = fix_contour_orientation(np.asarray(zyx_points),
verbose=verbose,
**kwargs)
#Fix Scaling
trnsfmdpnts = points_resample(
zyx_points,
original_dims=fix_dimension_orientation(
vol.fullsizedimensions, **kwargs),
resample_dims=tifffile.imread(
pth_update(vol.resampled_for_elastix_vol)).shape,
verbose=verbose)
#write out points for transformix
transformfile = [
xx for xx in listall(os.path.join(vol.inverse_elastixfld))
if os.path.basename(vol.full_sizedatafld_vol)[:-5] in xx
and 'atlas2reg2sig' in xx
and 'reg2sig_TransformParameters.1.txt' in xx
][0]
tmpdst = os.path.join(os.path.dirname(src),
'coordinate_to_location_tmp')
output = points_transform(src=trnsfmdpnts[:, :3],
dst=tmpdst,
transformfile=transformfile,
verbose=True)
#collect from transformix
xyz_points = collect_points_post_transformix(output)
#now ID:
pix_ids = transformed_pnts_to_allen_helper_func(xyz_points,
ann=ann,
order='XYZ')
#pixel id to transform
aid = kwargs[
'allen_id_table'] if 'allen_id_table' in kwargs else kwargs[
'volumes'][0].allen_id_table
structures = structure_lister(pd.read_excel(aid), *pix_ids)
#update dataframe
df[brain + ' xyz points atlas space'] = df[brain][:3].tolist() + [
str(s.tolist()[0]) for zyx in xyz_points
]
df[brain + ' structures'] = df[brain][:3].tolist() + [
str(s.tolist()[0]) for s in structures
]
if not dst: dst = src[:-5] + '_output.xlsx'
df.to_excel(dst)
print('Saved as {}'.format(dst))
return
#
#zyx_list=[(1920, 326,322), (1920, 305,314), (1920, 1333, 1020), (1920, 1367, 1020), (1920, 1350, 1000), (1920, 1350, 1030)]
zyx_list = [(1920, 1350, 1020)]
#extract ROIs, fill contours, find nonzero pixels
im = tifffile.imread(fl)
#rois = read_roi_zip(roipth)
#blnk = np.zeros((im.shape))#.astype('uint32')
#for roi in rois:
# cv2.fillPoly(blnk, [np.int32(roi)], 255)
#plt.ion(); plt.figure(); plt.imshow(blnk[0])
#fix orientations
from tools.imageprocessing.orientation import fix_contour_orientation, fix_dimension_orientation
zyx_list = fix_contour_orientation(np.asarray(zyx_list), **kwargs)
z, y, x = fix_dimension_orientation(
kwargs['volumes'][0].fullsizedimensions, **kwargs)
#account for point transform due to resizing
nx4centers = np.ones((len(zyx_list), 4))
nx4centers[:, :-1] = zyx_list #inv
zr, yr, xr = tifffile.imread(
pth_update(kwargs['volumes'][0].resampled_for_elastix_vol)).shape
trnsfrmmatrix = np.identity(4) * (
zr / z, yr / y, xr / x, 1) ###downscale to "resampledforelastix size"
#nx4 * 4x4 to give transform
trnsfmdpnts = nx4centers.dot(trnsfrmmatrix) ##z,y,x
#write out points for transformix
txtflnm = 'rois_xyz'
os.remove(pth + '/' + txtflnm)