def cell_detect_qc_wrapper(src=False, **kwargs):
'''Function to take in pth to brain folder, and output qc data
src = destination of package (main folder)
'''
#load
if src: kwargs.update(load_kwargs(src))
if not src: kwargs.update(load_kwargs(**kwargs))
#run for each cellch
for cellch in [xx for xx in kwargs['volumes'] if xx.ch_type == 'cellch']:
1
#cell_detect(src = cellch.full_sizedatafld_vol, dst = os.path.join(os.path.dirname(os.path.dirname(cellch.full_sizedatafld_vol)), 'cells', os.path.basename(cellch.full_sizedatafld_vol)), **kwargs)
return
def run_downsizing(pth, cores=12):
"""
function to take a whole brain volume processed using BrainPipe and downsize it to a
specific resolution in XY
useful if you are trying to make an atlas with a specific final resolution (e.g. 25um in XYZ)
"""
print(os.path.basename(pth))
kwargs = load_kwargs(
pth
) #note: this function specifically relies on the paramter dictionary made when processing
#if you move the processed directory to another location, the paths initially saved in this dictionary will be incorrect
#so suggest you do not move processed directories, or see the BrainPipe scripts on how to correct it if you did
regvol = kwargs["volumes"][0]
print(kwargs["volumes"])
fszdt = regvol.full_sizedatafld_vol
dst = os.path.join(
pth, "downsized_for_atlas") #set destination for downsized planes
if not os.path.exists(dst): os.mkdir(dst) #make dest directory
plns = [
os.path.join(fszdt, xx) for xx in os.listdir(fszdt) if "tif" in xx
]
plns.sort()
iterlst = [(pln, dst) for pln in plns]
p = mp.Pool(cores)
p.starmap(downsize, iterlst)
def detect_contours_in_3d_checker(jobid, pln_chnk=50, **kwargs):
'''Not utilized yet
'''
kwargs = load_kwargs(**kwargs)
outdr = kwargs['outputdirectory']
vols = kwargs['volumes']
reg_vol = [xx for xx in vols if xx.ch_type == 'regch'][0]
###set volume to use
vol = [xx for xx in vols if xx.ch_type != 'regch'][jobid]
if vol.ch_type == 'cellch':
detect3dfld = reg_vol.celldetect3dfld
coordinatesfld = reg_vol.cellcoordinatesfld
elif vol.ch_type == 'injch':
detect3dfld = reg_vol.injdetect3dfld
coordinatesfld = reg_vol.injcoordinatesfld
zmax = vols[0].fullsizedimensions[0]
if len(os.listdir(detect3dfld)) != int(ceil(zmax / pln_chnk)):
writer(
outdr,
'\n\n***************************STEP 4 FAILED*********************\n{} files found in {}. Should have {}.'
.format(len(os.listdir(contourdetect3dfld)),
contourdetect3dfld[contourdetect3dfld.rfind('/') + 1:],
int(ceil(zmax / pln_chnk))))
else:
writer(
outdr,
'\n\n***************************STEP 4 SUCCESS*********************\n{} files found in {}. Should have {}.'
.format(len(os.listdir(contourdetect3dfld)),
contourdetect3dfld[contourdetect3dfld.rfind('/') + 1:],
int(ceil(zmax / pln_chnk))))
return
def elastix_wrapper(jobid, cores=5, **kwargs):
'''Wrapper to handle most registration operations.
jobid =
0: 'normal registration'
1: 'cellchannel inverse'
2: 'injchannel inverse'
'''
#inputs
kwargs = load_kwargs(**kwargs)
sys.stdout.write('\nElastix in:\n')
sys.stdout.flush()
os.system('which elastix')
#'normal' registration
if jobid == 0: elastix_registration(jobid, cores=cores, **kwargs)
#cellchannel inverse
if jobid == 1:
make_inverse_transform(
[xx for xx in kwargs['volumes'] if xx.ch_type == 'cellch'][0],
cores=cores,
**kwargs)
#injchannel inverse -- ##FIXME think about limiting the search to only the cerebellum
if jobid == 2:
#make inverse transform
transformfile = make_inverse_transform(
[xx for xx in kwargs['volumes'] if xx.ch_type == 'injch'][0],
cores=cores,
**kwargs)
#detect injection site ##FIXME need to define image and pass in appropriate thresh/filter-kernels
inj = [xx for xx in kwargs['volumes'] if xx.ch_type == 'injch'][0]
#array = find_site(inj.ch_to_reg_to_atlas+'/result.1.tif', thresh=10, filter_kernel=(5,5,5))
array = find_site(inj.resampled_for_elastix_vol,
thresh=10,
filter_kernel=(5, 5, 5)).astype(int)
#old version
#array = inj_detect_using_labels(threshold = .15, resampledforelastix = True, num_labels_to_keep=1, show = False, save = True, masking = True, **kwargs)
#apply resizing point transform
txtflnm = point_transform_due_to_resizing(array,
chtype='injch',
**kwargs)
#run transformix on points
points_file = point_transformix(txtflnm, transformfile)
#convert registered points into structure counts
transformed_pnts_to_allen(points_file,
ch_type='injch',
point_or_index=None,
**kwargs)
return
def fill_params(expt_name, stepid, jobid):
params = {}
# slurm params
params["stepid"] = stepid
params["jobid"] = jobid
# experiment params
# going one folder up to get to fullsizedata
params["expt_name"] = os.path.basename(
os.path.abspath(os.path.dirname(expt_name)))
# find cell channel tiff directory from parameter dict
kwargs = load_kwargs(os.path.dirname(expt_name))
vol = [vol for vol in kwargs["volumes"]]
src = vol.full_sizedatafld_vol
assert os.path.isdir(src), "nonexistent data directory"
print("\n\n data directory: {}".format(src))
params["cellch_dir"] = src
params["scratch_dir"] = "/jukebox/scratch/ejdennis"
params["data_dir"] = os.path.join(params["scratch_dir"],
params["expt_name"])
# changed paths after cnn run
params["cnn_data_dir"] = os.path.join(params["scratch_dir"],
params["expt_name"])
params["cnn_dir"] = os.path.join(params["cnn_data_dir"],
"output_chnks") # set cnn patch directory
params["reconstr_arr"] = os.path.join(params["cnn_data_dir"],
"reconstructed_array.npy")
params["output_dir"] = expt_name
# pre-processing params
params["dtype"] = "float32"
params["cores"] = 8
params["verbose"] = True
params["cleanup"] = False
# cnn window size for lightsheet =
# typically 20, 192, 192 for 4x, 20, 32, 32 for 1.3x
params["patchsz"] = (60, 3840, 3328)
params["stridesz"] = (40, 3648, 3136)
params["window"] = (20, 192, 192)
params["inputshape"] = get_dims_from_folder(src)
params["patchlist"] = make_indices(params["inputshape"],
params["stridesz"])
# post-processing params
params["threshold"] = (0.5, 1) # h129 = 0.6; prv = 0.85
params["zsplt"] = 30
params["ovlp_plns"] = 30
return params
def fix_orientation(imstack, axes=None, verbose=False, **kwargs):
'''Function to fix orientation of imaging series relative to atlas. Note assumes inputs of tuple of STRINGS**** not ints.
This allows for '-0' which is need if one wants to reverse the x axis.
Assumes XYZ orientation ('0','1','2'). To reverse the order of an axis add a '-'. I.e. to flip the Y axis: ('0', '-1', '2').
Order of operations is reversing of axes BEFORE swapping axes. E.g: ('2','-0','1'). This means reverse the X axis, then move: X->Y, Z->X, Y->Z.
Inputs
--------------------
imstack: np.array of image
axes (optional): tuple of strs ('0','1','2')
verbose (optional): gives information about shape before and after
kwargs (optional): will look for kwargs['finalorientation'] = ('0','1','2')
Returns
--------------------
imstack: numpy array reorientated
'''
#handle inputs
if not axes:
try:
axes = kwargs['finalorientation']
except KeyError:
#reload param dictionary from original run_tracing file and update kwargs
import tools.utils.io as io
sys.path.append(kwargs['packagedirectory'])
import run_tracing
kwargs.update(run_tracing.params)
io.save_kwargs(**kwargs)
#reload kwargs now that it has been updated
kwargs = io.load_kwargs(kwargs['outputdirectory'])
axes = kwargs['finalorientation']
#verbosity:
if verbose:
shape_before = imstack.shape
origax = axes
#handle reversing of an axis
imstack = reverse_axis(imstack, axes)
#change from XYZ to ZYX (np's) convention and remove negatives from axes and change to ints now that axes have been flipped
axes = [abs(2 - abs(int(xx))) for xx in axes][::-1]
#swap axes:
imstack = imstack.transpose(axes)
#verbosity:
if verbose:
sys.stdout.write(
'\n\nfix_orientation function:\n "finalorientation" command (xyz): {}\n Shape before(xyz): {}\n Shape after(xyz): {}\n\n'
.format(origax, shape_before[::-1], imstack.shape[::-1]))
sys.stdout.flush()
return imstack
def terastitcher_from_params(**params):
"""
"""
assert params["stitchingmethod"] in [
"terastitcher", "Terastitcher", "TeraStitcher"
]
kwargs = pth_update(load_kwargs(**params))
kwargs["cores"] = params["cores"] if "cores" in params else 12
terastitcher_wrapper(**kwargs)
return
def get_resampledvol_n_dimensions(dct):
""" works around param dict in case paths were missaved """
try:
kwargs = load_kwargs(dct)
vol = [xx for xx in kwargs["volumes"] if xx.ch_type =="cellch"][0]
resampled_vol = vol.resampled_for_elastix_vol
resampled_dims = tifffile.imread(resampled_vol).shape
except FileNotFoundError:
fls = listdirfull(os.path.dirname(dct), ".tif"); fls.sort()
resampled_vol = fls[-1] #will be the last one, bc of the 647 channel
resampled_dims = tifffile.imread(resampled_vol).shape
return resampled_dims, resampled_vol
def get_fullsizedimensions(dct):
""" works around param dict in case paths were missaved """
try:
kwargs = load_kwargs(dct)
vol = [xx for xx in kwargs["volumes"] if xx.ch_type =="cellch"][0]
zf = len(listdirfull(vol.full_sizedatafld_vol, ".tif"))
yf,xf = tifffile.imread(listdirfull(vol.full_sizedatafld_vol, "tif")[0]).shape
fullsizedimensions = tuple((zf, yf, xf))
except: #if param dict is messed up
fsz = os.path.join(os.path.dirname(dct), "full_sizedatafld")
vols = os.listdir(fsz); vols.sort()
src = os.path.join(fsz, vols[len(vols)-1]) #hack - try to load param_dict instead?
if not os.path.isdir(src): src = os.path.join(fsz, vols[len(vols)-2])
zf = len(listdirfull(src, ".tif"))
yf,xf = tifffile.imread(listdirfull(src, "tif")[0]).shape
fullsizedimensions = tuple((zf, yf, xf))
return fullsizedimensions
def correct_kwargs(src):
'''Temporary adjustment to correct kwargs after setting up folders in step 0 locally.
Input: source path of output directory
'''
#import kwargs
kwargs=load_kwargs(src)
#change packagedirectory (which defaults to my lightsheet_copy for some reason???)
kwargs['packagedirectory'] = os.path.join(src,'lightsheet')
kwargs['parameterfolder'] = os.path.join(src,'lightsheet/parameterfolder')
#save kwargs
save_kwargs(os.path.join(src, 'param_dict.p'), **kwargs)
#return kwargs to check
return kwargs
def generate_transformed_cellcount(dataframe, dst, transformfiles, lightsheet_parameter_dictionary, verbose=False):
"""Function to take a csv file and generate an input to transformix
Inputs
----------------
dataframe = preloaded pandas dataframe
dst = destination to save files
transformfiles = list of all elastix transform files used, and in order of the original transform****
lightsheet_parameter_file = .p file generated from lightsheet package
"""
#set up locations
transformed_dst = os.path.join(dst, "transformed_points"); makedir(transformed_dst)
#make zyx numpy arry
zyx = dataframe[["z","y","x"]].values
#adjust for reorientation THEN rescaling, remember full size data needs dimension change releative to resample
fullsizedimensions = get_fullsizedimensions(lightsheet_parameter_dictionary)
kwargs = load_kwargs(lightsheet_parameter_dictionary)
zyx = fix_contour_orientation(zyx, verbose=verbose, **kwargs) #now in orientation of resample
resampled_dims, resampled_vol = get_resampledvol_n_dimensions(lightsheet_parameter_dictionary)
zyx = points_resample(zyx, original_dims = fix_dimension_orientation(fullsizedimensions,
**kwargs), resample_dims = resampled_dims, verbose = verbose)[:, :3]
#make into transformix-friendly text file
pretransform_text_file = create_text_file_for_elastix(zyx, transformed_dst)
#copy over elastix files
transformfiles = modify_transform_files(transformfiles, transformed_dst)
change_transform_parameter_initial_transform(transformfiles[0], "NoInitialTransform")
#run transformix on points
points_file = point_transformix(pretransform_text_file, transformfiles[-1], transformed_dst)
#convert registered points into structure counts
converted_points = unpack_pnts(points_file, transformed_dst)
return converted_points
def transformed_pnts_to_allen(points_file,
ann,
ch_type="injch",
point_or_index=None,
allen_id_table_pth=False,
**kwargs):
"""function to take elastix point transform file and return anatomical locations of those points
point_or_index=None/point/index: determines which transformix output to use: point is more accurate, index is pixel value(?)
Elastix uses the xyz convention rather than the zyx numpy convention
NOTE: this modification does not output out a single excel file, but a data frame
Inputs
-----------
points_file =
ch_type = "injch" or "cellch"
allen_id_table_pth (optional) pth to allen_id_table
ann = annotation file
Returns
-----------
df = data frame containing voxel counts
"""
kwargs = load_kwargs(**kwargs)
#####inputs
assert type(points_file) == str
if point_or_index == None:
point_or_index = "OutputPoint"
elif point_or_index == "point":
point_or_index = "OutputPoint"
elif point_or_index == "index":
point_or_index = "OutputIndexFixed"
#
vols = kwargs["volumes"]
reg_vol = [xx for xx in vols if xx.ch_type == "regch"][0]
####load files
if not allen_id_table_pth:
allen_id_table = pd.read_excel(
os.path.join(reg_vol.packagedirectory,
"supp_files/allen_id_table.xlsx")
) ##use for determining neuroanatomical locations according to allen
else:
allen_id_table = pd.read_excel(allen_id_table_pth)
#####inputs
assert type(points_file) == str
point_or_index = 'OutputPoint'
#get points
with open(points_file, "r") as f:
lines = f.readlines()
f.close()
#####populate post-transformed array of contour centers
sys.stdout.write("\n\n{} points detected\n\n".format(len(lines)))
arr = np.empty((len(lines), 3))
for i in range(len(lines)):
arr[i,
...] = lines[i].split()[lines[i].split().index(point_or_index) +
3:lines[i].split().index(point_or_index) +
6] #x,y,z
pnts = transformed_pnts_to_allen_helper_func(arr, ann)
pnt_lst = [xx for xx in pnts if xx != 0]
#check to see if any points where found
if len(pnt_lst) == 0:
raise ValueError("pnt_lst is empty")
else:
sys.stdout.write("\nlen of pnt_lst({})\n\n".format(len(pnt_lst)))
#generate dataframe with column
df = count_structure_lister(allen_id_table, *pnt_lst)
return df
def pool_injections_inversetransform(**kwargs):
"""Function to pool several injection sites.
Assumes that the basic registration AND inverse transform using elastix has been run.
If not, runs inverse transform. Additions to analyze_injection.py and pool_injections_for_analysis().
Inputs
-----------
kwargs:
"inputlist": inputlist, #list of folders generated previously from software
"channel": "01",
"channel_type": "injch",
"filter_kernel": (5,5,5), #gaussian blur in pixels (if registered to ABA then 1px likely is 25um)
"threshold": 10 (int, value to use for thresholding, this value represents the number of stand devs above the mean of the gblurred image)
"num_sites_to_keep": #int, number of injection sites to keep, useful if multiple distinct sites
"injectionscale": 45000, #use to increase intensity of injection site visualizations generated - DOES NOT AFFECT DATA
"imagescale": 2, #use to increase intensity of background site visualizations generated - DOES NOT AFFECT DATA
"reorientation": ("2","0","1"), #use to change image orientation for visualization only
"crop": #use to crop volume, values below assume horizontal imaging and sagittal atlas
False
cerebellum: "[:,390:,:]"
caudal midbrain: "[:,300:415,:]"
midbrain: "[:,215:415,:]"
thalamus: "[:,215:345,:]"
anterior cortex: "[:,:250,:]"
"dst": "/home/wanglab/Downloads/test", #save location
"save_individual": True, #optional to save individual images, useful to inspect brains, which you can then remove bad brains from list and rerun function
"colormap": "plasma",
"atlas": "/jukebox/LightSheetTransfer/atlas/sagittal_atlas_20um_iso.tif", #whole brain atlas
Optional:
----------
"save_array": path to folder to save out numpy array per brain of binarized detected site
"save_tif": saves out tif volume per brain of binarized detected site
"dpi": dots per square inch to save at
Returns
----------------count_threshold
a pooled image consisting of max IP of reorientations provide in kwargs.
a list of structures (csv file) with pixel counts, pooling across brains.
if save individual will save individual images, useful for inspection and/or visualization
"""
inputlist = kwargs["inputlist"]
dst = kwargs["dst"]
makedir(dst)
injscale = kwargs["injectionscale"] if "injectionscale" in kwargs else 1
imagescale = kwargs["imagescale"] if "imagescale" in kwargs else 1
axes = kwargs["reorientation"] if "reorientation" in kwargs else ("0", "1",
"2")
cmap = kwargs["colormap"] if "colormap" in kwargs else "plasma"
save_array = kwargs["save_array"] if "save_array" in kwargs else False
save_tif = kwargs["save_tif"] if "save_tif" in kwargs else False
num_sites_to_keep = kwargs[
"num_sites_to_keep"] if "num_sites_to_keep" in kwargs else 1
ann = sitk.GetArrayFromImage(sitk.ReadImage(kwargs["annotation"]))
#if kwargs["crop"]: (from original analyze injection function, no functionality here if points file exist)
# ann = eval("ann{}".format(kwargs["crop"]))
nonzeros = []
#not needed as mapped points from point_transformix used
#id_table = kwargs["id_table"] if "id_table" in kwargs else "/jukebox/temp_wang/pisano/Python/lightsheet/supp_files/allen_id_table.xlsx"
#allen_id_table = pd.read_excel(id_table)
for i in range(len(inputlist)): #to iteratre through brains
pth = inputlist[i] #path of each processed brain
print(" loading:\n {}".format(pth))
dct = load_kwargs(pth) #load kwargs of brain as dct
try:
inj_vol = [xx for xx in dct["volumes"] if xx.ch_type == "injch"
][0] #set injection channel volume
im = tifffile.imread(inj_vol.resampled_for_elastix_vol
) #load inj_vol as numpy array
if kwargs["crop"]:
im = eval("im{}".format(kwargs["crop"])) #; print im.shape
#run find site function to segment inj site using non-registered resampled for elastix volume - pulled directly from tools.registration.register.py and tools.analysis.analyze_injection.py
array = find_site(im,
thresh=kwargs["threshold"],
filter_kernel=kwargs["filter_kernel"],
num_sites_to_keep=num_sites_to_keep) * injscale
if save_array:
np.save(
os.path.join(dst,
"{}".format(os.path.basename(pth)) + ".npy"),
array.astype("uint16"))
if save_tif:
tifffile.imsave(
os.path.join(dst,
"{}".format(os.path.basename(pth)) + ".tif"),
array.astype("uint16"))
#optional "save_individual"
if kwargs["save_individual"]:
im = im * imagescale
a = np.concatenate((np.max(
im, axis=0), np.max(array.astype("uint16"), axis=0)),
axis=1)
b = np.concatenate((np.fliplr(
np.rot90(np.max(fix_orientation(im, axes=axes), axis=0),
k=3)),
np.fliplr(
np.rot90(np.max(fix_orientation(
array.astype("uint16"), axes=axes),
axis=0),
k=3))),
axis=1)
plt.figure()
plt.imshow(np.concatenate((b, a), axis=0), cmap=cmap, alpha=1)
plt.axis("off")
plt.savefig(os.path.join(
dst, "{}".format(os.path.basename(pth)) + ".pdf"),
dpi=300,
transparent=True)
plt.close()
#find all nonzero pixels in resampled for elastix volume
print(" finding nonzero pixels for voxel counts...\n")
nz = np.nonzero(array)
nonzeros.append(zip(*nz)) #<-for pooled image
#find transform file
inverse_fld = inj_vol.inverse_elastixfld
inj_fld = listdirfull(inverse_fld, "inj")[0]
atlas2reg2sig_fld = listdirfull(inj_fld, "atlas2reg2sig")[0]
transformfile = os.path.join(atlas2reg2sig_fld,
"reg2sig_TransformParameters.1.txt")
if not os.path.exists(transformfile): #if transformed points exist
print(
"Transform file file not found. Running elastix inverse transform... \n"
)
transformfile = make_inverse_transform(
[xx for xx in dct["volumes"] if xx.ch_type == "injch"][0],
cores=6,
**dct)
else:
print("Inverse transform exists. \n")
#apply resizing point transform
txtflnm = point_transform_due_to_resizing(array,
chtype="injch",
**dct)
#run transformix on points
points_file = point_transformix(txtflnm, transformfile)
tdf = transformed_pnts_to_allen(points_file,
ann,
ch_type="injch",
point_or_index=None,
**dct) #map to allen atlas
if i == 0:
df = tdf.copy()
countcol = "count" if "count" in df.columns else "cell_count"
df.drop([countcol], axis=1, inplace=True)
df[os.path.basename(pth)] = tdf[countcol]
except:
print(
"could not recover injection site, inspect manually for parameter dictionary errors or missing inj channel \n\n"
)
#cell counts to csv
df.to_csv(os.path.join(dst, "voxel_counts.csv"))
print("\n\nCSV file of cell counts, saved as {}\n\n\n".format(
os.path.join(dst, "voxel_counts.csv")))
#condense nonzero pixels
nzs = [
str(x) for xx in nonzeros for x in xx
] #this list has duplicates if two brains had the same voxel w label
c = Counter(nzs)
arr = np.zeros(im.shape)
print("Collecting nonzero pixels for pooled image...")
tick = 0
#generating pooled array where voxel value = total number of brains with that voxel as positive
for k, v in c.items():
k = [int(xx) for xx in k.replace("(", "").replace(")", "").split(",")]
arr[k[0], k[1], k[2]] = int(v)
tick += 1
if tick % 50000 == 0: print(" {}".format(tick))
#load atlas and generate final figure
print("Generating final figure...")
atlas = tifffile.imread(kwargs["atlas"]) #reads atlas
print(
"Zooming in atlas..."
) #necessary to have a representative heat map as these segmentations are done from the resized volume, diff dimensions than atlas
zoomed_atlas = zoom(
atlas, 1.3) #zooms atlas; different than original analyze_injection.py
sites = fix_orientation(arr, axes=axes)
#cropping
if kwargs["crop"]:
zoomed_atlas = eval("zoomed_atlas{}".format(kwargs["crop"]))
zoomed_atlas = fix_orientation(zoomed_atlas, axes=axes)
my_cmap = eval("plt.cm.{}(np.arange(plt.cm.RdBu.N))".format(cmap))
my_cmap[:1, :4] = 0.0
my_cmap = mpl.colors.ListedColormap(my_cmap)
my_cmap.set_under("w")
plt.figure()
plt.imshow(np.max(zoomed_atlas, axis=0), cmap="gray")
plt.imshow(np.max(sites, axis=0), alpha=0.99, cmap=my_cmap)
plt.colorbar()
plt.axis("off")
dpi = int(kwargs["dpi"]) if "dpi" in kwargs else 300
plt.savefig(os.path.join(dst, "heatmap.pdf"), dpi=dpi, transparent=True)
plt.close()
print("Saved as {}".format(os.path.join(dst, "heatmap.pdf")))
return df
def overlay_qc(args):
#unpacking this way for multiprocessing
fld, folder_suffix, output_folder, verbose, doubletransform, make_volumes = args
#init error file
error_file = os.path.join(os.path.join(output_folder, os.path.basename(fld)), "errors.txt")
try:
#get 3dunet cell dataframe csv file
input_csv = listdirfull(os.path.join(fld, folder_suffix), ".csv")
assert len(input_csv) == 1, "multiple csv files"
dataframe = pd.read_csv(input_csv[0])
#location to save out
dst = os.path.join(output_folder, os.path.basename(fld)); makedir(dst)
#EXAMPLE USING LIGHTSHEET - assumes marking centers in the "raw" full sized cell channel. This will transform those
#centers into "atlas" space (in this case the moving image)
#in this case the "inverse transform has the atlas as the moving image in the first step,
#and the autofluorescence channel as the moving image in the second step
#NOTE - it seems that the registration of cell to auto is failing on occasion....thus get new files...
################################
cell_inverse_folder = listdirfull(os.path.join(fld, "elastix_inverse_transform"), "cellch")[0]
a2r = listall(cell_inverse_folder, "atlas2reg_TransformParameters"); a2r.sort()
r2s = listall(cell_inverse_folder, "reg2sig_TransformParameters"); r2s.sort() #possibly remove
#IMPORTANT. the idea is to apply cfos->auto->atlas
transformfiles = r2s + a2r if doubletransform else a2r #might get rid of r2s
lightsheet_parameter_dictionary = os.path.join(fld, "param_dict.p")
converted_points = generate_transformed_cellcount(dataframe, dst, transformfiles,
lightsheet_parameter_dictionary, verbose=verbose)
#load and convert to single voxel loc
zyx = np.asarray([str((int(xx[0]), int(xx[1]), int(xx[2]))) for xx in np.nan_to_num(np.load(converted_points))])
from collections import Counter
zyx_cnt = Counter(zyx)
#check...
if make_volumes:
#manually call transformix
kwargs = load_kwargs(lightsheet_parameter_dictionary)
resampled_dims, resampled_vol = get_resampledvol_n_dimensions(lightsheet_parameter_dictionary)
transformed_vol = os.path.join(dst, "transformed_volume"); makedir(transformed_vol)
if not doubletransform:
transformfiles = [os.path.join(fld, "elastix/TransformParameters.0.txt"), os.path.join(fld,
"elastix/TransformParameters.1.txt")]
transformfiles = modify_transform_files(transformfiles, transformed_vol) #copy over elastix files
transformix_command_line_call(resampled_vol, transformed_vol, transformfiles[-1])
else:
v=[xx for xx in kwargs["volumes"] if xx.ch_type == "cellch"][0]
#sig to reg
tps = [listall(os.path.dirname(v.ch_to_reg_to_atlas), "/TransformParameters.0")[0],
listall(os.path.dirname(v.ch_to_reg_to_atlas), "/TransformParameters.1")[0]]
#reg to atlas
transformfiles = tps+[os.path.join(fld, "elastix/TransformParameters.0.txt"),
os.path.join(fld, "elastix/TransformParameters.1.txt")]
transformfiles = modify_transform_files(transformfiles, transformed_vol) #copy over elastix files
transformix_command_line_call(resampled_vol, transformed_vol, transformfiles[-1])
#cell_registered channel
cell_reg = tifffile.imread(os.path.join(transformed_vol, "result.tif"))
tifffile.imsave(os.path.join(transformed_vol, "result.tif"), cell_reg, compress=1)
cell_cnn = np.zeros_like(cell_reg)
tarr = []; badlist=[]
for zyx,v in zyx_cnt.items():
z,y,x = [int(xx) for xx in zyx.replace("(","",).replace(")","").split(",")]
tarr.append([z,y,x])
try:
cell_cnn[z,y,x] = v*100
except:
badlist.append([z,y,x])
#apply x y dilation
r = 2
selem = ball(r)[int(r/2)]
cell_cnn = cell_cnn.astype("uint8")
cell_cnn = np.asarray([cv2.dilate(cell_cnn[i], selem, iterations = 1) for i in range(cell_cnn.shape[0])])
tarr=np.asarray(tarr)
if len(badlist)>0:
print("{} errors in mapping with cell_cnn shape {}, each max dim {}, \npossibly due to a registration overshoot \
or not using double transform\n\n{}".format(len(badlist), cell_cnn.shape, np.max(tarr,0), badlist))
merged = np.stack([cell_reg, cell_cnn, np.zeros_like(cell_reg)], -1)
tifffile.imsave(os.path.join(transformed_vol, "merged.tif"), merged)#, compress=1)
#out = np.concatenate([cell_cnn, cell_reg, ], 0)
#####check at the resampled for elastix phase before transform
#make zyx numpy arry
zyx = dataframe[["z","y","x"]].values
fullsizedimensions = get_fullsizedimensions(lightsheet_parameter_dictionary)
zyx = fix_contour_orientation(zyx, verbose=verbose, **kwargs) #now in orientation of resample
zyx = points_resample(zyx, original_dims = fix_dimension_orientation(fullsizedimensions, **kwargs),
resample_dims = resampled_dims, verbose = verbose)[:, :3]
#cell channel
cell_ch = tifffile.imread(resampled_vol)
cell_cnn = np.zeros_like(cell_ch)
tarr = []; badlist=[]
for _zyx in zyx:
z,y,x = [int(xx) for xx in _zyx]
tarr.append([z,y,x])
try:
cell_cnn[z,y,x] = 100
except:
badlist.append([z,y,x])
tarr = np.asarray(tarr)
merged = np.stack([cell_ch, cell_cnn, np.zeros_like(cell_ch)], -1)
tifffile.imsave(os.path.join(transformed_vol, "resampled_merged.tif"), merged)#, compress=1)
except Exception as e:
print(e)
with open(error_file, "a") as err_fl:
err_fl.write("\n\n{} {}\n\n".format(fld, e))
return outpth
if __name__ == "__main__":
print(sys.argv)
jobid = int(os.environ["SLURM_ARRAY_TASK_ID"])
src = "/jukebox/wang/pisano/tracing_output/eaat4"
dst = "/jukebox/wang/zahra/eaat4_screening/201910_analysis"
brains = listdirfull(src)
brain = brains[jobid]
kwargs = load_kwargs(brain)
cellvol = [
vol for vol in kwargs["volumes"]
if vol.ch_type == "cellch" or vol.ch_type == "injch"
][0]
fullszfld = cellvol.full_sizedatafld_vol
imgs = [os.path.join(fullszfld, xx) for xx in os.listdir(fullszfld)]
imgs.sort()
stk = np.array([tif.imread(img) for img in imgs])[:, 1700:, :]
#stk = tif.imread(src).astype("uint16")
clh = np.array([
equalize_adapthist(img,
clip_limit=0.05,
kernel_size=(50, 100),
def make_inverse_transform(vol_to_process, cores=5, **kwargs):
'''Script to perform inverse transform and return path to elastix inverse parameter file
Returns:
---------------
transformfile
'''
sys.stdout.write('starting make_inverse_transform, this will take time...')
############inputs
kwargs = load_kwargs(kwargs['outputdirectory'])
outdr = kwargs['outputdirectory']
vols = kwargs['volumes']
reg_vol = [xx for xx in vols if xx.ch_type == 'regch'][0]
AtlasFile = reg_vol.atlasfile
parameterfolder = reg_vol.parameterfolder
###############
###images need to have been stitched, resized, and saved into single tiff stack ###
###resize to ~220% total size of atlas (1.3x/dim) ###
reg_vol.add_resampled_for_elastix_vol(reg_vol.downsized_vol +
'_resampledforelastix.tif')
#resample_par(cores, reg_vol, AtlasFile, svlocname=reg_vol_resampled, singletifffile=True, resamplefactor=1.2)
if not os.path.exists(reg_vol.resampled_for_elastix_vol):
print('Resizing')
#resample(reg_vol, AtlasFile, svlocname=reg_vol_resampled, singletifffile=True, resamplefactor=1.3)
resample_par(cores,
reg_vol.downsized_vol + '.tif',
AtlasFile,
svlocname=reg_vol.resampled_for_elastix_vol,
singletifffile=True,
resamplefactor=1.3)
print('Past Resizing')
vol_to_process.add_resampled_for_elastix_vol(vol_to_process.downsized_vol +
'_resampledforelastix.tif')
if not os.path.exists(vol_to_process.resampled_for_elastix_vol):
print('Resizing')
resample_par(cores,
vol_to_process.downsized_vol + '.tif',
AtlasFile,
svlocname=vol_to_process.resampled_for_elastix_vol,
singletifffile=True,
resamplefactor=1.3)
print('Past Resizing')
####setup
parameters = []
[
parameters.append(os.path.join(parameterfolder, files))
for files in os.listdir(parameterfolder)
if files[0] != '.' and files[-1] != '~'
]
parameters.sort()
###set up save locations
svlc = os.path.join(outdr, 'elastix_inverse_transform')
makedir(svlc)
svlc = os.path.join(
svlc, '{}_{}'.format(vol_to_process.ch_type, vol_to_process.brainname))
makedir(svlc)
###Creating LogFile
#writer(svlc, 'Starting elastix...AtlasFile: {}\n parameterfolder: {}\n svlc: {}\n'.format(AtlasFile, parameterfolder, svlc))
writer(
svlc,
'Order of parameters used in Elastix:{}\n...\n\n'.format(parameters))
##register: 1) atlas->reg 2) reg->sig NOTE these are intentionally backwards so applying point transform can be accomplished
#atlas(mv)->reg (fx)
atlas2reg = os.path.join(
svlc,
reg_vol.resampled_for_elastix_vol[reg_vol.resampled_for_elastix_vol.
rfind('/') + 1:-4] + '_atlas2reg')
makedir(atlas2reg)
e_out_file, e_transform_file = elastix_command_line_call(
fx=reg_vol.resampled_for_elastix_vol,
mv=AtlasFile,
out=atlas2reg,
parameters=parameters)
#reg(mv)->sig(fx)
reg2sig = os.path.join(
svlc, vol_to_process.resampled_for_elastix_vol[
vol_to_process.resampled_for_elastix_vol.rfind('/') + 1:-4] +
'_reg2sig')
makedir(reg2sig)
e_out_file, e_transform_file = elastix_command_line_call(
fx=vol_to_process.resampled_for_elastix_vol,
mv=reg_vol.resampled_for_elastix_vol,
out=reg2sig,
parameters=parameters)
##set up transform series:
atlas2reg2sig = os.path.join(
svlc, vol_to_process.resampled_for_elastix_vol[
vol_to_process.resampled_for_elastix_vol.rfind('/') + 1:-4] +
'_atlas2reg2sig')
makedir(atlas2reg2sig)
#copy transform paramters
[
shutil.copy(os.path.join(reg2sig, xx),
os.path.join(atlas2reg2sig, 'reg2sig_' + xx))
for xx in os.listdir(reg2sig) if 'TransformParameters' in xx
]
[
shutil.copy(os.path.join(atlas2reg, xx),
os.path.join(atlas2reg2sig, 'atlas2reg_' + xx))
for xx in os.listdir(atlas2reg) if 'TransformParameters' in xx
]
#connect transforms by setting regtoatlas TP0's initial transform to sig->reg transform
tps = [
os.path.join(atlas2reg2sig, xx) for xx in os.listdir(atlas2reg2sig)
if 'TransformParameters' in xx
]
tps.sort(
reverse=True
) #they are now in order recent to first, thus first is regtoatlas_TransformParameters.1.txt
for x in range(len(tps)):
if not x == len(tps) - 1:
change_transform_parameter_initial_transform(tps[x], tps[x + 1])
assert os.path.exists(tps[0])
writer(svlc, '***Elastix Registration Successfully Completed***\n')
writer(svlc, '\ne_transform_file is {}'.format(tps[0]))
####################
sys.stdout.write('complted make_inverse_transform')
return tps[0]
def identify_structures_w_contours(jobid,
cores=5,
make_color_images=False,
overlay_on_original_data=False,
consider_only_multipln_contours=False,
**kwargs):
'''function to take 3d detected contours and apply elastix transform
'''
#######################inputs and setup#################################################
###inputs
kwargs = load_kwargs(**kwargs)
outdr = kwargs['outputdirectory']
vols = kwargs['volumes']
reg_vol = [xx for xx in vols if xx.ch_type == 'regch'][0]
###get rid of extra jobs
if jobid >= len([xx for xx in vols if xx.ch_type != 'regch'
]): ###used to end jobs if too many are called
print('jobid({}) >= volumes {}'.format(
jobid, len([xx for xx in vols if xx.ch_type != 'regch'])))
return
###volumes to process: each job represents a different contour volume
vol_to_process = [xx for xx in vols if xx.ch_type != 'regch'][jobid]
ch = vol_to_process.channel
print(vol_to_process.ch_type)
#find appropriate folders for contours
if vol_to_process.ch_type == 'cellch':
detect3dfld = reg_vol.celldetect3dfld
coordinatesfld = reg_vol.cellcoordinatesfld
elif vol_to_process.ch_type == 'injch':
detect3dfld = reg_vol.injdetect3dfld
coordinatesfld = reg_vol.injcoordinatesfld
#set scale and atlas
xscl, yscl, zscl = reg_vol.xyz_scale ###micron/pixel
zmx, ymx, xmx = reg_vol.fullsizedimensions
AtlasFile = reg_vol.atlasfile
print('Using {} CORES'.format(cores))
try:
p
except NameError:
p = mp.Pool(cores)
resizefactor = kwargs['resizefactor']
brainname = reg_vol.brainname
############################################################################################################
#######################use regex to sort np files by ch and then by zpln####################################
############################################################################################################
fl = [f for f in os.listdir(detect3dfld)
if '.p' in f and 'ch' in f] #sorted for raw files
reg = re.compile(r'(.*h+)(?P<ch>\d{2})(.*)(.p)')
matches = map(reg.match, fl)
##load .np files
sys.stdout.write(
'\njobid({}), loading ch{} .p files to extract contour_class objects....'
.format(jobid, ch))
contour_class_lst = []
for fl in [
os.path.join(detect3dfld, ''.join(xx.groups())) for xx in matches
if xx.group('ch')[-2:] in ch
]:
tmpkwargs = {}
pckl = open(fl, 'rb')
tmpkwargs.update(pickle.load(pckl))
pckl.close()
if consider_only_multipln_contours == False:
tmplst = tmpkwargs['single']
[tmplst.append(xx) for xx in tmpkwargs['multi']]
elif consider_only_multipln_contours == True:
tmplst = tmpkwargs['multi']
[contour_class_lst.append(xx) for xx in tmplst]
sys.stdout.write('\ndone loading contour_class objects.\n')
#check for successful loading
if len(contour_class_lst) == 0:
print('Length of contours in ch{} was {}, ending process...'.format(
jobid, len(contour_class_lst)))
try:
p.terminate()
except:
1
return
############################################################################################################
##############################make color files##############################################################
############################################################################################################
if make_color_images == True:
sys.stdout.write('\nmaking 3d planes...')
sys.stdout.flush()
valid_plns = range(0, zmx + 1)
svlc = os.path.join(outdr, 'ch{}_3dcontours'.format(ch))
removedir(svlc)
if overlay_on_original_data == False:
ovly = False
elif overlay_on_original_data == True:
ovly = True
iterlst = []
[
iterlst.append(
(ch, svlc, contour_class_lst, valid_plns, outdr,
vol_to_process, resizefactor, contour_class_lst,
consider_only_multipln_contours, ovly, core, cores))
for core in range(cores)
]
p.starmap(ovly_3d, iterlst)
lst = os.listdir(svlc)
lst1 = [os.path.join(svlc, xx) for xx in lst]
lst1.sort()
del lst
del iterlst
###load ims and return dct of keys=str(zpln), values=np.array
sys.stdout.write(
'\n3d planes made, saved in {},\nnow compressing into single tifffile'
.format(svlc))
imstack = tifffile.imread(lst1)
del lst1
if len(imstack.shape) > 3:
imstack = np.squeeze(imstack)
try: ###check for orientation differences, i.e. from horiztonal scan to sagittal for atlas registration
imstack = np.swapaxes(imstack, *kwargs['swapaxes'])
except:
pass
tiffstackpth = os.path.join(
outdr, '3D_contours_ch{}_{}'.format(ch, brainname))
tifffile.imsave(tiffstackpth, imstack.astype('uint16'))
del imstack
gc.collect()
shutil.rmtree(svlc)
sys.stdout.write('\ncolor image stack made for ch{}'.format(ch))
else:
sys.stdout.write('\nmake_color_images=False, not creating images')
############################################################################################################
######################apply point transform and make transformix input file#################################
############################################################################################################
###find centers and add 1's to make nx4 array for affine matrix multiplication to account for downsizing
###everything is in PIXELS
contourarr = np.empty((len(contour_class_lst), 3))
for i in range(len(contour_class_lst)):
contourarr[i, ...] = contour_class_lst[
i].center ###full sized dimensions: if 3x3 tiles z(~2000),y(7680),x(6480) before any rotation
try:
contourarr = swap_cols(
contourarr, *kwargs['swapaxes']
) ###change columns to account for orientation changes between brain and atlas: if horizontal to sagittal==>x,y,z relative to horizontal; zyx relative to sagittal
z, y, x = swap_cols(np.array([
vol_to_process.fullsizedimensions
]), *kwargs['swapaxes'])[
0] ##convert full size cooridnates into sagittal atlas coordinates
sys.stdout.write('\nSwapping Axes')
except: ###if no swapaxes then just take normal z,y,x dimensions in original scan orientation
z, y, x = vol_to_process.fullsizedimensions
sys.stdout.write('\nNo Swapping of Axes')
d1, d2 = contourarr.shape
nx4centers = np.ones((d1, d2 + 1))
nx4centers[:, :-1] = contourarr
###find resampled elastix file dim
print(os.listdir(outdr))
print([xx.channel for xx in vols if xx.ch_type == 'regch'])
with tifffile.TiffFile([
os.path.join(outdr, f) for f in os.listdir(outdr)
if 'resampledforelastix' in f and 'ch{}'.format(
[xx.channel for xx in vols if xx.ch_type == 'regch'][0]) in f
][0]) as tif:
zr = len(tif.pages)
yr, xr = tif.pages[0].shape
tif.close()
####create transformmatrix
trnsfrmmatrix = np.identity(4) * (
zr / z, yr / y, xr / x, 1) ###downscale to "resampledforelastix size"
sys.stdout.write('trnsfrmmatrix:\n{}\n'.format(trnsfrmmatrix))
#nx4 * 4x4 to give transform
trnsfmdpnts = nx4centers.dot(trnsfrmmatrix) ##z,y,x
sys.stdout.write('first three transformed pnts:\n{}\n'.format(
trnsfmdpnts[0:3]))
#create txt file, with elastix header, then populate points
txtflnm = '{}_zyx_transformedpnts_ch{}.txt'.format(brainname, ch)
pnts_fld = os.path.join(outdr, 'transformedpoints_pretransformix')
makedir(pnts_fld)
transforminput = os.path.join(pnts_fld, txtflnm)
removedir(transforminput) ###prevent adding to an already made file
writer(pnts_fld, 'index\n{}\n'.format(len(trnsfmdpnts)), flnm=txtflnm)
sys.stdout.write(
'\nwriting centers to transfomix input points text file: {}....'.
format(transforminput))
stringtowrite = '\n'.join([
'\n'.join(['{} {} {}'.format(i[2], i[1], i[0])]) for i in trnsfmdpnts
]) ####this step converts from zyx to xyz*****
writer(pnts_fld, stringtowrite, flnm=txtflnm)
#[writer(pnts_fld, '{} {} {}\n'.format(i[2],i[1],i[0]), flnm=txtflnm, verbose=False) for i in trnsfmdpnts] ####this step converts from zyx to xyz*****
sys.stdout.write('...done writing centers.')
sys.stdout.flush()
del trnsfmdpnts, trnsfrmmatrix, nx4centers, contourarr
gc.collect()
############################################################################################################
####################################elastix for inverse transform###########################################
############################################################################################################
transformfile = make_inverse_transform(vol_to_process, cores, **kwargs)
assert os.path.exists(transformfile)
sys.stdout.write(
'\n***Elastix Inverse Transform File: {}***'.format(transformfile))
############################################################################################################
####################################transformix#############################################################
############################################################################################################
if make_color_images != False:
#apply transform to 3d_tiffstack:
transformimageinput = tiffstackpth
elastixpth = os.path.join(outdr, 'elastix')
trnsfrm_outpath = os.path.join(
elastixpth, '3D_contours_ch{}_{}'.format(ch, brainname))
makedir(trnsfrm_outpath)
writer(trnsfrm_outpath, '\nProcessing ch{} 3D...'.format(ch))
#transformfiles=[os.path.join(elastixpth, xx) for xx in os.listdir(os.path.join(outdr, 'elastix')) if "TransformParameters" in xx]; mxx=max([xx[-5] for xx in transformfiles])
#transformfile=os.path.join(elastixpth, 'TransformParameters.{}.txt'.format(mxx))
trnsfrm_out_file = os.path.join(trnsfrm_outpath,
'result.tif') #output of transformix
transformimageinput_resized = transformimageinput[:-4] + '_resampledforelastix.tif'
print('Resizing {}'.format(transformimageinput_resized))
resample_par(cores,
transformimageinput,
AtlasFile,
svlocname=transformimageinput_resized,
singletifffile=True,
resamplefactor=1.7)
sp.call([
'transformix', '-in', transformimageinput_resized, '-out',
trnsfrm_outpath, '-tp', transformfile
])
writer(trnsfrm_outpath,
'\n Transformix File Generated: {}'.format(trnsfrm_out_file))
writer(
trnsfrm_outpath, '\n Passing colorcode: {} file as {}'.format(
ch, os.path.join(trnsfrm_outpath, 'depthcoded.png')))
###depth coded image of transformix result; not functional yet
#depth.colorcode(trnsfrm_out_file, trnsfrm_outpath)
#getvoxels(trnsfrm_out_file, os.path.join(trnsfrm_outpath, 'zyx_voxels_{}.npy'.format(ch)))
#allen_compare(AtlasFile, svlc, trnsfrm_outpath)
##if successful delete contour cooridnates and maybe contourdetect3d flds
############################################################
##############apply transform to points#####################
elastixpth = os.path.join(outdr, 'elastix_inverse_transform')
trnsfrm_outpath = os.path.join(elastixpth, 'ch{}_3dpoints'.format(ch))
makedir(trnsfrm_outpath)
writer(trnsfrm_outpath,
'\n***********Starting Transformix for: {}***********'.format(ch))
sys.stdout.flush()
#transformfiles=[os.path.join(elastixpth, xx) for xx in os.listdir(os.path.join(outdr, 'elastix')) if "TransformParameters" in xx]; mxx=max([xx[-5] for xx in transformfiles])
#transformfile=os.path.join(elastixpth, 'TransformParameters.{}.txt'.format(mxx))
trnsfrm_out_file = os.path.join(
trnsfrm_outpath, 'outputpoints.txt') #MIGHT NEED TO CHANGE THIS
sp.call([
'transformix', '-def', transforminput, '-out', trnsfrm_outpath, '-tp',
transformfile
])
#sp.call(['transformix', '-def', 'all', '-out', trnsfrm_outpath, '-tp', transformfile]) ##displacement field
writer(trnsfrm_outpath,
'\n Transformix File Generated: {}'.format(trnsfrm_out_file))
####################################################################################
##############generate list and image overlaid onto allen atlas#####################
####################################################################################
name = 'job{}_{}'.format(jobid, vol_to_process.ch_type)
transformed_pnts_to_allen(trnsfrm_out_file, ch, cores, name=name, **kwargs)
writer(outdr, '*************STEP 5*************\n Finished')
print('end of script')
try:
p.terminate()
except:
1
return
def check_registration_injection(pth, inputs, cerebellum_only = True, axis = 0):
'''Function to output registered brain images from processed directories.
Useful to determine 'bad' registration.
Inputs:
pth = pdf file path
inputs = Directories containing processed data
'''
pdf_pages = PdfPages(pth) #compiles into multiple pdfs
#iterate through inputs
for src in inputs:
#load kwargs
dct = load_kwargs(src)
#set output directory
outdr = dct['outputdirectory']
#set atlas file path
atl = tifffile.imread(dct['AtlasFile'])
#determine elastix output path
elastix_out = os.path.join(outdr, 'elastix')
#set registration channel file path
reg = tifffile.imread(os.path.join(elastix_out, 'result.1.tif'))
vol = [xx for xx in dct['volumes'] if xx.ch_type == 'injch'][0] #get injection volume
#read transformed injch image
print('Reading registered injection channel image\n {}'.format(outdr))
im = tifffile.imread(os.path.dirname(vol.ch_to_reg_to_atlas)+'/result.tif')#.astype('uint8')
print('\nPlotting images...\n')
figs = plt.figure(figsize=(8.27, 11.69))
#starting to plot figures
plt.subplot(131)
#plot the result and atlas next to each other
plt.imshow(atl[300], cmap = 'gray'); plt.axis('off'); plt.title('Atlas', fontsize = 10)
plt.subplot(132)
plt.imshow(reg[300], cmap = 'gray'); plt.axis('off'); plt.title('Registered brain', fontsize = 10)
#plot the max intensity zplane for the injection channel
plt.subplot(133)
a = np.max(im, axis = axis) # coronal view = 1; saggital view = 0
plt.imshow(a, cmap = 'plasma', alpha = 1); plt.axis('off'); plt.title('Injection site', fontsize = 10)
#fix title
# brainname = re.search('(?<=_)(\w+)(?=_1d3x)', vol.brainname)
if cerebellum_only:
#add title to page
plt.text(0.5,0.65, os.path.basename(src), transform = figs.transFigure, size = 16) #.group(0)
else:
#add title to page
plt.text(0.1,0.70, os.path.basename(src), transform = figs.transFigure, size = 16)
#done with the page
pdf_pages.savefig(dpi = 300, bbox_inches = 'tight')
#write PDF document contain composite of all brains
pdf_pages.close()
print('Saved as {}'.format(pth))
def pool_injections_for_analysis(**kwargs):
'''Function to pool several injection sites. Assumes that the basic registration using this software has been run.
Inputs
-----------
kwargs:
'inputlist': inputlist, #list of folders generated previously from software
'channel': '01',
'channel_type': 'injch',
'filter_kernel': (3,3,3), #gaussian blur in pixels (if registered to ABA then 1px likely is 25um)
'threshold': 3 (int, value to use for thresholding, this value represents the number of stand devs above the mean of the gblurred image)
'num_sites_to_keep': int, number of injection sites to keep, useful if multiple distinct sites
'injectionscale': 45000, #use to increase intensity of injection site visualizations generated - DOES NOT AFFECT DATA
'imagescale': 2, #use to increase intensity of background site visualizations generated - DOES NOT AFFECT DATA
'reorientation': ('2','0','1'), #use to change image orientation for visualization only
'crop': #use to crop volume, values below assume horizontal imaging and sagittal atlas
False
cerebellum: '[:,390:,:]'
caudal midbrain: '[:,300:415,:]'
midbrain: '[:,215:415,:]'
thalamus: '[:,215:345,:]'
anterior cortex: '[:,:250,:]'
'dst': '/home/wanglab/Downloads/test', #save location
'save_individual': True, #optional to save individual images, useful to inspect brains, which you can then remove bad brains from list and rerun function
'colormap': 'plasma',
'atlas': '/jukebox/wang/pisano/Python/allenatlas/average_template_25_sagittal_forDVscans.tif',
'annotation':'/jukebox/wang/pisano/Python/allenatlas/annotation_25_ccf2015_forDVscans.nrrd',
'id_table': '/jukebox/temp_wang/pisano/Python/lightsheet/supp_files/allen_id_table.xlsx',
Optional:
----------
'save_array': path to folder to save out numpy array per brain of binarized detected site
'save_tif': saves out tif volume per brain of binarized detected site
'dpi': dots per square inch to save at
'crop_atlas':(notfunctional) similiar to crop. Use when you would like to greatly restrain the cropping for injsite detection, but you want to display a larger area of overlay.
this will 0 pad the injection sites to accomodate the difference in size. Note this MUST be LARGER THAN crop.
Returns
----------------count_threshold
a pooled image consisting of max IP of reorientations provide in kwargs.
a list of structures (csv file) with pixel counts, pooling across brains.
if save individual will save individual images, useful for inspection and/or visualization
'''
inputlist = kwargs['inputlist']
dst = kwargs['dst']
makedir(dst)
injscale = kwargs['injectionscale'] if 'injectionscale' in kwargs else 1
imagescale = kwargs['imagescale'] if 'imagescale' in kwargs else 1
axes = kwargs['reorientation'] if 'reorientation' in kwargs else ('0', '1',
'2')
cmap = kwargs['colormap'] if 'colormap' in kwargs else 'plasma'
id_table = kwargs[
'id_table'] if 'id_table' in kwargs else '/jukebox/temp_wang/pisano/Python/lightsheet/supp_files/allen_id_table.xlsx'
count_threshold = kwargs[
'count_threshold'] if 'count_threshold' in kwargs else 10
save_array = kwargs['save_array'] if 'save_array' in kwargs else False
save_tif = kwargs['save_tif'] if 'save_tif' in kwargs else False
num_sites_to_keep = kwargs[
'num_sites_to_keep'] if 'num_sites_to_keep' in kwargs else 1
nonzeros = []
ann = sitk.GetArrayFromImage(sitk.ReadImage(kwargs['annotation']))
if kwargs['crop']: ann = eval('ann{}'.format(kwargs['crop']))
allen_id_table = pd.read_excel(id_table)
for i in range(len(inputlist)):
pth = inputlist[i]
print('\n\n_______\n{}'.format(os.path.basename(pth)))
dct = load_kwargs(pth)
#print dct['AtlasFile']
try:
vol = [
xx for xx in dct['volumes']
if xx.ch_type == kwargs['channel_type']
and xx.channel == kwargs['channel']
][0]
except:
vol = [
xx for xx in dct['volumes']
if xx.ch_type == "cellch" and xx.channel == kwargs['channel']
][0]
#done to account for different versions
if os.path.exists(vol.ch_to_reg_to_atlas + '/result.1.tif'):
impth = vol.ch_to_reg_to_atlas + '/result.1.tif'
elif os.path.exists(vol.ch_to_reg_to_atlas
) and vol.ch_to_reg_to_atlas[-4:] == '.tif':
impth = vol.ch_to_reg_to_atlas
elif os.path.exists(
os.path.dirname(vol.ch_to_reg_to_atlas) + '/result.1.tif'):
impth = os.path.dirname(vol.ch_to_reg_to_atlas) + '/result.1.tif'
elif os.path.exists(
os.path.dirname(vol.ch_to_reg_to_atlas) + '/result.tif'):
impth = os.path.dirname(vol.ch_to_reg_to_atlas) + '/result.tif'
print(' loading:\n {}'.format(pth))
im = tifffile.imread(impth)
if kwargs['crop']:
im = eval('im{}'.format(kwargs['crop'])) #; print im.shape
#segment
arr = find_site(im,
thresh=kwargs['threshold'],
filter_kernel=kwargs['filter_kernel'],
num_sites_to_keep=num_sites_to_keep) * injscale
if save_array:
np.save(
os.path.join(dst, '{}'.format(os.path.basename(pth)) + '.npy'),
arr.astype('float32'))
if save_tif:
tifffile.imsave(
os.path.join(dst, '{}'.format(os.path.basename(pth)) + '.tif'),
arr.astype('float32'))
#optional 'save_individual'
if kwargs['save_individual']:
im = im * imagescale
a = np.concatenate(
(np.max(im, axis=0), np.max(arr.astype('uint16'), axis=0)),
axis=1)
b = np.concatenate((np.fliplr(
np.rot90(np.max(fix_orientation(im, axes=axes), axis=0), k=3)),
np.fliplr(
np.rot90(np.max(fix_orientation(
arr.astype('uint16'), axes=axes),
axis=0),
k=3))),
axis=1)
plt.figure()
plt.imshow(np.concatenate((b, a), axis=0), cmap=cmap, alpha=1)
plt.axis('off')
plt.savefig(os.path.join(
dst, '{}'.format(os.path.basename(pth)) + '.pdf'),
dpi=300,
transparent=True)
plt.close()
#cell counts to csv
print(' finding nonzero pixels for voxel counts...')
nz = np.nonzero(arr)
nonzeros.append(zip(*nz)) #<-for pooled image
pos = transformed_pnts_to_allen_helper_func(
np.asarray(zip(*[nz[2], nz[1], nz[0]])), ann)
tdf = count_structure_lister(allen_id_table, *pos)
if i == 0:
df = tdf.copy()
countcol = 'count' if 'count' in df.columns else 'cell_count'
df.drop([countcol], axis=1, inplace=True)
df[os.path.basename(pth)] = tdf[countcol]
df.to_csv(os.path.join(dst, 'voxel_counts.csv'))
print('\n\nCSV file of cell counts, saved as {}\n\n\n'.format(
os.path.join(dst, 'voxel_counts.csv')))
#condense nonzero pixels
nzs = [
str(x) for xx in nonzeros for x in xx
] #this list has duplicates if two brains had the same voxel w label
c = Counter(nzs)
array = np.zeros(im.shape)
print('Collecting nonzero pixels for pooled image...')
tick = 0
#generating pooled array where voxel value = total number of brains with that voxel as positive
for k, v in c.iteritems():
k = [int(xx) for xx in k.replace('(', '').replace(')', '').split(',')]
array[k[0], k[1], k[2]] = int(v)
tick += 1
if tick % 50000 == 0: print(' {}'.format(tick))
#load atlas and generate final figure
print('Generating final figure...')
atlas = tifffile.imread(kwargs['atlas'])
arr = fix_orientation(array, axes=axes)
#cropping
#if 'crop_atlas' not in kwargs:
if kwargs['crop']: atlas = eval('atlas{}'.format(kwargs['crop']))
atlas = fix_orientation(atlas, axes=axes)
#elif 'crop_atlas' in kwargs:
#if kwargs['crop_atlas']: atlas = eval('atlas{}'.format(kwargs['crop_atlas']))
#atlas = fix_orientation(atlas, axes=axes)
#accomodate for size difference
#d0,d1,d2 = [(x-y)/2 for x,y in zip(atlas.shape, arr.shape)]
#arr = np.pad(arr,((d0,d0),(d1,d1),(d2,d2)), mode='constant')
##allows for a single pixel shift - if needed
#d0,d1,d2 = [(x-y) for x,y in zip(atlas.shape, arr.shape)]
#arr = np.pad(arr,((d0,0),(d1,0),(d2,0)), mode='constant')
my_cmap = eval('plt.cm.{}(np.arange(plt.cm.RdBu.N))'.format(cmap))
my_cmap[:1, :4] = 0.0
my_cmap = mpl.colors.ListedColormap(my_cmap)
my_cmap.set_under('w')
plt.figure()
plt.imshow(np.max(atlas, axis=0), cmap='gray')
plt.imshow(np.max(arr, axis=0), alpha=0.99, cmap=my_cmap)
plt.colorbar()
plt.axis('off')
dpi = int(kwargs['dpi']) if 'dpi' in kwargs else 300
plt.savefig(os.path.join(dst, 'heatmap.pdf'), dpi=dpi, transparent=True)
plt.close()
print('Saved as {}'.format(os.path.join(dst, 'heatmap.pdf')))
return df
def point_transform_due_to_resizing(array,
chtype='injch',
svlc=False,
**kwargs):
'''Function to take npy array, find nonzero pixels, apply point transform (due to resizing) and package them into a file for final elastix point transform
Inputs
-------------
array = np array, tif, or path to numpy array from tools.objectdetection.injdetect.inj_detect_using_labels ZYX****
chtype = 'injch' or 'cellch'
svlc =
False: savesfile into (outdr, 'transformedpoints_pretransformix'). Strongly recommended to use this as it will then work with the rest of package
True
Returns
---------------
txtflnm = pth to file containing transformed points BEFORE elastix transformation
NOTE THIS FUNCTION ASSUMES ARRAY AND ATLAS HAVE SAME Z,Y,X (DOES NOT TAKE INTO ACCOUNT SWAPPING OF AXES)
'''
if type(array) == str:
if array[-4:] == '.npy': array = np.load(array)
if array[-4:] == '.tif': array = tifffile.imread(array)
kwargs = load_kwargs(**kwargs)
outdr = kwargs['outputdirectory']
brainname = [xx for xx in kwargs['volumes']
if xx.ch_type == 'regch'][0].brainname
vol = [xx for xx in kwargs['volumes'] if xx.ch_type == chtype][0]
#array dimensions
z, y, x = array.shape
#compare size of array with 'resampledforelastixsize'
with tifffile.TiffFile([
os.path.join(outdr, f) for f in os.listdir(outdr)
if 'resampledforelastix.tif' in f and not '3D_contours' in f
and 'ch{}'.format([
xx.channel for xx in kwargs['volumes'] if xx.ch_type == 'regch'
][0]) in f
][0]) as tif:
zr = len(tif.pages)
yr, xr = tif.pages[0].shape
tif.close()
nonzeropixels = np.argwhere(array > 0)
nx4centers = np.ones(
(len(nonzeropixels), 4)) #FIXME: the logic needs to be checked
nx4centers[:, :-1] = nonzeropixels
####create transformmatrix
trnsfrmmatrix = np.identity(4) * (
zr / z, yr / y, xr / x, 1) ###downscale to "resampledforelastix size"
sys.stdout.write('\n\n Transfrom matrix:\n{}\n'.format(trnsfrmmatrix))
#nx4 * 4x4 to give transform
trnsfmdpnts = nx4centers.dot(trnsfrmmatrix) ##z,y,x
sys.stdout.write('\nfirst three transformed pnts:\n{}\n'.format(
trnsfmdpnts[0:3]))
#create txt file, with elastix header, then populate points
txtflnm = '{}_zyx_transformedpnts_{}.txt'.format(brainname, vol.ch_type)
#
if not svlc:
pnts_fld = os.path.join(outdr, 'transformedpoints_pretransformix')
makedir(pnts_fld)
if svlc:
pnts_fld = os.path.join(svlc)
makedir(pnts_fld)
transforminput = os.path.join(pnts_fld, txtflnm)
removedir(transforminput) ###prevent adding to an already made file
writer(pnts_fld, 'index\n{}\n'.format(len(trnsfmdpnts)), flnm=txtflnm)
sys.stdout.write(
'\nwriting centers to transfomix input points text file...')
stringtowrite = '\n'.join([
'\n'.join(['{} {} {}'.format(i[2], i[1], i[0])]) for i in trnsfmdpnts
]) ####this step converts from zyx to xyz*****
writer(pnts_fld, stringtowrite, flnm=txtflnm)
#[writer(pnts_fld, '{} {} {}\n'.format(i[2],i[1],i[0]), flnm=txtflnm, verbose=False) for i in trnsfmdpnts] ####this step converts from zyx to xyz*****
sys.stdout.write('...done writing centers.\nSaved in {}'.format(pnts_fld))
sys.stdout.flush()
del trnsfmdpnts, trnsfrmmatrix, nx4centers
return os.path.join(pnts_fld, txtflnm)
#print newim.shape
imstack = tifffile.imread(
'/home/wanglab/LightSheetData/marm_ghazanfar/post_processed/marm_01/smallctx/marm_01_smallctx_488_647_90msec_z3um_20hfds_01na_resized_ch00_resampledforelastix.tif'
)
kwargs = {}
#kwargs['finalorientation']= ('1', '-0','2')
kwargs['finalorientation'] = ('2', '0', '-1')
newim = fix_orientation(imstack, axes=None, **kwargs)
sitk.Show(sitk.GetImageFromArray(imstack))
sitk.Show(sitk.GetImageFromArray(newim))
from tools.utils.io import load_kwargs
pth = '/home/wanglab/wang/pisano/tracing_output/bl6_ts/20150804_tp_bl6_ts01'
kwargs = load_kwargs(pth)
axes = ('-2', '-0', '-1')
contour_array = np.zeros((2, 3))
contour_array[0] = np.asarray([3, 4, 5])
contour_array[1] = np.asarray([300, 400, 500])
contour_array = fix_contour_orientation(contour_array,
axes=axes,
verbose=True)
zyx = (500, 4444, 3456)
zyx = fix_dimension_orientation(zyx,
axes=[int(xx) for xx in axes],
verbose=True)
def elastix_registration(jobid, cores=5, **kwargs):
'''Function to take brainvolumes and register them to AtlasFiles using elastix parameters in parameterfolder.
Inputs
---------------
cores = for parallelization
optional kwargs:
secondary_registration (optional): False (default) - apply transform determined from regch->atlas to other channels
True - register other channel(s) to regch then apply transform determined from regch->atlas
(useful if imaging conditions were different between channel and regch, i.e. added horizontal foci, sheet na...etc)
kwargs overrides explicit 'secondary_registration' input to function
Required inputs via kwargs:
brainname='AnimalID'
brainpath= pathtofolder
AtlasFile=pathtoatlas ###atlas is a tif stack
parameterfolder ##contains parameter files: Order1_filename.txt, Order2_filename.txt,....
svlc=pathtosavedirectory
maskfile(optional)=creates a nested folder inside svlc and runs registration with mask
To run in parallel use: parallel_elastixlooper
'''
###inputs
outdr = kwargs['outputdirectory']
kwargs = load_kwargs(outdr)
#check to see if masking, cropping or normal atlas
if 'maskatlas' in kwargs:
AtlasFile = generate_masked_atlas(**kwargs)
elif 'cropatlas' in kwargs:
AtlasFile = generate_cropped_atlas(**kwargs)
else:
AtlasFile = kwargs['AtlasFile']
###make variables for volumes:
vols = kwargs['volumes']
reg_vol = [xx for xx in vols if xx.ch_type == 'regch'][0]
#images need to have been stitched, resized, and saved into single tiff stack
#resize to ~220% total size of atlas (1.3x/dim)
sys.stdout.write('Beginning registration on {}'.format(reg_vol.brainname))
sys.stdout.flush()
reg_vol.add_resampled_for_elastix_vol(reg_vol.downsized_vol +
'_resampledforelastix.tif')
if not os.path.exists(reg_vol.resampled_for_elastix_vol):
sys.stdout.write('\n Resizing {}'.format(reg_vol.downsized_vol))
sys.stdout.flush()
resample_par(cores,
reg_vol.downsized_vol + '.tif',
AtlasFile,
svlocname=reg_vol.resampled_for_elastix_vol,
singletifffile=True,
resamplefactor=1.3)
[
vol.add_registration_volume(reg_vol.resampled_for_elastix_vol)
for vol in vols
]
sys.stdout.write('...completed resizing\n')
sys.stdout.flush()
###find elastix parameters files and sort, set up parameters and logfiles
parameters = []
[
parameters.append(os.path.join(reg_vol.parameterfolder, files))
for files in os.listdir(reg_vol.parameterfolder)
if files[0] != '.' and files[-1] != '~'
]
parameters.sort()
svlc = os.path.join(outdr, 'elastix')
makedir(svlc)
writer(
svlc,
'Starting elastix...AtlasFile: {}\n parameterfolder: {}\n svlc: {}\n'
.format(AtlasFile, reg_vol.parameterfolder, svlc))
writer(
svlc,
'Order of parameters used in Elastix:{}\n...\n\n'.format(parameters))
#optionally generate MHD file for better scaling in elastix (make both mhds if present since one tiff and one mhd doesn't work well)
resampled_zyx_dims = False
if False and 'atlas_scale' in kwargs:
atlasfilecopy = AtlasFile
AtlasFile = convert_to_mhd(
AtlasFile,
dims=kwargs['atlas_scale'],
dst=os.path.join(
kwargs['outputdirectory'],
os.path.splitext(os.path.basename(kwargs['AtlasFile']))[0]) +
'.mhd')
#copy reg vol and calculate effective distance/pixel scale
reg_volcopy = reg_vol.resampled_for_elastix_vol
resampled_zyx_dims = [
cc * dd for cc, dd in zip(kwargs['xyz_scale'][::-1], [
float(bb) / float(aa) for aa, bb in zip(
tifffile.imread(reg_vol.resampled_for_elastix_vol).shape,
reg_vol.fullsizedimensions)
])
]
#note convert_to_mhd dims are in XYZ
reg_vol.add_resampled_for_elastix_vol(
convert_to_mhd(
reg_vol.resampled_for_elastix_vol,
dims=resampled_zyx_dims[::-1],
dst=os.path.join(
kwargs['outputdirectory'],
os.path.splitext(
os.path.basename(
reg_vol.resampled_for_elastix_vol))[0]) + '.mhd'))
#ELASTIX
e_out_file, transformfile = elastix_command_line_call(
AtlasFile, reg_vol.resampled_for_elastix_vol, svlc, parameters)
#optionally generate MHD file for better scaling in elastix
if False and 'atlas_scale' in kwargs:
removedir(AtlasFile)
removedir(AtlasFile[-3:] + '.raw')
AtlasFile = atlasfilecopy
removedir(reg_vol.resampled_for_elastix_vol)
removedir(reg_vol.resampled_for_elastix_vol + '.raw')
reg_vol.add_resampled_for_elastix_vol(reg_volcopy)
#RG movie for visual inspection of image registration
color_movie_merger(AtlasFile, e_out_file, svlc, reg_vol.brainname)
#RGB movie with blue channel=pre-registered stack
bluechannel = os.path.join(svlc, 'combinedmovies',
reg_vol.brainname + '_resized_bluechannel.tif')
resample(
reg_vol.downsized_vol + '.tif',
AtlasFile,
svlocname=bluechannel,
singletifffile=True
) ##needs to be resample(not resample_par) as you need EXTACT dimensions
color_movie_merger(AtlasFile,
e_out_file,
svlc,
reg_vol.brainname + '_bluechanneloriginal',
movie5=bluechannel)
#Make gridline transform file
gridfld, tmpgridline = gridcompare(svlc, reg_vol)
sp.call([
'transformix', '-in', tmpgridline, '-out', gridfld, '-tp',
transformfile
])
combine_images(str(reg_vol.resampled_for_elastix_vol), AtlasFile,
os.path.join(gridfld, 'result.tif'), e_out_file, svlc,
reg_vol.brainname)
shutil.rmtree(gridfld)
#Apply transforms to other channels
writer(
svlc,
'\n...\nStarting Transformix on channel files...\n\nChannels to process are {}\n*****\n\n'
.format([x.downsized_vol for x in vols]))
#type of transform and channels to apply transform to
secondary_registration = kwargs[
'secondary_registration'] if 'secondary_registration' in kwargs else True
transform_function = apply_transformix_and_register if secondary_registration else apply_transformix
vols_to_register = [xx for xx in vols if xx.ch_type != 'regch']
#appy transform
[
transform_function(vol, reg_vol, svlc, cores, AtlasFile, parameters,
transformfile, resampled_zyx_dims)
for vol in vols_to_register
]
writer(svlc, '\nPast transformix step')
###make final output image if a cellch and injch exist
if any([True
for vol in vols_to_register if vol.ch_type == 'cellch']) and any(
[True for vol in vols_to_register if vol.ch_type == 'injch']):
injch = [
vol.registration_volume for vol in vols_to_register
if vol.ch_type == 'injch'
][0]
cellch = [
vol.registration_volume for vol in vols_to_register
if vol.ch_type == 'cellch'
][0]
inj_and_cells(svlc, cellch, injch, AtlasFile)
####### check to see if script finished due to an error
if os.path.exists(e_out_file) == False:
writer(
svlc,
'****ERROR****GOTTEN TO END OF SCRIPT,\nTHIS ELASTIX OUTPUT FILE DOES NOT EXIST: {0} \n'
.format(e_out_file))
######write out logfile describing parameters input into function
writer(
svlc,
"\nelastixlooper has finished using:\nbrainname: {}\nAtlasFile: {}\nparameterfolder: {}\nparameter files {}\nsvlc: {}"
.format(reg_vol.brainname, AtlasFile, reg_vol.parameterfolder,
parameters, svlc))
###update volumes in kwargs and pickle
vols_to_register.append(reg_vol)
kwargs.update(dict([('volumes', vols_to_register)]))
pckloc = os.path.join(outdr, 'param_dict.p')
pckfl = open(pckloc, 'wb')
pickle.dump(kwargs, pckfl)
pckfl.close()
writer(
outdr,
"\n\n*************STEP 3************************\nelastix has completed using:\nbrainname: {}\nAtlasFile: {}\nparameterfolder: {}\nparameter files {}\nsvlc: {}\n****************\n"
.format(reg_vol.brainname, AtlasFile, reg_vol.parameterfolder,
parameters, svlc))
return
cb_roi[z, y, x] = 1 #make a mask of structure in annotation space
cb_roi = cb_roi.astype(bool) #add mask of structure to dictionary
#mask all the regions that are not cerebellum in the segmentation
for site in sites:
site[~cb_roi] = 0
#%%
#visualization
atl = fix_orientation(tifffile.imread(atl_pth)[:, 450:, :], ("2", "0", "1"))
#masked_sites = np.array([fix_orientation(site[:, 450:, :], ("2", "0", "1")) for site in sites])
for i, site in enumerate(sites):
masked_site = fix_orientation(site[:, 450:, :], ("2", "0", "1"))
kwargs = load_kwargs(os.path.join(data, os.path.basename(imgs[i])[:-15]))
cellvol = [
vol for vol in kwargs["volumes"]
if vol.ch_type == "cellch" or vol.ch_type == "injch"
][0]
regvol = fix_orientation(
tifffile.imread(cellvol.ch_to_reg_to_atlas)[:, 450:, :],
("2", "0", "1"))
tifffile.imsave(
os.path.join(dst,
os.path.basename(imgs[i])[:-15] + "maxproj_.tif"),
np.max(regvol, axis=0))
# merged = np.stack([regvol, masked_site, np.zeros_like(atl)], -1)
# tifffile.imsave(os.path.join(dst, os.path.basename(imgs[i])), merged.astype("uint16"))
#
#my_cmap = eval("plt.cm.{}(np.arange(plt.cm.RdBu.N))".format("viridis"))
def transformed_pnts_to_allen(trnsfrm_out_file,
ch,
cores,
point_or_index=None,
name=False,
**kwargs):
'''function to take elastix point transform file and return anatomical locations of those points
point_or_index=None/point/index: determines which transformix output to use: point is more accurate, index is pixel value(?)
Elastix uses the xyz convention rather than the zyx numpy convention
###ASSUMES INPUT OF XYZ
'''
#####inputs
assert type(trnsfrm_out_file) == str
if point_or_index == None:
point_or_index = 'OutputPoint'
elif point_or_index == 'point':
point_or_index = 'OutputPoint'
elif point_or_index == 'index':
point_or_index = 'OutputIndexFixed'
try: #check to see if pool processes have already been spawned
p
except NameError:
p = mp.Pool(cores)
kwargs = load_kwargs(**kwargs)
vols = kwargs['volumes']
reg_vol = [xx for xx in vols if xx.ch_type == 'regch'][0]
####load files
id_table = pd.read_excel(
os.path.join(kwargs['packagedirectory'], 'supp_files/id_table.xlsx')
) ##use for determining neuroanatomical locations according to allen
ann = sitk.GetArrayFromImage(sitk.ReadImage(
kwargs['annotationfile'])) ###zyx
with open(trnsfrm_out_file, "rb") as f:
lines = f.readlines()
f.close()
#####populate post-transformed array of contour centers
sys.stdout.write('\n{} points detected'.format(len(lines)))
arr = np.empty((len(lines), 3))
for i in range(len(lines)):
arr[i,
...] = lines[i].split()[lines[i].split().index(point_or_index) +
3:lines[i].split().index(point_or_index) +
6] #x,y,z
#optional save out of points
np.save(kwargs['outputdirectory'] + '/injection/zyx_voxels.npy',
np.asarray([(z, y, x) for x, y, z in arr]))
pnts = transformed_pnts_to_allen_helper_func(arr, ann)
pnt_lst = [xx for xx in pnts if xx != 0]
if len(pnt_lst) == 0:
raise ValueError('pnt_lst is empty')
else:
sys.stdout.write('\nlen of pnt_lst({})'.format(len(pnt_lst)))
imstack = brain_structure_keeper(
ann, True,
*pnt_lst) ###annotation file, true=to depict density, list of pnts
df = count_structure_lister(id_table, *pnt_lst)
#########save out imstack and df
nametosave = '{}{}_{}'.format(name, ch, reg_vol.brainname)
tifffile.imsave(
os.path.join(kwargs['outputdirectory'],
nametosave + '_structure_density_map.tif'), imstack)
excelfl = os.path.join(kwargs['outputdirectory'],
nametosave + '_stuctures_table.xlsx')
df.to_excel(excelfl)
print('file saved as: {}'.format(excelfl))
try:
p.terminate()
except:
1
return
def inj_detect_using_labels(threshold = .1, resampledforelastix = False, num_labels_to_keep=100, show = False, save = True, masking = False, **kwargs):
'''Loads, thresholds and then applied scipy.ndimage.labels to find connected structures.
Inputs
-------
threshold = % of maximum pixel intensity to make zero
resampledforelastix= False:use downsized image (still fairly large); True: use 'resampledforelastix' image (smaller and thus faster)
num_labels_to_keep= optional; take the top xx labels
show = optional, dispalys the threholded image and the image after scipy.ndimage.label has been applied
save = True: saves npy array; False = returns the numpy array
masking = (optional) if True: using BD's masking algorithm *BEFORE* of thresholding
Returns
--------
if save=False: returns thresholded numpy array
if save=True: returns path to saved npfl
'''
sys.stdout.write('\n\nStarting Injection site detection, you need ~3x Memory of size of volume to run\n\n'); sys.stdout.flush()
kwargs = load_kwargs(**kwargs)
#load inj vol
if not resampledforelastix: injvol = tifffile.imread([xx for xx in kwargs['volumes'] if xx.ch_type == 'injch'][0].downsized_vol+'.tif')
if resampledforelastix: injvol = tifffile.imread([xx for xx in kwargs['volumes'] if xx.ch_type == 'injch'][0].downsized_vol+'_resampledforelastix.tif')
injvolcp = injvol.copy()
#bd's masking using regression
if masking:
sys.stdout.write('\nStarting Masking Step...'); sys.stdout.flush()
#load reg vol
if not resampledforelastix: regvol = [xx for xx in kwargs['volumes'] if xx.ch_type == 'regch'][0].downsized_vol+'.tif'
if resampledforelastix: regvol = [xx for xx in kwargs['volumes'] if xx.ch_type == 'regch'][0].downsized_vol+'_resampledforelastix.tif'
#masking regression
maskfld = os.path.join(kwargs['outputdirectory'], 'injection', 'mask'); makedir(maskfld)
injvol = make_mask(injvol, regvol, step=0.05, slope_thresh=0.4, init_window=200, out=maskfld, despeckle_kernel=5, imsave_kwargs={'compress': 5}, save_plots=True, verbose=True, **kwargs)
sys.stdout.write('\n\nCompleted Masking Step'); sys.stdout.flush()
#threshold bottom xx% of pixels
injvol[injvol <= np.max(injvol)*threshold] = 0
#look for connected pixels
sys.stdout.write('\nLooking for connected pixels....'); sys.stdout.flush()
lbl, numfeat=label(injvol)
if show: sitk.Show(sitk.GetImageFromArray(injvol))
if show: sitk.Show(sitk.GetImageFromArray(lbl))
del injvol
sys.stdout.write('\n {} number of unique labels detected, if a large number, increase the threshold value'.format(numfeat)); sys.stdout.flush()
#get pixelid, pixelcounts
pxvl_num = Counter([lbl[tuple(xx)] for xx in np.argwhere(lbl>0)])
[sys.stdout.write('\n{} pixels at value {}'.format(num, pxvl)) for pxvl, num in pxvl_num.iteritems()]; sys.stdout.flush()
#format into list
num_pxvl = [[num, pxvl] for pxvl, num in pxvl_num.iteritems()]
sys.stdout.write('\nKeeping the {} largest label(s)'.format(num_labels_to_keep)); sys.stdout.flush()
#remove smaller labels
num_pxvl.sort(reverse=True)
rmlst=num_pxvl[num_labels_to_keep:]
for n, pxvl in rmlst:
lbl[lbl==pxvl] = 0
[sys.stdout.write('\n Kept {} of pixel id({})'.format(xx[0], xx[1])) for xx in num_pxvl[:num_labels_to_keep]]; sys.stdout.flush()
#remove nonzero pixels from original (preserving their original values)
injvol = injvolcp * lbl.astype('bool'); del injvolcp
#save out points:
if save:
if not resampledforelastix: svpth = os.path.join(kwargs['outputdirectory'], 'injection', '{}labelskept_{}threshold_injectionpixels_downsized.tif').format(num_labels_to_keep, threshold)
if resampledforelastix: svpth = os.path.join(kwargs['outputdirectory'], 'injection','{}labelskept_{}threshold_injectionpixels_resampledforelastix.tif').format(num_labels_to_keep, threshold)
tifffile.imsave(svpth,injvol)
return svpth
else:
return injvol
def find_location(src,
dst=False,
correspondence_type='post_elastix',
verbose=False):
'''
Function to transform an excel sheet (e.g.: lightsheet/supp_files/sample_coordinate_to_location.xlsx) and output transformed locations.
Suggestion is to use imagej to find XYZ coordinates to input into excel sheet.
Inputs
----------------
src = excelsheet
correspondence_type =
'post_elastix': your coordinates are the corresponding post-registered elastix file (outputfolder/elastix/..../result....tif)
'full_size_data': you coordinates are from the "full_sizedatafld" where:
Z== #### in 'file_name_Z####.tif'
X,Y are the pixel of that tif file
Returns
----------------
dst = (optional) output excelfile. Ensure path ends with '.xlsx'
'''
#from __future__ import division
#import shutil, os, tifffile, cv2, numpy as np, pandas as pd, sys, SimpleITK as sitk
#from tools.utils.io import listdirfull, load_kwargs, writer, makedir
#from tools.conv_net.read_roi import read_roi, read_roi_zip
from tools.registration.register import transformed_pnts_to_allen_helper_func
from tools.registration.transform import structure_lister
from tools.utils.io import load_kwargs, listdirfull, listall
import SimpleITK as sitk
import pandas as pd, numpy as np, os
from skimage.external import tifffile
if correspondence_type == 'post_elastix':
print(
'This function assumes coordinates are from the corresponding post-registered elastix file. \nMake sure the excel file has number,<space>number,<space>number and not number,number,number'
)
#inputs
df = pd.read_excel(src)
for brain in df.columns[1:]:
print(brain)
#load and find files
kwargs = load_kwargs(
df[brain][df['Inputs'] == 'Path to folder'][0])
ann = sitk.GetArrayFromImage(
sitk.ReadImage(kwargs['annotationfile']))
#Look up coordinates to pixel value
xyz_points = np.asarray([(int(xx.split(',')[0]),
int(xx.split(',')[1]),
int(xx.split(',')[2]))
for xx in df[brain][3:].tolist()])
xyz_points = transformed_pnts_to_allen_helper_func(xyz_points,
ann=ann,
order='XYZ')
#pixel id to transform
if 'allen_id_table' in kwargs:
structures = structure_lister(
pd.read_excel(kwargs['allen_id_table']), *xyz_points)
else:
structures = structure_lister(
pd.read_excel(kwargs['volumes'][0].allen_id_table),
*xyz_points)
#update dataframe
df[brain + ' point transform'] = df[brain][:3].tolist() + [
str(s.tolist()[0]) for s in structures
]
if not dst: dst = src[:-5] + '_output.xlsx'
df.to_excel(dst)
print('Saved as {}'.format(dst))
if correspondence_type == 'full_size_data':
from tools.imageprocessing.orientation import fix_dimension_orientation, fix_contour_orientation
from tools.utils.directorydeterminer import pth_update
from tools.registration.register import collect_points_post_transformix
from tools.registration.transform import points_resample, points_transform
print(
'This function assumes coordinates are from the corresponding "full_sizedatafld". \nMake sure the excel file has number,<space>number,<space>number and not number,number,number'
)
#inputs
df = pd.read_excel(src)
for brain in df.columns[1:]:
#load and find files
kwargs = load_kwargs(
df[brain][df['Inputs'] == 'Path to folder'][0])
ann = sitk.GetArrayFromImage(
sitk.ReadImage(kwargs['annotationfile']))
ch_type = str(
df[brain][df['Inputs'] == 'Channel Type'].tolist()[0])
vol = [xx for xx in kwargs['volumes'] if xx.ch_type == ch_type][0]
#Look up coordinates to pixel value
zyx_points = np.asarray([(int(xx.split(',')[2]),
int(xx.split(',')[1]),
int(xx.split(',')[0]))
for xx in df[brain][3:].tolist()])
#Fix orientation
zyx_points = fix_contour_orientation(np.asarray(zyx_points),
verbose=verbose,
**kwargs)
#Fix Scaling
trnsfmdpnts = points_resample(
zyx_points,
original_dims=fix_dimension_orientation(
vol.fullsizedimensions, **kwargs),
resample_dims=tifffile.imread(
pth_update(vol.resampled_for_elastix_vol)).shape,
verbose=verbose)
#write out points for transformix
transformfile = [
xx for xx in listall(os.path.join(vol.inverse_elastixfld))
if os.path.basename(vol.full_sizedatafld_vol)[:-5] in xx
and 'atlas2reg2sig' in xx
and 'reg2sig_TransformParameters.1.txt' in xx
][0]
tmpdst = os.path.join(os.path.dirname(src),
'coordinate_to_location_tmp')
output = points_transform(src=trnsfmdpnts[:, :3],
dst=tmpdst,
transformfile=transformfile,
verbose=True)
#collect from transformix
xyz_points = collect_points_post_transformix(output)
#now ID:
pix_ids = transformed_pnts_to_allen_helper_func(xyz_points,
ann=ann,
order='XYZ')
#pixel id to transform
aid = kwargs[
'allen_id_table'] if 'allen_id_table' in kwargs else kwargs[
'volumes'][0].allen_id_table
structures = structure_lister(pd.read_excel(aid), *pix_ids)
#update dataframe
df[brain + ' xyz points atlas space'] = df[brain][:3].tolist() + [
str(s.tolist()[0]) for zyx in xyz_points
]
df[brain + ' structures'] = df[brain][:3].tolist() + [
str(s.tolist()[0]) for s in structures
]
if not dst: dst = src[:-5] + '_output.xlsx'
df.to_excel(dst)
print('Saved as {}'.format(dst))
return
def pool_injections_for_analysis(**kwargs):
"""Function to pool several injection sites. Assumes that the basic registration using this software has been run.
Inputs
-----------
kwargs:
"inputlist": inputlist, #list of folders generated previously from software
"inputtype": "main_folder", "tiff" #specify the type of input. main_folder is the lightsheetpackage"s folder, tiff is the file
to use for injection site segmentation
"channel": "01",
"channel_type": "injch",
"filter_kernel": (3,3,3), #gaussian blur in pixels (if registered to ABA then 1px likely is 25um)
"threshold": 3 (int, value to use for thresholding, this value represents the number of stand devs above the mean of the gblurred
image)
"num_sites_to_keep": int, number of injection sites to keep, useful if multiple distinct sites
"injectionscale": 45000, #use to increase intensity of injection site visualizations generated - DOES NOT AFFECT DATA
"imagescale": 2, #use to increase intensity of background site visualizations generated - DOES NOT AFFECT DATA
"reorientation": ("2","0","1"), #use to change image orientation for visualization only
"crop": #use to crop volume, values below assume horizontal imaging and sagittal atlas
False
cerebellum: "[:,390:,:]"
caudal midbrain: "[:,300:415,:]"
midbrain: "[:,215:415,:]"
thalamus: "[:,215:345,:]"
anterior cortex: "[:,:250,:]"
"dst": "/home/wanglab/Downloads/test", #save location
"save_individual": True, #optional to save individual images, useful to inspect brains, which you can then remove bad brains from
list and rerun function
"colormap": "plasma",
"atlas": "/jukebox/wang/pisano/Python/allenatlas/average_template_25_sagittal_forDVscans.tif",
"annotation":"/jukebox/wang/pisano/Python/allenatlas/annotation_25_ccf2015_forDVscans.nrrd",
"id_table": "/jukebox/temp_wang/pisano/Python/lightsheet/supp_files/allen_id_table.xlsx",
Optional:
----------
"save_array": path to folder to save out numpy array per brain of binarized detected site
"save_tif": saves out tif volume per brain of binarized detected site
"dpi": dots per square inch to save at
"crop_atlas":(notfunctional) similiar to crop. Use when you would like to greatly restrain the cropping for injsite detection,
but you want to display a larger area of overlay.
this will 0 pad the injection sites to accomodate the difference in size. Note this MUST be LARGER THAN crop.
Returns
----------------count_threshold
a pooled image consisting of max IP of reorientations provide in kwargs.
a list of structures (csv file) with pixel counts, pooling across brains.
if save individual will save individual images, useful for inspection and/or visualization
"""
inputlist = kwargs["inputlist"]
inputtype = kwargs["inputtype"] if "inputtype" in kwargs else "main_folder"
dst = kwargs["dst"]
makedir(dst)
injscale = kwargs["injectionscale"] if "injectionscale" in kwargs else 1
imagescale = kwargs["imagescale"] if "imagescale" in kwargs else 1
axes = kwargs["reorientation"] if "reorientation" in kwargs else ("0", "1",
"2")
cmap = kwargs["colormap"] if "colormap" in kwargs else "plasma"
id_table = kwargs[
"id_table"] if "id_table" in kwargs else "/jukebox/LightSheetTransfer/atlas/ls_id_table_w_voxelcounts.xlsx"
save_array = kwargs["save_array"] if "save_array" in kwargs else False
save_tif = kwargs["save_tif"] if "save_tif" in kwargs else False
num_sites_to_keep = kwargs[
"num_sites_to_keep"] if "num_sites_to_keep" in kwargs else 1
nonzeros = []
ann = sitk.GetArrayFromImage(sitk.ReadImage(kwargs["annotation"]))
if kwargs["crop"]: ann = eval("ann{}".format(kwargs["crop"]))
allen_id_table = pd.read_excel(id_table)
for i in range(len(inputlist)):
pth = inputlist[i]
print("\n\n_______\n{}".format(os.path.basename(pth)))
#find the volume by loading the param dictionary generated using the light-sheet package
if inputtype == "main_folder":
dct = load_kwargs(pth)
#print dct["AtlasFile"]
try:
vol = [
xx for xx in dct["volumes"]
if xx.ch_type in kwargs["channel_type"]
and xx.channel == kwargs["channel"]
][0]
except:
vol = [xx for xx in dct["volumes"] if xx.ch_type != "regch"][0]
#risky for 3 channel images, but needed if param dict initially is mislabelled
#done to account for different versions
if os.path.exists(vol.ch_to_reg_to_atlas + "/result.1.tif"):
impth = vol.ch_to_reg_to_atlas + "/result.1.tif"
elif os.path.exists(vol.ch_to_reg_to_atlas
) and vol.ch_to_reg_to_atlas[-4:] == ".tif":
impth = vol.ch_to_reg_to_atlas
elif os.path.exists(
os.path.dirname(vol.ch_to_reg_to_atlas) + "/result.1.tif"):
impth = os.path.dirname(
vol.ch_to_reg_to_atlas) + "/result.1.tif"
elif os.path.exists(
os.path.dirname(vol.ch_to_reg_to_atlas) + "/result.tif"):
impth = os.path.dirname(vol.ch_to_reg_to_atlas) + "/result.tif"
else: #"tiff", use the file given
impth = pth
print(" loading:\n {}".format(pth))
im = tifffile.imread(impth)
if kwargs["crop"]:
im = eval("im{}".format(kwargs["crop"])) #; print im.shape
#segment
arr = find_site(im,
thresh=kwargs["threshold"],
filter_kernel=kwargs["filter_kernel"],
num_sites_to_keep=num_sites_to_keep) * injscale
if save_array:
np.save(
os.path.join(save_array,
"{}".format(os.path.basename(pth)) + ".npy"),
arr.astype("float32"))
if save_tif:
tifffile.imsave(
os.path.join(save_tif,
"{}".format(os.path.basename(pth)) + ".tif"),
arr.astype("float32"))
#optional "save_individual"
if kwargs["save_individual"]:
im = im * imagescale
a = np.concatenate(
(np.max(im, axis=0), np.max(arr.astype("uint16"), axis=0)),
axis=1)
b = np.concatenate((np.fliplr(
np.rot90(np.max(fix_orientation(im, axes=axes), axis=0), k=3)),
np.fliplr(
np.rot90(np.max(fix_orientation(
arr.astype("uint16"), axes=axes),
axis=0),
k=3))),
axis=1)
plt.figure()
plt.imshow(np.concatenate((b, a), axis=0), cmap=cmap, alpha=1)
plt.axis("off")
plt.savefig(os.path.join(
dst, "{}".format(os.path.basename(pth)) + ".pdf"),
dpi=300,
transparent=True)
plt.close()
#cell counts to csv
print(" finding nonzero pixels for voxel counts...")
nz = np.nonzero(arr)
nonzeros.append(list(zip(*nz))) #<-for pooled image
pos = transformed_pnts_to_allen_helper_func(
np.asarray(list(zip(*[nz[2], nz[1], nz[0]]))), ann)
tdf = count_structure_lister(allen_id_table, *pos)
if i == 0:
df = tdf.copy()
countcol = "count" if "count" in df.columns else "cell_count"
df.drop([countcol], axis=1, inplace=True)
df[os.path.basename(pth)] = tdf[countcol]
df.to_csv(os.path.join(dst, "voxel_counts.csv"), index=False)
print("\n\nCSV file of cell counts, saved as {}\n\n\n".format(
os.path.join(dst, "voxel_counts.csv")))
#condense nonzero pixels
nzs = [
str(x) for xx in nonzeros for x in xx
] #this list has duplicates if two brains had the same voxel w label
c = Counter(nzs)
array = np.zeros(im.shape)
print("Collecting nonzero pixels for pooled image...")
tick = 0
#generating pooled array where voxel value = total number of brains with that voxel as positive
for k, v in c.items():
k = [int(xx) for xx in k.replace("(", "").replace(")", "").split(",")]
array[k[0], k[1], k[2]] = int(v)
tick += 1
if tick % 50000 == 0: print(" {}".format(tick))
#load atlas and generate final figure
print("Generating final figure...")
atlas = tifffile.imread(kwargs["atlas"])
arr = fix_orientation(array, axes=axes)
#cropping
if kwargs["crop"]: atlas = eval("atlas{}".format(kwargs["crop"]))
atlas = fix_orientation(atlas, axes=axes)
my_cmap = eval("plt.cm.{}(np.arange(plt.cm.RdBu.N))".format(cmap))
my_cmap[:1, :4] = 0.0
my_cmap = mpl.colors.ListedColormap(my_cmap)
my_cmap.set_under("w")
plt.figure()
plt.imshow(np.max(atlas, axis=0), cmap="gray")
plt.imshow(np.max(arr, axis=0), alpha=0.99, cmap=my_cmap)
plt.colorbar()
plt.axis("off")
dpi = int(kwargs["dpi"]) if "dpi" in kwargs else 300
plt.savefig(os.path.join(dst, "heatmap.pdf"), dpi=dpi, transparent=True)
plt.close()
print("Saved as {}".format(os.path.join(dst, "heatmap.pdf")))
return df
flds = [
os.path.join(src, xx) for xx in os.listdir(src)
if os.path.isdir(os.path.join(src, xx))
]
for fld in flds:
start = time.time()
brain = os.path.basename(fld)
#load in ROIS - clicked in SAGITTAL volume
roi_pth = fld + "/{}_20190602_fiber_points_RoiSet.zip".format(brain)
#check if path exists obv
if os.path.exists(roi_pth):
#have to load kwargs...
kwargs = load_kwargs(fld)
#get rois
zyx_rois_sag = np.asarray(
[[int(yy) for yy in xx.replace(".roi", "").split("-")]
for xx in read_roi_zip(roi_pth, include_roi_name=True)])
#slice into diff orientations
zyx_rois_cor = np.asarray([[xx[1], xx[2], xx[0]]
for xx in zyx_rois_sag])
zyx_rois_hor = np.asarray([[xx[2], xx[1], xx[0]]
for xx in zyx_rois_sag])
#make destination path
dst = os.path.join(fld, "points_merged_to_atlas")
if not os.path.exists(dst): os.mkdir(dst)
def transformed_pnts_to_allen(points_file,
ch_type='injch',
point_or_index=None,
allen_id_table_pth=False,
**kwargs):
'''function to take elastix point transform file and return anatomical locations of those points
point_or_index=None/point/index: determines which transformix output to use: point is more accurate, index is pixel value(?)
Elastix uses the xyz convention rather than the zyx numpy convention
Inputs
-----------
points_file =
ch_type = 'injch' or 'cellch'
allen_id_table_pth (optional) pth to allen_id_table
Returns
-----------
excelfl = path to excel file
'''
kwargs = load_kwargs(**kwargs)
#####inputs
assert type(points_file) == str
if point_or_index == None:
point_or_index = 'OutputPoint'
elif point_or_index == 'point':
point_or_index = 'OutputPoint'
elif point_or_index == 'index':
point_or_index = 'OutputIndexFixed'
#
vols = kwargs['volumes']
reg_vol = [xx for xx in vols if xx.ch_type == 'regch'][0]
####load files
if not allen_id_table_pth:
allen_id_table = pd.read_excel(
os.path.join(reg_vol.packagedirectory,
'supp_files/allen_id_table.xlsx')
) ##use for determining neuroanatomical locations according to allen
else:
allen_id_table = pd.read_excel(allen_id_table_pth)
ann = sitk.GetArrayFromImage(sitk.ReadImage(
kwargs['annotationfile'])) ###zyx
with open(points_file, "rb") as f:
lines = f.readlines()
f.close()
#####populate post-transformed array of contour centers
sys.stdout.write('\n{} points detected\n\n'.format(len(lines)))
arr = np.empty((len(lines), 3))
for i in range(len(lines)):
arr[i,
...] = lines[i].split()[lines[i].split().index(point_or_index) +
3:lines[i].split().index(point_or_index) +
6] #x,y,z
#optional save out of points
np.save(kwargs['outputdirectory'] + '/injection/zyx_voxels.npy',
np.asarray([(z, y, x) for x, y, z in arr]))
pnts = transformed_pnts_to_allen_helper_func(arr, ann)
pnt_lst = [xx for xx in pnts if xx != 0]
#check to see if any points where found
if len(pnt_lst) == 0:
raise ValueError('pnt_lst is empty')
else:
sys.stdout.write('\nlen of pnt_lst({})\n\n'.format(len(pnt_lst)))
#generate dataframe with column
df = count_structure_lister(allen_id_table, *pnt_lst)
#save df
nametosave = '{}_{}'.format(reg_vol.brainname, ch_type)
excelfl = os.path.join(kwargs['outputdirectory'],
nametosave + '_stuctures_table.xlsx')
df.to_excel(excelfl)
print('\n\nfile saved as: {}'.format(excelfl))
return excelfl
