def point_transformix(txtflnm, transformfile, dst=False):
'''apply elastix transform to points
Inputs
-------------
txtflnm = list of points that have resizingtransform
transformfile = elastix transform file
dst = optional folder
Returns
---------------
trnsfrm_out_file = pth to file containing post transformix points
'''
sys.stdout.write('\n***********Starting Transformix***********')
from subprocess import check_output
#set paths
if not dst:
trnsfrm_outpath = os.path.join(os.path.dirname(transformfile),
'posttransformix')
makedir(trnsfrm_outpath)
if dst:
trnsfrm_outpath = os.path.join(dst, 'posttransformix')
makedir(trnsfrm_outpath)
trnsfrm_out_file = os.path.join(trnsfrm_outpath, 'outputpoints.txt')
#run transformix point transform
call = 'transformix -def {} -out {} -tp {}'.format(txtflnm,
trnsfrm_outpath,
transformfile)
print(check_output(call, shell=True))
writer(trnsfrm_outpath,
'\n Transformix File Generated: {}'.format(trnsfrm_out_file))
return trnsfrm_out_file
def generate_masked_atlas(binarize = False, **kwargs):
'''Wrapper to call mask_atlas and save in appropriate place
Inputs
-------------------
binarize: (optional) if true convert all nonzero pixels to 255
kwargs used:
ouputdirectory
AtlasFile
maskatlas
Returns:
-------------------
path to newly saved masked atlas
'''
#make folder to store masked atlas
fld = os.path.join(kwargs['outputdirectory'], 'maskedatlas')
makedir(fld)
#generate masked_atlas
masked_atlas = mask_atlas(**kwargs)
if binarize:
masked_atlas[masked_atlas>0]=255
masked_atlas = change_bitdepth(masked_atlas)
#save masked_atlas
masked_atlas_pth = os.path.join(fld, 'masked_atlas.tif')
tifffile.imsave(masked_atlas_pth, masked_atlas)
return masked_atlas_pth
def overlay(AtlasFile, svlc, nm_to_sv, arr, gaussianblur=False):
'''function to plot coordinates using depth color overlay.
Inputs:
pth=pth to allen annoation
AtlasFile=pth to atlas (grayscale)
svlc=pth to folder to save
nm_to_sv=text to identify this structure
arr= np array of [zyx] PIXEL COORDINATES to plot
gaussianblur (optional):False/True
'''
im = sitk.GetArrayFromImage(sitk.ReadImage(AtlasFile))
im = np.zeros(im.shape).astype('uint8')
for i in tuple(map(tuple, arr.astype('int'))):
im[i] = 255
#optional
if gaussianblur:
from scipy.ndimage.filters import gaussian_filter
im = gaussian_filter(im, (2,2,2)).astype('uint8')
im[im>0]=255
im = gaussian_filter(im, (2,2,2)).astype('uint8')
im[im>0]=255
makedir(svlc)
svlc2=os.path.join(svlc, nm_to_sv)
makedir(svlc2)
impth=os.path.join(svlc2, str(nm_to_sv)+'.tif')
tifffile.imsave(impth, im.astype('uint8'))
allen_compare(AtlasFile, svlc, impth)
return
def blend_lightsheets(flds, dst, cores, cleanup=False):
"""0=L, 1=R
"""
#make sure l and r are in appropriate positions
if np.any(["right_lightsheet" in xx for xx in flds]) and np.any(
["left_lightsheet" in xx for xx in flds]):
l = [xx for xx in flds if "left_lightsheet" in xx][0]
r = [xx for xx in flds if "right_lightsheet" in xx][0]
flds = [l, r]
#
st = time.time()
name = os.path.basename(dst)
ch = dst[-2:]
sys.stdout.write("\nStarting blending of {}...".format(dst))
sys.stdout.flush()
ydim0, xdim0 = sitk.GetArrayFromImage(
sitk.ReadImage(listall(flds[0], keyword=".tif")[0])).shape
ydim1, xdim1 = sitk.GetArrayFromImage(
sitk.ReadImage(listall(flds[1], keyword=".tif")[0])).shape
ydim = ydim0 if ydim0 < ydim1 else ydim1
xdim = xdim0 if xdim0 < xdim1 else xdim1
#alpha=np.tile(scipy.stats.logistic.cdf(np.linspace(-250, 250, num=xdim)), (ydim, 1))
fls0 = listall(flds[0], keyword=".tif")
fls0.sort()
fls1 = listall(flds[1], keyword=".tif")
fls1.sort()
fls = fls0 if len(fls0) < len(fls1) else fls1
makedir(dst)
#makedir(os.path.join(dst, name))
iterlst = [{
"xdim": xdim,
"ydim": ydim,
"fl0": fls0[i],
"channel": ch,
"fl1": fls1[i],
"dst": dst,
"name": name,
"zplane": i
} for i, fl0 in enumerate(fls)]
if cores >= 2:
p = mp.Pool(cores)
p.map(blend, iterlst)
p.terminate()
else:
[blend(dct) for dct in iterlst]
#if cleanup: [shutil.rmtree(xx) for xx in flds]
sys.stdout.write("\n...finished in {} minutes.\n".format(np.round(
(time.time() - st) / 60),
decimals=2))
sys.stdout.flush()
return
def apply_classifier(classifier, raw_src, cnn_src, collect_cnn = False, size = (3,12,12), pad=False, cores=10, numZSlicesPerSplit=50, overlapping_planes = 15, verbose=True, save=True, maxip=0):
'''
classifier = pretrained random forest or path to pretrained random forest
raw_src = folder of tiffs of raw input data
cnn_src = folder of tiffs from cnn output
size
pad = if True, pad the edges of objects determined. False: remove edge cases, usually better since they will be few in number
cores = number of cores for parallelization, larger the number the less memory efficient
numZSlicesPerSplit: chunk of zplanes to process at once. Adjust this and cores based on memory constraints.
overlapping_planes: number of planes on each side to overlap by, this should be a comfortable amount larger than the maximum z distances of a single object
save (optional): #optional save to prevent rerun of jobs
collect_cnn = optional to include cnn data for random forest input
Returns
----------------
a dictionary consisting of k=centers, v=[corresponding pixel indices determine by CNN, maximum intensity, list of maximum radius/plane]
'''
#handle inputs
threshold = 1
zyx_search_range = (2,10,10)
zdim = len(listdirfull(cnn_src, keyword='.tif'))
#optional save to prevent rerun of jobs
if save:
save = cnn_src+'_apply_classifier_tmp'
makedir(save)
#run
if verbose: sys.stdout.write('\n Thesholding, determining connected pixels, identifying center of masses, applying classifier\n\n'); sys.stdout.flush(); st = time.time()
rng = range(0, zdim, numZSlicesPerSplit); jobs = len(rng);
iterlst=[(cnn_src, raw_src, collect_cnn, z, zdim, numZSlicesPerSplit, overlapping_planes, threshold, classifier, size, zyx_search_range, pad, job, jobs, verbose, save, maxip) for job, z in enumerate(rng)]
#par vs not par
if cores > 1:
p = mp.Pool(cores)
center_pixels_intensity_radius_lst = p.starmap(apply_classifier_helper, iterlst)
p.terminate()
else:
center_pixels_intensity_radius_lst = []
for i in iterlst:
center_pixels_intensity_radius_lst.append(apply_classifier_helper(i))
#optional reload:
if save:
center_pixels_intensity_radius_lst = [load_dictionary(xx) for xx in listdirfull(save)]
shutil.rmtree(save)
#unpack
if verbose: sys.stdout.write('\n...finished, formatting dictionary...'); sys.stdout.flush()
center_pixels_intensity_radius_dct = {}; [center_pixels_intensity_radius_dct.update(xx) for xx in center_pixels_intensity_radius_lst]
if 'None' in center_pixels_intensity_radius_dct: del center_pixels_intensity_radius_dct['None']
if verbose: print ('Total time {} minutes'.format(round((time.time() - st) / 60)))
if verbose: print('{} centers found.'.format(len(center_pixels_intensity_radius_dct)))
return center_pixels_intensity_radius_dct
def move_images_after_stitching(ts_out, dst, channel):
#make sure destination exists
makedir(dst)
imgs = listall(ts_out, '.tif')
imgs.sort()
#parellelize
iterlst = [(i, img, dst, channel) for i, img in enumerate(imgs)]
p = mp.Pool(6)
p.starmap(move_images_after_stitching_par, iterlst)
return dst
def generate_transformed_cellcount(dataframe,
dst,
transformfiles,
lightsheet_parameter_dictionary,
verbose=False):
'''Function to take a csv file and generate an input to transformix
Inputs
----------------
dataframe = preloaded pandas dataframe
dst = destination to save files
transformfiles = list of all elastix transform files used, and in order of the original transform****
lightsheet_parameter_file = .p file generated from lightsheet package
'''
#set up locations
transformed_dst = os.path.join(dst, 'transformed_points')
makedir(transformed_dst)
#make zyx numpy arry
zyx = dataframe[['z', 'y', 'x']].values
#adjust for reorientation THEN rescaling, remember full size data needs dimension change releative to resample
kwargs = load_dictionary(lightsheet_parameter_dictionary)
vol = [xx for xx in kwargs['volumes'] if xx.ch_type == 'cellch'][0]
fullsizedimensions = get_fullsizedims_from_kwargs(
kwargs
) #don't get from kwargs['volumes'][0].fullsizedimensions it's bad! use this instead
zyx = fix_contour_orientation(zyx, verbose=verbose,
**kwargs) #now in orientation of resample
zyx = points_resample(
zyx,
original_dims=fix_dimension_orientation(fullsizedimensions, **kwargs),
resample_dims=tifffile.imread(vol.resampled_for_elastix_vol).shape,
verbose=verbose)[:, :3]
#make into transformix-friendly text file
pretransform_text_file = create_text_file_for_elastix(zyx, transformed_dst)
#copy over elastix files
transformfiles = modify_transform_files(transformfiles, transformed_dst)
change_transform_parameter_initial_transform(transformfiles[0],
'NoInitialTransform')
#run transformix on points
points_file = point_transformix(pretransform_text_file, transformfiles[-1],
transformed_dst)
#convert registered points into structure counts
converted_points = unpack_pnts(points_file, transformed_dst)
return converted_points
def make_jobs(image_dictionary):
'''Simple function to create job dct for parallelization
'''
jobdct={}
lslst = ['left_lightsheet', 'right_lightsheet']
for channel in image_dictionary['channels']:
for lightsheet in range(image_dictionary['lightsheets']):
if image_dictionary['lightsheets'] == 2:
side=['_C00_', '_C01_'][lightsheet]
dct = copy.deepcopy(image_dictionary)
dct['zchanneldct']={k:{kk:[xx for xx in vv if side in xx]} for k,v in dct['zchanneldct'].iteritems() for kk,vv in v.iteritems()}
name = os.path.basename(os.path.dirname((dct['zchanneldct'].values()[0].values()[0][0])))
out = os.path.join(dst, '{}_{}'.format(name, lslst[lightsheet])); makedir(out)
jobdct['channel{}_lightsheet{}'.format(channel, lightsheet)] = copy.deepcopy({'job': 'channel{}_lightsheet{}'.format(channel, lightsheet), 'name': name, 'lightsheet': lslst[lightsheet], 'channel': channel, 'dst':dct['dst'], 'dct': copy.deepcopy(dct), 'out':out, 'cores':1})
return jobdct
def inj_and_cells(elastixfld, cellch, injch, AtlasFile, threshold=0.075):
###assumes sagittal files
###set paths for
###generate Allen Atlas background
allen = os.path.join(elastixfld, 'allenatlas_inj')
makedir(allen)
allentiff = tifffile.imread(AtlasFile[:-4] + '.tif')[:, 390:, :]
allentiffloc = os.path.join(allen, 'allen_for_inj.tif')
tifffile.imsave(allentiffloc, allentiff)
depth.colorcode(allentiffloc, allen)
grayscale = [
os.path.join(allen, 'proj0.png'),
os.path.join(allen, 'proj1.png'),
os.path.join(allen, 'proj2.png')
]
####parsing out injection site
ninjfld = os.path.join(elastixfld, 'injection_and_cells')
makedir(ninjfld)
injstack = tifffile.imread(injch)[:, 390:, :] ##assumes sagittal ABA 25um
ninjstackloc = os.path.join(ninjfld, 'injectionsite.tif')
#normalize and threshold
mxx = injstack.max()
injstack = (injstack - injstack.min()) / (injstack.max() - injstack.min())
injstack[injstack < threshold] = 0
injstack = injstack * mxx
tifffile.imsave(ninjstackloc, injstack.astype('uint16'))
####applying depth coding
depth.colorcode(ninjstackloc, ninjfld)
###apply overlay
color = [
os.path.join(ninjfld, 'proj0.png'),
os.path.join(ninjfld, 'proj1.png'),
os.path.join(ninjfld, 'proj2.png')
]
nametosave = [
os.path.join(ninjfld, "proj0_overlay.png"),
os.path.join(ninjfld, "proj1_overlay.png"),
os.path.join(ninjfld, "proj2_overlay.png")
]
for x in range(3):
print(grayscale[x], color[x], nametosave[x])
depth.overlay(grayscale[x], color[x], nametosave[x], alpha=0.95)
depth.layout(nametosave[1], nametosave[0], nametosave[2], ninjfld)
print(grayscale)
print(color)
print(nametosave)
writer(elastixfld, "\nFinished inj_and_cells depth coloring overlay")
return
def resize(src, resizefactor, cores):
"""
src = fullsizedata_fld
"""
#find files
fls = listall(src, keyword=".tif")
#calc resize
y, x = tifffile.imread(fls[0], multifile=False).shape
yr = int(y / resizefactor)
xr = int(x / resizefactor)
#set up dsts
[
makedir(
os.path.join(os.path.dirname(src), xx[:-4] + "resized_" + xx[-4:]))
for xx in os.listdir(src) if ".txt" not in xx
]
#parallelize
iterlst = [copy.deepcopy({"fl": fl, "xr": xr, "yr": yr}) for fl in fls]
p = mp.Pool(cores)
p.map(resize_helper, iterlst)
p.terminate()
return
def isolate_and_overlay(pth, AtlasFile, svlc, nm_to_sv,*args):
'''function to segment out allen brain structures and then depth color overlay.
Inputs:
pth=pth to allen annoation
AtlasFile=pth to atlas (grayscale)
svlc=pth to folder to save
nm_to_sv=text to identify this structure
*args=lst of structure ID's (pixel values) to keep
'''
im=isolate_structures(pth, *args)
makedir(svlc)
svlc2=os.path.join(svlc, nm_to_sv)
makedir(svlc2)
impth=os.path.join(svlc2, str(nm_to_sv)+'.tif')
tifffile.imsave(impth, im.astype('uint8'))
allen_compare(AtlasFile, svlc, impth)
return
def similarity_transform(fx, mv, dst, nm, level='fast', cleanup=True):
'''function for similarity transform
Inputs
-------------------
fx = fixed path (usually Atlas for 'normal' noninverse transforms)
mv = moving path (usually volume to register for 'normal' noninverse transforms)
dst = location to save
nm = 'name of file to save'
level = 'fast', 'intermediate', 'slow' : links to parameter files of certain complexity
cleanup = if False do not remove files
Returns
-------------
path to file
path to transform file (if cleanup == False)
'''
transform_to_use = {
'slow':
'/jukebox/wang/pisano/Python/lightsheet/supp_files/similarity_transform_slow.txt',
'intermediate':
'/jukebox/wang/pisano/Python/lightsheet/supp_files/similarity_transform_intermediate.txt',
'fast':
'/jukebox/wang/pisano/Python/lightsheet/supp_files/similarity_transform_fast.txt'
}[level]
#make dir
makedir(dst)
out = os.path.join(dst, 'tmp')
makedir(out)
fl, tp = elastix_command_line_call(fx,
mv,
out=out,
parameters=[transform_to_use])
#move and delete
if nm[-4:] != '.tif': nm = nm + '.tif'
dstfl = os.path.join(dst, nm)
shutil.move(fl, dstfl)
if cleanup: shutil.rmtree(out)
print('saved as {}'.format(dstfl))
if cleanup: return dstfl
if not cleanup: return dstfl, tp
def points_transform(src, dst, transformfile, verbose=False):
'''Function to apply a tranform given a numpy.
***Assumes ZYX and that any orientation changes have already been done.***
src: numpy array or list of np arrays of dims nx3
dst: folder to save
'''
src = np.asarray(src)
assert src.shape[-1] == 3, 'src must be a nx3 array'
#create txt file, with elastix header, then populate points
makedir(dst)
pnts_fld = os.path.join(dst, 'transformedpoints_pretransformix')
makedir(pnts_fld)
transforminput = os.path.join(pnts_fld, 'zyx_transformedpnts.txt')
#prevent adding to an already made file
removedir(transforminput)
writer(pnts_fld, 'index\n{}\n'.format(len(src)), flnm=transforminput)
if verbose:
sys.stdout.write(
'\nwriting centers to transfomix input points text file: {}....'.
format(transforminput))
#this step converts from zyx to xyz*****
stringtowrite = '\n'.join(
['\n'.join(['{} {} {}'.format(i[2], i[1], i[0])]) for i in src])
writer(pnts_fld, stringtowrite, flnm=transforminput)
if verbose:
sys.stdout.write('...done writing centers.')
sys.stdout.flush()
#elastix for inverse transform
trnsfrm_out_file = point_transformix(txtflnm=transforminput,
dst=dst,
transformfile=transformfile)
assert os.path.exists(
trnsfrm_out_file), 'Error Transform file does not exist: {}'.format(
trnsfrm_out_file)
#if verbose: sys.stdout.write('\n***Elastix Inverse Transform File: {}***'.format(transformfile))
return trnsfrm_out_file
def blend_lightsheets_pre_stitching(left, right, dst, cores):
st = time.time()
name = os.path.basename(dst)
ch = dst[-2:]
sys.stdout.write("\nStarting blending of {}...".format(dst))
sys.stdout.flush()
ydim0, xdim0 = sitk.GetArrayFromImage(sitk.ReadImage(left[0])).shape
ydim1, xdim1 = sitk.GetArrayFromImage(sitk.ReadImage(right[0])).shape
#checks
assert ydim0 == ydim1
assert xdim0 == xdim1
assert len(left) == len(right)
makedir(dst)
iterlst = [{
"xdim": xdim0,
"ydim": ydim0,
"fl0": left[i],
"channel": ch,
"fl1": right[i],
"dst": dst,
"name": name,
"zplane": i,
"stitched": False
} for i, l in enumerate(left)]
if cores >= 2:
p = mp.Pool(cores)
p.map(blend, iterlst)
p.terminate()
else:
[blend(dct) for dct in iterlst]
#if cleanup: [shutil.rmtree(xx) for xx in flds]
sys.stdout.write("\n...finished in {} minutes.\n".format(np.round(
(time.time() - st) / 60),
decimals=2))
sys.stdout.flush()
return
def blend_lightsheets(name, flds, dst, cores):
'''
'''
sys.stdout.write('\nStarting blending of {}...'.format(name)); sys.stdout.flush()
ydim, xdim =tifffile.imread(listall(flds[0], keyword='.tif')[0]).shape
alpha=np.tile(scipy.stats.logistic.cdf(np.linspace(-250, 250, num=xdim)), (ydim, 1))
fls0 = listall(flds[0], keyword='.tif'); fls0.sort()
fls1 = listall(flds[1], keyword='.tif'); fls1.sort()
assert set([os.path.basename(xx) for xx in fls0]) == set([os.path.basename(xx) for xx in fls1]), 'uneven number of z planes between L and R lightsheets'
makedir(os.path.join(dst, name))
iterlst=[{'alpha':alpha, 'fl0':fl0, 'fl1':fls1[i], 'dst':dst, 'name':name, 'zplane':i} for i,fl0 in enumerate(fls0)]
if cores>=2:
p=mp.Pool(cores)
p.map(blend, iterlst)
p.terminate()
else:
[blend(dct) for dct in iterlst]
[shutil.rmtree(xx) for xx in flds]
sys.stdout.write('completed.\n'.format(name)); sys.stdout.flush()
return
def generate_transformed_cellcount(dataframe, dst, transformfiles, lightsheet_parameter_dictionary, verbose=False):
"""Function to take a csv file and generate an input to transformix
Inputs
----------------
dataframe = preloaded pandas dataframe
dst = destination to save files
transformfiles = list of all elastix transform files used, and in order of the original transform****
lightsheet_parameter_file = .p file generated from lightsheet package
"""
#set up locations
transformed_dst = os.path.join(dst, "transformed_points"); makedir(transformed_dst)
#make zyx numpy arry
zyx = dataframe[["z","y","x"]].values
#adjust for reorientation THEN rescaling, remember full size data needs dimension change releative to resample
fullsizedimensions = get_fullsizedimensions(lightsheet_parameter_dictionary)
kwargs = load_kwargs(lightsheet_parameter_dictionary)
zyx = fix_contour_orientation(zyx, verbose=verbose, **kwargs) #now in orientation of resample
resampled_dims, resampled_vol = get_resampledvol_n_dimensions(lightsheet_parameter_dictionary)
zyx = points_resample(zyx, original_dims = fix_dimension_orientation(fullsizedimensions,
**kwargs), resample_dims = resampled_dims, verbose = verbose)[:, :3]
#make into transformix-friendly text file
pretransform_text_file = create_text_file_for_elastix(zyx, transformed_dst)
#copy over elastix files
transformfiles = modify_transform_files(transformfiles, transformed_dst)
change_transform_parameter_initial_transform(transformfiles[0], "NoInitialTransform")
#run transformix on points
points_file = point_transformix(pretransform_text_file, transformfiles[-1], transformed_dst)
#convert registered points into structure counts
converted_points = unpack_pnts(points_file, transformed_dst)
return converted_points
def generate_transformed_cellcount(points, dst, transformfiles):
""" makes an input to transformix """
#set up locations
transformed_dst = os.path.join(dst, "transformed_points")
makedir(transformed_dst)
#make zyx numpy arry
zyx = np.asarray(points)
transformfiles = modify_transform_files(transformfiles, transformed_dst)
change_transform_parameter_initial_transform(transformfiles[0],
"NoInitialTransform")
pretransform_text_file = create_text_file_for_elastix(zyx, transformed_dst)
#run transformix on points
points_file = point_transformix(pretransform_text_file, transformfiles[-1],
transformed_dst)
#convert registered points into structure counts
converted_points = unpack_pnts(points_file, transformed_dst)
return converted_points
def generate_cropped_atlas(**kwargs):
'''Wrapper to call crop_atlas and save in appropriate place
Inputs
-------------------
kwargs used:
ouputdirectory
AtlasFile
maskatlas
Returns:
-------------------
path to newly saved cropped atlas
'''
#make folder to store masked atlas
fld = os.path.join(kwargs['outputdirectory'], 'croppedatlas')
makedir(fld)
#generate masked_atlas
cropped_atlas = crop_atlas(**kwargs)
#save masked_atlas
cropped_atlas_pth = os.path.join(fld, 'cropped_atlas.tif')
tifffile.imsave(cropped_atlas_pth, cropped_atlas)
return cropped_atlas_pth
ch = "Ex_642_Em_2"
#for array job parallelization
print(os.environ["SLURM_ARRAY_TASK_ID"])
jobid = int(os.environ["SLURM_ARRAY_TASK_ID"])
# jobid=0
#set brain name
brain = os.path.join(src, brains[jobid])
start = time.time()
#need to change this config depending on processing pipeline
cellvol = fast_scandir(os.path.join(brain,ch,"stitched"))[-1]
#link to parameters
a2r0 = os.path.join(brain, "elastix_inverse_transform/TransformParameters.0.txt")
a2r1 = os.path.join(brain, "elastix_inverse_transform/TransformParameters.1.txt")
#set destination directory
braindst = os.path.join(scratch_dir, os.path.basename(brain))
makedir(braindst)
aldst = os.path.join(braindst, "transformed_annotations"); makedir(aldst)
#transformix
# transformfiles = modify_transform_files(transformfiles=[a2r0, a2r1, r2s0, r2s1], dst = aldst)
transformfiles = modify_transform_files(transformfiles=[a2r0, a2r1], dst = aldst)
[change_interpolation_order(xx,0) for xx in transformfiles]
#change the parameter in the transform files that outputs 16bit images instead
for fl in transformfiles:# Read in the file
with open(fl, "r") as file:
filedata = file.read()
# Replace the target string
filedata = filedata.replace('(ResultImagePixelType "short")', '(ResultImagePixelType "float")')
# Write the file out again
with open(fl, "w") as file:
file.write(filedata)
#run transformix
filtered = gfilt(clh**power, filter_kernel)
#tif.imsave("/home/wanglab/Desktop/filtered_z200-300.tif", filtered[200:300].astype("float32"))
#
#plt.figure()
#plt.imshow(filtered[300], "gist_yarg")
thresholded = filtered > filtered.mean() + thresh * filtered.std()
labelled, nlab = label(thresholded)
#plt.figure()
sizes = [np.sum(labelled == i) for i in range(1, nlab + 1)]
vals = [i + 1 for i in np.argsort(sizes)[::-1]]
arr = np.in1d(labelled, vals).reshape(labelled.shape)
seg_dst = os.path.join(dst, "rawdata_segmentations")
makedir(seg_dst)
tif.imsave(
os.path.join(seg_dst, "%s.tif" % os.path.basename(brain)),
np.in1d(labelled, vals).reshape(labelled.shape).astype("uint16"))
im = tif.imread(imgs[0])
arr_fullsz = np.zeros((len(imgs), im.shape[0], im.shape[1]))
print(arr_fullsz.shape)
print(arr.shape)
#################################################################################################################################################################
arr_fullsz[:, 1700:, :] = arr
#plt.imshow(arr_fullsz[300])
zf, yf, xf = 1, (1 / 3), (1 / 3)
#this isn't a perfect Quality control and you can expect some pixel shifts
#qc_overlay_transform_type = 'affine_only_reg_to_sig';#both for reg+atlas, and only affine for sig and reg. This is what clearmap does
#qc_overlay_transform_type = 'all' #both for auto+atlas and for sig+auto
qc_overlay_transform_type = 'single'
#don't consider reg with sig at all
#%%
###############################
#run, don't touch below
###############################
df = pd.read_csv(path_to_csv_file)
for i, row in df.iterrows():
#set up destination
dst = os.path.join(row['dst'], dst_suffix)
makedir(dst)
#lightsheet package output folder
fld = row['fld']
#path to mat file
cell_centers_path = os.path.join(row['prefix_path_to_mat_file'],
mat_file_suffix)
print(
'\n\n_________________\nFolder: {}\nDestination: {}\nMatfile: {}\n'
.format(fld, dst, cell_centers_path))
#loads
kwargs = load_kwargs(fld)
regvol = [xx for xx in kwargs['volumes'] if xx.ch_type == 'regch'][0]
np.sum(xx[4:7]),
np.sum(xx[7:10]), xx[10], xx[11], xx[12],
np.sum(xx[13:16])
] for xx in expr_all_as_frac_of_inj])
primary_pool = np.array([np.argmax(e) for e in frac_of_inj_pool])
#get n's after pooling
primary_lob_n = np.array([
len(np.where(primary_pool == i)[0]) for i in range(max(primary_pool) + 1)
])
#normalization of inj site
frac_of_inj_pool_norm = np.asarray(
[brain / brain.sum() for brain in frac_of_inj_pool])
lr_brains = list(lr_dist.keys())
atl_dst = os.path.join(dst, "pma_to_aba")
makedir(atl_dst)
id_table = pd.read_excel(df_pth)
#%%
#------------------------------------------------------------------------------------------------------------------------------
#NOTE THAT ONLY HAVE TO DO THIS ONCE!!!! DO NOT NEED TO DO AGAIN UNLESS DOUBLE CHECKIHG
#transform points to allen atlas space
#get brains that we actually need to get cell counts from
src = "/jukebox/wang/pisano/tracing_output/antero_4x_analysis/201903_antero_pooled_cell_counts_thalamus/transformed_points"
post_transformed = [
os.path.join(
src,
os.path.join(xx, "transformed_points/posttransformed_zyx_voxels.npy"))
for xx in lr_brains
]
transformfiles = [
def pool_injections_for_analysis(**kwargs):
inputlist = kwargs['inputlist']
dst = kwargs['dst']; makedir(dst)
injscale = kwargs['injectionscale'] if 'injectionscale' in kwargs else 1
imagescale = kwargs['imagescale'] if 'imagescale' in kwargs else 1
axes = kwargs['reorientation'] if 'reorientation' in kwargs else ('0','1','2')
cmap = kwargs['colormap'] if 'colormap' in kwargs else 'plasma'
id_table = kwargs['id_table'] if 'id_table' in kwargs else '/jukebox/temp_wang/pisano/Python/lightsheet/supp_files/allen_id_table.xlsx'
save_tif = kwargs['save_tif'] if 'save_tif' in kwargs else False
num_sites_to_keep = kwargs['num_sites_to_keep'] if 'num_sites_to_keep' in kwargs else 1
nonzeros = []
ann = sitk.GetArrayFromImage(sitk.ReadImage(kwargs['annotation']))
if kwargs['crop']: ann = eval('ann{}'.format(kwargs['crop']))
allen_id_table=pd.read_excel(id_table)
for i in range(len(inputlist)):
impth = inputlist[i]
animal = os.path.basename(os.path.dirname(os.path.dirname(os.path.dirname(impth))))
print('\n\n_______\n{}'.format(animal))
print(' loading:\n {}'.format(animal))
im = tifffile.imread(impth)
if kwargs['crop']: im = eval('im{}'.format(kwargs['crop']))#; print im.shape
#reorient to coronal?
#segment
arr = find_site(im, thresh=kwargs['threshold'], filter_kernel=kwargs['filter_kernel'], num_sites_to_keep=num_sites_to_keep)*injscale
if save_tif: tifffile.imsave(os.path.join(dst,'{}'.format(os.path.dirname(impth))+'_inj.tif'), arr.astype('float32'))
#optional 'save_individual'
if kwargs['save_individual']:
im = im*imagescale
a=np.concatenate((np.max(im, axis=0), np.max(arr.astype('uint16'), axis=0)), axis=1)
b=np.concatenate((np.fliplr(np.rot90(np.max(fix_orientation(im, axes=axes), axis=0),k=3)), np.fliplr(np.rot90(np.max(fix_orientation(arr.astype('uint16'), axes=axes), axis=0),k=3))), axis=1)
plt.figure()
plt.imshow(np.concatenate((b,a), axis=0), cmap=cmap, alpha=1); plt.axis('off')
plt.savefig(os.path.join(dst,'{}'.format(animal)+'.pdf'), dpi=300, transparent=True)
plt.close()
#cell counts to csv
print(' finding nonzero pixels for voxel counts...')
nz = np.nonzero(arr)
nonzeros.append(zip(*nz)) #<-for pooled image
pos = transformed_pnts_to_allen_helper_func(np.asarray(zip(*[nz[2], nz[1], nz[0]])), ann)
tdf = count_structure_lister(allen_id_table, *pos)
if i == 0:
df = tdf.copy()
countcol = 'count' if 'count' in df.columns else 'cell_count'
df.drop([countcol], axis=1, inplace=True)
df[animal] = tdf[countcol]
df.to_csv(os.path.join(dst,'voxel_counts.csv'))
print('\n\nCSV file of cell counts, saved as {}\n\n\n'.format(os.path.join(dst,'voxel_counts.csv')))
#condense nonzero pixels
nzs = [str(x) for xx in nonzeros for x in xx] #this list has duplicates if two brains had the same voxel w label
c = Counter(nzs)
array = np.zeros(im.shape)
print('Collecting nonzero pixels for pooled image...')
tick = 0
#generating pooled array where voxel value = total number of brains with that voxel as positive
for k,v in c.iteritems():
k = [int(xx) for xx in k.replace('(','').replace(')','').split(',')]
array[k[0], k[1], k[2]] = int(v)
tick+=1
if tick % 50000 == 0: print(' {}'.format(tick))
#load atlas and generate final figure
print('Generating final figure...')
atlas = tifffile.imread(kwargs['atlas'])
arr = fix_orientation(array, axes=axes)
#cropping
#if 'crop_atlas' not in kwargs:
if kwargs['crop']: atlas = eval('atlas{}'.format(kwargs['crop']))
atlas = fix_orientation(atlas, axes=axes)
my_cmap = eval('plt.cm.{}(np.arange(plt.cm.RdBu.N))'.format(cmap))
my_cmap[:1,:4] = 0.0
my_cmap = mpl.colors.ListedColormap(my_cmap)
my_cmap.set_under('w')
plt.figure()
plt.imshow(np.max(atlas, axis=0), cmap='gray')
plt.imshow(np.max(arr, axis=0), alpha=0.99, cmap=my_cmap); plt.colorbar(); plt.axis('off')
dpi = int(kwargs['dpi']) if 'dpi' in kwargs else 300
plt.savefig(os.path.join(dst,'heatmap.pdf'), dpi=dpi, transparent=True);
plt.close()
print('Saved as {}'.format(os.path.join(dst,'heatmap.pdf')))
return df
def overlay_qc(args):
#unpacking this way for multiprocessing
fld, folder_suffix, output_folder, verbose, doubletransform, make_volumes = args
try:
#get 3dunet cell dataframe csv file
input_csv = listdirfull(os.path.join(fld, folder_suffix), ".csv")
assert len(input_csv) == 1, "multiple csv files"
dataframe = pd.read_csv(input_csv[0])
#location to save out
dst = os.path.join(output_folder, os.path.basename(fld)); makedir(dst)
#EXAMPLE USING LIGHTSHEET - assumes marking centers in the "raw" full sized cell channel. This will transform those
#centers into "atlas" space (in this case the moving image)
#in this case the "inverse transform has the atlas as the moving image in the first step,
#and the autofluorescence channel as the moving image in the second step
#NOTE - it seems that the registration of cell to auto is failing on occasion....thus get new files...
################################
cell_inverse_folder = listdirfull(os.path.join(fld, "elastix_inverse_transform"), "cellch")[0]
a2r = listall(cell_inverse_folder, "atlas2reg_TransformParameters"); a2r.sort()
r2s = listall(cell_inverse_folder, "reg2sig_TransformParameters"); r2s.sort() #possibly remove
#IMPORTANT. the idea is to apply cfos->auto->atlas
transformfiles = r2s + a2r if doubletransform else a2r #might get rid of r2s
lightsheet_parameter_dictionary = os.path.join(fld, "param_dict.p")
converted_points = generate_transformed_cellcount(dataframe, dst, transformfiles,
lightsheet_parameter_dictionary, verbose=verbose)
#load and convert to single voxel loc
zyx = np.asarray([str((int(xx[0]), int(xx[1]), int(xx[2]))) for xx in np.nan_to_num(np.load(converted_points))])
from collections import Counter
zyx_cnt = Counter(zyx)
#check...
if make_volumes:
#manually call transformix
kwargs = load_dictionary(lightsheet_parameter_dictionary)
vol = [xx for xx in kwargs["volumes"] if xx.ch_type == "cellch"][0].resampled_for_elastix_vol
transformed_vol = os.path.join(dst, "transformed_volume"); makedir(transformed_vol)
if not doubletransform:
transformfiles = [os.path.join(fld, "elastix/TransformParameters.0.txt"), os.path.join(fld,
"elastix/TransformParameters.1.txt")]
transformfiles = modify_transform_files(transformfiles, transformed_vol) #copy over elastix files
transformix_command_line_call(vol, transformed_vol, transformfiles[-1])
else:
v=[xx for xx in kwargs["volumes"] if xx.ch_type == "cellch"][0]
#sig to reg
tps = [listall(os.path.dirname(v.ch_to_reg_to_atlas), "/TransformParameters.0")[0],
listall(os.path.dirname(v.ch_to_reg_to_atlas), "/TransformParameters.1")[0]]
#reg to atlas
transformfiles = tps+[os.path.join(fld, "elastix/TransformParameters.0.txt"),
os.path.join(fld, "elastix/TransformParameters.1.txt")]
transformfiles = modify_transform_files(transformfiles, transformed_vol) #copy over elastix files
transformix_command_line_call(vol, transformed_vol, transformfiles[-1])
#cell_registered channel
cell_reg = tifffile.imread(os.path.join(transformed_vol, "result.tif"))
tifffile.imsave(os.path.join(transformed_vol, "result.tif"), cell_reg, compress=1)
cell_cnn = np.zeros_like(cell_reg)
tarr = []; badlist=[]
for zyx,v in zyx_cnt.items():
z,y,x = [int(xx) for xx in zyx.replace("(","",).replace(")","").split(",")]
tarr.append([z,y,x])
try:
cell_cnn[z,y,x] = v*100
except:
badlist.append([z,y,x])
#apply x y dilation
r = 2
selem = ball(r)[int(r/2)]
cell_cnn = cell_cnn.astype("uint8")
cell_cnn = np.asarray([cv2.dilate(cell_cnn[i], selem, iterations = 1) for i in range(cell_cnn.shape[0])])
tarr=np.asarray(tarr)
if len(badlist)>0:
print("{} errors in mapping with cell_cnn shape {}, each max dim {}, \npossibly due to a registration overshoot \
or not using double transform\n\n{}".format(len(badlist), cell_cnn.shape, np.max(tarr,0), badlist))
merged = np.stack([cell_cnn, cell_reg, np.zeros_like(cell_reg)], -1)
tifffile.imsave(os.path.join(transformed_vol, "merged.tif"), merged)#, compress=1)
#out = np.concatenate([cell_cnn, cell_reg, ], 0)
#####check at the resampled for elastix phase before transform...this mapping looks good...
if make_volumes:
#make zyx numpy arry
zyx = dataframe[["z","y","x"]].values
kwargs = load_dictionary(lightsheet_parameter_dictionary)
vol = [xx for xx in kwargs["volumes"] if xx.ch_type =="cellch"][0]
fullsizedimensions = get_fullsizedims_from_kwargs(kwargs) #don"t get from kwargs["volumes"][0].fullsizedimensions it"s bad! use this instead
zyx = fix_contour_orientation(zyx, verbose=verbose, **kwargs) #now in orientation of resample
zyx = points_resample(zyx, original_dims = fix_dimension_orientation(fullsizedimensions, **kwargs),
resample_dims = tifffile.imread(vol.resampled_for_elastix_vol).shape, verbose = verbose)[:, :3]
#cell channel
cell_ch = tifffile.imread(vol.resampled_for_elastix_vol)
cell_cnn = np.zeros_like(cell_ch)
tarr = []; badlist=[]
for _zyx in zyx:
z,y,x = [int(xx) for xx in _zyx]
tarr.append([z,y,x])
try:
cell_cnn[z,y,x] = 100
except:
badlist.append([z,y,x])
tarr=np.asarray(tarr)
merged = np.stack([cell_cnn, cell_ch, np.zeros_like(cell_ch)], -1)
tifffile.imsave(os.path.join(transformed_vol, "resampled_merged.tif"), merged)#, compress=1)
except Exception as e:
print(e)
with open(error_file, "a") as err_fl:
err_fl.write("\n\n{} {}\n\n".format(fld, e))
except Exception as e:
print(e)
with open(error_file, "a") as err_fl:
err_fl.write("\n\n{} {}\n\n".format(fld, e))
if __name__ == "__main__":
#goal is to transform cooridnates, voxelize based on number of cells and overlay with reigstered cell signal channel...
#set paths and make folders
dst = "/jukebox/wang/zahra/h129_qc"
input_folder = "/jukebox/wang/pisano/tracing_output/antero_4x/"
folder_suffix = "3dunet_output/pooled_cell_measures"
#thalamus
output_folder = os.path.join(dst, "rtn_thal_transformed_points"); makedir(output_folder)
qc_folder = os.path.join(dst, "rtn_thal_qc"); makedir(qc_folder)
error_file = os.path.join(qc_folder, "errors.txt")
#inputs
brains = ["20160622_db_bl6_crii_52hr_01", "20160622_db_bl6_unk_01", "20160801_db_cri_02_1200rlow_52hr", "20160801_db_l7_cri_01_mid_64hr",
"20160822_tp_bl6_crii_250r_01", "20160822_tp_bl6_crii_1250r_05", "20160822_tp_bl6_crii_1500r_06", "20160823_tp_bl6_cri_250r_01",
"20160823_tp_bl6_cri_500r_02", "20160912_tp_bl6_lob7_750r_04", "20160916_tp_lob6_ml_01", "20160916_tp_lob6_ml_04", "20160916_tp_lob7_250r_05",
"20160920_tp_bl6_lob7_250r_02", "20160920_tp_bl6_lob7_500r_03", "20160920_tp_bl6_lob7_ml_01", "20161129_db_bl6_lob6b_ml50r_53hr",
"20161201_db_bl6_lob6b_500r_53d5hr", "20161203_tp_bl6_crii_250r_08", "20161203_tp_bl6_crii_750r_09",
"20161203_tp_bl6_lob7_500r_05", "20161203_tp_bl6_lob7_1000r_06", "20161203_tp_bl6_lob7_1500r_07",
"20161203_tp_bl6_lob7_ml_04", "20161205_tp_bl6_sim_250r_02", "20161205_tp_bl6_sim_750r_03",
"20161205_tp_bl6_sim_1250r_04", "20161205_tp_bl6_sim_1750r_05","20161207_db_bl6_lob6a_500r_53hr",
"20161207_db_bl6_lob6a_850r_53hr", "20161208_db_bl6_cri_50r_pv_53hr","20170115_tp_bl6_lob6b_500r_05",
"20170115_tp_bl6_lob6b_1000r_06","20170116_tp_bl6_lob6b_lpv_07","20170130_tp_bl6_sim_rlat_05",
"20170207_db_bl6_crii_1300r_02", "20170207_db_bl6_crii_rlat_03","20170212_tp_bl6_lob8_ml_05",
def ovly_3d(ch, svlc, tplst, valid_plns, outdr, vol_to_process, resizefactor,
contour_class_lst, multiplane, ovly, core, cores):
'''function to apply independent colors to contours that have been detected on multiple planes, and save images.
This has been parallelized and calculates a range based on the number of cores.
'''
####assign random color to each group of contours, the break up groups, and sort by plane
#use this for looking at multi contours only
if multiplane != False:
multilist = [x for x in tplst if len(x.plns) > 1]
tplst = multilist
contour_color_lst = []
for contours in tplst:
color = (random.randint(50, 255), random.randint(50, 255),
random.randint(50, 255))
for plns in contours.plns.itervalues():
contour_color_lst.append([plns, color]) ###break apart list
#sort by z
try: #sort by z
contour_color_lst.sort()
except ValueError: #done to suppress error
pass
############################################load find full size files############################################
makedir(svlc)
dr = vol_to_process.full_sizedatafld_vol
#dr='/home/wanglab/wang/pisano/tracing_output/H129_EGFP_NLS_vc22_01_lob5_intensityrescale/full_sizedatafld/ch01'
fl = [f for f in os.listdir(dr) if '.tif' in f]
reg = re.compile(r'(.*C+)(?P<ch>[0-9]{1,2})(.*Z+)(?P<z>[0-9]{1,4})(.tif)'
) ###regex used since code changes file format
matches = map(reg.match, fl)
##find index of z,y,x,ch in matches
z_indx = matches[0].span('z')
####make dictionary where each pln is a key, values are all point that should be drawn
zdct = {}
for i in range(len(contour_color_lst)):
pln = contour_color_lst[i] ##check
try:
zdct[str(pln[0][0]).zfill(4)].append(pln)
except KeyError:
zdct[str(pln[0][0]).zfill(4)] = []
zdct[str(pln[0][0]).zfill(4)].append(pln)
############parse jobs:
chnkrng = chunkit(core, cores, zdct)
########### apply points to each pln
if ovly == True: ###ovly 3d contours onto data
flpth = ''.join(matches[0].groups())
y, x = tifffile.imread(os.path.join(dr, flpth)).shape
dsf = resizefactor
for i in range(chnkrng[0], chnkrng[1]):
try:
pln = zdct.keys()[i]
cntrs = zdct[zdct.keys()[i]]
ovly_helper(flpth, z_indx, dr, svlc, x, y, dsf, pln, cntrs)
del cntrs, pln
gc.collect()
except:
pass
###make zpln range and make empty planes on the zplns where no contours are found
try:
plnlst = [int(plns) for plns in zdct.iterkeys()]
plnrng = range(min(plnlst),
max(plnlst) + 1)
nocontour_plns = set(plnrng).difference(set(plnlst))
############parse jobs:
chnkrng = chunkit(core, cores, nocontour_plns)
print(
'\n\ncontours not found on planes: {}\nmaking empty images for those planes...'
.format(nocontour_plns))
for i in range(chnkrng[0], chnkrng[1]):
pln = list(nocontour_plns)[i]
blnk_ovly_helper(flpth, z_indx, dr, svlc, x, y, dsf, pln)
del pln
gc.collect()
return
except ValueError: #no contours found
print('No contour found in ch{}, not making color stack...'.format(
ch))
return
elif ovly == False: ###make downsized 3d contours
flpth = ''.join(matches[0].groups())
y, x = tifffile.imread(os.path.join(dr, flpth)).shape
dsf = resizefactor
for i in range(chnkrng[0], chnkrng[1]):
try:
pln = zdct.keys()[i]
cntrs = zdct[zdct.keys()[i]]
no_ovly_helper(flpth, z_indx, dr, svlc, x, y, dsf, pln, cntrs)
del cntrs, pln
gc.collect()
except:
pass
###make zpln range and make empty planes on the zplns where no contours are found
try:
plnlst = [int(plns) for plns in zdct.iterkeys()]
plnrng = range(min(plnlst),
max(plnlst) + 1)
nocontour_plns = set(plnrng).difference(set(plnlst))
if len(nocontour_plns) != 0:
print(
'\n\ncontours not found on planes: {}\nmaking empty images for those planes...'
.format(nocontour_plns))
############parse jobs:
chnkrng = chunkit(core, cores, nocontour_plns)
for i in range(chnkrng[0], chnkrng[1]):
try:
pln = list(nocontour_plns)[i]
blnk_helper(flpth, z_indx, dr, svlc, x, y, dsf, pln)
del pln
gc.collect()
except IndexError:
pass
return
except ValueError: #no contours found
print('No contour found in ch{}, not making color stack...'.format(
ch))
return
def identify_structures_w_contours(jobid,
cores=5,
make_color_images=False,
overlay_on_original_data=False,
consider_only_multipln_contours=False,
**kwargs):
'''function to take 3d detected contours and apply elastix transform
'''
#######################inputs and setup#################################################
###inputs
kwargs = load_kwargs(**kwargs)
outdr = kwargs['outputdirectory']
vols = kwargs['volumes']
reg_vol = [xx for xx in vols if xx.ch_type == 'regch'][0]
###get rid of extra jobs
if jobid >= len([xx for xx in vols if xx.ch_type != 'regch'
]): ###used to end jobs if too many are called
print('jobid({}) >= volumes {}'.format(
jobid, len([xx for xx in vols if xx.ch_type != 'regch'])))
return
###volumes to process: each job represents a different contour volume
vol_to_process = [xx for xx in vols if xx.ch_type != 'regch'][jobid]
ch = vol_to_process.channel
print(vol_to_process.ch_type)
#find appropriate folders for contours
if vol_to_process.ch_type == 'cellch':
detect3dfld = reg_vol.celldetect3dfld
coordinatesfld = reg_vol.cellcoordinatesfld
elif vol_to_process.ch_type == 'injch':
detect3dfld = reg_vol.injdetect3dfld
coordinatesfld = reg_vol.injcoordinatesfld
#set scale and atlas
xscl, yscl, zscl = reg_vol.xyz_scale ###micron/pixel
zmx, ymx, xmx = reg_vol.fullsizedimensions
AtlasFile = reg_vol.atlasfile
print('Using {} CORES'.format(cores))
try:
p
except NameError:
p = mp.Pool(cores)
resizefactor = kwargs['resizefactor']
brainname = reg_vol.brainname
############################################################################################################
#######################use regex to sort np files by ch and then by zpln####################################
############################################################################################################
fl = [f for f in os.listdir(detect3dfld)
if '.p' in f and 'ch' in f] #sorted for raw files
reg = re.compile(r'(.*h+)(?P<ch>\d{2})(.*)(.p)')
matches = map(reg.match, fl)
##load .np files
sys.stdout.write(
'\njobid({}), loading ch{} .p files to extract contour_class objects....'
.format(jobid, ch))
contour_class_lst = []
for fl in [
os.path.join(detect3dfld, ''.join(xx.groups())) for xx in matches
if xx.group('ch')[-2:] in ch
]:
tmpkwargs = {}
pckl = open(fl, 'rb')
tmpkwargs.update(pickle.load(pckl))
pckl.close()
if consider_only_multipln_contours == False:
tmplst = tmpkwargs['single']
[tmplst.append(xx) for xx in tmpkwargs['multi']]
elif consider_only_multipln_contours == True:
tmplst = tmpkwargs['multi']
[contour_class_lst.append(xx) for xx in tmplst]
sys.stdout.write('\ndone loading contour_class objects.\n')
#check for successful loading
if len(contour_class_lst) == 0:
print('Length of contours in ch{} was {}, ending process...'.format(
jobid, len(contour_class_lst)))
try:
p.terminate()
except:
1
return
############################################################################################################
##############################make color files##############################################################
############################################################################################################
if make_color_images == True:
sys.stdout.write('\nmaking 3d planes...')
sys.stdout.flush()
valid_plns = range(0, zmx + 1)
svlc = os.path.join(outdr, 'ch{}_3dcontours'.format(ch))
removedir(svlc)
if overlay_on_original_data == False:
ovly = False
elif overlay_on_original_data == True:
ovly = True
iterlst = []
[
iterlst.append(
(ch, svlc, contour_class_lst, valid_plns, outdr,
vol_to_process, resizefactor, contour_class_lst,
consider_only_multipln_contours, ovly, core, cores))
for core in range(cores)
]
p.starmap(ovly_3d, iterlst)
lst = os.listdir(svlc)
lst1 = [os.path.join(svlc, xx) for xx in lst]
lst1.sort()
del lst
del iterlst
###load ims and return dct of keys=str(zpln), values=np.array
sys.stdout.write(
'\n3d planes made, saved in {},\nnow compressing into single tifffile'
.format(svlc))
imstack = tifffile.imread(lst1)
del lst1
if len(imstack.shape) > 3:
imstack = np.squeeze(imstack)
try: ###check for orientation differences, i.e. from horiztonal scan to sagittal for atlas registration
imstack = np.swapaxes(imstack, *kwargs['swapaxes'])
except:
pass
tiffstackpth = os.path.join(
outdr, '3D_contours_ch{}_{}'.format(ch, brainname))
tifffile.imsave(tiffstackpth, imstack.astype('uint16'))
del imstack
gc.collect()
shutil.rmtree(svlc)
sys.stdout.write('\ncolor image stack made for ch{}'.format(ch))
else:
sys.stdout.write('\nmake_color_images=False, not creating images')
############################################################################################################
######################apply point transform and make transformix input file#################################
############################################################################################################
###find centers and add 1's to make nx4 array for affine matrix multiplication to account for downsizing
###everything is in PIXELS
contourarr = np.empty((len(contour_class_lst), 3))
for i in range(len(contour_class_lst)):
contourarr[i, ...] = contour_class_lst[
i].center ###full sized dimensions: if 3x3 tiles z(~2000),y(7680),x(6480) before any rotation
try:
contourarr = swap_cols(
contourarr, *kwargs['swapaxes']
) ###change columns to account for orientation changes between brain and atlas: if horizontal to sagittal==>x,y,z relative to horizontal; zyx relative to sagittal
z, y, x = swap_cols(np.array([
vol_to_process.fullsizedimensions
]), *kwargs['swapaxes'])[
0] ##convert full size cooridnates into sagittal atlas coordinates
sys.stdout.write('\nSwapping Axes')
except: ###if no swapaxes then just take normal z,y,x dimensions in original scan orientation
z, y, x = vol_to_process.fullsizedimensions
sys.stdout.write('\nNo Swapping of Axes')
d1, d2 = contourarr.shape
nx4centers = np.ones((d1, d2 + 1))
nx4centers[:, :-1] = contourarr
###find resampled elastix file dim
print(os.listdir(outdr))
print([xx.channel for xx in vols if xx.ch_type == 'regch'])
with tifffile.TiffFile([
os.path.join(outdr, f) for f in os.listdir(outdr)
if 'resampledforelastix' in f and 'ch{}'.format(
[xx.channel for xx in vols if xx.ch_type == 'regch'][0]) in f
][0]) as tif:
zr = len(tif.pages)
yr, xr = tif.pages[0].shape
tif.close()
####create transformmatrix
trnsfrmmatrix = np.identity(4) * (
zr / z, yr / y, xr / x, 1) ###downscale to "resampledforelastix size"
sys.stdout.write('trnsfrmmatrix:\n{}\n'.format(trnsfrmmatrix))
#nx4 * 4x4 to give transform
trnsfmdpnts = nx4centers.dot(trnsfrmmatrix) ##z,y,x
sys.stdout.write('first three transformed pnts:\n{}\n'.format(
trnsfmdpnts[0:3]))
#create txt file, with elastix header, then populate points
txtflnm = '{}_zyx_transformedpnts_ch{}.txt'.format(brainname, ch)
pnts_fld = os.path.join(outdr, 'transformedpoints_pretransformix')
makedir(pnts_fld)
transforminput = os.path.join(pnts_fld, txtflnm)
removedir(transforminput) ###prevent adding to an already made file
writer(pnts_fld, 'index\n{}\n'.format(len(trnsfmdpnts)), flnm=txtflnm)
sys.stdout.write(
'\nwriting centers to transfomix input points text file: {}....'.
format(transforminput))
stringtowrite = '\n'.join([
'\n'.join(['{} {} {}'.format(i[2], i[1], i[0])]) for i in trnsfmdpnts
]) ####this step converts from zyx to xyz*****
writer(pnts_fld, stringtowrite, flnm=txtflnm)
#[writer(pnts_fld, '{} {} {}\n'.format(i[2],i[1],i[0]), flnm=txtflnm, verbose=False) for i in trnsfmdpnts] ####this step converts from zyx to xyz*****
sys.stdout.write('...done writing centers.')
sys.stdout.flush()
del trnsfmdpnts, trnsfrmmatrix, nx4centers, contourarr
gc.collect()
############################################################################################################
####################################elastix for inverse transform###########################################
############################################################################################################
transformfile = make_inverse_transform(vol_to_process, cores, **kwargs)
assert os.path.exists(transformfile)
sys.stdout.write(
'\n***Elastix Inverse Transform File: {}***'.format(transformfile))
############################################################################################################
####################################transformix#############################################################
############################################################################################################
if make_color_images != False:
#apply transform to 3d_tiffstack:
transformimageinput = tiffstackpth
elastixpth = os.path.join(outdr, 'elastix')
trnsfrm_outpath = os.path.join(
elastixpth, '3D_contours_ch{}_{}'.format(ch, brainname))
makedir(trnsfrm_outpath)
writer(trnsfrm_outpath, '\nProcessing ch{} 3D...'.format(ch))
#transformfiles=[os.path.join(elastixpth, xx) for xx in os.listdir(os.path.join(outdr, 'elastix')) if "TransformParameters" in xx]; mxx=max([xx[-5] for xx in transformfiles])
#transformfile=os.path.join(elastixpth, 'TransformParameters.{}.txt'.format(mxx))
trnsfrm_out_file = os.path.join(trnsfrm_outpath,
'result.tif') #output of transformix
transformimageinput_resized = transformimageinput[:-4] + '_resampledforelastix.tif'
print('Resizing {}'.format(transformimageinput_resized))
resample_par(cores,
transformimageinput,
AtlasFile,
svlocname=transformimageinput_resized,
singletifffile=True,
resamplefactor=1.7)
sp.call([
'transformix', '-in', transformimageinput_resized, '-out',
trnsfrm_outpath, '-tp', transformfile
])
writer(trnsfrm_outpath,
'\n Transformix File Generated: {}'.format(trnsfrm_out_file))
writer(
trnsfrm_outpath, '\n Passing colorcode: {} file as {}'.format(
ch, os.path.join(trnsfrm_outpath, 'depthcoded.png')))
###depth coded image of transformix result; not functional yet
#depth.colorcode(trnsfrm_out_file, trnsfrm_outpath)
#getvoxels(trnsfrm_out_file, os.path.join(trnsfrm_outpath, 'zyx_voxels_{}.npy'.format(ch)))
#allen_compare(AtlasFile, svlc, trnsfrm_outpath)
##if successful delete contour cooridnates and maybe contourdetect3d flds
############################################################
##############apply transform to points#####################
elastixpth = os.path.join(outdr, 'elastix_inverse_transform')
trnsfrm_outpath = os.path.join(elastixpth, 'ch{}_3dpoints'.format(ch))
makedir(trnsfrm_outpath)
writer(trnsfrm_outpath,
'\n***********Starting Transformix for: {}***********'.format(ch))
sys.stdout.flush()
#transformfiles=[os.path.join(elastixpth, xx) for xx in os.listdir(os.path.join(outdr, 'elastix')) if "TransformParameters" in xx]; mxx=max([xx[-5] for xx in transformfiles])
#transformfile=os.path.join(elastixpth, 'TransformParameters.{}.txt'.format(mxx))
trnsfrm_out_file = os.path.join(
trnsfrm_outpath, 'outputpoints.txt') #MIGHT NEED TO CHANGE THIS
sp.call([
'transformix', '-def', transforminput, '-out', trnsfrm_outpath, '-tp',
transformfile
])
#sp.call(['transformix', '-def', 'all', '-out', trnsfrm_outpath, '-tp', transformfile]) ##displacement field
writer(trnsfrm_outpath,
'\n Transformix File Generated: {}'.format(trnsfrm_out_file))
####################################################################################
##############generate list and image overlaid onto allen atlas#####################
####################################################################################
name = 'job{}_{}'.format(jobid, vol_to_process.ch_type)
transformed_pnts_to_allen(trnsfrm_out_file, ch, cores, name=name, **kwargs)
writer(outdr, '*************STEP 5*************\n Finished')
print('end of script')
try:
p.terminate()
except:
1
return
from tools.utils.io import listdirfull, makedir, load_memmap_arr, load_np, listall, load_kwargs
from skimage.external import tifffile
import skimage
from tools.utils.directorydeterminer import directorydeterminer
from scipy.ndimage.interpolation import zoom
from tools.registration.transform_cell_counts import generate_transformed_cellcount, get_fullsizedims_from_kwargs, points_resample
from tools.registration.transform_list_of_points import modify_transform_files
from tools.imageprocessing.orientation import fix_contour_orientation, fix_dimension_orientation
import matplotlib.gridspec as gridspec
from tools.conv_net.functions.dilation import dilate_with_element
from skimage.morphology import ball
if __name__ == '__main__':
#set up
dst = '/home/wanglab/wang/pisano/tracing_output/qc/antero_no_jg'; makedir(dst)
lst = [xx for xx in listdirfull('/home/wanglab/wang/pisano/tracing_output/antero_4x') if 'jg' not in os.path.basename(xx)]
dst = '/home/wanglab/wang/pisano/tracing_output/qc/antero_only_jg'; makedir(dst)
lst = [xx for xx in listdirfull('/home/wanglab/wang/pisano/tracing_output/antero_4x') if 'jg' in os.path.basename(xx)]
cnn_transform_type = 'affine_only_reg_to_sig';#both for regwatlas, and only affine for sig adn reg #'all', 'single': don't consider reg with sig at all
volume_transform_type = 'single';#both for regwatlas, and only affine for sig adn reg #'all', 'single': don't consider reg with sig at all
verbose = True
generate_registered_overlay = False #this looks bad
generate_downsized_overlay = True #this looks better
#fld = '/home/wanglab/wang/pisano/tracing_output/antero_4x/20170130_tp_bl6_sim_1750r_03'
#loop
for fld in lst:
try:
print fld
brain = os.path.join(src, brains[jobid])
start = time.time()
#accessing parameter dictionary
cellvol = "/jukebox/wang/pisano/tracing_output/bl6_ts/20150804_tp_bl6_ts04/full_sizedatafld/20150804_tp_bl6_ts04_488w_647_z3um_250msec_1hfds_ch01"
a2r0 = "/jukebox/scratch/zmd/save/contra_ipsi_projection_studies_20191125/20150804_tp_bl6_ts04/elastix_inverse_transform/TransformParameters.0.txt"
a2r1 = "/jukebox/scratch/zmd/save/contra_ipsi_projection_studies_20191125/20150804_tp_bl6_ts04/elastix_inverse_transform/TransformParameters.1.txt"
# r2s0 = "/jukebox/wang/pisano/tracing_output/bl6_ts/20150804_tp_bl6_ts04/elastix_inverse_transform/cellch_20150804_tp_bl6_ts04_555_z3um_70msec_3hfds/20150804_tp_bl6_ts04_555_z3um_70msec_3hfds_resized_ch00_resampledforelastix_atlas2reg2sig/reg2sig_TransformParameters.0.txt"
# r2s1 = "/jukebox/LightSheetTransfer/tp/20200701_12_55_28_20170207_db_bl6_crii_rpv_01/elastix_inverse_transform/TransformParameters.1.txt"
#set destination directory
braindst = os.path.join(scratch_dir, os.path.basename(brain))
makedir(braindst)
aldst = os.path.join(braindst, "transformed_annotations")
makedir(aldst)
#transformix
# transformfiles = modify_transform_files(transformfiles=[a2r0, a2r1, r2s0], dst = aldst)
transformfiles = modify_transform_files(transformfiles=[a2r0, a2r1],
dst=aldst)
[change_interpolation_order(xx, 0) for xx in transformfiles]
#change the parameter in the transform files that outputs 16bit images instead
for fl in transformfiles: # Read in the file
with open(fl, "r") as file:
filedata = file.read()
# Replace the target string
secondary = np.array([np.argsort(e)[-2] for e in expr_all_as_frac_of_inj])
#pooled injections
ak_pool = np.array(["Lob. I-III, IV-V", "Lob. VIa, VIb, VII", "Lob. VIII, IX, X",
"Simplex", "Crus I", "Crus II", "PM, CP"])
frac_of_inj_pool = np.array([[np.sum(xx[:4]),np.sum(xx[4:7]),np.sum(xx[7:10]), xx[10], xx[11], xx[12], np.sum(xx[13:16])]
for xx in expr_all_as_frac_of_inj])
primary_pool = np.array([np.argmax(e) for e in frac_of_inj_pool])
#get n's after pooling
primary_lob_n = np.array([len(np.where(primary_pool == i)[0]) for i in range(max(primary_pool)+1)])
#normalization of inj site
frac_of_inj_pool_norm = np.asarray([brain/brain.sum() for brain in frac_of_inj_pool])
#%%
lr_brains = list(lr_dist.keys())
atl_dst = os.path.join(dst, "pma_to_aba"); makedir(atl_dst)
id_table = pd.read_excel(df_pth)
#------------------------------------------------------------------------------------------------------------------------------
#NOTE THAT ONLY HAVE TO DO THIS ONCE!!!! DO NOT NEED TO DO AGAIN UNLESS DOUBLE CHECKIHG
#transform points to allen atlas space
#get brains that we actually need to get cell counts from
src = "/jukebox/wang/pisano/tracing_output/antero_4x_analysis/201903_antero_pooled_cell_counts_thalamus/transformed_points"
post_transformed = [os.path.join(src, os.path.join(xx, "transformed_points/posttransformed_zyx_voxels.npy")) for xx in lr_brains]
transformfiles = ["/jukebox/wang/zahra/aba_to_pma/TransformParameters.0.txt",
"/jukebox/wang/zahra/aba_to_pma/TransformParameters.1.txt"]
#collect
# for fl in post_transformed:
# arr = np.load(fl)
