def write_tiff(data, fname='tmp/data', digit=None, ext='tiff'):
"""
Write image data to a tiff file.
Overwrite existing data and infer data-type from the data.
Parameters
----------
data : ndarray
Array data to be saved.
fname : str
File name to which the data is saved. ``.tiff`` extension
will be appended if it doesn't already have one.
digit : int
Append this number to fname using a folder e.g. {fname}/{digit}.{ext}
"""
# Add the extension and digit.
if digit is not None:
fname = os.path.join(fname, str(digit))
if not str(fname).endswith(ext):
fname = ".".join([fname, ext])
# Convert to absolute path.
fname = os.path.abspath(fname)
# Create the directory if it doesn't exist.
dname = os.path.dirname(os.path.abspath(fname))
if not os.path.exists(dname):
os.makedirs(dname)
# Save the file.
tifffile.imsave(fname, data)
def mip(path,qtprocessbar=None, customSaveDir=None, normalize=False):
if debug is True: print clrmsg.DEBUG, "Creating normalized Maximum Intensity Projection (MIP):", path
img = tf.imread(path)
if qtprocessbar:
qtprocessbar.setValue(10)
QtGui.QApplication.processEvents()
fpath,fname = os.path.split(path)
if customSaveDir:
fname_mip = os.path.join(customSaveDir, "MIP_"+fname)
fname_mip_norm = os.path.join(customSaveDir, "MIP_norm_"+fname)
else:
fname_mip = os.path.join(fpath, "MIP_"+fname)
fname_mip_norm = os.path.join(fpath, "MIP_norm_"+fname)
if len(img.shape) == 4:
img = np.amax(img, axis=1)
if normalize:
if debug is True: print clrmsg.DEBUG, "Normalizing..."
img = norm_img(img)
if debug is True: print clrmsg.DEBUG, "Saving..."
tf.imsave(fname_mip_norm if normalize else fname_mip, img, imagej=True)
if debug is True: print clrmsg.DEBUG, " ...done"
elif len(img.shape) == 3:
img = np.amax(img, axis=0)
if normalize:
if debug is True: print clrmsg.DEBUG, "Normalizing..."
img = norm_img(img)
if debug is True: print clrmsg.DEBUG, "Saving..."
tf.imsave(fname_mip_norm if normalize else fname_mip, img)
if debug is True: print clrmsg.DEBUG, " ...done"
else: print clrmsg.ERROR, "I'm sorry, I don't know this image shape: {0}".format(img.shape)
def relabel_volume(dir, outdir):
files = sorted(os.listdir(dir))
out = None
out_is_there = False
for f in files:
i = tif.imread(os.path.join(dir,f))
if (out_is_there):
out = numpy.dstack([out, i])
else:
out = i
out_is_there = True
print '3d volume', out.shape
import skimage
from skimage.segmentation import relabel_sequential
relabeled,fm,im = skimage.segmentation.relabel_sequential(out)
print 'Max', relabeled.max()
for z in range(relabeled.shape[2]):
tif.imsave(os.path.join(outdir,str(z)+'.tif'),relabeled[:,:,z].astype(numpy.uint32))
print 'stored', z
def theoreticalOTF(self):
"""
Returns a theoretical OTF.
OTF = 2 * (psi - cos(psi)sin(psi)) / pi
psi = inv_cos(lambda * wavelength / 2 * NA)
Reference:
https://www.microscopyu.com/microscopy-basics/modulation-transfer-function
I'm assuming that the formula in this reference is for the FT of the PSF and
not the FT of the square root of the PSF.
"""
tmp = (1.5 * self.wavelength * self.k / (2.0 * self.NA))
tmp[(tmp > 1.0)] = 1.0
tmp[(tmp < -1.0)] = -1.0
psi = numpy.arccos(tmp)
otf = 2.0 * (psi - numpy.cos(psi)*numpy.sin(psi)) / numpy.pi
otf = otf/numpy.sum(otf)
if False:
tifffile.imsave("otf.tif", otf.astype(numpy.float32))
return otf
def start(self):
self.main.recWidget.writable = False
self.main.tree.writable = False
self.main.liveviewButton.setEnabled(False)
path = self.main.recWidget.folderEdit.text()
name = '3Dcalibration_step{}'.format(self.step)
savename = guitools.getUniqueName(os.path.join(path, name) + '.tiff')
steps = (self.rangeUm // self.step).magnitude
self.main.focusWidget.zMove(-0.5*steps*self.step)
self.main.focusWidget.zMove(self.step)
time.sleep(0.1)
stack = np.zeros((int(steps), self.main.shape[0], self.main.shape[1]),
dtype=np.uint16)
for s in np.arange(steps, dtype=int):
self.main.focusWidget.zMove(self.step)
time.sleep(0.1) # Waiting for the motor to get to new position
image = self.main.img.image.astype(np.uint16)
stack[s] = image
tiff.imsave(savename, stack, software='Tormenta',
photometric='minisblack',
resolution=(1/self.main.umxpx, 1/self.main.umxpx),
extratags=[('resolution_unit', 'H', 1, 3, True)])
self.main.focusWidget.zMove(-0.5*steps*self.step)
self.main.recWidget.writable = True
self.main.tree.writable = True
self.main.liveviewButton.setEnabled(True)
self.sigDone.emit()
def visualize_image_for_hand_labelling(file_id):
"""Saves out a visualization of the image ``file_id`` that's appropriate for hand-labelling"""
im = get_bounded_im(file_id)
im = two_channel_to_color(im)
target_path = os.path.join(LABELS_FOLDER, '{}_corrected.bmp'.format(file_id))
tifffile.imsave(target_path, im)
def writeData(filename, data):
"""Write image data to tif file
Arguments:
filename (str): file name
data (array): image data
Returns:
str: tif file name
"""
d = len(data.shape);
if d == 2:
#tiff.imsave(filename, data);
tiff.imsave(filename, data.transpose([1,0]));
elif d == 3:
#tiff.imsave(filename, data.transpose([2,0,1]));
tiff.imsave(filename, data.transpose([2,1,0]));
elif d == 4:
#tiffile (z,y,x,c)
t = tiff.TiffWriter(filename, bigtiff = True);
#t.save(data.transpose([2,0,1,3]), photometric = 'minisblack', planarconfig = 'contig');
t.save(data.transpose([2,1,0,3]), photometric = 'minisblack', planarconfig = 'contig')
t.close();
else:
raise RuntimeError('writing multiple channel data to tif not supported');
return filename;
def test_movie_of_cell(self):
fake_mdf_1 = pd.DataFrame({
"Object": [1],
"X": [100],
"Y": [100]
})
movie = movie_of_cell(
fake_mdf_1,
fixture("single_page.tif"),
200, 200)
self.assertIsInstance(movie, np.ndarray)
self.assertEqual(movie.shape, (1, 200, 200))
# test the pillow code path
movie = movie_of_cell(
fake_mdf_1,
[fixture("single_page.tif")],
200, 200)
self.assertIsInstance(movie, np.ndarray)
self.assertEqual(movie.shape, (1, 200, 200))
fake_mdf_4 = pd.DataFrame({
"Object": [1, 1, 1, 1],
"X": [100, 120, 140, 160],
"Y": [200, 180, 160, 140],
})
movie = movie_of_cell(
fake_mdf_4,
fixture("multi_page.tif"),
200, 200)
self.assertIsInstance(movie, np.ndarray)
self.assertEqual(movie.shape, (4, 200, 200))
if WRITE_OUTPUT:
tf.imsave("test_movie_of_cell.tif", movie, photometric="minisblack")
def saveTiffStack(fname, data, useLibTiff = False):
"""Save data in file fname
"""
if useLibTiff:
raise NotImplementedError
from tifffile import imsave
imsave(str(fname), data.swapaxes(1,2))
def test_unknown_axes_tags(self):
"""
This test is related to https://github.com/ilastik/ilastik/issues/1487
Here, we generate a 3D tiff file with scikit-learn and try to read it
"""
import tifffile
from distutils import version
# TODO(Dominik) remove version checking once tifffile dependency is fixed
# ilastik tiffile version >= 2000.0.0
# latest tifffile version is 0.13.0 right now
tifffile_version_ilastik_ref = version.StrictVersion("2000.0.0")
tifffile_version_ref = version.StrictVersion("0.7.0")
tifffile_version = version.StrictVersion(tifffile.__version__)
testshapes = [((10, 20), "yx"), ((10, 20, 30), "zyx"), ((10, 20, 30, 3), "zyxc"), ((5, 10, 20, 30, 3), "tzyxc")]
with tempdir() as d:
for test_shape, test_axes in testshapes:
data = numpy.random.randint(0, 256, test_shape).astype(numpy.uint8)
tiff_path = "{}/myfile_{}.tiff".format(d, test_axes)
# TODO(Dominik) remove version checking once dependencies for
# skimage are >= 0.13.0 for all flavours of ilastik
if (tifffile_version > tifffile_version_ilastik_ref) or (tifffile_version < tifffile_version_ref):
tifffile.imsave(tiff_path, data)
else:
tifffile.imsave(tiff_path, data, metadata={"axes": "QQQ"})
op = OpTiffReader(graph=Graph())
op.Filepath.setValue(tiff_path)
assert op.Output.ready()
assert_array_equal(data, op.Output[:].wait())
assert op.Output.meta.axistags == vigra.defaultAxistags(test_axes)
def measure_ufo(out_path, metric, axis, width, height):
pm = Ufo.PluginManager()
sched = Ufo.Scheduler()
graph = Ufo.TaskGraph()
input_path = 'data/measure.tif'
image = make_input(width, height)
tifffile.imsave(input_path, image)
reader = pm.get_task('read')
measure = pm.get_task('measure')
output = Ufo.OutputTask()
reader.props.path = input_path
measure.props.axis = axis
measure.props.metric = metric
graph.connect_nodes(reader, measure)
graph.connect_nodes(measure, output)
sched.run(graph)
buf = output.get_output_buffer()
gpu_result = ufo.numpy.asarray(buf)
write_image(out_path, gpu_result)
def roundtrip(self, dtype, x):
f = NamedTemporaryFile(suffix='.tif')
fname = f.name
f.close()
imsave(fname, x)
y = imread(fname)
assert_array_equal(x, y)
def main():
fpath = '2018-11-14/TL150_tune_0001.dax'
DM = DaxReader(global_datapath + fpath)
print(DM.image_height)
print(DM.image_width)
print(DM.number_frames)
stack = []
for cc in range(DM.number_frames):
frame = DM.loadAFrame(cc)
stack.append(frame.astype('float64'))
stack = np.array(stack)
zz, dom_z = psfstack_survey(stack)
tf.imsave('imstack.tif', stack.astype('uint16'))
fig = plt.figure()
ax = fig.add_subplot(111)
ax.imshow(stack[dom_z])
psf_collection, centers = psf_finder(stack, dom_z)
ax.scatter(centers[:,1], centers[:,0], s = 150, facecolors = 'none',edgecolors = 'yellow', linewidth = 2 )
ax.axis('off')
plt.tight_layout()
fig.savefig('PSF_extract')
print("# of psfs:",len(psf_collection))
ii = 0
for psf in psf_collection:
print(centers[ii])
ax.imshow(psf[:, 23, :], cmap = 'Greys_r', extent = [-0.088*24, 0.088*24, -3, 3])
ax.axis('off')
fig.savefig('psf_xz'+str(ii)+'tune1')
ii+=1
def remove_dark(A, folder):
"""
This function will subtract the dark files from the data files
Parameters
----------
A : list
list of tiff files
Returns
-------
clean_data : array
dark subtracted data , clean data
shape (number of clean images, detectore shape 0, detecotor shape 1)
"""
clean_data_arr = [] # save the cleaned data
for name in A:
if "dark" in name: # check the dark files
dark_data = imread(name)
print ("+++++ bad", name)
else:
arr = imread(name)
print ("good", name)
# clean the data
clean_data = arr - dark_data
#print (os.path.join(name))
imsave(name, clean_data)
clean_data_arr.append(clean_data)
return np.asarray(clean_data_arr)
def newImage(self, new_image):
fitting.PeakFinder.newImage(self, new_image)
#
# If does not already exist, create filter objects from
# the best fit spline at different z value of the PSF.
#
# As not all PSFs will be maximal in the center we can't just
# use the image intensity at the center as the starting
# height value. Instead we will use the intensity at the
# peak center of the convolved image, then adjust this
# value by the height_rescale parameter.
#
if (len(self.mfilter) == 0):
for mfilter_z in self.mfilter_z:
psf = self.s_to_psf.getPSF(mfilter_z,
shape = new_image.shape,
normalize = False)
psf_norm = psf/numpy.sum(psf)
self.height_rescale.append(1.0/numpy.sum(psf * psf_norm))
self.mfilter.append(matchedFilterC.MatchedFilter(psf_norm))
# Save a picture of the PSF for debugging purposes.
if False:
print("psf max", numpy.max(psf))
temp = 10000.0 * psf + 100.0
filename = "psf_{0:.3f}.tif".format(mfilter_z)
tifffile.imsave(filename, temp.astype(numpy.uint16))
self.taken = []
for i in range(len(self.mfilter)):
self.taken.append(numpy.zeros(new_image.shape, dtype=numpy.int32))
def saveImg(img,fn,enc="uint16",scale=True,maxVal=None): """Saves image as tif file. ``scale`` triggers the image to be scaled to either the maximum range of encoding or ``maxVal``. See also :py:func:`scaleToEnc`. Args: img (numpy.ndarray): Image to save. fn (str): Filename. Keyword Args: enc (str): Encoding of image. scale (bool): Scale image. maxVal (int): Maximum value to which image is scaled. Returns: str: Filename. """ #Fill nan pixels with 0 img=np.nan_to_num(img) #Scale img if scale: img=scaleToEnc(img,enc,maxVal=maxVal) tifffile.imsave(fn,img) return fn
def recolor_volume(dir, outdir, database_file):
conn = sqlite3.connect(database_file)
cursor = conn.cursor()
cursor.execute('SELECT * FROM relabelMap')
result = cursor.fetchall()
mergeTable = {}
for r in result:
mergeTable[r[0]] = r[1:]
print 'loaded colortable.'
# print mergeTable
files = os.listdir(dir)
for f in files:
if (f.startswith('.')):
continue
i = tif.imread(os.path.join(dir,f))
for oldid in mergeTable.keys():
i[i==oldid] = mergeTable[oldid]
tif.imsave(os.path.join(outdir,f), i)
def HSVizualizer(inputDirectory, filenames, outputDirectory):
"""
Args:
inputDirectory: (str)
filenames: (list) filenames to analyze and draw
outputDirectory: (str) Where to place output
Returns:
"""
for filename in filenames:
raw = tiff.imread(inputDirectory+filename)
nframes, frame_width, frame_height = raw.shape
#outstack is in RGB color so we need an extra 3 dimensions
outStack = np.zeros((nframes-1, frame_width, frame_height, 3), dtype='uint8')
for i in xrange(nframes-1):
t1 = time.time()
print "Start frame "+str(i)+"..."+filename
frame = raw[i]
next_frame = raw[i+1]
flow = cv2.calcOpticalFlowFarneback(frame, next_frame, None, 0.5, 3, 8, 4, 7, 1.5, 0)
outStack[i] = draw_hsv(flow)
print "Finish frame "+str(i)+" in "+str(time.time()-t1)+" s."
#print outStack.shape
tiff.imsave(outputDirectory+filename+'_HSV.tif', outStack)
print "All done in "+str(time.time()-t0)+" s"
def main(args):
tiff_fileno = args.filenumber
for arg in args.ge2:
if not arg.endswith('.ge2'):
if args.verbose:
print('Skipping %s, file format not recognized.' % (arg))
continue
if args.dezing:
raise NotImplementedError('dezinging has not been implemented')
continue
if args.summed:
out = "%s.tif" % (arg.rsplit('.ge2', 1)[0])
if os.path.exists(out):
if args.verbose:
print('Skipping %s, %s already exists' % (arg,out))
continue
if args.verbose:
print('Converting %s to %s'% (arg, out), end='\n')
tifffile.imsave(out, load_ge2(arg, summed=args.summed))
if args.delete:
os.remove(arg)
if args.verbose:
print('Removed %s' % arg)
else:
stem = args.output if args.output else arg.rsplit('.ge2', 1)[0]
for i, d in enumerate(AdeptFile(arg)):
out = '%s_%05d.tif' % (stem, tiff_fileno)
if args.verbose:
print("saving image %d from %s as %s" % (i+1, arg, out))
tifffile.imsave(out, d)
tiff_fileno += 1
def shift_stack_onfile(fpath, shift_coord, new_path = None, partial = False, srange = (0, 500), sub = False):
'''
Open a stack, take the shifted coordinates and correct the stack on file.
'''
if partial:
print("Only save part of the stack.")
with tf.TiffFile(fpath) as tif:
raw_img = tif.asarray()[srange[0]:srange[1]]
shift_coord = shift_coord[srange[0]:srange[1]]
tif.close()
else:
print("save the whole stack.")
with tf.TiffFile(fpath) as tif:
raw_img = tif.asarray()
n_slice, ny, nx = raw_img.shape
# subpixel correction interpolation needed
for ii in range(n_slice):
frame = raw_img[ii]
raw_img[ii] = interpolation.shift(frame, shift = shift_coord[ii], order = 0)
if (ii%100 == 0):
print("Finished ", ii, "slices.")
if new_path is None:
tf.imsave(fpath, raw_img.astype('uint16'))
else:
tf.imsave(new_path, raw_img.astype('uint16'))
def test_extract_window(self):
for i, (x, y) in enumerate(((0, 0), (100, 100), (250, 250))):
window = extract_window(self.single_page, x, y, 200, 200)
self.assertIsInstance(window, np.ndarray)
self.assertEqual(window.shape, (200, 200))
if WRITE_OUTPUT:
tf.imsave("test_extract_window_%d.tif" % i, window)
def tiffSave(parent,array,path,cellname,cellNumber):
array=np.swapaxes(array,0,1)
numints=len(str(array.shape[2]))
fullname=path+cellname
w=0
while w!=array.shape[2]:
currentslice="Exporting cell " + str(cellNumber) + " of " + str(int(np.sum(parent.verifiedcells))) + ": Exporting image " + str(w+1) + " of " + str(array.shape[2])
parent.progress.set(currentslice)
parent.frames[ExtractGUI2].update_idletasks()
intdiff=numints-len(str(w))
ints=0
intstr=""
while ints!=intdiff:
intstr=intstr+"0"
ints += 1
intstr=intstr+str(w)
tempfilename=fullname+"_slice"+intstr+".tiff"
tempfile=array[:,:,w].astype("uint8")
tifffile.imsave(tempfilename,tempfile)
w += 1
currentslice="Exporting cell " + str(cellNumber) + " of " + str(int(np.sum(parent.verifiedcells))) + ": Stacking TIFFs"
parent.progress.set(currentslice)
parent.frames[ExtractGUI2].update_idletasks()
mptiffstring="convert " + path + "*.tiff " + fullname +"_EM.tiff"
subprocess.call(mptiffstring,shell=True)
currentslice="Exporting cell " + str(cellNumber) + " of " + str(int(np.sum(parent.verifiedcells))) + ": Cleaning up TIFFs"
parent.progress.set(currentslice)
parent.frames[ExtractGUI2].update_idletasks()
tempfiles=glob.glob(fullname+"_slice*.tiff")
for f in tempfiles:
os.remove(f)
def initFindAndFit(parameters):
"""
Initialize and return a fitting.PeakFinderFitter object.
"""
# Create pupil function object.
[min_z, max_z] = parameters.getZRange()
pupil_fn = pupilFn.PupilFunction(pf_filename = parameters.getAttr("pupil_function"),
zmin = min_z * 1.0e+3,
zmax = max_z * 1.0e+3)
# PSF debugging.
if False:
tifffile.imsave("pupil_fn_psf.tif", pupil_fn.getPSF(0.1).astype(numpy.float32))
# Check that the PF and camera pixel sizes agree.
diff = abs(parameters.getAttr("pixel_size") - pupil_fn.getPixelSize()*1.0e3)
assert (diff < 1.0e-6), "Incorrect pupil function?"
# Create peak finder.
finder = fitting.PeakFinderArbitraryPSF(parameters = parameters,
psf_object = pupil_fn)
# Create cubicFitC.CSplineFit object.
mfitter = initFitter(finder, parameters, pupil_fn)
# Create peak fitter.
fitter = fitting.PeakFitterArbitraryPSF(mfitter = mfitter,
parameters = parameters)
# Specify which properties we want from the analysis.
properties = ["background", "error", "height", "iterations", "significance", "sum", "x", "y", "z"]
return fitting.PeakFinderFitter(peak_finder = finder,
peak_fitter = fitter,
properties = properties)
def mip():
files = tkFileDialog.askopenfilenames(parent=root,title='Choose image(stack) files')
if not files: return
for filename in files:
if filename.endswith('.tif'):
print "Creating normalized Maximum Intensity Projection (MIP):", filename
img = tf.imread(filename)
fpath,fname = os.path.split(filename)
fname_norm = os.path.join(fpath,"MIP_"+fname)
if len(img.shape) == 4:
img_MIP = np.zeros((img.shape[0],img.shape[2],img.shape[3]), dtype=img.dtype)
for i in range(0,img.shape[0]):
for ii in range(0,img.shape[2]):
for iii in range(0,img.shape[3]):
img_MIP[i,ii,iii] = img[i,:,ii,iii].max()
img_MIP = norm_img(img_MIP)
tf.imsave(fname_norm, img_MIP, imagej=True)
elif len(img.shape) == 3:
img_MIP = np.zeros((img.shape[1],img.shape[2]), dtype=img.dtype)
for i in range(0,img.shape[1]):
for ii in range(0,img.shape[2]):
img_MIP[i,ii] = img[:,i,ii].max()
img_MIP = norm_img(img_MIP)
tf.imsave(fname_norm, img_MIP)
else: print "I'm sorry, I don't know this image shape: {0}".format(img.shape)
print " ...done"
print "Maximum Intensity Projection finished."
print "="*40
def sima_export_frames(dataset, path, filenames, startIdx=0, stopIdx=None,
ftype="tiff"):
"""
Export a sima.ImagingDataset to individual tiffs.
Works around sima only producing multipage tiffs.
Parameters
----------
dataset : sima.ImagingDataset
The sima.ImagingDataset to be exported
path : string
Full path to target directory for exported tiffs
filenames : list of strings
Filenames to be used for the export. While these can be
full paths, only the file name part will be used.
startIdx : int, optional
Index of first frame to be exported (inclusive). Default: 0
stopIdx : int, optional
Index of last frame to be exported (exclusive). Default: None
ftype : stf, optional
file type, one of "tiff" or "raw". Default: "tiff"
"""
try:
assert(len(filenames) == dataset.sequences[0].shape[0])
except AssertionError as err:
print(len(filenames), dataset.sequences[0].shape[0])
raise err
assert(ftype in ["tiff", "raw"])
if ftype == "raw":
if startIdx != 0 or stopIdx is not None:
raise RuntimeError(
"Can only export complete dataset in raw format")
if not os.path.exists(path):
os.makedirs(path)
if stopIdx is None or stopIdx > len(filenames):
stopIdx = len(filenames)
save_frames = sima.sequence._fill_gaps(
iter(dataset.sequences[0]), iter(dataset.sequences[0]))
if ftype == "tiff":
for nf, frame in enumerate(save_frames):
if nf >= startIdx and nf < stopIdx:
tifffile.imsave(
os.path.join(path, os.path.basename(filenames[nf])),
np.array(frame[0]).squeeze().astype(
np.uint16))
elif ftype == "raw":
sys.stdout.write("Reading files...")
sys.stdout.flush()
t0 = time.time()
arr = np.array(
[frame for frame in save_frames]).squeeze().astype(
np.uint16)
sys.stdout.write(" done in {0:.2f}s\n".format(time.time()-t0))
compress_np(
arr, path, THOR_RAW_FN, dataset.sequences[0].shape,
compress=True)
def single_tiff_setup(n_images, fmt='foo-{:05}.tif'):
with tempdir() as d:
data = np.ones((512, 512), dtype=np.float32)
for i in range(n_images):
tifffile.imsave(d.path(fmt.format(i)), data)
yield d
def export_tif(self, filepath, filename, export_dtype):
frames = file_io.load_file(filepath)
frames = frames.astype(export_dtype)
try:
tiff.imsave(filename, frames)
except:
qtutil.critical(filename + ' could not be written. Ensure that export dtype is properly set above. '
'This fixes this issue in most cases.', self)
def write_tiff(file_name, data):
"""
The default TIFF writer which uses :py:mod:`tifffile` module.
Return the written file name.
"""
tifffile.imsave(file_name, data)
return file_name
def _savePicture(self,data,filename):
"""
Saves a picture in png 16 bits, greyscale.
"""
print 'shape of picture : ',data.shape
tiff.imsave(filename,data)
return
def save(self,file_name):
'''
Save the timeseries in various formats
parameters
----------
file_name: name of file. Possible formats are tif, avi, npz and hdf5
'''
name,extension = os.path.splitext(file_name)[:2]
print extension
if extension == '.tif': # load avi file
# raise Exception('not implemented')
np.clip(self,np.percentile(self,1),np.percentile(self,99.99999),self)
minn,maxx = np.min(self),np.max(self)
data = 65536 * (self-minn)/(maxx-minn)
data = data.astype(np.int32)
imsave(file_name, self.astype(np.float32))
elif extension == '.npz':
np.savez(file_name,input_arr=self, start_time=self.start_time,fr=self.fr,meta_data=self.meta_data,file_name=self.file_name)
elif extension == '.avi':
codec=cv2.cv.FOURCC('I','Y','U','V')
np.clip(self,np.percentile(self,1),np.percentile(self,99),self)
minn,maxx = np.min(self),np.max(self)
data = 255 * (self-minn)/(maxx-minn)
data = data.astype(np.uint8)
y,x = data[0].shape
vw = cv2.VideoWriter(file_name, codec, self.fr, (x,y), isColor=True)
for d in data:
vw.write(cv2.cvtColor(d, cv2.COLOR_GRAY2BGR))
vw.release()
elif extension == '.mat':
if self.meta_data[0] is None:
savemat(file_name,{'input_arr':np.rollaxis(self,axis=0,start=3), 'start_time':self.start_time,'fr':self.fr,'meta_data':[],'file_name':self.file_name})
else:
savemat(file_name,{'input_arr':np.rollaxis(self,axis=0,start=3), 'start_time':self.start_time,'fr':self.fr,'meta_data':self.meta_data,'file_name':self.file_name})
elif extension == '.hdf5':
with h5py.File(file_name, "w") as f:
dset=f.create_dataset("mov",data=np.asarray(self))
dset.attrs["fr"]=self.fr
dset.attrs["start_time"]=self.start_time
dset.attrs["file_name"]=self.file_name
dset.attrs["meta_data"]=cpk.dumps(self.meta_data)
else:
print extension
raise Exception('Extension Unknown')
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("-i", "--input", help="path to an image", type=str)
parser.add_argument("-l",
"--label_input",
help="path to an image",
type=str)
parser.add_argument("-o",
"--output",
help="path to save an output",
type=str)
parser.add_argument("-f", "--function", help="function to use", type=str)
parser.add_argument("param",
nargs="*",
help="input argument file path",
type=lambda kv: kv.split("="))
args = parser.parse_args()
if args.output is None:
args.output = 'temp.tif'
img = imread(args.input)
labels = imread(args.label_input).astype(np.uint16)
func = globals()[args.function]
param = dict(args.param)
for key, value in param.iteritems():
param[key] = ast.literal_eval(value)
seg = func(img, label=labels, holder=None, **param).astype(np.float32)
tiff.imsave(args.output, seg)
"imagescale": 2,
"crop": "[:,450:,:]",
"dst": "/jukebox/wang/zahra/tracing_projects/mapping_paper/revision_images/short_timepoint_counts/short_time_series_inj",
"save_individual": True,
"save_tif": True,
"colormap": "plasma",
"atlas": "//LightSheetTransfer/atlas/annotation_sagittal_atlas_20um_iso_16bit.tif",
"annotation": "/jukebox/LightSheetTransfer/atlas/annotation_sagittal_atlas_20um_iso_16bit.tif",
"id_table": "/jukebox/LightSheetTransfer/atlas/ls_id_table_w_voxelcounts_16bit.xlsx"
}
#segment injection site
for i, vol in enumerate(inputlist):
arr = find_site(vol, dct["threshold"], dct["filter_kernel"])
plt.imshow(np.max(arr,axis=0), cmap="Greys")
tifffile.imsave(os.path.join(dct["dst"], brains[i]+".tif"), arr.astype("uint16")*65535) #save to inj site destination
#transform atlas to segmentation space
#this is because i did not run the reg --> atlas transform
#btw this will take long, and i don't recommend parallelizing bc of transformix
for i, vol in enumerate(inputlist):
invol = dct["annotation"]
#run transformix
outpth = os.path.join(dct["dst"], brains[i]); makedir(outpth)
outpth = run_transformix(invol, outpth, transformfiles[i])
#save out
tifffile.imsave(os.path.join(dct["dst"],brains[i]+"_annotation.tif"),
img.astype("uint16"))
shutil.rmtree(outpth) #delete original transformed file
def save_image(img, extension):
if extension == 'npz':
np.savez('res.' + extension, img)
else:
imsave('res.' + extension, img)
temp = predict(np.rot90(img, 3),
model,
patch_sz=PATCH_SZ,
n_classes=N_CLASSES)
#print(temp.transpose([2,0,1])[0][0][0], temp.transpose([2,0,1])[3][12][13])
# print("Case 6", temp.shape, mymat.shape)
mymat = np.mean(np.array(
[np.rot90(temp, -3).transpose(2, 0, 1), mymat]),
axis=0)
else:
temp = predict(img,
model,
patch_sz=PATCH_SZ,
n_classes=N_CLASSES).transpose([2, 0, 1])
#print(temp[0][0][0], temp[3][12][13])
# print("Case 7", temp.shape, mymat.shape)
mymat = np.mean(np.array([temp, mymat]), axis=0)
#print(mymat[0][0][0], mymat[3][12][13])
map = picture_from_mask(mymat, 0.5)
#mask = predict(img, model, patch_sz=PATCH_SZ, n_classes=N_CLASSES).transpose([2,0,1]) # make channels first
#map = picture_from_mask(mask, 0.5)
#tiff.imsave('result.tif', (255*mask).astype('uint8'))
tiff.imsave('data/restest{}.tif'.format(test_id),
(255 * mymat).astype('uint8'))
tiff.imsave('data/maptest{}.tif'.format(test_id), map)
# print("kappa",kappa(cv2.imread('data/testout/maptest{}.tif'.format(test_id)),cv2.imread('data/testgt.tif')))
# print("f1Score",f1Score(cv2.imread('data/testout/maptest{}.tif'.format(test_id)),cv2.imread('data/testgt.tif')))
imgs_downsized.sort()
z = len(imgs_downsized)
y, x = tifffile.imread(imgs_downsized[0]).shape
arr = np.zeros((z, y, x))
atl = tifffile.imread(atlpth)
atlz, atly, atlx = atl.shape #get shape, sagittal
#read all the downsized images into a single volume
print("reading planes into a single volume")
sys.stdout.flush()
for i, img in enumerate(imgs_downsized):
if i % 500 == 0:
print(f"Plane {i+1}/{len(imgs_downsized)}")
try:
arr[i, :, :] = tifffile.imread(img) #horizontal
except:
print(img)
#switch to sagittal
arrsag = np.swapaxes(arr, 2, 0)
z, y, x = arrsag.shape
print((z, y, x))
print("\n**********downsizing....heavy!**********\n")
sys.stdout.flush()
arrsagd = zoom(arrsag,
((atlz * 1.4 / z), (atly * 1.4 / y), (atlx * 1.4 / x)),
order=1)
tifffile.imsave(
os.path.join(dst_dir_downsized,
f"downsized_for_atlas_ch{channel}.tif"),
arrsagd.astype("uint16"))
def execute(self):
##########################################################################
if args.mode == 'per_img':
fname = os.path.basename(os.path.splitext(args.input_fname)[0])
print(fname)
reader = io.cziReader.CziReader(args.input_fname)
img = reader.load()
img = np.squeeze(img, axis=0)
img = img.astype(np.float32)
struct_img = img[:, args.structure_ch, :, :]
if args.structure_type == 1: # ACTB
from python_image_analysis.seg_actb import ACTB as SegModule
elif args.structure_type == 2: # DSP
from python_image_analysis.seg_dsp import DSP as SegModule
elif args.structure_type == 3: # golgi
from python_image_analysis.seg_golgi import ST6GAL1 as SegModule
elif args.structure_type == 4: # lamp1
from python_image_analysis.seg_lamp1 import LAMP1 as SegModule
elif args.structure_type == 5: # myosin
from python_image_analysis.seg_myosin import MYH10 as SegModule
elif args.structure_type == 6: # sec61b
from python_image_analysis.seg_sec61 import SEC61B as SegModule
elif args.structure_type == 7: #tubulin
from python_image_analysis.seg_tubulin import TUBA1B as SegModule
elif args.structure_type == 8: # ZO1
from python_image_analysis.seg_zo1 import ZO1 as SegModule
else:
print('unsupported structure type')
quit()
bw = SegModule(struct_img, args.drug_type)
bw = bw.astype(np.uint8)
bw[bw > 0] = 255
nm = args.output_dir + 'seg_' + str(
args.structure_type) + '_' + str(
args.drug_type) + '_' + fname + '.tiff'
imsave(nm, bw)
elif args.mode == 'per_csv':
from python_image_analysis.seg_actb import ACTB
from python_image_analysis.seg_dsp import DSP
from python_image_analysis.seg_golgi import ST6GAL1
from python_image_analysis.seg_lamp1 import LAMP1
from python_image_analysis.seg_myosin import MYH10
from python_image_analysis.seg_sec61 import SEC61B
from python_image_analysis.seg_tubulin import TUBA1B
from python_image_analysis.seg_zo1 import ZO1
import pandas as pd
df = pd.read_csv(args.input_dir)
for idx in range(len(df)):
filename = df['link'].iloc[idx]
fname = os.path.basename(os.path.splitext(filename)[0])
reader = io.cziReader.CziReader(filename)
img = reader.load()
img = np.squeeze(img, axis=0)
img = img.astype(np.float32)
#TODO: another column in csv for strucutre channel index
struct_img = img[:, 3, :, :]
structure_name = df['structure'].iloc[idx]
if caseless_equal(df['well'].iloc[idx], 'Vehicle'):
drug_type = 0
elif caseless_equal(df['drug'].iloc[idx], 'Brefeldin'):
drug_type = 1
elif caseless_equal(df['drug'].iloc[idx], 'Paclitaxol'):
drug_type = 2
elif caseless_equal(df['drug'].iloc[idx], 'Staurosporine'):
drug_type = 3
elif caseless_equal(df['drug'].iloc[idx],
's-Nitro-Blebbistatin'):
drug_type = 4
elif caseless_equal(df['drug'].iloc[idx], 'Rapamycin'):
drug_type = 5
else:
print('error in understanding drug type')
print(df['drug'].iloc[idx])
quit()
if structure_name == 'beta-actin': # ACTB
bw = ACTB(struct_img, drug_type)
elif structure_name == 'desmoplakin': # DSP
bw = DSP(struct_img, drug_type)
elif structure_name == 'golgi': # golgi
bw = ST6GAL1(struct_img, drug_type)
elif structure_name == 'lamp1': # lamp1
bw = LAMP1(struct_img, drug_type)
elif structure_name == 'myosin': # myosin
bw = MYH10(struct_img, drug_type)
elif structure_name == 'sec61b': # sec61b
bw = SEC61B(struct_img, drug_type)
elif structure_name == 'tubulin': #tubulin
bw = TUBA1B(struct_img, drug_type)
elif structure_name == 'zo1': # ZO1
bw = ZO1(struct_img, drug_type)
else:
print('error in understanding structure name')
print(df['structure'].iloc[idx])
quit()
bw = bw.astype(np.uint8)
bw[bw > 0] = 255
if drug_type == 0:
nm = args.output_dir + 'seg_' + structure_name + '_Vehicle_' + fname + '.tiff'
else:
nm = args.output_dir + 'seg_' + structure_name + '_' + df[
'drug'].iloc[idx] + '_' + fname + '.tiff'
imsave(nm, bw)
def gen_model_eval_data(params: Dict[str, Union[List[float], np.ndarray]],
shape: int,
use_noise: bool,
use_psf: bool,
use_subsampling: bool,
samples_per_param: int = 18,
dists=DEFAULT_DIST):
"""
Analyze how the performance of a CARE model is affected by different parameters of degradation and image content
:param model: The model to evaluate
:param params: The parameters to test as a dictionary. The keys must be parameters to test and the values must
be values to test
:param shape: The size of the test images generated to evaluate the performance of the model
:param use_noise: Whether to use add noise to generated images
:param use_psf: Whether to convolve the generated images with some PSF
:param use_subsampling: Whether to downsample the generate images
:return: A dictionary containing the results
"""
from tqdm.auto import tqdm
import warnings
import tifffile
import os
import shutil
import time
import matplotlib.pyplot as plt
results = {}
if os.path.exists('eval_data'):
shutil.rmtree('eval_data')
os.mkdir('eval_data')
time.sleep(5)
with warnings.catch_warnings():
warnings.filterwarnings('ignore', module='scipy')
warnings.filterwarnings('ignore', module='scikits.fitting')
for param in tqdm(params, desc='Param'):
os.mkdir(f'eval_data/{param}')
Y, X, psfas, psfbs = _gen_param_influence_data(
param,
params[param],
shape,
use_noise,
use_psf,
use_subsampling,
samples_per_param=samples_per_param,
dists=dists)
# print(param, params[param], shape, use_noise, use_psf,
# use_subsampling, samples_per_param, dists)
# plt.imshow(np.sum(Y[0][0][:, :, :], axis=0))
# plt.show()
# plt.imshow(np.sum(X[0][0][:, :, :, 0], axis=0))
# plt.show()
# plt.hist(X[0][0].ravel(), bins=50)
# plt.show()
# print(X[0][0].shape, X[0][0].dtype, X[0][0].min(), X[0][0].max(), np.any(np.isnan(X[0][0])), X[0][0].flags)
for v_i in range(len(X)):
os.mkdir(f'eval_data/{param}/{params[param][v_i]}')
os.mkdir(f'eval_data/{param}/{params[param][v_i]}/Y')
os.mkdir(f'eval_data/{param}/{params[param][v_i]}/X')
os.mkdir(f'eval_data/{param}/{params[param][v_i]}/psfas')
os.mkdir(f'eval_data/{param}/{params[param][v_i]}/psfbs')
# os.mkdir(f'eval_data/{param}/{params[param][v_i]}/X_psf')
for s in range(len(X[v_i])):
tifffile.imsave(
f'eval_data/{param}/{params[param][v_i]}/Y/{s}.tif',
Y[v_i][s])
tifffile.imsave(
f'eval_data/{param}/{params[param][v_i]}/X/{s}.tif',
X[v_i][s])
tifffile.imsave(
f'eval_data/{param}/{params[param][v_i]}/psfas/{s}.tif',
psfas[v_i][s])
tifffile.imsave(
f'eval_data/{param}/{params[param][v_i]}/psfbs/{s}.tif',
psfbs[v_i][s])
def run(self):
tifffile.imsave(self.filename, self.tiff)
self.done.emit()
self.terminate()
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Jun 29 15:08:27 2019
@author: wanglab
"""
import numpy as np, os, tifffile
src = "/home/wanglab/mounts/wang/pisano/tracing_output/retro_4x/20180313_jg_bl6f_prv_23/full_sizedatafld/20180313_jg23_4x_647_008na_1hfds_z7d5um_300msec_10povlp_ch00"
#trying to get hypothal
plns = list(range(620, 670))
zplns = [[
os.path.join(src, xx) for xx in os.listdir(src) if "Z0{}".format(pln) in xx
][0] for pln in plns]
zplns.sort()
assert len(zplns) == 50
y, x = tifffile.imread(zplns[0]).shape
arr = np.zeros((len(zplns), y, x))
for i, zpln in enumerate(zplns):
arr[i] = tifffile.imread(zpln)
tifffile.imsave("/home/wanglab/Desktop/prv_23_z620-670_hypothal.tif", arr)
# generation depth image, without interpolation
img_depth = np.zeros((img_height, img_width, 1),
dtype=np.float32) # 32 bit
for i in tqdm(range(0, px.shape[1])):
if img_width > px[0, i] > 0 and img_height > py[0, i] > 0:
depth = math.sqrt((Xc - pt_xyz[i, 0])**2 + (Yc - pt_xyz[i, 1])**2 +
(Zc - pt_xyz[i, 2])**2)
if img_depth[int(py[0, i]), int(px[0, i])] == 0:
img_depth[int(py[0, i]), int(px[0, i])] = depth
elif img_depth[int(py[0, i]), int(px[0, i])] != 0 and \
depth <= img_depth[int(py[0, i]), int(px[0, i])]: # only save the closest distance in that pixel
img_depth[int(py[0, i]), int(px[0, i])] = depth
# set "nan" to pixel where no point projected
# mask = (img_depth == 0)
# img_depth[mask[:, :] == True] = np.nan
#
# where_are_NaNs = np.isnan(img_depth)
# img_depth[where_are_NaNs] = 0.0
kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (15, 15))
closing = cv2.morphologyEx(img_depth, cv2.MORPH_CLOSE, kernel)
tifffile.imsave(os.path.join(save_path, img_name), closing)
# test = tifffile.imread(os.path.join(save_path, img_name+'_dist.tif'))
def imsave(filename, arr):
ext = os.path.splitext(filename)[-1]
if ext== '.tif' or ext=='tiff':
tifffile.imsave(filename, arr)
else:
cv2.imwrite(filename, arr)
elif i == 3:
#was previously rotating by 90 deg. This and 270 deg rotation does not work with
#rectangular images like ours.
#180 works fine bc its a flip
#circle back and add augmentation after run completes
temp = predict(img, model, patch_sz=PATCH_SZ, n_classes=N_CLASSES)
print("Case 4", temp.shape, mymat.shape)
mymat = np.mean(np.array([temp, mymat]), axis=0)
elif i == 4:
temp = predict(np.rot90(img, 2),
model,
patch_sz=PATCH_SZ,
n_classes=N_CLASSES)
print("Case 5", temp.shape, mymat.shape)
mymat = np.mean(np.array([np.rot90(temp, -2), mymat]), axis=0)
elif i == 5:
#was a 270 degree rotation, which doesn't work with rectangular image
#removing for now, will replace
temp = predict(img, model, patch_sz=PATCH_SZ, n_classes=N_CLASSES)
print("Case 6", temp.shape, mymat.shape)
mymat = np.mean(np.array([temp, mymat]), axis=0)
else:
temp = predict(img, model, patch_sz=PATCH_SZ, n_classes=N_CLASSES)
print("Case 7", temp.shape, mymat.shape)
mymat = np.mean(np.array([temp, mymat]), axis=0)
#create classified map
map = picture_from_mask(mymat, 0.5)
tiff.imsave('output/result.tif', (255 * mymat).astype('uint8'))
tiff.imsave('output/map.tif', map)
# print(index, path, file)
start = 0
index = re.findall('\d+(?=of)', path)
# index = re.findall('(?<=shift_)\d*',path)
index = int(index[0])
dwell = re.findall('(?<=x)\d*', file)
dwell = int(dwell[0])
num_patterns = dwell
skip = 0
AverageStack = AS.creatAverageStack(path, file, num_patterns, 1)
averagedpattern = AverageStack.serStack(start, dwell, skip)
averagedstack[shifts - index, :, :] = averagedpattern
averagedstack = averagedstack.astype('float32')
imsave(folder + '/averagedStack.tif', averagedstack, shape=averagedstack.shape)
# %% export one shift raw to tiff
path = '/Volumes/Seagate Backup Plus Drive/TQD_20180702_Si/Si110_CL4o1_ser9999_area4'
file = 'series_93_x9999.raw'
num_patterns = 9999
AverageStack = AS.creatAverageStack(path, file, num_patterns, 1)
averagedpattern = AverageStack.serStack(0, 9999, 0)
averagedpattern = averagedpattern[0]
plt.imshow(np.log(averagedpattern))
savepath = '/Volumes/Seagate Backup Plus Drive/TQD_20180702_Si/Si110_CL4o1_ser9999_area4/tifffiles'
imsave(savepath + '/averagedStack.tif',
averagedpattern,
shape=averagedpattern.shape)
# %% regular autoshift: export patterns, head and end for estimate dwell
import argparse
parser = argparse.ArgumentParser(
description='''Filter on class 0 and formalizing class 1''')
parser.add_argument('-c0', '--class0', help='Image of class 0', required=True)
parser.add_argument('-c1', '--class1', help='Image of class 1', required=True)
args = parser.parse_args()
update_progress(0)
# No mowing
# Select only meadows 13, grasslands 18 and woody moorlands 19
# Format each not mowed parcels (pixels) to 0 and everywhere else to white 255
GRT0 = imread(args.class0)
GRT0[np.logical_and(GRT0 != 13, GRT0 != 18, GRT0 != 19)] = 255
GRT0[np.logical_or(GRT0 == 13, GRT0 == 18, GRT0 == 19)] = 0
# Save the image class 0
imsave(args.class0, GRT0.astype(np.uint8))
# Mowing
# Format each mowed parcels (pixels) to 1 and everywhere else to white 255
GRT1 = imread(args.class1)
GRT1[GRT1 > 0] = 1
GRT1[GRT1 == 0] = 255
# Save the image class 1
imsave(args.class1, GRT1.astype(np.uint8))
update_progress(1)
def run(self, mojodir, outdir):
print 'Convert MOJO dir', mojodir
self.__mojo_dir = mojodir
# load largest image, combine to tile, store as tif
data_path = self.__mojo_dir + '/ids/w=00000000/'
for z in os.listdir(data_path):
data_path2 = os.path.join(data_path, z)
print data_path2
images = os.listdir(data_path2)
tile = {}
for i in images:
location = os.path.splitext(i)[0].split(',')
for l in location:
l = l.split('=')
exec(l[0] + '=int("' + l[1] + '")')
if not x in tile:
tile[x] = {}
# tile[x][y] = tif.imread(os.path.join(data_path,i))
hdf5_file = h5py.File(os.path.join(data_path2, i))
list_of_names = []
hdf5_file.visit(list_of_names.append)
tile[x][y] = hdf5_file[list_of_names[0]].value
hdf5_file.close()
row = None
first_row = True
# go through rows of each tile
for r in tile.keys():
column = None
first_column = True
print len(tile[r])
for c in tile[r]:
if first_column:
column = tile[r][c]
first_column = False
else:
column = np.concatenate((column, tile[r][c]), axis=0)
if first_row:
row = column
first_row = False
else:
row = np.concatenate((row, column), axis=1)
tile = row
outfile = os.path.join(outdir, z + '.tif')
tif.imsave(outfile, tile)
print 'stored', outfile
resolution
), # rp1[ii].minor_axis_length, rp1[ii].major_axis_length,
rp1[ii].solidity))
# Plot centroids
ratio_plot = np.zeros(lab_img.shape)
centroids = np.array([np.array(rr.centroid) for rr in rp1])
centroids = centroids.astype(np.uint16)
ratio_plot[centroids[:, 0], centroids[:, 1], centroids[:, 2]] = ratios
# tiff.imshow(dilate(np.max(ratio_plot, axis=0), size=5).astype(np.float32))
centroid_dir = os.path.join(outdir, 'centroids')
mkdir(centroid_dir)
oname = os.path.join(centroid_dir,
os.path.basename(fname))[:-4] + '_centroids.tif'
tiff.imsave(oname,
dilate(np.max(ratio_plot, axis=0), size=5).astype(np.float32))
# Plot full nuclei
ratio_plot = np.zeros(lab_img.shape)
for ii, reg in enumerate(rp1):
ratio_plot[lab_img == reg.label] = ratios[ii]
full_dir = os.path.join(outdir, 'full')
mkdir(full_dir)
oname = os.path.join(full_dir, os.path.basename(fname))[:-4] + '_full.tif'
fp.tiffsave(oname, ratio_plot.astype(np.float32), resolution=resolution)
fp.tiffsave(oname[:-4] + '_maxproj.tif',
np.array([np.max(ratio_plot.astype(np.float32), axis=0)]),
resolution=resolution)
################################################################################
end.record()
torch.cuda.synchronize()
end_time = start.elapsed_time(end)
mean_time += end_time
# Use only first sparse image, corresponding to the selected volume.
if net_get_params(net).n_frames>1:
curr_img_sparse = curr_img_sparse[:,0,...].unsqueeze(1)
if args.writeToVideo>0:
video_logs['input_frames'][0,ix,...] = curr_img_stack[0,0,...]
video_logs['sparse_reconstruction'][0,ix,...] = curr_img_sparse[0,0,...]
video_logs['dense_reconstruction'][0,ix,...] = torch.nn.functional.relu(curr_img_sparse[0,0,...]-curr_img_sparse[0,0,...])
video_logs['MIP_sparse_3D_reconstruction'][0,ix,...] = volume_2_projections(prediction.permute(0,2,3,1).unsqueeze(1))[0,0,...]
if args.writeVolsToStack>0:
imsave(save_folder + '/XLFM_stack_S/XLFM_stack_'+ "%03d" % ix + '.tif', prediction[0,...].cpu().numpy())
# plt.clf()
# plt.subplot(1,3,1)
# plt.imshow(curr_img_stack[0,0,...].float().cpu().detach().numpy())
# plt.subplot(1,3,2)
# plt.imshow(curr_img_sparse[0,0,...].float().cpu().detach().numpy())
# plt.subplot(1,3,3)
# plt.imshow(volume_2_projections(prediction.permute(0,2,3,1).unsqueeze(1))[0,0,...].float().detach().cpu().numpy())
# plt.title('Time: '+str(int(end_time)) + ' ms')
# plt.show()
# Extract lenslet images
curr_img_sparse = dataset.extract_views(curr_img_sparse, dataset.lenslet_coords, dataset.subimage_shape)[:,0,...]
if args.write_to_tb:
if local_volumes.shape == prediction.shape:
def export_TF(self, name, description, authors, test_img, axes, patch_shape, fname=None):
"""
name: String
Name of the model.
description: String
A short description of the model e.g. on what data it was trained.
authors: String
Comma seperated list of author names.
patch_shape: The shape of the patches used in model.train().
"""
if fname is None:
fname = self.logdir / 'export.bioimage.io.zip'
else:
fname = Path(fname)
input_n_dims = len(test_img.shape)
if 'C' in axes:
input_n_dims -=1
assert input_n_dims == self.config.n_dim, 'Input and network dimensions do not match.'
assert test_img.shape[axes.index('X')] == test_img.shape[axes.index('Y')], 'X and Y dimensions are not of same length.'
test_output = self.predict(test_img, axes)
# Extract central slice of Z-Stack
if 'Z' in axes:
z_dim = axes.index('Z')
if z_dim != 0:
test_output = np.moveaxis(test_output, z_dim, 0)
test_output = test_output[int(test_output.shape[0]/2)]
# CSBDeep Export
meta = {
'type': self.__class__.__name__,
'version': package_version,
'probabilistic': self.config.probabilistic,
'axes': self.config.axes,
'axes_div_by': self._axes_div_by(self.config.axes),
'tile_overlap': self._axes_tile_overlap(self.config.axes),
}
export_SavedModel(self.keras_model, str(fname), meta=meta)
# CSBDeep Export Done
# Replace : with -
name = name.replace(':', ' -')
yml_dict = self.get_yml_dict(name, description, authors, test_img, axes, patch_shape=patch_shape)
yml_file = self.logdir / 'model.yaml'
'''default_flow_style must be set to TRUE in order for the output to display arrays as [x,y,z]'''
yaml = YAML(typ='rt')
yaml.default_flow_style = False
with open(yml_file, 'w') as outfile:
yaml.dump(yml_dict, outfile)
input_file = self.logdir / 'testinput.tif'
output_file = self.logdir / 'testoutput.tif'
imsave(input_file, test_img)
imsave(output_file, test_output)
with ZipFile(fname, 'a') as myzip:
myzip.write(yml_file, arcname=os.path.basename(yml_file))
myzip.write(input_file, arcname=os.path.basename(input_file))
myzip.write(output_file, arcname=os.path.basename(output_file))
print("\nModel exported in BioImage ModelZoo format:\n%s" % str(fname.resolve()))
def predict_folds(config, loader, fold_idcs):
"""Evaluate the model versus each fold
"""
for fold_idx in fold_idcs:
paths_weights_fold = dict()
paths_weights_fold['segm'] = \
os.path.join(config['path_weights'], 'segm', f'fold_{fold_idx}')
handlers_ckpt = dict()
handlers_ckpt['segm'] = CheckpointHandler(paths_weights_fold['segm'])
paths_ckpt_sel = dict()
paths_ckpt_sel['segm'] = handlers_ckpt['segm'].get_last_ckpt()
# Initialize and configure the model
model = (dict_models[config['model_segm']]
(input_channels=config['input_channels'],
output_channels=config['output_channels'],
center_depth=config['center_depth'],
pretrained=config['pretrained'],
restore_weights=config['restore_weights'],
path_weights=paths_ckpt_sel['segm']))
model = nn.DataParallel(model).to(maybe_gpu)
model.eval()
with tqdm(total=len(loader), desc=f'Eval, fold {fold_idx}') as prog_bar:
for i, data_batch in enumerate(loader):
xs, ys_true = data_batch['xs'], data_batch['ys']
xs, ys_true = xs.to(maybe_gpu), ys_true.to(maybe_gpu)
if config['model_segm'] == 'unet_lext':
ys_pred = model(xs)
elif config['model_segm'] == 'unet_lext_aux':
ys_pred, _ = model(xs)
else:
msg = f"Unknown model {config['model_segm']}"
raise ValueError(msg)
ys_pred_softmax = nn.Softmax(dim=1)(ys_pred)
ys_pred_softmax_np = ys_pred_softmax.detach().to('cpu').numpy()
data_batch['pred_softmax'] = ys_pred_softmax_np
# Rearrange the batch
data_dicts = [{k: v[n] for k, v in data_batch.items()}
for n in range(len(data_batch['image']))]
for k, data_dict in enumerate(data_dicts):
dir_base = os.path.join(
config['path_predicts'],
data_dict['patient'], data_dict['release'], data_dict['sequence'])
fname_base = os.path.splitext(
os.path.basename(data_dict['path_rel_image']))[0]
# Save the predictions
dir_predicts = os.path.join(dir_base, 'mask_folds')
if not os.path.exists(dir_predicts):
os.makedirs(dir_predicts)
fname_full = os.path.join(
dir_predicts,
f'{fname_base}_fold_{fold_idx}.tiff')
tmp = (data_dict['pred_softmax'] * 255).astype(np.uint8, casting='unsafe')
tifffile.imsave(fname_full, tmp, compress=9)
prog_bar.update(1)
# test and export images #
#############################
if (i % 10 == 0):
pred = model.predict(conf_raw)
if (np.shape(pred_old)[0] > 1):
if (i > 20):
if (np.array_equal(pred_old, pred)):
print("CONVERGED TO BAD RESULT")
break
for j in range(0, 16):
tifffile.imsave(
"tiffs/pred0/0predicted_affins%i" % j,
np.asarray(pred, dtype=np.float32)[0, j, :, :, 0])
tifffile.imsave(
"tiffs/pred1/1predicted_affins%i" % j,
np.asarray(pred, dtype=np.float32)[0, j, :, :, 1])
tifffile.imsave(
"tiffs/pred2/2predicted_affins%i" % j,
np.asarray(pred, dtype=np.float32)[0, j, :, :, 2])
tifffile.imsave(
"tiffs/act0/0actual_affins%i" % j,
np.asarray(conf_aff, dtype=np.float32)[0, j, :, :, 0])
tifffile.imsave(
"tiffs/act1/1actual_affins%i" % j,
np.asarray(conf_aff, dtype=np.float32)[0, j, :, :, 1])
tifffile.imsave(
"tiffs/act2/2actual_affins%i" % j,
def generate_files(opts, test):
# Save training datasets into the npz format.
# https://kite.com/python/docs/numpy.lib.npyio.NpzFile
# https://docs.scipy.org/doc/numpy/reference/generated/numpy.savez.html
# Save testing datasets into tiff images.
dataset = get_dataset(opts, test)
if not test:
# Training data will be saved in npz files.
# DTYPE: np.float32
# FORMAT:
# Multiple npz files where each one contains one training sample:
# {'X': (n_channel, depth, height, width),
# 'Y': (n_channel, depth, height, width)}
if not os.path.exists(opts.npz_dataset_dir):
os.makedirs(opts.npz_dataset_dir)
else:
print('The training dataset already exists.')
sys.exit(0)
for i in range(0, opts.num_train_pairs):
# Here, as training data have different sizes, each training sample
# will be saved into a single npz file.
output_file = os.path.join(opts.npz_dataset_dir,
'train_{:02d}.npz'.format(i))
data = dataset[i]
# Add the channel dimension.
X = np.expand_dims(data[0], axis=0).astype(np.float32)
Y = np.expand_dims(data[1], axis=0).astype(np.float32)
np.savez(output_file, X=X, Y=Y)
print('The training npz file has been created: %s' % (output_file))
else:
# Each image for prediction is saved as signal_{index}.tiff
# The corresponding label, if present, is saved as target_{index}.tiff
if not os.path.exists(opts.tiff_dataset_dir):
os.makedirs(opts.tiff_dataset_dir)
else:
print('The testing dataset already exists.')
sys.exit(0)
source_output_dir = os.path.join(opts.tiff_dataset_dir)
if not os.path.exists(source_output_dir):
os.makedirs(source_output_dir)
label_output_dir = os.path.join(opts.tiff_dataset_dir, 'ground_truth')
if not os.path.exists(label_output_dir):
os.makedirs(label_output_dir)
for i in range(0, opts.num_test_pairs):
name = dataset.get_information(i)['path_czi']
print('Processing %s' % name)
data = dataset[i]
path_tiff = os.path.join(source_output_dir,
'test_image_{:02d}.tiff'.format(i))
if not os.path.isfile(path_tiff):
tifffile.imsave(path_tiff, data[0])
print('Saved:', path_tiff)
if len(data) == 2 and not opts.no_target:
path_tiff = os.path.join(label_output_dir,
'test_image_{:02d}.tiff'.format(i))
if not os.path.isfile(path_tiff):
tifffile.imsave(path_tiff, data[1])
print('Saved:', path_tiff)
def setUp(self):
"""
Set up temporary test directory and mock S3 bucket connection
"""
# Test metadata parameters
self.channel_idx = 1
self.slice_idx = 2
self.time_idx = 3
self.channel_name = "TESTCHANNEL"
# Mock S3 dir
self.storage_dir = "raw_frames/ISP-2005-06-09-20-00-00-0001"
# Create temporary directory and write temp image
self.tempdir = TempDirectory()
self.temp_path = self.tempdir.path
# Temporary frame
self.im = np.ones((10, 15), dtype=np.uint16)
self.im[2:5, 3:12] = 50000
# Metadata
mmmetadata = self._get_mmmeta()
ijmeta = self._get_ijmeta()
extra_tags = [('MicroManagerMetadata', 's', 0, mmmetadata, True)]
# Save test ome tif file
self.file_path1 = os.path.join(self.temp_path, "test_Pos1.ome.tif")
tifffile.imsave(
self.file_path1,
self.im,
ijmetadata=ijmeta,
extratags=extra_tags,
)
mmmetadata = self._get_mmmeta(pos_idx=3)
extra_tags = [('MicroManagerMetadata', 's', 0, mmmetadata, True)]
# Save test ome tif file
self.file_path3 = os.path.join(self.temp_path, "test_Pos3.ome.tif")
tifffile.imsave(
self.file_path3,
self.im,
ijmetadata=ijmeta,
extratags=extra_tags,
)
# Setup mock S3 bucket
self.mock = mock_s3()
self.mock.start()
self.conn = boto3.resource('s3', region_name='us-east-1')
self.bucket_name = 'czbiohub-imaging'
self.conn.create_bucket(Bucket=self.bucket_name)
# Instantiate file parser class
self.storage_class = aux_utils.get_storage_class('s3')
self.frames_inst = ometif_splitter.OmeTiffSplitter(
data_path=self.temp_path,
storage_dir="raw_frames/ISP-2005-06-09-20-00-00-0001",
storage_class=self.storage_class,
)
# Get path to json schema file
dir_name = os.path.dirname(__file__)
self.schema_file_path = os.path.realpath(
os.path.join(dir_name, '..', '..', 'metadata_schema.json'), )
# Upload data
self.frames_inst.get_frames_and_metadata(
schema_filename=self.schema_file_path,
positions='[1, 3]',
)
ext = image[centers[j,0]-armsize:centers[j,0]+armsize, centers[j,1]-armsize:centers[j,1]+ armsize]
#ext_temp = (ext-minint[j])*1./(maxint[j]-minint[j])*255.
ext_temp = (ext-np.min(ext))*1./(np.max(ext)-np.min(ext))*255.
ext_temp = np.array(ext_temp)
ext_temp = ext_temp.astype(np.uint8)
#test if blank image
if n==xi:
seg = segmentationClass.Segmentation(ext_temp, gauss = 1, gaussMask = 2, size = 70, maxFilter = 12)
seg.findMaximaOnFG([])
if seg.areamax > 15.**2*np.pi*2.5 or np.sum(seg.FG)>15.**2*np.pi*3 or seg.areanum > 3 or seg.areanum==0:
bllist.append(j)
else:
os.chdir(outpath)
if not os.path.isdir(pos_dir):
os.mkdir(pos_dir)
os.chdir(outpath+'/'+pos_dir)
if not os.path.isdir(name_dir):
os.mkdir(name_dir)
os.chdir(outpath+'/'+pos_dir+'/'+name_dir)
if not os.path.isdir('images'):
os.mkdir('images')
tiff.imsave("images/{0}_x{1:04d}_y{2:04d}_t{3:04d}.tif".format(listvideo[m].split('.')[0], centers[j,0], centers[j,1], ind2), ext)
#
else:
os.chdir(outpath+'/'+pos_dir+'/'+name_dir)
tiff.imsave("images/{0}_x{1:04d}_y{2:04d}_t{3:04d}.tif".format(listvideo[m].split('.')[0], centers[j,0], centers[j,1], ind2), ext)
def stop(self, doc):
tifffile.imsave(self.fname, self.data)
self.data = None
self.desc = None
self.fname = None
def run_multi(batch_dir, UUID, output):
n_processes = os.environ['_MESMERIZE_N_THREADS']
n_processes = int(n_processes)
file_path = os.path.join(batch_dir, UUID)
filename = [file_path + '_input.tiff']
input_params = pickle.load(open(file_path + '.params', 'rb'))
seq = tifffile.TiffFile(filename[0]).asarray()
seq_shape = seq.shape
# assume default tzxy
for z in range(seq.shape[1]):
tifffile.imsave(f'{file_path}_z{z}.tiff', seq[:, z, :, :])
del seq
print('*********** Creating Process Pool ***********')
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local', n_processes=n_processes, single_thread=False, ignore_preexisting=True
)
num_components = 0
output_files = []
for z in range(seq_shape[1]):
print(f"Plane {z} / {seq_shape[1]}")
filename = [f'{file_path}_z{z}.tiff']
print('Creating memmap')
memmap_fname = cm.save_memmap(
filename,
base_name=f'memmap-{UUID}',
order='C',
border_to_0=input_params['border_pix'],
dview=dview
)
Yr, dims, T = cm.load_memmap(memmap_fname)
Y = np.reshape(Yr.T, [T] + list(dims), order='F')
Ain = None
# seed components
# see if it's an h5 file produced by the nuset_segment GUI
try:
hdict = HdfTools.load_dict(os.path.join(f'{file_path}.ain'), 'data')
Ain = hdict[f'sparse_mask'][str(z)]
except Exception as e:
output['warnings'] = f'Could not seed components, make sure that ' \
f'the .ain file exists in the batch dir: {e}'
#print(Ain)
#raise Exception
# seeded
if Ain is not None:
input_params['cnmf_kwargs'].update(
{
'only_init': False,
'rf': None
}
)
cnmf_params = CNMFParams(params_dict=input_params['cnmf_kwargs'])
cnm = cnmf.CNMF(
dview=dview,
n_processes=n_processes,
Ain=Ain,
params=cnmf_params,
)
cnm.fit(Y)
if input_params['refit']:
cnm = cnm.refit(Y, dview=dview)
cnm.params.set('quality', input_params['eval_kwargs'])
cnm.estimates.evaluate_components(
Y,
cnm.params,
dview=dview
)
cnm.estimates.select_components(use_object=True)
num_components += len(cnm.estimates.C)
out_filename = f'{UUID}_results_z{z}.hdf5'
cnm.save(out_filename)
output_files.append(out_filename)
os.remove(filename[0])
output.update(
{
'output': UUID,
'status': 1,
'output_files': output_files,
'num_components': num_components
}
)
dview.terminate()
return output
for _ in tqdm.tqdm( range( n_iterations ) ):
noisy, clear = get_training_data_inputs(batch_size)
for nc in range( n_critcs ):
critic_model.train_on_batch([clear, noisy], [valid, fake, dummy])
noisy_to_clean_generator.train_on_batch(noisy, valid)
last_loss = cycle_training_multi_gpu_model.train_on_batch( [noisy, clear], [zero_output, zero_output] )
write_model( f'{cycle_training_model_directory}_{loop+1}', cycle_training_model )
write_model( f'{noisy_to_clean_model_directory}_{loop+1}', noisy_to_clean_model )
noisy_to_clean_multi_gpu_model = noisy_to_clean_model
denoised_experimental = noisy_to_clean_multi_gpu_model.predict( noisy_images[:batch_size] )
denoised_experimental = (denoised_experimental + 1.0) / 2.0
denoised_experimental = np.asarray( denoised_experimental * ( (1 << 16) -1 ), dtype='uint16' )
tifffile.imsave( f'./denoised_image-{training_index}_{loop+1}.tif', denoised_experimental, compress=6 )
send_message( f'cycle denoising of {training_index}: {loop+1}/{n_loops} done, last loss [noisy, clear] is {last_loss}.' )
_a, _b = copy.deepcopy( noisy_images[:4] ), denoised_experimental[:4]
for idx in range(4):
imageio.imsave( './tmp_denoised.png', combine( np.squeeze(_a[idx]), np.squeeze(_b[idx]) ) )
send_photo( './tmp_denoised.png' )
import inspect
import os
file_name = inspect.getfile(inspect.currentframe())
file_directory = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
send_message( f'training finished with file {file_directory}{file_name} of {n_training_loops} loops with batch size {batch_size}' )
send_message( f'cycle model has been saved to {cycle_training_model_directory}' )
send_message( f'noisy to clean model has been saved to {noisy_to_clean_model_directory}' )
def merge_blocks(
input_dir: Union[Path, str],
basename: str,
output_dir: Union[None, Path, str] = None,
squeeze: bool = False,
verbose: bool = False
) -> bool:
"""Merge 2D or 3D blocks into a full image.
@param input_dir: Path or str
Full path of the folder where the files to be merged are contained.
@param basename: str
Common base name of all block files (see below).
@param output_dir: None, Path or str, default = None
Full path to the target directory. Set to None (or omit), to save in the folder of the source TIFF file.
@param squeeze: Bool
If True, squeeze any singleton dimension.
@param verbose: Bool
If True, print a lot of information.
Files (blocks) are expected to have filenames following this pattern: {input_dir} / {basename}_c####_r####.tif
@return True if the process was successful, false otherwise.
"""
# Try to get the list of files to merge
file_list = natsorted(glob.glob(str(Path(input_dir) / f"{basename}_c????_r????.tif")))
if len(file_list) == 0:
print("No files found. Aborting.")
return False
# Make sure output_dir is defined and exists
if output_dir is not None:
work_dir = Path(output_dir)
work_dir.mkdir(parents=True, exist_ok=True)
else:
output_dir = Path(input_dir)
# Full path of the first image
first_block_file_name = Path(input_dir) / f"{basename}_c0000_r0000.tif"
# Inform
print(f"Load and process block image {first_block_file_name}")
# Read the first image to get the size of the individual block
block = imread(first_block_file_name)
# Image size
block_height = block.shape[-2]
block_width = block.shape[-1]
# Step sizes
step_y = block_height // 2
step_x = block_width // 2
# Get the numbers of blocks per row and column by scanning the file names
last_row = -1
last_col = -1
for file_name in file_list:
c = int(file_name[-14:-10])
if c > last_col:
last_col = c
r = int(file_name[-8:-4])
if r > last_row:
last_row = r
# Calculate the final image size
image_height = step_y * (last_col + 2)
image_width = step_x * (last_row + 2)
# Inform
if verbose:
print(f"Block size = ({block_height}, {block_width}); "
f"grid size (with overlap) = ({last_row}, {last_col}), "
f"final image size = ({image_height}, {image_width})")
# Allocate memory for the final image
image_size = list(block.shape)
image_size[-2] = image_height
image_size[-1] = image_width
image_dtype = block.dtype
image = np.zeros(tuple(image_size), dtype=image_dtype)
# Define the operation grid
centers_y = range(step_y, image_height, step_y)
centers_x = range(step_x, image_width, step_x)
# Process the image row first
for c, center_y in enumerate(centers_y):
for r, center_x in enumerate(centers_x):
# Do not read the very first block, since we already did
if c == 0 and r == 0:
# Inform
if verbose:
print(f"First block already loaded")
else:
# Build the expected block file name
block_file_name = Path(input_dir) / f"{basename}_c{c:04d}_r{r:04d}.tif"
# Inform
print(f"Load and process block image {block_file_name}")
# Load the block
block = imread(block_file_name)
# Inform
if verbose:
print(f"Insert block into region y = "
f"{center_y - step_y}:{center_y + step_y}, "
f"x = {center_x - step_x}:{center_x + step_x}"
)
# Insert current block; averaging the overlapping areas where needed
if c == 0 and r == 0:
image[..., (center_y - step_y): (center_y + step_y), (center_x - step_x): (center_x + step_x)] = block
elif c == 0 and r > 0:
# Only average on the left
if verbose:
print(f"Reading area [{center_y - step_y}:{center_y + step_y}, {center_x - step_x}:{center_x}] from image.")
print(f"Reading area [0:{block_height}, 0:{step_x}] from block.")
print(f"Storing averaged overlapping area to [0:{block_height}, 0:{step_x}] in block.")
print(f"Storing modified block in area to [{center_y - step_y}:{center_y + step_y}, {center_x - step_x}:{center_x + step_x}] in image.")
overlap_image = image[..., (center_y - step_y): (center_y + step_y), (center_x - step_x): center_x].astype(np.float32)
working_block = block.astype(np.float32)
working_block[..., 0:block_height, 0: step_x] = (0.5 * (working_block[..., 0: block_height, 0: step_x] + overlap_image)).astype(image_dtype)
image[..., (center_y - step_y): (center_y + step_y), (center_x - step_x): (center_x + step_x)] = working_block
elif c > 0 and r == 0:
# Only average at the top
if verbose:
print(f"Reading area [{center_y - step_y}:{center_y}, {center_x - step_x}:{center_x + step_x}] from image.")
print(f"Reading area [0:{step_y}, 0:{block_width}] from block.")
print(f"Storing averaged overlapping area to [0:{block_height}, 0:{step_x}] in block.")
print(f"Storing modified block in area to [{center_y - step_y}:{center_y + step_y}, {center_x - step_x}:{center_x + step_x}] in image.")
overlap_image = image[..., (center_y - step_y): (center_y), (center_x - step_x): center_x + step_x].astype(np.float32)
working_block = block.astype(np.float32)
working_block[..., 0:step_y, 0: block_width] = (0.5 * (working_block[..., 0:step_y, 0: block_width] + overlap_image)).astype(image_dtype)
image[..., (center_y - step_y): (center_y + step_y), (center_x - step_x): (center_x + step_x)] = working_block
else:
# Average both left and up (three segments)
if verbose:
print(f"Reading area 1 [{center_y - step_y}:{center_y}, {center_x - step_x}:{center_x}] from image.")
print(f"Reading area 2 [{center_y - step_y}:{center_y}, {center_x}:{center_x + step_x}] from image.")
print(f"Reading area 3 [{center_y}:{center_y + step_y}, {center_x - step_x}:{center_x}] from image.")
print(f"Reading area 1 [0:{step_y}, 0:{step_x}] from block.")
print(f"Reading area 2 [0:{step_y}, {step_x}:{block_width}] from block.")
print(f"Reading area 3 [{step_y}:{block_height}, 0:{step_x}] from block.")
print(f"Averaging areas 1 to [0:{step_y}, 0:{step_x}] in block.")
print(f"Averaging areas 2 to [0:{step_y}, {step_x}:{block_width}] in block.")
print(f"Averaging areas 3 to [{step_y}:{block_height}, 0:{step_x}] in block.")
print(f"Storing modified block in area to [{center_y - step_y}:{center_y + step_y}, {center_x - step_x}:{center_x + step_x}] in image.")
overlap_image_area_1 = image[..., (center_y - step_y): center_y, (center_x - step_x): center_x].astype(np.float32)
overlap_image_area_2 = image[..., (center_y - step_y): center_y, center_x: (center_x + step_x)].astype(np.float32)
overlap_image_area_3 = image[..., center_y: center_y + step_y, (center_x - step_x): center_x].astype(np.float32)
working_block = block.astype(np.float32)
working_block[..., 0:step_y, 0:step_x] = (0.5 * (working_block[..., 0:step_y, 0: step_x] + overlap_image_area_1)).astype(image_dtype)
working_block[..., 0:step_y, step_x:block_width] = (0.5 * (working_block[..., 0:step_y, step_x:block_width] + overlap_image_area_2)).astype(image_dtype)
working_block[..., step_y:block_height, 0:step_x] = (0.5 * (working_block[..., step_y:block_height, 0: step_x] + overlap_image_area_3)).astype(image_dtype)
image[..., (center_y - step_y): (center_y + step_y), (center_x - step_x): (center_x + step_x)] = working_block
# If squeeze is True, remove any singleton dimension
if squeeze:
image = image.squeeze()
# Save the reconstructed image
out_file_name = Path(output_dir) / f"{basename}.tif"
imsave(out_file_name, image)
# Return success
return True
df["num_voxels"] = missing_struct_voxels+all_struct_voxels
df["type"] = ["eroded"]*len(missing_struct_voxels) + ["original"]*len(all_struct_voxels)
sns.stripplot(x = "num_voxels", y = "type", data = df, color = "crimson", orient = "h")
sns.boxplot(x = "num_voxels", y = "type", data = df, orient = "h", showfliers=False, showcaps=False,
boxprops={'facecolor':'None'})
plt.xlim([0, 200000])
plt.xlabel("Total number of voxels in structure")
plt.ylabel("Structures 'zero'ed' out vs. all original structures")
plt.savefig(os.path.join(fig_dst, "boxplot_total_voxels_org_vs_eroded.pdf"), bbox_inches = "tight")
#%%
#export missing structures name, id, and total voxel count
dataf = pd.DataFrame()
dataf["name"] = missing_struct_names
dataf["id"] = missing_struct_ids
dataf["parent_name"] = missing_struct_parents
dataf["voxels_in_structure"] = missing_struct_voxels
dataf.to_csv(os.path.join(src, "structures_zeroed_after_80um_erosion_allen_annotation_2017_ids_fixed_v2.csv"))
#%%
#make a volume where only the "zeroed" out structures are displayed, zero out the rest
zr_ann = ann.copy()
for iid in missing:
zr_ann[zr_ann == iid] = 0
tifffile.imsave(r"Z:\zahra\kelly_cell_detection_analysis\non_zeroed_out_structures_in_org_atlas_for_vis.tif", zr_ann)
def create_blocks(
input_tiff_file: Union[Path, str],
output_dir: Union[None, Path, str] = None,
block_size: Tuple[int, int] = (512, 512),
verbose: bool = False) -> bool:
"""Breaks a 2D or 3D data set into blocks of size (height, widht) and full depth.
@param input_tiff_file: Path or str
Full path of the TIFF file to read.
@param output_dir: None, Path or str, default = None
Full path to the target directory. Set to None (or omit), to save in the folder of the source TIFF file.
@param block_size: Tuple(height, width), default = (512, 512)
Size of the block (height, width); all other dimensions are preserved completely.
@param verbose: Bool
If True, print a lot of information.
Blocks are saved with filenames following this pattern: {work_dir} / {basename}_c####_r####.tif
@return True if the process was successful, false otherwise.
"""
# Try opening the file; if it fails, return False
try:
img = imread(input_tiff_file)
except:
print(f"Could not read file {input_tiff_file}. Aborting.")
return False
# Prepare path and file name information
if output_dir is not None:
work_dir = Path(output_dir)
work_dir.mkdir(parents=True, exist_ok=True)
else:
work_dir = Path(input_tiff_file).parent
basename = Path(input_tiff_file).stem
# Image size
image_height = img.shape[-2]
image_width = img.shape[-1]
# Step sizes
step_y = block_size[0] // 2
step_x = block_size[1] // 2
# Define the operation grid
centers_y = range(step_y, image_height, step_y)
centers_x = range(step_x, image_width, step_x)
# Process the image row first
for c, center_y in enumerate(centers_y):
for r, center_x in enumerate(centers_x):
# Inform
if verbose:
print(f"Extract region y = {center_y - step_y}:{center_y + step_y}, x = {center_x - step_x}:{center_x + step_x}")
# Extract current block
block = img[..., (center_y - step_y): (center_y + step_y), (center_x - step_x): (center_x + step_x)]
# Build file name
out_file_name = work_dir / f"{basename}_c{c:04d}_r{r:04d}.tif"
# Save the block
imsave(str(out_file_name), block)
# Inform
print(f"Saved file {out_file_name.stem}")
return True
def xyOffset(image1, image2, scale, center=None):
"""
Note that the search is limited to a X by X region
in the center of the overlap between the two images.
"""
image1 = image1 - numpy.median(image1)
image2 = image2 - numpy.median(image2)
result = xyCorrelate(image1, image2)
if False:
import tifffile
tifffile.imsave("corr_image1.tif", image1.astype(numpy.float32))
tifffile.imsave("corr_image2.tif", image2.astype(numpy.float32))
tifffile.imsave("corr_result.tif", result.astype(numpy.float32))
# These are the coordinates of the image center.
mx = int(round(0.5 * result.shape[0]))
my = int(round(0.5 * result.shape[1]))
# This is the area to search, 30 pixels * scale.
s_size = int(30 * int(scale))
sx = s_size
sy = s_size
# Adjust if the image is really small.
if mx < (s_size + 5):
sx = mx - 5
if my < (s_size + 5):
sy = my - 5
# Use center position provided by the user.
if isinstance(center, list):
rx = int(round(center[0]) + sx)
ry = int(round(center[1]) + sy)
if False:
print(rx)
print(
numpy.argmax(
numpy.max(result[mx - sx:mx + sx + 1, my - sy:my + sy + 1],
axis=1)))
print(ry)
print(
numpy.argmax(
numpy.max(result[mx - sx:mx + sx + 1, my - sy:my + sy + 1],
axis=0)))
# Otherwise find local maximum near the image center.
else:
rx = numpy.argmax(
numpy.max(result[mx - sx:mx + sx + 1, my - sy:my + sy + 1],
axis=1))
ry = numpy.argmax(
numpy.max(result[mx - sx:mx + sx + 1, my - sy:my + sy + 1],
axis=0))
# Adjust from maxima search area to full image size.
rx += (mx - sx)
ry += (my - sy)
#
# Fit a gaussian to the maxima.
#
# FIXME: Should use elliptical gaussian with variable axis?
#
# FIXME: Should fit a larger / smaller area?
#
[fit, success
] = gaussfit.fitSymmetricGaussian(result[rx - 5:rx + 6, ry - 5:ry + 6],
2.0)
if 0:
fit_fn = gaussfit.symmetricGaussian(*fit)
surf = numpy.zeros((11, 11))
fp = open("fit.txt", "w")
fp.write("x, y, correlation, fit\n")
for x in range(11):
for y in range(11):
surf[x, y] = fit_fn(x, y)
fp.write(
str(x) + ", " + str(y) + ", " +
str(result[rx - 5 + x, ry - 5 + y]) + ", " +
str(surf[x, y]) + "\n")
fp.close()
if (fit[0] == fit[1]) or (fit[2] < 2.0) or (fit[2] > 8.0) or (
fit[3] < 2.0) or (fit[3] > 8.0):
print("Bad fit center:", fit[2], fit[3])
success = False
return [
result[mx - sx:mx + sx + 1,
my - sy:my + sy + 1], rx + fit[2] - 5.0 - 0.5 * result.shape[0],
ry + fit[3] - 5.0 - 0.5 * result.shape[1], success
]
