def stitch_images(path_to_files, output_path, *args):
from xml_parser import parse_phenix_xml as parse_xml
from tqdm import tqdm
from image_stitcher import channel_stitch
from skimage.external import tifffile
path_to_xml_file = path_to_files + 'Index.idx.xml'
dictionary = parse_xml(path_to_xml_file)
for wells in tqdm(dictionary):
d = {
channels: images
for channels, images in dictionary[wells].items()
}
stitching_input = [(wells, channels, images)
for channels, images in d.items()]
stitched_list = [
channel_stitch(path_to_files, stitching_input[index][0],
stitching_input[index][1],
stitching_input[index][2])
for index, item in enumerate(stitching_input)
]
with tifffile.TiffWriter(output_path + wells + "_" +
"image_stack.tiff",
imagej=True) as stack:
for item in stitched_list:
stack.save(item, photometric='minisblack')
def register(tiff_path, trans_mat, out_tiff_path, highres = True, compress = 1, pad = True):
'''
tiff_path: tiff file name
trans_mat: translation matrix, can be obtained by roi2mat()
highres: optional, if set to True, will multiply the trans_mat by compress (which is set to 3 by default)
compress: as above
pad: whehther or not pad the periphery to zeros. If false, will crop the tiff
This function returns a dict of metadata, and writes the tiff to current working directory
'''
with tifffile.TiffFile(tiff_path) as tif:
# read tiff
im_in = tif.asarray()
# read metadata as tif_tags (dict)
tif_tags = tif.pages[0].tags.values()
# register using trans_mat
im_out = translate(im_in,trans_mat,hi_res=highres,compression=compress,padzeros=pad)
im_out = im_out.astype('uint16')
# save registered tiff, no compression
with tifffile.TiffWriter(out_tiff_path, bigtiff = highres) as tif:
for i in range(im_out.shape[0]):
tif.save(im_out[i])
return tif_tags
def make_PMT_iamge(self, obj_position=None):
"""
Take PMT image at certain objective position.
Parameters
----------
obj_position : float, optional
The target objective position. The default is None.
"""
if obj_position != None:
self.pi_device_instance.move(obj_position)
# Get the image.
self.galvo_image = self.RasterScanins.run()
# plt.figure()
# plt.imshow(self.galvo_image)
# plt.show()
meta_infor = "index_" + str(self.each_pos_index) + "_pos_" + str(obj_position)
if True:
with skimtiff.TiffWriter(
os.path.join(self.saving_dir, meta_infor + ".tif")
) as tif:
tif.save(self.galvo_image.astype("float32"), compress=0)
time.sleep(0.5)
def test_tiff_imread(tmpdir, seed, nframes, shape, dtype):
np.random.seed(seed)
dirpth = tmpdir.mkdir("test_imread")
dtype = np.dtype(dtype).type
low, high = 0.0, 1.0
if isinstance(dtype, numbers.Integral):
low, high = np.iinfo(dtype).min, np.iinfo(dtype).max
a = np.random.uniform(low=low, high=high, size=shape).astype(dtype)
fn = str(dirpth.join("test.tiff"))
with tifffile.TiffWriter(fn) as fh:
for i in range(len(a)):
fh.save(a[i])
d = dask_imread.imread(fn, nframes=nframes)
if nframes == -1:
nframes = shape[0]
assert min(nframes, shape[0]) == max(d.chunks[0])
if shape[0] % nframes == 0:
assert nframes == d.chunks[0][-1]
else:
assert (shape[0] % nframes) == d.chunks[0][-1]
dau.assert_eq(a, d)
def __loop(self):
# display start time
# time format
fmt = "%H:%M:%S"
start = datetime.datetime.now()
print("[{}] Starting…".format(start.strftime(fmt)))
if self.skip is True:
print('Skipping file renaming')
else:
self.__sortRename()
print('Now building the stacks')
from skimage.external import tifffile
sortedPath = self.folder.joinpath('sorted')
for stage in sortedPath.iterdir():
print('Processing stage: {}'.format(stage.name))
for color in self.colors:
print('Processing color: {}'.format(color))
filelist = list(stage.glob('love*' + color + '*.tif'))
# need it as string for tifffile
slist = sorted([str(x) for x in filelist])
outfile = str(stage.joinpath('stack' + color + '.tif'))
if not os.path.isfile(outfile):
with tifffile.TiffWriter(
outfile, imagej=True,
software='stackbuilder.py') as stack:
stack.save(tifffile.TiffSequence(slist).asarray())
def tiffs_to_stack(directory):
if not directory[-1] == '/':
directory = directory + '/'
stack_fn = os.path.join(directory, 'stack.tiff')
print('Creating tiff stack from {}'.format(directory))
with tifffile.TiffWriter(stack_fn) as stack:
for filename in sorted(glob.glob(directory + '*.tif')):
stack.save(tifffile.imread(filename))
def do(self, data):
out = data
output_file = "./output/" + self.get("file_name") + ".tif"
with tff.TiffWriter(output_file) as tif:
tif.save(out)
return out
def __init__(self, s_fname_out, b_delete_existing=False):
if os.path.isfile(s_fname_out) and not b_delete_existing:
raise ValueError(
"Requested output file already exist. Die in order to prevent data loss."
)
self.s_fname_out = s_fname_out
self.oc_tiff = tifffile.TiffWriter(self.s_fname_out,
bigtiff=True,
software="mendouscopy")
def stack_folder_tif(folder_path, folder_name, new_path, flip):
file_list = os.listdir(folder_path)
file_list.sort()
new_file_path = os.path.join(new_path, folder_name) + '.tif'
with tifffile_sk.TiffWriter(new_file_path) as stack:
for filename in file_list:
np_im = tifffile_sk.imread(os.path.join(folder_path, filename))
if flip:
np_im = np.flipud(np_im)
stack.save(np_im)
def writeTiff(path, image, name, dtype='float32'):
""" saves a 2d or 3d numpy array to multi-page tiff file as (x,y,z)
:param path: base path
:param image: 2d / 3d numpy array
:param name: image name (.tif will be added)
:param dtype: dtype of image (float32)
"""
filename = path + name + ".tif"
try:
if image.size * image.dtype.itemsize > 2000 * 2**20:
writer1 = tifffile.TiffWriter(filename, bigtiff=True)
else:
writer1 = tifffile.TiffWriter(filename)
if len(image.shape) > 2:
writer1.save(image.transpose((2, 0, 1)).astype(dtype=dtype),
photometric="minisblack")
else:
writer1.save(image.astype(dtype=dtype), photometric="minisblack")
writer1.close()
del writer1
os.chmod(filename, 0o777)
except (IOError, OSError):
myDir, myName = os.path.split(filename)
newFilename = tempfile.mktemp(suffix='_' + myName,
prefix='tmp',
dir=myDir)
if image.size * image.dtype.itemsize > 2000 * 2**20:
writer1 = tifffile.TiffWriter(filename, bigtiff=True)
else:
writer1 = tifffile.TiffWriter(filename)
logger.error(
'error saving tiff:' + str(filename) + ' changed name to temp: ',
str(newFilename))
if len(image.shape) > 2:
writer1.save(image.transpose((2, 0, 1)).astype(dtype=dtype),
photometric="minisblack")
else:
writer1.save(image.astype(dtype=dtype), photometric="minisblack")
writer1.close()
del writer1
os.chmod(newFilename, 0o777)
def writeTiff4D(path, image, name, dtype='float32'):
""" saves a 4d numpy array to multi-page tiff as (x,y,z*t)
:param path: base path
:param image: 4d numpy array assumes 2nd and 3rd dimensions are x,y
:param name: image name (.tif will be added)
:param dtype: dtype of image (float32)
"""
image = image.transpose((3, 0, 1, 2))
si = image.shape
filename = path + name + ".tif"
if image.size * image.dtype.itemsize > 2000 * 2**20:
writer1 = tifffile.TiffWriter(filename, bigtiff=True)
else:
writer1 = tifffile.TiffWriter(filename)
out = image.reshape(si[0] * si[1], si[2], si[3], order="F")
writer1.save(out.astype(dtype=dtype), photometric="minisblack")
writer1.close()
del writer1
os.chmod(path + name + ".tif", 0o777)
def write_ometiff(output_path, array6D, omexml_string):
with tifffile.TiffWriter(output_path) as tif:
for i in range(len(array6D)):
# check if this is the first Series
if i != 0:
# for other series, no need to add the omexml. The contiguous=False allows to add the series one by one to the same file
tif.save(array6D[i],
photometric='minisblack',
metadata={'axes': 'TZCXY'},
contiguous=False)
else:
tif.save(array6D[i],
description=omexml_string,
photometric='minisblack',
metadata={'axes': 'TZCXY'})
def StopStreaming(self, saving_dir = None):
# =====================================================================
# Stop the streaming and save the file.
# - saving_dir: directory in which the video is saved.
# =====================================================================
self.isStreaming = False
# Stop the acquisition
self.hcam.stopAcquisition()
if saving_dir != None:
self.isSaving = True
# Save the file.
with skimtiff.TiffWriter(saving_dir, append = True)\
as tif:
for eachframe in range(self.imageCount):
image = np.resize(self.video_list[eachframe], (self.dims[1], self.dims[0]))
tif.save(image, compress=0, description=self.metaData)
self.isSaving = False
def evaluate_focus(self, obj_position=None):
"""
Evaluate the focus degree of certain objective position.
Parameters
----------
obj_position : float, optional
The target objective position. The default is None.
Returns
-------
degree_of_focus : float
Degree of focus.
"""
if obj_position != None:
self.pi_device_instance.move(obj_position)
# Get the image.
if self.source_of_image == "PMT":
self.galvo_image = self.galvo.run()
plt.figure()
plt.imshow(self.galvo_image)
plt.show()
if False:
with skimtiff.TiffWriter(
os.path.join(
r'M:\tnw\ist\do\projects\Neurophotonics\Brinkslab\Data\Xin\2020-11-17 gaussian fit auto-focus cells\trial_11',
str(obj_position).replace(".", "_") +
'.tif')) as tif:
tif.save(self.galvo_image.astype('float32'), compress=0)
degree_of_focus = ProcessImage.local_entropy(
self.galvo_image.astype('float32'))
time.sleep(0.2)
return degree_of_focus
dataDir = '/home/nick/data/2pdata/imag003'
session = '002_010'
Fn = 'imag003_{}.tif'.format(session)
fullFile = os.path.join(dataDir, Fn)
im0 = tifffile.TiffFile(fullFile)
im0ar = im0.asarray()
f0 = np.mean(im0ar[:10, :, :], axis=0)
# f0 = np.mean(im0ar, axis=0)
# f0_resize=np.reshape(f0.repeat(249), np.shape(im0ar))
f0_repeat = np.repeat(f0[np.newaxis, :, :], im0ar.shape[0], axis=0)
dff = (im0ar - f0_repeat) / f0_repeat
# avgImg = np.mean(im0.asarray(), axis=0)
avgImg = np.mean(dff, axis=0)
# plt.clf()
# plt.imshow(avgImg)
# plt.show()
output = True
outputFn = '/tmp/temp_dff.tiff'
if output:
# with tifffile.TiffWriter(outputFn) as tif:
with tifffile.TiffWriter(outputFn, imagej=True) as tif:
for indFrame in range(dff.shape[0]):
tif.save(dff.astype('uint16')[indFrame, :, :], compress=0)
def metoctiff(data_array,
ullat_radians,
urlat_radians,
lllat_radians,
lrlat_radians,
uclat_radians,
lclat_radians,
ullon_radians,
urlon_radians,
lllon_radians,
lrlon_radians,
uclon_radians,
lclon_radians,
data_start_datetime,
data_end_datetime,
product_name,
platform_name,
data_units,
output_filename,
requested_data_min,
requested_data_max,
scale_data_min=1,
scale_data_max=255,
missing_value=0,
product_description='None',
mpl_cmap=None,
existing_image=None,
gzip_output=False):
'''
Generate a metoctiff with valid headers from existing image OR data found in data_array,
with appropriate colormap, data ranges, tags
NOTE: If you include the "existing_image" option, it will just read in an arbitrary existing image and plot it
QUALITATIVELY with the appropriate colormap / lat / lons, but no quantitative information.
+------------------+-----------+-------------------------------------------------------+
| Parameters: | Type: | Description: |
+==================+===========+=======================================================+
| data_array: | *ndarray* | Array of data to scale properly for metoctiff |
+------------------+-----------+-------------------------------------------------------+
| ullat_radians: | *float* | upper left lat of the data_array in radians |
| | | used for metoctiff "extratags" header |
+------------------+-----------+-------------------------------------------------------+
| urlat_radians: | *float* | upper right lat of the data_array in radians |
| | | used for metoctiff "extratags" header |
+------------------+-----------+-------------------------------------------------------+
| lllat_radians: | *float* | lower left lat of the data_array in radians |
| | | used for metoctiff "extratags" header |
+------------------+-----------+-------------------------------------------------------+
| lrlat_radians: | *float* | lower right lat of the data_array in radians |
| | | used for metoctiff "extratags" header |
+------------------+-----------+-------------------------------------------------------+
| ullon_radians: | *float* | upper left lon of the data_array in radians |
| | | used for metoctiff "extratags" header |
+------------------+-----------+-------------------------------------------------------+
| urlon_radians: | *float* | upper right lon of the data_array in radians |
| | | used for metoctiff "extratags" header |
+------------------+-----------+-------------------------------------------------------+
| lllon_radians: | *float* | lower left lon of the data_array in radians |
| | | used for metoctiff "extratags" header |
+------------------+-----------+-------------------------------------------------------+
| lrlon_radians: | *float* | lower right lon of the data_array in radians |
| | | used for metoctiff "extratags" header |
+------------------+-----------+-------------------------------------------------------+
|data_start_datetime:|*datetime*| datetime object indicating the data start datetime |
| | | used for metoctiff "description" header DATA_START_TIME |
+------------------+-----------+-------------------------------------------------------+
|data_end_datetime:| *datetime*| datetime object indicating the data end datetime |
| | | used for metoctiff "description" header DATA_END_TIME |
+------------------+-----------+-------------------------------------------------------+
| product_name: | *str* | string of current product name of the data_array |
| | | used for metoctiff "description" header DATA_NAME |
+------------------+-----------+-------------------------------------------------------+
| platform_name: | *str* | string of current platform name of the data_array |
| | | used for metoctiff "description" header PLATFORM_NAME |
+------------------+-----------+-------------------------------------------------------+
| data_units: | *str* | string of current units of the data_array |
| | | used for metoctiff "description" header DATA_UNITS |
+------------------+-----------+-------------------------------------------------------+
| output_filename: | *str* | string of output filename to write the metoctiff |
+------------------+-----------+-------------------------------------------------------+
|requested_data_min| *float* | Minimum allowed value for the actual data, |
| | | for scaling from scale_data_min to scale_data_max |
| | | metoctiff description tag: |
| | | DATA_RANGE=scale_data_min, |
| | | scale_data_max, |
| | | requested_data_min, |
| | | requested_data_max/scale_data_max |
| | | |
+------------------+-----------+-------------------------------------------------------+
|requested_data_max| *float* | Maximum allowed value for the actual data, |
| | | for scaling from scale_data_min to scale_data_max |
| | | metoctiff description tag: |
| | | DATA_RANGE=scale_data_min, |
| | | scale_data_max, |
| | | requested_data_min, |
| | | requested_data_max/scale_data_max |
+------------------+-----------+-------------------------------------------------------+
|scale_data_min | *uint8* | Minimum allowed value for the scaled data, |
| | | for scaling from scale_data_min to scale_data_max |
| | DEFAULT | metoctiff description tag: |
| | 1 | DATA_RANGE=scale_data_min, |
| | | scale_data_max, |
| | | requested_data_min, |
| | | requested_data_max/scale_data_max |
| | | |
+------------------+-----------+-------------------------------------------------------+
|scale_data_max | *uint8* | Maximum allowed value for the scaled data, |
| | | for scaling from scale_data_min to scale_data_max |
| | DEFAULT | metoctiff description tag: |
| | 255 | DATA_RANGE=scale_data_min, |
| | | scale_data_max, |
| | | requested_data_min, |
| | | requested_data_max/scale_data_max |
+------------------+-----------+-------------------------------------------------------+
|missing_value | *uint8* | Value that ATCF considers missing/bad data, |
| | | between 0 and 255 |
| | DEFAULT | metoctiff description tag: |
| | 0 | DATA_RANGE=scale_data_min, |
| | | scale_data_max, |
| | | requested_data_min, |
| | | requested_data_max/scale_data_max |
+------------------+-----------+-------------------------------------------------------+
| mpl_cmap: |*ColorMap* | matplotlib ColorMap object to create 255 color palette|
| | | to map the scaled 1-255 data values in the jif |
| | | !!! ColorMap must match the range specified in |
| | | requested_data_min to requested_data_max |
| | | if you want specific colors to match specific |
| | | values !!! |
| | | |
+------------------+-----------+-------------------------------------------------------+
|existing_image: | *str* | string of full path to an existing image |
| | *ndarray* | RGB or RGBA array of 0 to 1 |
| | | NOTE: Use of this option basically ignores most |
| | | everything else! Just reads it in and writes |
| | | it back out qualitatively, with the |
| | | appropriate colors and metoctiff headers |
+------------------+-----------+-------------------------------------------------------+
| gzip_output: |*bool* | Flag to determine whether to gzip the output |
+------------------+-----------+-------------------------------------------------------+
'''
output_products = []
LOG.info('Creating metoctiff image file, gzip_output=%s', gzip_output)
#
# Get the image lat lon corners for the metoctiff extratags
# Added the image flip.
#
rsULLat = int(numpy.rad2deg(ullat_radians) * 100000)
rsULLon = int(numpy.rad2deg(ullon_radians) * 100000)
rsURLat = int(numpy.rad2deg(urlat_radians) * 100000)
rsURLon = int(numpy.rad2deg(urlon_radians) * 100000)
rsLLLat = int(numpy.rad2deg(lllat_radians) * 100000)
rsLLLon = int(numpy.rad2deg(lllon_radians) * 100000)
rsLRLat = int(numpy.rad2deg(lrlat_radians) * 100000)
rsLRLon = int(numpy.rad2deg(lrlon_radians) * 100000)
rsUCLat = int(numpy.rad2deg(uclat_radians) * 100000)
rsUCLon = int(numpy.rad2deg(uclon_radians) * 100000)
rsBCLat = int(numpy.rad2deg(lclat_radians) * 100000)
rsBCLon = int(numpy.rad2deg(lclon_radians) * 100000)
#
# NOTE: We are now passing Center Lat and Center Lon directly -
# these calculations fail when crossing the dateline.
# Get the center lat lon values of image for the metoctiff extratags
#
# rsUCLat = (rsULLat + rsURLat) / 2
# rsUCLon = (rsULLon + rsURLon) / 2
# rsBCLat = (rsLLLat + rsLRLat) / 2
# rsBCLon = (rsLLLon + rsLRLon) / 2
#
# Additional info for extratags required for metocTiff
#
nProjection = 4 # 1 = Polar Stereographic 2 = Lambert Conformal 4 = Mercator 8 = Normal.
# It's likely that mercator is analogous to 'cyl' in pyproj'''
rsStandard1 = 0 # only used if lambert conformal projection is specified
rsStandard2 = 0 # only used if lambert conformal projection is specified
if rsBCLat >= 0:
Hemisphere = 1 # northern
else:
Hemisphere = 2 # southern
# Otherwise, if we passed a existing_image, read it in and grab data from
# temporary image
if existing_image is not None:
scaledata, jif_cmap, requested_bounds, scale_bounds = get_data_from_image(
existing_image)
scale_data_min = scale_bounds[0]
scale_data_max = scale_bounds[1]
requested_data_min = requested_bounds[0]
requested_data_max = requested_bounds[1]
else:
scaledata, jif_cmap = get_data_from_equations(
data_array, mpl_cmap, requested_data_min, requested_data_max,
scale_data_min, scale_data_max, missing_value)
#
# Info for "description" tag included in metoctiff file.
# ATCF relies heavily on this description in order to display the tiff correctly
#
data_name = product_name
platform = platform_name
data_start_dtstr = data_start_datetime.strftime(
'%a, %d %b %Y %H:%M:%S GMT')
data_end_dtstr = data_end_datetime.strftime('%a, %d %b %Y %H:%M:%S GMT')
#
# Write out the tiff file with all of the appropriate METOCTiff headers!
# Note it appears we need to use little endian order (big endian looks correct,
# but doesn't work for interrogating values in ATCF), and uint8
#
# Note: Any data ranges are scaled to 0 to 249 in get_data_from_equations
#
# for byteorderstr in ['le', 'be']:
# for dtype in ['uint8', 'uint16', 'float64', 'float32']:
for byteorderstr in ['le']:
for dtype in ['uint8']:
if byteorderstr == 'le':
byteorder = '<'
elif byteorderstr == 'be':
byteorder = '>'
# Just output filename - we've determined little endian, uint8, from data itself is the way to go.
curr_output_filename = output_filename
szDescription = 'DATA_PLATFORM="{0}";'.format(platform) + \
'DATA_NAME="{0}";'.format(data_name) + \
'DATA_START_TIME="{0}";'.format(data_start_dtstr) + \
'DATA_END_TIME="{0}";'.format(data_end_dtstr) + \
'DATA_UNITS="{0}";'.format(data_units)
LOG.info('DATA_RANGE tag from %s to %s', requested_data_min,
requested_data_max)
# From def get_data_from_equations
# m = (float(scale_data_max) - scale_data_min) / (float(requested_data_max) - requested_data_min)
# b = scale_data_min - m * float(requested_data_min)
# scaledata = m * data_array + b
# So realdata = (1/m) * (scaledata - b)
# DATA range uses the 1/m scaling factor, as well as scale_data_min, scale_data_max, and requested_data_min
# in order to recover the real data from the scaled values.
# To determine MAX_DATA_VAL from scaling_term in DATA_RANGE:
# scaling_term*(SCALE_DATA_MAX-SCALE_DATA_MIN) + MIN_DATA_VAL
scaling_term = (float(requested_data_max) - requested_data_min) / (
float(scale_data_max) - scale_data_min)
datarange = 'DATA_RANGE="{0},{1},{2},{3:0.6f},{4}";'.format(
scale_data_min, scale_data_max, requested_data_min,
scaling_term, product_description)
# print(datarange)
szDescription = '{0}{1}'.format(szDescription, datarange)
scaledata = scaledata.astype(dtype)
from geoips2.filenames.base_paths import make_dirs
make_dirs(os.path.dirname(curr_output_filename))
with tf.TiffWriter(curr_output_filename,
byteorder=byteorder) as mtif:
LOG.info('Writing METOCTIFF jif file: %s...',
curr_output_filename)
mtif.save(scaledata,
colormap=jif_cmap,
description=szDescription,
metadata=None,
extratags=[(284, 'H', 1, 1, True),
(33000, 'i', 1, nProjection, True),
(33001, 'i', 1, rsStandard1, True),
(33002, 'i', 1, rsStandard2, True),
(33003, 'i', 1, Hemisphere, True),
(33004, 'i', 1, rsULLat, True),
(33005, 'i', 1, rsULLon, True),
(33006, 'i', 1, rsLLLat, True),
(33007, 'i', 1, rsLLLon, True),
(33008, 'i', 1, rsURLat, True),
(33009, 'i', 1, rsURLon, True),
(33010, 'i', 1, rsLRLat, True),
(33011, 'i', 1, rsLRLon, True),
(33012, 'i', 1, rsBCLat, True),
(33013, 'i', 1, rsBCLon, True),
(33014, 'i', 1, rsUCLat, True),
(33015, 'i', 1, rsUCLon, True)])
LOG.info('MTIFSUCCESS %s', curr_output_filename)
output_products = [curr_output_filename]
# Sanity Check
LOG.info('Min/Max Left Lat %s %s', rsLLLat / 10**5 * 1.0,
rsULLat / 10**5 * 1.0)
LOG.info('Min/Max Right Lat %s %s', rsLRLat / 10**5 * 1.0,
rsURLat / 10**5 * 1.0)
LOG.info('Bottom/Top Center Lat %s %s', rsBCLat / 10**5 * 1.0,
rsUCLat / 10**5 * 1.0)
LOG.info('Min/Center/Max Lower Lon %s %s %s', rsLLLon / 10**5 * 1.0,
rsBCLon / 10**5, rsLRLon / 10**5 * 1.0)
LOG.info('Min/Center/Max Upper Lon %s %s %s', rsULLon / 10**5 * 1.0,
rsUCLon / 10**5 * 1.0, rsURLon / 10**5 * 1.0)
if gzip_output is True:
try:
#gzip the file and remove original
LOG.info('Gzipping output to file %s.gz', output_filename)
with open(output_filename, 'rb') as uncompressedFile, gzip.open(
output_filename + '.gz', 'wb') as compressedFile:
shutil.copyfileobj(uncompressedFile, compressedFile)
if os.path.isfile(output_filename):
os.remove(output_filename)
output_products = [output_filename + '.gz']
except Exception as err:
LOG.info('{0}: {1} >> {2}'.format(
type(err).__name__, str(err.__doc__), str(err.args)))
return output_products
def convert_tiff(s_fname_in, s_fname_out):
na_data = tifffile.imread(s_fname_in)
i_nframes, i_nrows, i_ncols = na_data.shape
with tifffile.TiffWriter(s_fname_out, bigtiff=False) as h_file:
for i_frame_idx in range(i_nframes):
h_file.save(na_data[i_frame_idx, ...])
def metoctiff(self, sector, output_filename):
log.info('Creating metoctiff image file.')
#
# Get the image lat lon corners for the metoctiff tags
# Added the image flip.
#
corners = numpy.flipud(sector.area_definition.corners)
rsULLat = int(numpy.rad2deg(corners[3].lat) * 100000)
rsULLon = int(numpy.rad2deg(corners[3].lon) * 100000)
rsURLat = int(numpy.rad2deg(corners[2].lat) * 100000)
rsURLon = int(numpy.rad2deg(corners[2].lon) * 100000)
rsLLLat = int(numpy.rad2deg(corners[0].lat) * 100000)
rsLLLon = int(numpy.rad2deg(corners[0].lon) * 100000)
rsLRLat = int(numpy.rad2deg(corners[1].lat) * 100000)
rsLRLon = int(numpy.rad2deg(corners[1].lon) * 100000)
#
# Get the center lat lon values of image for the metoctiff tags
#
rsUCLat = (rsULLat + rsURLat) / 2
rsUCLon = (rsULLon + rsURLon) / 2
rsBCLat = (rsLLLat + rsLRLat) /2
rsBCLon = (rsLLLon + rsLRLon) /2
#
# Info for extra tags required for metocTiff
#
nProjection = 4 # 1 = Polar Stereographic 2 = Lambert Conformal 4 = Mercator 8 = Normal. It's likely that mercator is analogous to 'cyl' in pyproj
rsStandard1 = 0 #only used if lamber conformal projection is specified
rsStandard2 = 0 #only used if lamber conformal projection is specified
if rsBCLat >= 0:
Hemisphere = 1 #northern
else:
Hemisphere = 2 #southern
#
# The ATCF app relies heavily upon tag 270 (description) to ingest metoctiffs. Setup the szDescription variables
#
platform = self.title.satellite
data_name = self.title.product
if 'img' not in self.product.images.keys():
log.info('Only single channel imagery supported at this time, skipping METOCTIFF')
return None
data_max = int(self.product.images['img'].max)
data_min = int(self.product.images['img'].min)
if self.product.images['img'].units:
data_units = self.product.images['img'].units
else:
data_units = 'unknown'
#
# The image is normalized data 0-1 and the imagey needs to be 0-255
#
image_max = int(self.image.max() * 255)
image_min = int(self.image.min() * 255)
#
# Setup the start and end time strings and convert them to metoctiff format
#
geoips_start_time = self.start_datetime
data_start_time = geoips_start_time.strftime('%a, %d %b %Y %H:%M:%S GMT')
geoips_end_time = self.end_datetime
data_end_time = geoips_end_time.strftime('%a, %d %b %Y %H:%M:%S GMT')
#
#
#The item discription string is written to tiff tag 270 and is needed by ATCF
#
szDescription = 'DATA_PLATFORM='+'"'+platform+'"'+';'+'DATA_NAME='+'"'+data_name+'"'+';'+'DATA_START_TIME='+'"'+data_start_time+'"'+';'+'DATA_END_TIME='+'"'+data_end_time+'"'+';'+'DATA_UNITS='+'"'+data_units+'"'+';'+'DATA_RANGE='+'"'+str(image_min)+','+str(image_max)+','+str(data_min)+','+str(data_max)+',None'+'"'+';'
#
# Create the 8 bit image to pass to the tiff writer.
#
data_tbs = self.image[:,:,0:3] * 255
data_tbsint = numpy.flipud(data_tbs.astype(numpy.uint8))
#TODO
#set the color map based on the product in data_name = self.title.product
if data_name in ['Visible']:
colorMap = color_map_greyScale
else:
colorMap = color_map_color
#colorMap = color_map_color
# if passing in tag 320 as an extratag, TiffWriter expects a 1d array with all red values, then green values, then blue values of length 2**(data.itemsize*8)
# if using TiffWriter.save colormap parameter, the object passed must be shape (3, 2**(data.itemsize*8)) and dtype uint16
# not sure why reversing the color map is appropriate here, but it is
r = colorMap[0:256]
r.reverse()
g = colorMap[256:512]
g.reverse()
b = colorMap[512:]
b.reverse()
#create the object to pass into the colormap parameter of TiffWriter.save
clrmap = numpy.array([r,g,b],dtype=numpy.uint16)
try:
# write out the file
with tf.TiffWriter(output_filename) as mtif:
# data shape should be image depth, height (length), width
# after transposing just take one dimension so that a single page mtif pops out the other side
# have only successfully loaded into ATCF mtif's with photometric (tag 262) as palette. This setting is inferred from the data shape and the value of the colormap.
# setting metadata=None prevents some double writing of tags that can occur during the mtif.save process that the user (me or you) may not expect to be written
mtif.save(data_tbsint.transpose(2,0,1)[0,:,:],colormap=clrmap,description=szDescription,metadata=None,extratags=[(284,'H',1,1,True),(33000,'i',1,nProjection,True),(33001,'i',1,rsStandard1,True),(33002,'i',1,rsStandard2,True),(33003,'i',1,Hemisphere,True),(33004,'i',1,rsULLat,True),(33005,'i',1,rsULLon,True),(33006,'i',1,rsLLLat,True),(33007,'i',1,rsLLLon,True),(33008,'i',1,rsURLat,True),(33009,'i',1,rsURLon,True),(33010,'i',1,rsLRLat,True),(33011,'i',1,rsLRLon,True),(33012,'i',1,rsBCLat,True),(33013,'i',1,rsBCLon,True),(33014,'i',1,rsUCLat,True),(33015,'i',1,rsUCLon,True)])
except Exception as err:
log.info('{0}: {1} >> {2}'.format(type(err).__name__,str(err.__doc__),str(err.args)))
for j in range(num_bacteria):
bac_image = cv2.rectangle(image, (int(bac_cells[j, 0]), int(bac_cells[j, 1])), (int(bac_cells[j, 2]),
int(bac_cells[j, 3])), (0, 255, 0), -1, 8)
bac_image_final = cv2.cvtColor(bac_image, cv2.COLOR_BGR2GRAY)
bac_image_16_reform = bac_image_final.reshape(pixels, resolution, pixels, resolution).sum(3).sum(1)
bac_image_16 = npy.uint16(bac_image_16_reform)
non_zer_find = npy.nonzero(bac_image_16)
bac_image_16[bac_image_16!=0] = 1
filename_bin = filename.replace(".tif", "_binary.tif")
#filename_bin.replace(".tif", "binary.tif")
#print(filename_bin)
cv2.imwrite(filename_bin, bac_image_16)
#Opening big tiff file
with tifffile.TiffWriter(filename, bigtiff=True) as tif:
#Convert image to array
bac_array = npy.array(bac_image_final)
max_cell_value = npy.amax(bac_array)
#Find pixels where bacteria cells are
index_max = npy.argwhere(bac_array == max_cell_value)
#Give cells autofluorescence
for m in range(len(index_max)):
bac_array[index_max[m, 0], index_max[m, 1]] = npy.random.poisson(cell_background, 1)
bac_array_2 = npy.array(bac_array)
for i in range(track_steps):
if i in integrate_array_2:
init_time = npy.where(integrate_array_2 == i)
init_time = init_time[0]
def run(self):
self.timed_filming_done.emit(False)
self.mmc.clearCircularBuffer()
self.mmc.prepareSequenceAcquisition(self.name)
timestr = time.strftime("%m-%d-%Y_%H;%M;%S")
video_name = "Videos/video_" + "{}".format(timestr) + ".tif"
self.frames = 0
if self.mode == "Continuous":
print("Starting continuous video")
self.mmc.startContinuousSequenceAcquisition(self.interval)
start_time = time.time()
while not self.isInterruptionRequested():
if self.mmc.isBufferOverflowed():
print("Circular buffer overflowed")
self.mmc.stopSequenceAcquisition()
break
self.rec_time = time.time() - start_time
self.mmc.stopSequenceAcquisition()
with skimtiff.TiffWriter(video_name, append = True, imagej = True)\
as tif:
while self.mmc.getRemainingImageCount() > 0:
self.image = self.mmc.popNextImage()
self.frames += 1
tif.save(self.image, compress=0)
print("Done writing " + str(self.frames) + " frames, recorded for "\
+ str(round(self.rec_time,1)) + " seconds." )
self.timed_filming_done.emit(True)
elif self.mode == "Timed":
print("Start recording " + str(self.tot_ims) + " frames.")
self.mmc.startContinuousSequenceAcquisition(self.interval)
'''
The self.interval only works for some camera models. It makes no
difference for the Hamamatsu orca flash 4!
'''
start_time = time.time()
while self.mmc.getRemainingImageCount() < self.tot_ims - 1:
if self.mmc.isBufferOverflowed():
print("Circular buffer overflowed")
self.mmc.stopSequenceAcquisition()
break
if self.isInterruptionRequested():
print("Stopped recording manually")
self.mmc.stopSequenceAcquisition()
break
self.mmc.stopSequenceAcquisition()
self.rec_time = time.time() - start_time
with skimtiff.TiffWriter(video_name, append = True, imagej = True) \
as tif:
while self.mmc.getRemainingImageCount() > 0:
self.image = self.mmc.popNextImage()
self.frames += 1
tif.save(self.image, compress=0)
print("Done writing " + str(self.frames) + " frames, recorded for " \
+ str(round(self.rec_time,1)) + " seconds." )
self.timed_filming_done.emit(True)
#----------------------Beta modes and testing functions----------------
elif self.mode == "Timed_v2":
'''
This mode works perfectly fine while testing with the democam, but
for some reason the hamamatsu camera can't hande the
startSequenceAcquisitionfunction if called upon the second time.
This mode would be ideal (as startsequenceacquisition automaticaLly
stops when the circular buffer overflows) but for now can't be used.
'''
print("Starting timed video")
self.mmc.startSequenceAcquisition(self.name, self.tot_ims, \
self.interval, True)
with skimtiff.TiffWriter(video_name, append = True, imagej = True)\
as tif:
for n in range(0, self.tot_ims):
if self.mmc.isBufferOverflowed():
print("Circular buffer overflowed")
self.mmc.stopSequenceAcquisition()
break
elif self.isInterruptionRequested():
print("Stopped recording manually")
self.mmc.stopSequenceAcquisition()
break
while self.mmc.getRemainingImageCount() == 0:
"""
I'm using a for loop, so the if getRemainingImageCount>0
function does not work here (the number of iterations
would not be correct.) Therefore I'm using a while loop
with a sleep. Could also use a pass statement but that
would be more computationally expensive.
"""
time.sleep(0.001) #waiting a ms
self.image = self.mmc.popNextImage()
tif.save(self.image, compress=0, photometric='minisblack')
self.frames += 1
print("Done writing " + str(self.frames) + " frames.")
self.timed_filming_done.emit(True)
elif self.mode == "framerate_tester":
'''
This mode has been built to check the framerate of the camera
without writing out files. Should not be used except for testing.
'''
print("Start recording " + str(self.tot_ims) +
" frames with the tester.")
self.mmc.startContinuousSequenceAcquisition(self.interval)
start_time = time.time()
while self.mmc.getRemainingImageCount() < self.tot_ims - 1:
if self.mmc.isBufferOverflowed():
print("Circular buffer overflowed")
self.mmc.stopSequenceAcquisition()
break
if self.isInterruptionRequested():
print("Stopped recording manually")
self.mmc.stopSequenceAcquisition()
break
self.mmc.stopSequenceAcquisition()
self.rec_time = time.time() - start_time
self.frames = self.mmc.getRemainingImageCount()
print("Done writing " + str(self.frames) + " frames, recorded for " \
+ str(round(self.rec_time,1)) + "seconds")
self.done = True
def process_data(data_path, params):
"""
Process raw data, segment it, and extract features
Arguments:
data_path str The path to raw data
params dict A dictionary of parameters
Return:
pandas.DataFrame The extracted features for each cell
"""
global MATLAB
print("Processing TIFFs for feature extraction...")
data_set = params['data_set']
input_path = params['input_path']
filter_window = params['filter_window']
gamma = params['gamma']
channel = params['channel']
pixel_size = params['pixel_size']
rolling_ball_size = params['rolling_ball_size']
tiff_path = params['tiff_path']
mip_path = params['mip_path']
keep_imgs = params['keep_imgs']
tiff_path.mkdir(mode=0o755, parents=True, exist_ok=True)
raw_path = tiff_path / "8-bit"
raw_path.mkdir(mode=0o755, parents=True, exist_ok=True)
# Get TIFF stacks
files = glob.glob(str(data_path) + "/*.tif")
if len(files) <= 0:
# TODO: Make this...not crappy
files = glob.glob(str(data_path) + "/*.TIF")
if len(files) <= 0:
print("Could not find any TIFF files!")
exit(1)
files.sort(key=lambda x: str(len(x)) + x)
frame_i = 1
for file in files:
with tifffile.TiffFile(file) as tif:
if pixel_size is None and 'XResolution' in tif.pages[0].tags:
pixel_size = tif.pages[0].tags['XResolution'].value
dtype = tif.pages[0].tags['XResolution'].dtype
if len(pixel_size) == 2:
pixel_size = pixel_size[0]
if dtype == '1I':
# Convert from inches to microns
pixel_size = pixel_size * 3.937E-5
elif dtype == '2I':
# Convert from meters to microns
pixel_size = pixel_size * 1E-6
for i in range(len(tif.pages)):
print(" Processing frame " + str(frame_i) + "...")
raw, img = process_image(tif.pages[i].asarray(), channel,
filter_window, gamma, pixel_size,
rolling_ball_size)
file_name = str(frame_i).zfill(4) + ".tif"
tifffile.TiffWriter(str(tiff_path / file_name)).save(
img, resolution=(pixel_size, pixel_size, None))
tifffile.TiffWriter(str(raw_path / file_name)).save(
raw, resolution=(pixel_size, pixel_size, None))
frame_i += 1
# cmd = [
# str(FIJI_PATH),
# "--headless",
# str(PREPROCESSOR_PATH),
# str(data_path) + "/",
# str(tiff_path) + "/",
# "--filter-window=" + str(filter_window),
# "--gamma=" + str(gamma),
# "--channel=" + str(channel),
# "--pixel-size=" + str(pixel_size),
# "--rolling-ball-size=" + str(rolling_ball_size)
# ]
# try:
# subprocess.check_call(cmd)
# except subprocess.CalledProcessError:
# print(colorize("red", "Unable to process TIFFs"))
# exit(1)
frame_paths = list(tiff_path.glob("*.tif"))
# Filter out ._ files OS X likes to make
frame_paths = list(filter(lambda x: x.name[:2] != "._", frame_paths))
frame_paths = [str(x) for x in frame_paths]
frame_paths.sort()
TEMP_PATH.mkdir(mode=0o755, parents=True, exist_ok=True)
tmp_label = str(time())
tmp_csv_path = TEMP_PATH / (tmp_label + ".csv")
tmp_mask_path = TEMP_PATH / (tmp_label)
tmp_mask_path.mkdir(exist_ok=True)
out = io.StringIO()
if MATLAB is None:
print("Starting matlab engine...")
MATLAB = matlab.engine.start_matlab()
print("Running MATLAB feature extraction...")
start = time()
MATLAB.cd(str(MATLAB_PATH))
promise = MATLAB.process_video(frame_paths,
pixel_size,
str(tmp_csv_path),
str(tmp_mask_path),
stdout=out,
background=True)
# MATLAB.process_video(frame_paths, pixel_size, str(tmp_csv_path), str(tmp_mask_path))
bar = progressbar.ProgressBar(term_width=35,
max_value=progressbar.UnknownLength,
widgets=[progressbar.BouncingBar()])
while promise.done() is not True:
bar.update(1)
sleep(0.2)
bar.finish()
print("Finished in " + str((time() - start) / 60) + " min")
data = pd.read_csv(str(tmp_csv_path), dtype={'particle_id': str})
data['x_conversion'] = pixel_size
data['y_conversion'] = pixel_size
data['median'] = data['mean_nuc']
data['area'] = data['area_nuc']
data['sum'] = data['mean_nuc'] * data['area_nuc']
print("Building tracks...")
frames = sorted(data['frame'].unique())
data['min_frame'] = (
data['particle_id'].str.split(".").str[0]).astype('int')
data['orig_particle_id'] = data['particle_id']
for i in tqdm(frames[:-1], ncols=90, unit="frames"):
data = tracks.make_tracks(data, i, 10, 3)
data.drop_duplicates(subset=['particle_id', 'frame'], inplace=True)
data = base_transform(data, params)
# Filter out particles that are too near each other
data = data.loc[(data['nearest_neighbor_distance'] >= 8 * pixel_size), :]
# Build MIP for each particle
# particle_imgs = {} # MIP over the entire video
# ref_particle_imgs = {} # MIP for the first 3 frames
# captured_frames = {} # Number of frames we've captured per pid
# print("Building MIP for each particle...")
# for i in tqdm(frames, ncols=90, unit="frames"):
# mask = np.matrix(sio.loadmat(str(tmp_mask_path / (str(i) + ".mat")))['Lnuc'], dtype=np.uint8)
# img = cv2.imread(frame_paths[(i-1)], cv2.IMREAD_GRAYSCALE)
# for pid in data.loc[(data['frame'] == i), 'particle_id']:
# # Convert the mask ID (which uses pre-track-building IDs)
# # to the new ID as built in tracks.make_tracks
# matlab_id = data.loc[( (data['frame'] == i) & (data['particle_id'] == pid) ), 'orig_particle_id']
# matlab_id = int(matlab_id.iloc[0].split(".")[1])
# this_mask = mask.copy()
# this_mask[this_mask != matlab_id] = 0
# this_mask[this_mask == matlab_id] = 1
# if pid not in particle_imgs:
# particle_imgs[pid] = np.zeros((500,500), dtype=np.uint8)
# captured_frames[pid] = 0
# # Get just the masked nucleus
# fg = cv2.bitwise_and(img, img, mask=this_mask)
# # Crop to 500x500, centered on the nuclear mask
# coords = data.loc[( (data['frame'] == i) & (data['particle_id'] == pid) ), [ 'x', 'y' ]]
# x = int(round(coords['x'].iloc[0]/pixel_size))
# y = int(round(coords['y'].iloc[0]/pixel_size))
# fg = hatchvid.crop_frame(fg, x, y, 500, 500)
# # Make a MIP of the previous MIP and this img
# particle_imgs[pid] = np.amax([ particle_imgs[pid], fg ], axis=0)
# captured_frames[pid] += 1
# if captured_frames[pid] < 3:
# # Make the reference frame
# ref_particle_imgs[pid] = particle_imgs[pid].copy()
# if keep_imgs:
# img_path = (tiff_path / "cells" / pid).resolve()
# img_path.mkdir(parents=True, exist_ok=True)
# cv2.imwrite(str(img_path / (str(i) + ".tif")), fg)
# data['mip_sum'] = 0.0
# data['mip_cyto_sum'] = 0.0
# data['mip_normalized_sum'] = 0.0
# for pid, img in particle_imgs.items():
# idx = (data['particle_id'] == pid)
# data.loc[idx, 'mip_sum'] = np.sum(img)
# mask = ref_particle_imgs[pid]
# threshold, mask = cv2.threshold(mask, 1, 255, cv2.THRESH_BINARY_INV)
# cyto = cv2.bitwise_and(img, img, mask=mask)
# data.loc[idx, 'mip_cyto_sum'] = np.sum(cyto)
# data.loc[idx, 'mip_normalized_sum'] = np.sum(cyto)/np.sum(img)
# # Clear out the old images
# if mip_path.exists():
# shutil.rmtree(mip_path)
# mip_path.mkdir(mode=0o755, parents=True, exist_ok=True)
# # Write out our MIPs
# for pid, img in particle_imgs.items():
# cv2.imwrite(str(mip_path / (pid + ".tif")), img)
# cv2.imwrite(str(mip_path / (pid + "-ref.tif")), ref_particle_imgs[pid])
if tmp_csv_path.exists():
tmp_csv_path.unlink()
if tmp_mask_path.exists():
shutil.rmtree(tmp_mask_path)
return data
def FluorescenceAnalysis(self, folder, round_num, save_mask=True):
"""
# =============================================================================
# Given the folder and round number, return a dictionary for the round
# that contains each scanning position as key and structured array of detailed
# information about each identified cell as content.
#
# Returned structured array fields:
# - BoundingBox of cell ROI
# - Mean intensity of whole cell area
# - Mean intensity of cell membrane part
# - Contour soma ratio
# =============================================================================
Parameters
----------
folder : string.
The directory to folder where the screening data is stored.
round_num : string.
The target round number of analysis.
save_mask: bool.
Whether to save segmentation masks.
Returns
-------
cell_Data : pd.DataFrame.
Sum of return from func: retrieveDataFromML, for whole round.
"""
RoundNumberList, CoordinatesList, fileNameList = self.retrive_scanning_scheme(
folder, file_keyword="Zmax")
# RoundNumberList, CoordinatesList, fileNameList = self.retrive_scanning_scheme(folder, file_keyword = 'Zfocus')
if not os.path.exists(
os.path.join(folder, "MLimages_{}".format(round_num))):
# If the folder is not there, create the folder to store ML segmentations
os.mkdir(os.path.join(folder, "MLimages_{}".format(round_num)))
for EachRound in RoundNumberList:
cells_counted_in_round = 0
background_substraction = False
# =============================================================================
# For background_substraction
# =============================================================================
# If background images are taken
background_images_folder = os.path.join(
folder, "background {}".format(EachRound))
# print(background_images_folder)
if os.path.exists(background_images_folder):
# If the background image is taken to substract out
background_substraction = True
print("Run background substraction.")
# Get all the background files names
background_fileNameList = []
for file in os.listdir(background_images_folder):
if "calculated background" not in file:
if "tif" in file or "TIF" in file:
background_fileNameList.append(
os.path.join(background_images_folder, file))
background_image = ProcessImage.image_stack_calculation(
background_fileNameList, operation="mean")
# # Smooth the background image
# background_image = ProcessImage.average_filtering(
# background_image, filter_side_length = 25)
# Save the individual file.
with skimtiff.TiffWriter(
os.path.join(background_images_folder,
"calculated background.tif"),
imagej=True,
) as tif:
tif.save(background_image.astype(np.uint16), compress=0)
if EachRound == round_num:
# Start numbering cells at each round
self.cell_counted_inRound = 0
for EachCoord in CoordinatesList:
# =============================================================================
# For fluorescence:
# =============================================================================
print(EachCoord)
# -------------- readin image---------------
for Eachfilename in enumerate(fileNameList):
if (EachCoord in Eachfilename[1]
and EachRound in Eachfilename[1]):
if "Zmax" in Eachfilename[1]:
try:
ImgNameInfor = Eachfilename[1][
0:Eachfilename[1].index(
"_PMT"
)] # get rid of '_PMT_0Zmax.tif' in the name.
except:
ImgNameInfor = Eachfilename[1][
0:Eachfilename[1].index(
"_Cam"
)] # get rid of '_Cam_Zmax.tif' in the name.
elif "Zfocus" in Eachfilename[1]:
ImgNameInfor = Eachfilename[1][
0:len(Eachfilename[1]) -
16] # get rid of '_PMT_0Zfocus.tif' in the name.
elif "Zpos1" in Eachfilename[1]:
ImgNameInfor = Eachfilename[1][0:len(
Eachfilename[1]
)] # get rid of '_PMT_0Zfocus.tif' in the name.
_imagefilename = os.path.join(
folder, Eachfilename[1])
# ------------------------------------------
# =========================================================================
# USING MASKRCNN...
# =========================================================================
# Imagepath = self.Detector._fixPathName(_imagefilename)
Rawimage = imread(_imagefilename)
# Background substraction
if background_substraction == True:
Rawimage = np.abs(Rawimage - background_image)
camera_dark_level = 100
# # Normalize to the illumination intensity
# Rawimage = np.uint16(Rawimage \
# / ((background_image - camera_dark_level)\
# /(np.amin(background_image) - camera_dark_level)))
# if ClearImgBef == True:
# # Clear out junk parts to make it esaier for ML detection.
# RawimageCleared = self.preProcessMLimg(Rawimage, smallest_size=300, lowest_region_intensity=0.16)
# else:
# RawimageCleared = Rawimage.copy()
image = ProcessImage.convert_for_MaskRCNN(Rawimage)
# Run the detection on input image.
results = self.Detector.detect([image])
MLresults = results[0]
if save_mask == True:
fig, ax = plt.subplots()
# Set class_names = [None,None,None,None] to mute class name display.
visualize.display_instances(
image,
MLresults["rois"],
MLresults["masks"],
MLresults["class_ids"],
class_names=[None, None, None, None],
ax=ax,
centre_coors=MLresults["Centre_coor"],
Centre_coor_radius=2,
WhiteSpace=(0, 0),
) # MLresults['class_ids'],MLresults['scores'],
# ax.imshow(fig)
fig.tight_layout()
# Save the detection image
fig_name = os.path.join(
folder, "MLimages_{}\{}.tif".format(
round_num, ImgNameInfor))
plt.savefig(fname=fig_name,
dpi=200,
pad_inches=0.0,
bbox_inches="tight")
# segmentationImg = Image.fromarray(fig) #generate an image object
# segmentationImg.save(os.path.join(folder, 'MLimages_{}\{}.tif'.format(round_num, ImgNameInfor)))#save as tif
# Use retrieveDataFromML from ImageProcessing.py to extract numbers.
if self.cell_counted_inRound == 0:
(
cell_Data,
self.cell_counted_inRound,
total_cells_counted_in_coord,
) = ProcessImage.retrieveDataFromML(
Rawimage,
MLresults,
str(ImgNameInfor),
self.cell_counted_inRound,
show_each_cell=False)
else:
(
Cell_Data_new,
self.cell_counted_inRound,
total_cells_counted_in_coord,
) = ProcessImage.retrieveDataFromML(
Rawimage,
MLresults,
str(ImgNameInfor),
self.cell_counted_inRound,
show_each_cell=False)
if len(Cell_Data_new) > 0:
cell_Data = cell_Data.append(Cell_Data_new)
# Count in total how many flat and round cells are identified.
cells_counted_in_round += total_cells_counted_in_coord
print("Number of round/flat cells in this round: {}".format(
cells_counted_in_round))
# Save to excel
cell_Data.to_excel(
os.path.join(
os.path.join(
folder,
round_num + "_" +
datetime.now().strftime("%Y-%m-%d_%H-%M-%S") +
"_CellsProperties.xlsx",
)))
return cell_Data
def main(bq, prob_map_dir, outdir, testing_data_dir, min_distance,
label_threshold, black_threshold):
table_service = bq.service('table')
try:
import tables
except ImportError:
logging.warn("pytables services not available")
#some hyperparameters
minimum = 50
directory = prob_map_dir
blk_threshold = 0.05
min_dis = min_distance
label_thresh = label_threshold
files = os.listdir(directory)
original_input = testing_data_dir + os.listdir(testing_data_dir)[0]
output_files = []
with tifffile.TiffFile(original_input) as tiff:
imMeta = tiff.is_imagej
for file in files:
print file
#slice_ind = seeds_slice_id # input('Please input the slice number for seeds:')
blk_threshold = black_threshold #input('Please input the threhold for black voxels')
#slice_ind = slice_ind-1
path = os.path.join(directory, file)
img = sitk.ReadImage(path)
img = sitk.GetArrayFromImage(img)
img = img.astype('float32')
seg_new = img / np.amax(img)
slice_ind = slice_det(seg_new)
seg_new[seg_new < blk_threshold] = 0
mid_slice = seg_new[slice_ind]
mask_mid = peak_local_max(-mid_slice,
min_distance=min_dis,
indices=False,
exclude_border=1)
#plt.imshow(mid_slice,cmap='gray')
#plt.show()
mask_mid = mask_mid * 1
mask_mid = binary_opening(1 - mask_mid)
mask_mid = binary_closing(mask_mid)
distance = ndi.distance_transform_edt(1 - mask_mid)
#mask_mid = peak_local_max(distance,min_distance=30,indices=False,threshold_rel=0.2)
mask_mid_bin = np.zeros_like(distance)
mask_mid_bin[distance > label_thresh] = 1
masks = label(mask_mid_bin)
#masks_img = sitk.GetImageFromArray((masks*255).astype('uint8'))
#sitk.WriteImage(masks_img,'/home/tom/result/'+filename+'det.png')
uni, counts = np.unique(masks, return_counts=True)
for i in uni:
if counts[i] < minimum:
masks[masks == i] = 0
uni, counts = np.unique(masks, return_counts=True)
mask_temp = masks
masks = mask_temp
#plt.imshow(masks)
#plt.show()
#mask = watershed(seg_new[slice_ind],masks)
#one_cell = np.zeros_like(seg_new)
#one_cell[slice_ind] = mask
#masks = propagate(seg_new,one_cell,slice_ind)
mask_3d = np.zeros_like(seg_new)
mask_3d[slice_ind] = masks
#last = np.zeros((512,512))
#first = np.zeros((512,512))
#last[400:402,259:262]=30
#first[375:380,245:250]=1
#mask_3d[5]=last
#mask_3d[18*5]=last
#mask_3d[0]=first
#masks_img = sitk.GetImageFromArray((masks*255).astype('uint8'))
#sitk.WriteImage(masks_img,'/home/tom/result/'+'seeds.png')
#masks = random_walker(seg_new,mask_3d,beta=10,mode='bf')
masks = watershed(seg_new, mask_3d, watershed_line=True)
#CRF
#distance_one_cell = ndi.distance_transform_edt(one_cell)
#distance_one_cell[distance_one_cell>15] = np.amax(distance_one_cell)
#distance_one_cell = distance_one_cell/np.amax(distance_one_cell)
#prob_map = np.multiply(1-prob_map,one_cell)
#pro = CRFProcessor.CRF3DProcessor()
#seg_new = np.transpose(prob_map,(1,2,0))
#labels = np.zeros((512,512,12,2))
#seg_new[seg_new>0.01] = 1
#labels[:,:,:,0] = seg_new
#labels[:,:,:,1] = 1-seg_new
#distance_one_cell = np.transpose(distance_one_cell,(1,2,0))
#result = pro.set_data_and_run(seg_new,labels)
#result = np.transpose(result,(2,0,1))
#print np.unique(result)
#plt.imshow(mask_mid_bin)
#plt.show()
#masks = resize(masks,(masks.shape[0]/5,masks.shape[1],masks.shape[2]),mode='constant')
#masks_img = sitk.GetImageFromArray((masks).astype('uint8'))
#file_name = os.path.splitext(file)[0]
#outfile = os.path.join(outdir,file_name+'-seg.tif')
#sitk.WriteImage(masks_img,outfile)
with tifffile.TiffWriter('source/result/' + file + 'seg.tif') as tif:
for i in range(masks.shape[0]):
tif.save(masks[i], extratags=[(270, 's', 1, imMeta)])
labeled_image = label_segmentation('source/result/' + file + 'seg.tif')
num_cells = len(np.unique(masks)) - 1
coordinates = seg_img_coord(masks)
#for label in np.unique(seg):
adj_table = compute_cell_adjacent_table(masks)
adj_table_done = tuple([(k, v) for k, v in adj_table.items()])
#df = pd.DataFrame(adj_table)
#df.to_csv("result/adj_table.csv")
center = cell_center(masks)
points = compute_conjunction_points(masks, adj_table)
points_done = tuple([(k, [x for xs in v for x in xs])
for k, v in points.items()])
#np.savetxt("result/points.csv", points, delimiter=",")
cell_vol = cell_volumn(masks)
cell_vol_done = tuple([(k, v) for k, v in cell_vol.items()])
#df1 = pd.DataFrame(cell_volumn(masks))
#df1.to_csv("result/cell_volumn.csv")
output_files.append('source/result/' + file + 'seg.tif')
tfu = adj_table.values()
b = np.full([len(tfu), len(max(tfu, key=lambda x: len(x)))],
fill_value=np.nan)
for i, j in enumerate(tfu):
b[i][0:len(j)] = j
with h5py.File('source/hdf/PlantCellSegmentation.h5', 'w') as hf:
grp0 = hf.create_group("Cell Labels")
grp1 = hf.create_group("Surface Coordinates")
grp2 = hf.create_group("Cell Center")
grp3 = hf.create_group("Cell Volume")
grp4 = hf.create_group("Adjacency Table")
grp5 = hf.create_group("Segmented Image")
grp2.create_dataset("Cell Center",
data=(np.array((center.values()))))
grp3.create_dataset("Cell Volume",
data=(np.array((cell_vol.values()))))
grp4.create_dataset("Adjacency Table", data=b)
grp0.create_dataset("Cell Labels", data=np.array((center.keys())))
grp5.create_dataset("Segmented Image",
data=masks,
compression='gzip',
compression_opts=9)
for ix in range(num_cells):
grp1.create_dataset("Cell Label: {}".format(ix),
compression='gzip',
compression_opts=9,
data=(np.array(
(coordinates.values())))[ix])
#outtable_xml_adj_table = table_service.store_array(adj_table_done, name='adj_table')
#outtable_xml_points = table_service.store_array(points_done, name='points')
#outtable_xml_cell_vol = table_service.store_array(cell_vol_done, name='cell_vol')
#print(output_files)
return output_files, adj_table, points, cell_vol, coordinates, center
def evaluate_focus(self, obj_position = None):
"""
Evaluate the focus degree of certain objective position.
Parameters
----------
obj_position : float, optional
The target objective position. The default is None.
Returns
-------
degree_of_focus : float
Degree of focus.
"""
if obj_position != None:
self.pi_device_instance.move(obj_position)
# Get the image.
if self.source_of_image == "PMT":
self.galvo_image = self.galvo.run()
plt.figure()
plt.imshow(self.galvo_image)
plt.show()
if False:
with skimtiff.TiffWriter(os.path.join(r'M:\tnw\ist\do\projects\Neurophotonics\Brinkslab\Data\Xin\2020-11-17 gaussian fit auto-focus cells\trial_11', str(obj_position).replace(".", "_")+ '.tif')) as tif:
tif.save(self.galvo_image.astype('float32'), compress=0)
degree_of_focus = ProcessImage.local_entropy(self.galvo_image.astype('float32'))
elif self.source_of_image == "Camera":
# First configure the AOTF.
self.AOTF_runner = DAQmission()
# Find the AOTF channel key
for key in self.imaging_conditions:
if 'AO' in key:
# like '488AO'
AOTF_channel_key = key
# Set the AOTF first.
self.AOTF_runner.sendSingleDigital('blankingall', True)
self.AOTF_runner.sendSingleAnalog(AOTF_channel_key, self.imaging_conditions[AOTF_channel_key])
# Snap an image from camera
self.camera_image = self.HamamatsuCam_ins.SnapImage(self.imaging_conditions['exposure_time'])
time.sleep(0.5)
# Set back AOTF
self.AOTF_runner.sendSingleDigital('blankingall', False)
self.AOTF_runner.sendSingleAnalog(AOTF_channel_key, 0)
plt.figure()
plt.imshow(self.camera_image)
plt.show()
if False:
with skimtiff.TiffWriter(os.path.join(r'M:\tnw\ist\do\projects\Neurophotonics\Brinkslab\Data\Xin\2021-03-06 Camera AF\beads', str(obj_position).replace(".", "_")+ '.tif')) as tif:
tif.save(self.camera_image.astype('float32'), compress=0)
degree_of_focus = ProcessImage.variance_of_laplacian(self.camera_image.astype('float32'))
time.sleep(0.2)
return degree_of_focus
BASEDIR = os.path.dirname(os.path.abspath(__file__))
RAWDIR = os.path.join(BASEDIR, 'raw')
OUTDIR = os.path.join(BASEDIR, 'out')
if __name__ == '__main__':
######################################################################################
# user settings
# file type specifier
FILETYPE = '/*.tif'
# specify glob directory: all file matching FILETYPE will be considered
IMAGEDIR = RAWDIR # glob directory
outname = 'stack.tif'
# Takes all tif images in a specified directory and stacks them together.
# As of now, this will create a z-stack from planar 2d images without calibration.
######################################################################################
ensure_dir(os.path.join(BASEDIR, OUTDIR))
with tifffile.TiffWriter(os.path.join(OUTDIR, outname)) as stack:
for filename in glob.glob(IMAGEDIR + FILETYPE):
# the photometric keyword specifies the image colorspace
stack.save(tifffile.imread(filename), photometric='minisblack')
def PMT_image_processing(self):
"""
Reconstruct the image from np array and save it.
Returns
-------
None.
"""
for imageSequence in range(self.repeatnum):
try:
self.PMT_image_reconstructed_array = self.data_collected_0[np.where(self.PMT_data_index_array == imageSequence+1)]
Dataholder_average = np.mean(self.PMT_image_reconstructed_array.reshape(self.averagenum, -1), axis=0)
Value_yPixels = int(self.lenSample_1/self.ScanArrayXnum)
self.PMT_image_reconstructed = np.reshape(Dataholder_average, (Value_yPixels, self.ScanArrayXnum))
self.PMT_image_reconstructed = self.PMT_image_reconstructed[:, 50:550] # Crop size based on: M:\tnw\ist\do\projects\Neurophotonics\Brinkslab\Data\Xin\2019-12-30 2p beads area test 4um
# self.PMT_image_reconstructed = self.PMT_image_reconstructed[:, 70:326] # for 256*256 images
# Evaluate the focus degree of re-constructed image.
self.FocusDegree_img_reconstructed = ProcessImage.local_entropy(self.PMT_image_reconstructed.astype('float32'))
print('FocusDegree_img_reconstructed is {}'.format(self.FocusDegree_img_reconstructed))
# Save the individual file.
with skimtiff.TiffWriter(os.path.join(self.scansavedirectory, 'Round'+str(self.RoundWaveformIndex[0]) + '_Grid' + str(self.Grid_index) +'_Coords'+str(self.currentCoordsSeq)+'_R'+str(self.CurrentPosIndex[0])+'C'+str(self.CurrentPosIndex[1])+'_PMT_'+str(imageSequence)+'Zpos'+str(self.ZStackOrder)+'.tif'), imagej = True) as tif:
tif.save(self.PMT_image_reconstructed.astype('float32'), compress=0, metadata = {"FocusPos: " : str(self.FocusPos)})
plt.figure()
plt.imshow(self.PMT_image_reconstructed, cmap = plt.cm.gray) # For reconstructed image we pull out the first layer, getting 2d img.
plt.show()
#---------------------------------------------For multiple images in one z pos, Stack the arrays into a 3d array--------------------------------------------------------------------------
# if imageSequence == 0:
# self.PMT_image_reconstructed_stack = self.PMT_image_reconstructed[np.newaxis, :, :] # Turns into 3d array
# else:
# self.PMT_image_reconstructed_stack = np.concatenate((self.PMT_image_reconstructed_stack, self.PMT_image_reconstructed[np.newaxis, :, :]), axis=0)
# print(self.PMT_image_reconstructed_stack.shape)
#---------------------------------------------Calculate the z max projection-----------------------------------------------------------------------
if self.repeatnum == 1: # Consider one repeat image situlation
if self.ZStackNum > 1:
if self.ZStackOrder == 1:
self.PMT_image_maxprojection_stack = self.PMT_image_reconstructed[np.newaxis, :, :]
else:
self.PMT_image_maxprojection_stack = np.concatenate((self.PMT_image_maxprojection_stack, self.PMT_image_reconstructed[np.newaxis, :, :]), axis=0)
else:
self.PMT_image_maxprojection_stack = self.PMT_image_reconstructed[np.newaxis, :, :]
# Save the max projection image
if self.ZStackOrder == self.ZStackNum:
self.PMT_image_maxprojection = np.max(self.PMT_image_maxprojection_stack, axis=0)
# Save the zmax file.
with skimtiff.TiffWriter(os.path.join(self.scansavedirectory, 'Round'+str(self.RoundWaveformIndex[0])+ '_Grid' + str(self.Grid_index) + '_Coords'+str(self.currentCoordsSeq)+'_R'+str(self.CurrentPosIndex[0])+'C'+str(self.CurrentPosIndex[1])+'_PMT_'+str(imageSequence)+'Zmax'+'.tif'), imagej = True) as tif:
tif.save(self.PMT_image_maxprojection.astype('float32'), compress=0, metadata = {"FocusPos: " : str(self.FocusPos)})
except:
print('No.{} image failed to generate.'.format(imageSequence))
def process_data(data_path, params):
"""
Process raw data, segment it, and extract features
Arguments:
data_path str The path to raw data
params dict A dictionary of parameters
Return:
pandas.DataFrame The extracted features for each cell
"""
data_set = params['data_set']
input_path = params['input_path']
channel = params['channel']
pixel_size = params['pixel_size']
tiff_path = params['tiff_path']
mip_path = params['mip_path']
keep_imgs = params['keep_imgs']
tiff_path.mkdir(mode=0o755, parents=True, exist_ok=True)
mip_path.mkdir(mode=0o755, parents=True, exist_ok=True)
raw_path = tiff_path / "8-bit"
raw_path.mkdir(mode=0o755, parents=True, exist_ok=True)
# Get TIFF stacks
files = list(data_path.glob("*.tif"))
if len(files) <= 0:
# TODO: Make this...not crappy
files = list(data_path.glob("*.TIF"))
# Filter out ._ files OS X likes to make
files = list(filter(lambda x: x.name[:2] != "._", files))
if len(files) <= 0:
print("Could not find any TIFF files!")
exit(1)
files = [ str(x) for x in files ]
files.sort(key=lambda x: str(len(x)) + x)
# Frame data to store
frame_shape = None
data = {
'particle_id': [],
'mask_id': [],
'frame': [],
'x': [],
'x_px': [],
'y': [],
'y_px': [],
'area': [],
'mean': [],
'min': [],
'max': []
}
frame_i = 1
all_masks = []
raw_paths = []
with yaspin(text="Processing TIFFs for feature extraction") as spinner:
spinner.spinner = Spinners.dots8
for file in files:
with tifffile.TiffFile(file) as tif:
if pixel_size is None and 'XResolution' in tif.pages[0].tags:
pixel_size = tif.pages[0].tags['XResolution'].value
dtype = tif.pages[0].tags['XResolution'].dtype
if len(pixel_size) == 2:
pixel_size = pixel_size[0]
if dtype == '1I':
# Convert from inches to microns
pixel_size = pixel_size*3.937E-5
elif dtype == '2I':
# Convert from meters to microns
pixel_size = pixel_size*1E-6
for i in range(len(tif.pages)):
spinner.text = "Processing TIFFs for feature extraction (frame " + str(frame_i) + ")"
img = tif.pages[i].asarray()
# Get the signal channel
if len(img.shape) == 3:
# channel is 1-indexed, python is 0-indexed
img = img[:,:, (channel-1)]
# Pre-processing
sys.stdout = open(os.devnull, 'w') # Suppress print
i_norm = intensity_normalization(img, [ 10.0, 5.0 ])
i_smooth = image_smoothing_gaussian_3d(i_norm, sigma=1)
sys.stdout = sys.__stdout__
# Store frame size for later
if frame_shape is None:
frame_shape = img.shape
# Expand image to 3D
i_smooth = np.repeat(i_smooth[np.newaxis, :, :], 3, axis=0)
# Masked object thresholding
pre_seg_1, mo_mask = MO(i_smooth, 'ave', 100, extra_criteria=True, return_object=True)
# S2 filter for detecting extra spots
extra_spots = dot_2d_slice_by_slice_wrapper(i_smooth, [[ 2, 0.025 ]])
pre_seg_2 = np.logical_or(pre_seg_1, extra_spots)
# S2 filter for detecting dark spots
dark_spots = dot_2d_slice_by_slice_wrapper(1-i_smooth, [[ 2, 0.025], [1, 0.025]])
pre_seg_2[dark_spots > 0] = 0
# Size filtering
seg = morphology.remove_small_objects(pre_seg_2>0, min_size=400, connectivity=1, in_place=False)
# Return to 2D
seg = seg[1, :, :]
i_smooth = i_smooth[1,:,:]
i_norm_8bit = exposure.rescale_intensity(i_norm, out_range=( 0, 255 )).astype(np.uint8)
i_smooth_8bit = exposure.rescale_intensity(i_smooth, out_range=( 0, 255 )).astype(np.uint8)
# Label regions
masks = measure.label(seg)
# Read props
props = measure.regionprops(masks, intensity_image=i_norm_8bit)
# Perform a flood fill on all segments
st_dev = np.std(i_smooth_8bit)
for region in props:
centroid = np.round(region.centroid).astype(np.uint32)
new_mask = segmentation.flood(i_smooth_8bit, ( centroid[0], centroid[1] ), tolerance=st_dev/2)
new_mask = ndi.morphology.binary_fill_holes(new_mask)
new_mask = morphology.binary_closing(new_mask,selem=morphology.disk(4))
# Sanity check the size of our flood
new_mask = measure.label(new_mask)
new_props = measure.regionprops(new_mask, intensity_image=i_norm_8bit)
if len(new_props) <= 0:
continue
new_region = new_props[0]
max_flood_size = (pixel_size**2)*10000 if pixel_size is not None else 10000
region_size = new_region.area*(pixel_size**2) if pixel_size is not None else new_region.area
if region_size > max_flood_size:
# If our flood is bigger than a 10000 um^2
continue
# Update masks
masks[( (new_mask == 1) | (masks == region.label) )] = region.label
all_masks.append(masks.copy())
# Read props
props = measure.regionprops(masks, intensity_image=i_norm_8bit)
for region in props:
data['particle_id'].append(str(frame_i) + "." + str(region.label))
data['mask_id'].append(region.label)
data['frame'].append(frame_i)
data['x'].append(region.centroid[1])
data['x_px'].append(int(region.centroid[1]))
data['y'].append(region.centroid[0])
data['y_px'].append(int(region.centroid[0]))
data['area'].append(region.area)
data['mean'].append(region.mean_intensity)
data['min'].append(region.min_intensity)
data['max'].append(region.max_intensity)
# Write out normalized and smoothed images
file_name = str(frame_i).zfill(4) + ".tif"
tifffile.TiffWriter(str(tiff_path / file_name)).save(i_smooth_8bit, resolution=(pixel_size, pixel_size, None))
tifffile.TiffWriter(str(raw_path / file_name)).save(i_norm_8bit, resolution=(pixel_size, pixel_size, None))
raw_paths.append(raw_path / file_name)
frame_i += 1
spinner.ok("✅")
all_masks = np.stack(all_masks, axis=0)
data = pd.DataFrame(data)
data = data.astype({
'particle_id': 'str',
'mask_id': 'int',
'frame': 'int',
'x_px': 'int',
'y_px': 'int'
})
# Convert pixels to um
if pixel_size is None:
pixel_size = 1
data['unit_conversion'] = 'px'
else:
data['unit_conversion'] = 'um/px'
data['area'] = data['area']*(pixel_size**2)
data['x'] = data['x']*pixel_size
data['y'] = data['y']*pixel_size
data['x_conversion'] = pixel_size
data['y_conversion'] = pixel_size
# Fake median and sum
data['median'] = data['mean']
data['sum'] = data['area']*data['mean']
# Assign particles to tracks
print("Building tracks...")
frames = sorted(data['frame'].unique())
data['orig_particle_id'] = data['particle_id']
data['min_frame'] = data['frame']
for i in tqdm(frames[:-1], ncols=90, unit="frames"):
data = tracks.make_tracks(data, i, 10, 3)
data.drop_duplicates(subset=[ 'particle_id', 'frame' ], inplace=True)
data.drop('min_frame', axis='columns', inplace=True)
# "Fill in" gaps where we lost tracking
data.sort_values(by=[ 'particle_id', 'frame' ], inplace=True)
data['track_id'] = 0
data = data.groupby([ 'particle_id' ]).apply(id_track)
missing_particle_ids = data.loc[( data['track_id'] > 0 ), 'particle_id'].unique()
if len(missing_particle_ids) > 0:
print("Interpolating cell masks...")
missing_data = None
for particle_id in tqdm(missing_particle_ids, ncols=90, unit="cells"):
p_data = data.loc[(data['particle_id'] == particle_id),:]
# Find the frame sets that we are missing
missing_frames = np.setdiff1d(frames, p_data['frame'].unique())
missing_frames = pd.DataFrame({
'frame': missing_frames
}, dtype='int')
missing_frames.sort_values(by=[ 'frame' ])
missing_frames['track_id'] = (missing_frames['frame'].diff() > 1).cumsum()
missing_frames.drop_duplicates(subset=[ 'track_id' ], inplace=True)
orig_particle_id = p_data['orig_particle_id'].iloc[0]
for missing_frame in missing_frames['frame'].unique():
ref_frame = missing_frame-1
if ref_frame not in p_data['frame'].unique():
continue
# Interpolate data
missing_data = {
'particle_id': [],
'mask_id': [],
'frame': [],
'x': [],
'x_px': [],
'y': [],
'y_px': [],
'area': [],
'mean': [],
'min': [],
'max': [],
'orig_particle_id': [],
'track_id': [],
}
mask_id = p_data.loc[( p_data['frame'] == ref_frame ), 'mask_id'].iloc[0]
# Find ending frame for this gap
# If there is no end, skip
stop_frame = p_data.loc[(p_data['frame'] > missing_frame), 'frame'].unique()
if len(stop_frame) <= 0:
continue
stop_frame = np.min(stop_frame)
if stop_frame == np.max(frames):
continue
missing_frame -= 1
while(missing_frame < stop_frame):
# Advance
ref_frame = missing_frame
missing_frame += 1
file_name = str(missing_frame).zfill(4) + ".tif"
i_norm_8bit = cv2.imread(str(raw_path / file_name), cv2.IMREAD_GRAYSCALE)
i_smooth_8bit = cv2.imread(str(tiff_path / file_name), cv2.IMREAD_GRAYSCALE)
# Frame is 1-indexed, but all_masks is 0-indexed
masks = all_masks[(ref_frame-1),:,:].copy()
masks[(masks != mask_id)] = 0
props = measure.regionprops(masks, intensity_image=i_norm_8bit)
if len(props) <= 0:
continue
region = props[0]
st_dev = np.std(i_smooth_8bit)
centroid = np.round(region.centroid).astype(np.uint32)
new_mask = segmentation.flood(i_smooth_8bit, ( centroid[0], centroid[1] ), tolerance=st_dev/2)
new_mask = ndi.morphology.binary_fill_holes(new_mask)
new_mask = morphology.binary_closing(new_mask,selem=morphology.disk(4))
new_mask = measure.label(new_mask)
props = measure.regionprops(new_mask, intensity_image=i_norm_8bit)
region = props[0]
max_flood_size = (pixel_size**2)*10000
if region.area*(pixel_size**2) > max_flood_size:
# If our flood is bigger than a 10000 um^2
continue
# Update masks stack
masks = all_masks[(missing_frame-1),:,:].copy()
masks[(new_mask == mask_id)] = mask_id
all_masks[(missing_frame-1),:,:] = masks
# Add in new data
missing_data['particle_id'].append(particle_id)
missing_data['mask_id'].append(mask_id)
missing_data['frame'].append(missing_frame)
missing_data['x'].append(region.centroid[1])
missing_data['x_px'].append(int(region.centroid[1]))
missing_data['y'].append(region.centroid[0])
missing_data['y_px'].append(int(region.centroid[0]))
missing_data['area'].append(region.area)
missing_data['mean'].append(region.mean_intensity)
missing_data['min'].append(region.min_intensity)
missing_data['max'].append(region.max_intensity)
missing_data['orig_particle_id'].append(orig_particle_id)
missing_data['track_id'] = 0
if missing_data is not None:
missing_data = pd.DataFrame(missing_data)
missing_data = missing_data.astype({
'particle_id': 'str',
'mask_id': 'int',
'frame': 'int',
'x_px': 'int',
'y_px': 'int'
})
missing_data['unit_conversion'] = data['unit_conversion'].iloc[0]
missing_data['area'] = missing_data['area']*(pixel_size**2)
missing_data['x'] = missing_data['x']*pixel_size
missing_data['y'] = missing_data['y']*pixel_size
missing_data['x_conversion'] = pixel_size
missing_data['y_conversion'] = pixel_size
missing_data['median'] = missing_data['mean']
missing_data['sum'] = missing_data['area']*missing_data['mean']
data = pd.concat([ data, missing_data ], sort=False, ignore_index=True)
data.sort_values(by=[ 'particle_id', 'frame' ], inplace=True)
data['track_id'] = 0
data = data.groupby([ 'particle_id' ]).apply(id_track)
params['frame_width'] = frame_shape[1]
params['frame_height'] = frame_shape[0]
data = base_transform(data, params)
# Build MIP for each particle
particle_imgs = {} # MIP over the entire video
ref_particle_imgs = {} # MIP for the first 3 frames
captured_frames = {} # Number of frames we've captured per pid
print("Building MIP for each particle...")
prev_img = None
for i in tqdm(frames, ncols=90, unit="frames"):
masks = all_masks[(i-1),:,:].copy()
img = cv2.imread(str(raw_paths[(i-1)]), cv2.IMREAD_GRAYSCALE)
pids = data.loc[( data['frame'] == i ), 'particle_id'].unique()
for pid in pids:
mask = masks.copy()
mask_ids = data.loc[( (data['particle_id'] == pid) & (data['frame'] == i) ), 'mask_id'].unique()
if len(mask_ids) <= 0:
continue
mask_id = mask_ids[0]
mask[( mask != mask_id )] = 0
mask[( mask == mask_id )] = 1
mask = mask.astype(np.uint8)
if pid not in particle_imgs:
particle_imgs[pid] = np.zeros((200,200), dtype=np.uint8)
captured_frames[pid] = 0
# Get just the masked nucleus
fg = cv2.bitwise_and(img, img, mask=mask)
# Crop to 200x200, centered on the nuclear mask
coords = data.loc[( (data['frame'] == i) & (data['particle_id'] == pid) ), [ 'x_px', 'y_px' ]]
x = coords['x_px'].iloc[0]
y = coords['y_px'].iloc[0]
fg = hatchvid.crop_frame(fg, x, y, 200, 200)
# Make a MIP of the previous MIP and this img
particle_imgs[pid] = np.amax([ particle_imgs[pid], fg ], axis=0)
captured_frames[pid] += 1
if captured_frames[pid] < 3:
# Make the reference frame
ref_particle_imgs[pid] = particle_imgs[pid].copy()
# Clear out the old images
if mip_path.exists():
shutil.rmtree(mip_path)
mip_path.mkdir(mode=0o755, parents=True, exist_ok=True)
# Write out our MIPs
data['mip_sum'] = 0.0
data['mip_masked_sum'] = 0.0
for pid, img in particle_imgs.items():
idx = (data['particle_id'] == pid)
data.loc[idx, 'mip_sum'] = np.sum(img)
mask = ref_particle_imgs[pid]
threshold, mask = cv2.threshold(mask, 1, 255, cv2.THRESH_BINARY_INV)
masked = cv2.bitwise_and(img, img, mask=mask)
data.loc[idx, 'mip_masked_sum'] = np.sum(masked)
cv2.imwrite(str(mip_path / (pid + ".tif")), img)
cv2.imwrite(str(mip_path / (pid + "-ref.tif")), ref_particle_imgs[pid])
fourcc = cv2.VideoWriter_fourcc(*'mp4v')
writer = cv2.VideoWriter(str(mip_path / ("test.mp4")), fourcc, 10, (all_masks.shape[1], all_masks.shape[2]), True)
for i in frames:
masks = all_masks[(i-1),:,:]
img = cv2.imread(str(raw_paths[(i-1)]), cv2.IMREAD_GRAYSCALE)
labelled = label2rgb(masks, image=img)
labelled = exposure.rescale_intensity(labelled, in_range=(0,1), out_range='uint8').astype(np.uint8)
writer.write(labelled)
writer.release()
return data
def analyze_images_in_folder(
self,
folder,
generate_zmax=False,
show_result=True,
save_mask=True,
save_excel=True,
):
"""
Given the folder, perform general analysis over the images in it.
Parameters
----------
folder : str
Path to the folder.
generate_zmax : bool, optional
Whether to calcaulate the z-max projection first. The default is False.
show_result : bool, optional
If show the machine learning segmentation results. The default is True.
save_mask : bool, optional
DESCRIPTION. The default is True.
save_excel : bool, optional
DESCRIPTION. The default is True.
Returns
-------
cell_Data : pd.dataframe
DESCRIPTION.
"""
flat_cell_counted_in_folder = 0
total_cells_counted_in_folder = 0
background_substraction = False
root_folder = folder
# If need to do zmax projection first
if generate_zmax == True:
ProcessImage.cam_screening_post_processing(root_folder)
# Here a new folder for maxProjection is generated inside, change the path
folder = os.path.join(root_folder, "maxProjection")
# If background images are taken
if os.path.exists(os.path.join(root_folder, "background")):
# If the background image is taken to substract out
background_substraction = True
print("Run background substraction.")
# Get all the background files names
background_fileNameList = []
for file in os.listdir(os.path.join(root_folder, "background")):
if "calculated background" not in file:
if "tif" in file or "TIF" in file:
background_fileNameList.append(
os.path.join(root_folder, "background", file))
# Average over multiple images
background_image = ProcessImage.image_stack_calculation(
background_fileNameList, operation="mean")
# # Smooth the image
# background_image = ProcessImage.average_filtering(
# background_image, filter_side_length = 25)
# Save the individual file.
with skimtiff.TiffWriter(
os.path.join(root_folder, "background",
"calculated background.tif"),
imagej=True,
) as tif:
tif.save(background_image.astype(np.uint16), compress=0)
# Get a list of file names
fileNameList = []
for file in os.listdir(folder):
if "tif" in file and "LED" not in file:
fileNameList.append(file)
print(fileNameList)
# Analyse each image
for image_file_name in fileNameList:
print(image_file_name)
Rawimage = imread(os.path.join(folder, image_file_name))
if background_substraction == True:
Rawimage = np.abs(Rawimage - background_image).astype(
np.uint16)
camera_dark_level = 100
# # Normalize to the illumination intensity
# Rawimage = np.uint16(Rawimage \
# / ((background_image - camera_dark_level)\
# /(np.amin(background_image) - camera_dark_level)))
# Analyze each image
# Run the detection on input image.
MLresults = self.DetectionOnImage(Rawimage,
axis=None,
show_result=show_result)
if save_mask == True:
if not os.path.exists(os.path.join(folder, "ML_masks")):
# If the folder is not there, create the folder
os.mkdir(os.path.join(folder, "ML_masks"))
fig, ax = plt.subplots()
# Set class_names = [None,None,None,None] to mute class name display.
visualize.display_instances(
Rawimage,
MLresults["rois"],
MLresults["masks"],
MLresults["class_ids"],
class_names=[None, None, None, None],
ax=ax,
centre_coors=MLresults["Centre_coor"],
Centre_coor_radius=2,
WhiteSpace=(0, 0),
) # MLresults['class_ids'],MLresults['scores'],
# ax.imshow(fig)
fig.tight_layout()
# Save the detection Rawimage
fig_name = os.path.join(
folder,
"ML_masks",
"ML_mask_{}.png".format(
image_file_name[0:len(image_file_name) - 4]),
)
plt.savefig(fname=fig_name,
dpi=200,
pad_inches=0.0,
bbox_inches="tight")
if flat_cell_counted_in_folder == 0:
(
cell_Data,
flat_cell_counted_in_folder,
total_cells_counted_in_coord,
) = ProcessImage.retrieveDataFromML(
Rawimage, MLresults, image_file_name,
flat_cell_counted_in_folder)
else:
(
Cell_Data_new,
flat_cell_counted_in_folder,
total_cells_counted_in_coord,
) = ProcessImage.retrieveDataFromML(
Rawimage, MLresults, image_file_name,
flat_cell_counted_in_folder)
if len(Cell_Data_new) > 0:
cell_Data = cell_Data.append(Cell_Data_new)
total_cells_counted_in_folder += total_cells_counted_in_coord
if save_excel == True:
# Save to excel
cell_Data.to_excel(
os.path.join(
folder,
"CellsProperties_{}flat_outof_{}cells.xlsx".format(
flat_cell_counted_in_folder,
total_cells_counted_in_folder),
))
return cell_Data
