def czi_predict_images(learn,
czi_in,
dest,
category,
tag=None,
size=128,
max_imgs=None):
with czifile.CziFile(czi_in) as czi_f:
under_tag = f'_' if tag is None else f'_{tag}_'
dest_folder = Path(dest / category)
dest_folder.mkdir(exist_ok=True, parents=True)
proc_axes, proc_shape = get_czi_shape_info(czi_f)
channels = proc_shape['C']
depths = proc_shape['Z']
times = proc_shape['T']
if not max_imgs is None: times = min(max_imgs, times)
x, y = proc_shape['X'], proc_shape['Y']
data, img_info = img_to_float(czi_f.asarray())
orig_dtype = data.dtype
img_max = data.max()
print(f'czi: x:{x} y:{y} t:{times} c:{channels} z:{depths} {img_max}')
channels_bar = progress_bar(
range(channels)) if channels > 1 else range(channels)
depths_bar = progress_bar(
range(depths)) if depths > 1 else range(depths)
times_bar = progress_bar(range(times)) if times > 1 else range(times)
for c in channels_bar:
for z in depths_bar:
preds = []
origs = []
if (depths > 1) or (channels > 1):
pred_out = dest_folder / f'{czi_in.stem}_c{c:02d}_z{z:02d}_{under_tag}_pred.tif'
orig_out = dest_folder / f'{czi_in.stem}_c{c:02d}_z{z:02d}_{under_tag}_orig.tif'
else:
pred_out = dest_folder / f'{czi_in.stem}_{under_tag}_pred.tif'
orig_out = dest_folder / f'{czi_in.stem}_{under_tag}_orig.tif'
if not pred_out.exists():
for t in times_bar:
idx = build_index(
proc_axes, {
'T': t,
'C': c,
'Z': z,
'X': slice(0, x),
'Y': slice(0, y)
})
img = data[idx].copy()
pred = unet_image_from_tiles_blend(learn,
img[None],
tile_sz=size,
img_info=img_info)
preds.append(pred[None])
origs.append(img[None])
if len(preds) > 0:
all_y = img_to_uint8(np.concatenate(preds))
imageio.mimwrite(pred_out, all_y, bigtiff=True)
all_y = img_to_uint8(np.concatenate(origs))
imageio.mimwrite(orig_out, all_y, bigtiff=True)
nClass = UNet2D.hp['nClasses']
imagePath = args.imagePath
dapiChannel = args.channel
dsFactor = args.scalingFactor
parentFolder = os.path.dirname(os.path.dirname(imagePath))
fileName = os.path.basename(imagePath)
fileNamePrefix = fileName.split(os.extsep, 1)
print(fileName)
fileType = fileNamePrefix[1]
if fileType == 'ome.tif' or fileType == 'btf':
I = skio.imread(imagePath, img_num=dapiChannel, plugin='tifffile')
elif fileType == 'tif':
I = tifffile.imread(imagePath, key=dapiChannel)
elif fileType == 'czi':
with czifile.CziFile(imagePath) as czi:
image = czi.asarray()
I = image[0, 0, dapiChannel, 0, 0, :, :, 0]
elif fileType == 'nd2':
with ND2Reader(iFile) as fullStack:
I = fullStack[dapiChannel]
if args.classOrder == -1:
args.classOrder = range(nClass)
rawI = I
print(type(I))
hsize = int((float(I.shape[0]) * float(dsFactor)))
vsize = int((float(I.shape[1]) * float(dsFactor)))
I = resize(I, (hsize, vsize))
I = im2double(
def process_czi(item, category, mode):
tif_srcs = []
base_name = item.stem
with czifile.CziFile(item) as czi_f:
data = czi_f.asarray()
axes, shape = get_czi_shape_info(czi_f)
channels = shape['C']
depths = shape['Z']
times = shape['T']
x, y = shape['X'], shape['Y']
mid_depth = depths // 2
depth_range = range(max(0, mid_depth - 2), min(depths, mid_depth + 2))
is_multi = (times > 1) or (depths > 1)
data = czi_f.asarray()
all_rmax = data.max()
all_mi, all_ma = np.percentile(data, [2, 99.99])
dtype = data.dtype
for channel in range(channels):
for z in depth_range:
for t in range(times):
idx = build_index(
axes, {
'T': t,
'C': channel,
'Z': z,
'X': slice(0, x),
'Y': slice(0, y)
})
img = data[idx]
mi, ma = np.percentile(img, [2, 99.99])
if dtype == np.uint8: rmax = 255.
else: rmax = img.max()
tif_srcs.append({
'fn': item,
'ftype': 'czi',
'multi': int(is_multi),
'category': category,
'dsplit': mode,
'uint8': dtype == np.uint8,
'mi': mi,
'ma': ma,
'rmax': rmax,
'all_rmax': all_rmax,
'all_mi': all_mi,
'all_ma': all_ma,
'mean': img.mean(),
'sd': img.std(),
'nc': channels,
'nz': depths,
'nt': times,
'z': mid_depth,
't': t,
'c': channel,
'x': x,
'y': y
})
return tif_srcs
def __init__(self, path_czi):
with czifile.CziFile(path_czi) as czi:
self.czi_np = czi.asarray()
self.axes = czi.axes
self.metadata = czi.metadata
def get_tile_puller(tile_stat, crap_func, t_frames, z_frames):
fn = tile_stat['fn']
ftype = tile_stat['ftype']
nz = tile_stat['nz']
nt = tile_stat['nt']
half_z = z_frames // 2
half_t = t_frames // 2
if ftype == 'czi':
img_f = czifile.CziFile(fn)
proc_axes, proc_shape = get_czi_shape_info(img_f)
img_data = img_f.asarray()
img_data = img_data.astype(np.float32)
def czi_get(istat):
c, z, t, x, y, mi, ma, is_uint8, rmax, all_rmax, all_ma = [
istat[fld] for fld in [
'c', 'z', 't', 'x', 'y', 'mi', 'ma', 'uint8', 'rmax',
'all_rmax', 'all_ma'
]
]
if is_uint8:
mi, ma, rmax = 0., 255.0, 255.0
all_ma, all_rmax = 255.0, 255.0
t_slice = slice(t - half_t, t + half_t + 1) if half_t > 0 else t
z_slice = slice(z - half_z, z + half_z + 1) if half_z > 0 else z
idx = build_index(
proc_axes, {
'C': c,
'T': t_slice,
'Z': z_slice,
'X': slice(0, x),
'Y': slice(0, y)
})
img = img_data[idx].copy()
img /= all_rmax
if len(img.shape) <= 2: img = img[None]
return img
img_get = czi_get
img_get._to_close = img_f
else:
pil_img = PIL.Image.open(fn)
def pil_get(istat):
c, z, t, x, y, mi, ma, is_uint8, rmax, all_rmax, all_ma = [
istat[fld] for fld in [
'c', 'z', 't', 'x', 'y', 'mi', 'ma', 'uint8', 'rmax',
'all_rmax', 'all_ma'
]
]
if half_t > 0: n_start, n_end = t - half_t, t + half_t + 1
elif half_z > 0: n_start, n_end = z - half_z, z + half_z + 1
else: n_start, n_end = 0, 1
if is_uint8:
mi, ma, rmax = 0., 255.0, 255.0
all_ma, all_rmax = 255.0, 255.0
img_array = []
for ix in range(n_start, n_end):
pil_img.seek(ix)
pil_img.load()
img = np.array(pil_img)
if len(img.shape) > 2: img = img[:, :, 0]
img_array.append(img.copy())
img = np.stack(img_array)
img = img.astype(np.float32)
img /= all_rmax
return img
img_get = pil_get
img_get._to_close = pil_img
def puller(istat, tile_folder, crap_folder, close_me=False):
if close_me:
img_get._to_close.close()
return None
id = istat['index']
fn = Path(istat['fn'])
tile_sz = istat['tile_sz']
c, z, t, x, y, mi, ma, is_uint8, rmax = [
istat[fld]
for fld in ['c', 'z', 't', 'x', 'y', 'mi', 'ma', 'uint8', 'rmax']
]
raw_data = img_get(istat)
img_data = (np.iinfo(np.uint8).max * raw_data).astype(np.uint8)
thresh = np.percentile(img_data, 2)
thresh_pct = (img_data > thresh).mean() * 0.30
frame_count = img_data.shape[0]
mid_frame = frame_count // 2
crop_img, box = draw_random_tile(img_data[mid_frame], istat['tile_sz'],
thresh, thresh_pct)
crop_img.save(tile_folder / f'{id:06d}_{fn.stem}.tif')
if crap_func and crap_folder:
if frame_count > 1:
crap_data = []
for i in range(frame_count):
frame_img = img_data[i, box[0]:box[2], box[1]:box[3]]
crap_frame = crap_func(frame_img)
crap_data.append(np.array(crap_frame))
multi_array = np.stack(crap_data)
np.save(crap_folder / f'{id:06d}_{fn.stem}.npy', multi_array)
else:
crap_img = crap_func(crop_img)
crap_img.save(crap_folder / f'{id:06d}_{fn.stem}.tif')
info = dict(istat)
info['id'] = id
info['box'] = box
info['tile_sz'] = tile_sz
crop_data = np.array(crop_img)
info['after_mean'] = crop_data.mean()
info['after_sd'] = crop_data.std()
info['after_max'] = crop_data.max()
info['after_min'] = crop_data.min()
return info
return puller
def print_czi_info(fn):
with czifile.CziFile(fn) as czi_f:
proc_axes, proc_shape = get_czi_shape_info(czi_f)
print(fn.stem, proc_shape)
def __init__(self, czifilename, renormalize=False):
self._fo = czifile.CziFile(czifilename)
# sanity check dimensions
self._Z = self._fo.axes.find('Z')
if self._Z < 0:
self._Z = None
self._Y = self._fo.axes.find('Y')
self._X = self._fo.axes.find('X')
assert self._Y >= 0
assert self._X == self._Y + 1
self._slc = slice(self._Y, self._X + 1)
# check interleaved pixel config
assert (self._slc.stop + 1) == len(self._fo.axes), (self._slc.stop,
len(self._fo.axes))
assert self._fo.shape[self._slc.stop] in [1, 3], self._fo.shape
assert self._fo.axes[self._slc] == 'YX', self._fo.axes[self._slc]
# check multi-channel config
self._C = self._fo.axes.find('C')
assert self._C >= 0
channel = self._fo.start[self._C]
assert self._fo.shape[self._C] == 1 or self._fo.shape[self._slc.stop] == 1, \
"do not understand multi-channel interleaved shape %s" % self._fo.shape
# sort out segments into channel zoom tiers
self._channel_tiers = [dict() for c in range(self._fo.shape[self._C])]
self._channel_tier_maps = [
dict() for c in range(self._fo.shape[self._C])
]
v0 = None
v1 = None
tile_size = (0, 0)
for entry in self._fo.subblock_directory:
assert len(entry.shape) == len(self._fo.shape)
assert entry.axes == self._fo.axes
channel = entry.start[self._C]
# HACK: discard all but middle plane if there is a Z stack
if self._Z is not None:
if entry.start[self._Z] != (entry.shape[self._Z] / 2):
continue
# infer zoom from canvas size to tile size ratio
zoom = map(lambda c, t: c / t, entry.shape[self._slc],
entry.stored_shape[self._slc])
assert zoom[0] == zoom[1]
zoom = zoom[0]
if zoom not in self._channel_tiers[channel]:
self._channel_tiers[channel][zoom] = []
# tile bounding box in canvas coordinates
bbox = (entry.start[self._slc],
tuple(
map(lambda s, l: s + l, entry.start[self._slc],
entry.shape[self._slc])))
self._channel_tiers[channel][zoom].append((bbox, entry))
# Figure total shape too...
if v0 is None:
v0 = bbox[0]
v1 = bbox[1]
else:
v0 = map(lambda v, b: min(v, b), v0, bbox[0])
v1 = map(lambda v, b: max(v, b), v1, bbox[1])
tile_size = tuple(
map(lambda a, b: max(a, b), tile_size,
entry.stored_shape[self._slc]))
# build up tile bbox maps for each zoom tier for efficient intersection tests
for channel in range(self._fo.shape[self._C]):
for zoom, bbox_entries in self._channel_tiers[channel].items():
# pack as (v0.y, v0.x, v1.y, v1.x) for each entry
bboxes = np.zeros((len(bbox_entries), 4), dtype=np.int32)
for i in range(bboxes.shape[0]):
bboxes[i, 0:2] = np.array(bbox_entries[i][0][0],
dtype=np.int32)
bboxes[i, 2:4] = np.array(bbox_entries[i][0][1],
dtype=np.int32)
# for fancy-indexing by boolean arrays, repeat entry indexes into array
boxmap = np.array(range(len(bbox_entries)), dtype=np.int32)
self._channel_tier_maps[channel][zoom] = (bboxes, boxmap)
channels = self._fo.metadata.findall(
'Metadata/DisplaySetting/Channels/Channel')
assert channels, 'found no Metadata/DisplaySetting/Channels/Channel elements in CZI metadata'
self._channel_names_long = [c.get('Name') for c in channels]
self._channel_names = [c.find('ShortName').text for c in channels]
self._channel_colors = [
c.find('Color').text if c.find('Color') is not None else None
for c in channels
]
self._bbox_native = (tuple(v0), tuple(v1))
self._bbox_zeroed = ((0, 0), (v1[0] - v0[0], v1[1] - v0[1]))
self._tile_size = tile_size
self._zoom_levels = self._channel_tiers[0].keys()
self._zoom_levels.sort()
sys.stderr.write(
'CZI %s tile-size %s %s\n channels: %s\n bounding-box: %s native or shape %s\n zoom levels: %s\n'
% (' '.join(
map(lambda d, s: '%s=%d' %
(d, s), self._fo.axes, self._fo.shape)),
'x'.join(map(str, self._tile_size)), self._fo.dtype, ', '.join([
'%s (%s %s)' %
(self._channel_names[i], self._channel_names_long[i],
self._channel_colors[i])
for i in range(self._fo.shape[self._C])
]), self._bbox_native, 'x'.join(map(
str, self._bbox_zeroed[1])), self._zoom_levels))
if renormalize and self._fo.dtype not in [np.uint8]:
sys.stderr.write(
'Finding per-channel value ranges for dynamic normalization...\n'
)
self._channel_ranges = [
self._value_range(channel)
for channel in range(self._fo.shape[self._C])
]
else:
self._channel_ranges = None
# HACK: try to configure a tile cache for row-major tile grid traversals
# 1. assume we will output tiles no larger than current tile size
# 2. assume that up to 3 tile rows might be active due to overlapping source tiles
overlap_factor = max([
float(e.text) for e in self._fo.metadata.findall(
'Metadata/Experiment/ExperimentBlocks/AcquisitionBlock/SubDimensionSetups/RegionsSetup/SampleHolder/Overlap'
)
])
row_tile_count = math.ceil(
self._bbox_zeroed[1][1] /
(self._tile_size[1] - self._tile_size[1] * overlap_factor))
self._tile_cache_size = row_tile_count * 3 + 1
# use dict as cache... key: (entry, dtype), value: [data, clock]
self._tile_cache = dict()
self._tile_cache_clock = 0
sys.stderr.write(
'Estimating %d tiles per row at zoom level 1 or %d tile cache size\n'
% (row_tile_count, self._tile_cache_size))
# get per-dimension distances and turn meter value into micrometer
self.mpps = dict([(dx.get('Id'), float(dx.find('Value').text) * 1E6)
for dx in self._fo.metadata.findall(
'Metadata/Scaling/Items/Distance')])
sys.stderr.write('Image reported microns per pixel: %s\n' % self.mpps)
import numpy as np
import czifile as zis
import imgfileutils as imf
#filename = r"C:\Temp\input\DTScan_ID4.czi"
#filename = r"C:\Users\m1srh\OneDrive - Carl Zeiss AG\Testdata_Zeiss\CZI_Testfiles\W96_B2+B4_S=2_T=2=Z=4_C=3_Tile=4x8.czi"
filename = r"C:\Users\m1srh\Documents\ImageSC\P2A_B6_M1_GIN-0004-Scene-2-ScanRegion1_unc.czi"
#filename = r"C:\Users\m1srh\Documents\ImageSC\P2A_B6_M1_GIN-0004-Scene-2-ScanRegion1_AF750.czi"
#filename = r"C:\Users\m1srh\Documents\ImageSC\P2A_B6_M1_GIN-0004-Scene-2-ScanRegion1_AF594.czi"
#filename = r"C:\Users\m1srh\Documents\ImageSC\P2A_B6_M1_GIN-0004-Scene-2-ScanRegion1_AF488.czi"
#filename = r"C:\Users\m1srh\Documents\ImageSC\P2A_B6_M1_GIN-0004-Scene-2-ScanRegion1_DAPI.czi"
# get CZI object
czi = zis.CziFile(filename)
# parse the XML into a dictionary
metadatadict_czi = czi.metadata(raw=False)
# get the complete scene information
allscenes = metadatadict_czi['ImageDocument']['Metadata']['Information'][
'Image']['Dimensions']['S']['Scenes']['Scene']
centerposX = []
centerposY = []
# check if there is a list of scenes
if isinstance(allscenes, list):
for s in range(len(allscenes)):
for k, v in allscenes[s].items():
print(k, ':', v)
# get the SceneCenter Position for all scenes
centerposX.append(
def parse_metadata(self):
"""
Looks for individual data within the input files and stores them into a dictionary
"""
czi_file = czi.CziFile(self.dir + self.filename) # Open file
# First, stores the relevant metadata from the XML metadata header
from_xml = czi_file.metadata(raw=False)['ImageDocument']['Metadata']
'''
XML metadata structure: stared items are (sub-)dictionaries, dashed items are k, v pairs
ImageDocument
Metadata
CustomAttributes
- NeedsPyramidRecalculation
Information
Document
- UserName
- CreationDate
Image
- SizeX
- SizeY
- SizeM
- OriginalCompressionMethod
- OriginalEncodingQuality
- Dimensions
Scaling
- Metadata
AutoScaling
- Type
- CreationDateTime
Items
- Distance
DisplaySetting
Channels
Channel
- BitCountRange
- PixelType
- DyeName
- ColorMode
- Id
- Name
ToneMapping
- Mode
- EnhanceClipLimit
- EnhanceRegionSizePercentage
- ViewerRotations
- Layers
'''
self.metadata['SizeX_pixels'] = from_xml['Information']['Image'][
'SizeX']
self.metadata['SizeY_pixels'] = from_xml['Information']['Image'][
'SizeY']
self.metadata['SizeM'] = from_xml['Information']['Image']['SizeM']
self.metadata['EnhanceClipLimit'] = from_xml['DisplaySetting'][
'ToneMapping']['EnhanceClipLimit']
self.metadata['EnhanceRegionSizePercentage'] = from_xml[
'DisplaySetting']['ToneMapping']['EnhanceRegionSizePercentage']
for element in from_xml['Scaling']['Items']['Distance']:
self.metadata['Calib' + element['Id'] +
'_microns'] = element['Value'] * 1000000.0
# Second, look for additional metadata from the segments
# Looks at the image sub blocks within the file
sub_blocks = czi_file.subblocks()
for subBlock in sub_blocks:
bloc_nb = -1
'''
Extracts the following infos from the subblock:
the block number, its start, size, start coordinate and stored size
once done, everything is pushed to the metadata dictionary with a key in the form:
TileXX_dimension_parameterName
'''
for sub in subBlock.directory_entry.dimension_entries:
# Lazy way: supposing the M tag will always be present in subblocks BEFORE X or Y blocks
if sub.dimension == 'M':
bloc_nb = sub.start
else:
self.metadata['Tile' + str(bloc_nb) + "_" + sub.dimension +
"_start"] = sub.start
self.metadata['Tile' + str(bloc_nb) + "_" + sub.dimension +
"_size"] = sub.size
self.metadata['Tile' + str(bloc_nb) + "_" + sub.dimension +
"_start_coordinate"] = sub.start_coordinate
self.metadata['Tile' + str(bloc_nb) + "_" + sub.dimension +
"_stored_size"] = sub.stored_size
'''
Now the tile number has been extracted, get the stage coordinates and
pushes to the metadata dictionary with a key in the form TileXX_parameterName_units
'''
for key, value in subBlock.metadata(raw=False)['Tags'].items():
self.metadata['Tile' + str(bloc_nb) + '_' + key +
'_mm'] = value / 1000.0
'''
Quite tricky to recompute the stage coordinates of the central position...
First, computes the stage calibration from the two first tiles:
- Computes the slope delta positions in mm / delta positions in pixels.
'''
slope = (self.metadata['Tile1_StageXPosition_mm'] -
self.metadata['Tile0_StageXPosition_mm']) / (
self.metadata['Tile1_X_start'] -
self.metadata['Tile0_X_start'])
slope = (int(slope * 10000)
) / 10000 # limit to 4 digits because... it seems to work ;-)
'''
Second, computes the X/Y stage offset from the first tile:
- Slope*img_size/2: coordinate of the center of the image relative to the stage if the origin was top-let
- -start_coordinate: takes into account the offset of the stage (origin is not zero)
- +tile0_size/2*pixel_size: takes into account the fact that the coordinates are relative to the center of
the first tile, converting coordinates using the image calibration
'''
self.metadata['ImageCenterPosition_X_mm'] = slope * (
self.metadata['SizeX_pixels'] /
2) + self.metadata['Tile0_StageXPosition_mm'] - (
self.metadata['Tile0_X_stored_size'] /
2) * self.metadata['CalibX_microns'] / 1000
self.metadata['ImageCenterPosition_X_mm'] = int(
self.metadata['ImageCenterPosition_X_mm'] * 100) / 100
self.metadata['ImageCenterPosition_Y_mm'] = self.metadata[
'Tile1_StageYPosition_mm']
czi_file.close()
The dzidir destination defaults to a name derived from the CZI
filename if not specified.
""" % (mesg)
if __name__ == "__main__":
if len(sys.argv) <= 1:
raise ValueError(usage('First argument must be a CZI filepath.'))
if len(sys.argv) > 3:
raise ValueError(usage('At most two arguments are expected.'))
try:
f = open(sys.argv[1])
try:
fo = czifile.CziFile(sys.argv[1])
except Exception, e:
raise ValueError(
usage('%s. First argument must be a CZI file.' % e))
except:
raise ValueError(usage('First argument must be a readable file.'))
# if len(sys.argv) > 2:
# if not isinstance(sys.argv[2], int):
# raise ValueError(usage('Second argument must be a valid zoom level.'))
ret = main(*sys.argv[1:])
sys.exit(ret)
def extract_fovs(file_path: Path,
out_path: Path):
""" Extract individual FOVs from a czi file
When CZI files are loaded by BioFormats, it will generally try to mosaic
images together by stage position if the image was captured with the
intention of mosaicing images together. At the time this function was
written, there was no clear way of extracting individual FOVs so this
algorithm was created.
Every field of view in each z-slice, channel, and timepoint contained in a
CZI file is saved as an individual image.
Args:
file_path (Path): Path to CZI file
out_path (Path): Path to output directory
"""
with ProcessManager.process(file_path.name):
logger.info('Starting extraction from ' + str(file_path) + '...')
if Path(file_path).suffix != '.czi':
TypeError("Path must be to a czi file.")
base_name = Path(file_path.name).stem
# Load files without mosaicing
czi = czifile.CziFile(file_path,detectmosaic=False)
subblocks = [s for s in czi.filtered_subblock_directory if s.mosaic_index is not None]
ind = {'X': [],
'Y': [],
'Z': [],
'C': [],
'T': [],
'Row': [],
'Col': []}
# Get the indices of each FOV
for s in subblocks:
scene = [dim.start for dim in s.dimension_entries if dim.dimension=='S']
if scene is not None and scene[0] != 0:
continue
for dim in s.dimension_entries:
if dim.dimension=='X':
ind['X'].append(dim.start)
elif dim.dimension=='Y':
ind['Y'].append(dim.start)
elif dim.dimension=='Z':
ind['Z'].append(dim.start)
elif dim.dimension=='C':
ind['C'].append(dim.start)
elif dim.dimension=='T':
ind['T'].append(dim.start)
row_conv = {y:row for (y,row) in zip(np.unique(np.sort(ind['Y'])),range(0,len(np.unique(ind['Y']))))}
col_conv = {x:col for (x,col) in zip(np.unique(np.sort(ind['X'])),range(0,len(np.unique(ind['X']))))}
ind['Row'] = [row_conv[y] for y in ind['Y']]
ind['Col'] = [col_conv[x] for x in ind['X']]
with BioReader(file_path) as br:
metadata = br.metadata
chan_names = br.cnames
for s,i in zip(subblocks,range(0,len(subblocks))):
Z = None if len(ind['Z'])==0 else ind['Z'][i]
C = None if len(ind['C'])==0 else ind['C'][i]
T = None if len(ind['T'])==0 else ind['T'][i]
out_file_path = out_path.joinpath(_get_image_name(base_name,
row=ind['Row'][i],
col=ind['Col'][i],
Z=Z,
C=C,
T=T))
dims = [_get_image_dim(s,'Y'),
_get_image_dim(s,'X'),
_get_image_dim(s,'Z'),
_get_image_dim(s,'C'),
_get_image_dim(s,'T')]
data = s.data_segment().data().reshape(dims)
write_thread(out_file_path,
data,
metadata,
chan_names[C])
from io import BytesIO
from matplotlib import pyplot
import czifile
print(czifile.__version__)
file_lineselect = '/Volumes/Anup_2TB/raw_data/beiquelab/zen/data_anup/20190829/s1c1s1/Image 30.czi'
with czifile.CziFile(file_lineselect) as czi:
print(czi)
print(czi.metadata())
line_scan = czi.asarray()
for attachment in czi.attachments():
print(attachment)
embedded_czifile = czifile.CziFile(
BytesIO(
next(attachment for attachment in czi.attachments()
if attachment.attachment_entry.name == 'Image').data(
raw=True)))
print(embedded_czifile)
print(embedded_czifile.metadata())
embedded_image = embedded_czifile.asarray()
for line in line_scan.squeeze():
pyplot.plot(line)
for image in embedded_image.squeeze():
a = (a - mi) / (ma - mi + eps)
#%%
from fastai import *
from fastai.vision import *
img = Image(tensor(a[None]))
#%%
img
#%%
info = []
for fn in progress_bar(fns):
if fn.parts[-4] != 'random' and 'great quality' not in fn.stem:
try:
with czifile.CziFile(fn) as czi_f:
data = czi_f.asarray()
mi, ma = np.percentile(data, [2, 99.8])
info.append({
'fn': fn,
'ndtype': czi_f.dtype,
'category': fn.parts[-3],
'maxval': data.max(),
'mi': mi,
'ma': ma
})
except:
print('exception', fn)
pass
df = pd.DataFrame(info)
def Import_czi(filename,par_obj,win_obj):
win_obj.diag = dialog_import(par_obj,win_obj)
def import_data_fn(self):
deltat= 1000/float(self.text_1)
data_array = czi_fn.imread(str(filename))
scanObject(filename,par_obj,[deltat,float(self.text_2)/1000000],data_array,0,0);
win_obj.bleachCorr1 = False
win_obj.bleachCorr2 = False
win_obj.DeltatEdit.setText(str(deltat));
win_obj.label.generateList()
self.win_obj.image_status_text.showMessage("Correlating carpet: File " +str(self.win_obj.file_import.file_index+1)+' of '+str(self.win_obj.file_import.file_list.__len__()))
self.win_obj.app.processEvents()
if win_obj.last_in_list == False:
print( 'moving to next file')
win_obj.file_import.load_next_file()
else:
print ('finished with all files')
win_obj.file_import.post_initial_import()
win_obj.diag.import_data_fn = import_data_fn
#There is only carpet in each file.
czifile = czi_fn.CziFile(str(filename))
filename.replace('\\', '/')
xml = czifile.metadata()
root = ET.XML(xml)
suggest_line_time = "0.0"
for neighbor in root:
#print(neighbor)
for element in neighbor.iter():
if element.tag == "PixelTime":
suggest_pixel_time = np.round(float(element.text)*1e6,8)
if element.tag == "LineTime":
suggest_line_time = np.round(1./float(element.text),8)
name = str(filename).split('/')[-1]
win_obj.diag.stack_ind = {}
win_obj.diag.stack_ind['title'] = name
win_obj.diag.stack_ind['name'] =""
win_obj.diag.czifile = czifile
win_obj.last_in_file = False
if win_obj.last_in_list == True:
win_obj.last_in_file = True
if win_obj.yes_to_all == None:
win_obj.diag.suggest_pixel_time = suggest_pixel_time
win_obj.diag.create_line_sampling(suggest_line_time)
else:
win_obj.diag.text_1 = win_obj.diag.win_obj.text_1
win_obj.diag.text_2 = win_obj.diag.win_obj.text_2
win_obj.diag.ok_1 = True
win_obj.diag.ok_2 = True
win_obj.diag.import_data_fn(win_obj.diag)
def czi_movie_to_synth(czi_fn,
dest,
category,
mode,
single=True,
multi=False,
tiles=None,
scale=4,
n_tiles=5,
n_frames=5,
crappify_func=None):
base_name = czi_fn.stem
if single:
hr_dir = ensure_folder(dest / 'hr' / mode / category)
lr_dir = ensure_folder(dest / 'lr' / mode / category)
lrup_dir = ensure_folder(dest / 'lrup' / mode / category)
with czifile.CziFile(czi_fn) as czi_f:
data = czi_f.asarray()
axes, shape = get_czi_shape_info(czi_f)
channels = shape['C']
depths = shape['Z']
times = shape['T']
x,y = shape['X'], shape['Y']
for channel in range(channels):
for depth in range(depths):
for t in range(times):
save_name = f'{channel:02d}_{depth:02d}_{t:06d}_{base_name}'
idx = build_index( axes, {'T': t, 'C':channel, 'Z': depth, 'X':slice(0,x), 'Y':slice(0,y)})
img_data = data[idx].astype(np.float32).copy()
img_max = img_data.max()
if img_max != 0: img_data /= img_max
image_to_synth(img_data, dest, mode, hr_dir, lr_dir, lrup_dir, save_name,
single, multi, tiles, n_tiles, n_frames, scale, crappify_func)
if multi:
with czifile.CziFile(czi_fn) as czi_f:
proc_axes, proc_shape = get_czi_shape_info(czi_f)
channels = proc_shape['C']
depths = proc_shape['Z']
times = proc_shape['T']
x,y = proc_shape['X'], proc_shape['Y']
data = czi_f.asarray()
for channel in range(channels):
img_max = None
timerange = list(range(0,times-n_frames+1, n_frames))
if len(timerange) >= n_frames:
hr_mt_dir = ensure_folder(dest / f'hr_mt_{n_frames:02d}' / mode / category)
lr_mt_dir = ensure_folder(dest / f'lr_mt_{n_frames:02d}' / mode / category)
lrup_mt_dir = ensure_folder(dest / f'lrup_mt_{n_frames:02d}' / mode / category)
for time_col in timerange:
save_name = f'{channel:02d}_T{time_col:05d}-{(time_col+n_frames-1):05d}_{base_name}'
idx = build_index(proc_axes, {'T': slice(time_col,time_col+n_frames), 'C': channel, 'X':slice(0,x),'Y':slice(0,y)})
img_data = data[idx].astype(np.float32).copy()
img_max = img_data.max()
if img_max != 0: img_data /= img_max
_,h,w = img_data.shape
adjh, adjw = (h//4) * 4, (w//4)*4
hr_imgs = img_data[:,0:adjh, 0:adjw]
lr_imgs = []
lrup_imgs = []
for i in range(hr_imgs.shape[0]):
hr_img = hr_imgs[i]
crap_img = crappify_func(hr_img).astype(np.float32).copy() if crappify_func else hr_img
lr_img = npzoom(crap_img, 1/scale, order=0).astype(np.float32).copy()
lr_imgs.append(lr_img)
lrup_img = npzoom(lr_img, scale, order=0).astype(np.float32).copy()
lrup_imgs.append(lrup_img)
lr_imgs = np.array(lr_imgs).astype(np.float32).copy()
lrup_imgs = np.array(lrup_imgs).astype(np.float32).copy()
hr_img = hr_imgs[hr_imgs.shape[0]//2].astype(np.float32).copy()
hr_mt_name, lr_mt_name, lrup_mt_name = [d / save_name for d in [hr_mt_dir, lr_mt_dir, lrup_mt_dir]]
np.save(hr_mt_name, hr_img)
np.save(lr_mt_name, lr_imgs)
np.save(lrup_mt_name, lrup_imgs)
make_multi_tiles(tiles, category, n_tiles, scale, hr_img, lr_imgs, lrup_imgs,
save_name, dest, n_frames, mode, 't')
if depths >= n_frames:
hr_mz_dir = ensure_folder(dest / f'hr_mz_{n_frames:02d}' / mode / category)
lr_mz_dir = ensure_folder(dest / f'lr_mz_{n_frames:02d}' / mode / category)
lrup_mz_dir = ensure_folder(dest / f'lrup_mz_{n_frames:02d}' / mode / category)
mid_depth = depths // 2
start_depth = mid_depth - n_frames//2
end_depth = mid_depth + n_frames//2
depthrange = slice(start_depth,end_depth+1)
save_name = f'{channel:02d}_Z{start_depth:05d}-{end_depth:05d}_{base_name}'
idx = build_index(proc_axes, {'Z': depthrange, 'C': channel, 'X':slice(0,x),'Y':slice(0,y)})
img_data = data[idx].astype(np.float32).copy()
img_max = img_data.max()
if img_max != 0: img_data /= img_max
_,h,w = img_data.shape
adjh, adjw = (h//4) * 4, (w//4)*4
hr_imgs = img_data[:,0:adjh, 0:adjw]
lr_imgs = []
lrup_imgs = []
for i in range(hr_imgs.shape[0]):
hr_img = hr_imgs[i]
crap_img = crappify_func(hr_img).astype(np.float32).copy() if crappify_func else hr_img
lr_img = npzoom(crap_img, 1/scale, order=0).astype(np.float32).copy()
lr_imgs.append(lr_img)
lrup_img = npzoom(lr_img, scale, order=0).astype(np.float32).copy()
lrup_imgs.append(lrup_img)
lr_imgs = np.array(lr_imgs).astype(np.float32).copy()
lrup_imgs = np.array(lrup_imgs).astype(np.float32).copy()
hr_img = hr_imgs[hr_imgs.shape[0]//2].astype(np.float32).copy()
hr_mz_name, lr_mz_name, lrup_mz_name = [d / save_name for d in [hr_mz_dir, lr_mz_dir, lrup_mz_dir]]
np.save(hr_mz_name, hr_img)
np.save(lr_mz_name, lr_imgs)
np.save(lrup_mz_name, lrup_imgs)
make_multi_tiles(tiles, category, n_tiles, scale, hr_img, lr_imgs, lrup_imgs,
save_name, dest, n_frames, mode, 'z')
def process_czi(fn, processor, proc_func, out_fn, baseline_dir, n_depth=1, n_time=1, mode='L'):
stats = []
with czifile.CziFile(fn) as czi_f:
proc_axes, proc_shape = get_czi_shape_info(czi_f)
channels = proc_shape['C']
depths = proc_shape['Z']
times = proc_shape['T']
x, y = proc_shape['X'], proc_shape['Y']
data = czi_f.asarray().astype(np.float32)
data, img_info = img_to_float(data)
# mi, ma = img_info['mi'], img_info['ma']
# data = np.clip(data, mi, ma)
# data -= mi
# img_info['real_max'] = data.max()
# img_info['ma'] = data.max()
# img_info['mi'] = 0.
if depths < n_depth: return
if times < n_time: return
if n_depth > 1: # this is busted
offset_frames = n_depth // 2
for c in range(channels):
for t in range(times):
for z in range(offset_frames, depths - offset_frame):
depth_slice = slice(z-offset_frames, z+offset_frame+1)
idx = build_index(
proc_axes, {
'T': t,
'C': c,
'Z': depth_slice,
'X': slice(0, x),
'Y': slice(0, y)
})
img = data[idx].copy()
tag = f'{c}_{t}_{z+offset_frames}_'
save_name = f'{proc_name}_{item.stem}_{tag}'
pred_img = proc_func(img, img_info=img_info, mode=mode)
pred_img8 = (pred_img * np.iinfo(np.uint8).max).astype(np.uint8)
PIL.Image.fromarray(pred_img8).save(out_fn)
elif n_time > 1:
offset_frames = n_time // 2
for c in range(channels):
for z in range(depths):
imgs = []
time_range = list(range(offset_frames, times - offset_frames))
for t in progress_bar(time_range):
time_slice = slice(t-offset_frames, t+offset_frames+1)
idx = build_index(
proc_axes, {
'T': time_slice,
'C': c,
'Z': z,
'X': slice(0, x),
'Y': slice(0, y)
})
img = data[idx].copy()
pred_img = proc_func(img, img_info=img_info, mode=mode)
pred_img8 = (pred_img * np.iinfo(np.uint8).max).astype(np.uint8)
imgs.append(pred_img8[None])
all_y = np.concatenate(imgs)
if processor!='bilinear':
fldr_name = f'{out_fn.parent}/{processor}'
else:
fldr_name = out_fn.parent.parent.parent/processor/out_fn.parent.stem
save_name = f'{fn.stem}.tif'
if c > 1 or z > 1:
fldr_name = fldr_name/f'{c}_{z}'
out_fldr = ensure_folder(fldr_name)
if all_y.size < 4e9:
imageio.mimwrite(out_fldr/save_name, all_y)
else:
imageio.mimwrite(out_fldr/save_name, all_y, bigtiff=True)
#imageio.mimwrite(out_fldr/save_name, all_y)
#imageio.mimwrite((out_fldr/save_name).with_suffix('.mp4'), all_y, fps=30, macro_block_size=None)
else:
imgs = []
for c in range(channels):
for z in range(depths):
for t in range(times):
idx = build_index(
proc_axes, {
'T': t,
'C': c,
'Z': z,
'X': slice(0, x),
'Y': slice(0, y)
})
img = data[idx].copy()
pred_img = proc_func(img, img_info=img_info, mode=mode)
pred_img8 = (pred_img * np.iinfo(np.uint8).max).astype(np.uint8)
imgs.append(pred_img8[None])
all_y = np.concatenate(imgs)
if processor!='bilinear':
fldr_name = f'{out_fn.parent}/{processor}'
else:
fldr_name = out_fn.parent.parent.parent/processor/out_fn.parent.stem
save_name = f'{fn.stem}.tif'
out_fldr = ensure_folder(fldr_name)
if all_y.size < 4e9:
imageio.mimwrite(out_fldr/save_name, all_y)
else:
imageio.mimwrite(out_fldr/save_name, all_y, bigtiff=True)
def estimate_channel_shading(
filepath,
channel,
noise_threshold=None,
metric='median',
z_range=[],
quantile_threshold=0.8,
polynomial_order=3,
outputdir='',
):
"""Estimate the x- and y-profiles for a channel in a czi file.
# TODO: generalize to other formats than czi
# TODO: automate threshold estimate
# TODO: implement more metrics
# TODO: implement logging throughout.
# TODO: write corrected output for Fiji BigStitcher.
# TODO: try getting median over 4D concatenation directly (forego xy)
"""
# Get the list of subblocks for the channel.
if filepath.endswith('.czi'): # order of subblock_directory: CZM
czi = czifile.CziFile(filepath)
zstack0 = czi.subblock_directory[0]
dtype = zstack0.dtype
tilesize = [
zstack0.shape[zstack0.axes.index('Y')],
zstack0.shape[zstack0.axes.index('X')]
]
n_planes = czi.shape[czi.axes.index('Z')]
n_chs = czi.shape[czi.axes.index('C')]
sbd_channel = [sbd for sbd in czi.subblock_directory[channel::n_chs]]
else:
print('Sorry, only czi implemented for now...')
return
# Prepare the output directory.
datadir, filename = os.path.split(filepath)
dataset, ext = os.path.splitext(filename)
outputdir = outputdir or os.path.join(datadir, 'shading')
os.makedirs(outputdir, exist_ok=True)
fstem = '{}_ch{:02d}_{}'.format(dataset, channel, metric)
filepath = os.path.join(outputdir, '{}.log'.format(fstem))
logging.basicConfig(filename=filepath, level=logging.INFO)
# Compute median values per plane for X and Y concatenation.
meds_X, meds_Y = [], []
img_fitted = np.ones(tilesize, dtype='float')
for plane in range(n_planes):
msg = 'Processing ch{:02d}:plane{:03d}'.format(channel, plane)
logger.info(msg)
#print(msg)
dstack = []
for sbd in sbd_channel[plane::n_planes]:
dstack.append(np.squeeze(sbd.data_segment().data()))
for ax, meds in {0: meds_X, 1: meds_Y}.items():
dstacked = np.concatenate(dstack, axis=ax)
ma_data = np.ma.masked_array(dstacked, dstacked < noise_threshold)
meds.append(np.ma.median(ma_data, axis=ax))
# Estimate the profiles from the medians.
out = [metric, z_range, quantile_threshold, polynomial_order]
for dim, meds in {'X': meds_X, 'Y': meds_Y}.items():
out.append(np.stack(meds, axis=0))
data_sel, dm, z_sel = select_z_range(out[-1], z_range,
quantile_threshold)
data_mean, data_norm, data_norm_mean = normalize_profile(data_sel)
data_fitted, data_fitted_norm = fit_profile(data_norm_mean,
polynomial_order)
if dim == 'X':
img_fitted *= data_fitted_norm
elif dim == 'Y':
img_fitted *= data_fitted_norm[:, np.newaxis]
# Save medians and parameters.
fstem = '{}_ch{:02d}'.format(dataset, channel)
filepath = os.path.join(outputdir, '{}_shading.pickle'.format(fstem))
with open(filepath, 'wb') as f:
pickle.dump(out, f, pickle.HIGHEST_PROTOCOL)
# Save estimated shading image.
img = np.array(img_fitted * np.iinfo(dtype).max, dtype=dtype)
filepath = os.path.join(outputdir, '{}_shading.tif'.format(fstem))
imsave(filepath, img)
# Print a report page to pdf.
generate_report(outputdir, dataset, channel=channel)
def CZIMetadatatoDictionaries(InputDirectory, CziName):
czi = czifile.CziFile(InputDirectory + str(CziName))
czi_array = czifile.imread(InputDirectory +
str(CziName)) #read the czi file.
czi_array = czi_array.squeeze(
) #take out the dimensions that are not important
#print(czi_array.shape)
####### Extract the metadata
metadata = czi.metadata #reading the metadata from CZI
root = ET.fromstring(metadata) #loading metadata into XML object
##### Making a dictionry from all of the channel data only if it has ID that hs the channel number as the key and the dye name as the value
ChannelDictionary = {}
for neighbor in root.iter('Channel'):
TempDict = {}
TempDict = neighbor.attrib
if 'Id' in TempDict: #for the metadata lines that start with ID
#print(TempDict) #test
Search = r"(\w+):(\d)" #separate the channel:1 into two parts .. only keep the number
Result = re.search(Search, TempDict['Id'])
Search2 = r"(\w+)-(.+)"
Result2 = re.search(Search2, TempDict['Name'])
ChannelDictionary[Result2.group(1)] = Result.group(
2
) #make a new dictionary where that number (0 based!) is the channel/key to the and the value is the dye name
#print(ChannelDictionary)
####### pull out the channels and make stacks
if "AF405" in ChannelDictionary.keys():
AF405index = ChannelDictionary["AF405"]
AF405Stack = czi_array[int(AF405index), ...]
else:
print("AF405 is not in this file")
AF488Stack = 'empty'
if "AF488" in ChannelDictionary.keys():
AF488index = ChannelDictionary["AF488"]
AF488Stack = czi_array[int(AF488index), ...]
else:
print("AF488 is not in this file")
AF488Stack = 'empty'
if "AF647" in ChannelDictionary.keys():
AF647index = ChannelDictionary["AF647"]
AF647Stack = czi_array[int(AF647index), ...]
else:
print("AF647 is not in this file")
AF647Stack = 'empty'
if "AF546" in ChannelDictionary.keys():
AF546index = ChannelDictionary["AF546"]
AF546Stack = czi_array[int(AF546index), ...]
elif "At550" in ChannelDictionary.keys():
AF546index = ChannelDictionary["At550"]
AF546Stack = czi_array[int(AF546index), ...]
else:
print("AF546 is not in this file")
AF546Stack = 'empty'
return (AF405Stack, AF488Stack, AF647Stack, AF546Stack)
def read_czi(filename,
trim=False,
swapaxes=True,
return_metadata=False,
metadata_only=False):
"""Read a czi file into an ndarray
Args:
filename: string
Path to czi file
trim: bool
If true, remove last frame if it contains blank slices
swapaxes: bool
If true, switches first two axes to produce a stack order ctzxy
return_metadata: bool
If true, returns a tuple of stack, first distance, z interval
metadata_only: bool
If true, just return the metadata features starting_positions
and z_interval. Stack is returned as None.
Returns:
stack: ndarray
Image stack in dimensions [t,c,z,x,y] (no swap) or
[c,t,z,x,y] (swapped)
starting_positions: list of floats
List of the position, in microns, of the first slice in the Z
stack of each file, taken from czi file metadata.
z_interval: float
Size of Z slice, in microns, taken from czi metadata
"""
def frame_incomplete(stack3d):
"""Determine if frame is incomplete."""
for slice in stack3d:
# If only value in slice is 0, frame is incomplete.
if ((np.min(slice) == 0) & (np.max(slice) == 0)):
return True
return False
if not metadata_only:
stack = czifile.imread(filename)
stack = np.squeeze(stack)
# Trim off last frame if incomplete.
if trim:
if frame_incomplete(stack[-1, 0]):
stack = stack[:-1]
if (swapaxes):
stack = np.swapaxes(stack, 0, 1)
else:
stack = None
if return_metadata:
handle = czifile.CziFile(filename)
metadata = handle.metadata()
root = ET.fromstring(metadata)
# Pull first distance and z interval, convert to microns.
first_dist = float(
root.findall('.//ZStackSetup')[0][8][0][0].text) * 1e6
#last_dist = root.findall('.//ZStackSetup')[0][9][0][0].text
z_interval = float(
root.findall('.//ZStackSetup')[0][10][0][0].text) * 1e6
handle.close()
return stack, first_dist, z_interval
else:
return stack
def read_linescan_timeseries_with_czifile(self):
with czifile.CziFile(self.fname) as czi:
attachDir = czi.attachment_directory
subblkdir = czi.subblock_directory
print(subblkdir)
for attachment in czi.attachments():
print(attachment.attachment_entry.name)
print(attachment.attachment_entry.content_file_type)
if (attachment.attachment_entry.name == 'TimeStamps'):
self.timestamps = attachment.data()
if 'SizeT' in self.metadata:
self.timestamps = self.timestamps[0:self.
metadata['SizeT'] -
1]
self.t0 = self.timestamps[0]
print('Reading timestamps successful')
if (attachment.attachment_entry.name == 'Image'):
# embedded_czifile = czifile.CziFile(
# BytesIO(
# next(
# attachment for attachment in czi.attachments()
# if attachment.attachment_entry.name == 'Image'
# ).data(raw=True)
# )
# )
# self.embedded_image = embedded_czifile.asarray()
# self.embedded_image
extract_attachments(self, czi)
print('Reading imageattachment successful')
if (attachment.attachment_entry.name == 'EventList'):
events = attachment.data()
eventtimes = []
eventdict = {'time': 0, 'type': '', 'name': ''}
for item in events:
# print(item.description)
self.eventlist.append({
'time':
item.time,
'type':
item.EV_TYPE[item.event_type],
'name':
item.description
})
self.eventtimes = [
item['time'] for item in self.eventlist
]
print('Reading eventlist successful')
if ((self.metadata['AcquisitionMode'] == 'Line')
and (self.metadata['isTimeSeries'] == True)):
print("Detected: Zeiss linescan in timeseries")
self.data = czi.asarray()
imgShAll = np.array(czi.shape)
imgAxAll = list(czi.axes)
# find index of Channel axis
axIdC = [
idx for idx, item in enumerate(imgAxAll) if item == 'C'
].pop()
# find index of X axis
axIdX = [
idx for idx, item in enumerate(imgAxAll) if item == 'X'
].pop()
# find index of T axis
axIdT = [
idx for idx, item in enumerate(imgAxAll) if item == 'T'
].pop()
# self.data.resize(imgShAll[axIdC],imgShAll[axIdT],imgShAll[axIdX]) # CTX
self.data.resize(imgShAll[axIdX], imgShAll[axIdT],
imgShAll[axIdC]) # XTC
def read_metadata_main(self, fname, metadata_elements):
with czifile.CziFile(fname) as czi:
metadata_xmlstr = czi.metadata(raw=True)
self.metadata = metadata_xmlstr_to_metadata_dict(metadata_xmlstr)
