def func2():
root_path = "/home/jjx/Biology/data/data_modified_emb_gt_48_center_crop/test/"
# root_path = "/home/jjx/Biology/test/test"
root_path = "/home/jjx/Biology/NeuralTrack2/test/test/"
paths = glob.glob(os.path.join(root_path, "ins/*"))
paths.sort()
out_dir = "./result/test_test/"
mkdir_if_not_exist(out_dir)
for path in tqdm(paths):
name = os.path.basename(path)
ins = orderedLabel(tifffile.imread(path))
over_seg = orderedLabel(tifffile.imread(os.path.join(root_path, "over_segs", name)))
under_seg = orderedLabel(tifffile.imread(os.path.join(root_path, "under_segs", name)))
proj_ins = np.max(ins, axis=0).astype(int)
proj_over_seg = np.max(over_seg, axis=0).astype(int)
proj_under_seg = np.max(under_seg, axis=0).astype(int)
fig, axs = plt.subplots(1, 3, figsize=(30, 10))
axs[0].imshow(proj_ins, cmap=plt.cm.gist_ncar)
axs[0].set_title("Ins", fontsize=40)
axs[1].imshow(proj_over_seg, cmap=plt.cm.gist_ncar)
axs[1].set_title("overSeg", fontsize=40)
axs[2].imshow(proj_under_seg, cmap=plt.cm.gist_ncar)
axs[2].set_title("underSeg", fontsize=40)
fig.savefig(os.path.join(out_dir, name.split(".")[0]+"_color.jpg"))
plt.close(fig)
def get_data(cell_name, auxilin_dir=auxilin_dir, normalize=True):
'''Loads in X and Y for one cell
Returns
-------
X : np.ndarray
has shape (W, H, num_images)
Y : np.ndarray
has shape (W, H, num_images)
'''
cell_name = 'Cell1_1s'
data_dir = oj(auxilin_dir, 'A7D2', cell_name) # 'A7D2', 'EGFB-GAK-F6'
fname1 = os.listdir(oj(data_dir, 'TagRFP'))[0]
fname2 = os.listdir(oj(data_dir, 'EGFP'))[0]
X = imread(oj(
data_dir, 'TagRFP',
fname1)) #.astype(np.float32) # X = RFP(clathrin) (num_images x H x W)
Y = imread(oj(
data_dir, 'EGFP',
fname2)) #.astype(np.float32) # Y = EGFP (auxilin) (num_image x H x W)
if normalize:
X = (X - np.mean(X)) / np.std(X)
Y = (Y - np.mean(Y)) / np.std(Y)
return X, Y
def load_imgs(imgs_paths):
"""Returns images as numpy array from the imgs_paths
PARAMS:
imgs_paths (str / list(str)): path of the images to load (relative or absolute)
type str or list (changes type of the output)
RETURNS:
images (np.array / list(np.array)): loaded images as numpy arrays
single array if imgs_paths in of type str
list of arrays if imgs_paths in of type list
"""
if isinstance(imgs_paths, str):
print("Loading: {}".format(imgs_paths))
images = tifffile.imread(imgs_paths)
elif isinstance(imgs_paths, list):
# Returns loaded images as a list
images = []
for filename in imgs_paths:
print("Loading: {}".format(filename))
images.append(tifffile.imread(filename))
else:
raise ValueError("Wrong type from imgs_paths: {}".format(
type(imgs_paths)))
return images
def _create(
output,
images,
bgnds=[],
darks=[],
groupname="original",
images_dsetname="images",
bgnds_dsetname="bgnds",
darks_dsetname="darks",
dtype="i2",
generator=True,
):
fd = h5py.File(output, "w")
grp = fd.create_group(groupname)
ny, nx = imread(images[0]).shape # All images are same shape
if len(bgnds) > 0:
bgnds_dset = grp.create_dataset(bgnds_dsetname, (len(bgnds), ny, nx), dtype=dtype)
if len(darks) > 0:
darks_dset = grp.create_dataset(darks_dsetname, (len(darks), ny, nx), dtype=dtype)
images_dset = grp.create_dataset(images_dsetname, (len(images), ny, nx), dtype=dtype)
for i, im in enumerate(bgnds):
bgnds_dset[i, :, :] = imread(im)[:, :]
for i, im in enumerate(darks):
darks_dset[i, :, :] = imread(im)[:, :]
for i, im in enumerate(images):
images_dset[i, :, :] = imread(im)[:, :]
if generator:
yield i, im
def pull_item(self, idx):
img = tifffile.imread(os.path.join(self.data_root, self.tif_paths[idx]))
gt_box = self.generate_gtbox(os.path.join(self.data_root, self.json_paths[idx]), label=0)
temp = 0
while(not gt_box):
if temp == (len(self.list_id) - 1):
idx = 0
else:
temp = idx
idx = min(idx+1, len(self.list_id)-1)
img = tifffile.imread(os.path.join(self.data_root, self.tif_paths[idx]))
gt_box = self.generate_gtbox(os.path.join(self.data_root, self.json_paths[idx]), label=0)
img0 = np.sum(img, axis = 0) > 0
img1 = np.sum(img, axis = 1) > 0
img2 = np.sum(img, axis = 2) > 0
img0 = img0.astype(np.int)
img1 = img1.astype(np.int)
img2 = img2.astype(np.int)
img0[img0>0] = 1
img1[img1>0] = 1
img2[img2>0] = 1
img_ = np.stack([img0, img1, img2], axis=0)
height, width, length = img.shape
img = (img>0).astype(np.int)
img = np.expand_dims(img, axis=0)
#gt_box = np.array(gt_box)
return torch.from_numpy(img), gt_box, height, width, length, torch.from_numpy(img_)
def load_tiff(pwd, start_index=None, end_index=None):
'''
Usage:
- just read all the image under the pwd
'''
if os.path.isdir(pwd):
img_stack = []
names = []
for imgname in os.listdir(pwd):
if imgname.endswith('.tif'):
names.append(imgname)
names = sorted(names, key = lambda x: int(filter(str.isdigit, x)))
if (start_index) and (end_index) and (0 <= start_index < end_index) and (start_index < end_index <= len(names)):
names = names[start_index:end_index]
for z,imgname in enumerate(names):
msg = "reading %d-th frame" % (z+1)
end = len(names)
bar('info')(msg, z+1, end)
img_pwd = os.path.join(pwd, imgname)
if img_pwd.endswith('.tif'):
img = tiff.imread(img_pwd)
img_stack.append(img)
return np.array(img_stack)
else:
return tiff.imread(pwd)
def make_memmap_from_tiff_list(src,
dst,
cores=8,
dtype="float32",
verbose=True):
"""
Function to make a memory mapped array from a list of tiffs
"""
if type(src) == str and os.path.isdir(src):
src = listdirfull(src, keyword=".tif")
src.sort()
im = tifffile.imread(src[0])
if not dtype: dtype = im.dtype
#init
dst = os.path.join(dst, "input_memmap_array.npy")
memmap = load_memmap_arr(dst,
mode="w+",
dtype=dtype,
shape=tuple([len(src)] + list(im.shape)))
#run
if cores <= 1:
for i, s in enumerate(src):
memmap[i, ...] = tifffile.imread(s)
memmap.flush()
else:
iterlst = [(i, s, dst, verbose) for i, s in enumerate(src)]
p = mp.Pool(cores)
p.starmap(make_memmap_from_tiff_list_helper, iterlst)
p.terminate
return dst
def get_images(cell_dir: str):
'''Loads in X and Y for one cell
Params
------
cell_dir
Path to directory for one cell
Returns
-------
X : np.ndarray
has shape (W, H, num_images)
Y : np.ndarray
has shape (W, H, num_images)
'''
for name in os.listdir(oj(cell_dir, 'TagRFP')):
if 'tif' in name:
fname1 = name
for name in os.listdir(oj(cell_dir, 'EGFP')):
if 'tif' in name:
fname2 = name
print(cell_dir)
X = imread(oj(cell_dir, 'TagRFP', fname1)) # .astype(np.float32) # X = RFP(clathrin) (num_images x H x W)
Y = imread(oj(cell_dir, 'EGFP', fname2)) # .astype(np.float32) # Y = EGFP (auxilin) (num_image x H x W)
return X, Y
def load_image(self, filepath, bit_depth=8):
"""Loads images from .png files
-1 channel: soft tissue contrast only
-2 channels: soft and bone tissue contrasts
-3 channels: lung, soft and bone tissue contrasts"""
def normalize(image, bit_depth=8, rg=(-1, 1)):
"""Linearly rescales data to the wanted range (rg)"""
rg_width = float(rg[1] - rg[0])
return image /(2**bit_depth -1) *rg_width +rg[0]
if self.n_channels == 1:
#Loads only the soft tissue contrast channel
image = tifffile.imread(filepath)
if len(image.shape) == 3:
return normalize(np.expand_dims(image[...,1], axis=-1), bit_depth)
else:
return normalize(np.expand_dims(image, axis=-1), bit_depth)
elif self.n_channels == 2:
return normalize(tifffile.imread(filepath)[...,1:], bit_depth)
elif self.n_channels == 3:
return normalize(tifffile.imread(filepath), bit_depth)
else:
raise ValueError(f"Wrong number of channels {self.n_channels}")
def DownloadFromDrive(file_name, formato="tif", path=os.getcwd(), force=False):
""" Downloads an image from google drive and deletes it at the end
:param file_name: file to download
:param formato: formato de el asset a descargar
:param path: path to download the image
:param force: if we want to force the donwload (overwrites if exist)
:return: the full path to the donwloaded image
"""
image_name_full_original = os.path.join(path,
addFormat(file_name, formato))
if os.path.isfile(image_name_full_original) and not force:
return image_name_full_original
# Download file from drive
drive = AuthDrive()
file_list = drive.ListFile({
'q':
"title contains '{}' and trashed=false".format(file_name)
}).GetList()
# expr = re.compile("%s-(\d+)-(\d+)\.%s" % (file_name, formato))
expr = re.compile("(%s)(-\d+-\d+)?\.%s" % (file_name, formato))
f_downs = []
for file_down in file_list:
title = file_down.attr["metadata"]["title"]
fmo = expr.fullmatch(title)
if fmo is not None:
image_name_full = os.path.join(path, addFormat(title, formato))
logger.info("Downloading image %s from drive" % title)
file_down.GetContentFile(image_name_full)
f_downs.append(image_name_full)
# Delete image from drive
filele = drive.CreateFile()
filele.auth.service.files().delete(
fileId=file_down['id']).execute()
if len(f_downs) == 1:
return f_downs[0]
if len(f_downs) == 4:
# Case when the image is divided in 4 patches.
from skimage.external import tifffile
f_downs = sorted(f_downs)
up = np.concatenate([tifffile.imread(f) for f in f_downs[:2]], axis=1)
down = np.concatenate([tifffile.imread(f) for f in f_downs[2:]],
axis=1)
tifffile.imsave(image_name_full_original,
np.concatenate((up, down), axis=0))
for fd in f_downs:
os.remove(fd)
return image_name_full_original
raise IOError("files {} dont know how to concat them".format(f_downs))
def rate_sample_single(sample_ind,
sample_info,
num_segment=10000,
dims_info=None,
rate=0.1):
tiff = tifffile.imread(sample_info[0])
gt = tifffile.imread(sample_info[-1])
gt.resize(tiff.shape)
gt_padded = image_pad(gt, dims_info.pad_size)
inds = np.arange(gt.size)
range_mask = dims_info.range_mask_get(tiff.shape)
inds_f = inds[np.logical_and(gt > 0, range_mask).flatten()]
inds_b = inds[np.logical_and(gt == 0, range_mask).flatten()]
if len(inds_f) > 0:
num_segment_b = int(rate * num_segment)
num_segment_f = num_segment - num_segment_b
inds_sel_f = np.random.choice(inds_f, num_segment_f)
inds_sel_b = np.random.choice(inds_b, num_segment_b)
#inds_sel = list(inds_sel_b) + list(inds_sel_f)
segments = []
for i in inds_sel_f:
segments.append([sample_ind, i, 1])
for i in inds_sel_b:
segments.append([sample_ind, i, 0])
#segments = [[sample_ind,i,1] for i in inds_sel]
return segments
else:
num_segment_b = int(rate * num_segment)
segments = [[sample_ind, i, 0] for i in inds_sel_b]
return segments
def stich_single(image_filepath, mask_filepath):
image_paths = glob.glob(image_filepath)
mask_paths = glob.glob(mask_filepath)
image_paths = sorted(image_paths)
mask_paths = sorted(mask_paths)
img = tifffile.imread(image_paths[8])
mask = tifffile.imread(mask_paths[8])
print(img.max(), mask.max())
all_images = []
for image_path in image_paths:
all_images.append(tifffile.imread(image_path) / RGB_bits)
all_masks = []
for mask_path in mask_paths:
all_masks.append(tifffile.imread(mask_path) / mask_bits)
stiched = gldata.stich_images(all_images)
stiched_masks = gldata.stich_images(all_masks)
return stiched, stiched_masks
def func2():
root_path = "/home/jjx/Biology/logs/test_synthesize_embcls_tcl_64_center_crop_merge_0.5/visual_results/"
out_path = "./results/test_synthesize_embcls_tcl_64_center_crop_merge_0.5/"
os.makedirs(out_path, exist_ok=True)
log_file = os.path.join(out_path, "metric_PAT_test_synthesize_embcls_tcl_64_center_crop_merge_0.5.txt")
logger = set_logger(log_file, "log_metric_pat")
names = os.listdir(root_path)
names.sort()
precs = []
for name in tqdm(names):
ins_paths = glob.glob(os.path.join(root_path, name, "*_gt.tif"))
for ins_path in ins_paths:
rd = os.path.basename(ins_path).split(".")[0]
ins_name = "_".join(rd.split("_")[:-1])
seg = tifffile.imread(os.path.join(root_path, name, ins_name+"_merge.tif"))
gt = tifffile.imread(os.path.join(root_path, name, ins_name+"_gt.tif"))
# over_seg = orderedLabel(tifffile.imread(os.path.join(root_path, name, ins_name+"_over_seg.tif")))
# under_seg = orderedLabel(tifffile.imread(os.path.join(root_path, name, ins_name+"_under_seg.tif")))
# cross_seg = tifffile.imread(os.path.join(root_path, name, ins_name+"_cross_seg.tif"))
seg = seg[10:-10, 10:-10, 10:-10]
gt = gt[10:-10, 10:-10, 10:-10]
logger.info("{}_{}:".format(name, ins_name))
precision, _ = pat_precision(seg, gt, logger)
precs.append(precision)
logger.info("{}_{} precision: {}".format(name, ins_name, precision))
logger.info("")
logger.info("Average of total {} is {}".format(len(precs), np.mean(precs)))
def read_EM(path):
x_train = tifffile.imread(path + 'train-volume.tif')
x_train = img_as_float32(x_train)
t_train = tifffile.imread(path + 'train-labels.tif')
t_train = img_as_float32(t_train)
x_test = tifffile.imread(path + 'test-volume.tif')
x_test = img_as_float32(x_test)
return x_train, t_train, x_test
def blend(dct):
'''0=L, 1=R'''
fl0 = dct['fl0']; fl1 = dct['fl1']
alpha=dct['alpha']; im0 = tifffile.imread(fl0); dtype = im0.dtype; im1 = tifffile.imread(fl1)
ch = '_C'+dct['fl0'][dct['fl0'].rfind('channel')+7:dct['fl0'].rfind('channel')+9]
tifffile.imsave(os.path.join(dct['dst'], dct['name'], dct['name']+ch+'_Z'+str(dct['zplane']).zfill(4)+'.tif'), (im0*(1-alpha) + im1* (alpha)).astype(dtype), compress=1)
[os.remove(xx) for xx in [fl0,fl1]]
return
def check_cell_center_to_resampled(brain,
zstart,
zstop,
dst,
annotation=False):
"""
maps cnn cell center coordinates to resampled stack
inputs:
brain = path to lightsheet processed directory
zstart = beginning of zslice
zstop = end of zslice
dst = path of tif stack to save
NOTE: 20+ PLANES CAN OVERLOAD MEMORY
"""
start = time.time()
#doing things without loading parameter dict, could become a problem
tifs = [xx for xx in os.listdir(brain) if xx[-4:] == ".tif"]
tifs.sort()
raw = tifffile.imread(tifs[len(tifs) - 1])
pth = os.path.join(brain, "clearmap_cluster_output/cells.npy")
cells = np.load(pth) #this is in x,y,z!!!!!!!!!!!!!!!
cells = cells[(cells[:, 2] >= zstart) &
(cells[:, 2] <= zstop - 1)] #-1 to account for range
cell_centers = np.zeros(raw.shape)
for i, r in enumerate(cells):
cell_centers[r[2] - zstart, r[1] - 1:r[1] + 1,
r[0] - 1:r[0] + 1] = 50000
rbg = np.stack([
raw.astype("uint16"),
cell_centers.astype("uint16"),
np.zeros_like(raw)
], -1)
#add the annotation volume transformed-to-raw-space as the 'blue' channel in the RGB stack (3 color overlay)
if annotation:
#add 'blue' channel to rbg stack
ann = tifffile.imread(annotation)
rbg = np.stack([
raw.astype("uint16"),
cell_centers.astype("uint16"),
ann("uint16")
], -1)
resize_merged_stack(
rbg,
os.path.join(
dst, "{}_raw_cell_centers_resized_z{}-{}.tif".format(
os.path.basename(brain), zstart, zstop)), "uint16", 6)
print("took %0.1f seconds to make merged maps for %s" %
((time.time() - start), brain))
def func6():
root_path = "/home/jjx/Biology/logs/test_dsmc_emb_center_crop_synthesize/visual_results/"
root_path = "/home/jjx/Biology/logs/test_synthesize_embcls_tcl_64_center_crop_merge_0.5/visual_results/"
out_path = "./result/test_synthesize_embcls_tcl_64_center_crop_merge_0.5/"
mkdir_if_not_exist(out_path)
names = os.listdir(root_path)
names.sort()
for name in tqdm(names):
# print(name)
ins_paths = glob.glob(os.path.join(root_path, name, "*_gt.tif"))
fig, axs = plt.subplots(len(ins_paths), 4, figsize=(40, 10*len(ins_paths)))
for i, ins_path in enumerate(ins_paths):
rd = os.path.basename(ins_path).split(".")[0]
ins_name = "_".join(rd.split('_')[:-1])
merge = orderedLabel(tifffile.imread(os.path.join(root_path, name, ins_name+"_merge.tif")))
gt = orderedLabel(tifffile.imread(os.path.join(root_path, name, ins_name+"_gt.tif")))
over_seg = orderedLabel(tifffile.imread(os.path.join(root_path, name, "over_seg.tif")))
under_seg = orderedLabel(tifffile.imread(os.path.join(root_path, name, "under_seg.tif")))
proj_merge = np.max(merge, axis=0).astype(int)
proj_gt = np.max(gt, axis=0).astype(int)
proj_over_seg = np.max(over_seg, axis=0).astype(int)
proj_under_seg = np.max(under_seg, axis=0).astype(int)
if len(ins_paths) == 1:
axs[0].imshow(proj_gt, cmap=plt.cm.gist_ncar)
axs[0].set_title(ins_name+"_gt", fontsize=40)
axs[1].imshow(proj_merge, cmap=plt.cm.gist_ncar)
axs[1].set_title(ins_name+"_merge", fontsize=40)
axs[2].imshow(proj_over_seg, cmap=plt.cm.gist_ncar)
axs[2].set_title(ins_name+"_over", fontsize=40)
axs[3].imshow(proj_under_seg, cmap=plt.cm.gist_ncar)
axs[3].set_title(ins_name+"_under", fontsize=40)
else:
axs[i, 0].imshow(proj_gt, cmap=plt.cm.gist_ncar)
axs[i, 0].set_title(ins_name+"_gt", fontsize=40)
axs[i, 1].imshow(proj_seg, cmap=plt.cm.gist_ncar)
axs[i, 1].set_title(ins_name+"_seg", fontsize=40)
axs[i, 2].imshow(proj_over_seg, cmap=plt.cm.gist_ncar)
axs[i, 2].set_title(ins_name+"_over", fontsize=40)
axs[i, 3].imshow(proj_under_seg, cmap=plt.cm.gist_ncar)
axs[i, 3].set_title(ins_name+"_under", fontsize=40)
fig.savefig(os.path.join(out_path, name+"_color.jpg"))
plt.close(fig)
def func2():
root_path = "/home/jjx/Biology/logs/test_dsc_emb_center_crop/visual_results/"
out_path = "./results/statistic_gt/"
os.makedirs(out_path, exist_ok=True)
log_file = os.path.join(out_path, "metric_statistic_gt.txt")
logger = set_logger(log_file, "log_metric_ffn")
names = os.listdir(root_path)
names.sort()
omit_ratio = []
split_ratio = []
merged_ratio = []
correct_ratio = []
ERLs = []
lens = []
for name in tqdm(names):
ins_paths = glob.glob(os.path.join(root_path, name, "*_gt.tif"))
for ins_path in ins_paths:
rd = os.path.basename(ins_path).split(".")[0]
ins_name = "_".join(rd.split("_")[:-1])
seg = tifffile.imread(
os.path.join(root_path, name, ins_name + "_seg.tif"))
gt = tifffile.imread(
os.path.join(root_path, name, ins_name + "_gt.tif"))
seg_skel = skeleton_func(seg,
nums_cpu=1,
anisotropy=(200, 200, 1000),
return_arr=False)
gt_skel = skeleton_func(gt,
nums_cpu=1,
anisotropy=(200, 200, 1000),
return_arr=False)
len_skels = []
for i in gt_skel:
skel = gt_skel[i]
coords = skel.vertices / (200, 200, 1000)
len_s = 0
for e in skel.edges:
len_s += np.linalg.norm(coords[int(e[0])] -
coords[int(e[1])])
len_skels.append(len_s)
logger.info("{}_{}:".format(name, ins_name))
logger.info("average length: {}".format(np.mean(len_skels)))
logger.info("")
lens.append(np.mean(len_skels))
logger.info("Average of total {} is {}".format(len(lens), np.mean(lens)))
def build_iscat_fluo_training(iscat_filepaths, fluo_filepaths):
"""Creates iscat training data and target in data/iscat_seg/[REF_FRAMES / MASKS] for the iSCAT cell segmentation task with fluorent images used as training
ARGS:
iscat_filepaths (list(str)): filepaths of all the iSCAT images to input as returned by utilities.load_data_paths()
fluo_filepaths (list(str)): filepaths of all the fluo images to input as returned by utilities.load_data_paths()
"""
print("Building iSCAT/fluo dataset...")
OUT_PATH = DATA_PATH + 'iscat_fluo/'
os.makedirs(os.path.join(OUT_PATH, 'REF_FRAMES/'), exist_ok=True)
os.makedirs(os.path.join(OUT_PATH, 'MASKS/'), exist_ok=True)
fluo_tirf_imgs = (tifffile.imread(fluo_tirf_img_path)
for fluo_tirf_img_path in fluo_filepaths)
fluo_iscat_imgs = (tifffile.imread(fluo_iscat_img_path)
for fluo_iscat_img_path in iscat_filepaths)
min_size, max_size = 1, 13
for i, (tirf, iscat) in enumerate(zip(fluo_tirf_imgs, fluo_iscat_imgs)):
tirf = processing.coregister(tirf, 1.38)
tirf = ndimage.median_filter(tirf, 5)
tirf = tirf > 300
tirf = ndimage.binary_closing(
tirf, processing.structural_element('circle',
(1, 10, 10))).astype('float32')
iscat = processing.image_correction(iscat)
iscat = processing.enhance_contrast(iscat,
'stretching',
percentile=(1, 99))
iscat = processing.fft_filtering(iscat, min_size, max_size, True)
iscat = processing.enhance_contrast(iscat,
'stretching',
percentile=(3, 97))
# Discards the first image
for j in range(1, tirf.shape[0]):
try:
if (tirf != 0).any():
imageio.imsave(
OUT_PATH + 'REF_FRAMES/' +
"iscat_{}_{}.png".format(i + 1, j + 1),
rescale(iscat[int(j * iscat.shape[0] /
tirf.shape[0])]))
imageio.imsave(
OUT_PATH +
'MASKS/' + "fluo_{}_{}.png".format(i + 1, j + 1),
rescale(tirf[j]))
except IndexError:
# In case the dimensions don't match -> goes to next slice
print("IndexError")
continue
def mask_similarity_transformed_atlas(auto, ann, atlas, dst, verbose=True):
'''
Function to similarity transform atlas, mask based on ostu's method and save out
Inputs
-----------
auto = path to autofluor image from clearmap cluster: './clearmap_cluster_output/autofluo_resampled.tif'
atlas = path to atlas
ann = path to annotation file
dst = location to save files: './clearmap_cluster_output'
Return
----------
masked_atlas_pth
masked_ann_pth
'''
#setup directory
sim_dst = os.path.join(dst, 'similarity_transform')
makedir(dst)
#do similarity transform on atlas to roughly align them - we might consider affine
sim_atlas, tp = similarity_transform(fx=auto,
mv=atlas,
dst=sim_dst,
nm='atlas_similarity_to_auto.tif',
level='intermediate',
cleanup=False)
#transform ann
transformix_command_line_call(src=ann, dst=sim_dst, transformfile=tp)
sim_ann = os.path.join()
#binarize auto
if verbose:
sys.stdout.write('Thresholding similarity tranformed atlas...')
sys.stdout.flush()
mask = otsu_threshold(auto, verbose=verbose)
#mask atlas
if verbose: print('Masking...')
masked_atlas = tifffile.imread(sim_atlas)
masked_ann = tifffile.imread(sim_ann)
masked_atlas[mask == 0] = 0
masked_ann[mask == 0] = 0
#save out
masked_atlas_pth = os.path.join(dst, 'atlas_similarity_to_auto_masked.tif')
masked_ann_pth = os.path.join(dst, 'ann_similarity_to_auto_masked.tif')
if verbose: print('Saving as {}'.format(masked_atlas_pth))
tifffile.imsave(masked_atlas_pth, masked_atlas)
tifffile.imsave(masked_ann_pth, masked_ann)
return masked_atlas_pth, masked_ann_pth
def read_image_ms(path,train_ratio):
img = get_list(path)
d=len(img)
train_size = int(d*train_ratio)
test_size=d-train_size
random.seed(0)
random.shuffle(img)
MS_train = img[:train_size]
MS_test = img[train_size:]
for i in range(test_size):
print('number:',i)
print(MS_test[i])
MS_train_data = numpy.empty([train_size,200,200,4])
MS_test_data = numpy.empty([test_size,200,200,4])
while train_size>0:
imag=tifffile.imread(MS_train[train_size-1])
shape_array = imag.shape
std_array=tuple(numpy.array([200,200,4]))
if (shape_array==std_array)==True:
MS_train_data[train_size-1,:,:,:]=imag.reshape([-1,200,200,4])
else:
os.remove(MS_train[train_size-1])
train_size=train_size-1
while test_size>0:
imag=tifffile.imread(MS_test[test_size-1])
shape_array = imag.shape
std_array=tuple(numpy.array([200,200,4]))
if (shape_array==std_array)==True:
MS_test_data[test_size-1,:,:,:]=imag.reshape([-1,200,200,4])
else:
os.remove(MS_test[test_size-1])
test_size=test_size-1
return MS_train_data,MS_test_data
def roi_checker():
roi1024 = imread('roi.1024.tif')
roi2048 = imread('roi.2048.tif')
#Now that i have the rois loaded i want to see which ROI is not in the other
roi1024unique = np.unique(roi1024)
roi2048unique = np.unique(roi2048)
if len(np.setdiff1d(roi2048unique, roi1024unique)) == 1:
roiToRemove = np.setdiff1d(roi2048unique, roi1024unique)
#Now that we have found the roi giving us a head ache, lets return its value
return np.where(roi2048unique == roiToRemove)[0]
def __getitem__(self, index):
"""Return a data point and its metadata information.
Parameters:
index (int) -- a random integer for data indexing
Returns a dictionary that contains A, B, A_paths and B_paths
A (tensor) -- an image in the input domain
B (tensor) -- its corresponding image in the target domain
A_paths (str) -- image paths
B_paths (str) -- image paths
"""
patch_size = self.opt.patch_size
stride = patch_size - 2 * 5 #5는 patch_offset
#patch_offset 5-8
height = 512
width = 512
patch_offset=5
mod_h = height + stride - np.mod(height - 2 * patch_offset, stride)
mod_w = width + stride - np.mod(width - 2 * patch_offset, stride)
num_patches = (mod_h // stride) * (mod_w // stride)
A_path = self.A_paths[index//num_patches] # make sure index is within then range
# make sure index is within then range
index_B = index % self.B_size
B_path = self.B_paths[index_B//num_patches]
A_img = imread(A_path)
B_img = imread(B_path)
#A_img, B_img = self.get_patch(A_img, B_img)
#print(index)
print(index%num_patches)
A_patch_arr = self.make_patches(A_img)
A_img = A_patch_arr[index%num_patches]
B_patch_arr = self.make_patches(B_img)
B_img = B_patch_arr[index%num_patches]
A_img = np.transpose(A_img ,(1,2,0))
B_img = np.transpose(B_img ,(1,2,0))
A_img = torchvision.transforms.functional.to_tensor(A_img)
B_img = torchvision.transforms.functional.to_tensor(B_img)
A = (A_img-0.2974482899666286)/0.3435000343241166
B = (B_img-0.2974482899666286)/0.3435000343241166
#A.shape (c,h,w)
return {'A': A, 'B': B, 'A_paths': A_path, 'B_paths': B_path}
def check_clearmap_cfos_output(pth, inputs, axis = 0):
'''Function to output cfos registered images from clearmap processed directories.
Useful to determine 'bad' registration or cell count.
Inputs:
pth = pdf file path
inputs = Directories containing processed data
'''
pdf_pages = PdfPages(pth) #compiles into multiple pdfs
#iterate through inputs
for src in inputs:
#load kwargs
dct = load_clearmap_kwargs(src)
#set output directory
outdr = dct['outputdirectory']
#determine elastix cfos to auto output path
elastix_out = os.path.join(outdr, 'clearmap_cluster_output/elastix_auto_to_atlas')
#set registered volume path
auto_2_atl = tifffile.imread(os.path.join(elastix_out, 'result.1.tif'))
#read image
print('Reading autofluo image\n {}'.format(outdr))
auto = tifffile.imread(os.path.join(outdr, 'clearmap_cluster_output/autofluo_for_cfos_resampled.tif'))#.astype('uint8')
print('Reading signal channel image\n {}'.format(outdr))
cfos = tifffile.imread(os.path.join(outdr, 'clearmap_cluster_output/cfos_resampled.tif'))#.astype('uint8')
print('\nPlotting images...\n')
figs = plt.figure(figsize=(8.27, 11.69))
#starting to plot figures
plt.subplot(131)
#plot the result and atlas next to each other
plt.imshow(auto[350], cmap = 'gray'); plt.axis('off'); plt.title('Autofluorescent image', fontsize = 10)
plt.subplot(132)
plt.imshow(auto_2_atl[221], cmap = 'gray'); plt.axis('off'); plt.title('Registered Atlas', fontsize = 10)
plt.subplot(133)
a = np.max(cfos, axis = axis) # coronal view = 1; saggital view = 0
plt.imshow(a, cmap = 'plasma', alpha = 1); plt.axis('off'); plt.title('Viral Spread', fontsize = 10)
#add title to page
plt.text(0.1,0.70, os.path.basename(src), transform = figs.transFigure, size = 16)
#done with the page
pdf_pages.savefig(dpi = 300, bbox_inches = 'tight')
#write PDF document contain composite of all brains
pdf_pages.close()
print('Saved as {}'.format(pth))
def get_resampledvol_n_dimensions(dct):
""" works around param dict in case paths were missaved """
try:
kwargs = load_kwargs(dct)
vol = [xx for xx in kwargs["volumes"] if xx.ch_type =="cellch"][0]
resampled_vol = vol.resampled_for_elastix_vol
resampled_dims = tifffile.imread(resampled_vol).shape
except FileNotFoundError:
fls = listdirfull(os.path.dirname(dct), ".tif"); fls.sort()
resampled_vol = fls[-1] #will be the last one, bc of the 647 channel
resampled_dims = tifffile.imread(resampled_vol).shape
return resampled_dims, resampled_vol
def setUp(self):
# make x, y ranges
self.x = np.arange(64)
self.y = np.arange(128)
# make meshgrid
self.xx, self.yy = np.meshgrid(self.x, self.y)
self.noisy_data = tif.imread(os.path.join(os.path.dirname(__file__),
'..', 'fixtures',
'noisy_data.tif'))
self.raw_data = tif.imread(os.path.join(os.path.dirname(__file__),
'..', 'fixtures',
'raw_data.tif'))
self.myg = Gauss2D(self.noisy_data)
def func1():
no_emb_path = "/home/jjx/Biology/logs/test_emb_48_with_branch_no_emb/visual_results/"
dsc_path = "/home/jjx/Biology/logs/test_emb_48_with_branch_dsc_48/visual_results/"
root_path = "/home/jjx/Biology/logs/test_emb_48_with_branch_tcl_48/visual_results/"
paths = glob.glob(os.path.join(root_path, "*"))
out_dir = "./result/test_emb_48_/"
mkdir_if_not_exist(out_dir)
for path in paths:
name = path.split("/")[-1]
print(path, name)
gt = orderedLabel(tifffile.imread(os.path.join(path, "ins_gt.tif")))
tif_tcl = orderedLabel(tifffile.imread(os.path.join(path, "ins_pred.tif")))
tif_no_emb = orderedLabel(tifffile.imread(os.path.join(no_emb_path, name, "ins_pred.tif")))
tif_dsc = orderedLabel(tifffile.imread(os.path.join(dsc_path, name, "ins_pred.tif")))
w, h, d = tif_tcl.shape
project_tif_tcl = np.max(tif_tcl, axis=0).astype(int)
project_tif_dsc= np.max(tif_dsc, axis=0).astype(int)
project_tif_no_emb = np.max(tif_no_emb, axis=0).astype(int)
project_gt = np.max(gt, axis=0).astype(int)
# proj = np.hstack([project_tif, -1*np.ones((h, 1)), project_gt])
# proj = np.pad(proj, ((1, 1), (1, 1)), 'constant', constant_values=-1)
# plt.imsave(os.path.join(out_dir, name+"_color.jpg"), proj, cmap=plt.cm.gist_ncar)
# plt.imsave(os.path.join(out_dir, name+"_color.jpg"), proj, cmap=plt.cm.gray)
fig, axs = plt.subplots(1, 4, figsize=(60, 15))
axs[0].imshow(project_gt, cmap=plt.cm.gist_ncar)
axs[0].set_title("Ground Truth", fontsize=60)
axs[1].imshow(project_tif_no_emb, cmap=plt.cm.gist_ncar)
axs[1].set_title("no emb", fontsize=60)
axs[2].imshow(project_tif_dsc, cmap=plt.cm.gist_ncar)
axs[2].set_title("dsc", fontsize=60)
axs[3].imshow(project_tif_tcl, cmap=plt.cm.gist_ncar)
axs[3].set_title("tcl", fontsize=60)
# for i in range(4):
# axs[i].set_xlabel("Width", fontsize=60)
# axs[i].set_xlabel("Height", fontsize=60)
fig.savefig(os.path.join(out_dir, name+"_color.jpg"))
plt.close(fig)
def crop_atlas(dct = None, atlasfile=None, verbose=False, **kwargs):
'''Helper function to crop atlas.
Assumes zero based numerics: i.e. that the first entry in a dimension is 0 NOT 1.
Uses numpy's ZYX convention
Inputs:
-----------------------
dct (optional): dictionary consisting of x,y,z ranges to keep.
NOTE: ranges to keep must be
e.g.
dct = {'x': all, 'y': '125:202', 'z': '75:125'}
atlasfile (optional): path to atlas file
Returns:
-----------------------
atlas: np array of cropped atlas
'''
#handle inputs
if not dct:
dct = kwargs['cropatlas']
#load atlas
if not atlasfile:
atlas = tifffile.imread(kwargs['AtlasFile'])
else:
atlas = tifffile.imread(atlasfile)
if verbose: print ('Generating a cropped atlas\n Keeping dims: {}'.format(dct))
#works through each dimension
for ii in range(3):
#set dimension
dim = ['x', 'y', 'z'][ii]
#if dimension is not present keep all
if not dim in dct: dct[dim] = ':'
if str(dct[dim]) == '<built-in function all>': dct[dim] = ':'
#crop areas
crop = np.ones(atlas.shape, dtype=bool)
#keep areas within indices
exec('crop = atlas[{},{},{}]'.format(dct['z'], dct['y'], dct['x']))
#atlas[mask] = 0
return crop
def pixel_shift(a, b):
"""Function to shift the higher of two images intensity towards the lower ones pixel intensity based on median values (more robust to extremes)
"""
a = tifffile.imread(a) if type(a) == str else a
b = tifffile.imread(b) if type(b) == str else b
dtype = a.dtype
am = np.median(a)
bm = np.median(b)
delta = float(np.abs(am - bm))
if am > bm: a = (a - delta).clip(min=0).astype(dtype)
if bm > am: b = (b - delta).clip(min=0).astype(dtype)
return a, b
def __getitem__(self, idx):
img = tifffile.imread(os.path.join(self.data_root, self.tif_paths[idx]))
img = (img - MEAN) / STD
img = img[:300, :300, :300]
img = np.expand_dims(img, axis=0)
gt = tifffile.imread(os.path.join(self.data_root, self.gt_paths[idx]))
gt = gt[:300, :300, :300]
gt = np.expand_dims(gt, axis=0)
ins = tifffile.imread(os.path.join(self.data_root, self.ins_paths[idx]))
ins = ins[:300, :300, :300]
return torch.from_numpy(gt.astype(np.float)), torch.from_numpy(ins.astype(np.float)), torch.from_numpy(img.astype(np.float))
def inj_and_cells(elastixfld, cellch, injch, AtlasFile, threshold=0.075):
###assumes sagittal files
###set paths for
###generate Allen Atlas background
allen = os.path.join(elastixfld, 'allenatlas_inj')
makedir(allen)
allentiff = tifffile.imread(AtlasFile[:-4] + '.tif')[:, 390:, :]
allentiffloc = os.path.join(allen, 'allen_for_inj.tif')
tifffile.imsave(allentiffloc, allentiff)
depth.colorcode(allentiffloc, allen)
grayscale = [
os.path.join(allen, 'proj0.png'),
os.path.join(allen, 'proj1.png'),
os.path.join(allen, 'proj2.png')
]
####parsing out injection site
ninjfld = os.path.join(elastixfld, 'injection_and_cells')
makedir(ninjfld)
injstack = tifffile.imread(injch)[:, 390:, :] ##assumes sagittal ABA 25um
ninjstackloc = os.path.join(ninjfld, 'injectionsite.tif')
#normalize and threshold
mxx = injstack.max()
injstack = (injstack - injstack.min()) / (injstack.max() - injstack.min())
injstack[injstack < threshold] = 0
injstack = injstack * mxx
tifffile.imsave(ninjstackloc, injstack.astype('uint16'))
####applying depth coding
depth.colorcode(ninjstackloc, ninjfld)
###apply overlay
color = [
os.path.join(ninjfld, 'proj0.png'),
os.path.join(ninjfld, 'proj1.png'),
os.path.join(ninjfld, 'proj2.png')
]
nametosave = [
os.path.join(ninjfld, "proj0_overlay.png"),
os.path.join(ninjfld, "proj1_overlay.png"),
os.path.join(ninjfld, "proj2_overlay.png")
]
for x in range(3):
print(grayscale[x], color[x], nametosave[x])
depth.overlay(grayscale[x], color[x], nametosave[x], alpha=0.95)
depth.layout(nametosave[1], nametosave[0], nametosave[2], ninjfld)
print(grayscale)
print(color)
print(nametosave)
writer(elastixfld, "\nFinished inj_and_cells depth coloring overlay")
return
def tile_and_correct_wrapper(params):
from skimage.external.tifffile import imread
import numpy as np
import cv2
try:
cv2.setNumThreads(1)
except:
1 # 'Open CV is naturally single threaded'
from caiman.motion_correction import tile_and_correct
img_name, out_fname, idxs, shape_mov, template, strides, overlaps, max_shifts,\
add_to_movie, max_deviation_rigid, upsample_factor_grid, newoverlaps, newstrides = params
imgs = imread(img_name, key=idxs)
mc = np.zeros(imgs.shape, dtype=np.float32)
shift_info = []
for count, img in enumerate(imgs):
if count % 10 == 0:
print(count)
mc[count], total_shift, start_step, xy_grid = tile_and_correct(img, template, strides, overlaps, max_shifts, add_to_movie=add_to_movie, newoverlaps=newoverlaps, newstrides=newstrides,
upsample_factor_grid=upsample_factor_grid, upsample_factor_fft=10, show_movie=False, max_deviation_rigid=max_deviation_rigid)
shift_info.append([total_shift, start_step, xy_grid])
if out_fname is not None:
outv = np.memmap(out_fname, mode='r+', dtype=np.float32,
shape=shape_mov, order='F')
outv[:, idxs] = np.reshape(
mc.astype(np.float32), (len(imgs), -1), order='F').T
return shift_info, idxs, np.nanmean(mc, 0)
def readMapFilesTif(lNames, path='./tif/', bVerbose=False):
"""readMapFilesTif(lNames, path='./tif/', bVerbose=False)
Read and scale a list of EDS maps with the ROI as the last element.
Convert to float images, scaling to the such that the maximum counts
in the most intense map has a gray value of 1.0
"""
import numpy as np
from skimage.external.tifffile import imread
imgs = []
maxCts = []
for name in lNames:
fPath = path + name + '.tif'
if bVerbose:
print(fPath)
img = imread(fPath)
img = img.astype(float)
maxCts.append(np.max(img))
imgs.append(img)
l = len(imgs)
if bVerbose:
print(l)
print(maxCts)
maxG = np.max(maxCts[0:l-1])
roiG = maxCts[l-1]
if bVerbose:
print(maxG, roiG)
for i in range(l-1):
imgs[i] /= maxG
imgs[l-1] /= roiG
return imgs, maxCts
def imgs_read(imgs_p):
imgs = []
for img_p in imgs_p:
img_array = tifffile.imread(img_p)
img = torch.from_numpy(img_array.astype(int))
imgs.append(img)
return imgs
def stack_to_tif(stack_path, compress=1, wipe=False):
"""
:param stack_path: string, full path of .stack file to be converted to .tif
:param compress: int, compression level to use for tif file
:param wipe: bool, if True stack file will be deleted after saving as tif
:return:
"""
from numpy import array_equal
from os import remove
from os.path import split, sep
from numpy import fromfile
import volTools as volt
from skimage.external import tifffile as tif
dims = volt.getStackDims(split(stack_path)[0] + sep)
im = fromfile(stack_path, dtype='int16')
im = im.reshape(dims[-1::-1])
tif_path = stack_path.split('.')[0] + '.tif'
tif.imsave(tif_path, im, compress=compress)
if wipe:
check_file = tif.imread(tif_path)
if array_equal(check_file, im):
print('Deleting {0}...'.format(stack_path))
remove(stack_path)
else:
print('{0} and {1} differ... something went wrong!'.format(stack_path, tif_path))
def test_subtract():
image = imread(resource('test00001.tiff'))
img_sub = subtract(image, 0.001)
# subtracting background should reduce max intensity
assert np.max(img_sub) <= np.max(image)
# subtracting background should raise min intensity
assert np.min(img_sub) >= np.min(image)
def convert_to_volume(options):
# data = tifffile.imread(options.input_file)
path, files = hack(options.input_file)
os.chdir(path)
data = tifffile.imread(files)
reshapedData = hytra.util.axesconversion.adjustOrder(data, options.tif_input_axes, options.output_axes)
logging.getLogger('stack_to_h5.py').info("Saving h5 volume of shape {}".format(data.shape))
vigra.writeHDF5(reshapedData, options.output_file, options.output_path)
def transform(f):
path = f
city_dir_name = f.split("/")[-3]
image = tifffile.imread(path)
bands = []
for band in range(8):
bands.append(equalize_adapthist(image[..., band]) * 2047)
img = np.array(np.stack(bands, axis=-1), dtype="uint16")
clahe_city_dir = os.path.join(wdata_dir, city_dir_name)
os.makedirs(clahe_city_dir, exist_ok=True)
mul_dir = os.path.join(clahe_city_dir, 'CLAHE-MUL-PanSharpen')
os.makedirs(mul_dir, exist_ok=True)
tifffile.imsave(os.path.join(mul_dir, f.split("/")[-1]), img, planarconfig='contig')
def segmentation_to_hdf5(options):
"""
The generated segmentation is one HDF5 dataset per timestep,
and each of these datasets has shape 1(t),x,y,z,1(c).
"""
out_h5 = h5py.File(options.hdf5Path, 'w')
for timeframe in range(len(options.tif_input_files)):
data = tifffile.imread(options.tif_input_files[timeframe])
if timeframe == 0:
logging.getLogger('segmentation_to_hdf5.py').info("Found image of shape {}".format(data.shape))
data = hytra.util.axesconversion.adjustOrder(data, options.tif_input_axes, 'txyzc')
if timeframe == 0:
logging.getLogger('segmentation_to_hdf5.py').info("Changed into shape {}".format(data.shape))
internalPath = options.hdf5ImagePath % (timeframe, timeframe + 1, data.shape[1], data.shape[2], data.shape[3])
out_h5.create_dataset(internalPath, data=data, dtype='u2', compression='gzip')
time = timeframe
logging.getLogger('segmentation_to_hdf5.py').info("Saved {} timeframes".format(time))
def choixDossierEtStack(self, pathDossier): #, tilt=True):
""" Cette fonction ouvre toutes les images d'un dossier et les met en stack.
Args:
pathDossier: le chemin du dossier contenant les images.
tilt: pour le tri, si ce sont des images avec (tilt=xdeg) dans leurs noms.
Returns:
Le stack d'images recalées dans le sens croissant des angles.
"""
l = listdir(pathDossier)
# filtrage, on ne garde que les tif + enlève fichiers cachés
l = [x for x in l if x.endswith('.tif') and not x.startswith('.')]
#if tilt:
if len(l[0].split('=')) > 2:
l = sorted(l, key=lambda x: float(x.split('=')[-1].split('d')[0]))
#angles = sorted([float(x.split('=')[-1].split('d')[0]) for x in l])
else: # pour images avec les heures, test
l = list(ifilter(lambda x: len(x.split('-')) > 3, l))
l = sorted(l, key=lambda i: float(i.split('-')[2].split('h')[0]))
s = []
# voir si lire info dans l'image puis hdf5 quelque part
#self.sauvegardeInfosTif()
for i in l:
tmp = imread(pathDossier + '/' + i)
tmp = tmp[:884,...]
if tmp.shape[-1] == 3: # si rgb
tmp = rgb2gray(tmp)
s.append(normaliser(tmp))
return np.array(s)
def parser(pth, size = (228, 228)):
'''Function to take image and ROI zip pairs made using ImageJ, generate two lists corresponding to im and masked image.
Note, many of the image masks will be blank images, particularly with a sparsely labeled section.
Parameters
-----------
pth: path to folder containing tif and roi.zip
size: tuple, x by y output parsed image size
Returns
-----------
parsed_im: list of parsed images to appropriate size
parsed_mask: list of parsed binary mask of filled contours (pixelvalue=1)
'''
#set pth to zip and load im
try:
zipfl = [os.path.join(pth, xx) for xx in os.listdir(pth) if 'zip' in xx[-3:]][0]
im = tifffile.imread([os.path.join(pth, xx) for xx in os.listdir(pth) if 'tif' in xx[-3:]][0])
except IndexError:
print('missing files for:\n {}'.format(pth))
return
#unpack zipfl into a list of roi contours; note convention change ImageJ=x,y, cv2=y,x
roilst=[]
zp=zipfile.ZipFile(zipfl)
for i in zp.namelist():
roilst.append(np.fliplr(np.int32(read_roi(zp.open(i)))))
#create a binary mask of im:
mask = np.zeros(im.shape)
cv2.fillPoly(mask, roilst, 1)
#parse up im and mask
parsed_im = generate_blocks(im,size)
parsed_mask = generate_blocks(mask,size)
print('Completed parsing: {}'.format(pth[pth.rfind('/')+1:]))
return parsed_im, parsed_mask
def load_tiff(sc, pwd, start_index=None, end_index=None):
'''
Usage:
- just read all the image under the pwd
'''
if os.path.isdir(pwd):
img_stack = []
names = []
for imgname in os.listdir(pwd):
if imgname.endswith('.tif'):
names.append(imgname)
names = sorted(names, key = lambda x: int(filter(str.isdigit, x)))
if (start_index) and (end_index) and (0 <= start_index < end_index) and (start_index < end_index <= len(names)):
names = names[start_index:end_index]
n = len(names)
def func(name):
img_pwd = os.path.join(pwd, name)
if img_pwd.endswith('.tif'):
img = tiff.imread(img_pwd)
return img
rdd_file = sc.parallelize(names)
return np.squeeze(np.array(rdd_file.map(func).collect()))
else:
return tiff.imread(pwd)
def data_loader(q, ):
for f in tqdm(sorted(os.listdir(test_dir))):
img_path = os.path.join(test_dir, f)
arr = tifffile.imread(img_path)
image = np.stack([arr[..., 4], arr[..., 2], arr[..., 1], arr[..., 0], arr[..., 3], arr[..., 5], arr[..., 6], arr[..., 7]], axis=-1)
if stretch_and_mean:
image = stretch_8bit(image) * 255
mean_band = mean_bands[city_id]
for band in range(len(mean_band)):
image[..., band] -= mean_band[band]
if ohe_city_flag:
ohe_city = np.zeros((image.shape[0], image.shape[1], 4), dtype="float32")
ohe_city[..., cities.index(city_id)] = 2047
image = np.concatenate([image, ohe_city], axis=-1)
image = np.array(image, dtype="float32")
if preprocessing_function == 'caffe':
image_rgb = image[..., :3]
image_bgr = image_rgb[..., ::-1]
image[..., :3] = image_bgr
if not stretch_and_mean:
image = image / 8. - 127.5
else:
if stretch_and_mean:
image = image / 255
else:
image = image / 1024. - 1
padded = np.zeros((1312, 1312, channels), dtype="float32")
padded[:1300, :1300, :] = image
image = padded
q.put((f, image))
q.put((None, None))
as of march 2019 numpy and scikit-image need updating to newer versions using conda
conda install -c anaconda numpy=1.16.0
conda install -c anaconda scikit-image=0.14.2
as does tensor flow gpu - the version installed by flowdec have a dll load error.
watch out for cuda / tensorflow versions compatabiliries.
conda install -c anaconda tensorflow-gpu
I got a load dll error before this install, but tensor flow updated to v1.13 tested ok
"""
from skimage.external.tifffile import imsave, imread
from flowdec import data as fd_data
from flowdec import restoration as fd_restoration
# Load test image from same dir as we execute in
raw = imread('C1-YeastTNA1_1516_conv_RG_26oC_003.tif')
# Load psf kernel image from same dir
kernel = imread('gpsf_3D_1514_a3_001_WF-sub105.tif')
# Run the deconvolution process and note that deconvolution initialization is best kept separate from
# execution since the "initialize" operation corresponds to creating a TensorFlow graph, which is a
# relatively expensive operation and should not be repeated across multiple executions
n_iter = 500
algo = fd_restoration.RichardsonLucyDeconvolver(raw.ndim).initialize()
res = algo.run(fd_data.Acquisition(data=raw, kernel=kernel), niter=n_iter).data
# save the result,
print('Saving result image TIFF file')
# using skimage.external.tifffile.imsave
imsave(('result' + str(n_iter) + 'iterations.tif'), res)
def load(file_name,fr=30,start_time=0,meta_data=None,subindices=None,shape=None,num_frames_sub_idx=np.inf):
'''
load movie from file.
Parameters
-----------
file_name: string
name of file. Possible extensions are tif, avi, npy, (npz and hdf5 are usable only if saved by calblitz)
fr: float
frame rate
start_time: float
initial time for frame 1
meta_data: dict
same as for calblitz.movie
subindices: iterable indexes
for loading only portion of the movie
shape: tuple of two values
dimension of the movie along x and y if loading from a two dimensional numpy array
Returns
-------
mov: calblitz.movie
'''
# case we load movie from file
if os.path.exists(file_name):
name, extension = os.path.splitext(file_name)[:2]
if extension == '.tif' or extension == '.tiff': # load avi file
if subindices is not None:
input_arr = imread(file_name)[subindices, :, :]
else:
input_arr = imread(file_name)
input_arr = np.squeeze(input_arr)
elif extension == '.avi': # load avi file
if subindices is not None:
raise Exception('Subindices not implemented')
cap = cv2.VideoCapture(file_name)
try:
length = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
except:
print('Roll back top opencv 2')
length = int(cap.get(cv2.cv.CV_CAP_PROP_FRAME_COUNT))
width = int(cap.get(cv2.cv.CV_CAP_PROP_FRAME_WIDTH))
height = int(cap.get(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT))
input_arr=np.zeros((length, height,width),dtype=np.uint8)
counter=0
ret=True
while True:
# Capture frame-by-frame
ret, frame = cap.read()
if not ret:
break
input_arr[counter]=frame[:,:,0]
counter=counter+1
if not counter%100:
print(counter)
# When everything done, release the capture
cap.release()
cv2.destroyAllWindows()
elif extension == '.npy': # load npy file
if subindices is not None:
input_arr=np.load(file_name)[subindices]
else:
input_arr=np.load(file_name)
if input_arr.ndim==2:
if shape is not None:
d,T=np.shape(input_arr)
d1,d2=shape
input_arr=np.transpose(np.reshape(input_arr,(d1,d2,T),order='F'),(2,0,1))
else:
raise Exception('Loaded vector is 2D , you need to provide the shape parameter')
elif extension == '.mat': # load npy file
input_arr=loadmat(file_name)['data']
input_arr=np.rollaxis(input_arr,2,-3)
if subindices is not None:
input_arr=input_arr[subindices]
elif extension == '.npz': # load movie from saved file
if subindices is not None:
raise Exception('Subindices not implemented')
with np.load(file_name) as f:
return movie(**f)
elif extension== '.hdf5':
with h5py.File(file_name, "r") as f:
attrs=dict(f['mov'].attrs)
#print attrs
if meta_data in attrs:
attrs['meta_data']=cpk.loads(attrs['meta_data'])
if subindices is None:
# fr=f['fr'],start_time=f['start_time'],file_name=f['file_name']
return movie(f['mov'],**attrs)
else:
return movie(f['mov'][subindices],**attrs)
elif extension == '.mmap':
filename=os.path.split(file_name)[-1]
fpart=filename.split('_')[1:-1]
Yr, dims, T = load_memmap(filename)
d1, d2 = dims
images = np.reshape(Yr.T, [T] + list(dims), order='F')
print('mmap')
return movie(images,fr=fr)
elif extension == '.sbx':
print('sbx')
return movie(sbxread(file_name[:-4],num_frames_sub_idx).transpose([0,3,2,1]),fr=fr)
else:
raise Exception('Unknown file type')
else:
raise Exception('File not found!')
return movie(input_arr,fr=fr,start_time=start_time,file_name=os.path.split(file_name)[-1], meta_data=meta_data)
help='Filename of the sporozyte tiff')
parser.add_argument('--nucleus', required=True, type=str, dest='nucleusFilename',
help='Filename of the nucleus tiff')
parser.add_argument('--input-axes', type=str, dest='inputAxes', default='txy',
help='Axes string defining the input volume shape')
parser.add_argument('--output-axes', type=str, dest='outputAxes', default='txyc',
help='Axes string defining the output volume shape. The last one should be channels (c)')
parser.add_argument('--3channel-out', type=str, dest='threeChannelOut', required=True, help='Filename of the resulting 3 channel HDF5')
parser.add_argument('--nucleus-channel-out', type=str, dest='nucleusChannelOut', required=True, help='Filename of the resulting nucleus channel HDF5')
parser.add_argument('--uint8', action='store_true', dest='use_uint8', help="Add this flag to force conversion to uint 8")
parser.add_argument('--normalize', action='store_true', dest='normalize', help="Add this flag to force normalization between 0 and 1 if uint8 is not specified, 0 to 255 otherwise")
args = parser.parse_args()
# read data
sporoChannel = tiffile.imread(args.sporoFilename)
nucleusChannel = tiffile.imread(args.nucleusFilename)
print("Found input images of dimensions {} and {}".format(sporoChannel.shape, nucleusChannel.shape))
if args.normalize:
# normalize to range 0-1
sporoChannel = (sporoChannel-np.min(sporoChannel))/float(np.max(sporoChannel)-np.min(sporoChannel))
nucleusChannel = (nucleusChannel-np.min(sporoChannel))/float(np.max(nucleusChannel)-np.min(sporoChannel))
# adjust axes
sporoChannel = hytra.util.axesconversion.adjustOrder(sporoChannel, args.inputAxes, args.outputAxes)
nucleusChannel = hytra.util.axesconversion.adjustOrder(nucleusChannel, args.inputAxes, args.outputAxes)
desiredType = 'float32'
if args.use_uint8:
desiredType = 'uint8'
import os
import matplotlib.pyplot as plt
from scipy import ndimage as ndi
from skimage.morphology import disk
from skimage.filters import median
from skimage.external.tifffile import imread
from skimage.filters import threshold_otsu, threshold_adaptive
gitDir = os.getenv('GIT_HOME')
relImg = '/OSImageAnalysis/images/'
imgFile = 'latex.tif'
imgPath = gitDir + relImg + imgFile
image = imread(imgPath).astype(float)
sz = 5
# med_img = median(image, disk(5))
med_img = ndi.filters.median_filter(image, size=(sz,sz))
lowpass = ndi.gaussian_filter(image, 16)
bks = med_img - lowpass
global_thresh = threshold_otsu(bks)
binary_global = bks > global_thresh
block_size = 35
binary_adaptive = threshold_adaptive(med_img, block_size, offset=10)
binary_fill = ndi.binary_fill_holes(1-binary_global).astype(int)
def save_memmap(filenames,base_name='Yr',resize_fact=(1,1,1),remove_init=0,idx_xy=None):
""" Saves efficiently a list of tif files into a memory mappable file
Parameters
----------
filenames: list
list of tif files
base_name: str
the base yused to build the file name. IT MUST NOT CONTAIN "_"
resize_fact: tuple
x,y, and z downampling factors (0.5 means downsampled by a factor 2)
remove_init: int
number iof frames to remove at the begining of each tif file (used for resonant scanning images if laser in rutned on trial by trial)
idx_xy: tuple size 2
for selecting slices of the original FOV, for instance idx_xy=(slice(150,350,None),slice(150,350,None))
Return
-------
fname_new: the name of the mapped file, the format is such that the name will contain the frame dimensions and the number of f
"""
order='F'
Ttot=0;
for idx,f in enumerate(filenames):
print f
if os.path.splitext(f)[-1] == '.hdf5':
import calblitz as cb
if idx_xy is None:
Yr=np.array(cb.load(f))[remove_init:]
else:
Yr=np.array(cb.load(f))[remove_init:,idx_xy[0],idx_xy[1]]
else:
if idx_xy is None:
Yr=imread(f)[remove_init:]
else:
Yr=imread(f)[remove_init:,idx_xy[0],idx_xy[1]]
fx,fy,fz=resize_fact
if fx!=1 or fy!=1 or fz!=1:
try:
import calblitz as cb
Yr=cb.movie(Yr,fr=1)
Yr=Yr.resize(fx=fx,fy=fy,fz=fz)
except:
print('You need to install the CalBlitz package to resize the movie')
raise
[T,d1,d2]=Yr.shape;
Yr=np.transpose(Yr,(1,2,0))
Yr=np.reshape(Yr,(d1*d2,T),order=order)
if idx==0:
fname_tot=base_name+'_d1_'+str(d1)+'_d2_'+str(d2)+'_order_'+str(order)
big_mov=np.memmap(fname_tot,mode='w+',dtype=np.float32,shape=(d1*d2,T),order=order);
else:
big_mov=np.memmap(fname_tot,dtype=np.float32,mode='r+',shape=(d1*d2,Ttot+T),order=order)
# np.save(fname[:-3]+'npy',np.asarray(Yr))
big_mov[:,Ttot:Ttot+T]=np.asarray(Yr,dtype=np.float32)+1e-10
big_mov.flush()
Ttot=Ttot+T;
fname_new=fname_tot+'_frames_' +str(Ttot) + '_.mmap'
os.rename(fname_tot,fname_new)
return fname_new
def edges_visualisation(options):
'''
Script for visualising the WRONG EDGES signalled by the CTC evaluation tool in TRA_log.txt.
For this we show raw data, segmentation, the segmentation after tracking and the groundtruth for frame t and t+1.
Note! Everything has to be in CTC-Format!!
The edges to be added will be shown in Cyan in the groundtruth.
The redundant edges to be deleted in tomato orange in the tracking result.
The wrong semantics will be shown in gold in the tracking result.
The output images for all problematic frames will be written in the ouptut directory. If this directory
is not specified, then the results open in normal pyplot pop-up windows.
'''
if options.output == None:
logging.getLogger('wrong_edges_visualisation.py').info("Figures will open in pop-up windows. Specify an output path if you want to save them all.")
with open(options.txt_path) as txt:
content = txt.readlines()
edges_to_be_added = []
redundant_edges = []
incorrect_semantics = []
problematic_frames = []
set_redundant_edges = False
set_edges_to_be_added = False
set_edges_incorrect_semantics = False
for line in content:
if 'Edges To Be Added' in line:
set_edges_to_be_added = True
continue
elif 'Redundant Edges To Be Deleted' in line:
set_edges_to_be_added = False
set_redundant_edges = True
continue
elif 'Edges with Incorrect Semantics' in line:
set_redundant_edges = False
set_edges_to_be_added = False
set_edges_incorrect_semantics = True
continue
elif '====' in line:
break # reached the end of the error part or TRA_log.txt
if set_redundant_edges or set_edges_to_be_added or set_edges_incorrect_semantics:
fromFrame, toFrame = line.split(' -> ')
# reading both from- and toFrame is sensible, because they are not always consecutive
t = '*'+fromFrame[3:5].strip().zfill(3)
t_next = '*'+toFrame[3:5].strip().zfill(3)
if [t, t_next] not in problematic_frames:
problematic_frames.append([t, t_next])
if set_edges_to_be_added:
if [t, t_next] not in edges_to_be_added:
edges_to_be_added.append([t, t_next])
index = edges_to_be_added.index([t, t_next])
# create a list that contains the frames pattern as first two elements and the GT labels of the last two
edges_to_be_added[index].append(int(fromFrame[-3:-1].strip('=')))
edges_to_be_added[index].append(int(toFrame[-4:-2].strip('=')))
elif set_redundant_edges:
if [t, t_next] not in redundant_edges:
redundant_edges.append([t, t_next])
index = redundant_edges.index([t, t_next])
# create a list that contains the frames pattern as first two elements and the GT labels of the last two
redundant_edges[index].append(int(fromFrame[-3:-1].strip('=')))
redundant_edges[index].append(int(toFrame[-4:-2].strip('=')))
elif set_edges_incorrect_semantics:
if [t, t_next] not in incorrect_semantics:
incorrect_semantics.append([t, t_next])
index = incorrect_semantics.index([t, t_next])
# create a list that contains the frames pattern as first two elements and the GT labels of the last two
incorrect_semantics[index].append(int(fromFrame[-3:-1].strip('=')))
incorrect_semantics[index].append(int(toFrame[-4:-2].strip('=')))
for frames in sorted(problematic_frames):
# extract the frame number
logging.getLogger('wrong_edges_visualisation.py').debug("Looking at error {}".format(frames))
rawim = []
gtim = []
segim =[]
traim = []
for i in range(2):
rawim.append(tifffile.imread(glob.glob(options.raw_path + frames[i] + '.tif')))
gtim.append(tifffile.imread(glob.glob(options.gt_path +frames[i] + '.tif')))
segim.append(tifffile.imread(glob.glob(options.seg_path +frames[i] +'.tif')))
traim.append(tifffile.imread(glob.glob(options.tra_path +frames[i] +'.tif')))
# generating a suitable colormap for showing errors of the tracking result
colors0 = np.array([[0,0,0]]) # background
colors1 = np.array([[0,0,0]])
for grayval in range(1,traim[0].max()+1):
colors0 = np.vstack((colors0, [0,float(grayval)/traim[0].max(), 1-float(grayval)/traim[0].max()]))
for grayval in range(1,traim[1].max()+1):
colors1 = np.vstack((colors1, [0,float(grayval)/traim[1].max(), 1-float(grayval)/traim[1].max()]))
tra_colors0 = colors0.copy()
tra_colors1 = colors1.copy()
# visualise edges to be added as cyan blobs in the tracking result
if any(frames == [x[0],x[1]] for x in edges_to_be_added):
logging.getLogger('wrong_edges_visualisation.py').debug("EDGE TO BE ADDED")
errors = [element for element in edges_to_be_added if [element[0], element[1]] == frames]
for error in errors:
colors0[(int(error[2]))] = [0.5, 1, 1] # error[2] is the label
colors1[(int(error[3]))] = [0.5, 1, 1] # error[2] is the label
# edges to be deleted as tomato orange blobs in the tracking result
if any(frames == [x[0],x[1]] for x in redundant_edges):
logging.getLogger('wrong_edges_visualisation.py').debug("REDUNDANT EDGE")
errors = [element for element in redundant_edges if [element[0], element[1]] == frames]
for error in errors:
tra_colors0[(int(error[2]))] = [1, 0.2, 0.1]
tra_colors1[(int(error[3]))] = [1, 0.2, 0.1]
# edges with incorrect semantics in gold in the tracking result
if any(frames == [x[0],x[1]] for x in incorrect_semantics):
logging.getLogger('wrong_edges_visualisation.py').debug("INCORRECT SEMANTICS")
errors = [element for element in incorrect_semantics if [element[0], element[1]] == frames]
for error in errors:
tra_colors0[(int(error[2]))] = [1, 0.95, 0]
tra_colors1[(int(error[3]))] = [1, 0.95, 0]
colormap0 = matplotlib.colors.ListedColormap(colors0, N=traim[0].max()+1)
colormap1 = matplotlib.colors.ListedColormap(colors1, N=traim[1].max()+1)
tracolormap0 = matplotlib.colors.ListedColormap(tra_colors0, N=traim[0].max()+1)
tracolormap1 = matplotlib.colors.ListedColormap(tra_colors1, N=traim[1].max()+1)
fig, axes = plt.subplots(nrows=2, ncols=4)
# cmap can/should be adjusted here. 'flag' is best for Fluo-SIM 01
name, raw0, axraw0= tifffile.imshow(rawim[0], figure=fig, subplot=241, title='raw image FRAME {}'.format(frames[0].strip('*')), cmap='flag')
axraw0.colorbar.remove()
raw0.axis('off')
name, seg0, axseg0 = tifffile.imshow(segim[0], figure=fig, subplot=242, title='segmentation', cmap=colormap0, vmin=0, vmax=gtim[0].max()+1)
axseg0.colorbar.remove()
seg0.axis('off')
name, tra0, axtra0 = tifffile.imshow(traim[0], figure=fig, subplot=243, title='tracking result', cmap=tracolormap0, vmin=0, vmax=traim[0].max()+1)
axtra0.colorbar.remove()
tra0.axis('off')
name, gt0, axgt0 = tifffile.imshow(gtim[0], figure=fig, subplot=244, title='groundtruth', cmap=colormap0, vmin=0, vmax=gtim[1].max()+1)
axgt0.colorbar.remove()
gt0.axis('off')
name, raw1, axraw1= tifffile.imshow(rawim[1], figure=fig, subplot=245, title='raw image FRAME {}'.format(frames[1].strip('*')), cmap='flag')
axraw1.colorbar.remove()
raw1.axis('off')
name, seg1, axseg1 = tifffile.imshow(segim[1], figure=fig, subplot=246, title='segmentation', cmap=colormap1, vmin=0, vmax=gtim[1].max()+1)
axseg1.colorbar.remove()
seg1.axis('off')
name, tra1, axtra1 = tifffile.imshow(traim[1], figure=fig, subplot=247, title='tracking result', cmap=tracolormap1, vmin=0, vmax=traim[1].max()+1)
axtra1.colorbar.remove()
tra1.axis('off')
name, gt1, axgt1 = tifffile.imshow(gtim[1], figure=fig, subplot=248, title='groundtruth', cmap=colormap1, vmin=0, vmax=gtim[1].max()+1)
axgt1.colorbar.remove()
gt1.axis('off')
fig.tight_layout()
plt.subplots_adjust(wspace = 0.2)
if options.output == None:
plt.show()
else:
plt.savefig(options.output+'/error_frame{}_to_{}'.format(error[0].strip('*'), error[1].strip('*')))
plt.close()
if options.output != None:
logging.getLogger('wrong_edges_visualisation.py').info("Done writing the images. You can open them to view the errors.")
def error_visualisation(options):
'''
Script for visualising the false negatives signalled by the CTC evaluation tool in TRA_log.txt.
For this we show raw data, segmentation, the segmentation after tracking and the groundtruth.
Note! Everything has to be in CTC-Format!!
The false negatives (from tracking) will be shown in red in the groundtruth and in the segmentation.
The false positives from the segmentation will appear pink in the segmentation.
The mergers are shown in purple on the tracking image.
The output images for all problematic frames will be written in the ouptut directory. If this directory
is not specified, then the results open in normal pyplot pop-up windows.
'''
if options.output == None:
logging.getLogger('error_visualisation.py').info("Figures will open in pop-up windows. Specify an output path if you want to save them all.")
with open(options.txt_path) as txt:
content = txt.readlines()
problematic = []
merger = {}
for line in content:
if "[T=" in line:
break
if "T=" in line:
if ' Label=' in line:
merger.setdefault('*'+line[2:4].strip().zfill(3), [])
merger['*'+line[2:4].strip().zfill(3)].append(line[-3:-1].strip('='))
problematic.append('*'+line[2:4].strip().zfill(3))
for frame in sorted(list(set(problematic))):
# extract the frame number
logging.getLogger('error_visualisation.py').debug("Looking at frame {}".format(frame))
rawim = tifffile.imread(glob.glob(options.raw_path+frame+'.tif'))
gtim = tifffile.imread(glob.glob(options.gt_path+frame+'.tif'))
segim = tifffile.imread(glob.glob(options.seg_path+frame+'.tif'))
traim = tifffile.imread(glob.glob(options.tra_path+frame+'.tif'))
# generating a suitable colormap for showing errors of the tracking result
colors = np.array([[0,0,0]]) # background
for grayval in range(1,traim.max()+1):
colors = np.vstack((colors, [0,float(grayval)/traim.max(), 1-float(grayval)/traim.max()]))
tra_colors = colors.copy()
# visualise mergers as purple blobs in the tracking result
if frame in merger.keys():
for label in merger[frame]:
tra_colors[(int(label))] = [0.65, 0, 1]
tracolormap = matplotlib.colors.ListedColormap(tra_colors, N=traim.max()+1)
# groundtruth colormap
err_colors = np.array([[0,0,0]])
for grayval in range(1, gtim.max()+1):
position = np.where(gtim == grayval)
if position[0].shape[0]==0:
err_colors = np.vstack((err_colors, [1,0,0]))
else:
act_grayval = int(ceil(np.median(traim[position])))
if act_grayval == 0:
err_colors = np.vstack((err_colors, [1,0,0])) # false negatives in tracking result
else:
err_colors = np.vstack((err_colors, colors[act_grayval]))
gtcolormap = matplotlib.colors.ListedColormap(err_colors, N=gtim.max()+1)
# creating colormap fot the segmentation
seg_colors = np.array([[0,0,0]])
for grayval in range(1, segim.max()+1):
position = np.where(segim == grayval)
if position[0].shape[0]==0:
seg_colors = np.vstack((seg_colors, [1,0.8,0.9]))
else:
act_grayval = int(ceil(np.median(gtim[position])))
if act_grayval == 0:
seg_colors = np.vstack((seg_colors, [1,0.8,0.9])) # false positives will appear pink
else:
seg_colors = np.vstack((seg_colors, err_colors[act_grayval]))
segcolormap = matplotlib.colors.ListedColormap(seg_colors, N=gtim.max()+1)
fig, axes = plt.subplots(nrows=2, ncols=2)
# cmap can/should be adjusted here. 'flag' is best for Fluo-SIM 01
name, raw, axraw= tifffile.imshow(rawim, figure=fig, subplot=221, title='raw image FRAME {}'.format(frame.strip('*')), cmap='flag')
axraw.colorbar.remove()
raw.axis('off')
name, seg, axseg = tifffile.imshow(segim, figure=fig, subplot=222, title='segmentation', cmap=segcolormap, vmin=0, vmax=gtim.max()+1)
axseg.colorbar.remove()
seg.axis('off')
name, tra, axtra = tifffile.imshow(traim, figure=fig, subplot=223, title='tracking result', cmap=tracolormap, vmin=0, vmax=traim.max()+1)
axtra.colorbar.remove()
tra.axis('off')
name, gt, axgt = tifffile.imshow(gtim, figure=fig, subplot=224, title='groundtruth', cmap=gtcolormap, vmin=0, vmax=gtim.max()+1)
axgt.colorbar.remove()
gt.axis('off')
fig.tight_layout()
plt.subplots_adjust(hspace = 0.2)
if options.output == None:
plt.show()
else:
plt.savefig(options.output+'/error_frame{}'.format(frame.strip('*')))
plt.close()
if options.output != None:
logging.getLogger('error_visualisation.py').info("Done writing the images. You can open them to view the errors.")
def func(name):
img_pwd = os.path.join(pwd, name)
if img_pwd.endswith('.tif'):
img = tiff.imread(img_pwd)
return img
def load(file_name,fr=None,start_time=0,meta_data=None,subindices=None,shape=None):
'''
load movie from file.
Parameters
-----------
file_name: string
name of file. Possible extensions are tif, avi, npy, (npz and hdf5 are usable only if saved by calblitz)
fr: float
frame rate
start_time: float
initial time for frame 1
meta_data: dict
same as for calblitz.movie
subindices: iterable indexes
for loading only portion of the movie
shape: tuple of two values
dimension of the movie along x and y if loading from a two dimensional numpy array
Returns
-------
mov: calblitz.movie
'''
# case we load movie from file
if os.path.exists(file_name):
name,extension = os.path.splitext(file_name)[:2]
if extension == '.tif': # load avi file
print('Loading tif...')
if subindices is not None:
input_arr=imread(file_name)[subindices,:,:]
else:
input_arr=imread(file_name)
input_arr=np.squeeze(input_arr)
# with pims.open(file_name) as f:
# if len(f.frame_shape)==3:
# for ext_fr in f:
# if subindices is None:
# input_arr = np.array(ext_fr)
# else:
# input_arr = np.array([ext_fr[j] for j in subindices])
# elif len(f.frame_shape)==2:
# if subindices is None:
# input_arr = np.array(f)
# else:
# input_arr = np.array([f[j] for j in subindices])
# else:
# raise Exception('The input file has an unknown numberof dimensions')
# necessary for the way pims work with tiffs
# input_arr = input_arr[:,::-1,:]
elif extension == '.avi': # load avi file
#raise Exception('Use sintax mov=cb.load(filename)')
if subindices is not None:
raise Exception('Subindices not implemented')
cap = cv2.VideoCapture(file_name)
try:
length = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
except:
print 'Roll back top opencv 2'
length = int(cap.get(cv2.cv.CV_CAP_PROP_FRAME_COUNT))
width = int(cap.get(cv2.cv.CV_CAP_PROP_FRAME_WIDTH))
height = int(cap.get(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT))
input_arr=np.zeros((length, height,width),dtype=np.uint8)
counter=0
ret=True
while True:
# Capture frame-by-frame
ret, frame = cap.read()
if not ret:
break
input_arr[counter]=frame[:,:,0]
counter=counter+1
if not counter%100:
print counter
# When everything done, release the capture
cap.release()
cv2.destroyAllWindows()
elif extension == '.npy': # load npy file
if subindices is not None:
input_arr=np.load(file_name)[subindices]
else:
input_arr=np.load(file_name)
if input_arr.ndim==2:
if shape is not None:
d,T=np.shape(input_arr)
d1,d2=shape
input_arr=np.transpose(np.reshape(input_arr,(d1,d2,T),order='F'),(2,0,1))
else:
raise Exception('Loaded vector is 2D , you need to provide the shape parameter')
elif extension == '.mat': # load npy file
input_arr=loadmat(file_name)['data']
input_arr=np.rollaxis(input_arr,2,-3)
if subindices is not None:
input_arr=input_arr[subindices]
elif extension == '.npz': # load movie from saved file
if subindices is not None:
raise Exception('Subindices not implemented')
with np.load(file_name) as f:
return movie(**f)
elif extension== '.hdf5':
with h5py.File(file_name, "r") as f:
attrs=dict(f['mov'].attrs)
attrs['meta_data']=cpk.loads(attrs['meta_data'])
if subindices is None:
# fr=f['fr'],start_time=f['start_time'],file_name=f['file_name']
return movie(f['mov'],**attrs)
else:
return movie(f['mov'][subindices],**attrs)
else:
raise Exception('Unknown file type')
else:
raise Exception('File not found!')
return movie(input_arr,fr=fr,start_time=start_time,file_name=file_name, meta_data=meta_data)
def create_label_volume(options):
# read image
# label_volume = vigra.impex.readVolume('/export/home/lparcala/Fluo-N2DH-SIM/01_GT/TRA/man_track000.tif')
# label_volume = tifffile.imread(options.input_tif) # this could work, if tifffile would work.
path, files = hack(options.input_tif)
os.chdir(path)
label_volume = tifffile.imread(files)
logging.getLogger('ctc_gt_to_hdf5.py').info("Found dataset of size {}".format(label_volume.shape))
label_volume = hytra.util.axesconversion.adjustOrder(label_volume, options.tif_input_axes, 'xyztc')
timeaxis = label_volume.shape[3]
split_events = find_splits(options.input_track, options.start_frame)
if not os.path.exists(options.output_file):
#create new folder for gt files
os.mkdir(options.output_file)
else:
# as h5py somehow appends the old file instead of overwriting, do it manually
if os.path.isfile(options.output_file):
os.remove(options.output_file)
elif os.path.isdir(options.output_file):
import shutil
shutil.rmtree(options.output_file)
os.mkdir(options.output_file)
# store object ids per frame and generate mappings
objects_per_frame = []
mapping_per_frame = {}
for frame in range(timeaxis):
label_image = label_volume[..., frame, 0]
mapping_per_frame[frame] = find_label_image_remapping(label_image)
# handle frame zero
if not options.single_frames:
# one holistic volume file
out_h5 = h5py.File(options.output_file, 'w')
ids = out_h5.create_group('ids')
tracking = out_h5.create_group('tracking')
# create empty tracking group for first frame
tracking_frame = tracking.create_group(format(0, options.filename_zero_padding))
else:
frame = 0
out_h5 = h5py.File(options.output_file + format(frame, options.filename_zero_padding) + '.h5', 'w')
tracking_frame = out_h5.create_group('tracking')
save_label_image_for_frame(options, label_volume, out_h5, 0, mapping_per_frame)
for frame in range(timeaxis):
label_image = label_volume[..., frame, 0]
objects = np.unique(label_image)
objects_per_frame.append(set(objects))
if not options.single_frames:
ids.create_dataset(format(frame, options.filename_zero_padding), data=objects, dtype='u2')
# handle all further frames
for frame in range(1, timeaxis):
if options.single_frames:
out_h5 = h5py.File(options.output_file + format(frame, options.filename_zero_padding) + '.h5', 'w')
tracking_frame = out_h5.create_group('tracking')
else:
tracking_frame = tracking.create_group(format(frame, options.filename_zero_padding))
save_label_image_for_frame(options, label_volume, out_h5, frame, mapping_per_frame)
# intersect track id sets of both frames, and place moves in HDF5 file
tracks_in_both_frames = objects_per_frame[frame - 1] & objects_per_frame[frame] - set([0])
moves = zip(list(tracks_in_both_frames), list(tracks_in_both_frames))
# add the found splits as both, mitosis and split events
if frame in split_events.keys():
splits_in_frame = split_events[frame]
# make sure all splits have the same dimension
splits = []
for key, value in splits_in_frame.iteritems():
value = [v for v in value if v in objects_per_frame[frame]]
if key not in objects_per_frame[frame - 1]:
logging.getLogger('ctc_gt_to_hdf5.py').warning("Parent {} of split is not in previous frame {}. Ignored".format(key, frame - 1))
continue
if len(value) > 1:
if len(value) > 2:
logging.getLogger('ctc_gt_to_hdf5.py').warning("Cutting off children of {} in timestep {}".format(key, frame))
# cut off divisions into more than 2
splits.append([key] + value[0:2])
elif len(value) == 1:
# store as move
logging.getLogger('ctc_gt_to_hdf5.py').warning("Store move ({},{}) instead of split into one in timestep {}".format(key, value[0], frame))
moves.append((key, value[0]))
if len(splits) > 0:
splits = np.array(splits)
if options.index_remapping:
splits = remap_events(splits, mapping_per_frame[frame - 1], mapping_per_frame[frame])
tracking_frame.create_dataset("Splits", data=splits, dtype='u2')
mitosis = [splits[i][0] for i in range(splits.shape[0])]
tracking_frame.create_dataset("Mitosis", data=np.array(mitosis), dtype='u2')
if len(moves) > 0:
if options.index_remapping:
moves = remap_events(np.array(moves), mapping_per_frame[frame - 1], mapping_per_frame[frame])
tracking_frame.create_dataset("Moves", data=moves, dtype='u2')
# a little output so that the user knows whats going on
print('Running mip on', arg['<myfile>'])
# Need to take the first system argument as the filename for a TIF file
# test if filename has tiff in it
filename = arg['<myfile>']
# try our main block
try:
if '.tif' in filename or '.tiff' in filename:
# Import skimage so we have access to tiff loading
from skimage.external import tifffile as tif
# here's the real danger zone, did the user give us a real file?
try:
data = tif.imread(filename)
except FileNotFoundError as er:
raise er
if arg['--log']:
import numpy as np
if data.min() > 0:
data = np.log(data)
else:
print(filename, 'had negative numbers, log not taken')
# Trying to set the cmap here opens a new figure window
# need to set up kwargs for efficient argument passing
# plt.set_cmap('gnuplot2')
# plot the data
fig, ax = mip(data)
# readjust the white space (maybe move this into main code later)
T = h1+h2
# Entropy value is calculated
T = -T*scalar
# location where the maximum entropy
# occurs is the threshold for the renyi entropy
location = T.argmax(axis=0)
# location value is used as the threshold
thresh = location - s
return thresh
img = tifffile.imread('ct-example.tif', key=0)
print(img.shape, img.dtype, np.amin(img), np.amax(img))
pdf, cumsum_pdf, s, e, bins = computeRenyiArrays(img)
thr = calcThreshold(pdf, cumsum_pdf, s, e, 3.0)
print("threshold alpha(3) = %.2f" % thr)
t1 = calcThresholdAlpha1(pdf, cumsum_pdf, s, e)
print("threshold alpha(1) = %.2f" % t1)
plt.plot(bins,pdf);
hFig = plt.gcf();
ax = hFig.gca();
# ax.set_ylim(0,50000)
ax.set_yscale('log');
