def load_stack_of_tiff_as_numpy(folder_path, output_filename):
# path = "/home/abailoni_local/trendyTukan_localdata0/datasets/battery_data/Daten for AlbertoBailoni"
# Z, Y, X
# TODO: generalize
collected_dataset = np.empty((100, 1080, 1197, 2), dtype='uint8')
for filename in os.listdir(folder_path):
if filename.endswith(".tiff"):
im = Image.open(os.path.join(folder_path, filename))
imarray = np.array(im)
parts = filename.split("_")
slice_nb = int(parts[-1].split(".")[0]) - 1
if parts[0] == "2ND":
collected_dataset[slice_nb, :, 4:, 0] = imarray
else:
imarray = imarray[:-4]
collected_dataset[slice_nb, :, :-4, 1] = imarray
# channel_nb = 0 if parts[0] == "2ND" else 1
# collected_dataset[slice_nb, :, ]
# print(slice_nb, channel_nb)
# print(imarray.shape, parts[0])
# all_arrays[]
# print(os.path.join(directory, filename))
# continue
else:
continue
print(collected_dataset.max())
writeHDF5(collected_dataset,
os.path.join(folder_path, output_filename),
"data")
def save_infer_output(self, output):
print("Saving....")
if self.get("export_path") is not None:
dir_path = os.path.join(
self.get("export_path"),
self.get("name_experiment", default="generic_experiment"))
else:
try:
# Only works for my experiments saving on trendyTukan, otherwise it will throw an error:
trendyTukan_path = get_trendytukan_drive_dir()
except ValueError:
raise ValueError(
"TrendyTukan drive not found. Please specify an `export_path` in the config file."
)
dir_path = os.path.join(
trendyTukan_path, "projects/pixel_embeddings",
self.get("name_experiment", default="generic_experiment"))
check_dir_and_create(dir_path)
filename = os.path.join(
dir_path,
"predictions_sample_{}.h5".format(self.get("loaders/infer/name")))
print("Writing to ", self.get("inner_path_output", 'data'))
writeHDF5(output.astype(np.float16), filename,
self.get("inner_path_output", 'data'))
print("Saved to ", filename)
# Dump configuration to export folder:
self.dump_configuration(os.path.join(dir_path,
"prediction_config.yml"))
def save_infer_output(self, output):
import h5py
import numpy as np
print("Saving....")
from segmfriends.utils.various import check_dir_and_create
dir_path = os.path.join(
get_trendytukan_drive_path(), "projects/pixel_embeddings",
self.get("name_experiment", default="generic_experiment"))
check_dir_and_create(dir_path)
filename = os.path.join(
dir_path,
"predictions_sample_{}.h5".format(self.get("loaders/infer/name")))
print("Writing to ", self.get("inner_path_output", 'data'))
from segmfriends.utils.various import writeHDF5
writeHDF5(output.astype(np.float16), filename,
self.get("inner_path_output", 'data'))
print("Saved to ", filename)
# Dump configuration to export folder:
self.dump_configuration(os.path.join(dir_path,
"prediction_config.yml"))
"datasets/CREMI/official_test_samples/sample_{}_padded_20160601.hdf".
format(sample))
out_file = os.path.join(
get_trendytukan_drive_path(),
"datasets/CREMI/official_test_samples/full_aligned_samples/sample_{}_aligned_plus_raw_mask.hdf"
.format(sample))
import h5py
GT_mask_file = os.path.join(
get_trendytukan_drive_path(),
"datasets/CREMI/official_test_samples/full_aligned_samples/sample_{}_GT_mask_temp.hdf"
.format(sample))
mask_inner_path = "volumes/labels/mask"
GT_box = np.zeros(padded_shape, dtype="uint32")
GT_box[slice_GT_mask] = 1
from segmfriends.utils.various import writeHDF5
writeHDF5(GT_box, GT_mask_file, mask_inner_path)
from cremi_tools.alignment import realign
# GT_mask_file = os.path.join(get_trendytukan_drive_path(), "datasets/CREMI/alignment_experiments/sample_{}_backaligned.hdf".format(sample))
realign(raw_file,
sample,
out_file,
labels_file=GT_mask_file,
labels_key=mask_inner_path)
os.remove(GT_mask_file)
def run_method_on_graph(method_type,
true_assign,
k=None,
p=None,
n=None,
spectral_method_name=None,
linkage_criteria=None,
add_cannot_link_constraints=None,
signed_edge_weights=None,
multicut_solver_type="kernighanLin",
graph=None,
A_p=None,
A_n=None,
experiment_name=None,
eta=None,
gauss_sigma=None,
project_directory=None,
save_output_segm=False,
affinities=None,
offsets=None,
output_shape=None):
# Run clustering:
tick = time.time()
# print(method_type)
if method_type == "GASP":
# TODO: assert
print(linkage_criteria, add_cannot_link_constraints)
node_labels, _ = run_GASP(
graph,
signed_edge_weights,
# edge_sizes=edge_sizes,
linkage_criteria=linkage_criteria,
add_cannot_link_constraints=add_cannot_link_constraints,
use_efficient_implementations=False,
# **additional_kwargs
)
elif method_type == "multicut":
print(multicut_solver_type)
node_labels = multicut(graph,
None,
None,
signed_edge_weights,
solver_type=multicut_solver_type)
elif method_type == "spectral":
c = Cluster((A_p, A_n))
print(spectral_method_name)
try:
if spectral_method_name == "BNC":
node_labels = c.spectral_cluster_bnc(k=k, normalisation='sym')
elif spectral_method_name == "L-sym":
node_labels = c.spectral_cluster_laplacian(k=k,
normalisation='sym')
elif spectral_method_name == "SPONGE":
# FIXME: not sure about this...
# node_labels = c.geproblem_laplacian(k = k, normalisation='additive')
node_labels = c.SPONGE(k=k)
elif spectral_method_name == "SPONGE-sym":
# node_labels = c.geproblem_laplacian(k = k, normalisation='multiplicative')
node_labels = c.SPONGE_sym(k=k)
else:
raise NotImplementedError
except np.linalg.LinAlgError:
print("#### LinAlgError ({}) ####".format(spectral_method_name))
return
else:
raise NotImplementedError
runtime = time.time() - tick
# Compute scores and stats:
RAND_score = adjusted_rand_score(node_labels, true_assign)
counts = np.bincount(node_labels.astype('int64'))
nb_clusters = (counts > 0).sum()
biggest_clusters = np.sort(counts)[::-1][:10]
# Save stuff/results...
print(runtime, RAND_score)
# print(nb_clusters, biggest_clusters)
# Save config setup:
new_results = {}
# TODO: delete this crap
new_results["method_type"] = method_type
new_results["k"] = k
new_results["p"] = p
new_results["n"] = n
new_results["eta"] = eta
new_results["guass_sigma"] = gauss_sigma
new_results["spectral_method_name"] = spectral_method_name
new_results["linkage_criteria"] = linkage_criteria
new_results["add_cannot_link_constraints"] = add_cannot_link_constraints
new_results["multicut_solver_type"] = multicut_solver_type
new_results["experiment_name"] = experiment_name
new_results["project_directory"] = project_directory
ID = str(np.random.randint(1000000000))
new_results["ID"] = ID
experiment_dir_path = os.path.join(project_directory, experiment_name)
check_dir_and_create(experiment_dir_path)
check_dir_and_create(os.path.join(experiment_dir_path, 'scores'))
result_file = os.path.join(
experiment_dir_path, 'scores',
'{}_{}_{}_{}.json'.format(ID, method_type, spectral_method_name,
linkage_criteria))
# Actually save results:
new_results["runtime"] = runtime
new_results["RAND_score"] = RAND_score
new_results["nb_clusters"] = int(nb_clusters)
new_results["biggest_clusters"] = [int(size) for size in biggest_clusters]
# Save output:
if save_output_segm:
assert output_shape is not None
assert affinities is not None
assert offsets is not None
check_dir_and_create(os.path.join(experiment_dir_path, 'out_segms'))
export_file = os.path.join(
experiment_dir_path, 'out_segms',
'{}_{}_{}.h5'.format(method_type, spectral_method_name,
linkage_criteria))
from segmfriends.utils.various import writeHDF5
writeHDF5(node_labels.reshape(output_shape),
export_file,
'segm',
compression='gzip')
# Delete small segments:
from GASP.segmentation.watershed import SizeThreshAndGrowWithWS
hmap_kwargs = {"offset_weights": [1.0, 1.0], "used_offsets": [1, 2]}
size_grower = SizeThreshAndGrowWithWS(20,
offsets,
hmap_kwargs=hmap_kwargs)
segm_WS = size_grower(affinities, node_labels.reshape(output_shape))
writeHDF5(segm_WS, export_file, 'segm_WS', compression='gzip')
RAND_score_WS = adjusted_rand_score(segm_WS.flatten(), true_assign)
new_results["RAND_score_WS"] = RAND_score_WS
with open(result_file, 'w') as f:
json.dump(new_results, f, indent=4, sort_keys=True)
def get_kwargs_iter_CREMI(fixed_kwargs,
kwargs_to_be_iterated,
init_kwargs_iter=None,
nb_iterations=1):
kwargs_iter = init_kwargs_iter if isinstance(init_kwargs_iter,
list) else []
iter_collected = {}
KEYS_TO_ITER = [
'method_type', 'spectral_method_name', 'linkage_criteria',
'add_cannot_link_constraints'
]
for key in KEYS_TO_ITER:
if key in fixed_kwargs:
iter_collected[key] = [fixed_kwargs[key]]
elif key in kwargs_to_be_iterated:
iter_collected[key] = kwargs_to_be_iterated[key]
else:
raise ValueError("Iter key {} was not passed!".format(key))
fixed_kwargs = deepcopy(fixed_kwargs)
dataset = fixed_kwargs.pop("dataset")
# Load the data:
for _ in range(nb_iterations):
if dataset == "CREMI":
from .load_datasets import get_dataset_data, CREMI_crop_slices, get_dataset_offsets
affs, GT = get_dataset_data(
dataset='CREMI',
sample="B",
crop_slice_str=":,18:28,1020:1120,990:1090",
run_connected_components=True)
n = 10000
# p = fixed_kwargs.pop("p")
print("Creating pixel graph:")
from GASP.utils.graph import build_pixel_lifted_graph_from_offsets
offsets = np.array(get_dataset_offsets("CREMI"))
graph, is_local_edge, true_assign, edge_sizes = build_pixel_lifted_graph_from_offsets(
GT.shape, offsets, GT_label_image=GT)
true_assign = true_assign.astype('uint64')
assert (edge_sizes == 1).all()
uv_ids = graph.uvIds()
k = np.unique(true_assign).shape[0]
# TODO: check sign of affinities
edge_weights = graph.edgeValues(np.rollaxis(affs, 0, 4))
# FIXME: always use additive cost for the moment...
# Compute log costs:
signed_edge_weights = edge_weights - 0.5
print("Create positive and negative graph adj. matrices:")
A_p, A_n = from_edge_list_to_adj_matrix(uv_ids,
signed_edge_weights)
A_p = sparse.csr_matrix(A_p)
A_n = sparse.csr_matrix(A_n)
# Dump data:
experiment_dir_path = os.path.join(
fixed_kwargs['project_directory'],
fixed_kwargs['experiment_name'])
check_dir_and_create(experiment_dir_path)
check_dir_and_create(os.path.join(experiment_dir_path, 'segms'))
check_dir_and_create(os.path.join(experiment_dir_path,
'out_segms'))
export_file = os.path.join(experiment_dir_path, 'out_segms',
'inputs.h5')
from segmfriends.utils.various import writeHDF5
writeHDF5(affs.astype('float32'),
export_file,
'affs',
compression='gzip')
writeHDF5(GT, export_file, 'gt', compression='gzip')
# Start collecting kwargs:
for method_type in iter_collected["method_type"]:
if method_type == "spectral":
for spectral_method_name in iter_collected[
"spectral_method_name"]:
new_kwargs = {}
new_kwargs.update(fixed_kwargs)
iterated_kwargs = {
'method_type': method_type,
'true_assign': true_assign,
'spectral_method_name': spectral_method_name,
'A_p': A_p,
'A_n': A_n,
'n': n,
'eta': None,
'p': None,
'k': k,
'save_output_segm': True,
'output_shape': GT.shape,
'affinities': affs,
'offsets': get_dataset_offsets("CREMI")
}
new_kwargs.update({
k: v
for k, v in iterated_kwargs.items()
if k not in new_kwargs
})
kwargs_iter.append(new_kwargs)
elif method_type == "GASP":
for linkage in iter_collected["linkage_criteria"]:
for CNC in iter_collected[
"add_cannot_link_constraints"]:
new_kwargs = {}
new_kwargs.update(fixed_kwargs)
iterated_kwargs = {
'method_type': method_type,
'true_assign': true_assign,
'linkage_criteria': linkage,
'add_cannot_link_constraints': CNC,
'signed_edge_weights': signed_edge_weights,
'graph': graph,
'eta': None,
'n': n,
'p': None,
'k': k,
'save_output_segm': True,
'output_shape': GT.shape,
'affinities': affs,
'offsets': get_dataset_offsets("CREMI")
}
new_kwargs.update({
k: v
for k, v in iterated_kwargs.items()
if k not in new_kwargs
})
kwargs_iter.append(new_kwargs)
elif method_type == "multicut":
new_kwargs = {}
new_kwargs.update(fixed_kwargs)
iterated_kwargs = {
'method_type': method_type,
'true_assign': true_assign,
'signed_edge_weights': signed_edge_weights,
'graph': graph,
'eta': None,
'n': n,
'p': None,
'k': k,
'save_output_segm': True,
'output_shape': GT.shape,
'affinities': affs,
'offsets': get_dataset_offsets("CREMI")
}
new_kwargs.update({
k: v
for k, v in iterated_kwargs.items()
if k not in new_kwargs
})
kwargs_iter.append(new_kwargs)
else:
raise NotImplementedError
else:
raise NotImplementedError
return kwargs_iter
affs, affs_valid_mask = compute_affinities(GT.astype('int64'),
offsets,
ignore_label=0,
have_ignore_label=True)
# Where it is not valid, we should not predict a boundary label:
affs[affs_valid_mask == 0] = 1
# Combine left and right affinities:
segment_mask = np.logical_and(affs[0], affs[1])
# This functions erode binary mask (segments 1, boundary 0)
eroded_segment_mask = segment_mask.copy()
for z in range(eroded_segment_mask.shape[0]):
eroded_segment_mask[z] = vigra.filters.multiBinaryErosion(
segment_mask[z], radius=2.)
boundary_mask = np.logical_not(eroded_segment_mask)
# Max int32 value:
BOUNDARY_VALUE = 2147483647
modified_GT = GT.copy()
modified_GT[boundary_mask] = BOUNDARY_VALUE
print("Done")
from segmfriends.utils.various import writeHDF5
legacy_mod = "_OLD" if sample == "B" and old_raw else ""
writeHDF5(modified_GT, data_path,
'segmentations/groundtruth_plus_boundary2' + legacy_mod)
scores_dir = os.path.join(project_dir, exp_name, "scores")
prev_score_file = os.path.join(
scores_dir, filename.replace(".h5", ".yml"))
result_config = yaml2dict(prev_score_file)
result_config['score_WS'] = evals
new_score_file_path = prev_score_file.replace(
".yml", "{}.yml".format(postfix))
with open(new_score_file_path, 'w') as f:
# json.dump(config_to_save, f, indent=4, sort_keys=True)
yaml.dump(result_config, f)
# Save new segm:
out_segm_path = pred_file.replace(".h5",
"{}.h5".format(postfix))
writeHDF5(segm.astype('uint32'), out_segm_path, "segm_WS")
if PREPARE_SUBMISSION:
from vaeAffs.postproc.utils import prepare_submission
path_bbox_slice = config["volume_config"][
"paths_padded_boxes"]
prepare_submission(sample,
out_segm_path,
inner_path_segm="segm_WS",
path_bbox_slice=path_bbox_slice[sample],
ds_factor=(1, 2, 2))
glia_mask = readHDF5(glia_prediction_path, "glia_mask")
for exp_name, invert in EXP_NAMES:
pred_dir = os.path.join(project_dir, exp_name)
for item in os.listdir(pred_dir):
if os.path.isfile(os.path.join(pred_dir, item)):
filename = item
if not filename.endswith(".h5") or filename.startswith(
".") or not filename.startswith("predictions_sample"):
continue
pred_file = os.path.join(pred_dir, filename)
# Load glia mask and predictions:
# TODO: add crop slice
print("Loading affs for ", exp_name)
affs = readHDF5(pred_file, "data")
print("Computing...")
new_affs = compute_boundary_mask_from_label_image(
affs,
glia_mask,
invert_affs=invert,
offsets=offsets,
combination_method=combination_method,
set_invalid_values_to=-1)
print("Saving...")
writeHDF5(new_affs, pred_file,
"affs_plus_glia_{}".format(combination_method))
affs[affs_valid_mask==0] = 1
# Combine left and right affinities:
segment_mask = np.logical_and(affs[0], affs[1])
# This functions erode binary out_mask (segments 1, boundary 0)
eroded_segment_mask = segment_mask.copy()
for z in range(eroded_segment_mask.shape[0]):
eroded_segment_mask[z] = vigra.filters.multiBinaryErosion(segment_mask[z], radius=2.)
boundary_mask = np.logical_not(eroded_segment_mask)
BOUNDARY_LABEL = 2
DEFECTED_LABEL = 3
out_mask = glia.copy()
out_mask[boundary_mask] = BOUNDARY_LABEL
# Mask defected slices:
for slc in defected_slices[sample]:
out_mask[parse_data_slice(slc)] = DEFECTED_LABEL
# Copy GT from previous (to avoid weird connected components problems):
for z in copy_from_previous[sample]:
GT[z] = GT[z-1]
print("Now writing...")
from segmfriends.utils.various import writeHDF5
writeHDF5(out_mask, data_path, 'volumes/labels/various_masks')
writeHDF5(GT, data_path, 'volumes/labels/neuron_ids_fixed')
with open(csv_file_path, mode='w') as f:
employee_writer = csv.writer(f,
delimiter=';',
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
for i in range(3):
employee_writer.writerow(
[str(crop_slice[i].start),
str(crop_slice[i].stop)])
print(crop_slice)
GT = GT[crop_slice]
# Write affs and mask in target file:
target_path = os.path.join(
get_trendytukan_drive_path(),
"datasets/CREMI/padded_data/cropped_aligned_samples/sample_{}.h5".
format(sample))
target_path_2x = os.path.join(
get_trendytukan_drive_path(),
"datasets/CREMI/padded_data/cropped_aligned_samples/sample_{}_2x.h5".
format(sample))
raw = readHDF5(source_path, "volumes/raw", crop_slice=crop_slice)
print("Saving...")
writeHDF5(GT, target_path, gt_inner_path)
writeHDF5(raw, target_path, "volumes/raw")
print("Downscaling and saving...")
writeHDF5(zoom(GT, (1, 0.5, 0.5), order=0), target_path_2x, gt_inner_path)
writeHDF5(zoom(raw, (1, 0.5, 0.5), order=3), target_path_2x, "volumes/raw")
print("Saving...")
# target_path_old = os.path.join(get_hci_home_path(),
# "datasets/CREMI/official_test_samples/cropped_aligned_samples/sample{}_cropped{}.h5".format(
# sample, POSTFIX))
target_path = os.path.join(get_hci_home_path(),
"datasets/CREMI/official_test_samples/cropped_aligned_samples/sample{}_cropped{}.h5".format(sample, POSTFIX))
# if include_affs:
# affs_path = os.path.join(get_trendytukan_drive_path(), "datasets/CREMI/constantin_affs/test_samples/sample{}.h5".format(sample))
# affs_inner_path = "affinities"
# affs = readHDF5(affs_path, affs_inner_path, crop_slice=(slice(None), ) + crop_slice)
# writeHDF5(affs, target_path, "volumes/affinities")
# raw = readHDF5(source_path, "volumes/raw")
# # raw = readHDF5(target_path_old, "volumes/raw_2x")
# if sample in blacked_out:
# for blk in blacked_out[sample]:
# print("blacking out ", blk)
# raw[blk] = 0
# mask_gt = readHDF5(source_path, mask_inner_path, dtype="uint16", crop_slice=crop_slice)
# writeHDF5(raw, target_path, "volumes/raw")
# writeHDF5(mask_big_pad.astype('uint16'), target_path, "volumes/labels/mask_gt")
# writeHDF5(mask_gt, target_path, "volumes/labels/mask_gt")
if downscale:
writeHDF5(zoom(mask_big_pad, (1, 0.5, 0.5), order=0), target_path, "volumes/labels/mask_raw_2x")
# writeHDF5(zoom(raw, (1, 0.5, 0.5), order=3), target_path, "volumes/raw_2x")
# writeHDF5(raw, target_path, "volumes/raw_2x")
print("Saving...")
target_path = os.path.join(
get_trendytukan_drive_path(),
"datasets/CREMI/official_test_samples/cropped_aligned_samples/sample{}_cropped.h5"
.format(sample))
if include_affs:
affs_path = os.path.join(
get_trendytukan_drive_path(),
"datasets/CREMI/constantin_affs/test_samples/sample{}.h5".format(
sample))
affs_inner_path = "affinities"
affs = readHDF5(affs_path,
affs_inner_path,
crop_slice=(slice(None), ) + crop_slice)
writeHDF5(affs, target_path, "volumes/affinities")
raw = readHDF5(source_path, "volumes/raw", crop_slice=crop_slice)
if sample is blacked_out:
for blk in blacked_out[sample]:
raw[blk] = 0
mask_gt = readHDF5(source_path,
mask_inner_path,
dtype="uint16",
crop_slice=crop_slice)
writeHDF5(raw, target_path, "volumes/raw")
writeHDF5(mask_big_pad[crop_slice].astype('uint16'), target_path,
"volumes/labels/mask_big_pad")
writeHDF5(mask_gt, target_path, "volumes/labels/mask_gt")
# Combine left and right affinities:
segment_mask = np.logical_and(affs[0], affs[1])
# This functions erode binary out_mask (segments 1, boundary 0)
eroded_segment_mask = segment_mask.copy()
for z in range(eroded_segment_mask.shape[0]):
eroded_segment_mask[z] = vigra.filters.multiBinaryErosion(
segment_mask[z], radius=1.)
boundary_mask = np.logical_not(eroded_segment_mask)
out_mask = glia.copy()
out_mask[boundary_mask] = BOUNDARY_LABEL
# # Make sure not to have added some boundary inside glia:
# out_mask[glia == GLIA_LABEL] = GLIA_LABEL
# Mask defected slices:
for slc in defected_slices[sample]:
out_mask[parse_data_slice(slc)] = DEFECTED_LABEL
# # Copy GT from previous (to avoid weird connected components problems):
# for z in copy_from_previous[sample]:
# GT[z] = GT[z-1]
print("Now writing...")
from segmfriends.utils.various import writeHDF5
writeHDF5(out_mask, data_path,
'volumes/labels/various_masks_noDefects_thinBound')
# writeHDF5(GT, data_path, 'volumes/labels/neuron_ids_fixed')
"A": [],
"B": [15, 16, 44, 45],
"C": [14, 74],
}
for sample in ["A", "B", "C"]:
# -----------
# LOAD data
# -----------
data_path = os.path.join(
get_abailoni_hci_home_path(),
"../ialgpu1_local_home/datasets/cremi/SOA_affinities/sample{}_train.h5"
.format(sample))
old_raw = True
legacy_mod = "_OLD" if sample == "B" and old_raw else ""
with h5py.File(data_path, 'r') as f:
GT = f['segmentations/groundtruth_plus_boundary' + legacy_mod][:]
# Max int32 value - 1:
DEFECTED_VALUE = 2147483647 - 1
for slc_idx in defected_slices[sample]:
GT[slc_idx] = DEFECTED_VALUE
from segmfriends.utils.various import writeHDF5
writeHDF5(GT, data_path,
'segmentations/groundtruth_plus_boundary_defectMod' + legacy_mod)