os.makedirs(partition_output_dir, exist_ok=True)
print("output path:", partition_path)
if os.path.exists(partition_path):
print("Exiting, path exists.")
else:
ModelsHeader = ModelsTableHeader()
models_df = pd.read_csv(models_table, dtype=ModelsHeader.dtype_dict)
model_df = models_df[models_df[ModelsHeader.model_name] == model_name]
overlap = model_df.iloc[0][ModelsHeader.overlap]
box_shape = int(model_df.iloc[0][ModelsHeader.box_size])
subtomogram_shape = (box_shape, box_shape, box_shape)
DTHeader = DatasetTableHeader(processing_tomo=processing_tomo)
df = pd.read_csv(dataset_table)
df[DTHeader.tomo_name] = df[DTHeader.tomo_name].astype(str)
print("Partitioning tomo", tomo_name)
tomo_df = df[df[DTHeader.tomo_name] == tomo_name]
path_to_raw = tomo_df.iloc[0][DTHeader.processing_tomo]
path_to_lamella = tomo_df.iloc[0][DTHeader.filtering_mask]
raw_dataset = load_tomogram(path_to_dataset=path_to_raw)
if isinstance(path_to_lamella, float):
print("No filtering mask file available.")
partition_tomogram(dataset=raw_dataset,
output_h5_file_path=partition_path,
subtomo_shape=subtomogram_shape,
run_job = False
else:
run_job = True
if run_job:
output_dir_tomo, data_partition = testing_partition_path(
output_dir=config.work_dir, tomo_name=tomo_name, fold=fold)
segmentation_label = model_name
box_shape = [config.box_size, config.box_size, config.box_size]
tomo_output_dir, output_path = get_probability_map_path(
config.output_dir, model_name, tomo_name, config.pred_class)
os.makedirs(tomo_output_dir, exist_ok=True)
DTHeader = DatasetTableHeader(processing_tomo=config.processing_tomo)
df = pd.read_csv(config.dataset_table, dtype={DTHeader.tomo_name: str})
df[DTHeader.tomo_name] = df[DTHeader.tomo_name].astype(str)
tomo_df = df[df[DTHeader.tomo_name] == tomo_name]
print("config.processing_tomo", config.processing_tomo)
tomo_path = tomo_df.iloc[0][config.processing_tomo]
tomo = load_tomogram(path_to_dataset=tomo_path)
output_shape = tomo.shape
del tomo
subtomos_internal_path = os.path.join(
h5_internal_paths.PREDICTED_SEGMENTATION_SUBTOMOGRAMS,
segmentation_label)
assemble_tomo_from_subtomos(output_path=output_path,
partition_file_path=data_partition,
models_table = os.path.join(models_table, "models.csv")
model_name = config["model_path"][:-4]
write_on_table = True
ModelsHeader = ModelsTableHeader()
models_df = pd.read_csv(models_table, dtype=ModelsHeader.dtype_dict)
model_df = models_df[models_df[ModelsHeader.model_name] == model_name]
overlap = model_df.iloc[0][ModelsHeader.overlap]
test_partition = config["prediction"]['partition_name']
processing_tomo = config["prediction"]['processing_tomo']
box_shape = int(model_df.iloc[0][ModelsHeader.box_size])
subtomogram_shape = (box_shape, box_shape, box_shape)
DTHeader = DatasetTableHeader(processing_tomo=processing_tomo,
partition_name=test_partition)
df = pd.read_csv(dataset_table)
df[DTHeader.tomo_name] = df[DTHeader.tomo_name].astype(str)
if write_on_table:
for tomo_name in tomo_list:
print("Partitioning tomo", tomo_name)
output_dir = config['pred_output_dir']
output_dir_tomo = os.path.join(output_dir, tomo_name)
os.makedirs(output_dir_tomo, exist_ok=True)
partition_path = os.path.join(output_dir_tomo, test_partition + ".h5")
print("output path:", partition_path)
if os.path.isfile(partition_path):
print("Partition exists already.")
else:
max_cluster_size = motl_parameters['max_cluster_size']
dataset_table = config['dataset_table']
ModelsHeader = ModelsTableHeader()
models_df = pd.read_csv(models_table,
dtype={
ModelsHeader.model_name: str,
ModelsHeader.segmentation_names: str
})
model_df = models_df[models_df[ModelsHeader.model_name] == model_name]
semantic_names = model_df.iloc[0]['semantic_classes'].split(',')
semantic_class = semantic_names[class_number]
ignore_border_thickness = motl_parameters['ignore_border_thickness']
filtering_mask = motl_parameters['region_mask']
DTHeader = DatasetTableHeader(filtering_mask=filtering_mask)
df = pd.read_csv(dataset_table)
df[DTHeader.tomo_name] = df[DTHeader.tomo_name].astype(str)
for tomo_name in tomo_list:
output_dir = os.path.join(config["pred_output_dir"], "predictions")
output_dir = build_prediction_output_dir(base_output_dir=output_dir,
label_name="",
model_name=model_name,
tomo_name=tomo_name,
semantic_class=semantic_class)
motls_in_dir = [file for file in os.listdir(output_dir) if 'motl_' in file]
assert len(motls_in_dir) == 1, "only one motive list can be filtered."
csv_motl = os.path.join(output_dir, motls_in_dir[0])
tomo_df = df[df[DTHeader.tomo_name] == tomo_name]
"/pr_radius_" + str(config.pr_tolerance_radius) + \
"/detected/.{fold}.done_pp_snakemake".format(fold=str(fold))
from networks.utils import get_training_testing_lists
if isinstance(fold, int):
tomo_training_list, tomo_testing_list = get_training_testing_lists(
config=config, fold=fold)
if tomo_name in tomo_testing_list:
run_job = True
else:
run_job = False
else:
run_job = True
if run_job:
DTHeader = DatasetTableHeader(semantic_classes=config.semantic_classes)
df = pd.read_csv(config.dataset_table)
df[DTHeader.tomo_name] = df[DTHeader.tomo_name].astype(str)
print("Processing tomo", tomo_name)
pred_output_dir = os.path.join(config.output_dir, "predictions")
tomo_output_dir = build_prediction_output_dir(
base_output_dir=pred_output_dir,
label_name="",
model_name=model_name,
tomo_name=tomo_name,
semantic_class=config.pred_class)
print(tomo_output_dir)
os.makedirs(tomo_output_dir, exist_ok=True)
motl_in_dir = [
file for file in os.listdir(tomo_output_dir) if 'motl_' == file[:5]
dtype={ModelsHeader.model_name: str,
ModelsHeader.segmentation_names: str})
model_df = models_df[models_df[ModelsHeader.model_name] == model_name]
print(model_df)
assert model_df.shape[0] == 1
overlap = model_df.iloc[0][ModelsHeader.overlap]
box_shape = int(model_df.iloc[0][ModelsHeader.box_size])
box_shape = [box_shape, box_shape, box_shape]
semantic_names = model_df.iloc[0]['semantic_classes'].split(',')
semantic_class = semantic_names[class_number]
dataset_table = config['dataset_table']
test_partition = config["prediction"]['partition_name']
DTHeader = DatasetTableHeader(partition_name=test_partition)
df = pd.read_csv(dataset_table)
df[DTHeader.tomo_name] = df[DTHeader.tomo_name].astype(str)
output_dir = os.path.join(output_dir, "predictions")
output_dir = os.path.join(output_dir, model_name)
for tomo_name in tomo_list:
tomo_output_dir = os.path.join(output_dir, tomo_name)
tomo_output_dir = os.path.join(tomo_output_dir, semantic_class)
os.makedirs(tomo_output_dir, exist_ok=True)
output_path = os.path.join(tomo_output_dir, "prediction.mrc")
if os.path.isfile(output_path):
print("The prediction file exists")
else:
tomo_df = df[df[DTHeader.tomo_name] == tomo_name]
run_job = False
else:
run_job = True
if run_job:
print("model_name", model_name)
prediction_path = get_post_processed_prediction_path(
output_dir=config.output_dir,
model_name=model_name,
tomo_name=tomo_name,
semantic_class=config.pred_class)
print(prediction_path)
assert os.path.isfile(
prediction_path), "The prediction file does not exist!"
DTHeader = DatasetTableHeader(semantic_classes=config.semantic_classes,
filtering_mask=config.region_mask)
df = pd.read_csv(config.dataset_table)
df[DTHeader.tomo_name] = df[DTHeader.tomo_name].astype(str)
clean_mask_name = DTHeader.masks_names[config.pred_class_number]
tomo_df = df[df[DTHeader.tomo_name] == tomo_name]
target_path = tomo_df.iloc[0][clean_mask_name]
prediction = load_tomogram(path_to_dataset=prediction_path)
contact_mode = config.contact_mode
if contact_mode == "intersection":
lamella_file = tomo_df.iloc[0][DTHeader.filtering_mask]
if str(lamella_file) == "nan":
prediction = load_tomogram(prediction_path)
else:
ModelsHeader.segmentation_names: str
})
model_df = models_df[models_df[ModelsHeader.model_name] == model_name]
print(model_df)
assert model_df.shape[0] == 1
overlap = model_df.iloc[0][ModelsHeader.overlap]
box_shape = int(model_df.iloc[0][ModelsHeader.box_size])
box_shape = [box_shape, box_shape, box_shape]
semantic_classes = model_df.iloc[0]['semantic_classes'].split(',')
semantic_class = semantic_classes[class_number]
motl_parameters = config['clustering_parameters']
filtering_mask = motl_parameters['region_mask']
dataset_table = config['dataset_table']
DTHeader = DatasetTableHeader(semantic_classes=semantic_classes)
df = pd.read_csv(dataset_table)
df[DTHeader.tomo_name] = df[DTHeader.tomo_name].astype(str)
output_dir = config["pred_output_dir"]
output_dir = os.path.join(output_dir, "predictions")
output_dir = os.path.join(output_dir, model_name)
for tomo_name in tomo_list:
print("Processing tomo", tomo_name)
output_dir = os.path.join(config["pred_output_dir"], "predictions")
tomo_output_dir = build_prediction_output_dir(
base_output_dir=output_dir,
label_name="",
model_name=model_name,
tomo_name=tomo_name,
semantic_class=semantic_class)
print("tomo_name", tomo_name)
partition_output_dir, partition_path = testing_partition_path(output_dir=config.work_dir,
tomo_name=tomo_name,
fold=fold)
print("partition_path =", partition_path)
os.makedirs(partition_output_dir, exist_ok=True)
if os.path.exists(partition_path):
print("Exiting, path exists.")
else:
overlap = config.overlap
box_size = config.box_size
box_shape = (box_size, box_size, box_size)
DTHeader = DatasetTableHeader(processing_tomo=config.processing_tomo, filtering_mask=config.region_mask)
df = pd.read_csv(config.dataset_table, dtype={"tomo_name": str})
df[DTHeader.tomo_name] = df[DTHeader.tomo_name].astype(str)
tomo_df = df[df[DTHeader.tomo_name] == tomo_name]
print(tomo_name, config.processing_tomo, tomo_df)
path_to_raw = tomo_df.iloc[0][config.processing_tomo]
intersecting_mask_path = tomo_df.iloc[0][config.region_mask]
raw_dataset = load_tomogram(path_to_dataset=path_to_raw, dtype=float)
if isinstance(intersecting_mask_path, float):
print("No region mask file available.")
intersecting_mask = np.ones_like(raw_dataset)
else:
intersecting_mask_path = tomo_df.iloc[0][config.region_mask]
intersecting_mask = load_tomogram(path_to_dataset=intersecting_mask_path)
tomo_list_spl.append(tomo)
return tomo_list_spl
tomo_training_list = split_list(
tomo_list_str=cv_data.loc[fold]["cv_training_list"])
tomo_evaluation_list = split_list(
tomo_list_str=cv_data.loc[fold]["cv_evaluation_list"])
logging_dir = os.path.join(output_dir, "logging")
model_dir = os.path.join(output_dir, "models")
models_table = os.path.join(model_dir, "models.csv")
box_shape = [box_size, box_size, box_size]
DTHeader = DatasetTableHeader(partition_name=partition_name,
semantic_classes=segmentation_names)
df = pd.read_csv(dataset_table, dtype={DTHeader.tomo_name: str})
model_path = os.path.join(model_dir, model_name + ".pkl")
log_model = os.path.join(logging_dir, model_name)
os.makedirs(log_model, exist_ok=True)
os.makedirs(model_dir, exist_ok=True)
device = get_device()
final_activation = nn.Sigmoid()
net_conf = {
'final_activation': final_activation,
'depth': depth,
'initial_features': initial_features,
"out_channels": output_classes,
ModelsHeader = ModelsTableHeader()
models_df = pd.read_csv(models_table, dtype=ModelsHeader.dtype_dict)
model_df = models_df[models_df[ModelsHeader.model_name] == model_name]
print(model_df)
assert model_df.shape[0] == 1
overlap = model_df.iloc[0][ModelsHeader.overlap]
box_shape = int(model_df.iloc[0][ModelsHeader.box_size])
box_shape = [box_shape, box_shape, box_shape]
segmentation_label = model_name
semantic_names = model_df.iloc[0][ModelsHeader.segmentation_names].split(',')
semantic_class = semantic_names[class_number]
dataset_table = config['dataset_table']
DTHeader = DatasetTableHeader()
output_dir = os.path.join(config['pred_output_dir'], "predictions")
output_dir = os.path.join(output_dir, model_name)
for tomo_name in tomo_list:
print("Processing tomo", tomo_name)
tomo_output_dir = os.path.join(output_dir, tomo_name)
tomo_output_dir = os.path.join(tomo_output_dir, semantic_class)
os.makedirs(tomo_output_dir, exist_ok=True)
df = pd.read_csv(dataset_table)
df[DTHeader.tomo_name] = df[DTHeader.tomo_name].astype(str)
tomo_df = df[df[DTHeader.tomo_name] == tomo_name]
x_dim = int(tomo_df.iloc[0][DTHeader.x_dim])
y_dim = int(tomo_df.iloc[0][DTHeader.y_dim])
z_dim = int(tomo_df.iloc[0][DTHeader.z_dim])