os.makedirs(partition_output_dir, exist_ok=True) print("output path:", partition_path) if os.path.exists(partition_path): print("Exiting, path exists.") else: ModelsHeader = ModelsTableHeader() models_df = pd.read_csv(models_table, dtype=ModelsHeader.dtype_dict) model_df = models_df[models_df[ModelsHeader.model_name] == model_name] overlap = model_df.iloc[0][ModelsHeader.overlap] box_shape = int(model_df.iloc[0][ModelsHeader.box_size]) subtomogram_shape = (box_shape, box_shape, box_shape) DTHeader = DatasetTableHeader(processing_tomo=processing_tomo) df = pd.read_csv(dataset_table) df[DTHeader.tomo_name] = df[DTHeader.tomo_name].astype(str) print("Partitioning tomo", tomo_name) tomo_df = df[df[DTHeader.tomo_name] == tomo_name] path_to_raw = tomo_df.iloc[0][DTHeader.processing_tomo] path_to_lamella = tomo_df.iloc[0][DTHeader.filtering_mask] raw_dataset = load_tomogram(path_to_dataset=path_to_raw) if isinstance(path_to_lamella, float): print("No filtering mask file available.") partition_tomogram(dataset=raw_dataset, output_h5_file_path=partition_path, subtomo_shape=subtomogram_shape,
run_job = False else: run_job = True if run_job: output_dir_tomo, data_partition = testing_partition_path( output_dir=config.work_dir, tomo_name=tomo_name, fold=fold) segmentation_label = model_name box_shape = [config.box_size, config.box_size, config.box_size] tomo_output_dir, output_path = get_probability_map_path( config.output_dir, model_name, tomo_name, config.pred_class) os.makedirs(tomo_output_dir, exist_ok=True) DTHeader = DatasetTableHeader(processing_tomo=config.processing_tomo) df = pd.read_csv(config.dataset_table, dtype={DTHeader.tomo_name: str}) df[DTHeader.tomo_name] = df[DTHeader.tomo_name].astype(str) tomo_df = df[df[DTHeader.tomo_name] == tomo_name] print("config.processing_tomo", config.processing_tomo) tomo_path = tomo_df.iloc[0][config.processing_tomo] tomo = load_tomogram(path_to_dataset=tomo_path) output_shape = tomo.shape del tomo subtomos_internal_path = os.path.join( h5_internal_paths.PREDICTED_SEGMENTATION_SUBTOMOGRAMS, segmentation_label) assemble_tomo_from_subtomos(output_path=output_path, partition_file_path=data_partition,
models_table = os.path.join(models_table, "models.csv") model_name = config["model_path"][:-4] write_on_table = True ModelsHeader = ModelsTableHeader() models_df = pd.read_csv(models_table, dtype=ModelsHeader.dtype_dict) model_df = models_df[models_df[ModelsHeader.model_name] == model_name] overlap = model_df.iloc[0][ModelsHeader.overlap] test_partition = config["prediction"]['partition_name'] processing_tomo = config["prediction"]['processing_tomo'] box_shape = int(model_df.iloc[0][ModelsHeader.box_size]) subtomogram_shape = (box_shape, box_shape, box_shape) DTHeader = DatasetTableHeader(processing_tomo=processing_tomo, partition_name=test_partition) df = pd.read_csv(dataset_table) df[DTHeader.tomo_name] = df[DTHeader.tomo_name].astype(str) if write_on_table: for tomo_name in tomo_list: print("Partitioning tomo", tomo_name) output_dir = config['pred_output_dir'] output_dir_tomo = os.path.join(output_dir, tomo_name) os.makedirs(output_dir_tomo, exist_ok=True) partition_path = os.path.join(output_dir_tomo, test_partition + ".h5") print("output path:", partition_path) if os.path.isfile(partition_path): print("Partition exists already.") else:
max_cluster_size = motl_parameters['max_cluster_size'] dataset_table = config['dataset_table'] ModelsHeader = ModelsTableHeader() models_df = pd.read_csv(models_table, dtype={ ModelsHeader.model_name: str, ModelsHeader.segmentation_names: str }) model_df = models_df[models_df[ModelsHeader.model_name] == model_name] semantic_names = model_df.iloc[0]['semantic_classes'].split(',') semantic_class = semantic_names[class_number] ignore_border_thickness = motl_parameters['ignore_border_thickness'] filtering_mask = motl_parameters['region_mask'] DTHeader = DatasetTableHeader(filtering_mask=filtering_mask) df = pd.read_csv(dataset_table) df[DTHeader.tomo_name] = df[DTHeader.tomo_name].astype(str) for tomo_name in tomo_list: output_dir = os.path.join(config["pred_output_dir"], "predictions") output_dir = build_prediction_output_dir(base_output_dir=output_dir, label_name="", model_name=model_name, tomo_name=tomo_name, semantic_class=semantic_class) motls_in_dir = [file for file in os.listdir(output_dir) if 'motl_' in file] assert len(motls_in_dir) == 1, "only one motive list can be filtered." csv_motl = os.path.join(output_dir, motls_in_dir[0]) tomo_df = df[df[DTHeader.tomo_name] == tomo_name]
"/pr_radius_" + str(config.pr_tolerance_radius) + \ "/detected/.{fold}.done_pp_snakemake".format(fold=str(fold)) from networks.utils import get_training_testing_lists if isinstance(fold, int): tomo_training_list, tomo_testing_list = get_training_testing_lists( config=config, fold=fold) if tomo_name in tomo_testing_list: run_job = True else: run_job = False else: run_job = True if run_job: DTHeader = DatasetTableHeader(semantic_classes=config.semantic_classes) df = pd.read_csv(config.dataset_table) df[DTHeader.tomo_name] = df[DTHeader.tomo_name].astype(str) print("Processing tomo", tomo_name) pred_output_dir = os.path.join(config.output_dir, "predictions") tomo_output_dir = build_prediction_output_dir( base_output_dir=pred_output_dir, label_name="", model_name=model_name, tomo_name=tomo_name, semantic_class=config.pred_class) print(tomo_output_dir) os.makedirs(tomo_output_dir, exist_ok=True) motl_in_dir = [ file for file in os.listdir(tomo_output_dir) if 'motl_' == file[:5]
dtype={ModelsHeader.model_name: str, ModelsHeader.segmentation_names: str}) model_df = models_df[models_df[ModelsHeader.model_name] == model_name] print(model_df) assert model_df.shape[0] == 1 overlap = model_df.iloc[0][ModelsHeader.overlap] box_shape = int(model_df.iloc[0][ModelsHeader.box_size]) box_shape = [box_shape, box_shape, box_shape] semantic_names = model_df.iloc[0]['semantic_classes'].split(',') semantic_class = semantic_names[class_number] dataset_table = config['dataset_table'] test_partition = config["prediction"]['partition_name'] DTHeader = DatasetTableHeader(partition_name=test_partition) df = pd.read_csv(dataset_table) df[DTHeader.tomo_name] = df[DTHeader.tomo_name].astype(str) output_dir = os.path.join(output_dir, "predictions") output_dir = os.path.join(output_dir, model_name) for tomo_name in tomo_list: tomo_output_dir = os.path.join(output_dir, tomo_name) tomo_output_dir = os.path.join(tomo_output_dir, semantic_class) os.makedirs(tomo_output_dir, exist_ok=True) output_path = os.path.join(tomo_output_dir, "prediction.mrc") if os.path.isfile(output_path): print("The prediction file exists") else: tomo_df = df[df[DTHeader.tomo_name] == tomo_name]
run_job = False else: run_job = True if run_job: print("model_name", model_name) prediction_path = get_post_processed_prediction_path( output_dir=config.output_dir, model_name=model_name, tomo_name=tomo_name, semantic_class=config.pred_class) print(prediction_path) assert os.path.isfile( prediction_path), "The prediction file does not exist!" DTHeader = DatasetTableHeader(semantic_classes=config.semantic_classes, filtering_mask=config.region_mask) df = pd.read_csv(config.dataset_table) df[DTHeader.tomo_name] = df[DTHeader.tomo_name].astype(str) clean_mask_name = DTHeader.masks_names[config.pred_class_number] tomo_df = df[df[DTHeader.tomo_name] == tomo_name] target_path = tomo_df.iloc[0][clean_mask_name] prediction = load_tomogram(path_to_dataset=prediction_path) contact_mode = config.contact_mode if contact_mode == "intersection": lamella_file = tomo_df.iloc[0][DTHeader.filtering_mask] if str(lamella_file) == "nan": prediction = load_tomogram(prediction_path) else:
ModelsHeader.segmentation_names: str }) model_df = models_df[models_df[ModelsHeader.model_name] == model_name] print(model_df) assert model_df.shape[0] == 1 overlap = model_df.iloc[0][ModelsHeader.overlap] box_shape = int(model_df.iloc[0][ModelsHeader.box_size]) box_shape = [box_shape, box_shape, box_shape] semantic_classes = model_df.iloc[0]['semantic_classes'].split(',') semantic_class = semantic_classes[class_number] motl_parameters = config['clustering_parameters'] filtering_mask = motl_parameters['region_mask'] dataset_table = config['dataset_table'] DTHeader = DatasetTableHeader(semantic_classes=semantic_classes) df = pd.read_csv(dataset_table) df[DTHeader.tomo_name] = df[DTHeader.tomo_name].astype(str) output_dir = config["pred_output_dir"] output_dir = os.path.join(output_dir, "predictions") output_dir = os.path.join(output_dir, model_name) for tomo_name in tomo_list: print("Processing tomo", tomo_name) output_dir = os.path.join(config["pred_output_dir"], "predictions") tomo_output_dir = build_prediction_output_dir( base_output_dir=output_dir, label_name="", model_name=model_name, tomo_name=tomo_name, semantic_class=semantic_class)
print("tomo_name", tomo_name) partition_output_dir, partition_path = testing_partition_path(output_dir=config.work_dir, tomo_name=tomo_name, fold=fold) print("partition_path =", partition_path) os.makedirs(partition_output_dir, exist_ok=True) if os.path.exists(partition_path): print("Exiting, path exists.") else: overlap = config.overlap box_size = config.box_size box_shape = (box_size, box_size, box_size) DTHeader = DatasetTableHeader(processing_tomo=config.processing_tomo, filtering_mask=config.region_mask) df = pd.read_csv(config.dataset_table, dtype={"tomo_name": str}) df[DTHeader.tomo_name] = df[DTHeader.tomo_name].astype(str) tomo_df = df[df[DTHeader.tomo_name] == tomo_name] print(tomo_name, config.processing_tomo, tomo_df) path_to_raw = tomo_df.iloc[0][config.processing_tomo] intersecting_mask_path = tomo_df.iloc[0][config.region_mask] raw_dataset = load_tomogram(path_to_dataset=path_to_raw, dtype=float) if isinstance(intersecting_mask_path, float): print("No region mask file available.") intersecting_mask = np.ones_like(raw_dataset) else: intersecting_mask_path = tomo_df.iloc[0][config.region_mask] intersecting_mask = load_tomogram(path_to_dataset=intersecting_mask_path)
tomo_list_spl.append(tomo) return tomo_list_spl tomo_training_list = split_list( tomo_list_str=cv_data.loc[fold]["cv_training_list"]) tomo_evaluation_list = split_list( tomo_list_str=cv_data.loc[fold]["cv_evaluation_list"]) logging_dir = os.path.join(output_dir, "logging") model_dir = os.path.join(output_dir, "models") models_table = os.path.join(model_dir, "models.csv") box_shape = [box_size, box_size, box_size] DTHeader = DatasetTableHeader(partition_name=partition_name, semantic_classes=segmentation_names) df = pd.read_csv(dataset_table, dtype={DTHeader.tomo_name: str}) model_path = os.path.join(model_dir, model_name + ".pkl") log_model = os.path.join(logging_dir, model_name) os.makedirs(log_model, exist_ok=True) os.makedirs(model_dir, exist_ok=True) device = get_device() final_activation = nn.Sigmoid() net_conf = { 'final_activation': final_activation, 'depth': depth, 'initial_features': initial_features, "out_channels": output_classes,
ModelsHeader = ModelsTableHeader() models_df = pd.read_csv(models_table, dtype=ModelsHeader.dtype_dict) model_df = models_df[models_df[ModelsHeader.model_name] == model_name] print(model_df) assert model_df.shape[0] == 1 overlap = model_df.iloc[0][ModelsHeader.overlap] box_shape = int(model_df.iloc[0][ModelsHeader.box_size]) box_shape = [box_shape, box_shape, box_shape] segmentation_label = model_name semantic_names = model_df.iloc[0][ModelsHeader.segmentation_names].split(',') semantic_class = semantic_names[class_number] dataset_table = config['dataset_table'] DTHeader = DatasetTableHeader() output_dir = os.path.join(config['pred_output_dir'], "predictions") output_dir = os.path.join(output_dir, model_name) for tomo_name in tomo_list: print("Processing tomo", tomo_name) tomo_output_dir = os.path.join(output_dir, tomo_name) tomo_output_dir = os.path.join(tomo_output_dir, semantic_class) os.makedirs(tomo_output_dir, exist_ok=True) df = pd.read_csv(dataset_table) df[DTHeader.tomo_name] = df[DTHeader.tomo_name].astype(str) tomo_df = df[df[DTHeader.tomo_name] == tomo_name] x_dim = int(tomo_df.iloc[0][DTHeader.x_dim]) y_dim = int(tomo_df.iloc[0][DTHeader.y_dim]) z_dim = int(tomo_df.iloc[0][DTHeader.z_dim])