for tomo_name in tomo_evaluation_list:
partition_output_dir, partition_path = fold_testing_partition_path(
output_dir=work_dir,
tomo_name=tomo_name,
model_name=model_name,
partition_name=test_partition,
fold=fold)
os.makedirs(partition_output_dir, exist_ok=True)
print("output path:", partition_path)
if os.path.exists(partition_path):
print("Exiting, path exists.")
else:
ModelsHeader = ModelsTableHeader()
models_df = pd.read_csv(models_table, dtype=ModelsHeader.dtype_dict)
model_df = models_df[models_df[ModelsHeader.model_name] == model_name]
overlap = model_df.iloc[0][ModelsHeader.overlap]
box_shape = int(model_df.iloc[0][ModelsHeader.box_size])
subtomogram_shape = (box_shape, box_shape, box_shape)
DTHeader = DatasetTableHeader(processing_tomo=processing_tomo)
df = pd.read_csv(dataset_table)
df[DTHeader.tomo_name] = df[DTHeader.tomo_name].astype(str)
print("Partitioning tomo", tomo_name)
def write_on_models_notebook(model_name: str,
label_name: str,
model_dir: str,
log_dir: str,
depth: int,
initial_features: int,
n_epochs: int,
training_paths_list: list,
split: float,
output_classes: int,
segmentation_names: list,
box_size: int,
partition_name: str,
processing_tomo: str,
retrain: str or bool,
path_to_old_model: str,
models_notebook_path: str,
encoder_dropout: float = np.nan,
decoder_dropout: float = np.nan,
batch_norm: bool = False,
overlap: int = 12,
cv_fold: int or None = None,
cv_test_tomos: list or None = None):
"""
:param model_name:
:param label_name:
:param model_dir:
:param log_dir:
:param depth:
:param initial_features:
:param n_epochs:
:param training_paths_list:
:param split:
:param output_classes:
:param segmentation_names:
:param retrain:
:param path_to_old_model:
:param models_notebook_path:
:param encoder_dropout:
:param decoder_dropout:
:param batch_norm:
:return:
"""
model_path = os.path.join(model_dir, model_name + ".pkl")
ModelsHeader = ModelsTableHeader()
training_paths = reduce(lambda x, y: x + ", " + y, training_paths_list)
segmentation_names = reduce(lambda x, y: x + ", " + y, segmentation_names)
print(training_paths, segmentation_names)
if cv_test_tomos is not None:
cv_testing_paths = reduce(lambda x, y: x + ", " + y, cv_test_tomos)
else:
cv_testing_paths = ""
now = datetime.datetime.now()
date = str(now.day) + "/" + str(now.month) + "/" + str(now.year)
mini_notebook_df = pd.DataFrame({ModelsHeader.model_name: model_name},
index=[0])
mini_notebook_df[ModelsHeader.model_name] = model_name
mini_notebook_df[ModelsHeader.label_name] = label_name
mini_notebook_df[ModelsHeader.model_path] = model_path
mini_notebook_df[ModelsHeader.log_path] = log_dir
mini_notebook_df[ModelsHeader.depth] = depth
mini_notebook_df[ModelsHeader.initial_features] = initial_features
mini_notebook_df[ModelsHeader.epochs] = n_epochs
mini_notebook_df[ModelsHeader.training_set] = training_paths
mini_notebook_df[ModelsHeader.testing_set] = cv_testing_paths
mini_notebook_df[ModelsHeader.fold] = cv_fold
mini_notebook_df[ModelsHeader.train_split] = split
mini_notebook_df[ModelsHeader.output_classes] = output_classes
mini_notebook_df[ModelsHeader.segmentation_names] = segmentation_names
mini_notebook_df[ModelsHeader.retrain] = str(retrain)
mini_notebook_df[ModelsHeader.old_model] = path_to_old_model
mini_notebook_df[ModelsHeader.training_date] = date
mini_notebook_df[ModelsHeader.batch_norm] = str(batch_norm)
mini_notebook_df[ModelsHeader.encoder_dropout] = encoder_dropout
mini_notebook_df[ModelsHeader.decoder_dropout] = decoder_dropout
mini_notebook_df[ModelsHeader.box_size] = box_size
mini_notebook_df[ModelsHeader.overlap] = overlap
mini_notebook_df[ModelsHeader.processing_tomo] = processing_tomo
mini_notebook_df[ModelsHeader.partition_name] = partition_name
models_notebook_dir = os.path.dirname(models_notebook_path)
makedirs(models_notebook_dir, exist_ok=True)
if os.path.isfile(models_notebook_path):
models_notebook_df = pd.read_csv(models_notebook_path,
dtype={ModelsHeader.model_name: str})
if model_name in models_notebook_df[ModelsHeader.model_name].values:
print("Substituting model row in models table")
index = models_notebook_df.index[models_notebook_df[
ModelsHeader.model_name] == model_name].tolist()
assert len(index) == 1
index = index[0]
models_notebook_df.iloc[index, :] = mini_notebook_df.iloc[0, :]
else:
models_notebook_df = models_notebook_df.append(mini_notebook_df,
sort="False")
models_notebook_df.to_csv(path_or_buf=models_notebook_path,
index=False)
else:
mini_notebook_df.to_csv(path_or_buf=models_notebook_path, index=False)
return