def parseDistribution(distribution):
distribution = normaliseString(distribution)
distribution_names = list(distributions.keys())
distribution_names += list(latent_distributions.keys())
for distribution_name in distribution_names:
if normaliseString(distribution_name) == distribution:
return distribution_name
raise ValueError("Distribution `{}` not found.".format(distribution))
def parseModelVersions(proposed_versions):
version_alias_sets = {
"end_of_training": ["eot", "end", "finish", "finished"],
"best_model": ["bm", "best", "optimal", "optimal_parameters", "op"],
"early_stopping": ["es", "early", "stop", "stopped"]
}
parsed_versions = []
if not isinstance(proposed_versions, list):
proposed_versions = [proposed_versions]
if proposed_versions == ["all"]:
parsed_versions = list(version_alias_sets.keys())
else:
for proposed_version in proposed_versions:
normalised_proposed_version = normaliseString(proposed_version)
parsed_version = None
for version, version_aliases in version_alias_sets.items():
if normalised_proposed_version == version \
or normalised_proposed_version in version_aliases:
parsed_version = version
break
if parsed_version:
parsed_versions.append(parsed_version)
else:
raise ValueError(
"`{}` is not a model version.".format(proposed_version))
return parsed_versions
def main(input_file_or_name, data_directory = "data",
log_directory = "log", results_directory = "results",
temporary_log_directory = None,
map_features = False, feature_selection = [], example_filter = [],
preprocessing_methods = [], noisy_preprocessing_methods = [],
split_data_set = True,
splitting_method = "default", splitting_fraction = 0.9,
model_type = "VAE", latent_size = 50, hidden_sizes = [500],
number_of_importance_samples = [5],
number_of_monte_carlo_samples = [10],
inference_architecture = "MLP",
latent_distribution = "gaussian",
number_of_classes = None,
parameterise_latent_posterior = False,
generative_architecture = "MLP",
reconstruction_distribution = "poisson",
number_of_reconstruction_classes = 0,
prior_probabilities_method = "uniform",
number_of_warm_up_epochs = 0,
kl_weight = 1,
proportion_of_free_KL_nats = 0.0,
batch_normalisation = True,
dropout_keep_probabilities = [],
count_sum = True,
number_of_epochs = 200, plotting_interval_during_training = None,
batch_size = 100, learning_rate = 1e-4,
run_id = None, new_run = False,
prediction_method = None, prediction_training_set_name = "training",
prediction_decomposition_method = None,
prediction_decomposition_dimensionality = None,
decomposition_methods = ["PCA"], highlight_feature_indices = [],
reset_training = False, skip_modelling = False,
model_versions = ["all"],
analyse = True, evaluation_set_name = "test", analyse_data = False,
analyses = ["default"], analysis_level = "normal", fast_analysis = False,
export_options = []):
# Setup
model_versions = parseModelVersions(model_versions)
## Analyses
if fast_analysis:
analyse = True
analyses = ["simple"]
analysis_level = "limited"
## Distributions
reconstruction_distribution = parseDistribution(
reconstruction_distribution)
latent_distribution = parseDistribution(latent_distribution)
## Model configuration validation
if not skip_modelling:
if run_id:
run_id = checkRunID(run_id)
model_valid, model_errors = validateModelParameters(
model_type, latent_distribution,
reconstruction_distribution, number_of_reconstruction_classes,
parameterise_latent_posterior
)
if not model_valid:
print("Model configuration is invalid:")
for model_error in model_errors:
print(" ", model_error)
print()
if analyse_data:
print("Skipping modelling.")
print("")
skip_modelling = True
else:
print("Modelling cancelled.")
return
## Binarisation
binarise_values = False
if reconstruction_distribution == "bernoulli":
if noisy_preprocessing_methods:
if noisy_preprocessing_methods[-1] != "binarise":
noisy_preprocessing_methods.append("binarise")
print("Appended binarisation method to noisy preprocessing,",
"because of the Bernoulli distribution.\n")
else:
binarise_values = True
## Data sets
if not split_data_set or analyse_data or evaluation_set_name == "full" \
or prediction_training_set_name == "full":
full_data_set_needed = True
else:
full_data_set_needed = False
# Data
print(title("Data"))
data_set = data.DataSet(
input_file_or_name,
directory = data_directory,
map_features = map_features,
feature_selection = feature_selection,
example_filter = example_filter,
preprocessing_methods = preprocessing_methods,
binarise_values = binarise_values,
noisy_preprocessing_methods = noisy_preprocessing_methods
)
if full_data_set_needed:
data_set.load()
if split_data_set:
training_set, validation_set, test_set = data_set.split(
splitting_method, splitting_fraction)
all_data_sets = [data_set, training_set, validation_set, test_set]
else:
splitting_method = None
training_set = data_set
validation_set = None
test_set = data_set
all_data_sets = [data_set]
evaluation_set_name = "full"
prediction_training_set_name = "full"
## Setup of log and results directories
log_directory = data.directory(log_directory, data_set,
splitting_method, splitting_fraction)
data_results_directory = data.directory(results_directory, data_set,
splitting_method, splitting_fraction, preprocessing = False)
results_directory = data.directory(results_directory, data_set,
splitting_method, splitting_fraction)
if temporary_log_directory:
main_temporary_log_directory = temporary_log_directory
temporary_log_directory = data.directory(temporary_log_directory,
data_set, splitting_method, splitting_fraction)
## Data analysis
if analyse and analyse_data:
print(subtitle("Analysing data"))
analysis.analyseData(
data_sets = all_data_sets,
decomposition_methods = decomposition_methods,
highlight_feature_indices = highlight_feature_indices,
analyses = analyses,
analysis_level = analysis_level,
export_options = export_options,
results_directory = data_results_directory
)
print()
## Full data set clean up
if not full_data_set_needed:
data_set.clear()
# Modelling
if skip_modelling:
print("Modelling skipped.")
return
print(title("Modelling"))
# Set the number of features for the model
feature_size = training_set.number_of_features
# Parse numbers of samples
number_of_monte_carlo_samples = parseSampleLists(
number_of_monte_carlo_samples)
number_of_importance_samples = parseSampleLists(
number_of_importance_samples)
# Use analytical KL term for single-Gaussian-VAE
if "VAE" in model_type:
if latent_distribution == "gaussian":
analytical_kl_term = True
else:
analytical_kl_term = False
# Change latent distribution to Gaussian mixture if not already set
if model_type == "GMVAE" and latent_distribution != "gaussian mixture":
latent_distribution = "gaussian mixture"
print("The latent distribution was changed to",
"a Gaussian-mixture model, because of the model chosen.\n")
# Set the number of classes if not already set
if not number_of_classes:
if training_set.has_labels:
number_of_classes = training_set.number_of_classes \
- training_set.number_of_excluded_classes
elif "mixture" in latent_distribution:
raise ValueError(
"For a mixture model and a data set without labels, "
"the number of classes has to be set."
)
else:
number_of_classes = 1
print(subtitle("Model setup"))
if model_type == "VAE":
model = VariationalAutoencoder(
feature_size = feature_size,
latent_size = latent_size,
hidden_sizes = hidden_sizes,
number_of_monte_carlo_samples =number_of_monte_carlo_samples,
number_of_importance_samples = number_of_importance_samples,
analytical_kl_term = analytical_kl_term,
inference_architecture = inference_architecture,
latent_distribution = latent_distribution,
number_of_latent_clusters = number_of_classes,
parameterise_latent_posterior = parameterise_latent_posterior,
generative_architecture = generative_architecture,
reconstruction_distribution = reconstruction_distribution,
number_of_reconstruction_classes = number_of_reconstruction_classes,
batch_normalisation = batch_normalisation,
dropout_keep_probabilities = dropout_keep_probabilities,
count_sum = count_sum,
number_of_warm_up_epochs = number_of_warm_up_epochs,
kl_weight = kl_weight,
log_directory = log_directory,
results_directory = results_directory
)
elif model_type == "GMVAE":
if prior_probabilities_method == "uniform":
prior_probabilities = None
elif prior_probabilities_method == "infer":
prior_probabilities = training_set.class_probabilities
elif prior_probabilities_method == "literature":
prior_probabilities = training_set.literature_probabilities
else:
prior_probabilities = None
if not prior_probabilities:
prior_probabilities_method = "uniform"
prior_probabilities_values = None
else:
prior_probabilities_values = list(prior_probabilities.values())
prior_probabilities = {
"method": prior_probabilities_method,
"values": prior_probabilities_values
}
model = GaussianMixtureVariationalAutoencoder(
feature_size = feature_size,
latent_size = latent_size,
hidden_sizes = hidden_sizes,
number_of_monte_carlo_samples = number_of_monte_carlo_samples,
number_of_importance_samples = number_of_importance_samples,
analytical_kl_term = analytical_kl_term,
prior_probabilities = prior_probabilities,
number_of_latent_clusters = number_of_classes,
proportion_of_free_KL_nats = proportion_of_free_KL_nats,
reconstruction_distribution = reconstruction_distribution,
number_of_reconstruction_classes = number_of_reconstruction_classes,
batch_normalisation = batch_normalisation,
dropout_keep_probabilities = dropout_keep_probabilities,
count_sum = count_sum,
number_of_warm_up_epochs = number_of_warm_up_epochs,
kl_weight = kl_weight,
log_directory = log_directory,
results_directory = results_directory
)
else:
raise ValueError("Model type not found: `{}`.".format(model_type))
print(model.description)
print()
print(model.parameters)
print()
## Training
print(subtitle("Model training"))
status, run_id = model.train(
training_set,
validation_set,
number_of_epochs = number_of_epochs,
batch_size = batch_size,
learning_rate = learning_rate,
plotting_interval = plotting_interval_during_training,
run_id = run_id,
new_run = new_run,
reset_training = reset_training,
temporary_log_directory = temporary_log_directory
)
# Remove temporary directories created and emptied during training
if temporary_log_directory and os.path.exists(main_temporary_log_directory):
removeEmptyDirectories(main_temporary_log_directory)
if not status["completed"]:
print(status["message"])
return
status_filename = "status"
if "epochs trained" in status:
status_filename += "-" + status["epochs trained"]
status_path = os.path.join(
model.logDirectory(run_id = run_id),
status_filename + ".log"
)
with open(status_path, "w") as status_file:
for status_field, status_value in status.items():
if status_value:
status_file.write(
status_field + ": " + str(status_value) + "\n"
)
print()
# Evaluation, prediction, and analysis
## Setup
if analyse:
if prediction_method:
predict_labels_using_model = False
elif "GM" in model.type:
predict_labels_using_model = True
prediction_method = "model"
else:
predict_labels_using_model = False
else:
predict_labels_using_model = False
evaluation_title_parts = ["evaluation"]
if analyse:
if prediction_method:
evaluation_title_parts.append("prediction")
evaluation_title_parts.append("analysis")
evaluation_title = enumerateListOfStrings(evaluation_title_parts)
print(title(evaluation_title.capitalize()))
### Set selection
for data_subset in all_data_sets:
clear_subset = True
if data_subset.kind == evaluation_set_name:
evaluation_set = data_subset
clear_subset = False
if prediction_method \
and data_subset.kind == prediction_training_set_name:
prediction_training_set = data_subset
clear_subset = False
if clear_subset:
data_subset.clear()
### Evaluation set
evaluation_subset_indices = analysis.evaluationSubsetIndices(
evaluation_set)
print("Evaluation set: {} set.".format(evaluation_set.kind))
### Prediction method
if prediction_method:
prediction_method = properString(
prediction_method,
PREDICTION_METHOD_NAMES
)
prediction_method_specifications = PREDICTION_METHOD_SPECIFICATIONS\
.get(prediction_method, {})
prediction_method_inference = prediction_method_specifications.get(
"inference", None)
prediction_method_fixed_number_of_clusters \
= prediction_method_specifications.get(
"fixed number of clusters", None)
prediction_method_cluster_kind = prediction_method_specifications.get(
"cluster kind", None)
if prediction_method_fixed_number_of_clusters:
number_of_clusters = number_of_classes
else:
number_of_clusters = None
if prediction_method_inference \
and prediction_method_inference == "transductive":
prediction_training_set = None
prediction_training_set_name = None
else:
prediction_training_set_name = prediction_training_set.kind
prediction_details = {
"method": prediction_method,
"number_of_classes": number_of_clusters,
"training_set_name": prediction_training_set_name,
"decomposition_method": prediction_decomposition_method,
"decomposition_dimensionality":
prediction_decomposition_dimensionality
}
print("Prediction method: {}.".format(prediction_method))
if number_of_clusters:
print("Number of clusters: {}.".format(number_of_clusters))
if prediction_training_set:
print("Prediction training set: {} set.".format(
prediction_training_set.kind))
prediction_id_parts = []
if prediction_decomposition_method:
prediction_decomposition_method = properString(
prediction_decomposition_method,
DECOMPOSITION_METHOD_NAMES
)
if not prediction_decomposition_dimensionality:
prediction_decomposition_dimensionality \
= DEFAULT_DECOMPOSITION_DIMENSIONALITY
prediction_id_parts += [
prediction_decomposition_method,
prediction_decomposition_dimensionality
]
prediction_details.update({
"decomposition_method": prediction_decomposition_method,
"decomposition_dimensionality":
prediction_decomposition_dimensionality
})
print("Decomposition method before prediction: {}-d {}.".format(
prediction_decomposition_dimensionality,
prediction_decomposition_method
))
prediction_id_parts.append(prediction_method)
if number_of_clusters:
prediction_id_parts.append(number_of_clusters)
if prediction_training_set \
and prediction_training_set.kind != "training":
prediction_id_parts.append(prediction_training_set.kind)
prediction_id = "_".join(map(
lambda s: normaliseString(str(s)).replace("_", ""),
prediction_id_parts
))
prediction_details["id"] = prediction_id
else:
prediction_details = {}
### Model parameter sets
model_parameter_set_names = []
if "end_of_training" in model_versions:
model_parameter_set_names.append("end of training")
if "best_model" in model_versions \
and betterModelExists(model, run_id = run_id):
model_parameter_set_names.append("best model")
if "early_stopping" in model_versions \
and modelStoppedEarly(model, run_id = run_id):
model_parameter_set_names.append("early stopping")
print("Model parameter sets: {}.".format(enumerateListOfStrings(
model_parameter_set_names)))
print()
## Model analysis
if analyse:
print(subtitle("Model analysis"))
analysis.analyseModel(
model = model,
run_id = run_id,
analyses = analyses,
analysis_level = analysis_level,
export_options = export_options,
results_directory = results_directory
)
## Results evaluation, prediction, and analysis
for model_parameter_set_name in model_parameter_set_names:
if model_parameter_set_name == "best model":
use_best_model = True
else:
use_best_model = False
if model_parameter_set_name == "early stopping":
use_early_stopping_model = True
else:
use_early_stopping_model = False
model_parameter_set_name = model_parameter_set_name.capitalize()
print(subtitle(model_parameter_set_name))
# Evaluation
model_parameter_set_name = model_parameter_set_name.replace(" ", "-")
print(heading("{} evaluation".format(model_parameter_set_name)))
if "VAE" in model.type:
transformed_evaluation_set, reconstructed_evaluation_set,\
latent_evaluation_sets = model.evaluate(
evaluation_set = evaluation_set,
evaluation_subset_indices = evaluation_subset_indices,
batch_size = batch_size,
predict_labels = predict_labels_using_model,
run_id = run_id,
use_best_model = use_best_model,
use_early_stopping_model = use_early_stopping_model
)
else:
transformed_evaluation_set, reconstructed_evaluation_set = \
model.evaluate(
evaluation_set = evaluation_set,
evaluation_subset_indices = evaluation_subset_indices,
batch_size = batch_size,
run_id = run_id,
use_best_model = use_best_model,
use_early_stopping_model = use_early_stopping_model
)
latent_evaluation_sets = None
print()
# Prediction
if analyse and "VAE" in model.type and prediction_method \
and not transformed_evaluation_set.has_predictions:
print(heading("{} prediction".format(model_parameter_set_name)))
latent_prediction_evaluation_set = latent_evaluation_sets["z"]
if prediction_method_inference \
and prediction_method_inference == "inductive":
latent_prediction_training_sets = model.evaluate(
evaluation_set = prediction_training_set,
batch_size = batch_size,
run_id = run_id,
use_best_model = use_best_model,
use_early_stopping_model = use_early_stopping_model,
output_versions = "latent",
log_results = False
)
latent_prediction_training_set \
= latent_prediction_training_sets["z"]
print()
else:
latent_prediction_training_set = None
if prediction_decomposition_method:
if latent_prediction_training_set:
latent_prediction_training_set, \
latent_prediction_evaluation_set \
= data.decomposeDataSubsets(
latent_prediction_training_set,
latent_prediction_evaluation_set,
method = prediction_decomposition_method,
number_of_components =
prediction_decomposition_dimensionality,
random = True
)
else:
latent_prediction_evaluation_set \
= data.decomposeDataSubsets(
latent_prediction_evaluation_set,
method = prediction_decomposition_method,
number_of_components =
prediction_decomposition_dimensionality,
random = True
)
print()
cluster_ids, predicted_labels, predicted_superset_labels \
= predict(
latent_prediction_training_set,
latent_prediction_evaluation_set,
prediction_method,
number_of_clusters
)
transformed_evaluation_set.updatePredictions(
predicted_cluster_ids = cluster_ids,
predicted_labels = predicted_labels,
predicted_superset_labels = predicted_superset_labels
)
reconstructed_evaluation_set.updatePredictions(
predicted_cluster_ids = cluster_ids,
predicted_labels = predicted_labels,
predicted_superset_labels = predicted_superset_labels
)
for variable in latent_evaluation_sets:
latent_evaluation_sets[variable].updatePredictions(
predicted_cluster_ids = cluster_ids,
predicted_labels = predicted_labels,
predicted_superset_labels = predicted_superset_labels
)
print()
# Analysis
if analyse:
print(heading("{} results analysis".format(model_parameter_set_name)))
analysis.analyseResults(
evaluation_set = transformed_evaluation_set,
reconstructed_evaluation_set = reconstructed_evaluation_set,
latent_evaluation_sets = latent_evaluation_sets,
model = model,
run_id = run_id,
decomposition_methods = decomposition_methods,
evaluation_subset_indices = evaluation_subset_indices,
highlight_feature_indices = highlight_feature_indices,
prediction_details = prediction_details,
best_model = use_best_model,
early_stopping = use_early_stopping_model,
analyses = analyses, analysis_level = analysis_level,
export_options = export_options,
results_directory = results_directory
)
# Clean up
if transformed_evaluation_set.version == "original":
transformed_evaluation_set.resetPredictions()
def decompose(values,
other_value_sets=[],
centroids={},
method=None,
number_of_components=None,
random=False):
# Setup
## Method
if method is None:
method = DEFAULT_DECOMPOSITION_METHOD
method = properString(normaliseString(method), DECOMPOSITION_METHOD_NAMES)
## Number of components
if number_of_components is None:
number_of_components = DEFAULT_DECOMPOSITION_DIMENSIONALITY
## Other value sets
if other_value_sets is not None \
and not isinstance(other_value_sets, (list, tuple)):
other_value_sets = [other_value_sets]
## Randomness
if random:
random_state = None
else:
random_state = 42
# Method
if method == "PCA":
if values.shape[1] <= MAXIMUM_FEATURE_SIZE_FOR_NORMAL_PCA \
and not scipy.sparse.issparse(values):
model = PCA(n_components=number_of_components)
else:
model = IncrementalPCA(n_components=number_of_components,
batch_size=100)
elif method == "SVD":
model = TruncatedSVD(n_components=number_of_components)
elif method == "ICA":
model = FastICA(n_components=number_of_components)
elif method == "t-SNE":
if number_of_components < 4:
tsne_method = "barnes_hut"
else:
tsne_method = "exact"
model = TSNE(n_components=number_of_components,
method=tsne_method,
random_state=random_state)
else:
raise ValueError("Method `{}` not found.".format(method))
# Fit and transform
values_decomposed = model.fit_transform(values)
if other_value_sets and method != "t_sne":
other_value_sets_decomposed = []
for other_values in other_value_sets:
other_value_decomposed = model.transform(other_values)
other_value_sets_decomposed.append(other_value_decomposed)
else:
other_value_sets_decomposed = None
if other_value_sets_decomposed and len(other_value_sets_decomposed) == 1:
other_value_sets_decomposed = other_value_sets_decomposed[0]
# Only supports centroids without data sets as top levels
if centroids and method == "PCA":
if "means" in centroids:
centroids = {"unknown": centroids}
W = model.components_
centroids_decomposed = {}
for distribution, distribution_centroids in centroids.items():
if distribution_centroids:
centroids_distribution_decomposed = {}
for parameter, values in distribution_centroids.items():
if parameter == "means":
shape = numpy.array(values.shape)
L = shape[-1]
reshaped_values = values.reshape(-1, L)
decomposed_values = model.transform(reshaped_values)
shape[-1] = number_of_components
new_values = decomposed_values.reshape(shape)
elif parameter == "covariance_matrices":
shape = numpy.array(values.shape)
L = shape[-1]
reshaped_values = values.reshape(-1, L, L)
B = reshaped_values.shape[0]
decomposed_values = numpy.empty((B, 2, 2))
for i in range(B):
decomposed_values[i] = W @ reshaped_values[i] @ W.T
shape[-2:] = number_of_components
new_values = decomposed_values.reshape(shape)
else:
new_values = values
centroids_distribution_decomposed[parameter] = new_values
centroids_decomposed[distribution] = \
centroids_distribution_decomposed
else:
centroids_decomposed[distribution] = None
if "unknown" in centroids_decomposed:
centroids_decomposed = centroids_decomposed["unknown"]
else:
centroids_decomposed = None
if other_value_sets != [] and centroids != {}:
return values_decomposed, other_value_sets_decomposed, \
centroids_decomposed
elif other_value_sets != []:
return values_decomposed, other_value_sets_decomposed
elif centroids != {}:
return values_decomposed, centroids_decomposed
else:
return values_decomposed