def parse_numbers_of_samples(proposed_numbers_of_samples): required_scenarios = ["training", "evaluation"] if isinstance(proposed_numbers_of_samples, (int, float)): proposed_numbers_of_samples = [ _parse_number_of_samples(proposed_numbers_of_samples) ] if isinstance(proposed_numbers_of_samples, list): if len(proposed_numbers_of_samples) == 1: proposed_numbers_of_samples *= 2 elif len(proposed_numbers_of_samples) > 2: raise ValueError( "List of number of samples can only contain one or two " "numbers.") parsed_numbers_of_samples = { scenario: _parse_number_of_samples(number) for scenario, number in zip(required_scenarios, proposed_numbers_of_samples) } elif isinstance(proposed_numbers_of_samples, dict): valid = True for scenario in required_scenarios: scenario_number = proposed_numbers_of_samples.get(scenario) try: scenario_number = _parse_number_of_samples(scenario_number) except TypeError as sample_number_parsing_error: if "integer" in sample_number_parsing_error: valid = False finally: proposed_numbers_of_samples[scenario] = scenario_number if valid: parsed_numbers_of_samples = proposed_numbers_of_samples else: raise ValueError( "To supply the numbers of samples as a dictionary, the " "dictionary must contain the keys {} with the number of " "samples for each given as an integer.".format( enumerate_strings( ["`{}`".format(s) for s in required_scenarios], conjunction="and"))) else: raise TypeError( "Expected an `int`, `list`, or `dict`; got `{}`.".format( type(proposed_numbers_of_samples))) return parsed_numbers_of_samples
def validate_model_parameters(reconstruction_distribution=None, number_of_reconstruction_classes=None, model_type=None, latent_distribution=None, parameterise_latent_posterior=None): # Validate piecewise categorical likelihood if reconstruction_distribution and number_of_reconstruction_classes: if number_of_reconstruction_classes > 0: piecewise_categorical_likelihood_errors = [] if reconstruction_distribution == "bernoulli": piecewise_categorical_likelihood_errors.append( "the Bernoulli distribution") if "zero-inflated" in reconstruction_distribution: piecewise_categorical_likelihood_errors.append( "zero-inflated distributions") if "constrained" in reconstruction_distribution: piecewise_categorical_likelihood_errors.append( "constrained distributions") if len(piecewise_categorical_likelihood_errors) > 0: piecewise_categorical_likelihood_error = ( "{} cannot be piecewise categorical.".format( capitalise_string( enumerate_strings( piecewise_categorical_likelihood_errors, conjunction="or")))) raise ValueError(piecewise_categorical_likelihood_error) # Validate parameterisation of latent posterior for VAE if model_type and latent_distribution and parameterise_latent_posterior: if "VAE" in model_type: if (not (model_type in ["VAE"] and latent_distribution == "gaussian mixture") and parameterise_latent_posterior): parameterise_error = ( "Cannot parameterise latent posterior parameters for {} " "or {} distribution.".format(model_type, latent_distribution)) raise ValueError(parameterise_error)
def evaluate(data_set_file_or_name, data_format=None, data_directory=None, map_features=None, feature_selection=None, example_filter=None, noisy_preprocessing_methods=None, preprocessing_methods=None, split_data_set=None, splitting_method=None, splitting_fraction=None, model_type=None, latent_size=None, hidden_sizes=None, number_of_importance_samples=None, number_of_monte_carlo_samples=None, inference_architecture=None, latent_distribution=None, number_of_classes=None, parameterise_latent_posterior=False, prior_probabilities_method=None, generative_architecture=None, reconstruction_distribution=None, number_of_reconstruction_classes=None, count_sum=None, proportion_of_free_nats_for_y_kl_divergence=None, minibatch_normalisation=None, batch_correction=None, dropout_keep_probabilities=None, number_of_warm_up_epochs=None, kl_weight=None, minibatch_size=None, run_id=None, models_directory=None, included_analyses=None, analysis_level=None, decomposition_methods=None, highlight_feature_indices=None, export_options=None, analyses_directory=None, evaluation_set_kind=None, sample_size=None, prediction_method=None, prediction_training_set_kind=None, model_versions=None, **keyword_arguments): """Evaluate model on data set.""" if split_data_set is None: split_data_set = defaults["data"]["split_data_set"] if splitting_method is None: splitting_method = defaults["data"]["splitting_method"] if splitting_fraction is None: splitting_fraction = defaults["data"]["splitting_fraction"] if models_directory is None: models_directory = defaults["models"]["directory"] if evaluation_set_kind is None: evaluation_set_kind = defaults["evaluation"]["data_set_name"] if sample_size is None: sample_size = defaults["models"]["sample_size"] if prediction_method is None: prediction_method = defaults["evaluation"]["prediction_method"] if prediction_training_set_kind is None: prediction_training_set_kind = defaults["evaluation"][ "prediction_training_set_kind"] if model_versions is None: model_versions = defaults["evaluation"]["model_versions"] if analyses_directory is None: analyses_directory = defaults["analyses"]["directory"] evaluation_set_kind = normalise_string(evaluation_set_kind) prediction_training_set_kind = normalise_string( prediction_training_set_kind) model_versions = parse_model_versions(model_versions) print(title("Data")) binarise_values = False if reconstruction_distribution == "bernoulli": if noisy_preprocessing_methods: if noisy_preprocessing_methods[-1] != "binarise": noisy_preprocessing_methods.append("binarise") else: binarise_values = True data_set = DataSet(data_set_file_or_name, data_format=data_format, directory=data_directory, map_features=map_features, feature_selection=feature_selection, example_filter=example_filter, preprocessing_methods=preprocessing_methods, binarise_values=binarise_values, noisy_preprocessing_methods=noisy_preprocessing_methods) if not split_data_set or evaluation_set_kind == "full": data_set.load() if split_data_set: training_set, validation_set, test_set = data_set.split( method=splitting_method, fraction=splitting_fraction) data_subsets = [data_set, training_set, validation_set, test_set] for data_subset in data_subsets: clear_data_subset = True if data_subset.kind == evaluation_set_kind: evaluation_set = data_subset clear_data_subset = False if data_subset.kind == prediction_training_set_kind: prediction_training_set = data_subset clear_data_subset = False if clear_data_subset: data_subset.clear() else: splitting_method = None splitting_fraction = None evaluation_set = data_set prediction_training_set = data_set evaluation_subset_indices = indices_for_evaluation_subset(evaluation_set) models_directory = build_directory_path( models_directory, data_set=evaluation_set, splitting_method=splitting_method, splitting_fraction=splitting_fraction) analyses_directory = build_directory_path( analyses_directory, data_set=evaluation_set, splitting_method=splitting_method, splitting_fraction=splitting_fraction) print(title("Model")) if number_of_classes is None: if evaluation_set.has_labels: number_of_classes = (evaluation_set.number_of_classes - evaluation_set.number_of_excluded_classes) model = _setup_model( data_set=evaluation_set, model_type=model_type, latent_size=latent_size, hidden_sizes=hidden_sizes, number_of_importance_samples=number_of_importance_samples, number_of_monte_carlo_samples=number_of_monte_carlo_samples, inference_architecture=inference_architecture, latent_distribution=latent_distribution, number_of_classes=number_of_classes, parameterise_latent_posterior=parameterise_latent_posterior, prior_probabilities_method=prior_probabilities_method, generative_architecture=generative_architecture, reconstruction_distribution=reconstruction_distribution, number_of_reconstruction_classes=number_of_reconstruction_classes, count_sum=count_sum, proportion_of_free_nats_for_y_kl_divergence=( proportion_of_free_nats_for_y_kl_divergence), minibatch_normalisation=minibatch_normalisation, batch_correction=batch_correction, dropout_keep_probabilities=dropout_keep_probabilities, number_of_warm_up_epochs=number_of_warm_up_epochs, kl_weight=kl_weight, models_directory=models_directory) if ("best_model" in model_versions and not better_model_exists(model, run_id=run_id)): model_versions.remove("best_model") if ("early_stopping" in model_versions and not model_stopped_early(model, run_id=run_id)): model_versions.remove("early_stopping") print(subtitle("Analysis")) analyses.analyse_model(model=model, run_id=run_id, included_analyses=included_analyses, analysis_level=analysis_level, export_options=export_options, analyses_directory=analyses_directory) print(title("Results")) print("Evaluation set: {} set.".format(evaluation_set.kind)) print("Model version{}: {}.".format( "" if len(model_versions) == 1 else "s", enumerate_strings([v.replace("_", " ") for v in model_versions], conjunction="and"))) if prediction_method: prediction_specifications = PredictionSpecifications( method=prediction_method, number_of_clusters=number_of_classes, training_set_kind=prediction_training_set.kind) print("Prediction method: {}.".format( prediction_specifications.method)) print("Number of clusters: {}.".format( prediction_specifications.number_of_clusters)) print("Prediction training set: {} set.".format( prediction_specifications.training_set_kind)) print() for model_version in model_versions: use_best_model = False use_early_stopping_model = False if model_version == "best_model": use_best_model = True elif model_version == "early_stopping": use_early_stopping_model = True print(subtitle(model_version.replace("_", " ").capitalize())) print( heading("{} evaluation".format( model_version.replace("_", "-").capitalize()))) (transformed_evaluation_set, reconstructed_evaluation_set, latent_evaluation_sets) = model.evaluate( evaluation_set=evaluation_set, evaluation_subset_indices=evaluation_subset_indices, minibatch_size=minibatch_size, run_id=run_id, use_best_model=use_best_model, use_early_stopping_model=use_early_stopping_model, output_versions="all") print() if sample_size: print( heading("{} sampling".format( model_version.replace("_", "-").capitalize()))) sample_reconstruction_set, __ = model.sample( sample_size=sample_size, minibatch_size=minibatch_size, run_id=run_id, use_best_model=use_best_model, use_early_stopping_model=use_early_stopping_model) print() else: sample_reconstruction_set = None if prediction_method: print( heading("{} prediction".format( model_version.replace("_", "-").capitalize()))) latent_prediction_training_sets = model.evaluate( evaluation_set=prediction_training_set, minibatch_size=minibatch_size, run_id=run_id, use_best_model=use_best_model, use_early_stopping_model=use_early_stopping_model, output_versions="latent", log_results=False) print() cluster_ids, predicted_labels, predicted_superset_labels = ( predict_labels( training_set=latent_prediction_training_sets["z"], evaluation_set=latent_evaluation_sets["z"], specifications=prediction_specifications)) evaluation_set_versions = [ transformed_evaluation_set, reconstructed_evaluation_set ] + list(latent_evaluation_sets.values()) for evaluation_set_version in evaluation_set_versions: evaluation_set_version.update_predictions( prediction_specifications=prediction_specifications, predicted_cluster_ids=cluster_ids, predicted_labels=predicted_labels, predicted_superset_labels=predicted_superset_labels) print() print( heading("{} analysis".format( model_version.replace("_", "-").capitalize()))) analyses.analyse_results( evaluation_set=transformed_evaluation_set, reconstructed_evaluation_set=reconstructed_evaluation_set, latent_evaluation_sets=latent_evaluation_sets, model=model, run_id=run_id, sample_reconstruction_set=sample_reconstruction_set, decomposition_methods=decomposition_methods, evaluation_subset_indices=evaluation_subset_indices, highlight_feature_indices=highlight_feature_indices, best_model=use_best_model, early_stopping=use_early_stopping_model, included_analyses=included_analyses, analysis_level=analysis_level, export_options=export_options, analyses_directory=analyses_directory) return 0