def initialize_experiment(): experiment = Experiment(data) experiment.add_classifier(TemporalEvidencesClassifier(data.features, data.target_names), name="Our method") experiment.add_classifier(NaiveBayesClassifier(data.features, data.target_names), name="Naive Bayes") return experiment
def initialize_experiment(): experiment = Experiment(data) experiment.add_classifier(TemporalEvidencesClassifier( data.features, data.target_names), name="Our method") experiment.add_classifier(NaiveBayesClassifier(data.features, data.target_names), name="Naive Bayes") return experiment
""" This dataset is partially dominated by one of the sensors, which makes the evaluation results less statistically sound, e.g. it leads to large confidence intervals when running 10-fold cross-validation. """ data = load_dataset("../datasets/houseB.csv", "../datasets/houseB.config") cutoff_results_at = 15 return data, cutoff_results_at #configuration data, cutoff_results_at = houseA() #run several classifiers on the same dataset, use 10-fold cross-validation experiment = Experiment(data) experiment.add_classifier(TemporalEvidencesClassifier(data.features, data.target_names), name="Our method") experiment.add_classifier(NaiveBayesClassifier(data.features, data.target_names), name="Naive Bayes") experiment.add_classifier(RandomClassifier(data.features, data.target_names), name="Random") results = experiment.run(folds=10) #print and plot results results.print_quality_comparison_at_cutoff( cutoff=1, metrics=["Recall", "Precision", "F1"]) results.print_runtime_comparison() plot_conf = plot.plot_config(config.plot_directory, sub_dirs=[data.name], img_type=config.img_type)
sys.path.append("..") import pandas from evaluation.experiment import Experiment from evaluation.metrics import quality_metrics from recsys.classifiers.temporal import TemporalEvidencesClassifier, configure_dynamic_cutoff from recsys.dataset import load_dataset #configuration data = load_dataset("../datasets/houseA.csv", "../datasets/houseA.config") methods_to_test = [("Fixed cutoff", None), ("dynamic cutoff=4", configure_dynamic_cutoff(1.0, 0.4, 4)), ("dynamic cutoff=2", configure_dynamic_cutoff(1.0, 0.4, 2))] #run all configured cutoffs with 10-fold cross-validation experiment = Experiment(data) for name, method in methods_to_test: experiment.add_classifier(TemporalEvidencesClassifier(data.features, data.target_names, postprocess=method), name=name) results = experiment.run(folds=10) #print results pandas.set_option('expand_frame_repr', False) pandas.set_option('max_columns', 4) print "Maximum 5 recommendations" results.print_quality_comparison_at_cutoff(cutoff=5, metrics=quality_metrics) print "Maximum 10 recommendations" results.print_quality_comparison_at_cutoff(cutoff=10, metrics=quality_metrics)
data = load_dataset("../datasets/houseA.csv", "../datasets/houseA.config") intervals_to_test = [ #test various settings for delta t_max ("Delta t_max=1200s", initialize_bins(start=0, end=60, width=10) + initialize_bins(start=60, end=1200, width=30)), ("Delta t_max=120s", initialize_bins(start=0, end=60, width=10) + initialize_bins(start=60, end=120, width=30)), ("Delta t_max=60s", initialize_bins(start=0, end=60, width=10)), ("Delta t_max=30s", initialize_bins(start=0, end=30, width=10)), ("Delta t_max=10s", initialize_bins(start=0, end=10, width=10)), #test various interval widths ("all intervals 2s wide", initialize_bins(start=0, end=300, width=2)), ("all intervals 4s wide", initialize_bins(start=0, end=300, width=4)), ("all intervals 6s wide", initialize_bins(start=0, end=300, width=6)), ("all intervals 8s wide", initialize_bins(start=0, end=300, width=8)), ("all intervals 30s wide", initialize_bins(start=0, end=300, width=30)), ("all intervals 50s wide", initialize_bins(start=0, end=300, width=50)), ("all intervals 100s wide", initialize_bins(start=0, end=300, width=100)) ] #run 10-fold cross-validation for each of the configured intervals experiment = Experiment(data) for (name, bins) in intervals_to_test: experiment.add_classifier(TemporalEvidencesClassifier(data.features, data.target_names, bins=bins), name=name) results = experiment.run(folds=10) results.print_quality_comparison_at_cutoff( cutoff=1, metrics=["Recall", "Precision", "F1"])
from recsys.classifiers.binning import initialize_bins from recsys.dataset import load_dataset #configuration data = load_dataset("../datasets/houseA.csv", "../datasets/houseA.config") intervals_to_test = [#test various settings for delta t_max ("Delta t_max=1200s", initialize_bins(start=0, end=60, width=10) + initialize_bins(start=60, end=1200, width=30)), ("Delta t_max=120s", initialize_bins(start=0, end=60, width=10) + initialize_bins(start=60, end=120, width=30)), ("Delta t_max=60s", initialize_bins(start=0, end=60, width=10)), ("Delta t_max=30s", initialize_bins(start=0, end=30, width=10)), ("Delta t_max=10s", initialize_bins(start=0, end=10, width=10)), #test various interval widths ("all intervals 2s wide", initialize_bins(start=0, end=300, width=2)), ("all intervals 4s wide", initialize_bins(start=0, end=300, width=4)), ("all intervals 6s wide", initialize_bins(start=0, end=300, width=6)), ("all intervals 8s wide", initialize_bins(start=0, end=300, width=8)), ("all intervals 30s wide", initialize_bins(start=0, end=300, width=30)), ("all intervals 50s wide", initialize_bins(start=0, end=300, width=50)), ("all intervals 100s wide", initialize_bins(start=0, end=300, width=100))] #run 10-fold cross-validation for each of the configured intervals experiment = Experiment(data) for (name, bins) in intervals_to_test: experiment.add_classifier(TemporalEvidencesClassifier(data.features, data.target_names, bins=bins), name=name) results = experiment.run(folds=10) results.print_quality_comparison_at_cutoff(cutoff=1, metrics=["Recall", "Precision", "F1"])
cutoff_results_at = 14 return data, cutoff_results_at def houseB(): """ This dataset is partially dominated by one of the sensors, which makes the evaluation results less statistically sound, e.g. it leads to large confidence intervals when running 10-fold cross-validation. """ data = load_dataset("../datasets/houseB.csv", "../datasets/houseB.config") cutoff_results_at = 15 return data, cutoff_results_at #configuration data, cutoff_results_at = houseA() #run several classifiers on the same dataset, use 10-fold cross-validation experiment = Experiment(data) experiment.add_classifier(TemporalEvidencesClassifier(data.features, data.target_names), name="Our method") experiment.add_classifier(NaiveBayesClassifier(data.features, data.target_names), name="Naive Bayes") experiment.add_classifier(RandomClassifier(data.features, data.target_names), name="Random") results = experiment.run(folds=10) #print and plot results results.print_quality_comparison_at_cutoff(cutoff=1, metrics=["Recall", "Precision", "F1"]) results.print_runtime_comparison() plot_conf = plot.plot_config(config.plot_directory, sub_dirs=[data.name], img_type=config.img_type) results.plot_quality_comparison(metrics=["Recall", "Precision", "F1"], plot_config=plot_conf, cutoff_results_at=cutoff_results_at)