def houseB():
"""
This dataset is partially dominated by one of the sensors, which makes the evaluation results less statistically
sound, e.g. it leads to large confidence intervals when running 10-fold cross-validation.
"""
data = load_dataset("../datasets/houseB.csv", "../datasets/houseB.config")
cutoff_results_at = 15
return data, cutoff_results_at
def houseB():
"""
This dataset is partially dominated by one of the sensors, which makes the evaluation results less statistically
sound, e.g. it leads to large confidence intervals when running 10-fold cross-validation.
"""
data = load_dataset("../datasets/houseB.csv", "../datasets/houseB.config")
cutoff_results_at = 15
return data, cutoff_results_at
def houseA():
"""
This dataset has 14 binary sensors, i.e. at most 14 services are typically available concurrently. The only anomaly
is right at the beginning of the dataset, where the current status of the sensors are not known. In this case more
than 14 services can be recommended. However, there will be few instances where this is the case and
recommendation results will be be statistically insignificant for these values. For this reason, when printing or
plotting results, cut the recommendation results at 14 services.
"""
data = load_dataset("../datasets/houseA.csv", "../datasets/houseA.config")
cutoff_results_at = 14
return data, cutoff_results_at
def houseA():
"""
This dataset has 14 binary sensors, i.e. at most 14 services are typically available concurrently. The only anomaly
is right at the beginning of the dataset, where the current status of the sensors are not known. In this case more
than 14 services can be recommended. However, there will be few instances where this is the case and
recommendation results will be be statistically insignificant for these values. For this reason, when printing or
plotting results, cut the recommendation results at 14 services.
"""
data = load_dataset("../datasets/houseA.csv", "../datasets/houseA.config")
cutoff_results_at = 14
return data, cutoff_results_at
def test_houseA():
"""
Check if recommendation results of the Naive Bayes classifier are as expected for the houseA dataset.
"""
expected_precision = [
0.36389964907400657, 0.35097921410756322, 0.33726522147404076,
0.32783328310438886, 0.32350609998195629, 0.31650421215426117,
0.31231855510715401, 0.30519493826462696, 0.29979871087041599,
0.29551593405461341, 0.29150527293056733, 0.28660658398334166,
0.28251775629715375, 0.27922900853036048, 0.27801361790520279,
0.27550796383161991, 0.27466099306467773, 0.27342926821355851,
0.27273588164873097, 0.27221800916720873, 0.2721689921161986,
0.27166460270134835, 0.2715840524866433, 0.27152878300894606,
0.27150502624152628, 0.27150496777883093
]
expected_recall = [
0.50280293229840445, 0.62440707201379908, 0.72229409228115571,
0.78654592496765852, 0.84260457093574814, 0.87365243639499779,
0.90426908150064678, 0.93014230271668819, 0.95170332039672267,
0.96593359206554552, 0.97714532125916342, 0.98835705045278133,
0.99396291504959033, 0.99525657611039242, 0.99568779646399308,
0.99568779646399308, 0.99611901681759374, 0.99655023717119451,
0.99655023717119451, 0.99655023717119451, 0.99655023717119451,
0.99655023717119451, 0.99655023717119451, 0.99655023717119451,
0.99655023717119451, 0.99655023717119451
]
expected_f1 = [
0.40969896551479879, 0.40872515566619289, 0.42494674577211028,
0.41795013553355859, 0.41402807045377227, 0.40755506280680126,
0.40304176773376232, 0.39634842720545965, 0.39155915552074971,
0.38766523517736751, 0.38402205223880148, 0.37854826039224554,
0.37434429038914058, 0.37108071816768462, 0.36996547420680476,
0.36801458947276205, 0.36727323996045197, 0.36622750241189023,
0.36561611066990368, 0.36516946998665389, 0.36510160971503941,
0.36461007637070081, 0.36450029640432352, 0.36442385338288585,
0.3643920355519879, 0.36439191983486152
]
#perform classification using NaiveBayes on houseA
dataset = load_dataset(houseA_csv, houseA_config)
cls = NaiveBayesClassifier(dataset.features, dataset.target_names)
cls = cls.fit(dataset.data, dataset.target)
results = cls.predict(dataset.data)
metrics = QualityMetricsCalculator(dataset.target, results).calculate()
assert_almost_equal(metrics["Precision"].values,
expected_precision,
decimal=3)
assert_almost_equal(metrics["Recall"].values, expected_recall, decimal=3)
assert_almost_equal(metrics["F1"].values, expected_f1, decimal=3)
def test_train():
"""
Test that the classifier correctly extracts all observations from the test dataset.
"""
#train the classifier
data = load_dataset(data_file)
cls = TemporalEvidencesClassifier(data.features, data.target_names)
cls = cls.fit(data.data, data.target)
#load expected sources and their observations from json file
expected_sources = sources_from_json(sources_file)
#compare expected with actual sources
assert_array_equal(sorted(cls.sources.keys()), sorted(expected_sources.keys()),)
for name in expected_sources.keys():
assert_source_equal(cls.sources[name], expected_sources[name])
def test_recommend():
"""
Test that the classifier generates the correct recommendations for the test dataset.
"""
#train the classifier and calculate recommendations
data = load_dataset(data_file)
cls = TemporalEvidencesClassifier(data.features, data.target_names)
cls = cls.fit(data.data, data.target)
actual_recommendations = cls.predict(data.data, include_conflict_theta=True)
#load expected results from json file
with open(recommendations_file, 'r') as infile:
expected_recommendations = json.load(infile)
#compare expected with actual results
for actual, expected in zip(actual_recommendations, expected_recommendations):
assert_recommendations_equal(actual, expected)
def test_train():
"""
Test that the classifier correctly extracts all observations from the test dataset.
"""
#train the classifier
data = load_dataset(data_file)
cls = TemporalEvidencesClassifier(data.features, data.target_names)
cls = cls.fit(data.data, data.target)
#load expected sources and their observations from json file
expected_sources = sources_from_json(sources_file)
#compare expected with actual sources
assert_array_equal(
sorted(cls.sources.keys()),
sorted(expected_sources.keys()),
)
for name in expected_sources.keys():
assert_source_equal(cls.sources[name], expected_sources[name])
def test_recommend():
"""
Test that the classifier generates the correct recommendations for the test dataset.
"""
#train the classifier and calculate recommendations
data = load_dataset(data_file)
cls = TemporalEvidencesClassifier(data.features, data.target_names)
cls = cls.fit(data.data, data.target)
actual_recommendations = cls.predict(data.data,
include_conflict_theta=True)
#load expected results from json file
with open(recommendations_file, 'r') as infile:
expected_recommendations = json.load(infile)
#compare expected with actual results
for actual, expected in zip(actual_recommendations,
expected_recommendations):
assert_recommendations_equal(actual, expected)
def test_houseA():
"""
Check if recommendation results of the Naive Bayes classifier are as expected for the houseA dataset.
"""
expected_precision = [0.36389964907400657, 0.35097921410756322, 0.33726522147404076, 0.32783328310438886,
0.32350609998195629, 0.31650421215426117, 0.31231855510715401, 0.30519493826462696,
0.29979871087041599, 0.29551593405461341, 0.29150527293056733, 0.28660658398334166,
0.28251775629715375, 0.27922900853036048, 0.27801361790520279, 0.27550796383161991,
0.27466099306467773, 0.27342926821355851, 0.27273588164873097, 0.27221800916720873,
0.2721689921161986, 0.27166460270134835, 0.2715840524866433, 0.27152878300894606,
0.27150502624152628, 0.27150496777883093]
expected_recall = [0.50280293229840445, 0.62440707201379908, 0.72229409228115571, 0.78654592496765852,
0.84260457093574814, 0.87365243639499779, 0.90426908150064678, 0.93014230271668819,
0.95170332039672267, 0.96593359206554552, 0.97714532125916342, 0.98835705045278133,
0.99396291504959033, 0.99525657611039242, 0.99568779646399308, 0.99568779646399308,
0.99611901681759374, 0.99655023717119451, 0.99655023717119451, 0.99655023717119451,
0.99655023717119451, 0.99655023717119451, 0.99655023717119451, 0.99655023717119451,
0.99655023717119451, 0.99655023717119451]
expected_f1 = [0.40969896551479879, 0.40872515566619289, 0.42494674577211028, 0.41795013553355859,
0.41402807045377227, 0.40755506280680126, 0.40304176773376232, 0.39634842720545965,
0.39155915552074971, 0.38766523517736751, 0.38402205223880148, 0.37854826039224554,
0.37434429038914058, 0.37108071816768462, 0.36996547420680476, 0.36801458947276205,
0.36727323996045197, 0.36622750241189023, 0.36561611066990368, 0.36516946998665389,
0.36510160971503941, 0.36461007637070081, 0.36450029640432352, 0.36442385338288585,
0.3643920355519879, 0.36439191983486152]
#perform classification using NaiveBayes on houseA
dataset = load_dataset(houseA_csv, houseA_config)
cls = NaiveBayesClassifier(dataset.features, dataset.target_names)
cls = cls.fit(dataset.data, dataset.target)
results = cls.predict(dataset.data)
metrics = QualityMetricsCalculator(dataset.target, results).calculate()
assert_almost_equal(metrics["Precision"].values, expected_precision, decimal=3)
assert_almost_equal(metrics["Recall"].values, expected_recall, decimal=3)
assert_almost_equal(metrics["F1"].values, expected_f1, decimal=3)
"""
import sys
sys.path.append("..")
import pandas
from recsys.classifiers.temporal import TemporalEvidencesClassifier
from recsys.dataset import load_dataset
from evaluation.metrics import results_as_dataframe
from evaluation import plot
import config
#configuration
data = load_dataset("../datasets/houseA.csv", "../datasets/houseA.config")
#run the classifier on the whole dataset
cls = TemporalEvidencesClassifier(data.features, data.target_names)
cls = cls.fit(data.data, data.target)
results = cls.predict(data.data, include_conflict_theta=True)
#extract conflict and uncertainty and convert recommendations to pandas representation
recommendations, conflict, uncertainty = zip(*results)
results = results_as_dataframe(data.target, list(recommendations))
#for each row, mark correct recommendations with "1", false recommendations with "0"
find_matches_in_row = lambda row: [1 if col == row.name else 0 for col in row]
results = results.apply(find_matches_in_row, axis=1)
#set uncertainty and conflict as multi-index
Further details for this experiment can be found in the paper in Section 6.6 and the dissertation in Section 5.5.7
"""
import sys
sys.path.append("..")
import pandas
from evaluation.experiment import Experiment
from evaluation.metrics import quality_metrics
from recsys.classifiers.temporal import TemporalEvidencesClassifier, configure_dynamic_cutoff
from recsys.dataset import load_dataset
#configuration
data = load_dataset("../datasets/houseA.csv", "../datasets/houseA.config")
methods_to_test = [("Fixed cutoff", None),
("dynamic cutoff=4", configure_dynamic_cutoff(1.0, 0.4, 4)),
("dynamic cutoff=2", configure_dynamic_cutoff(1.0, 0.4, 2))]
#run all configured cutoffs with 10-fold cross-validation
experiment = Experiment(data)
for name, method in methods_to_test:
experiment.add_classifier(TemporalEvidencesClassifier(data.features, data.target_names,
postprocess=method), name=name)
results = experiment.run(folds=10)
#print results
pandas.set_option('expand_frame_repr', False)
pandas.set_option('max_columns', 4)
print "Maximum 5 recommendations"
