def test_highLevelsktime(network=CNNClassifier()):
'''
truly generalised test with sktime tasks/strategies
load data, build task
construct classifier, build strategy
fit,
score
'''
print("start test_highLevelsktime()")
from sktime.highlevel.tasks import TSCTask
from sktime.highlevel.strategies import TSCStrategy
from sklearn.metrics import accuracy_score
train = load_italy_power_demand(split='TRAIN')
test = load_italy_power_demand(split='TEST')
task = TSCTask(target='class_val', metadata=train)
strategy = TSCStrategy(network)
strategy.fit(task, train.iloc[:10])
y_pred = strategy.predict(test.iloc[:10])
y_test = test.iloc[:10][task.target]
print(accuracy_score(y_test, y_pred))
print("End test_highLevelsktime()")
def test_pipeline(network=CNNClassifier()):
'''
slightly more generalised test with sktime pipelines
load data,
construct pipeline with classifier,
fit,
score
'''
print("Start test_pipeline()")
from sktime.pipeline import Pipeline
# just a simple (useless) pipeline for the purposes of testing
# that the keras network is compatible with that system
steps = [
('clf', network)
]
clf = Pipeline(steps)
X_train, y_train = load_italy_power_demand(split='TRAIN', return_X_y=True)
X_test, y_test = load_italy_power_demand(split='TEST', return_X_y=True)
hist = clf.fit(X_train[:10], y_train[:10])
print(clf.score(X_test[:10], y_test[:10]))
print("End test_pipeline()")
def test_hivecote_v1_on_power_demand():
"""Test of HIVE-COTEv1 on italy power demand."""
# load power demand data
X_train, y_train = load_italy_power_demand(split="train", return_X_y=True)
X_test, y_test = load_italy_power_demand(split="test", return_X_y=True)
indices = np.random.RandomState(0).permutation(100)
# train HIVE-COTE v1
hc1 = HIVECOTEV1(
random_state=0,
stc_params={
"n_estimators": 10,
"transform_contract_in_mins": 0.1
},
tsf_params={"n_estimators": 10},
rise_params={"n_estimators": 10},
cboss_params={
"n_parameter_samples": 25,
"max_ensemble_size": 5
},
)
hc1.fit(X_train, y_train)
score = hc1.score(X_test.iloc[indices], y_test[indices])
assert score >= 0.92
def test_highLevelsktime(network=CNNClassifier(nb_epochs=SMALL_NB_EPOCHS)):
"""
truly generalised test with sktime tasks/strategies
load data, build task
construct classifier, build strategy
fit,
score
"""
print("start test_highLevelsktime()")
from sktime.benchmarking.tasks import TSCTask
from sktime.benchmarking.strategies import TSCStrategy
from sklearn.metrics import accuracy_score
train = load_italy_power_demand(split="train")
test = load_italy_power_demand(split="test")
task = TSCTask(target="class_val", metadata=train)
strategy = TSCStrategy(network)
strategy.fit(task, train.iloc[:10])
y_pred = strategy.predict(test.iloc[:10])
y_test = test.iloc[:10][task.target]
print(accuracy_score(y_test, y_pred))
print("End test_highLevelsktime()")
def test_pipeline(network=CNNClassifier(nb_epochs=SMALL_NB_EPOCHS)):
"""
slightly more generalised test with sktime pipelines
load data,
construct pipeline with classifier,
fit,
score
"""
print("Start test_pipeline()")
from sklearn.pipeline import Pipeline
# just a simple (useless) pipeline for the purposes of testing
# that the keras network is compatible with that system
steps = [("clf", network)]
clf = Pipeline(steps)
X_train, y_train = load_italy_power_demand(split="train", return_X_y=True)
X_test, y_test = load_italy_power_demand(split="test", return_X_y=True)
clf.fit(X_train[:10], y_train[:10])
print(clf.score(X_test[:10], y_test[:10]))
print("End test_pipeline()")
def test_classification_functionality(shape_descriptor_function):
X_train, y_train = load_italy_power_demand(split="train", return_X_y=True)
X_test, y_test = load_italy_power_demand(split="test", return_X_y=True)
shp = ShapeDTW(shape_descriptor_function=shape_descriptor_function)
shp.fit(X_train, y_train)
assert shp.score(X_test, y_test) > 0
def test_weasel_on_power_demand():
# load power demand data
X_train, y_train = load_italy_power_demand(split="train", return_X_y=True)
X_test, y_test = load_italy_power_demand(split="test", return_X_y=True)
# train WEASEL
weasel = WEASEL(random_state=1, binning_strategy="kmeans")
weasel.fit(X_train, y_train)
score = weasel.score(X_test, y_test)
# print(score)
assert score >= 0.94
def test_drcif_on_power_demand():
# load power demand data
X_train, y_train = load_italy_power_demand(split="train", return_X_y=True)
X_test, y_test = load_italy_power_demand(split="test", return_X_y=True)
indices = np.random.RandomState(0).permutation(100)
# train DrCIF
drcif = DrCIF(n_estimators=20, random_state=0)
drcif.fit(X_train, y_train)
score = drcif.score(X_test.iloc[indices], y_test[indices])
assert score >= 0.92
def test_weasel_on_power_demand():
# load power demand data
X_train, y_train = load_italy_power_demand(split='train', return_X_y=True)
X_test, y_test = load_italy_power_demand(split='test', return_X_y=True)
# train WEASEL
weasel = WEASEL(random_state=47)
weasel.fit(X_train, y_train)
score = weasel.score(X_test, y_test)
print(score)
assert (score >= 0.94)
def test_rocket_on_power_demand():
# load power demand data
X_train, y_train = load_italy_power_demand(split="train", return_X_y=True)
X_test, y_test = load_italy_power_demand(split="test", return_X_y=True)
indices = np.random.RandomState(0).permutation(100)
# train ROCKET
rocket = ROCKETClassifier(num_kernels=1000, random_state=0)
rocket.fit(X_train, y_train)
score = rocket.score(X_test.iloc[indices], y_test[indices])
assert score >= 0.92
def test_stsf_on_power_demand():
# load power demand data
X_train, y_train = load_italy_power_demand(split="train", return_X_y=True)
X_test, y_test = load_italy_power_demand(split="test", return_X_y=True)
indices = np.random.RandomState(0).permutation(100)
# train STSF
stsf = SupervisedTimeSeriesForest(random_state=0, n_estimators=20)
stsf.fit(X_train, y_train)
score = stsf.score(X_test.iloc[indices], y_test[indices])
assert score >= 0.92
def test_boss_on_power_demand():
# load power demand data
X_train, y_train = load_italy_power_demand(split='train', return_X_y=True)
X_test, y_test = load_italy_power_demand(split='test', return_X_y=True)
indices = np.random.RandomState(0).permutation(100)
# train BOSS
boss = BOSSEnsemble(random_state=47)
boss.fit(X_train, y_train)
score = boss.score(X_test.iloc[indices], y_test[indices])
assert (score >= 0.80)
def test_cif_on_power_demand():
# load power demand data
X_train, y_train = load_italy_power_demand(split="train", return_X_y=True)
X_test, y_test = load_italy_power_demand(split="test", return_X_y=True)
indices = np.random.RandomState(0).permutation(100)
# train CIF
cif = CanonicalIntervalForest(n_estimators=100, random_state=0)
cif.fit(X_train, y_train)
score = cif.score(X_test.iloc[indices], y_test[indices])
assert score >= 0.92
def test_matrix_profile_classifier_on_power_demand():
"""Test of MatrixProfileClassifier on italy power demand."""
# load power demand data
X_train, y_train = load_italy_power_demand(split="train", return_X_y=True)
X_test, y_test = load_italy_power_demand(split="test", return_X_y=True)
indices = np.random.RandomState(0).permutation(100)
# train TSFresh classifier
mpc = MatrixProfileClassifier(random_state=0)
mpc.fit(X_train, y_train)
score = mpc.score(X_test.iloc[indices], y_test[indices])
assert score >= 0.88
def test_cboss_on_power_demand():
# load power demand data
X_train, y_train = load_italy_power_demand(split="train", return_X_y=True)
X_test, y_test = load_italy_power_demand(split="test", return_X_y=True)
indices = np.random.RandomState(0).permutation(100)
# train cBOSS
cboss = ContractableBOSS(n_parameter_samples=50,
max_ensemble_size=10,
random_state=0)
cboss.fit(X_train, y_train)
score = cboss.score(X_test.iloc[indices], y_test[indices])
assert score >= 0.9
def accuracy_test(network, lower=0.94, upper=1.0):
"""
Test the classifier accuracy against expected lower and upper bounds.
"""
print("Start accuracy_test:", network.__class__.__name__)
X_train, y_train = load_italy_power_demand(split="train", return_X_y=True)
X_test, y_test = load_italy_power_demand(split="test", return_X_y=True)
network.fit(X_train, y_train)
accuracy = network.score(X_test, y_test)
print(network.__class__.__name__, "accuracy:", accuracy)
assert lower < accuracy <= upper
def test_matrix_profile_classifier_on_power_demand():
"""Test of Catch22Classifier on italy power demand."""
# load power demand data
X_train, y_train = load_italy_power_demand(split="train", return_X_y=True)
X_test, y_test = load_italy_power_demand(split="test", return_X_y=True)
indices = np.random.RandomState(0).permutation(100)
# train catch22 classifier
rf = RandomForestClassifier(n_estimators=20)
c22c = Catch22Classifier(random_state=0, estimator=rf)
c22c.fit(X_train, y_train)
score = c22c.score(X_test.iloc[indices], y_test[indices])
assert score >= 0.86
def test_arsenal_on_power_demand(n_jobs):
# load power demand data
X_train, y_train = load_italy_power_demand(split="train", return_X_y=True)
X_test, y_test = load_italy_power_demand(split="test", return_X_y=True)
indices = np.random.RandomState(0).permutation(100)
# train Arsenal
arsenal = Arsenal(num_kernels=1000,
n_estimators=10,
random_state=0,
n_jobs=n_jobs)
arsenal.fit(X_train, y_train)
score = arsenal.score(X_test.iloc[indices], y_test[indices])
assert score >= 0.92
def test_tde_on_power_demand():
# load power demand data
X_train, y_train = load_italy_power_demand(split="train", return_X_y=True)
X_test, y_test = load_italy_power_demand(split="test", return_X_y=True)
indices = np.random.RandomState(0).permutation(100)
# train TDE
tde = TemporalDictionaryEnsemble(
n_parameter_samples=50,
max_ensemble_size=10,
randomly_selected_params=40,
random_state=0,
)
tde.fit(X_train, y_train)
score = tde.score(X_test.iloc[indices], y_test[indices])
assert score >= 0.92
def test_basic_univariate(network=CNNClassifier(nb_epochs=SMALL_NB_EPOCHS)):
"""
just a super basic test with gunpoint,
load data,
construct classifier,
fit,
score
"""
print("Start test_basic()")
X_train, y_train = load_italy_power_demand(split="train", return_X_y=True)
X_test, y_test = load_italy_power_demand(split="test", return_X_y=True)
network.fit(X_train[:10], y_train[:10])
print(network.score(X_test[:10], y_test[:10]))
print("End test_basic()")
def test_basic_univariate(network=CNNClassifier()):
'''
just a super basic test with gunpoint,
load data,
construct classifier,
fit,
score
'''
print("Start test_basic()")
X_train, y_train = load_italy_power_demand(split='TRAIN', return_X_y=True)
X_test, y_test = load_italy_power_demand(split='TEST', return_X_y=True)
hist = network.fit(X_train[:10], y_train[:10])
print(network.score(X_test[:10], y_test[:10]))
print("End test_basic()")
def test_basic_tuning(network=TunedDeepLearningClassifier(
base_model=CNNClassifier(),
param_grid=dict(nb_epochs=[50, 100], ),
cv_folds=3,
)):
"""
just a super basic test of the tuner
"""
print("Start test_basic_tuning()")
X_train, y_train = load_italy_power_demand(split="train", return_X_y=True)
X_test, y_test = load_italy_power_demand(split="test", return_X_y=True)
network.fit(X_train[:10], y_train[:10])
print(network.score(X_test[:10], y_test[:10]))
print("End test_basic_tuning()")
def test_tsfresh_classifier_on_power_demand(relevant_feature_extractor):
"""Test of TSFreshClassifier on italy power demand."""
# load power demand data
X_train, y_train = load_italy_power_demand(split="train", return_X_y=True)
X_test, y_test = load_italy_power_demand(split="test", return_X_y=True)
indices = np.random.RandomState(0).permutation(100)
# train TSFresh classifier
rf = RandomForestClassifier(n_estimators=20)
tsfc = TSFreshClassifier(
random_state=0,
estimator=rf,
relevant_feature_extractor=relevant_feature_extractor,
)
tsfc.fit(X_train, y_train)
score = tsfc.score(X_test.iloc[indices], y_test[indices])
assert score >= 0.9
def test_regressor(estimator=CNTCRegressor(nb_epochs=SMALL_NB_EPOCHS)):
"""
test a regressor
"""
print("Start test_regressor()")
X_train, y_train = load_italy_power_demand(split="train", return_X_y=True)
X_test, y_test = load_italy_power_demand(split="test", return_X_y=True)
# Create some regression values
y_train = np.zeros(len(y_train))
for i in range(len(X_train)):
y_train[i] = X_train.iloc[i].iloc[0].iloc[0]
y_test = np.zeros(len(y_test))
for i in range(len(X_test)):
y_test[i] = X_test.iloc[i].iloc[0].iloc[0]
estimator.fit(X_train[:10], y_train[:10])
estimator.predict(X_test[:10])
score = estimator.score(X_test[:10], y_test[:10])
print("Estimator score:", score)
print("End test_regressor()")
def test_basic_inmem(network=DeepLearnerEnsembleClassifier(
base_model=CNNClassifier(nb_epochs=50),
nb_iterations=2,
keep_in_memory=True,
model_save_directory=None,
verbose=True,
)):
"""
just a super basic test with gunpoint,
load data,
construct classifier,
fit,
score
"""
print("Start test_basic()")
X_train, y_train = load_italy_power_demand(split="train", return_X_y=True)
X_test, y_test = load_italy_power_demand(split="test", return_X_y=True)
network.fit(X_train[:10], y_train[:10])
print(network.score(X_test[:10], y_test[:10]))
print("End test_basic()")
def test_basic_saving(network=DeepLearnerEnsembleClassifier(
base_model=CNNClassifier(nb_epochs=50),
nb_iterations=2,
keep_in_memory=False,
model_save_directory="testResultsDELETE",
verbose=True,
)):
"""
just a super basic test with gunpoint,
load data,
construct classifier,
fit,
score
"""
print("Start test_basic()")
path = Path(network.model_save_directory)
# if the directory doesn't get cleaned up because of error in testing
if not path.exists():
path.mkdir()
X_train, y_train = load_italy_power_demand(split="train", return_X_y=True)
X_test, y_test = load_italy_power_demand(split="test", return_X_y=True)
network.fit(X_train[:10], y_train[:10])
print(network.score(X_test[:10], y_test[:10]))
(path /
(network.base_model.model_name + "_0.hdf5")).unlink() # delete file
(path /
(network.base_model.model_name + "_1.hdf5")).unlink() # delete file
path.rmdir() # directory should now be empty, fails if not
print("End test_basic()")
def test_is_fitted(network=CNNClassifier()):
"""
testing that the networks correctly recognise when they are not fitted
"""
X_train, y_train = load_italy_power_demand(split="train", return_X_y=True)
if isinstance(network, BaseRegressor):
# Create some regression values, taken from test_regressor
y_train = np.zeros(len(y_train))
for i in range(len(X_train)):
y_train[i] = X_train.iloc[i].iloc[0].iloc[0]
# try to predict without fitting: SHOULD fail
with pytest.raises(NotFittedError):
network.predict(X_train[:10])
def test_validation(network=MLPClassifier()):
"""
testing that the networks log validation predictions in the history object
"""
X_train, y_train = load_italy_power_demand(split="train", return_X_y=True)
X_train = X_train[:10]
y_train = y_train[:10]
if isinstance(network, BaseRegressor):
# Create some regression values, taken from test_regressor
y_train = np.zeros(len(y_train))
for i in range(len(X_train)):
y_train[i] = X_train.iloc[i].iloc[0].iloc[0]
network.fit(X_train, y_train, validation_X=X_train, validation_y=y_train)
hist = network.history.history
assert ('val_loss' in hist)
assert (isinstance(hist['val_loss'][0],
(float, np.single, np.double, np.float32, np.float64)))
model.fit(X, y)
return model
def euclidean_distance(first, second, best_dist=sys.float_info.max):
dist = 0
if isinstance(first, dict):
words = set(list(first) + list(second))
for word in words:
val_a = first.get(word, 0)
val_b = second.get(word, 0)
dist += (val_a - val_b) * (val_a - val_b)
if dist > best_dist:
return sys.float_info.max
else:
dist = np.sum([(first[n] - second[n]) * (first[n] - second[n]) for n in range(len(first))])
return dist
if __name__ == "__main__":
X_train, y_train = load_italy_power_demand(split='TRAIN', return_X_y=True)
X_test, y_test = load_italy_power_demand(split='TEST', return_X_y=True)
model = bop_pipeline(X_train, y_train)
model.predict(X_test)
model.fit(X, y)
return model
def euclidean_distance(first, second, best_dist=sys.float_info.max):
dist = 0
if isinstance(first, dict):
words = set(list(first) + list(second))
for word in words:
val_a = first.get(word, 0)
val_b = second.get(word, 0)
dist += (val_a - val_b) * (val_a - val_b)
if dist > best_dist:
return sys.float_info.max
else:
dist = np.sum(
[(first[n] - second[n]) * (first[n] - second[n]) for n in range(len(first))]
)
return dist
if __name__ == "__main__":
X_train, y_train = load_italy_power_demand(split="TRAIN", return_X_y=True)
X_test, y_test = load_italy_power_demand(split="TEST", return_X_y=True)
model = bop_pipeline(X_train, y_train)
model.predict(X_test)
