def test_highLevelsktime(network=CNNClassifier()): ''' truly generalised test with sktime tasks/strategies load data, build task construct classifier, build strategy fit, score ''' print("start test_highLevelsktime()") from sktime.highlevel.tasks import TSCTask from sktime.highlevel.strategies import TSCStrategy from sklearn.metrics import accuracy_score train = load_italy_power_demand(split='TRAIN') test = load_italy_power_demand(split='TEST') task = TSCTask(target='class_val', metadata=train) strategy = TSCStrategy(network) strategy.fit(task, train.iloc[:10]) y_pred = strategy.predict(test.iloc[:10]) y_test = test.iloc[:10][task.target] print(accuracy_score(y_test, y_pred)) print("End test_highLevelsktime()")
def test_pipeline(network=CNNClassifier()): ''' slightly more generalised test with sktime pipelines load data, construct pipeline with classifier, fit, score ''' print("Start test_pipeline()") from sktime.pipeline import Pipeline # just a simple (useless) pipeline for the purposes of testing # that the keras network is compatible with that system steps = [ ('clf', network) ] clf = Pipeline(steps) X_train, y_train = load_italy_power_demand(split='TRAIN', return_X_y=True) X_test, y_test = load_italy_power_demand(split='TEST', return_X_y=True) hist = clf.fit(X_train[:10], y_train[:10]) print(clf.score(X_test[:10], y_test[:10])) print("End test_pipeline()")
def test_hivecote_v1_on_power_demand(): """Test of HIVE-COTEv1 on italy power demand.""" # load power demand data X_train, y_train = load_italy_power_demand(split="train", return_X_y=True) X_test, y_test = load_italy_power_demand(split="test", return_X_y=True) indices = np.random.RandomState(0).permutation(100) # train HIVE-COTE v1 hc1 = HIVECOTEV1( random_state=0, stc_params={ "n_estimators": 10, "transform_contract_in_mins": 0.1 }, tsf_params={"n_estimators": 10}, rise_params={"n_estimators": 10}, cboss_params={ "n_parameter_samples": 25, "max_ensemble_size": 5 }, ) hc1.fit(X_train, y_train) score = hc1.score(X_test.iloc[indices], y_test[indices]) assert score >= 0.92
def test_highLevelsktime(network=CNNClassifier(nb_epochs=SMALL_NB_EPOCHS)): """ truly generalised test with sktime tasks/strategies load data, build task construct classifier, build strategy fit, score """ print("start test_highLevelsktime()") from sktime.benchmarking.tasks import TSCTask from sktime.benchmarking.strategies import TSCStrategy from sklearn.metrics import accuracy_score train = load_italy_power_demand(split="train") test = load_italy_power_demand(split="test") task = TSCTask(target="class_val", metadata=train) strategy = TSCStrategy(network) strategy.fit(task, train.iloc[:10]) y_pred = strategy.predict(test.iloc[:10]) y_test = test.iloc[:10][task.target] print(accuracy_score(y_test, y_pred)) print("End test_highLevelsktime()")
def test_pipeline(network=CNNClassifier(nb_epochs=SMALL_NB_EPOCHS)): """ slightly more generalised test with sktime pipelines load data, construct pipeline with classifier, fit, score """ print("Start test_pipeline()") from sklearn.pipeline import Pipeline # just a simple (useless) pipeline for the purposes of testing # that the keras network is compatible with that system steps = [("clf", network)] clf = Pipeline(steps) X_train, y_train = load_italy_power_demand(split="train", return_X_y=True) X_test, y_test = load_italy_power_demand(split="test", return_X_y=True) clf.fit(X_train[:10], y_train[:10]) print(clf.score(X_test[:10], y_test[:10])) print("End test_pipeline()")
def test_classification_functionality(shape_descriptor_function): X_train, y_train = load_italy_power_demand(split="train", return_X_y=True) X_test, y_test = load_italy_power_demand(split="test", return_X_y=True) shp = ShapeDTW(shape_descriptor_function=shape_descriptor_function) shp.fit(X_train, y_train) assert shp.score(X_test, y_test) > 0
def test_weasel_on_power_demand(): # load power demand data X_train, y_train = load_italy_power_demand(split="train", return_X_y=True) X_test, y_test = load_italy_power_demand(split="test", return_X_y=True) # train WEASEL weasel = WEASEL(random_state=1, binning_strategy="kmeans") weasel.fit(X_train, y_train) score = weasel.score(X_test, y_test) # print(score) assert score >= 0.94
def test_drcif_on_power_demand(): # load power demand data X_train, y_train = load_italy_power_demand(split="train", return_X_y=True) X_test, y_test = load_italy_power_demand(split="test", return_X_y=True) indices = np.random.RandomState(0).permutation(100) # train DrCIF drcif = DrCIF(n_estimators=20, random_state=0) drcif.fit(X_train, y_train) score = drcif.score(X_test.iloc[indices], y_test[indices]) assert score >= 0.92
def test_weasel_on_power_demand(): # load power demand data X_train, y_train = load_italy_power_demand(split='train', return_X_y=True) X_test, y_test = load_italy_power_demand(split='test', return_X_y=True) # train WEASEL weasel = WEASEL(random_state=47) weasel.fit(X_train, y_train) score = weasel.score(X_test, y_test) print(score) assert (score >= 0.94)
def test_rocket_on_power_demand(): # load power demand data X_train, y_train = load_italy_power_demand(split="train", return_X_y=True) X_test, y_test = load_italy_power_demand(split="test", return_X_y=True) indices = np.random.RandomState(0).permutation(100) # train ROCKET rocket = ROCKETClassifier(num_kernels=1000, random_state=0) rocket.fit(X_train, y_train) score = rocket.score(X_test.iloc[indices], y_test[indices]) assert score >= 0.92
def test_stsf_on_power_demand(): # load power demand data X_train, y_train = load_italy_power_demand(split="train", return_X_y=True) X_test, y_test = load_italy_power_demand(split="test", return_X_y=True) indices = np.random.RandomState(0).permutation(100) # train STSF stsf = SupervisedTimeSeriesForest(random_state=0, n_estimators=20) stsf.fit(X_train, y_train) score = stsf.score(X_test.iloc[indices], y_test[indices]) assert score >= 0.92
def test_boss_on_power_demand(): # load power demand data X_train, y_train = load_italy_power_demand(split='train', return_X_y=True) X_test, y_test = load_italy_power_demand(split='test', return_X_y=True) indices = np.random.RandomState(0).permutation(100) # train BOSS boss = BOSSEnsemble(random_state=47) boss.fit(X_train, y_train) score = boss.score(X_test.iloc[indices], y_test[indices]) assert (score >= 0.80)
def test_cif_on_power_demand(): # load power demand data X_train, y_train = load_italy_power_demand(split="train", return_X_y=True) X_test, y_test = load_italy_power_demand(split="test", return_X_y=True) indices = np.random.RandomState(0).permutation(100) # train CIF cif = CanonicalIntervalForest(n_estimators=100, random_state=0) cif.fit(X_train, y_train) score = cif.score(X_test.iloc[indices], y_test[indices]) assert score >= 0.92
def test_matrix_profile_classifier_on_power_demand(): """Test of MatrixProfileClassifier on italy power demand.""" # load power demand data X_train, y_train = load_italy_power_demand(split="train", return_X_y=True) X_test, y_test = load_italy_power_demand(split="test", return_X_y=True) indices = np.random.RandomState(0).permutation(100) # train TSFresh classifier mpc = MatrixProfileClassifier(random_state=0) mpc.fit(X_train, y_train) score = mpc.score(X_test.iloc[indices], y_test[indices]) assert score >= 0.88
def test_cboss_on_power_demand(): # load power demand data X_train, y_train = load_italy_power_demand(split="train", return_X_y=True) X_test, y_test = load_italy_power_demand(split="test", return_X_y=True) indices = np.random.RandomState(0).permutation(100) # train cBOSS cboss = ContractableBOSS(n_parameter_samples=50, max_ensemble_size=10, random_state=0) cboss.fit(X_train, y_train) score = cboss.score(X_test.iloc[indices], y_test[indices]) assert score >= 0.9
def accuracy_test(network, lower=0.94, upper=1.0): """ Test the classifier accuracy against expected lower and upper bounds. """ print("Start accuracy_test:", network.__class__.__name__) X_train, y_train = load_italy_power_demand(split="train", return_X_y=True) X_test, y_test = load_italy_power_demand(split="test", return_X_y=True) network.fit(X_train, y_train) accuracy = network.score(X_test, y_test) print(network.__class__.__name__, "accuracy:", accuracy) assert lower < accuracy <= upper
def test_matrix_profile_classifier_on_power_demand(): """Test of Catch22Classifier on italy power demand.""" # load power demand data X_train, y_train = load_italy_power_demand(split="train", return_X_y=True) X_test, y_test = load_italy_power_demand(split="test", return_X_y=True) indices = np.random.RandomState(0).permutation(100) # train catch22 classifier rf = RandomForestClassifier(n_estimators=20) c22c = Catch22Classifier(random_state=0, estimator=rf) c22c.fit(X_train, y_train) score = c22c.score(X_test.iloc[indices], y_test[indices]) assert score >= 0.86
def test_arsenal_on_power_demand(n_jobs): # load power demand data X_train, y_train = load_italy_power_demand(split="train", return_X_y=True) X_test, y_test = load_italy_power_demand(split="test", return_X_y=True) indices = np.random.RandomState(0).permutation(100) # train Arsenal arsenal = Arsenal(num_kernels=1000, n_estimators=10, random_state=0, n_jobs=n_jobs) arsenal.fit(X_train, y_train) score = arsenal.score(X_test.iloc[indices], y_test[indices]) assert score >= 0.92
def test_tde_on_power_demand(): # load power demand data X_train, y_train = load_italy_power_demand(split="train", return_X_y=True) X_test, y_test = load_italy_power_demand(split="test", return_X_y=True) indices = np.random.RandomState(0).permutation(100) # train TDE tde = TemporalDictionaryEnsemble( n_parameter_samples=50, max_ensemble_size=10, randomly_selected_params=40, random_state=0, ) tde.fit(X_train, y_train) score = tde.score(X_test.iloc[indices], y_test[indices]) assert score >= 0.92
def test_basic_univariate(network=CNNClassifier(nb_epochs=SMALL_NB_EPOCHS)): """ just a super basic test with gunpoint, load data, construct classifier, fit, score """ print("Start test_basic()") X_train, y_train = load_italy_power_demand(split="train", return_X_y=True) X_test, y_test = load_italy_power_demand(split="test", return_X_y=True) network.fit(X_train[:10], y_train[:10]) print(network.score(X_test[:10], y_test[:10])) print("End test_basic()")
def test_basic_univariate(network=CNNClassifier()): ''' just a super basic test with gunpoint, load data, construct classifier, fit, score ''' print("Start test_basic()") X_train, y_train = load_italy_power_demand(split='TRAIN', return_X_y=True) X_test, y_test = load_italy_power_demand(split='TEST', return_X_y=True) hist = network.fit(X_train[:10], y_train[:10]) print(network.score(X_test[:10], y_test[:10])) print("End test_basic()")
def test_basic_tuning(network=TunedDeepLearningClassifier( base_model=CNNClassifier(), param_grid=dict(nb_epochs=[50, 100], ), cv_folds=3, )): """ just a super basic test of the tuner """ print("Start test_basic_tuning()") X_train, y_train = load_italy_power_demand(split="train", return_X_y=True) X_test, y_test = load_italy_power_demand(split="test", return_X_y=True) network.fit(X_train[:10], y_train[:10]) print(network.score(X_test[:10], y_test[:10])) print("End test_basic_tuning()")
def test_tsfresh_classifier_on_power_demand(relevant_feature_extractor): """Test of TSFreshClassifier on italy power demand.""" # load power demand data X_train, y_train = load_italy_power_demand(split="train", return_X_y=True) X_test, y_test = load_italy_power_demand(split="test", return_X_y=True) indices = np.random.RandomState(0).permutation(100) # train TSFresh classifier rf = RandomForestClassifier(n_estimators=20) tsfc = TSFreshClassifier( random_state=0, estimator=rf, relevant_feature_extractor=relevant_feature_extractor, ) tsfc.fit(X_train, y_train) score = tsfc.score(X_test.iloc[indices], y_test[indices]) assert score >= 0.9
def test_regressor(estimator=CNTCRegressor(nb_epochs=SMALL_NB_EPOCHS)): """ test a regressor """ print("Start test_regressor()") X_train, y_train = load_italy_power_demand(split="train", return_X_y=True) X_test, y_test = load_italy_power_demand(split="test", return_X_y=True) # Create some regression values y_train = np.zeros(len(y_train)) for i in range(len(X_train)): y_train[i] = X_train.iloc[i].iloc[0].iloc[0] y_test = np.zeros(len(y_test)) for i in range(len(X_test)): y_test[i] = X_test.iloc[i].iloc[0].iloc[0] estimator.fit(X_train[:10], y_train[:10]) estimator.predict(X_test[:10]) score = estimator.score(X_test[:10], y_test[:10]) print("Estimator score:", score) print("End test_regressor()")
def test_basic_inmem(network=DeepLearnerEnsembleClassifier( base_model=CNNClassifier(nb_epochs=50), nb_iterations=2, keep_in_memory=True, model_save_directory=None, verbose=True, )): """ just a super basic test with gunpoint, load data, construct classifier, fit, score """ print("Start test_basic()") X_train, y_train = load_italy_power_demand(split="train", return_X_y=True) X_test, y_test = load_italy_power_demand(split="test", return_X_y=True) network.fit(X_train[:10], y_train[:10]) print(network.score(X_test[:10], y_test[:10])) print("End test_basic()")
def test_basic_saving(network=DeepLearnerEnsembleClassifier( base_model=CNNClassifier(nb_epochs=50), nb_iterations=2, keep_in_memory=False, model_save_directory="testResultsDELETE", verbose=True, )): """ just a super basic test with gunpoint, load data, construct classifier, fit, score """ print("Start test_basic()") path = Path(network.model_save_directory) # if the directory doesn't get cleaned up because of error in testing if not path.exists(): path.mkdir() X_train, y_train = load_italy_power_demand(split="train", return_X_y=True) X_test, y_test = load_italy_power_demand(split="test", return_X_y=True) network.fit(X_train[:10], y_train[:10]) print(network.score(X_test[:10], y_test[:10])) (path / (network.base_model.model_name + "_0.hdf5")).unlink() # delete file (path / (network.base_model.model_name + "_1.hdf5")).unlink() # delete file path.rmdir() # directory should now be empty, fails if not print("End test_basic()")
def test_is_fitted(network=CNNClassifier()): """ testing that the networks correctly recognise when they are not fitted """ X_train, y_train = load_italy_power_demand(split="train", return_X_y=True) if isinstance(network, BaseRegressor): # Create some regression values, taken from test_regressor y_train = np.zeros(len(y_train)) for i in range(len(X_train)): y_train[i] = X_train.iloc[i].iloc[0].iloc[0] # try to predict without fitting: SHOULD fail with pytest.raises(NotFittedError): network.predict(X_train[:10])
def test_validation(network=MLPClassifier()): """ testing that the networks log validation predictions in the history object """ X_train, y_train = load_italy_power_demand(split="train", return_X_y=True) X_train = X_train[:10] y_train = y_train[:10] if isinstance(network, BaseRegressor): # Create some regression values, taken from test_regressor y_train = np.zeros(len(y_train)) for i in range(len(X_train)): y_train[i] = X_train.iloc[i].iloc[0].iloc[0] network.fit(X_train, y_train, validation_X=X_train, validation_y=y_train) hist = network.history.history assert ('val_loss' in hist) assert (isinstance(hist['val_loss'][0], (float, np.single, np.double, np.float32, np.float64)))
model.fit(X, y) return model def euclidean_distance(first, second, best_dist=sys.float_info.max): dist = 0 if isinstance(first, dict): words = set(list(first) + list(second)) for word in words: val_a = first.get(word, 0) val_b = second.get(word, 0) dist += (val_a - val_b) * (val_a - val_b) if dist > best_dist: return sys.float_info.max else: dist = np.sum([(first[n] - second[n]) * (first[n] - second[n]) for n in range(len(first))]) return dist if __name__ == "__main__": X_train, y_train = load_italy_power_demand(split='TRAIN', return_X_y=True) X_test, y_test = load_italy_power_demand(split='TEST', return_X_y=True) model = bop_pipeline(X_train, y_train) model.predict(X_test)
model.fit(X, y) return model def euclidean_distance(first, second, best_dist=sys.float_info.max): dist = 0 if isinstance(first, dict): words = set(list(first) + list(second)) for word in words: val_a = first.get(word, 0) val_b = second.get(word, 0) dist += (val_a - val_b) * (val_a - val_b) if dist > best_dist: return sys.float_info.max else: dist = np.sum( [(first[n] - second[n]) * (first[n] - second[n]) for n in range(len(first))] ) return dist if __name__ == "__main__": X_train, y_train = load_italy_power_demand(split="TRAIN", return_X_y=True) X_test, y_test = load_italy_power_demand(split="TEST", return_X_y=True) model = bop_pipeline(X_train, y_train) model.predict(X_test)