def test_automated_orchestration_vs_manual(data_loader):
"""Test orchestration."""
data = data_loader(return_X_y=False)
dataset = RAMDataset(dataset=data, name="data")
task = TSCTask(target="class_val")
# create strategies
# clf = TimeSeriesForestClassifier(n_estimators=1, random_state=1)
clf = make_reduction_pipeline(
RandomForestClassifier(n_estimators=2, random_state=1))
strategy = TSCStrategy(clf)
# result backend
results = RAMResults()
orchestrator = Orchestrator(
datasets=[dataset],
tasks=[task],
strategies=[strategy],
cv=SingleSplit(random_state=1),
results=results,
)
orchestrator.fit_predict(save_fitted_strategies=False)
result = next(results.load_predictions(cv_fold=0,
train_or_test="test")) # get
# only first item of iterator
actual = result.y_pred
# expected output
task = TSCTask(target="class_val")
cv = SingleSplit(random_state=1)
train_idx, test_idx = next(cv.split(data))
train = data.iloc[train_idx, :]
test = data.iloc[test_idx, :]
strategy.fit(task, train)
expected = strategy.predict(test)
# compare results
np.testing.assert_array_equal(actual, expected)
def test_stat():
"""Test sign ranks."""
data = load_gunpoint(split="train", return_X_y=False)
dataset = RAMDataset(dataset=data, name="gunpoint")
task = TSCTask(target="class_val")
fc = ComposableTimeSeriesForestClassifier(n_estimators=1, random_state=1)
strategy_fc = TSCStrategy(fc, name="tsf")
pf = KNeighborsTimeSeriesClassifier()
strategy_pf = TSCStrategy(pf, name="pf")
# result backend
results = RAMResults()
orchestrator = Orchestrator(
datasets=[dataset],
tasks=[task],
strategies=[strategy_pf, strategy_fc],
cv=SingleSplit(random_state=1),
results=results,
)
orchestrator.fit_predict(save_fitted_strategies=False)
analyse = Evaluator(results)
metric = PairwiseMetric(func=accuracy_score, name="accuracy")
_ = analyse.evaluate(metric=metric)
ranks = analyse.rank(ascending=True)
pf_rank = ranks.loc[ranks.strategy == "pf",
"accuracy_mean_rank"].item() # 1
fc_rank = ranks.loc[ranks.strategy == "tsf",
"accuracy_mean_rank"].item() # 2
rank_array = [pf_rank, fc_rank]
rank_array_test = [1, 2]
_, sign_test_df = analyse.sign_test()
sign_array = [
[sign_test_df["pf"][0], sign_test_df["pf"][1]],
[sign_test_df["tsf"][0], sign_test_df["tsf"][1]],
]
sign_array_test = [[1, 1], [1, 1]]
np.testing.assert_equal([rank_array, sign_array],
[rank_array_test, sign_array_test])
def dummy_results():
results = RAMResults()
results.cv = PresplitFilesCV()
results.save_predictions(
strategy_name="alg1",
dataset_name="dataset1",
index=np.array([1, 2, 3, 4]),
y_true=np.array([1, 1, 1, 1]),
y_pred=np.array([1, 1, 1, 1]),
y_proba=None,
cv_fold=0,
train_or_test="test",
fit_estimator_start_time=pd.to_datetime(1605268800, unit="ms"),
fit_estimator_end_time=pd.to_datetime(1605268801, unit="ms"),
predict_estimator_start_time=pd.to_datetime(1605268800, unit="ms"),
predict_estimator_end_time=pd.to_datetime(1605268801, unit="ms"),
)
results.save_predictions(
strategy_name="alg1",
dataset_name="dataset2",
index=np.array([1, 2, 3, 4]),
y_true=np.array([0, 0, 0, 0]),
y_pred=np.array([0, 0, 0, 0]),
y_proba=None,
cv_fold=0,
train_or_test="test",
fit_estimator_start_time=pd.to_datetime(1605268800, unit="ms"),
fit_estimator_end_time=pd.to_datetime(1605268801, unit="ms"),
predict_estimator_start_time=pd.to_datetime(1605268800, unit="ms"),
predict_estimator_end_time=pd.to_datetime(1605268801, unit="ms"),
)
results.save_predictions(
strategy_name="alg2",
dataset_name="dataset1",
index=np.array([1, 2, 3, 4]),
y_true=np.array([1, 1, 1, 1]),
y_pred=np.array([0, 0, 0, 0]),
y_proba=None,
cv_fold=0,
train_or_test="test",
fit_estimator_start_time=pd.to_datetime(1605268800, unit="ms"),
fit_estimator_end_time=pd.to_datetime(1605268801, unit="ms"),
predict_estimator_start_time=pd.to_datetime(1605268800, unit="ms"),
predict_estimator_end_time=pd.to_datetime(1605268801, unit="ms"),
)
results.save_predictions(
strategy_name="alg2",
dataset_name="dataset2",
index=np.array([1, 2, 3, 4]),
y_true=np.array([0, 0, 0, 0]),
y_pred=np.array([1, 1, 1, 1]),
y_proba=None,
cv_fold=0,
train_or_test="test",
fit_estimator_start_time=pd.to_datetime(1605268800, unit="ms"),
fit_estimator_end_time=pd.to_datetime(1605268801, unit="ms"),
predict_estimator_start_time=pd.to_datetime(1605268800, unit="ms"),
predict_estimator_end_time=pd.to_datetime(1605268801, unit="ms"),
)
results.save_predictions(
strategy_name="alg3",
dataset_name="dataset1",
index=np.array([1, 2, 3, 4]),
y_true=np.array([1, 1, 1, 1]),
y_pred=np.array([1, 1, 0, 1]),
y_proba=None,
cv_fold=0,
train_or_test="test",
fit_estimator_start_time=pd.to_datetime(1605268800, unit="ms"),
fit_estimator_end_time=pd.to_datetime(1605268801, unit="ms"),
predict_estimator_start_time=pd.to_datetime(1605268800, unit="ms"),
predict_estimator_end_time=pd.to_datetime(1605268801, unit="ms"),
)
results.save_predictions(
strategy_name="alg3",
dataset_name="dataset2",
index=np.array([1, 2, 3, 4]),
y_true=np.array([0, 0, 0, 0]),
y_pred=np.array([0, 0, 1, 0]),
y_proba=None,
cv_fold=0,
train_or_test="test",
fit_estimator_start_time=pd.to_datetime(1605268800, unit="ms"),
fit_estimator_end_time=pd.to_datetime(1605268801, unit="ms"),
predict_estimator_start_time=pd.to_datetime(1605268800, unit="ms"),
predict_estimator_end_time=pd.to_datetime(1605268801, unit="ms"),
)
return results