def test_mysvc_reducer(self):
## 1) Build dataset
## ===================================================================
X, y = datasets.make_classification(n_samples=12,
n_features=10,
n_informative=2,
random_state=1)
## 2) run with Methods
## ===================================================================
my_svc1 = MySVC(C=1.0)
my_svc2 = MySVC(C=2.0)
two_svc_single = Methods(my_svc1, my_svc2)
two_svc_local = Methods(my_svc1, my_svc2)
two_svc_swf = Methods(my_svc1, my_svc2)
two_svc_single.reducer = MyReducer()
two_svc_local.reducer = MyReducer()
two_svc_swf.reducer = MyReducer()
for leaf in two_svc_single.walk_leaves():
print leaf.get_key()
for leaf in two_svc_local.walk_leaves():
print leaf.get_key()
for leaf in two_svc_swf.walk_leaves():
print leaf.get_key()
# top-down process to call transform
two_svc_single.run(X=X, y=y)
# buttom-up process to compute scores
res_single = two_svc_single.reduce()
### You can get below results:
### ==================================================================
### [{'MySVC(C=1.0)': array([ 1., 1.])}, {'MySVC(C=2.0)': array([ 1., 1.])}]
### 3) Run using local multi-processes
### ==================================================================
from epac.map_reduce.engine import LocalEngine
local_engine = LocalEngine(two_svc_local, num_processes=2)
two_svc_local = local_engine.run(**dict(X=X, y=y))
res_local = two_svc_local.reduce()
### 4) Run using soma-workflow
### ==================================================================
from epac.map_reduce.engine import SomaWorkflowEngine
sfw_engine = SomaWorkflowEngine(tree_root=two_svc_swf,
num_processes=2)
two_svc_swf = sfw_engine.run(**dict(X=X, y=y))
res_swf = two_svc_swf.reduce()
if not repr(res_swf) == repr(res_local):
raise ValueError("Cannot dump class definition")
if not repr(res_swf) == repr(res_single):
raise ValueError("Cannot dump class definition")
def test_mysvc_reducer(self):
## 1) Build dataset
## ===================================================================
X, y = datasets.make_classification(n_samples=12,
n_features=10,
n_informative=2,
random_state=1)
## 2) run with Methods
## ===================================================================
my_svc1 = MySVC(C=1.0)
my_svc2 = MySVC(C=2.0)
two_svc_single = Methods(my_svc1, my_svc2)
two_svc_local = Methods(my_svc1, my_svc2)
two_svc_swf = Methods(my_svc1, my_svc2)
two_svc_single.reducer = MyReducer()
two_svc_local.reducer = MyReducer()
two_svc_swf.reducer = MyReducer()
for leaf in two_svc_single.walk_leaves():
print(leaf.get_key())
for leaf in two_svc_local.walk_leaves():
print(leaf.get_key())
for leaf in two_svc_swf.walk_leaves():
print(leaf.get_key())
# top-down process to call transform
two_svc_single.run(X=X, y=y)
# buttom-up process to compute scores
res_single = two_svc_single.reduce()
### You can get below results:
### ==================================================================
### [{'MySVC(C=1.0)': array([ 1., 1.])}, {'MySVC(C=2.0)': array([ 1., 1.])}]
### 3) Run using local multi-processes
### ==================================================================
from epac.map_reduce.engine import LocalEngine
local_engine = LocalEngine(two_svc_local, num_processes=2)
two_svc_local = local_engine.run(**dict(X=X, y=y))
res_local = two_svc_local.reduce()
### 4) Run using soma-workflow
### ==================================================================
from epac.map_reduce.engine import SomaWorkflowEngine
sfw_engine = SomaWorkflowEngine(tree_root=two_svc_swf, num_processes=2)
two_svc_swf = sfw_engine.run(**dict(X=X, y=y))
res_swf = two_svc_swf.reduce()
if not repr(res_swf) == repr(res_local):
raise ValueError("Cannot dump class definition")
if not repr(res_swf) == repr(res_single):
raise ValueError("Cannot dump class definition")
result_train, result_test = train_test_split(result)
if result_train is result_test: # No CV in the EPAC workflow
accuracy = accuracy_score(result['y/true'], result['y/pred'])
output["acc/y"] = accuracy
else: # there was a CV in the EPAC workflow
accuracy = accuracy_score(result_test['y/true'], result_test['y/pred'])
output["acc/y/test"] = accuracy
output["acc/y/train"] = accuracy_score(result_train['y/true'], result_train['y/pred'])
if accuracy > max_accuracy:
# keep the key in the reduced result
best_result = Result(key=result['key'], **output)
return best_result # reducer return a single result
best_svc = Methods(SVMTransform(C=1.0), SVMTransform(C=2.0))
best_svc.reducer = KeepBest()
cv = CV(best_svc, cv_key="y", cv_type="stratified", n_folds=2,
reducer=None)
cv.run(X=X, y=y) # top-down process to call transform
cv.reduce() # buttom-up process
# User defined reducer receive a ResultSet (list of dict) and returns a ResultSet
# ===============================================================================
class AccuracySummary(Reducer):
"""This reducer summarize the results by accuracies and return a set of results"""
def reduce(self, result_set):
# if you want to a remote execution of your code, import should be done
# within methods
from epac.utils import train_test_split
from epac.map_reduce.results import ResultSet
