def run():
"""Run profiling."""
lc = LayerGenerator().get_sequential('stack', False, False)
cm = CMLog(verbose=False)
cm.monitor()
sleep(5)
t1 = int(np.floor(perf_counter() - cm._t0) * 10)
sleep(0.1)
x, z = make_friedman1(int(5 * 1e6))
sleep(5)
t2 = int(np.floor(perf_counter() - cm._t0) * 10)
sleep(0.1)
lc.fit(x, z)
t3 = int(np.floor(perf_counter() - cm._t0) * 10)
sleep(5)
while not hasattr(cm, 'cpu'):
cm.collect()
sleep(1)
return cm, t1, t2, t3
def test_get_layers():
"""[Utils] testing: test dummy estimator and preprocessing formatting."""
for p in [False, True]:
for cls in ['stack', 'blend', 'subset']:
layer = LayerGenerator().get_layer(cls, p, True)
lc = LayerGenerator().get_layer_container(cls, p, True)
assert isinstance(layer, Layer)
assert isinstance(lc, LayerContainer)
lc_fit,
lc_from_file,
lc_from_csv,
lc_predict,
lc_transform)
PROBA = True
PROCESSING = True
LEN = 12
WIDTH = 2
FOLDS = 3
PARTITIONS = 2
MOD, r = divmod(LEN, FOLDS)
assert r == 0
lg = LayerGenerator()
data = Data('subset', PROBA, PROCESSING, PARTITIONS, FOLDS)
X, y = data.get_data((LEN, WIDTH), MOD)
(F, wf), (P, wp) = data.ground_truth(X, y, subsets=PARTITIONS)
layer = lg.get_layer('subset', PROBA, PROCESSING, PARTITIONS, FOLDS)
lc = lg.get_layer_container('subset', PROBA, PROCESSING, PARTITIONS, FOLDS)
layer.indexer.fit(X)
cache = Cache(X, y, data)
def test_layer_fit():
"""[Parallel | Subset | Prep | Proba] test layer fit."""
import numpy as np
from mlens.ensemble import (SequentialEnsemble, SuperLearner, BlendEnsemble,
Subsemble)
from mlens.utils.dummy import (Data, PREPROCESSING, ESTIMATORS, ECM,
LayerGenerator)
FOLDS = 3
LEN = 24
WIDTH = 2
MOD = 2
data = Data('stack', False, True, FOLDS)
X, y = data.get_data((LEN, WIDTH), MOD)
lc_s = LayerGenerator().get_layer_container('stack', False, True)
lc_b = LayerGenerator().get_layer_container('blend', False, False)
lc_u = LayerGenerator().get_layer_container('subset', False, False)
def test_fit():
"""[Sequential] Test multilayer fitting."""
S = lc_s.fit(X, y, -1)[-1]
B = lc_b.fit(S, y, -1)[-1]
U = lc_u.fit(B, y, -1)[-1]
ens = SequentialEnsemble()
ens.add('stack', ESTIMATORS, PREPROCESSING)
ens.add('blend', ECM)
ens.add('subset', ECM)
import gc
from mlens.utils.dummy import LayerGenerator, Data, Cache
from mlens.utils.dummy import (layer_fit, layer_predict, layer_transform,
lc_fit, lc_from_file, lc_from_csv, lc_predict,
lc_transform, lc_feature_prop)
PROBA = True
PROCESSING = False
LEN = 6
WIDTH = 2
FOLDS = 3
MOD, r = divmod(LEN, FOLDS)
assert r == 0
lg = LayerGenerator()
data = Data('blend', PROBA, PROCESSING)
X, y = data.get_data((LEN, WIDTH), MOD)
(F, wf), (P, wp) = data.ground_truth(X, y)
layer = lg.get_layer('blend', PROBA, PROCESSING)
lc = lg.get_layer_container('blend', PROBA, PROCESSING)
lc_p = lg.get_layer_container('blend',
PROBA,
PROCESSING,
propagate_features=[1])
layer.indexer.fit(X)
cache = Cache(X, y, data)
"""
from mlens.utils.dummy import LayerGenerator, Data, Cache
from mlens.utils.dummy import (layer_fit, layer_predict, layer_transform,
lc_fit, lc_from_file, lc_from_csv, lc_predict,
lc_transform, lc_feature_prop)
PROBA = True
PROCESSING = False
LEN = 12
WIDTH = 2
FOLDS = 3
PARTITIONS = 2
MOD, r = divmod(LEN, FOLDS)
assert r == 0
lg = LayerGenerator()
data = Data('subset', PROBA, PROCESSING, PARTITIONS, FOLDS)
X, y = data.get_data((LEN, WIDTH), MOD)
(F, wf), (P, wp) = data.ground_truth(X, y, subsets=PARTITIONS)
layer = lg.get_layer('subset', PROBA, PROCESSING, PARTITIONS, FOLDS)
lc = lg.get_layer_container('subset', PROBA, PROCESSING, PARTITIONS, FOLDS)
lc_p = lg.get_layer_container('subset',
PROBA,
PROCESSING,
PARTITIONS,
FOLDS,
propagate_features=[1])
layer_transform,
lc_fit,
lc_from_file,
lc_from_csv,
lc_predict,
lc_transform)
PROBA = False
PROCESSING = True
LEN = 6
WIDTH = 2
FOLDS = 3
MOD, r = divmod(LEN, FOLDS)
assert r == 0
lg = LayerGenerator()
data = Data('stack', PROBA, PROCESSING, FOLDS)
X, y = data.get_data((LEN, WIDTH), MOD)
(F, wf), (P, wp) = data.ground_truth(X, y)
layer = lg.get_layer('stack', PROBA, PROCESSING, FOLDS)
lc = lg.get_layer_container('stack', PROBA, PROCESSING, FOLDS)
layer.indexer.fit(X)
cache = Cache(X, y, data)
def test_layer_fit():
"""[Parallel | Stack | Prep] test layer fit."""
"""
from mlens.utils.dummy import LayerGenerator, Data, Cache
from mlens.utils.dummy import (layer_fit, layer_predict, layer_transform,
lc_fit, lc_from_file, lc_from_csv, lc_predict,
lc_transform, lc_feature_prop)
PROBA = True
PROCESSING = False
LEN = 6
WIDTH = 2
FOLDS = 2
MOD, r = divmod(LEN, FOLDS)
assert r == 0
lg = LayerGenerator()
data = Data('stack', PROBA, PROCESSING, FOLDS)
X, y = data.get_data((LEN, WIDTH), MOD)
(F, wf), (P, wp) = data.ground_truth(X, y)
layer = lg.get_layer('stack', PROBA, PROCESSING, FOLDS)
lc = lg.get_layer_container('stack', PROBA, PROCESSING, FOLDS)
lc_p = lg.get_layer_container('stack',
PROBA,
PROCESSING,
FOLDS,
propagate_features=[1])
layer.indexer.fit(X)
import gc
from mlens.utils.dummy import LayerGenerator, Data, Cache
from mlens.utils.dummy import (layer_fit, layer_predict, layer_transform,
lc_fit, lc_from_file, lc_from_csv, lc_predict,
lc_transform)
PROBA = True
PROCESSING = False
LEN = 6
WIDTH = 2
FOLDS = 3
MOD, r = divmod(LEN, FOLDS)
assert r == 0
lg = LayerGenerator()
data = Data('blend', PROBA, PROCESSING)
X, y = data.get_data((LEN, WIDTH), MOD)
(F, wf), (P, wp) = data.ground_truth(X, y)
layer = lg.get_layer('blend', PROBA, PROCESSING)
lc = lg.get_layer_container('blend', PROBA, PROCESSING)
layer.indexer.fit(X)
cache = Cache(X, y, data)
def test_layer_fit():
"""[Parallel | Blend | No Prep | Proba] test layer fit."""
author: Sebastian Flennerhag
:copyirgh: 2017
:licence: MIT
"""
import numpy as np
from mlens.externals.sklearn.base import clone
from mlens.utils.checks import check_ensemble_build, assert_valid_estimator, \
check_is_fitted, assert_correct_format, check_initialized
from mlens.utils.dummy import LayerGenerator, OLS, Scale
from mlens.utils.exceptions import LayerSpecificationError, \
LayerSpecificationWarning, NotFittedError, ParallelProcessingError, \
ParallelProcessingWarning
lg = LayerGenerator()
LAYER = lg.get_layer('stack', False, True)
LAYER_CONTAINER = lg.get_layer_container('stack', False, True)
class Tmp(object):
"""Temporary class for mimicking ParallelProcessing status."""
def __init__(self, lyr, __initialized__, __fitted__):
self.__initialized__ = __initialized__
self.__fitted__ = __fitted__
self.layers = lyr
class Lyr(object):
"""Temporary layer class for storing raise on exception."""
Test base functionality.
"""
import numpy as np
from mlens.externals.sklearn.base import clone
from mlens.utils.dummy import Data, LayerGenerator
LEN = 6
WIDTH = 2
MOD = 2
data = Data('stack', False, True, n_splits=5)
X, y = data.get_data((LEN, WIDTH), MOD)
lc = LayerGenerator().get_layer_container('stack', False, False)
layer = LayerGenerator().get_layer('stack', False, False)
def test_clone():
"""[Ensemble | LayerContainer] Test cloning."""
cloned = clone(lc)
params = lc.get_params(deep=False)
params_cloned = cloned.get_params(deep=False)
for par, param in params.items():
if par == 'layers':
assert param is not params_cloned[par]
else:
assert param is params_cloned[par]