def elm_regressor(X, Y, X_CV, X_test, estimator, param, path, **kwargs):
""" Implementation of SIR kNN estimator specific for the NSIM model """
assert 'n_hidden' in param, "SFFNNIRKNN: 'n_hidden' not in param"
sys.path.insert(0, path)
options = estimator.get('options')
from elm import ELMRegressor
"""
ELM-Parameters (copied from file)
----------
`n_hidden` : int, optional (default=20)
Number of units to generate in the SimpleRandomLayer
`alpha` : float, optional (default=0.5)
Mixing coefficient for distance and dot product input activations:
activation = alpha*mlp_activation + (1-alpha)*rbf_width*rbf_activation
`rbf_width` : float, optional (default=1.0)
multiplier on rbf_activation
`activation_func` : {callable, string} optional (default='tanh')
Function used to transform input activation
It must be one of 'tanh', 'sine', 'tribas', 'inv_tribase', 'sigmoid',
'hardlim', 'softlim', 'gaussian', 'multiquadric', 'inv_multiquadric' or
a callable. If none is given, 'tanh' will be used. If a callable
is given, it will be used to compute the hidden unit activations.
`activation_args` : dictionary, optional (default=None)
Supplies keyword arguments for a callable activation_func
`user_components`: dictionary, optional (default=None)
dictionary containing values for components that woud otherwise be
randomly generated. Valid key/value pairs are as follows:
'radii' : array-like of shape [n_hidden]
'centers': array-like of shape [n_hidden, n_features]
'biases' : array-like of shape [n_hidden]
'weights': array-like of shape [n_hidden, n_features]
`regressor` : regressor instance, optional (default=None)
If provided, this object is used to perform the regression from hidden
unit activations to the outputs and subsequent predictions. If not
present, an ordinary linear least squares fit is performed
`random_state` : int, RandomState instance or None (default=None)
Control the pseudo random number generator used to generate the
hidden unit weights at fit time.
"""
elm_reg = ELMRegressor(n_hidden = param['n_hidden'], activation_func = options.get('activation_func', 'tanh'))
elm_reg = elm_reg.fit(X, Y)
Y_CV = elm_reg.predict(X_CV)
Y_test = elm_reg.predict(X_test)
return Y_CV, Y_test
def objective(params):
metricR2 = list()
for i in range(3):
#initialize
model = ELMRegressor(n_hidden=int(params['neurons']))
#model train
model.fit(X_train[i], y_train[i])
#Model validation
pred = model.predict(X_val[i])
metricR2.append(r2_score(y_val[i], pred))
print('\nR2 medio: ' + str(np.mean(np.array(metricR2))))
return -1.0 * np.mean(np.array(metricR2))
def load_estimator(estimator_kwargs, path_to_source):
"""
Loads a specific estimator. Path to source should contain source code for
the nsim algorithm ("github.com/soply/nsim_algorithm"), simple estimators ("github.com/soply/simple_estimation")
and ELM "github.com/dclambert/Python-ELM". Other estimators
can be added flexible.
Parameters
------------
estimator_kwargs : dict
Contains estimator name, and additional arguments if necessary.
path_to_source : string
Folder where source code for other estimators is hosted.
"""
name = estimator_kwargs['estimator']
copy_dict = copy.deepcopy(estimator_kwargs)
del copy_dict['estimator']
if name == 'SIRKnn':
sys.path.insert(0, path_to_source + '/simple_estimation/')
from simple_estimation.estimators.SIRKnn import SIRKnn
estim = SIRKnn(**copy_dict)
elif name == 'SAVEKnn':
sys.path.insert(0, path_to_source + '/simple_estimation/')
from simple_estimation.estimators.SAVEKnn import SAVEKnn
estim = SAVEKnn(**copy_dict)
elif name == 'PHDKnn':
sys.path.insert(0, path_to_source + '/simple_estimation/')
from simple_estimation.estimators.PHDKnn import PHDKnn
estim = PHDKnn(**copy_dict)
elif name == 'nsim':
sys.path.insert(0, path_to_source + '/nsim_algorithm/')
from nsim_algorithm.estimator import NSIM_Estimator
estim = NSIM_Estimator(**copy_dict)
elif name == 'knn':
from sklearn.neighnors import KNeighborsRegressor
estim = KNeighborsRegressor()
elif name == 'linreg':
from sklearn.linear_model import LinearRegression
estim = LinearRegression()
elif name == 'isotron':
sys.path.insert(0, path_to_source + '/simple_estimation/')
from simple_estimation.estimators.Isotron import IsotronCV
estim = IsotronCV(**copy_dict)
elif name == 'slisotron':
sys.path.insert(0, path_to_source + '/simple_estimation/')
from simple_estimation.estimators.Slisotron import SlisotronCV
estim = SlisotronCV(**copy_dict) # Super slow implementation
elif name == 'elm':
sys.path.insert(0, path_to_source + '/Python-ELM/')
from elm import ELMRegressor
estim = ELMRegressor(**copy_dict)
elif name == 'ffnn':
from simple_estimation.estimators.FeedForwardNetwork import FeedForwardNetwork
estim = FeedForwardNetwork(**copy_dict)
else:
raise NotImplementedError('Load estimator: Estimator does not exist.')
return estim
def make_regressors():
nh = 10
names = ["ELM(10,tanh,LR)"]
srhl_tanh = SimpleRandomHiddenLayer(n_hidden=nh,
activation_func='tanh',
random_state=0)
log_reg = LogisticRegression()
classifiers = [ELMRegressor(srhl_tanh, regressor=log_reg)]
return names, classifiers
# <codecell>
for af in RandomLayer.activation_func_names():
print af
elmc = ELMClassifier(activation_func=af, random_state=0)
tr, ts = res_dist(dgx, dgy, elmc, n_runs=100, random_state=0)
# <codecell>
elmc = ELMClassifier(n_hidden=500, activation_func='multiquadric')
tr, ts = res_dist(dgx, dgy, elmc, n_runs=100, random_state=0)
scatter(tr, ts, alpha=0.1, marker='D', c='r')
# <codecell>
elmr = ELMRegressor(random_state=0, activation_func='gaussian', alpha=0.0)
elmr.fit(xtoy_train, ytoy_train)
print elmr.score(xtoy_train, ytoy_train), elmr.score(xtoy_test, ytoy_test)
plot(xtoy, ytoy, xtoy, elmr.predict(xtoy))
# <codecell>
from sklearn import pipeline
from sklearn.linear_model import LinearRegression
elmr = pipeline.Pipeline([('rhl',
RandomLayer(random_state=0,
activation_func='multiquadric')),
('lr', LinearRegression(fit_intercept=False))])
elmr.fit(xtoy_train, ytoy_train)
print elmr.score(xtoy_train, ytoy_train), elmr.score(xtoy_test, ytoy_test)
plot(xtoy, ytoy, xtoy, elmr.predict(xtoy))
# <codecell>
for af in RandomLayer.activation_func_names():
print af
elmc = ELMClassifier(activation_func=af, random_state=0)
tr,ts = res_dist(dgx, dgy, elmc, n_runs=100, random_state=0)
# <codecell>
elmc = ELMClassifier(n_hidden=500, activation_func='multiquadric')
tr,ts = res_dist(dgx, dgy, elmc, n_runs=100, random_state=0)
scatter(tr, ts, alpha=0.1, marker='D', c='r')
# <codecell>
elmr = ELMRegressor(random_state=0, activation_func='gaussian', alpha=0.0)
elmr.fit(xtoy_train, ytoy_train)
print elmr.score(xtoy_train, ytoy_train), elmr.score(xtoy_test, ytoy_test)
plot(xtoy, ytoy, xtoy, elmr.predict(xtoy))
# <codecell>
from sklearn import pipeline
from sklearn.linear_model import LinearRegression
elmr = pipeline.Pipeline([('rhl', RandomLayer(random_state=0, activation_func='multiquadric')),
('lr', LinearRegression(fit_intercept=False))])
elmr.fit(xtoy_train, ytoy_train)
print elmr.score(xtoy_train, ytoy_train), elmr.score(xtoy_test, ytoy_test)
plot(xtoy, ytoy, xtoy, elmr.predict(xtoy))
# <codecell>
# <codecell>
print '-2'
for af in RandomLayer.activation_func_names():
print af
elmc = ELMClassifier(activation_func=af, random_state=0)
tr,ts = res_dist(dgx, dgy, elmc, n_runs=100, random_state=0)
# <codecell>
print "-3 multiquadric"
elmc = ELMClassifier(n_hidden=500, activation_func='multiquadric')
tr,ts = res_dist(dgx, dgy, elmc, n_runs=100, random_state=0)
#scatter(tr, ts, alpha=0.1, marker='D', c='r')
# <codecell>
print "-4 gaussian regressor"
elmr = ELMRegressor(random_state=0, activation_func='gaussian', alpha=0.0)
elmr.fit(xtoy_train, ytoy_train)
print elmr.score(xtoy_train, ytoy_train), elmr.score(xtoy_test, ytoy_test)
#plot(xtoy, ytoy, xtoy, elmr.predict(xtoy))
# <codecell>
print '-5 pipeline.multiquadric'
from sklearn import pipeline
from sklearn.linear_model import LinearRegression
elmr = pipeline.Pipeline([('rhl', RandomLayer(random_state=0, activation_func='multiquadric')),
('lr', LinearRegression(fit_intercept=False))])
elmr.fit(xtoy_train, ytoy_train)
print elmr.score(xtoy_train, ytoy_train), elmr.score(xtoy_test, ytoy_test)
#plot(xtoy, ytoy, xtoy, elmr.predict(xtoy))
# <codecell>
def make_regressor(func, nh):
srhl = SimpleRandomHiddenLayer(n_hidden=nh,
activation_func=func,
random_state=0)
return ELMRegressor(srhl, regressor=LogisticRegression())
y_test = y_test
X_train = X_train
X_test = X_test
else:
y_train = y_train[h:]
y_test = y_test[h:]
X_train = X_train[:-h]
X_test = X_test[:-h]
#Best hyperparam config
best = pruebas[k].best_trial
params = best['misc']['vals']
model = ELMRegressor(n_hidden=int(np.array(params['neurons'])))
model.fit(X_train, y_train)
#Model metrics
pred = model.predict(X_train)
train_loss.append(r2_score(y_train, pred))
pred = model.predict(X_test)
test_loss.append(r2_score(y_test, pred))
val_loss.append(best['result']['loss']*-1)
#Store metrics
metrics = {}
metrics['train'] = train_loss
metrics['val'] = val_loss
metrics['test'] = test_loss
test_features = sc.transform(test_features)
### Using Principal Component Analysis to reduce the dimensions of the data
# When not in use/testing, just change this section to remarks
from sklearn.decomposition import PCA
pca = PCA(n_components=6)
# IMPORTANT to only use fit.transform once here, then just transform the test set
train_features = pca.fit_transform(train_features)
test_features = pca.transform(test_features)
### Import the Extreme Learning Machine Regressor library
from elm import ELMRegressor
# Create a Extreme Learning Machine Regressor object and tune it accordingly
# In this case, I decided to create more hidden layers as it was consistently performing much better than a single layer without too much of a sacrifice to time performance
elm = ELMRegressor(n_hidden=30)
# Then, fit the Regressor with the training data to create a model
elm.fit(train_features, train_labels)
# The next section is different from the other algorithms as ELM's hidden layer constantly changes their parameters
# Hence, the following for loop will allow us to iterate through the ELM model multiple times to find the mean
total = 0
rmse_val = [] # To store rmse values for different x
for x in range(20):
x = x + 1
model = ELMRegressor(n_hidden=30)
model.fit(train_features, train_labels) # Fit the model
pred = model.predict(test_features) # Make prediction on test set
error = sqrt(mean_squared_error(test_labels, pred)) # Calculate rmse
total = total + error
ftr6 = float(trainData['highest'][i+2] - float(trainData['lowest'][i+2]))
tmpX = np.array([ftr1, ftr2, ftr3, ftr4, ftr5, ftr6])
X[i] = tmpX
#Make the output from training data
y = np.zeros(len(trainData) - 3)
for i in range(0, len(trainData) - 3):
if trainData['changerange'][i+3] >= 0:
y[i] = 1
else:
y[i] = -1
srhl_tanh = SimpleRandomHiddenLayer(n_hidden=10, activation_func='tanh', random_state=0)
srhl_rbf = RBFRandomHiddenLayer(n_hidden=200*2, gamma=0.1, random_state=0)
#create ELM instance
reg = ELMRegressor(srhl_tanh)
cla = ELMClassifier(srhl_tanh)
#fit the data
reg = reg.fit(X, y)
cla = cla.fit(X, y)
#Read testing data from testing.csv
testData = pd.read_csv('testing.csv')
pdt = np.zeros(len(testData) - 3)
pdt2 = np.zeros(len(testData) - 3)
for i in range(len(testData) - 3):
ftr1 = float(testData['close'][i]) - float(testData['open'][i])
ftr2 = float(testData['close'][i+1]) - float(testData['open'][i+1])
ftr3 = float(testData['close'][i+2]) - float(testData['open'][i+2])
ftr4 = float(testData['highest'][i] - float(testData['lowest'][i]))
tr, ts = res_dist(irx, iry, elmc, n_runs=100, random_state=0)
elmc = ELMClassifier(SimpleRandomHiddenLayer(activation_func='tribas'))
tr, ts = res_dist(irx, iry, elmc, n_runs=100, random_state=0)
elmc = ELMClassifier(SimpleRandomHiddenLayer(activation_func='sigmoid'))
tr, ts = res_dist(irx, iry, elmc, n_runs=100, random_state=0)
elmc = ELMClassifier(SimpleRandomHiddenLayer(activation_func='hardlim'))
tr, ts = res_dist(irx, iry, elmc, n_runs=100, random_state=0)
# <codecell>
hardlim = (lambda a: np.array(a > 0.0, dtype=float))
tribas = (lambda a: np.clip(1.0 - np.fabs(a), 0.0, 1.0))
elmr = ELMRegressor(
SimpleRandomHiddenLayer(random_state=0, activation_func=tribas))
elmr.fit(xtoy_train, ytoy_train)
print elmr.score(xtoy_train, ytoy_train), elmr.score(xtoy_test, ytoy_test)
plot(xtoy, ytoy, xtoy, elmr.predict(xtoy))
# <codecell>
rhl = SimpleRandomHiddenLayer(n_hidden=200)
elmr = ELMRegressor(hidden_layer=rhl)
tr, ts = res_dist(mrx, mry, elmr, n_runs=20, random_state=0)
# <codecell>
rhl = RBFRandomHiddenLayer(n_hidden=15, gamma=0.25)
elmr = ELMRegressor(hidden_layer=rhl)
elmr.fit(xtoy_train, ytoy_train)
elmc = ELMClassifier(SimpleRandomLayer(activation_func='tanh'))
tr, ts = res_dist(irx, iry, elmc, n_runs=100, random_state=0)
elmc = ELMClassifier(SimpleRandomLayer(activation_func='tribas'))
tr, ts = res_dist(irx, iry, elmc, n_runs=100, random_state=0)
elmc = ELMClassifier(SimpleRandomLayer(activation_func='sigmoid'))
tr, ts = res_dist(irx, iry, elmc, n_runs=100, random_state=0)
elmc = ELMClassifier(SimpleRandomLayer(activation_func='hardlim'))
tr, ts = res_dist(irx, iry, elmc, n_runs=100, random_state=0)
# <codecell>
elmr = ELMRegressor(SimpleRandomLayer(random_state=0,
activation_func='tribas'))
elmr.fit(xtoy_train, ytoy_train)
print elmr.score(xtoy_train, ytoy_train), elmr.score(xtoy_test, ytoy_test)
plot(xtoy, ytoy, xtoy, elmr.predict(xtoy))
# <codecell>
rhl = SimpleRandomLayer(n_hidden=200)
elmr = ELMRegressor(hidden_layer=rhl)
tr, ts = res_dist(mrx, mry, elmr, n_runs=20, random_state=0)
# <codecell>
rhl = RBFRandomLayer(n_hidden=15, gamma=0.25)
elmr = ELMRegressor(hidden_layer=rhl)
elmr.fit(xtoy_train, ytoy_train)