def test_zero_resid():
# smoke and regression tests
X = np.array([[1, 0], [0, 1], [0, 2.1], [0, 3.1]], dtype=np.float64)
y = np.array([0, 1, 2, 3], dtype=np.float64)
res = QuantReg(y, X).fit(0.5, bandwidth='chamberlain') #'bofinger')
res.summary()
assert_allclose(res.params, np.array([0.0, 0.96774163]), rtol=1e-4, atol=1e-20)
assert_allclose(res.bse, np.array([0.0447576, 0.01154867]), rtol=1e-4, atol=1e-20)
assert_allclose(res.resid, np.array([0.0, 3.22583680e-02, -3.22574272e-02,
9.40732912e-07]), rtol=1e-4, atol=1e-20)
X = np.array([[1, 0], [0.1, 1], [0, 2.1], [0, 3.1]], dtype=np.float64)
y = np.array([0, 1, 2, 3], dtype=np.float64)
res = QuantReg(y, X).fit(0.5, bandwidth='chamberlain')
res.summary()
assert_allclose(res.params, np.array([9.99982796e-08, 9.67741630e-01]),
rtol=1e-4, atol=1e-20)
assert_allclose(res.bse, np.array([0.04455029, 0.01155251]), rtol=1e-4, atol=1e-20)
assert_allclose(res.resid, np.array([-9.99982796e-08, 3.22583598e-02,
-3.22574234e-02, 9.46361860e-07]), rtol=1e-4, atol=1e-20)
def train(self, data, **kwargs):
if self.indexer is not None and isinstance(data, pd.DataFrame):
data = self.indexer.get_data(data)
lagdata, ndata = lagmat(data, maxlag=self.order, trim="both", original='sep')
mqt = QuantReg(ndata, lagdata).fit(0.5)
if self.alpha is not None:
uqt = QuantReg(ndata, lagdata).fit(1 - self.alpha)
lqt = QuantReg(ndata, lagdata).fit(self.alpha)
self.mean_qt = [k for k in mqt.params]
if self.alpha is not None:
self.upper_qt = [k for k in uqt.params]
self.lower_qt = [k for k in lqt.params]
if self.dist:
self.dist_qt = []
for alpha in np.arange(0.05,0.5,0.05):
lqt = QuantReg(ndata, lagdata).fit(alpha)
uqt = QuantReg(ndata, lagdata).fit(1 - alpha)
lo_qt = [k for k in lqt.params]
up_qt = [k for k in uqt.params]
self.dist_qt.append([lo_qt, up_qt])
self.shortname = "QAR(" + str(self.order) + ") - " + str(self.alpha)
def forecaster(returns, ff, loss='MSE'):
output = []
dates = sorted(list(ff.index))
dataset = ff.merge(returns, left_index=True, right_index=True)
columnNames = ['MktPremium', 'HML', 'Mom']
name = returns.columns.tolist()[0]
i = dates.index('200201')
for j in range(i, (len(dates))):
trainData = dataset.loc['199801':dates[j], :]
trainX = trainData[columnNames]
trainY = trainData[[name]]
model = LinearRegression()
if loss == 'MSE':
model = LinearRegression()
if loss == 'Ridge':
model = Ridge()
if loss == 'Lasso':
model = Lasso()
if loss == 'Hub':
model = HuberRegressor()
if loss == 'ElasticNet':
model = ElasticNet()
model.fit(trainX, trainY)
testData = pd.DataFrame(dataset.loc[dates[j], :]).T
testX = testData[columnNames]
prediction = model.predict(testX)
if loss == 'LAD':
model = QuantReg(endog=trainY, exog=trainX)
res = model.fit(q=0.5)
prediction = model.predict(res.params, exog=testX)
if loss == '1Q':
model = QuantReg(endog=trainY, exog=trainX)
res = model.fit(q=0.25)
prediction = model.predict(res.params, exog=testX)
if loss == '3Q':
model = QuantReg(endog=trainY, exog=trainX)
res = model.fit(q=0.75)
prediction = model.predict(res.params, exog=testX)
if loss in ['Lasso', 'Hub', 'ElasticNet', 'LAD', '1Q', '3Q']:
output.append(prediction[0])
else:
output.append(prediction[0][0])
return (name, output)
def fit_predict(self, train, val=None, test=None, **kwa):
model = QuantReg(train[1], train[0]).fit(q=0.5, max_iter=10000)
if val is None:
return model.predict(test[0])
else:
return model.predict(val[0]), model.predict(test[0])
def calcuTD(x, O, Y, SeqDepth, Grid, Tau):
TauGroup, D = Grid[x]
D = int(D)
try:
polyX, centre, scale, alpha, beta = poly.poly(O, D)
except Exception:
polyX = None
if polyX is not None:
colVars = ['var_' + str(j) for j in range(D)]
polydata = pd.concat([pd.DataFrame({'Y':Y}), pd.DataFrame(polyX, columns=colVars)], axis=1)
try:
rqfit = smf.quantreg('Y~' + '+'.join(colVars), polydata).fit(q=TauGroup)
revX = poly.predict_poly(polyX, centre, scale, alpha, beta, SeqDepth)
revX = pd.DataFrame(revX, columns=colVars)
pdvalsrq = rqfit.predict(revX)
if min(pdvalsrq) > 0:
S = QuantReg(pdvalsrq.values, tools.add_constant(SeqDepth)).fit(q=Tau).params[1]
else:
S = -50
except Exception:
S = -50
else:
S = -50
return S
def test_collinear_matrix():
X = np.array([[1, 0, .5], [1, 0, .8],
[1, 0, 1.5], [1, 0, .25]], dtype=np.float64)
y = np.array([0, 1, 2, 3], dtype=np.float64)
res_collinear = QuantReg(y, X).fit(0.5)
assert len(res_collinear.params) == X.shape[1]
def fit(self):
optimizedHyperParameters = self.optimizedHyperParameters
fixedHyperParameters = self.fixedHyperParameters
kernelName = optimizedHyperParameters["kernelName"]
trainX, trainY, validationX, validationY = self.dataset.getDataset(2)
self.model = QuantReg(trainY, trainX)
def test_fitted_residuals():
data = sm.datasets.engel.load_pandas().data
y, X = dmatrices('foodexp ~ income', data, return_type='dataframe')
res = QuantReg(y, X).fit(q=.1)
# Note: maxabs relative error with fitted is 1.789e-09
assert_almost_equal(np.array(res.fittedvalues), Rquantreg.fittedvalues, 5)
assert_almost_equal(np.array(res.predict()), Rquantreg.fittedvalues, 5)
assert_almost_equal(np.array(res.resid), Rquantreg.residuals, 5)
def setup_class(cls):
data = sm.datasets.engel.load_pandas().data
y, X = dmatrices('foodexp ~ income', data, return_type='dataframe')
cls.res1 = QuantReg(y, X).fit(q=.75,
vcov='iid',
kernel='epa',
bandwidth='hsheather')
cls.res2 = epanechnikov_hsheather_q75
def test_use_t_summary():
X = np.array([[1, 0], [0, 1], [0, 2.1], [0, 3.1]], dtype=np.float64)
y = np.array([0, 1, 2, 3], dtype=np.float64)
res = QuantReg(y, X).fit(0.5, bandwidth='chamberlain', use_t=True)
summ = res.summary()
assert 'P>|t|' in str(summ)
assert 'P>|z|' not in str(summ)
def test_alpha_summary():
X = np.array([[1, 0], [0, 1], [0, 2.1], [0, 3.1]], dtype=np.float64)
y = np.array([0, 1, 2, 3], dtype=np.float64)
res = QuantReg(y, X).fit(0.5, bandwidth='chamberlain', use_t=True)
summ_20 = res.summary(alpha=.2)
assert '[0.025 0.975]' not in str(summ_20)
assert '[0.1 0.9]' in str(summ_20)
def test_nontrivial_singular_matrix():
x_one = np.random.random(1000)
x_two = np.random.random(1000)*10
x_three = np.random.random(1000)
intercept = np.ones(1000)
y = np.random.random(1000)*5
X = np.column_stack((intercept, x_one, x_two, x_three, x_one))
assert np.linalg.matrix_rank(X) < X.shape[1]
res_singular = QuantReg(y, X).fit(0.5)
assert len(res_singular.params) == X.shape[1]
assert np.linalg.matrix_rank(res_singular.cov_params()) == X.shape[1] - 1
# prediction is correct even with singular exog
res_ns = QuantReg(y, X[:, :-1]).fit(0.5)
assert_allclose(res_singular.fittedvalues, res_ns.fittedvalues, rtol=0.01)
def train_LAD(x, y):
"""
训练LAD线性回归模型，并返回模型预测值
"""
X = sm.add_constant(x)
model = QuantReg(y, X)
model = model.fit(q=0.5)
re = model.predict(X)
return re
def get_quantreg(_y, what="slope", q=0.5):
if not np.ma.is_masked(_y):
_x = sm.add_constant(np.arange(len(_y)))
res=QuantReg(_y, _x).fit(q=0.5)
if what=="slope":
return res.params[1]
elif what=="pval":
return res.pvalues[1]
elif what=="intercept":
return res.params[0]
else:
return np.nan
def calcuSlope(i, LogData, SeqDepth, Genes, Tau):
if i % round(len(Genes) / 10) == 0:
print(i / round(len(Genes) / 10) * 10, '%')
X = Genes[i]
with warnings.catch_warnings():
warnings.filterwarnings("ignore")
validIdx = np.logical_not(np.isnan(
LogData.loc[X].values)) & (SeqDepth.values > 0)
mod = QuantReg(LogData.loc[X].values[validIdx],
tools.add_constant(np.log(SeqDepth.values[validIdx])))
# mod = smf.quantreg('response ~ variable',
# pd.DataFrame({'response': LogData.loc[X], 'variable': np.log(SeqDepth)}))
slope = mod.fit(q=Tau).params[1]
return slope
def plot_ar1_coef(series, granularity='30Min'):
"""
plots ar1 coeff
input: pandas series of prices
output: plots graph
"""
# Groupby granularity
series = series.groupby(
pd.TimeGrouper(freq=granularity)).last().fillna(method='ffill')
# returns and volatility
ret = series.pct_change()
vol = ret.rolling(vol_window).std()
dff = pd.concat([ret, vol], axis=1, join='inner')
dff.columns = ['ret', 'vol']
# add constant
dff = sm.add_constant(dff)
# y-variable
dff = dff.assign(y=dff.ret.shift(-1))
# dropna
dff.replace([np.inf, -np.inf], np.nan)
dff.dropna(inplace=True)
from statsmodels.regression.quantile_regression import QuantReg
mod = QuantReg(endog=dff.y, exog=dff.loc[:, ['const', 'ret']])
#
quantiles = np.arange(.01, .99, .01)
def fit_model(q):
res = mod.fit(q=q)
return [q, res.params['ret']] + \
res.conf_int().loc['ret'].tolist()
models = [fit_model(x) for x in quantiles]
models = pd.DataFrame(models, columns=['q', 'b', 'lb', 'ub'])
# plot the quantile regression params
import matplotlib.pyplot as plt
plt.title('AR1 Coefficient with {} granularity'.format(granularity))
plt.plot(models.q, models.b, color='b', label='1st Order AutoRegression')
plt.plot(models.q, models.ub, linestyle='dotted', color='b')
plt.plot(models.q, models.lb, linestyle='dotted', color='b')
#plt.plot(models.q, models.high_vol, color='red', label='High Volatility')
plt.axhline(y=0, color='black', linestyle='--')
plt.legend()
plt.show()
"""Huber""" reg2 = HuberRegressor(epsilon = 1) model2 = reg2.fit(x, y) y_pred2 = model2.predict(x_test) """L1""" dfx = pd.DataFrame(x, columns = ['x']) dfy = pd.DataFrame(y, columns = ['y']) exog = sm.add_constant(dfx['x']) endog = dfy['y'] dft = pd.DataFrame(x_test, columns = ['test']) qrmodel = QuantReg(endog, exog) result = qrmodel.fit(q=0.5) ypred_qr = np.dot(dft, result.params[1]) + result.params[0] #results.predict(dft) """Student-t""" tmodel = TLinearModel(endog, exog) results = tmodel.fit(df=0.6) ypred_t = np.dot(dft, results.params[1]) + results.params[0] #results.predict(dft) """Plot""" plt.xlim(xmin, xmax) plt.ylim(ymin, ymax)
def forecaster(returns, ff, loss='MSE'):
output = []
factorLoadings = []
varianceOfErrors = []
df = ff.merge(returns, left_index=True, right_index=True)
name = returns.columns.tolist()[0]
df[name] = df[name] - df['RF']
regressors = ['Mkt.Rf', 'HML', 'Mom', 'RMW', 'CMA']
for j in range(120, len(df.index.tolist())):
trainData = df.iloc[(j - 120):j, :]
trainX = trainData[regressors]
trainY = trainData[[name]]
model = LinearRegression()
if loss == 'MSE':
model = LinearRegression()
if loss == 'Ridge':
model = Ridge()
if loss == 'Lasso':
model = Lasso()
if loss == 'Hub':
model = HuberRegressor()
if True == trainY.isnull().values.any():
output.append(np.nan)
factorLoadings.append(np.zeros((1, 5)))
varianceOfErrors.append(np.nan)
continue
model.fit(trainX, trainY)
res = ''
if loss == 'LAD':
model = QuantReg(endog=trainY, exog=trainX)
res = model.fit(q=0.5)
if loss == '1Q':
model = QuantReg(endog=trainY, exog=trainX)
res = model.fit(q=0.25)
if loss == '3Q':
model = QuantReg(endog=trainY, exog=trainX)
res = model.fit(q=0.75)
if loss in ['LAD', '1Q', '3Q']:
factorLoadings.append(np.array(res.params))
else:
factorLoadings.append(model.coef_)
if loss not in ['Lasso', 'Hub', 'LAD', '1Q', '3Q']:
varianceOfErrors.append(
np.var(trainY - model.predict(trainX)).tolist()[0])
if loss in ['Lasso', 'Hub']:
varianceOfErrors.append(
np.var(np.array(trainY) - model.predict(trainX)))
if loss in ['LAD', '1Q', '3Q']:
varianceOfErrors.append(
np.var(
model.predict(res.params, exog=trainX) - np.array(trainY)))
testData = pd.DataFrame(df.iloc[j, :]).T
testX = testData[regressors]
if loss in ['LAD', '1Q', '3Q']:
prediction = model.predict(res.params, exog=testX)
else:
prediction = model.predict(testX)
if loss in ['Lasso', 'Hub', 'LAD', '1Q', '3Q']:
output.append(prediction[0])
else:
output.append(prediction[0][0])
return (name, output, factorLoadings, varianceOfErrors)
import time
xmin = [-1., -1.]
xmax = [2., 3.]
mu, invSig = ConstructRBF(xmin, xmax, [3, 3])
t0 = time.time()
data_x, data_f = GenerateSample(xmin,
xmax,
N_sample=300,
Func=Func,
NoiseFunc=NoiseFunc)
print 'GenerateSample/Computation time:', time.time() - t0
t0 = time.time()
Theta = np.array([FeaturesNG(x, mu, invSig) for x in data_x])
quant_reg = QuantReg(data_f, Theta)
fit1 = quant_reg.fit(q=0.1)
fit5 = quant_reg.fit(q=0.5)
fit9 = quant_reg.fit(q=0.95)
w1 = fit1.params
w5 = fit5.params
w9 = fit9.params
print fit9.summary()
print 'Parameters w1:', w1
print 'Parameters w5:', w5
print 'Parameters w9:', w9
print 'QuantReg/Computation time:', time.time() - t0
fp = file('/tmp/data.dat', 'w')
for x, f in zip(data_x, data_f):
fp.write('%f %f %f\n' % (x[0], x[1], f))
#rankg2 = df['V3.%s' % first_min_second[1]].argsort().values
#response = np.array((rankg1 - rankg2), dtype='d')# ** 3
#response = MinMaxScaler().fit_transform(response[:, None])[:, 0]
response = pd.DataFrame(response,
columns=['Rank%s-Rank%s' % first_min_second])
explanatory = df[features].copy()
#explanatory = pd.DataFrame(MinMaxScaler().fit_transform(explanatory.copy().values),
# columns=explanatory.columns)
#explanatory['intercept'] = np.ones(len(explanatory), dtype='d')
explanatory['is_catole'] = np.array(df['bairro'] == 'catole', dtype='d')
explanatory['is_centro'] = np.array(df['bairro'] == 'centro', dtype='d')
explanatory['is_liberdade'] = np.array(df['bairro'] == 'liberdade',
dtype='d')
model = QuantReg(response, explanatory)
max_left = 0.0
max_left_q = 0
max_right = 0.0
max_right_q = 0
rsqs = []
qs = []
util = []
values = {}
for name in explanatory.columns:
values[name] = np.zeros(10)
i = 0
for q in np.linspace(0.05, 0.95, 5):
values[name][i] = 0
def train_predict_stacking_linear_regression(df_learning, df_prod,
l_tuple_strategy_normalised):
for quantile in constants.LIST_QUANTILE:
to_keep = []
for strategy, normalize_by in l_tuple_strategy_normalised:
str_normalized = '_normed_by_' + normalize_by if normalize_by is not None else ''
to_keep.append('{}{}_quantile_{:.3f}'.format(
strategy, str_normalized, quantile))
# Remove NA columns
to_keep = df_learning[to_keep].notnull().all()
to_keep = to_keep[to_keep].index.tolist()
# We need to remove constants columns from the sampled data
df_learning_weighted = df_learning.sample(10000,
weights='weight',
replace=True,
random_state=1)
# Remove constants columns
cols_constants = df_learning_weighted[to_keep].std() == 0
cols_constants = cols_constants[cols_constants].index.tolist()
for col in cols_constants:
to_keep.remove(col)
# # Remove correlated features
# # Create correlation matrix
# corr_matrix = df_learning[to_keep].corr().abs().fillna(1)
# # Select upper triangle of correlation matrix
# upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(np.bool))
# # Find index of feature columns with correlation greater than 0.95
# to_drop = [column for column in upper.columns if any(upper[column] > 0.95)]
# to_keep.remove(to_drop)
# Drop duplicates columns
def getDuplicateColumns(df):
'''
Get a list of duplicate columns.
It will iterate over all the columns in dataframe and find the columns whose contents are duplicate.
:param df: Dataframe object
:return: List of columns whose contents are duplicates.
'''
duplicateColumnNames = set()
# Iterate over all the columns in dataframe
for x in range(df.shape[1]):
# Select column at xth index.
col = df.iloc[:, x]
# Iterate over all the columns in DataFrame from (x+1)th index till end
for y in range(x + 1, df.shape[1]):
# Select column at yth index.
otherCol = df.iloc[:, y]
# Check if two columns at x 7 y index are equal
if col.equals(otherCol):
duplicateColumnNames.add(df.columns.values[y])
return list(duplicateColumnNames)
cols_duplicate = getDuplicateColumns(df_learning_weighted[to_keep])
for cols in cols_duplicate:
to_keep.remove(cols)
# to_keep = df_learning_weighted[to_keep].T.drop_duplicates().T.columns # Not efficient but ok
X_learning_weighted = df_learning_weighted[to_keep].fillna(0)
X_learning = df_learning[to_keep].fillna(0)
X_prod = df_prod[to_keep].fillna(0)
y_learning_weighted = df_learning_weighted['sales']
# weight_learning = df_learning['weight']
if X_learning_weighted.nunique().max() != 1:
linear_model = QuantReg(y_learning_weighted, X_learning_weighted)
linear_model = linear_model.fit(q=quantile)
# print(linear_model.summary())
df_learning['quantile_{:.3f}'.format(
quantile)] = linear_model.predict(X_learning)
df_prod['quantile_{:.3f}'.format(quantile)] = linear_model.predict(
X_prod)
else:
df_learning['quantile_{:.3f}'.format(quantile)] = 0
df_prod['quantile_{:.3f}'.format(quantile)] = 0
return df_learning, df_prod
#for f1, f2 in itertools.combinations(orig.columns.copy(), 2):
# prod = orig[f1].values * orig[f2].values
# orig[f1 + '_times_' + f2] = prod
orig['is_catole'] = np.array(df['bairro'] == 'catole', dtype='d')
orig['is_centro'] = np.array(df['bairro'] == 'centro', dtype='d')
orig['is_liberdade'] = np.array(df['bairro'] == 'liberdade', dtype='d')
scaled = pd.DataFrame(StandardScaler().fit_transform(orig.copy().values),
columns=orig.columns)
print(orig.shape)
assert orig.shape == scaled.shape
# In[5]:
model = QuantReg(response, orig)
# In[6]:
for q in np.linspace(0.05, 0.95, 10):
print(q)
print(model.fit(q=q).summary())
print()
print()
# In[ ]:
# In[ ]:
# In[ ]:
def QuantileRegression(X, Y, quantile):
mod = QuantReg(Y, X)
res = mod.fit(q=quantile)
return res.params
def run(self):
"""
Build the POD models.
Notes
-----
This method build the quantile regression model. First the censored data
are filtered if needed. The Box Cox transformation is performed if it is
enabled. Then it builds the POD model for given data and computes using
bootstrap all the defects quantile needed to build the POD model at the
confidence level.
"""
# Run the preliminary run of the POD class
result = self._run(self._inputSample, self._outputSample, self._detection,
self._noiseThres, self._saturationThres, self._boxCox,
self._censored)
# get some results
self._defects = result['inputSample']
self._signals = result['signals']
self._detectionBoxCox = result['detectionBoxCox']
defectsSize = self._defects.getSize()
# create the quantile regression object
X = ot.NumericalSample(defectsSize, [1, 0])
X[:, 1] = self._defects
self._algoQuantReg = QuantReg(np.array(self._signals), np.array(X))
# Compute the defect quantile
defectMax = self._defects.getMax()[0]
defectList = []
for probLevel in self._quantile:
# fit the quantile regression and return the NMF
model = self._buildModel(1. - probLevel)
# Solve the model == detectionBoxCox with defects
# boundaries = [0, defectMax]
defectList.append(ot.Brent().solve(model, self._detectionBoxCox,
0, defectMax))
# create support of the interpolating function including
# point (0, 0) and point (defectMax, max(quantile))
xvalue = np.hstack([0, defectList, defectMax])
yvalue = np.hstack([0., self._quantile, self._quantile.max()])
interpModel = interp1d(xvalue, yvalue, kind='linear')
self._PODmodel = ot.PythonFunction(1, 1, interpModel)
############ Confidence interval with bootstrap ########################
# Compute a NsimulationSize defect sizes for all quantiles
data = ot.NumericalSample(self._size, 2)
data[:, 0] = self._inputSample
data[:, 1] = self._outputSample
# bootstrap of the data
bootstrapExp = ot.BootstrapExperiment(data)
# create a numerical sample which contains for all simulations the
# defect quantile value. The goal is to compute the QuantilePerComponent
# of the simulation for each defect quantile (columns)
self._defectsPerQuantile = ot.NumericalSample(self._simulationSize, self._quantile.size)
for i in range(self._simulationSize):
# generate a sample with replacement within data of the same size
bootstrapData = bootstrapExp.generate()
# run the preliminary analysis : censore checking and box cox
result = self._run(bootstrapData[:,0], bootstrapData[:,1], self._detection,
self._noiseThres, self._saturationThres,
self._boxCox, self._censored)
# get some results
defects = result['inputSample']
signals = result['signals']
detectionBoxCox = result['detectionBoxCox']
defectsSize = defects.getSize()
# new quantile regression algorithm
X = ot.NumericalSample(defectsSize, [1, 0])
X[:, 1] = defects
algoQuantReg = QuantReg(np.array(signals), np.array(X))
# compute the quantile defects
defectMax = defects.getMax()[0]
defectList = []
for probLevel in self._quantile:
fit = algoQuantReg.fit(1. - probLevel, max_iter=300, p_tol=1e-2)
def model(x):
X = ot.NumericalPoint([1, x[0]])
return ot.NumericalPoint(fit.predict(X))
model = ot.PythonFunction(1, 1, model)
# Solve the model == detectionBoxCox with defects
# boundaries = [-infinity, defectMax] : it allows negative defects
# when for small prob level, there is no intersection with
# the detection threshold for positive defects
defectList.append(ot.Brent().solve(model, detectionBoxCox,
-ot.SpecFunc.MaxNumericalScalar,
defectMax))
# add the quantile in the numerical sample as the ith simulation
self._defectsPerQuantile[i, :] = defectList
if self._verbose:
updateProgress(i, self._simulationSize, 'Computing defect quantile')
def fit(self, X, y):
self.model_ = QuantReg(y, smapi.add_constant(X))
self.model_result_ = self.model_.fit(q=self.q)
return self
def fit(self, X, y):
X = self.preprocess(X)
self.regressor = QuantReg(y, X)
self.regressor_fit = self.regressor.fit(q=self.quantile)
def setup_fun(kernel='gau', bandwidth='bofinger'):
data = sm.datasets.engel.load_pandas().data
y, X = dmatrices('foodexp ~ income', data, return_type='dataframe')
statsm = QuantReg(y, X).fit(vcov='iid', kernel=kernel, bandwidth=bandwidth)
stata = d[(kernel, bandwidth)]
return statsm, stata
def fit(self,X,*args,**kwargs):
"""
Fit a projection pursuit dimension reduction model.
Required input argument: X data as matrix or data frame
Optinal input arguments:
arg or kwarg:
y data as vector or 1D matrix
kwargs:
h, int: option to overrule class's n_components parameter in fit.
Convenient command line, yet should not be used in automated
loops, e.g. cross-validation.
dmetric, str: distance metric used internally. Defaults to 'euclidean'
mixing, bool: to estimate mixing matrix (only relevant for ICA)
Further parameters to the regression methods can be passed on
here as well as kwargs, e.g. quantile=0.8 for quantile regression.
kwargs only relevant if y specified:
"""
# Collect optional fit arguments
biascorr = kwargs.pop('biascorr',False)
if 'h' not in kwargs:
h = self.n_components
else:
h = kwargs.pop('h')
self.n_components = h
if 'dmetric' not in kwargs:
dmetric = 'euclidean'
else:
dmetric = kwargs.get('dmetric')
if 'mixing' not in kwargs:
mixing = False
else:
mixing = kwargs.get('mixing')
if 'y' not in kwargs:
na = len(args)
if na > 0: #Use of *args makes it sklearn consistent
flag = 'two-block'
y = args[0]
else:
flag = 'one-block'
y = 0 # to allow calls with 'y=y' in spit of no real y argument present
else:
flag = 'two-block'
y = kwargs.get('y')
if 'quantile' not in kwargs:
quantile = .5
else:
quantile = kwargs.get('quantile')
if self.regopt == 'robust':
if 'fun' not in kwargs:
fun = 'Hampel'
else:
fun = kwargs.get('fun')
if 'probp1' not in kwargs:
probp1 = 0.95
else:
probp1 = kwargs.get('probp1')
if 'probp2' not in kwargs:
probp2 = 0.975
else:
probp2 = kwargs.get('probp2')
if 'probp3' not in kwargs:
probp3 = 0.99
else:
probp3 = kwargs.get('probp3')
if self.projection_index == dicomo:
if self.pi_arguments['mode'] in ('M3','cos','c*k'):
if 'option' not in kwargs:
option = 1
else:
option = kwargs.get('option')
if option > 3:
print('Option value >3 will compute results, but meaning may be questionable')
# Initiate projection index
self.most = self.projection_index(**self.pi_arguments)
# Initiate some parameters and data frames
if self.copy:
X0 = copy.deepcopy(X)
self.X0 = X0
else:
X0 = X
X = convert_X_input(X0)
n,p = X0.shape
trimming = self.trimming
# Check dimensions
if h > min(n,p):
raise(MyException('number of components cannot exceed number of samples'))
if (self.projection_index == dicomo and self.pi_arguments['mode'] == 'kurt' and self.whiten_data==False):
warnings.warn('Whitening step is recommended for ICA')
# Pre-processing adjustment if whitening
if self.whiten_data:
self.center_data = True
self.scale_data = False
self.compression = False
print('All results produced are for whitened data')
# Centring and scaling
if self.scale_data:
if self.center=='mean':
scale = 'std'
elif ((self.center=='median')|(self.center=='l1median')):
scale = 'mad'
else:
scale = 'None'
warnings.warn('Without scaling, convergence to optima is not given')
# Data Compression for flat tables if required
if ((p>n) and self.compression):
V,S,U = np.linalg.svd(X.T,full_matrices=False)
X = np.matmul(U.T,np.diag(S))
n,p = X.shape
if (srs.mad(X)==0).any():
warnings.warn('Due to low scales in data, compression would induce zero scales.'
+ '\n' + 'Proceeding without compression.')
dimensions = False
if copy:
X = copy.deepcopy(X0)
else:
X = X0
else:
dimensions = True
else:
dimensions = False
# Initiate centring object and scale X data
centring = VersatileScaler(center=self.center,scale=scale,trimming=trimming)
if self.center_data:
Xs = centring.fit_transform(X)
mX = centring.col_loc_
sX = centring.col_sca_
else:
Xs = X
mX = np.zeros((1,p))
sX = np.ones((1,p))
fit_arguments = {}
# Data whitening (best practice for ICA)
if self.whiten_data:
V,S,U = np.linalg.svd(Xs.T,full_matrices=False)
del U
K = (V/S)[:,:p]
del V,S
Xs = np.matmul(Xs, K)
Xs *= np.sqrt(p)
# Presently, X and y need to be matrices
# Will be changed to use regular np.ndarray
Xs = np.matrix(Xs)
# Pre-process y data when available
if flag != 'one-block':
ny = y.shape[0]
y = convert_y_input(y)
if len(y.shape) < 2:
y = np.matrix(y).reshape((ny,1))
# py = y.shape[1]
if ny != n:
raise(MyException('X and y number of rows must agree'))
if self.copy:
y0 = copy.deepcopy(y)
self.y0 = y0
if self.center_data:
ys = centring.fit_transform(y)
my = centring.col_loc_
sy = centring.col_sca_
else:
ys = y
my = 0
sy = 1
ys = np.matrix(ys).astype('float64')
else:
ys = None
# Initializing output matrices
W = np.zeros((p,h))
T = np.zeros((n,h))
P = np.zeros((p,h))
B = np.zeros((p,h))
R = np.zeros((p,h))
B_scaled = np.zeros((p,h))
C = np.zeros((h,1))
Xev = np.zeros((h,1))
assovec = np.zeros((h,1))
Maxobjf = np.zeros((h,1))
# Initialize deflation matrices
E = copy.deepcopy(Xs)
f = ys
bi = np.zeros((p,1))
opt_args = {
'alpha': self.alpha,
'trimming': self.trimming,
'biascorr': biascorr,
'dmetric' : 'euclidean',
}
if self.optimizer=='grid':
# Define grid optimization ranges
if 'ndir' not in self.optimizer_options:
self.optimizer_options['ndir'] = 1000
optrange = np.sign(self.optrange)
optmax = self.optrange[1]
stop0s = np.arcsin(optrange[0])
stop1s = np.arcsin(optrange[1])
stop1c = np.arccos(optrange[0])
stop0c = np.arccos(optrange[1])
anglestart = max(stop0c,stop0s)
anglestop = max(stop1c,stop1s)
nangle = np.linspace(anglestart,anglestop,self.optimizer_options['ndir'],endpoint=False)
alphamat = np.matrix([np.cos(nangle), np.sin(nangle)])
opt_args['_stop0c'] = stop0c
opt_args['_stop0s'] = stop0s
opt_args['_stop1c'] = stop1c
opt_args['_stop1s'] = stop1s
opt_args['optmax'] = optmax
opt_args['optrange'] = self.optrange
opt_args['square_pi'] = self.square_pi
if optmax != 1:
alphamat *= optmax
if p>2:
anglestart = min(opt_args['_stop0c'],opt_args['_stop0s'])
anglestop = min(opt_args['_stop1c'],opt_args['_stop1s'])
nangle = np.linspace(anglestart,anglestop,self.optimizer_options['ndir'],endpoint=True)
alphamat2 = np.matrix([np.cos(nangle), np.sin(nangle)])
if optmax != 1:
alphamat2 *= opt_args['optmax']
# Arguments for grid plane
opt_args['alphamat'] = alphamat,
opt_args['ndir'] = self.optimizer_options['ndir'],
opt_args['maxiter'] = self.optimizer_options['maxiter']
if type(opt_args['ndir'] is tuple):
opt_args['ndir'] = opt_args['ndir'][0]
# Arguments for grid plane #2
grid_args_2 = {
'alpha': self.alpha,
'alphamat': alphamat2,
'ndir': self.optimizer_options['ndir'],
'trimming': self.trimming,
'biascorr': biascorr,
'dmetric' : 'euclidean',
'_stop0c' : stop0c,
'_stop0s' : stop0s,
'_stop1c' : stop1c,
'_stop1s' : stop1s,
'optmax' : optmax,
'optrange' : self.optrange,
'square_pi' : self.square_pi
}
if flag=='two-block':
grid_args_2['y'] = f
if flag=='two-block':
opt_args['y'] = f
# Itertive coefficient estimation
for i in range(0,h):
if self.optimizer=='grid':
if p==2:
wi,maximo = gridplane(E,self.most,
pi_arguments=opt_args
)
elif p>2:
afin = np.zeros((p,1)) # final parameters for linear combinations
Z = copy.deepcopy(E)
# sort variables according to criterion
meas = [self.most.fit(E[:,k],
**opt_args)
for k in np.arange(0,p)]
if self.square_pi:
meas = np.square(meas)
wi,maximo = gridplane(Z[:,0:2],self.most,opt_args)
Zopt = Z[:,0:2]*wi
afin[0:2]=wi
for j in np.arange(2,p):
projmat = np.matrix([np.array(Zopt[:,0]).reshape(-1),
np.array(Z[:,j]).reshape(-1)]).T
wi,maximo = gridplane(projmat,self.most,
opt_args
)
Zopt = Zopt*float(wi[0]) + Z[:,j]*float(wi[1])
afin[0:(j+1)] = afin[0:(j+1)]*float(wi[0])
afin[j] = float(wi[1])
tj = Z*afin
objf = self.most.fit(tj,
**{**fit_arguments,**opt_args}
)
if self.square_pi:
objf *= objf
# outer loop to run until convergence
objfold = copy.deepcopy(objf)
objf = -1000
afinbest = afin
ii = 0
maxiter_2j = 2**round(np.log2(self.optimizer_options['maxiter']))
while ((ii < self.optimizer_options['maxiter'] + 1) and (abs(objfold - objf)/abs(objf) > 1e-4)):
for j in np.arange(0,p):
projmat = np.matrix([np.array(Zopt[:,0]).reshape(-1),
np.array(Z[:,j]).reshape(-1)]).T
if j > 16:
divv = maxiter_2j
else:
divv = min(2**j,maxiter_2j)
wi,maximo = gridplane_2(projmat,
self.most,
q=afin[j],
div=divv,
pi_arguments=grid_args_2
)
Zopt = Zopt*float(wi[0,0]) + Z[:,j]*float(wi[1,0])
afin *= float(wi[0,0])
afin[j] += float(wi[1,0])
# % evaluate the objective function:
tj = Z*afin
objfold = copy.deepcopy(objf)
objf = self.most.fit(tj,
q=afin,
**opt_args
)
if self.square_pi:
objf *= objf
if objf!=objfold:
if self.constraint == 'norm':
afinbest = afin/np.sqrt(np.sum(np.square(afin)))
else:
afinbest = afin
ii +=1
if self.verbose:
print(str(ii))
#endwhile
afinbest = afin
wi = np.zeros((p,1))
wi = afinbest
Maxobjf[i] = objf
# endif;%if p>2;
else: # do not optimize by the grid algorithm
if self.trimming > 0:
warnings.warn('Optimization that involves a trimmed objective is not a quadratic program. The scipy-optimize result will be off!!')
if 'center' in self.pi_arguments:
if (self.pi_arguments['center']=='median'):
warnings.warn('Optimization that involves a median in the objective is not a quadratic program. The scipy-optimize result will be off!!')
constraint = {'type':'eq',
'fun': lambda x: np.linalg.norm(x) -1,
}
if len(self.optimizer_constraints)>0:
constraint = [constraint,self.optimizer_constraints]
wi = minimize(pp_objective,
E[0,:].transpose(),
args=(self.most,E,opt_args),
method=self.optimizer,
constraints=constraint,
options=self.optimizer_options).x
wi = np.matrix(wi).reshape((p,1))
wi /= np.sqrt(np.sum(np.square(wi)))
# Computing projection weights and scores
ti = E*wi
if self.optimizer != 'grid':
Maxobjf[i] = self.most.fit(E*wi,**opt_args)
nti = np.linalg.norm(ti)
pi = E.T*ti / (nti**2)
if self.whiten_data:
wi /= np.sqrt((wi**2).sum())
wi = K*wi
wi0 = wi
wi = np.array(wi)
if len(W[:,i].shape) == 1:
wi = wi.reshape(-1)
W[:,i] = wi
T[:,i] = np.array(ti).reshape(-1)
P[:,i] = np.array(pi).reshape(-1)
if flag != 'one-block':
criteval = self.most.fit(E*wi0,
**opt_args
)
if self.square_pi:
criteval *= criteval
assovec[i] = criteval
# Deflation of the datamatrix guaranteeing orthogonality restrictions
E -= ti*pi.T
# Calculate R-Weights
R = np.dot(W[:,0:(i+1)],pinv2(np.dot(P[:,0:(i+1)].T,W[:,0:(i+1)]),check_finite=False))
# Execute regression y~T if y is present. Generate regression estimates.
if flag != 'one-block':
if self.regopt=='OLS':
ci = np.dot(ti.T,ys)/(nti**2)
elif self.regopt == 'robust':
linfit = rm(fun=fun,probp1=probp1,probp2=probp2,probp3=probp3,
centre=self.center,scale=scale,
start_cutoff_mode='specific',verbose=self.verbose)
linfit.fit(ti,ys)
ci = linfit.coef_
elif self.regopt == 'quantile':
linfit = QuantReg(y,ti)
model = linfit.fit(q=quantile)
ci = model.params
# end regression if
C[i] = ci
bi = np.dot(R,C[0:(i+1)])
bi_scaled = bi
bi = np.multiply(np.reshape(sy/sX,(p,1)),bi)
B[:,i] = bi[:,0]
B_scaled[:,i] = bi_scaled[:,0]
# endfor; Loop for latent dimensions
# Re-adjust estimates to original dimensions if data have been compressed
if dimensions:
B = np.matmul(V[:,0:p],B)
B_scaled = np.matmul(V[:,0:p],B_scaled)
R = np.matmul(V[:,0:p],R)
W = np.matmul(V[:,0:p],W)
P = np.matmul(V[:,0:p],P)
bi = B[:,h-1]
if self.center_data:
Xs = centring.fit_transform(X0)
mX = centring.col_loc_
sX = centring.col_sca_
else:
Xs = X0
mX = np.zeros((1,p))
sX = np.ones((1,p))
bi = bi.astype("float64")
if flag != 'one-block':
# Calculate scaled and unscaled intercepts
if dimensions:
X = convert_X_input(X0)
if(self.center == "mean"):
intercept = sps.trim_mean(y - np.matmul(X,bi),trimming)
else:
intercept = np.median(np.reshape(y - np.matmul(X,bi),(-1)))
yfit = np.matmul(X,bi) + intercept
if not(scale == 'None'):
if (self.center == "mean"):
b0 = np.mean(ys - np.matmul(Xs.astype("float64"),bi))
else:
b0 = np.median(np.array(ys.astype("float64") - np.matmul(Xs.astype("float64"),bi)))
else:
b0 = intercept
# Calculate fit values and residuals
yfit = yfit
r = y - yfit
setattr(self,"coef_",B)
setattr(self,"intercept_",intercept)
setattr(self,"coef_scaled_",B_scaled)
setattr(self,"intercept_scaled_",b0)
setattr(self,"residuals_",r)
setattr(self,"fitted_",yfit)
setattr(self,"y_loadings_",C)
setattr(self,"y_loc_",my)
setattr(self,"y_sca_",sy)
setattr(self,"x_weights_",W)
setattr(self,"x_loadings_",P)
setattr(self,"x_rotations_",R)
setattr(self,"x_scores_",T)
setattr(self,"x_ev_",Xev)
setattr(self,"crit_values_",assovec)
setattr(self,"Maxobjf_",Maxobjf)
if self.whiten_data:
setattr(self,"whitening_",K)
if mixing:
setattr(self,"mixing_",np.linalg.pinv(W))
setattr(self,"x_loc_",mX)
setattr(self,"x_sca_",sX)
setattr(self,'scaling',scale)
if self.return_scaling_object:
setattr(self,'scaling_object_',centring)
return(self)
from scipy import stats
import statsmodels.api as sm
from statsmodels.regression.quantile_regression import QuantReg
sige = 0.1
nobs, k_vars = 500, 3
x = np.random.uniform(-1, 1, size=nobs)
x.sort()
exog = np.vander(x, k_vars + 1)[:, ::-1]
mix = 0.1 * stats.norm.pdf(
x[:, None], loc=np.linspace(-0.5, 0.75, 4), scale=0.01).sum(1)
y = exog.sum(1) + mix + sige * (np.random.randn(nobs) / 2 + 1)**3
p = 0.5
res_qr = QuantReg(y, exog).fit(p)
res_qr2 = QuantReg(y, exog).fit(0.1)
res_qr3 = QuantReg(y, exog).fit(0.75)
res_ols = sm.OLS(y, exog).fit()
params = [res_ols.params, res_qr2.params, res_qr.params, res_qr3.params]
labels = ['ols', 'qr 0.1', 'qr 0.5', 'qr 0.75']
plt.figure()
plt.plot(x, y, '.', alpha=0.5)
for lab, beta in zip(['ols', 'qr 0.1', 'qr 0.5', 'qr 0.75'], params):
print('%-8s' % lab, np.round(beta, 4))
fitted = np.dot(exog, beta)
lw = 2
plt.plot(x, fitted, lw=lw, label=lab)
plt.legend()
evals = [(dtrain, 'train'), (dvalid_xy, 'eval')]
model = xgb.train(xgb_params,
dtrain,
num_boost_round=num_boost_rounds,
evals=evals,
early_stopping_rounds=early_stopping_rounds,
verbose_eval=10)
valid_pred = model.predict(dvalid_x, ntree_limit=model.best_ntree_limit)
print("XGBoost validation set predictions:")
print(pd.DataFrame(valid_pred).head())
print("\nMean absolute validation error:")
mean_absolute_error(y_valid, valid_pred)
if OPTIMIZE_FUDGE_FACTOR:
mod = QuantReg(y_valid, valid_pred)
res = mod.fit(q=.5)
print("\nLAD Fit for Fudge Factor:")
print(res.summary())
fudge = res.params[0]
print("Optimized fudge factor:", fudge)
print("\nMean absolute validation error with optimized fudge factor: ")
print(mean_absolute_error(y_valid, fudge * valid_pred))
fudge **= FUDGE_FACTOR_SCALEDOWN
print("Scaled down fudge factor:", fudge)
print("\nMean absolute validation error with scaled down fudge factor: ")
print(mean_absolute_error(y_valid, fudge * valid_pred))
else:
fudge = 1.0
