def plot_linear_model():
X, y = linear_model()
clf = svm.SVC(kernel='linear',
gamma=0.01, coef0=0, degree=3)
clf.fit(X, y)
fig = pl.figure()
ax = pl.subplot(111, xticks=[], yticks=[])
ax.scatter(X[:, 0], X[:, 1], c=y, cmap=pl.cm.bone)
ax.scatter(clf.support_vectors_[:, 0],
clf.support_vectors_[:, 1],
s=80, edgecolors="k", facecolors="none")
delta = 1
y_min, y_max = -50, 50
x_min, x_max = -50, 50
x = np.arange(x_min, x_max + delta, delta)
y = np.arange(y_min, y_max + delta, delta)
X1, X2 = np.meshgrid(x, y)
Z = clf.decision_function(np.c_[X1.ravel(), X2.ravel()])
Z = Z.reshape(X1.shape)
levels = [-1.0, 0.0, 1.0]
linestyles = ['dashed', 'solid', 'dashed']
colors = 'k'
ax.contour(X1, X2, Z, levels,
colors=colors,
linestyles=linestyles)
def modelSklearn(self):
'''
There is no solution using
sklearn as of now.
'''
model = sk.linear_model(self.xTrain, self.yTrain)
predictions = model.predict(self.xValidation)
# TODO: Use some optimization along with a loss function to reduce error
return predictions
import sklearn sklearn.linear_model() ergregreg gwthgrthtrh
def train_linear_model(train_X,
train_y,
val_X,
val_y,
train_weights=None,
val_weights=None,
linear_model=sklearn.linear_model.Ridge,
plot_alphas=False,
optimize='r2'):
'''
Args
- train_X: np.array, shape [num_train, features_dim]
- train_y: np.array, shape [num_train]
- val_X: np.array, shape [num_val, features_dim]
- val_y: np.array, shape [num_val]
- train_weights: np.array, shape [num_train]
- val_weights: np.array, shape [num_val]
- linear_model: sklearn.linear_model
- plot_alphas: bool, whether to plot alphas
- optimize: str, one of ['r2', 'R2']
Returns
- best_model: sklearn.linear_model, learned model
- best_train_preds: np.array, shape [num_train], output of best_model on train_X
- best_val_preds: np.array, shape [num_val], output of best_model on val_X
'''
assert optimize in ['r2', 'R2']
alphas = 2**np.arange(-5, 40, 0.5)
train_R2s = np.zeros_like(alphas)
train_r2s = np.zeros_like(alphas)
val_R2s = np.zeros_like(alphas)
val_r2s = np.zeros_like(alphas)
best_model = None
best_train_preds = None
best_val_preds = None
for i, alpha in enumerate(alphas):
model = linear_model(alpha=alpha)
model.fit(X=train_X, y=train_y)
train_preds = model.predict(train_X)
if plot_alphas:
train_R2s[i] = calc_score(labels=train_y,
preds=train_preds,
metric='R2',
weights=train_weights)
train_r2s[i] = calc_score(labels=train_y,
preds=train_preds,
metric='r2',
weights=train_weights)
val_preds = model.predict(val_X)
val_R2 = calc_score(labels=val_y,
preds=val_preds,
metric='R2',
weights=val_weights)
val_r2 = calc_score(labels=val_y,
preds=val_preds,
metric='r2',
weights=val_weights)
if (best_model is None) \
or (optimize == 'r2' and val_r2 > np.max(val_r2s)) \
or (optimize == 'R2' and val_R2 > np.max(val_R2s)):
best_model = model
best_val_preds = val_preds
best_train_preds = train_preds
val_R2s[i] = val_R2
val_r2s[i] = val_r2
if plot_alphas:
best_index = np.argmax(val_r2s)
print('best alpha: {:e}'.format(alphas[best_index]))
plot_alpha_vs_r2(alphas, train_R2s, train_r2s, val_R2s, val_r2s)
return best_model, best_train_preds, best_val_preds
def train_linear_logo(features,
labels,
group_labels,
cv_groups,
test_groups,
weights=None,
linear_model=sklearn.linear_model.Ridge,
plot=True,
group_names=None,
return_weights=False,
verbose=False):
'''Leave-one-group-out cross-validated training of a linear model.
Args
- features: np.array, shape [N, D]
each feature dim should be normalized to 0 mean, unit variance
- labels: np.array, shape [N]
- group_labels: np.array, shape [N], type np.int32
- cv_groups: list of int, labels of groups to use for LOGO-CV
- test_groups: list of int, labels of groups to test on
- weights: np.array, shape [N]
- linear_model: sklearn.linear_model
- plot: bool, whether to plot MSE as a function of alpha
- group_names: list of str, names of the groups, only used when plotting
- return_weights: bool, whether to return the final trained model weights
- verbose: bool
Returns
- test_preds: np.array, predictions on indices from test_groups
- coefs: np.array, shape [D] (only returned if return_weights=True)
- intercept: float (only returned if return_weights=True)
'''
cv_indices = np.isin(group_labels, cv_groups).nonzero()[0]
test_indices = np.isin(group_labels, test_groups).nonzero()[0]
X = features[cv_indices]
y = labels[cv_indices]
groups = group_labels[cv_indices]
w = None if weights is None else weights[cv_indices]
alphas = 2**np.arange(-5, 35, 3.0)
preds = np.zeros([len(alphas), len(cv_indices)], dtype=np.float64)
group_mses = np.zeros([len(alphas), len(cv_groups)], dtype=np.float64)
leftout_group_labels = np.zeros(len(cv_groups), dtype=np.int32)
logo = sklearn.model_selection.LeaveOneGroupOut()
for i, alpha in enumerate(alphas):
if verbose:
print(f'\rAlpha: {alpha} ({i+1}/{len(alphas)})', end='')
# set random_state for deterministic data shuffling
model = linear_model(alpha=alpha, random_state=123)
for g, (train_indices,
val_indices) in enumerate(logo.split(X, groups=groups)):
train_X, val_X = X[train_indices], X[val_indices]
train_y, val_y = y[train_indices], y[val_indices]
train_w = None if w is None else w[train_indices]
val_w = None if w is None else w[val_indices]
model.fit(X=train_X, y=train_y, sample_weight=train_w)
val_preds = model.predict(val_X)
preds[i, val_indices] = val_preds
group_mses[i, g] = np.average((val_preds - val_y)**2,
weights=val_w)
leftout_group_labels[g] = groups[val_indices[0]]
if verbose:
print()
mses = np.average((preds - y)**2, axis=1, weights=w) # shape [num_alphas]
if plot:
h = max(3, len(group_names) * 0.2)
fig, ax = plt.subplots(1,
1,
figsize=[h * 2, h],
constrained_layout=True)
for g in range(len(cv_groups)):
group_name = group_names[leftout_group_labels[g]]
ax.scatter(x=alphas,
y=group_mses[:, g],
label=group_name,
c=[cm.tab20.colors[g % 20]])
ax.plot(alphas, mses, 'g-', label='Overall val mse')
ax.legend(loc='center left',
bbox_to_anchor=(1, 0.5),
title='Left-out Group')
ax.set(xlabel='alpha', ylabel='mse')
ax.set_xscale('log')
ax.grid(True)
plt.show()
best_alpha = alphas[np.argmin(mses)]
best_model = linear_model(alpha=best_alpha)
best_model.fit(X=X, y=y, sample_weight=w)
test_X, test_y, = features[test_indices], labels[test_indices]
test_preds = best_model.predict(test_X)
best_val_mse = np.min(mses)
test_w = None if weights is None else weights[test_indices]
test_mse = np.average((test_preds - test_y)**2, weights=test_w)
print(
f'best val mse: {best_val_mse:.3f}, best alpha: {best_alpha}, test mse: {test_mse:.3f}'
)
if not return_weights:
return test_preds
else:
coefs = best_model.coef_
intercept = best_model.intercept_
return test_preds, coefs, intercept
X_test = scaler.transform(X_test)
X_train = scipy.sparse.hstack((text_train_tfidf, X_train), format="csr")
X_test = scipy.sparse.hstack((text_test_tfidf, X_test), format="csr")
model = sklearn.linear_model.LogisticRegression(C=0.7, penalty="l2")
result = make_predictions(model, X_train, target, X_test)
io.save_result(test["PostId"], result)
return result
def make_dirs(dir_names):
for name in dir_names:
if not os.path.exists(name):
os.makedirs(name)
dir_names = ["input", "output", "w2v", "metafeatures"]
make_dirs(dir_names)
train, target, test = io.load_data()
text_train_tfidf, text_test_tfidf = get_tfidf(train, test)
preds1 = rf_model(train, target, test, text_train_tfidf,
text_test_tfidf)
preds2 = linear_model(train, target, test, text_train_tfidf,
text_test_tfidf)
result = 0.7*preds1 + 0.3*preds2
io.save_result(test["PostId"], result)
