dataset = load_boston()
X = dataset.data
y = dataset.target
_, true_features = X.shape
# add dummy feature
X = np.concatenate([X, np.random.randn(*X.shape)], axis=1)
feature_names = list(dataset.feature_names) + ["fake"] * true_features
# standardize
X = StandardScaler().fit_transform(X)
y = scale(y)
X_train, X_test, y_train, y_test = train_test_split(X, y)
model = LassoNetRegressor(hidden_dims=(10,), verbose=True, patience=(100, 5))
path = model.path(X_train, y_train)
n_selected = []
mse = []
lambda_ = []
for save in path:
model.load(save.state_dict)
y_pred = model.predict(X_test)
n_selected.append(save.selected.sum().cpu().numpy())
mse.append(mean_squared_error(y_test, y_pred))
lambda_.append(save.lambda_)
fig = plt.figure(figsize=(12, 12))
X = dataset.data
y = dataset.target
_, true_features = X.shape
# add dummy feature
X = np.concatenate([X, np.random.randn(*X.shape)], axis=1)
feature_names = list(dataset.feature_names) + ["fake"] * true_features
# standardize
X = StandardScaler().fit_transform(X)
y = scale(y)
X_train, X_test, y_train, y_test = train_test_split(X, y)
model = LassoNetRegressor(
hidden_dims=(10, ),
eps_start=0.1,
verbose=True,
)
path = model.path(X_train, y_train)
n_selected = []
mse = []
lambda_ = []
for save in path:
model.load(save.state_dict)
y_pred = model.predict(X_test)
n_selected.append(save.selected.sum())
mse.append(mean_squared_error(y_test, y_pred))
lambda_.append(save.lambda_)
X = dataset.data
y = dataset.target
_, true_features = X.shape
# add dummy feature
X = np.concatenate([X, np.random.randn(*X.shape)], axis=1)
feature_names = list(dataset.feature_names) + ["fake"] * true_features
# standardize
X = StandardScaler().fit_transform(X)
y = scale(y)
X_train, X_test, y_train, y_test = train_test_split(X, y)
model = LassoNetRegressor(
hidden_dims=(10,),
verbose=True,
)
path = model.path(X_train, y_train)
plot_path(model, path, X_test, y_test)
plt.savefig("diabetes.png")
plt.clf()
n_features = X.shape[1]
importances = model.feature_importances_.numpy()
order = np.argsort(importances)[::-1]
importances = importances[order]
ordered_feature_names = [feature_names[i] for i in order]
color = np.array(["g"] * true_features + ["r"] * (n_features - true_features))[order]
p = 200
n = 1000
X = np.random.rand(n, p)
y = (10 * np.sin(np.pi * X[:, 0] * X[:, 1]) + 20 * (X[:, 2] - 0.5)**2 +
10 * X[:, 3] + 5 * X[:, 4])
return X, y
X, y = friedman()
X = StandardScaler().fit_transform(X)
y -= y.mean()
y /= y.std()
X_train, X_test, y_train, y_test = train_test_split(X, y)
model = LassoNetRegressor(verbose=True,
path_multiplier=1.002,
hidden_dims=(10, 10))
path = model.path(X_train, y_train)
def score(self, X, y, sample_weight=None):
y_pred = self.predict(X)
return np.sqrt(1 - r2_score(y, y_pred, sample_weight=sample_weight))
model.score = partial(score, model)
import matplotlib.pyplot as plt
plot_path(model, path, X_test, y_test)
y = np.concatenate((y_train, y_val))
y_mean = y.mean()
y_std = y.std()
for y in [y_train, y_val, y_test]:
y -= y_mean
y /= y_std
def rrmse(y, y_pred):
return np.sqrt(1 - r2_score(y, y_pred))
if __name__ == "__main__":
model = LassoNetRegressor(
path_multiplier=1.001,
M=100_000,
hidden_dims=(10, 10),
torch_seed=0,
)
path = model.path(X_train, y_train, X_val=X_val, y_val=y_val)
print(
"rrmse:",
min(rrmse(y_test,
model.load(save).predict(X_test)) for save in path),
)
plot_path(model, path, X_test, y_test, score_function=rrmse)
plt.show()
def test_regressor():
X, y = load_diabetes(return_X_y=True)
model = LassoNetRegressor()
model.fit(X, y)
model.score(X, y)
y -= y.mean()
y /= y.std()
def rrmse(y, y_pred):
return np.sqrt(1 - r2_score(y, y_pred))
for path_multiplier in [1.01, 1.001]:
print("path_multiplier:", path_multiplier)
for M in [10, 100, 1_000, 10_000, 100_000]:
print("M:", M)
model = LassoNetRegressor(
hidden_dims=(10, 10),
random_state=0,
torch_seed=0,
path_multiplier=path_multiplier,
M=M,
)
path = model.path(X_train, y_train)
print(
"rrmse:",
min(rrmse(y_test, model.load(save).predict(X_test)) for save in path),
)
plot_path(model, path, X_test, y_test, score_function=rrmse)
plt.savefig(f"friedman_path({path_multiplier})_M({M}).jpg")
path_multiplier = 1.001
print("path_multiplier:", path_multiplier)
for M in [100, 1_000, 10_000, 100_000]:
print("M:", M)
from sklearn.datasets import fetch_openml
from lassonet import LassoNetRegressor
from sklearn.metrics import mean_squared_error
import matplotlib.pyplot as plt
X, y = fetch_openml(name="mnist_784", return_X_y=True)
filter = y == "3"
X = X[filter].values / 255
model = LassoNetRegressor(M=30, n_iters=(3000, 500), path_multiplier=1.05, verbose=True)
path = model.path(X, X)
img = model.feature_importances_.reshape(28, 28)
plt.title("Feature importance to reconstruct 3")
plt.imshow(img)
plt.colorbar()
plt.savefig("mnist-reconstruction-importance.png")
n_selected = []
score = []
lambda_ = []
for save in path:
model.load(save.state_dict)
X_pred = model.predict(X)
n_selected.append(save.selected.sum())
score.append(mean_squared_error(X_pred, X))
lambda_.append(save.lambda_)