def main():
# Load temperature data
data = pd.read_csv('mlfromscratch/data/TempLinkoping2016.txt', sep="\t")
time = np.atleast_2d(data["time"].values).T
temp = data["temp"].values
X = time # fraction of the year [0, 1]
y = temp
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4)
poly_degree = 13
model = ElasticNet(degree=15,
reg_factor=0.01,
l1_ratio=0.7,
learning_rate=0.001,
n_iterations=4000)
model.fit(X_train, y_train)
# Training error plot
n = len(model.training_errors)
training, = plt.plot(range(n),
model.training_errors,
label="Training Error")
plt.legend(handles=[training])
plt.title("Error Plot")
plt.ylabel('Mean Squared Error')
plt.xlabel('Iterations')
plt.show()
y_pred = model.predict(X_test)
mse = mean_squared_error(y_test, y_pred)
print("Mean squared error: %s (given by reg. factor: %s)" % (mse, 0.05))
y_pred_line = model.predict(X)
# Color map
cmap = plt.get_cmap('viridis')
# Plot the results
m1 = plt.scatter(366 * X_train, y_train, color=cmap(0.9), s=10)
m2 = plt.scatter(366 * X_test, y_test, color=cmap(0.5), s=10)
plt.plot(366 * X,
y_pred_line,
color='black',
linewidth=2,
label="Prediction")
plt.suptitle("Elastic Net")
plt.title("MSE: %.2f" % mse, fontsize=10)
plt.xlabel('Day')
plt.ylabel('Temperature in Celcius')
plt.legend((m1, m2), ("Training data", "Test data"), loc='lower right')
plt.show()
def main():
# Load temperature data
data = pd.read_csv('mlfromscratch/data/TempLinkoping2016.txt', sep="\t")
time = np.atleast_2d(data["time"].values).T
temp = data["temp"].values
X = time # fraction of the year [0, 1]
y = temp
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4)
poly_degree = 13
model = ElasticNet(degree=15,
reg_factor=0.01,
l1_ratio=0.7,
learning_rate=0.001,
n_iterations=4000)
model.fit(X_train, y_train)
# Training error plot
n = len(model.training_errors)
training, = plt.plot(range(n), model.training_errors, label="Training Error")
plt.legend(handles=[training])
plt.title("Error Plot")
plt.ylabel('Mean Squared Error')
plt.xlabel('Iterations')
plt.show()
y_pred = model.predict(X_test)
mse = mean_squared_error(y_test, y_pred)
print ("Mean squared error: %s (given by reg. factor: %s)" % (mse, 0.05))
y_pred_line = model.predict(X)
# Color map
cmap = plt.get_cmap('viridis')
# Plot the results
m1 = plt.scatter(366 * X_train, y_train, color=cmap(0.9), s=10)
m2 = plt.scatter(366 * X_test, y_test, color=cmap(0.5), s=10)
plt.plot(366 * X, y_pred_line, color='black', linewidth=2, label="Prediction")
plt.suptitle("Elastic Net")
plt.title("MSE: %.2f" % mse, fontsize=10)
plt.xlabel('Day')
plt.ylabel('Temperature in Celcius')
plt.legend((m1, m2), ("Training data", "Test data"), loc='lower right')
plt.show()
def main():
X, y = make_regression(n_samples=100, n_features=1, noise=20)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4)
n_samples, n_features = np.shape(X)
model = LinearRegression(n_iterations=100)
model.fit(X_train, y_train)
# Training error plot
n = len(model.training_errors)
training, = plt.plot(range(n),
model.training_errors,
label="Training Error")
plt.legend(handles=[training])
plt.title("Error Plot")
plt.ylabel('Mean Squared Error')
plt.xlabel('Iterations')
plt.show()
y_pred = model.predict(X_test)
mse = mean_squared_error(y_test, y_pred)
print("Mean squared error: %s" % (mse))
y_pred_line = model.predict(X)
# Color map
cmap = plt.get_cmap('viridis')
# Plot the results
m1 = plt.scatter(366 * X_train, y_train, color=cmap(0.9), s=10)
m2 = plt.scatter(366 * X_test, y_test, color=cmap(0.5), s=10)
plt.plot(366 * X,
y_pred_line,
color='black',
linewidth=2,
label="Prediction")
plt.suptitle("Linear Regression")
plt.title("MSE: %.2f" % mse, fontsize=10)
plt.xlabel('Day')
plt.ylabel('Temperature in Celcius')
plt.legend((m1, m2), ("Training data", "Test data"), loc='lower right')
plt.show()
def main():
X, y = make_regression(n_samples=1000, n_features=1, noise=10)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4)
model = LassoRegression(degree=15,
reg_factor=0.05,
learning_rate=0.001,
n_iterations=4000)
model.fit(X_train, y_train)
n = len(model.training_errors)
training, = plt.plot(range(n),
model.training_errors,
label='Training Error')
plt.legend(handles=[training])
plt.title('Error Plot')
plt.ylabel('Mean Squared Error')
plt.xlabel('Iterations')
plt.show()
y_pred = model.predict(X_test)
mse = mean_squared_error(y_test, y_pred)
print('Mean Squared Error: {} (given by reg_factor : {}'.format(mse, 0.5))
y_pred_line = model.predict(X)
cmap = plt.get_cmap('viridis')
# Plot the results
m1 = plt.scatter(366 * X_train, y_train, color=cmap(0.9), s=10)
m2 = plt.scatter(366 * X_test, y_test, color=cmap(0.5), s=10)
plt.plot(366 * X,
y_pred_line,
color='black',
linewidth=2,
label="Prediction")
plt.suptitle("Lasso Regression")
plt.title("MSE: {}".format(mse, fontsize=10))
plt.xlabel('X')
plt.ylabel('Y')
plt.legend((m1, m2), ("Training data", "Test data"), loc='lower right')
plt.show()
def main():
# Load temperature data
data = pd.read_csv('mlfromscratch/data/TempLinkoping2016.txt', sep="\t")
time = np.atleast_2d(data["time"].as_matrix()).T
temp = np.atleast_2d(data["temp"].as_matrix()).T
X = time # Time. Fraction of the year [0, 1]
y = temp[:, 0] # Temperature. Reduce to one-dim
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4)
n_samples, n_features = np.shape(X)
clf = LinearRegression()
clf.fit(X_train, y_train)
y_pred = clf.predict(X_test)
mse = mean_squared_error(y_test, y_pred)
print("Mean squared error: %s" % (mse))
y_pred_line = clf.predict(X)
# Color map
cmap = plt.get_cmap('viridis')
# Plot the results
m1 = plt.scatter(366 * X_train, y_train, color=cmap(0.9), s=10)
m2 = plt.scatter(366 * X_test, y_test, color=cmap(0.5), s=10)
plt.plot(366 * X,
y_pred_line,
color='black',
linewidth=2,
label="Prediction")
plt.suptitle("Polynomial Ridge Regression")
plt.title("MSE: %.2f" % mse, fontsize=10)
plt.xlabel('Day')
plt.ylabel('Temperature in Celcius')
plt.legend((m1, m2), ("Training data", "Test data"), loc='lower right')
plt.show()
def main():
X, y = make_regression(n_samples=100, n_features=1, noise=20)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4)
n_samples, n_features = np.shape(X)
model = LinearRegression(n_iterations=100)
model.fit(X_train, y_train)
# Training error plot
n = len(model.training_errors)
training, = plt.plot(range(n), model.training_errors, label="Training Error")
plt.legend(handles=[training])
plt.title("Error Plot")
plt.ylabel('Mean Squared Error')
plt.xlabel('Iterations')
plt.show()
y_pred = model.predict(X_test)
mse = mean_squared_error(y_test, y_pred)
print ("Mean squared error: %s" % (mse))
y_pred_line = model.predict(X)
# Color map
cmap = plt.get_cmap('viridis')
# Plot the results
m1 = plt.scatter(366 * X_train, y_train, color=cmap(0.9), s=10)
m2 = plt.scatter(366 * X_test, y_test, color=cmap(0.5), s=10)
plt.plot(366 * X, y_pred_line, color='black', linewidth=2, label="Prediction")
plt.suptitle("Linear Regression")
plt.title("MSE: %.2f" % mse, fontsize=10)
plt.xlabel('Day')
plt.ylabel('Temperature in Celcius')
plt.legend((m1, m2), ("Training data", "Test data"), loc='lower right')
plt.show()
def main():
print("-- Regression Tree --")
# Load temperature data
data = pd.read_csv('mlfromscratch/data/TempLinkoping2016.txt', sep="\t")
time = np.atleast_2d(data["time"].values).T
temp = np.atleast_2d(data["temp"].values).T
X = standardize(time) # Time. Fraction of the year [0, 1]
y = temp[:, 0] # Temperature. Reduce to one-dim
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)
model = RegressionTree()
model.fit(X_train, y_train)
y_pred = model.predict(X_test)
y_pred_line = model.predict(X)
# Color map
cmap = plt.get_cmap('viridis')
mse = mean_squared_error(y_test, y_pred)
print("Mean Squared Error:", mse)
# Plot the results
# Plot the results
m1 = plt.scatter(366 * X_train, y_train, color=cmap(0.9), s=10)
m2 = plt.scatter(366 * X_test, y_test, color=cmap(0.5), s=10)
m3 = plt.scatter(366 * X_test, y_pred, color='black', s=10)
plt.suptitle("Regression Tree")
plt.title("MSE: %.2f" % mse, fontsize=10)
plt.xlabel('Day')
plt.ylabel('Temperature in Celcius')
plt.legend((m1, m2, m3), ("Training data", "Test data", "Prediction"),
loc='lower right')
plt.show()
def main():
print("-- Gradient Boosting Regression --")
# Load temperature data
data = pd.read_csv('mlfromscratch/data/TempLinkoping2016.txt', sep="\t")
time = np.atleast_2d(data["time"].as_matrix()).T
temp = np.atleast_2d(data["temp"].as_matrix()).T
X = time.reshape((-1, 1)) # Time. Fraction of the year [0, 1]
X = np.insert(X, 0, values=1, axis=1) # Insert bias term
y = temp[:, 0] # Temperature. Reduce to one-dim
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5)
clf = GradientBoostingRegressor(debug=True)
clf.fit(X_train, y_train)
y_pred = clf.predict(X_test)
y_pred_line = clf.predict(X)
# Color map
cmap = plt.get_cmap('viridis')
mse = mean_squared_error(y_test, y_pred)
print("Mean Squared Error:", mse)
# Plot the results
m1 = plt.scatter(366 * X_train[:, 1], y_train, color=cmap(0.9), s=10)
m2 = plt.scatter(366 * X_test[:, 1], y_test, color=cmap(0.5), s=10)
m3 = plt.scatter(366 * X_test[:, 1], y_pred, color='black', s=10)
plt.suptitle("Regression Tree")
plt.title("MSE: %.2f" % mse, fontsize=10)
plt.xlabel('Day')
plt.ylabel('Temperature in Celcius')
plt.legend((m1, m2, m3), ("Training data", "Test data", "Prediction"),
loc='lower right')
plt.show()
def main():
print ("-- Regression Tree --")
# Load temperature data
data = pd.read_csv('mlfromscratch/data/TempLinkoping2016.txt', sep="\t")
time = np.atleast_2d(data["time"].values).T
temp = np.atleast_2d(data["temp"].values).T
X = standardize(time) # Time. Fraction of the year [0, 1]
y = temp[:, 0] # Temperature. Reduce to one-dim
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)
model = RegressionTree()
model.fit(X_train, y_train)
y_pred = model.predict(X_test)
y_pred_line = model.predict(X)
# Color map
cmap = plt.get_cmap('viridis')
mse = mean_squared_error(y_test, y_pred)
print ("Mean Squared Error:", mse)
# Plot the results
# Plot the results
m1 = plt.scatter(366 * X_train, y_train, color=cmap(0.9), s=10)
m2 = plt.scatter(366 * X_test, y_test, color=cmap(0.5), s=10)
m3 = plt.scatter(366 * X_test, y_pred, color='black', s=10)
plt.suptitle("Regression Tree")
plt.title("MSE: %.2f" % mse, fontsize=10)
plt.xlabel('Day')
plt.ylabel('Temperature in Celcius')
plt.legend((m1, m2, m3), ("Training data", "Test data", "Prediction"), loc='lower right')
plt.show()
def main():
# Load temperature data
data = pd.read_csv('mlfromscratch/data/TempLinkoping2016.txt', sep="\t")
time = np.atleast_2d(data["time"].values).T
temp = data["temp"].values
X = time # fraction of the year [0, 1]
y = temp
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4)
poly_degree = 15
# Finding regularization constant using cross validation
lowest_error = float("inf")
best_reg_factor = 0
print("Finding regularization constant using cross validation:")
k = 10
for reg_factor in np.arange(0, 0.1, 0.01):
cross_validation_sets = k_fold_cross_validation_sets(X_train,
y_train,
k=k)
mse = 0
for _X_train, _X_test, _y_train, _y_test in cross_validation_sets:
model = PolynomialRidgeRegression(degree=poly_degree,
reg_factor=reg_factor,
learning_rate=0.001,
n_iterations=10000)
model.fit(_X_train, _y_train)
y_pred = model.predict(_X_test)
_mse = mean_squared_error(_y_test, y_pred)
mse += _mse
mse /= k
# Print the mean squared error
print("\tMean Squared Error: %s (regularization: %s)" %
(mse, reg_factor))
# Save reg. constant that gave lowest error
if mse < lowest_error:
best_reg_factor = reg_factor
lowest_error = mse
# Make final prediction
model = PolynomialRidgeRegression(degree=poly_degree,
reg_factor=best_reg_factor,
learning_rate=0.001,
n_iterations=10000)
model.fit(X_train, y_train)
y_pred = model.predict(X_test)
mse = mean_squared_error(y_test, y_pred)
print("Mean squared error: %s (given by reg. factor: %s)" %
(mse, best_reg_factor))
y_pred_line = model.predict(X)
# Color map
cmap = plt.get_cmap('viridis')
# Plot the results
m1 = plt.scatter(366 * X_train, y_train, color=cmap(0.9), s=10)
m2 = plt.scatter(366 * X_test, y_test, color=cmap(0.5), s=10)
plt.plot(366 * X,
y_pred_line,
color='black',
linewidth=2,
label="Prediction")
plt.suptitle("Polynomial Ridge Regression")
plt.title("MSE: %.2f" % mse, fontsize=10)
plt.xlabel('Day')
plt.ylabel('Temperature in Celcius')
plt.legend((m1, m2), ("Training data", "Test data"), loc='lower right')
plt.show()
time_X = np.atleast_2d(data["day"].values).T
temp = data["temperature"].values
X = time_X # fraction of the year [0, 1]
y = temp
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=.4,
random_state=10)
# Since the data is shuffled by train_test_split, we sort the test data to plot it neatly
y_test = y_test[X_test.flatten().argsort()]
X_test = np.sort(X_test, axis=0)
poly = PolynomialFeatures(degree=5, include_bias=False)
poly_X_train = poly.fit_transform(X_train)
poly_X_test = poly.fit_transform(X_test)
model = LinearRegression()
model.fit(poly_X_train, y_train)
y_pred = model.predict(poly_X_test)
mse = mean_squared_error(y_pred, y_test)
print("Mean Squared Error: ", mse)
plt.scatter(X_train, y_train, c='g', s=20)
plt.scatter(X_test, y_test, c='b', s=20)
plt.plot(X_test, y_pred, linewidth=2, c='r')
plt.legend()
plt.show()
def main():
# Load temperature data
data = pd.read_csv('mlfromscratch/data/TempLinkoping2016.txt', sep="\t")
time = np.atleast_2d(data["time"].as_matrix()).T
temp = data["temp"].as_matrix()
X = time # fraction of the year [0, 1]
y = temp
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4)
poly_degree = 15
# Finding regularization constant using cross validation
lowest_error = float("inf")
best_reg_factor = None
print ("Finding regularization constant using cross validation:")
k = 10
for reg_factor in np.arange(0, 0.1, 0.01):
cross_validation_sets = k_fold_cross_validation_sets(
X_train, y_train, k=k)
mse = 0
for _X_train, _X_test, _y_train, _y_test in cross_validation_sets:
model = PolynomialRidgeRegression(degree=poly_degree,
reg_factor=reg_factor,
learning_rate=0.001,
n_iterations=10000)
model.fit(_X_train, _y_train)
y_pred = model.predict(_X_test)
_mse = mean_squared_error(_y_test, y_pred)
mse += _mse
mse /= k
# Print the mean squared error
print ("\tMean Squared Error: %s (regularization: %s)" % (mse, reg_factor))
# Save reg. constant that gave lowest error
if mse < lowest_error:
best_reg_factor = reg_factor
lowest_error = mse
# Make final prediction
model = PolynomialRidgeRegression(degree=poly_degree,
reg_factor=best_reg_factor,
learning_rate=0.001,
n_iterations=10000)
model.fit(X_train, y_train)
y_pred = model.predict(X_test)
mse = mean_squared_error(y_test, y_pred)
print ("Mean squared error: %s (given by reg. factor: %s)" % (lowest_error, best_reg_factor))
y_pred_line = model.predict(X)
# Color map
cmap = plt.get_cmap('viridis')
# Plot the results
m1 = plt.scatter(366 * X_train, y_train, color=cmap(0.9), s=10)
m2 = plt.scatter(366 * X_test, y_test, color=cmap(0.5), s=10)
plt.plot(366 * X, y_pred_line, color='black', linewidth=2, label="Prediction")
plt.suptitle("Polynomial Ridge Regression")
plt.title("MSE: %.2f" % mse, fontsize=10)
plt.xlabel('Day')
plt.ylabel('Temperature in Celcius')
plt.legend((m1, m2), ("Training data", "Test data"), loc='lower right')
plt.show()
