def NNCostFunction(thetas, X, y):
#取出神经网络层数
K = thetas.size
#取出样本数量
m, feature = X.shape
#给样本添加偏置项
tempX = np.hstack((np.ones((m, 1)), X))
#给theta添加偏置项
tempthetas = np.empty(K, dtype='O')
for i in np.arange(K):
#theta添加偏置项
innum, outnum = thetas[i].shape
tempthetas[i] = np.vstack((np.ones((1, outnum)), thetas[i]))
A = np.empty(K + 1, dtype='O')
A[0] = X
for k in np.arange(K):
if k == 0:
#第一层
a = gd.sigmoid2(np.dot(tempX, tempthetas[k]))
else:
#添加偏置项
a = np.hstack((np.ones((a.shape[0], 1)), a))
a = gd.sigmoid2(np.dot(a, tempthetas[k]))
#每层的计算值
A[k + 1] = a
#print(A)
#计算误差
#反向传播算法
delta = np.empty(K, dtype='O')
D = np.empty(K, dtype='O')
for k in np.arange(K - 1, -1, -1, np.uint8):
if k == K - 1:
delta[k] = A[k + 1] - y
else:
delta[k] = np.dot(delta[k + 1],
thetas[k + 1].T) * gd.sigmoidgradient(
np.dot(A[k], thetas[k]))
#计算每个神经元所有样本的总偏导数
D[k] = np.sum(delta[k] * A[k + 1], axis=0)
#print("delta:%s"%delta)
#print("D:%s"%D)
#total=y*np.log10(A[K])+ (1-y)*np.log10(1-A[K])
J = -(np.sum(y * np.log10(A[K]) + (1 - y) * np.log10(1 - A[K]))) / m
D = D / m
#返回每个神经元的偏导数和成本，进行梯度下降训练
return D, J
def findBestFitLine_using_gd(self, learning_rate=0.01):
#Normalizing the inpurt before applying Gradient Descent
x_temp_toScale = np.concatenate((self.X_train[:, 1:], self.Y_train), 1)
normalScalar = NormalScalar.NormalScalar()
normaled_val = normalScalar.fit_transform(x_temp_toScale)
X_train = np.append([[1]] * len(normaled_val), normaled_val[:, :-1], 1)
Y_train = normaled_val[:, -1:]
#Applying Gradient Descent on cost funtion J
(m, n) = X_train.shape
J = lambda theta: ((X_train @ theta.reshape(n, 1) - Y_train).T @ (
X_train @ theta.reshape(n, 1) - Y_train))[0][0] / (2 * m)
gd = GradientDescent.GradientDescent()
theta = gd.gradientDescent(J,
np.array([0 for i in range(n)]),
learning_rate=learning_rate,
delta_val=0.000001,
iterations=10000)
# Inverse Scaling the value of theta
theta[0] = normalScalar.std[-1] * (
theta[0] -
(sum(normalScalar.mean[:-1] *
(theta[1:] / normalScalar.std[:-1])))) + normalScalar.mean[-1]
theta[1:] = (theta[1:] / normalScalar.std[:-1]) * normalScalar.std[-1]
return theta
def main():
# ===============================导入数据===================================
data = np.loadtxt('ex1data1.txt', delimiter=',', usecols=(0, 1))
x = data[:, 0]
y = data[:, 1]
# ===============================数据准备===================================
m = x.size
theta = np.zeros((2, 1))
interations = 3000
alpha = 0.01
X = np.c_[np.ones((m, 1)), x]
Y = y[:, np.newaxis]
# ===============================梯度下降算法 ===============================
theta = gd.gradient_descent(X, Y, theta, alpha, interations)
print('theta0的结果是', theta[0], '，', 'theta1的结果是', theta[1])
# ===============================画图示意====================================
plt.figure(0)
plt.xlabel('Population of City in 10,000s')
plt.ylabel('Profit in $10,000s')
plt.scatter(x, y)
t = np.arange(0, 23, 0.01)
z = theta[0] + theta[1] * t
plt.plot(t, z, 'r')
# plt.show()
# plt.pause(1)
# plt.close()
# t=np.arange()
# ================================计算代价====================================
# theta = np.array((0, 0))
cost = CostFunction(x, y, theta)
print('最小代价是', cost)
# =============================将代价函数可视化================================
theta_0 = np.linspace(-10, 10, 100)
theta_1 = np.linspace(-1, 4, 100)
m1 = theta_0.size
m2 = theta_1.size
cost = np.zeros((m1, m2))
theta_0, theta_1 = np.meshgrid(theta_0, theta_1)
for i in range(m1):
for j in range(m2):
theta2 = np.array([[theta_0[i, j]], [theta_1[i, j]]])
cost[i, j] = CostFunction(x, y, theta2)
fig = plt.figure()
ax = Axes3D(fig)
ax.plot_surface(theta_0, theta_1, cost)
plt.figure()
plt.contour(theta_0, theta_1, cost, levels=20)
plt.scatter(theta[0], theta[1], c='r', marker='x')
# plt.show()
plt.pause(10)
plt.close()
def fitByGradintDescent(self,
init: np.array,
maxloop: int = 10000000,
sep: float = 0.00001,
eps: float = 1e-10):
import GradientDescent
gradientdescent = GradientDescent.GradientDescent(
self.GetLossValue, self.DLossFun)
gradientdescent.search(init=init, maxloop=maxloop, sep=sep, eps=eps)
self.theta_ = gradientdescent.ans_
def _getGrad(self):
k = len(self.X[0])
# w0 = np.zeros(k).reshape(-1, 1)
w0 = np.random.uniform(-1, 1, k).reshape(-1, 1)
# difffunc = lambda w: np.add(np.subtract(self.X.T.dot(self.X).dot(w), self.X.T.dot(self.t)), 0.5*self.lamb*np.sign(w))
difffunc = lambda w: -2.0*(self.X.T).dot(self.t) + 2.0*(self.X.T).dot(self.X).dot(w) + self.lamb*np.sign(w)
func = lambda w: ((self.t - self.X.dot(w)).T).dot(self.t - self.X.dot(w)) + self.lamb*np.sum(np.absolute(w))
return gd.GradientDescent(self.grad_eta, func, difffunc, k, x0 = w0, eps = self.grad_eps, Nsteps = self.grad_steps, ylim = self.grad_ylim)
def fitByStochasticGradintDescent(self,
init_theta: np.array,
t1: float = 50.0,
t0: float = 5,
n_iters: int = 1000):
import GradientDescent
stdgradescent = GradientDescent.StochasticGradientDescent(
getFunValue=self.GetLossValue, DFun=self.StochasticDLossFun)
stdgradescent.search(length=len(self.__X),
init_theta=init_theta,
t1=t1,
t0=t0,
n_iters=n_iters)
self.theta_ = stdgradescent.ans_
def least_mean_squares(example_param, batch_size, iterations,
learning_constant):
if isinstance(example_param, str):
examples = data_parsing(example_param)
elif isinstance(example_param, list):
examples = example_param
else:
raise AttributeError(
"Invalid data type: Please pass either file path or list of examples to build tree."
)
labels = [ex[LABEL_INDEX] for ex in examples]
x = [ex[0:len(ex) - 1] for ex in examples]
return GradientDescent.gradient_descent(x, labels, [0] * (len(x[0])),
batch_size, iterations,
learning_constant)
def est_k_s(self, threshold):
#0 idx is shape, or -k
#2 idx is scale, or a or sigma
if self.OptMethod == "gd":
# version of GradientDescent
theta = GradientDescent.GradientDescent(X=self.X,
step=0.0001,
acc=10e-06,
maxIter=1000,
showdetail=False,
threshold=threshold)
elif self.OptMethod == "pso":
# version of PSO
theta = PSOptim.OptByPSO(
X=self.X, threshold=threshold
) # if we use this, we need to install pyswarm package or put its code in the same filedir
shape = np.sum(np.log(1 - theta * self.X)) / self.lenX()
scale = -1 * shape / theta
return (shape, scale)
def main():
## linux machine
# df = load_data("../../data/baby-weights-dataset.csv")
## windows machine
df = load_data(
"C:\\Users\\51606\\Desktop\\dstoolkit\\data\\baby-weights-dataset.csv")
print("Data has been loaded!")
scaler = StandardScaler()
X = scaler.fit_transform(df.drop("BWEIGHT", axis=1))
X = np.insert(X, 0, 1, axis=1)
y = df["BWEIGHT"]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)
reg = gd.GradientDescent(method="sgd")
print(reg.algorithm)
print("***Training the model...***")
reg.fit(X_train, y_train)
print("***Training done!***")
print(reg.weights)
print(f"The number of iterations to stop: {reg.num_iter}")
print(f"The corresponding training MSE: {reg.mse}")
y_pred = reg.predict(X_test)
print(f"The testing MSE: {mse(y_test,y_pred)}")
plt.title(title)
plt.pause(0.001)
plt.figure()
imshow(style_img, title='Style Image')
plt.figure()
imshow(content_img, title='Content Image')
cnn = models.vgg19(pretrained=True).features.to(device).eval()
cnn_normalization_mean = torch.tensor([0.485, 0.456, 0.406]).to(device)
cnn_normalization_std = torch.tensor([0.229, 0.224, 0.225]).to(device)
input_img = content_img.clone()
# if you want to use white noise instead uncomment the below line:
# input_img = torch.randn(content_img.data.size(), device=device)
plt.figure()
imshow(input_img, title='Input Image')
output = gd.run_style_transfer(device, cnn, cnn_normalization_mean,
cnn_normalization_std, content_img, style_img,
input_img, 200, 9000000)
plt.figure()
imshow(output, title='Output Image')
plt.ioff()
plt.show()
import GradientDescent as gd
# Parameters
learning_rate = 0.5
n = 12
theta= -2
thetaValues =[]
costValues=[]
# Training Data
train_X = numpy.asarray([1, 2, 3, 4, 5 , 6])
train_Y = numpy.asarray([1, 2, 3, 4, 5 , 6])
for index in range(n):
predictedY = train_X * theta;
cost = gd.compute_error_for_given_points(theta,0,train_X,train_Y)
thetaValues.append(theta)
costValues.append(cost)
theta = theta + learning_rate;
minCost = numpy.min(costValues)
print("minimum Cost =",minCost)
minCostIndex = costValues.index(minCost)
thetaFinal = thetaValues[minCostIndex]
print("Final Theta value =",thetaFinal)
#gradient descent
m,b,mvalues,bvalues,cValues = gd.gradient_descent_runner(train_X,train_Y, -2, 0, 0.005, 1000)
print(m)
# Test Set
X_test = data_set_test
X_test = (X_test - X_test.mean()) / X_test.std()
X_test.insert(0, "Intercept", 1)
X_test = np.matrix(X_test)
# Initial Thetas
theta = np.matrix(np.zeros(shape=X.shape[1]))
# Parameters
learning_rate = 0.01
iteration = 500
print("\nRunning Linear Regression On Whole Set")
result = gd.gradient_descent(X, y, theta, learning_rate, iteration)
gd.plot_graph(iteration, result[1])
final_predictions = X.dot(result[0].T)
mae = gd.mean_absolute_error(final_predictions, y)
print("Mean Absolute Error: {0}".format(mae))
print("\nRunning Linear Regression On Split Sets")
splits = cv.cross_validation_split(data_set, 5)
cv.perform_gradient_on_splits(splits, learning_rate, iteration)
prediction_of_test_set = X_test.dot(result[0].T)
prediction_df = pd.DataFrame(prediction_of_test_set)
prediction_df.columns = ['SalePrice']
df_submission = pd.concat([data_test['Id'], prediction_df], axis=1)
def train(self, X, y):
X = [[1] + x for x in X]
self.beta = gd.maximize_batch(partial(logistic_log_likelihood, X, y), partial(gradient, X, y), start=[0.0 for _ in X[0]])
plt.title('')
#plt.ion()#Turn interactive mode on.
plt.legend()#显示图标
# ===================== Part 2: Gradient descent =====================
print('Running Gradient Descent...')
X=np.c_[np.ones(m),X] #y=theta1*x1+heta2*x2
alpha=0.01 #learning rate
iterations = 1500
theta=np.zeros(2) #theta1，theta2初始化为0
J_history=np.zeros(iterations)
print(theta.shape,J_history.shape)
theta , J_history=GD.gradientdescent(X,Y,theta,alpha,iterations)
print(theta,J_history,theta.shape)
X1=np.array(np.arange(0,20,0.01))
Y1=theta[0]+X1*theta[1]
plt.plot(X1, Y1, 'r-', label='line',linewidth=1)
plt.show()
input('Program paused. Press ENTER to continue')
# ============= Part 4: Visualizing J(theta_0, theta_1) =============
x_theta0=np.linspace(-10,10,100)
y_theta0=np.linspace(-1,4,100)
# 对x、y数据执行网格化
xx,yy=np.meshgrid(x_theta0,y_theta0)
# Rosenbrock
def f(x, a=1, b=5):
y = (a - x[0])**2 + b * (x[1] - x[0]**2)**2
return y
x = sp.IndexedBase('x')
gradients = np.array([sp.diff(f(x), x[i]) for i in range(2)])
grads = sp.lambdify(x, gradients, 'numpy')
x_ = np.array([-2, 2])
alpha = 1E-2
gd.GradientDescent(f, grads, x_, alpha)
gd.ConjugateGradient(f, grads, x_)
############################################
""" 문제해결형 과제 (2) """
############################################
""" Branin Function """
def draw_branin(levels):
a = 1
b = 5.1 / (4 * np.pi**2)
c = 5 / np.pi
r = 6
import DataInit, AddBias, GradientDescent, Normalization, runMachine import os.path import numpy as np BASE = os.path.dirname(os.path.abspath(__file__)) print(os.path.join(BASE, "DataNew1.csv")) Path = os.path.join(BASE, "DataNew1.csv") # Path = "..\MachineLearningMF\Data.txt" '''somethin somthin''' data = DataInit.DataInit() CostFuntion = runMachine.runMachine() GDescent = GradientDescent.GradientDescent() Norm = Normalization.Normalization() AddBias1 = AddBias.AddBias() '''loac csv''' data.loader(Path) # ,제거 '''theta init''' '''initiated optimal theta 17/9/2019''' # 동 별 theta값 입력 테스트 # theta = np.array([[theta0], [theta1], [theta2], [theta3], [theta4]]) theta = np.array([[0], [0], [0], [0], [0]]) '''Normalize''' data.x, mu, sigma = Norm.featureNormalize(data) '''Add Bias Column''' data = AddBias1.addB(data) '''remove annotations when you compute theta again''' '''run Gradient descent and Cost function''' theta = GDescent.runGradient(data, theta, 0.0001, 100000) #theta값 뽑아내기 theta = theta.reshape(1, 5) #행렬 곱셈하기 위해 형태바꾸기
def train(self, X, y):
X = [[1] + x for x in X]
self.beta = gd.maximize_batch(partial(logistic_log_likelihood, X, y),
partial(gradient, X, y),
start=[0.0 for _ in X[0]])
print("Normalizing features... ")
# scale features and set them to zero mean
X, mu, sigma = normalizeFeatures(X)
# add intercept term to X
X = np.append(np.ones((m, 1)), X, 1)
# Gradient Descent with multiple variables
alpha = 0.6
num_iters = 20
# initialize theta and run Gradient Descent
theta = zeros((num_features, 1))
theta, J_history = gd.performGradientDescent(X, y, theta, alpha, num_iters)
plt.plot(range(len(J_history)), J_history)
plt.xlabel("Number of iterations")
plt.ylabel("Cost J")
plt.show()
# display Gradient descent's result
print("Theta computed from gradient descent:")
print(theta.A1)
# estimate the price of a 1650 sq-ft, 3br house
# first columns is all-ones, so it doesn't need to be normalized
X1 = np.append(np.matrix([1]), (np.matrix([1650, 3]) - mu) / sigma, 1)
print("Predicted price of a 1650 sq-ft, 3 br house (using gradient descent):")
print("\t$", (X1 * theta).A1)
thetas = np.array([theta1, theta2])
X = np.array([[0.4, 0.5, 0.6], [0.1, 0.2, 0.3]]).reshape((2, 3))
y = np.array([[0, 1], [1, 0]]).reshape((2, 2))
images = ld.loadImage()
labels = ld.loadLabel()
m = len(labels)
X_train = images.reshape(m, 784)
eye = np.eye(10)
y_train = np.empty((m, 10))
for i in np.arange(m):
y_train[i] = eye[labels[i]]
#均值归一化
X_train = (X_train - np.mean(X_train, axis=0)) / 255
tempX = X_train[0:100]
tempy = y_train[0:100]
theta1 = np.random.random((784, 785)) / 100
theta2 = np.random.random((785, 10)) / 100
thetas = np.array([theta1, theta2])
grad, J = NNCostFunction(thetas, tempX, tempy)
print("J:%f" % J)
v = gd.nngradientDescent(tempX, tempy, thetas, grad)
#print('1');
def RunLogRegressGradientDescent(x, y):
learningRate = 0.1
initCoefEstimate = np.ones([1, x.shape[1]])
[errors, coefs] = gd.GradientDescent(ComputeGradient, x, y, learningRate,
initCoefEstimate)
return coefs
resultsPTest = p.test(testInput, testOutput)
print("Perceptron accuracy for train set : " + str(np.sum([x[0] for x in resultsPTrain[0]])/np.sum([x[1] for x in resultsPTrain[0]])))
print("Perceptron accuracy for test set : " + str(np.sum([x[0] for x in resultsPTest[0]])/np.sum([x[1] for x in resultsPTest[0]])))
IO.PlotCM(resultsPTrain[1], save = True, fileName = "perceptronConfusionTrain")
IO.PlotCM(resultsPTest[1], save = True, fileName = "perceptronConfusionTest")
'''
Assignment 5:
'''
'''
The different activation functions are in commented out lines in the module GradientDescent.py
'''
gd = GD.GradientDescent()
weights = np.random.rand(9)
weights = weights * 2 - 1
learningRate = 0.5
mseplot = []
accuracyplot = []
for i in range(1000):
weights = weights - learningRate * gd.grdmse(weights)
mseplot.append(gd.mse(weights))
a = np.abs(np.around(gd.xor_net(0,0,weights)) - 0)
b = np.abs(np.around(gd.xor_net(0,1,weights)) - 1)
def RunLinRegressGradientDescent(x, y):
learningRate = 0.02
initCoefEstimate = np.array([1.0, 1.0])
coefs = gd.GradientDescent(ComputeGradient, x, y, learningRate,
initCoefEstimate)
gp.Plot2DResults('Gradient Descent', x, y, coefs)
def test(self):
v1 = [1, 5, 8]
v2 = [3, 6, 7]
self.assertEqual(round(gd.distance(v1, v2), 5), 2.44949)
import GradientDescent as gd
# Rosenbrock
def f(x, a=1, b=100):
y = (a - x[0])**2 + b * (x[1] - x[0]**2)**2
return y
x = sp.IndexedBase('x')
gradients = np.array([sp.diff(f(x), x[i]) for i in range(2)])
grads = sp.lambdify(x, gradients, 'numpy')
x_ = np.array([-2., 2.])
gd.GradientDescent(f, grads, x_, alpha=1E-1, verbose=True)
gd.ConjugateGradient(f, grads, x_)
gd.momentum(f, grads, x_, alpha=7E-4, verbose=True)
gd.nesterov(f, grads, x_, alpha=7E-4, verbose=True)
""" Huge-variate : e.g. Deep Learning """
gd.adagrad(f, grads, x_, alpha=3.05E-0, verbose=True)
""" RMSProp: Geoffrey Hinton """
def rmsprop(f,
grads,
x,
alpha,
return y
x = sp.IndexedBase('x')
gradients = np.array([sp.diff(f(x), x[i]) for i in range(2)])
grads = sp.lambdify(x, gradients, 'numpy')
#### Branin fuction min value 4 ####
# Gradient Descent | ConjugateGradient
alpha = 1E-2 # learning rate
x_ = np.array([0., 15.])
gd.GradientDescent(f, grads, x_, alpha)
gd.ConjugateGradient(f, grads, x_, verbose=False)
x_ = np.array([5., 5.])
gd.GradientDescent(f, grads, x_, alpha)
gd.ConjugateGradient(f, grads, x_, verbose=False)
x_ = np.array([13., 5.])
gd.GradientDescent(f, grads, x_, alpha)
gd.ConjugateGradient(f, grads, x_, verbose=False)
x_ = np.array([20., 17.])
gd.GradientDescent(f, grads, x_, alpha)
gd.ConjugateGradient(f, grads, x_, verbose=False) # error
# Momentum
# This is required so that the changes are picked up properly by the Jupyter notebook
importlib.reload(GrD)
N_samples = 1000
[X, Y] = randomSampleGenerator(N_samples)
print(
'Sample logistic function which is being predicted: y = 3 +5*x1 - 2*x2'
)
[X_train,
Y_train] = [X[0:int(0.7 * N_samples), :], Y[0:int(0.7 * N_samples)]]
[X_test, Y_test] = [
X[int(0.7 * N_samples):N_samples, :], Y[int(0.7 * N_samples):N_samples]
]
GD = GrD.GradientDescent(method="batch",
loss="logloss",
lr=0.001,
epochs=10000)
print('\n\nGradient descent logloss: batch with no adaptive learning')
coeff = GD.fit(X_train, Y_train)
print('Optimized Coefficients:', coeff)
YPredict = GD.predict(X_test)
print('R squared score: ', GD.score(Y_test, YPredict))
GD = GrD.GradientDescent(method="batch",
loss="logloss",
lr=0.001,
epochs=10000,
adaptive=True)
print('\n\nGradient descent logloss: batch with adaptive learning')
coeff = GD.fit(X_train, Y_train)
print('Optimized Coefficients:', coeff)
def train(self,
epochs=10,
batch=128,
lr=0.001,
p_drop_conv=0.2,
p_drop_hidden=0.5,
saveModel=False,
trainDirectory='Default',
nExecutions=1,
generateStadistics=False):
print "\nLoading Data..."
trX, trY, teX, teY = self.__loadData()
for x in range(0, nExecutions):
X = T.ftensor4()
Y = T.fmatrix()
n1, n2, n3, n4, noise_py_x = self.__model(X, self.w, self.w2,
self.w3, self.w4,
self.w_o, p_drop_conv,
p_drop_hidden)
cost = T.mean(T.nnet.categorical_crossentropy(noise_py_x, Y))
params = [self.w, self.w2, self.w3, self.w4, self.w_o]
updates = GD.RMSprop(cost, params, lr)
executeTrain = theano.function(inputs=[X, Y],
outputs=cost,
updates=updates,
allow_input_downcast=True)
l1, l2, l3, l4, py_x = self.__model(X, self.w, self.w2, self.w3,
self.w4, self.w_o, 0., 0.)
y_x = T.argmax(py_x, axis=1)
predict = theano.function(inputs=[X],
outputs=y_x,
allow_input_downcast=True)
self.__initializeWeights()
createDirectory(trainingPath)
path = trainingPath + str(trainDirectory) + "/"
createDirectory(path)
path = path + "Execution" + str(x) + "/"
createDirectory(path)
# Start train
f = open(str(path) + 'Cost.txt', 'w')
f.write('Cost (Execution' + str(x) + ')\n')
f2 = open(str(path) + 'HitPercentage.txt', 'w')
f2.write('Hit Percentage (Execution' + str(x) + ')\n')
save = False
m = 0
maxMean = 0
print "\nExecution " + str(x)
print "---------------------------------------"
for i in range(epochs):
print " Iteration " + str(i) + ":"
error = 0
for start, end in zip(range(0, len(trX), batch),
range(batch, len(trX), batch)):
error = executeTrain(trX[start:end], trY[start:end])
mean = np.mean(np.argmax(teY, axis=1) == predict(teX))
print " Mean: " + str(mean)
self.__sensitivity(teX, teY, predict)
print " Cost: " + str(error)
if (mean < m):
save = True
maxMean = mean
if saveModel and save and mean > maxMean:
ModelPersistence.saveModel(params,
filePath=path + "best_model.np")
maxMean = mean
m = mean
f.write(str(error) + "\n")
f.flush()
f2.write(str(m) + "\n")
f2.flush()
f.close()
f2.close()
print
if generateStadistics: self.__generateStadistics(path, teX, teY)
# Plot all the executions together
if (nExecutions > 1):
costs = [(trainingPath + str(trainDirectory) + "/Execution" +
str(x) + "/Cost.txt") for x in range(nExecutions)]
means = [(trainingPath + str(trainDirectory) + "/Execution" +
str(x) + "/HitPercentage.txt")
for x in range(nExecutions)]
Graphics.plotLearningCurve(
costs, "Costs", trainingPath + str(trainDirectory) + "/", True)
Graphics.plotLearningCurve(
means, "Hit Percentages",
trainingPath + str(trainDirectory) + "/", True)
# Plotting the data
plotData(X, y)
plt.show()
# Gradient Descent
print("Running Gradient Descent ...")
X = np.append(np.ones((m, 1)), X, 1) # add a column of ones to X
theta = np.zeros((2, 1)) # initialize fitting parameters
# some gradient descent settings
iterations = 1500
alpha = 0.01
# compute and display initial cost (expected: 32.0727338775)
cost = gd.computeCost(X, y, theta)
print("initial cost: ", cost)
# run gradient descent
(theta, history) = gd.performGradientDescent(X, y, theta, alpha, iterations)
# print theta to screen
print("theta found by gradient descent: ", theta.A1)
# plot original data (plt.hold(True) doesn't seem to work?...)
plotData(X[:, 1], y)
# plot linear fit from gradient descent
plt.plot(X[:, 1], X * theta, '-')
plt.legend(["Training data", "Linear regression"])
plt.show()
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
df = pd.read_csv(
'https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data',
header=None)
y = df.iloc[0:100, 4].values
y = np.where(y == 'Iris-setosa', -1, 1)
X = df.iloc[0:100, [0, 2]].values
X_std = np.copy(X)
X_std[:, 0] = (X[:, 0] - X[:, 0].mean()) / X[:, 0].std()
X_std[:, 1] = (X[:, 1] - X[:, 1].mean()) / X[:, 1].std()
ada = gd.AdalineGD(n_iter=15, eta=0.01)
ada.fit(X_std, y)
pl.plot_decision_regions(X_std, y, classifier=ada)
plt.title('Gradient Descent')
plt.xlabel('sepal length')
plt.ylabel('petal length')
plt.legend(loc='upper left')
plt.show()
plt.plot(range(1, len(ada.cost_) + 1), ada.cost_, marker='o')
plt.xlabel('iterations')
plt.ylabel('Squared Error Sum')
plt.show()
# Plotting the data
plotData(X, y)
plt.show()
# Gradient Descent
print("Running Gradient Descent ...")
X = np.append(np.ones((m, 1)), X, 1) # add a column of ones to X
theta = np.zeros((2, 1)) # initialize fitting parameters
# some gradient descent settings
iterations = 1500
alpha = 0.01
# compute and display initial cost (expected: 32.0727338775)
cost = gd.computeCost(X, y, theta)
print("initial cost: ", cost)
# run gradient descent
(theta, history) = gd.performGradientDescent(X, y, theta, alpha, iterations)
# print theta to screen
print("theta found by gradient descent: ", theta.A1)
# plot original data (plt.hold(True) doesn't seem to work?...)
plotData(X[:, 1], y)
# plot linear fit from gradient descent
plt.plot(X[:, 1], X * theta, '-')
plt.legend(["Training data", "Linear regression"])
plt.show()
import numpy as np
import GradientDescent as GD
import matplotlib.pylab as plt
from scipy import linalg
#first column is the size of the house (in square feet), second column is the number of bedroom
data = np.loadtxt('ex1data2.txt', delimiter=',')
x = data[:, 0:2] #m*2
y = data[:, 2] #m*1
m = y.size
x_normalize, x_std, x_mean = GD.Feature_normalize(x)
alpha = 0.01
num_iters = 400
# Init theta and Run Gradient Descent
theta = np.zeros(3)
x_normalize = np.c_[np.ones(m), x_normalize]
theta1, j_history1 = GD.gradientdescent_mutli(x_normalize, y, alpha, theta,
num_iters)
theta, j_history2 = GD.gradientdescent_mutli(x_normalize, y, 0.02, theta,
num_iters)
theta, j_history3 = GD.gradientdescent_mutli(x_normalize, y, 0.015, theta,
num_iters)
#print(theta)
num_iteration_x = np.array(range(num_iters))
#Convergence of gradient descent with an appropriate learning rate
#in different learning rate
m = 10000
s = 0
W_perfect = numpy.matrix([2, 1])
b_perfect = float(1.5)
X1_input = numpy.matrix(numpy.random.uniform(-100, 100, (m, 1)))
X2_input = numpy.array(X1_input) * numpy.array(X1_input)
X_input = numpy.append(X1_input, X2_input, axis=1)
y_input = numpy.matrix(
numpy.random.normal((X_input @ W_perfect.T) + b_perfect, s))
cost = Cost.MSE(X_input, y_input)
[W_gd, b_gd, nIter] = GradientDescent.Solve(cost,
learning_rate=1e-1,
max_iterations=1e8,
epsilon=1e-16,
debug_step=10000)
[W_as, b_as] = cost.AnalyticSolve()
print('**********************************************')
print('Input parameters:', W_perfect, b_perfect)
print('Gradient descent:', W_gd, b_gd, '(' + str(nIter) + ')')
print('Analytical :', W_as, b_as)
print('**********************************************')
data = ''
for i in range(X_input.shape[0]):
data += str(X_input[i, 0]) + ' ' + str(y_input[i, 0]) + '\n'
# end for
out_data = open('linear_regression_data.txt', 'w')
