def NNCostFunction(thetas, X, y):
    #取出神经网络层数
    K = thetas.size
    #取出样本数量
    m, feature = X.shape
    #给样本添加偏置项
    tempX = np.hstack((np.ones((m, 1)), X))
    #给theta添加偏置项
    tempthetas = np.empty(K, dtype='O')
    for i in np.arange(K):
        #theta添加偏置项
        innum, outnum = thetas[i].shape
        tempthetas[i] = np.vstack((np.ones((1, outnum)), thetas[i]))
    A = np.empty(K + 1, dtype='O')
    A[0] = X
    for k in np.arange(K):
        if k == 0:
            #第一层
            a = gd.sigmoid2(np.dot(tempX, tempthetas[k]))
        else:

            #添加偏置项
            a = np.hstack((np.ones((a.shape[0], 1)), a))
            a = gd.sigmoid2(np.dot(a, tempthetas[k]))
        #每层的计算值
        A[k + 1] = a

    #print(A)
    #计算误差
    #反向传播算法
    delta = np.empty(K, dtype='O')
    D = np.empty(K, dtype='O')
    for k in np.arange(K - 1, -1, -1, np.uint8):
        if k == K - 1:
            delta[k] = A[k + 1] - y
        else:
            delta[k] = np.dot(delta[k + 1],
                              thetas[k + 1].T) * gd.sigmoidgradient(
                                  np.dot(A[k], thetas[k]))
        #计算每个神经元所有样本的总偏导数
        D[k] = np.sum(delta[k] * A[k + 1], axis=0)
    #print("delta:%s"%delta)
    #print("D:%s"%D)
    #total=y*np.log10(A[K])+ (1-y)*np.log10(1-A[K])
    J = -(np.sum(y * np.log10(A[K]) + (1 - y) * np.log10(1 - A[K]))) / m
    D = D / m
    #返回每个神经元的偏导数和成本,进行梯度下降训练
    return D, J
    def findBestFitLine_using_gd(self, learning_rate=0.01):

        #Normalizing the inpurt before applying Gradient Descent
        x_temp_toScale = np.concatenate((self.X_train[:, 1:], self.Y_train), 1)
        normalScalar = NormalScalar.NormalScalar()
        normaled_val = normalScalar.fit_transform(x_temp_toScale)
        X_train = np.append([[1]] * len(normaled_val), normaled_val[:, :-1], 1)
        Y_train = normaled_val[:, -1:]

        #Applying Gradient Descent on cost funtion J
        (m, n) = X_train.shape
        J = lambda theta: ((X_train @ theta.reshape(n, 1) - Y_train).T @ (
            X_train @ theta.reshape(n, 1) - Y_train))[0][0] / (2 * m)
        gd = GradientDescent.GradientDescent()
        theta = gd.gradientDescent(J,
                                   np.array([0 for i in range(n)]),
                                   learning_rate=learning_rate,
                                   delta_val=0.000001,
                                   iterations=10000)

        # Inverse Scaling the value of theta
        theta[0] = normalScalar.std[-1] * (
            theta[0] -
            (sum(normalScalar.mean[:-1] *
                 (theta[1:] / normalScalar.std[:-1])))) + normalScalar.mean[-1]
        theta[1:] = (theta[1:] / normalScalar.std[:-1]) * normalScalar.std[-1]

        return theta
Example #3
0
def main():
    # ===============================导入数据===================================
    data = np.loadtxt('ex1data1.txt', delimiter=',', usecols=(0, 1))
    x = data[:, 0]
    y = data[:, 1]

    # ===============================数据准备===================================
    m = x.size
    theta = np.zeros((2, 1))
    interations = 3000
    alpha = 0.01
    X = np.c_[np.ones((m, 1)), x]
    Y = y[:, np.newaxis]

    # ===============================梯度下降算法 ===============================
    theta = gd.gradient_descent(X, Y, theta, alpha, interations)
    print('theta0的结果是', theta[0], ',', 'theta1的结果是', theta[1])

    # ===============================画图示意====================================
    plt.figure(0)
    plt.xlabel('Population of City in 10,000s')
    plt.ylabel('Profit in $10,000s')
    plt.scatter(x, y)
    t = np.arange(0, 23, 0.01)
    z = theta[0] + theta[1] * t
    plt.plot(t, z, 'r')
    # plt.show()
    # plt.pause(1)
    # plt.close()
    # t=np.arange()

    # ================================计算代价====================================
    # theta = np.array((0, 0))
    cost = CostFunction(x, y, theta)
    print('最小代价是', cost)

    # =============================将代价函数可视化================================
    theta_0 = np.linspace(-10, 10, 100)
    theta_1 = np.linspace(-1, 4, 100)
    m1 = theta_0.size
    m2 = theta_1.size
    cost = np.zeros((m1, m2))
    theta_0, theta_1 = np.meshgrid(theta_0, theta_1)
    for i in range(m1):
        for j in range(m2):
            theta2 = np.array([[theta_0[i, j]], [theta_1[i, j]]])
            cost[i, j] = CostFunction(x, y, theta2)
    fig = plt.figure()
    ax = Axes3D(fig)
    ax.plot_surface(theta_0, theta_1, cost)
    plt.figure()
    plt.contour(theta_0, theta_1, cost, levels=20)
    plt.scatter(theta[0], theta[1], c='r', marker='x')
    # plt.show()
    plt.pause(10)
    plt.close()
 def fitByGradintDescent(self,
                         init: np.array,
                         maxloop: int = 10000000,
                         sep: float = 0.00001,
                         eps: float = 1e-10):
     import GradientDescent
     gradientdescent = GradientDescent.GradientDescent(
         self.GetLossValue, self.DLossFun)
     gradientdescent.search(init=init, maxloop=maxloop, sep=sep, eps=eps)
     self.theta_ = gradientdescent.ans_
    def _getGrad(self):
        k = len(self.X[0])
        # w0 = np.zeros(k).reshape(-1, 1)
        w0 = np.random.uniform(-1, 1, k).reshape(-1, 1)
        
        # difffunc = lambda w: np.add(np.subtract(self.X.T.dot(self.X).dot(w), self.X.T.dot(self.t)), 0.5*self.lamb*np.sign(w))
        difffunc = lambda w: -2.0*(self.X.T).dot(self.t) + 2.0*(self.X.T).dot(self.X).dot(w) + self.lamb*np.sign(w)

        func = lambda w: ((self.t - self.X.dot(w)).T).dot(self.t - self.X.dot(w)) + self.lamb*np.sum(np.absolute(w))

        return gd.GradientDescent(self.grad_eta, func, difffunc, k, x0 = w0, eps = self.grad_eps, Nsteps = self.grad_steps, ylim = self.grad_ylim)
 def fitByStochasticGradintDescent(self,
                                   init_theta: np.array,
                                   t1: float = 50.0,
                                   t0: float = 5,
                                   n_iters: int = 1000):
     import GradientDescent
     stdgradescent = GradientDescent.StochasticGradientDescent(
         getFunValue=self.GetLossValue, DFun=self.StochasticDLossFun)
     stdgradescent.search(length=len(self.__X),
                          init_theta=init_theta,
                          t1=t1,
                          t0=t0,
                          n_iters=n_iters)
     self.theta_ = stdgradescent.ans_
Example #7
0
def least_mean_squares(example_param, batch_size, iterations,
                       learning_constant):

    if isinstance(example_param, str):
        examples = data_parsing(example_param)
    elif isinstance(example_param, list):
        examples = example_param
    else:
        raise AttributeError(
            "Invalid data type: Please pass either file path or list of examples to build tree."
        )

    labels = [ex[LABEL_INDEX] for ex in examples]
    x = [ex[0:len(ex) - 1] for ex in examples]

    return GradientDescent.gradient_descent(x, labels, [0] * (len(x[0])),
                                            batch_size, iterations,
                                            learning_constant)
Example #8
0
 def est_k_s(self, threshold):
     #0 idx is shape, or -k
     #2 idx is scale, or a or sigma
     if self.OptMethod == "gd":
         # version of GradientDescent
         theta = GradientDescent.GradientDescent(X=self.X,
                                                 step=0.0001,
                                                 acc=10e-06,
                                                 maxIter=1000,
                                                 showdetail=False,
                                                 threshold=threshold)
     elif self.OptMethod == "pso":
         # version of PSO
         theta = PSOptim.OptByPSO(
             X=self.X, threshold=threshold
         )  # if we use this, we need to install pyswarm package or put its code in the same filedir
     shape = np.sum(np.log(1 - theta * self.X)) / self.lenX()
     scale = -1 * shape / theta
     return (shape, scale)
Example #9
0
def main():
    ## linux machine
    # df = load_data("../../data/baby-weights-dataset.csv")
    ## windows machine
    df = load_data(
        "C:\\Users\\51606\\Desktop\\dstoolkit\\data\\baby-weights-dataset.csv")
    print("Data has been loaded!")
    scaler = StandardScaler()
    X = scaler.fit_transform(df.drop("BWEIGHT", axis=1))
    X = np.insert(X, 0, 1, axis=1)
    y = df["BWEIGHT"]
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)
    reg = gd.GradientDescent(method="sgd")
    print(reg.algorithm)
    print("***Training the model...***")
    reg.fit(X_train, y_train)
    print("***Training done!***")
    print(reg.weights)
    print(f"The number of iterations to stop: {reg.num_iter}")
    print(f"The corresponding training MSE: {reg.mse}")
    y_pred = reg.predict(X_test)
    print(f"The testing MSE: {mse(y_test,y_pred)}")
Example #10
0
        plt.title(title)
    plt.pause(0.001)


plt.figure()
imshow(style_img, title='Style Image')

plt.figure()
imshow(content_img, title='Content Image')

cnn = models.vgg19(pretrained=True).features.to(device).eval()
cnn_normalization_mean = torch.tensor([0.485, 0.456, 0.406]).to(device)
cnn_normalization_std = torch.tensor([0.229, 0.224, 0.225]).to(device)

input_img = content_img.clone()
# if you want to use white noise instead uncomment the below line:
# input_img = torch.randn(content_img.data.size(), device=device)

plt.figure()
imshow(input_img, title='Input Image')

output = gd.run_style_transfer(device, cnn, cnn_normalization_mean,
                               cnn_normalization_std, content_img, style_img,
                               input_img, 200, 9000000)

plt.figure()
imshow(output, title='Output Image')

plt.ioff()
plt.show()
Example #11
0
import GradientDescent as gd

# Parameters
learning_rate = 0.5
n = 12
theta= -2
thetaValues =[]
costValues=[]

# Training Data
train_X = numpy.asarray([1, 2, 3, 4, 5 , 6])
train_Y = numpy.asarray([1, 2, 3, 4, 5 , 6])

for index in range(n):
    predictedY = train_X * theta;
    cost = gd.compute_error_for_given_points(theta,0,train_X,train_Y)
    thetaValues.append(theta)
    costValues.append(cost)
    theta = theta + learning_rate;

minCost = numpy.min(costValues) 
print("minimum Cost =",minCost)

minCostIndex = costValues.index(minCost)
thetaFinal = thetaValues[minCostIndex]
print("Final Theta value =",thetaFinal)

#gradient descent
m,b,mvalues,bvalues,cValues = gd.gradient_descent_runner(train_X,train_Y, -2, 0, 0.005, 1000)

print(m)
Example #12
0
# Test Set
X_test = data_set_test
X_test = (X_test - X_test.mean()) / X_test.std()
X_test.insert(0, "Intercept", 1)
X_test = np.matrix(X_test)

# Initial Thetas
theta = np.matrix(np.zeros(shape=X.shape[1]))

# Parameters
learning_rate = 0.01
iteration = 500

print("\nRunning Linear Regression On Whole Set")
result = gd.gradient_descent(X, y, theta, learning_rate, iteration)
gd.plot_graph(iteration, result[1])

final_predictions = X.dot(result[0].T)
mae = gd.mean_absolute_error(final_predictions, y)
print("Mean Absolute Error: {0}".format(mae))

print("\nRunning Linear Regression On Split Sets")
splits = cv.cross_validation_split(data_set, 5)
cv.perform_gradient_on_splits(splits, learning_rate, iteration)

prediction_of_test_set = X_test.dot(result[0].T)
prediction_df = pd.DataFrame(prediction_of_test_set)
prediction_df.columns = ['SalePrice']

df_submission = pd.concat([data_test['Id'], prediction_df], axis=1)
 def train(self, X, y):
     X = [[1] + x for x in X]
     self.beta = gd.maximize_batch(partial(logistic_log_likelihood, X, y), partial(gradient, X, y), start=[0.0 for _ in X[0]]) 
Example #14
0
plt.title('')
#plt.ion()#Turn interactive mode on.
plt.legend()#显示图标


# ===================== Part 2: Gradient descent =====================
print('Running Gradient Descent...')

X=np.c_[np.ones(m),X] #y=theta1*x1+heta2*x2
alpha=0.01 #learning rate
iterations = 1500
theta=np.zeros(2) #theta1,theta2初始化为0
J_history=np.zeros(iterations)

print(theta.shape,J_history.shape)
theta , J_history=GD.gradientdescent(X,Y,theta,alpha,iterations)
print(theta,J_history,theta.shape)
X1=np.array(np.arange(0,20,0.01))

Y1=theta[0]+X1*theta[1]
plt.plot(X1, Y1, 'r-', label='line',linewidth=1)
plt.show()

input('Program paused. Press ENTER to continue')

# ============= Part 4: Visualizing J(theta_0, theta_1) =============
x_theta0=np.linspace(-10,10,100)
y_theta0=np.linspace(-1,4,100)

# 对x、y数据执行网格化
xx,yy=np.meshgrid(x_theta0,y_theta0)
Example #15
0

# Rosenbrock
def f(x, a=1, b=5):
    y = (a - x[0])**2 + b * (x[1] - x[0]**2)**2
    return y


x = sp.IndexedBase('x')
gradients = np.array([sp.diff(f(x), x[i]) for i in range(2)])
grads = sp.lambdify(x, gradients, 'numpy')

x_ = np.array([-2, 2])
alpha = 1E-2

gd.GradientDescent(f, grads, x_, alpha)

gd.ConjugateGradient(f, grads, x_)

############################################
""" 문제해결형 과제 (2) """
############################################
""" Branin Function """


def draw_branin(levels):

    a = 1
    b = 5.1 / (4 * np.pi**2)
    c = 5 / np.pi
    r = 6
Example #16
0
import DataInit, AddBias, GradientDescent, Normalization, runMachine
import os.path
import numpy as np

BASE = os.path.dirname(os.path.abspath(__file__))

print(os.path.join(BASE, "DataNew1.csv"))
Path = os.path.join(BASE, "DataNew1.csv")

# Path = "..\MachineLearningMF\Data.txt"
'''somethin somthin'''
data = DataInit.DataInit()
CostFuntion = runMachine.runMachine()
GDescent = GradientDescent.GradientDescent()
Norm = Normalization.Normalization()
AddBias1 = AddBias.AddBias()
'''loac csv'''
data.loader(Path)  # ,제거
'''theta init'''
'''initiated optimal theta 17/9/2019'''
# 동 별 theta값 입력 테스트
# theta = np.array([[theta0], [theta1], [theta2], [theta3], [theta4]])
theta = np.array([[0], [0], [0], [0], [0]])
'''Normalize'''
data.x, mu, sigma = Norm.featureNormalize(data)
'''Add Bias Column'''
data = AddBias1.addB(data)
'''remove annotations when you compute theta again'''
'''run Gradient descent and Cost function'''
theta = GDescent.runGradient(data, theta, 0.0001, 100000)  #theta값 뽑아내기
theta = theta.reshape(1, 5)  #행렬 곱셈하기 위해 형태바꾸기
 def train(self, X, y):
     X = [[1] + x for x in X]
     self.beta = gd.maximize_batch(partial(logistic_log_likelihood, X, y),
                                   partial(gradient, X, y),
                                   start=[0.0 for _ in X[0]])
Example #18
0
print("Normalizing features... ")

# scale features and set them to zero mean
X, mu, sigma = normalizeFeatures(X)

# add intercept term to X
X = np.append(np.ones((m, 1)), X, 1)

# Gradient Descent with multiple variables

alpha = 0.6
num_iters = 20

# initialize theta and run Gradient Descent
theta = zeros((num_features, 1))
theta, J_history = gd.performGradientDescent(X, y, theta, alpha, num_iters)

plt.plot(range(len(J_history)), J_history)
plt.xlabel("Number of iterations")
plt.ylabel("Cost J")
plt.show()

# display Gradient descent's result
print("Theta computed from gradient descent:")
print(theta.A1)

# estimate the price of a 1650 sq-ft, 3br house
# first columns is all-ones, so it doesn't need to be normalized
X1 = np.append(np.matrix([1]), (np.matrix([1650, 3]) - mu) / sigma, 1)
print("Predicted price of a 1650 sq-ft, 3 br house (using gradient descent):")
print("\t$", (X1 * theta).A1)
    thetas = np.array([theta1, theta2])
    X = np.array([[0.4, 0.5, 0.6], [0.1, 0.2, 0.3]]).reshape((2, 3))
    y = np.array([[0, 1], [1, 0]]).reshape((2, 2))

    images = ld.loadImage()
    labels = ld.loadLabel()

    m = len(labels)

    X_train = images.reshape(m, 784)
    eye = np.eye(10)

    y_train = np.empty((m, 10))
    for i in np.arange(m):
        y_train[i] = eye[labels[i]]
    #均值归一化
    X_train = (X_train - np.mean(X_train, axis=0)) / 255

    tempX = X_train[0:100]
    tempy = y_train[0:100]

    theta1 = np.random.random((784, 785)) / 100
    theta2 = np.random.random((785, 10)) / 100
    thetas = np.array([theta1, theta2])

    grad, J = NNCostFunction(thetas, tempX, tempy)
    print("J:%f" % J)
    v = gd.nngradientDescent(tempX, tempy, thetas, grad)

    #print('1');
Example #20
0
def RunLogRegressGradientDescent(x, y):
    learningRate = 0.1
    initCoefEstimate = np.ones([1, x.shape[1]])
    [errors, coefs] = gd.GradientDescent(ComputeGradient, x, y, learningRate,
                                         initCoefEstimate)
    return coefs
resultsPTest = p.test(testInput, testOutput)

print("Perceptron accuracy for train set : " + str(np.sum([x[0] for x in resultsPTrain[0]])/np.sum([x[1] for x in resultsPTrain[0]])))
print("Perceptron accuracy for test set : " + str(np.sum([x[0] for x in resultsPTest[0]])/np.sum([x[1] for x in resultsPTest[0]])))

IO.PlotCM(resultsPTrain[1], save = True, fileName = "perceptronConfusionTrain")
IO.PlotCM(resultsPTest[1], save = True, fileName = "perceptronConfusionTest")

'''
Assignment 5:
'''

'''
The different activation functions are in commented out lines in the module GradientDescent.py
'''
gd = GD.GradientDescent()

weights = np.random.rand(9)
weights = weights * 2 - 1   

learningRate = 0.5

mseplot = []
accuracyplot = []

for i in range(1000):
    weights = weights - learningRate * gd.grdmse(weights)
    mseplot.append(gd.mse(weights))
    
    a = np.abs(np.around(gd.xor_net(0,0,weights)) - 0)
    b = np.abs(np.around(gd.xor_net(0,1,weights)) - 1)
def RunLinRegressGradientDescent(x, y):
    learningRate = 0.02
    initCoefEstimate = np.array([1.0, 1.0])
    coefs = gd.GradientDescent(ComputeGradient, x, y, learningRate,
                               initCoefEstimate)
    gp.Plot2DResults('Gradient Descent', x, y, coefs)
Example #23
0
    def test(self):

        v1 = [1, 5, 8]
        v2 = [3, 6, 7]

        self.assertEqual(round(gd.distance(v1, v2), 5), 2.44949)
Example #24
0
import GradientDescent as gd


# Rosenbrock
def f(x, a=1, b=100):
    y = (a - x[0])**2 + b * (x[1] - x[0]**2)**2
    return y


x = sp.IndexedBase('x')
gradients = np.array([sp.diff(f(x), x[i]) for i in range(2)])
grads = sp.lambdify(x, gradients, 'numpy')

x_ = np.array([-2., 2.])

gd.GradientDescent(f, grads, x_, alpha=1E-1, verbose=True)

gd.ConjugateGradient(f, grads, x_)

gd.momentum(f, grads, x_, alpha=7E-4, verbose=True)

gd.nesterov(f, grads, x_, alpha=7E-4, verbose=True)
""" Huge-variate : e.g. Deep Learning """
gd.adagrad(f, grads, x_, alpha=3.05E-0, verbose=True)
""" RMSProp: Geoffrey Hinton """


def rmsprop(f,
            grads,
            x,
            alpha,
    return y


x = sp.IndexedBase('x')
gradients = np.array([sp.diff(f(x), x[i]) for i in range(2)])
grads = sp.lambdify(x, gradients, 'numpy')

#### Branin fuction min value 4 ####

# Gradient Descent | ConjugateGradient

alpha = 1E-2  # learning rate

x_ = np.array([0., 15.])
gd.GradientDescent(f, grads, x_, alpha)
gd.ConjugateGradient(f, grads, x_, verbose=False)

x_ = np.array([5., 5.])
gd.GradientDescent(f, grads, x_, alpha)
gd.ConjugateGradient(f, grads, x_, verbose=False)

x_ = np.array([13., 5.])
gd.GradientDescent(f, grads, x_, alpha)
gd.ConjugateGradient(f, grads, x_, verbose=False)

x_ = np.array([20., 17.])
gd.GradientDescent(f, grads, x_, alpha)
gd.ConjugateGradient(f, grads, x_, verbose=False)  # error

# Momentum
    # This is required so that the changes are picked up properly by the Jupyter notebook
    importlib.reload(GrD)

    N_samples = 1000
    [X, Y] = randomSampleGenerator(N_samples)
    print(
        'Sample logistic function which is being predicted: y = 3 +5*x1 - 2*x2'
    )
    [X_train,
     Y_train] = [X[0:int(0.7 * N_samples), :], Y[0:int(0.7 * N_samples)]]
    [X_test, Y_test] = [
        X[int(0.7 * N_samples):N_samples, :], Y[int(0.7 * N_samples):N_samples]
    ]

    GD = GrD.GradientDescent(method="batch",
                             loss="logloss",
                             lr=0.001,
                             epochs=10000)
    print('\n\nGradient descent logloss: batch with no adaptive learning')
    coeff = GD.fit(X_train, Y_train)
    print('Optimized Coefficients:', coeff)
    YPredict = GD.predict(X_test)
    print('R squared score: ', GD.score(Y_test, YPredict))

    GD = GrD.GradientDescent(method="batch",
                             loss="logloss",
                             lr=0.001,
                             epochs=10000,
                             adaptive=True)
    print('\n\nGradient descent logloss: batch with adaptive learning')
    coeff = GD.fit(X_train, Y_train)
    print('Optimized Coefficients:', coeff)
Example #27
0
    def train(self,
              epochs=10,
              batch=128,
              lr=0.001,
              p_drop_conv=0.2,
              p_drop_hidden=0.5,
              saveModel=False,
              trainDirectory='Default',
              nExecutions=1,
              generateStadistics=False):
        print "\nLoading Data..."
        trX, trY, teX, teY = self.__loadData()

        for x in range(0, nExecutions):
            X = T.ftensor4()
            Y = T.fmatrix()
            n1, n2, n3, n4, noise_py_x = self.__model(X, self.w, self.w2,
                                                      self.w3, self.w4,
                                                      self.w_o, p_drop_conv,
                                                      p_drop_hidden)
            cost = T.mean(T.nnet.categorical_crossentropy(noise_py_x, Y))
            params = [self.w, self.w2, self.w3, self.w4, self.w_o]
            updates = GD.RMSprop(cost, params, lr)
            executeTrain = theano.function(inputs=[X, Y],
                                           outputs=cost,
                                           updates=updates,
                                           allow_input_downcast=True)
            l1, l2, l3, l4, py_x = self.__model(X, self.w, self.w2, self.w3,
                                                self.w4, self.w_o, 0., 0.)
            y_x = T.argmax(py_x, axis=1)
            predict = theano.function(inputs=[X],
                                      outputs=y_x,
                                      allow_input_downcast=True)

            self.__initializeWeights()
            createDirectory(trainingPath)
            path = trainingPath + str(trainDirectory) + "/"
            createDirectory(path)
            path = path + "Execution" + str(x) + "/"
            createDirectory(path)

            # Start train
            f = open(str(path) + 'Cost.txt', 'w')
            f.write('Cost (Execution' + str(x) + ')\n')
            f2 = open(str(path) + 'HitPercentage.txt', 'w')
            f2.write('Hit Percentage (Execution' + str(x) + ')\n')
            save = False
            m = 0
            maxMean = 0
            print "\nExecution " + str(x)
            print "---------------------------------------"
            for i in range(epochs):
                print " Iteration " + str(i) + ":"
                error = 0
                for start, end in zip(range(0, len(trX), batch),
                                      range(batch, len(trX), batch)):
                    error = executeTrain(trX[start:end], trY[start:end])
                mean = np.mean(np.argmax(teY, axis=1) == predict(teX))
                print "     Mean: " + str(mean)
                self.__sensitivity(teX, teY, predict)
                print "     Cost: " + str(error)
                if (mean < m):
                    save = True
                    maxMean = mean
                if saveModel and save and mean > maxMean:
                    ModelPersistence.saveModel(params,
                                               filePath=path + "best_model.np")
                    maxMean = mean
                m = mean
                f.write(str(error) + "\n")
                f.flush()
                f2.write(str(m) + "\n")
                f2.flush()
            f.close()
            f2.close()
            print

            if generateStadistics: self.__generateStadistics(path, teX, teY)

        # Plot all the executions together
        if (nExecutions > 1):
            costs = [(trainingPath + str(trainDirectory) + "/Execution" +
                      str(x) + "/Cost.txt") for x in range(nExecutions)]
            means = [(trainingPath + str(trainDirectory) + "/Execution" +
                      str(x) + "/HitPercentage.txt")
                     for x in range(nExecutions)]
            Graphics.plotLearningCurve(
                costs, "Costs", trainingPath + str(trainDirectory) + "/", True)
            Graphics.plotLearningCurve(
                means, "Hit Percentages",
                trainingPath + str(trainDirectory) + "/", True)
Example #28
0
# Plotting the data
plotData(X, y)
plt.show()

# Gradient Descent
print("Running Gradient Descent ...")

X = np.append(np.ones((m, 1)), X, 1)  # add a column of ones to X
theta = np.zeros((2, 1))  # initialize fitting parameters

# some gradient descent settings
iterations = 1500
alpha = 0.01

# compute and display initial cost (expected: 32.0727338775)
cost = gd.computeCost(X, y, theta)
print("initial cost: ", cost)

# run gradient descent
(theta, history) = gd.performGradientDescent(X, y, theta, alpha, iterations)

# print theta to screen
print("theta found by gradient descent: ", theta.A1)

# plot original data (plt.hold(True) doesn't seem to work?...)
plotData(X[:, 1], y)
# plot linear fit from gradient descent
plt.plot(X[:, 1], X * theta, '-')
plt.legend(["Training data", "Linear regression"])
plt.show()
Example #29
0
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
df = pd.read_csv(
    'https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data',
    header=None)

y = df.iloc[0:100, 4].values
y = np.where(y == 'Iris-setosa', -1, 1)

X = df.iloc[0:100, [0, 2]].values

X_std = np.copy(X)
X_std[:, 0] = (X[:, 0] - X[:, 0].mean()) / X[:, 0].std()
X_std[:, 1] = (X[:, 1] - X[:, 1].mean()) / X[:, 1].std()

ada = gd.AdalineGD(n_iter=15, eta=0.01)
ada.fit(X_std, y)

pl.plot_decision_regions(X_std, y, classifier=ada)
plt.title('Gradient Descent')
plt.xlabel('sepal length')
plt.ylabel('petal length')
plt.legend(loc='upper left')

plt.show()
plt.plot(range(1, len(ada.cost_) + 1), ada.cost_, marker='o')
plt.xlabel('iterations')
plt.ylabel('Squared Error Sum')
plt.show()
Example #30
0
# Plotting the data
plotData(X, y)
plt.show()

# Gradient Descent
print("Running Gradient Descent ...")

X = np.append(np.ones((m, 1)), X, 1) # add a column of ones to X
theta = np.zeros((2, 1)) # initialize fitting parameters

# some gradient descent settings
iterations = 1500
alpha = 0.01

# compute and display initial cost (expected: 32.0727338775)
cost = gd.computeCost(X, y, theta)
print("initial cost: ", cost)

# run gradient descent
(theta, history) = gd.performGradientDescent(X, y, theta, alpha, iterations)

# print theta to screen
print("theta found by gradient descent: ", theta.A1)

# plot original data (plt.hold(True) doesn't seem to work?...)
plotData(X[:, 1], y)
# plot linear fit from gradient descent
plt.plot(X[:, 1], X * theta, '-')
plt.legend(["Training data", "Linear regression"])
plt.show()
Example #31
0
import numpy as np
import GradientDescent as GD
import matplotlib.pylab as plt
from scipy import linalg
#first column is the size of the house (in square feet), second column is the number of bedroom
data = np.loadtxt('ex1data2.txt', delimiter=',')
x = data[:, 0:2]  #m*2
y = data[:, 2]  #m*1
m = y.size

x_normalize, x_std, x_mean = GD.Feature_normalize(x)
alpha = 0.01
num_iters = 400

# Init theta and Run Gradient Descent
theta = np.zeros(3)
x_normalize = np.c_[np.ones(m), x_normalize]
theta1, j_history1 = GD.gradientdescent_mutli(x_normalize, y, alpha, theta,
                                              num_iters)
theta, j_history2 = GD.gradientdescent_mutli(x_normalize, y, 0.02, theta,
                                             num_iters)
theta, j_history3 = GD.gradientdescent_mutli(x_normalize, y, 0.015, theta,
                                             num_iters)

#print(theta)

num_iteration_x = np.array(range(num_iters))

#Convergence of gradient descent with an appropriate learning rate
#in different learning rate
Example #32
0
m = 10000
s = 0
W_perfect = numpy.matrix([2, 1])
b_perfect = float(1.5)

X1_input = numpy.matrix(numpy.random.uniform(-100, 100, (m, 1)))
X2_input = numpy.array(X1_input) * numpy.array(X1_input)
X_input = numpy.append(X1_input, X2_input, axis=1)
y_input = numpy.matrix(
    numpy.random.normal((X_input @ W_perfect.T) + b_perfect, s))

cost = Cost.MSE(X_input, y_input)
[W_gd, b_gd, nIter] = GradientDescent.Solve(cost,
                                            learning_rate=1e-1,
                                            max_iterations=1e8,
                                            epsilon=1e-16,
                                            debug_step=10000)
[W_as, b_as] = cost.AnalyticSolve()

print('**********************************************')
print('Input parameters:', W_perfect, b_perfect)
print('Gradient descent:', W_gd, b_gd, '(' + str(nIter) + ')')
print('Analytical      :', W_as, b_as)
print('**********************************************')

data = ''
for i in range(X_input.shape[0]):
    data += str(X_input[i, 0]) + ' ' + str(y_input[i, 0]) + '\n'
# end for
out_data = open('linear_regression_data.txt', 'w')