def exponential_vel(self,t,c1,c2): v_num = c1*(c2+t)*math.tanh(1/2 * math.sqrt(c1*(c2+t)**2)) * math.sech(1/2 * math.sqrt(c1*(c2+t)**2)) v_den = math.sqrt(c1*(c2+t)**2) * (math.tanh(1/2 * math.sqrt(c1*(c2+t)**2))-1) v1 = v_num/v_den v2 = -math.sqrt(c1) * math.tanh(1/2 * math.sqrt(c1)*(c2+t)) return v1,v2
#sigmoidp = lambda x: math.exp(-x)/(1.0+math.exp(-x))**2 #sigmoidpp = lambda x: math.exp(-x)*(math.exp(-x)-1.0)/(1.0+math.exp(-x))**3 ap = 1.0 xp = 4.5 bp = 3.0 penalty = lambda x: ap * (0.5 * (math.tanh(bp * (x - xp)) - math.tanh(bp * (x + xp))) + 1.0) penaltyp = lambda x: ap * 0.5 * bp * ((1 / math.cosh(bp * (x - xp)))**2 - (1.0 / math.cosh(bp * (x + xp)))**2) sigmoid = lambda x: 0.5 * (math.tanh(beta * x) + 1.0) sigmoidp = lambda x: 0.5 * beta * (1.0 / math.cosh(beta * x))**2 sigmoidpp = lambda x: 0.5 * (-2.0) * beta * beta * math.tanh(beta * x) * ( 1.0 / math.sech(beta * x))**2 xmoid1 = lambda x: x xmoidp1 = lambda x: 1.0 xmoidpp1 = lambda x: 0.0 relu = lambda x: max(0.0, x) relup = lambda x: 0.0 if (x <= 0.0) else 1.0 relupp = lambda x: 0.0 #bias = [[0.01],[0.01],[0.001],[0.0001],[0.00001],[0.0000001]] bias = [[0.01], [0.01], [0.001]] bias = [] #machine = myfeedforward.MyFeedForward([3,60,120,1],[xmoid1,sigmoid,sigmoid,xmoid1],[xmoidp1,sigmoidp,sigmoidp,xmoidp1],[xmoidpp1,sigmoidpp,sigmoidpp,xmoidpp1]) machine = myfeedforward.MyFeedForward([3, 60, 120, 1], [xmoid1, relu, relu, xmoid1],
import matplotlib.pyplot as plt import matplotlib from types import NoneType from utilities import DataCleaner, comparePrediction #progress bar formatting widgets = [Percentage(), " ", Bar('|'), ' ', ETA()] # TODO softmax and possibly linear activationFunctions = { # activation functions 'logistic': lambda x: 1 / (1 + math.exp(-x)), 'tanh': lambda x: math.tanh(x), } activationFunctionsD = { # their derivatives 'logistic': lambda x: math.exp(x) / ((math.exp(x) + 1) ** 2), 'tanh': lambda x: (math.sech(x)) ** 2, } class Layer(): """ Represents each layer in a multilayer preceptron. Contains weights, inputs, outputs for each Layer """ def __init__(self, rows, columns, name="Hidden Layer", activation="logistic", weightsMultiplier=1): """ columns - neurons in current layer, rows - size of the previous layer """ self.weights = np.random.standard_normal(size=(rows, columns)) * weightsMultiplier # initialize weights matrix. size of weights manipulated with the multiplier. self.bias = np.zeros(shape=(rows)) # bias vector initialized with zeros self.inputs = np.zeros(shape=(rows)) # input vector self.outputs = np.zeros(shape=(rows)) # output vector
def main(): # alpha0 = 0.01 alpha = 0.0001 beta1 = 0.9 beta2 = 0.999 eps = 1e-8 beta = 2.0 #sigmoid = lambda x: 1.0/(1.0+math.exp(-x)) #sigmoidp = lambda x: math.exp(-x)/(1.0+math.exp(-x))**2 #sigmoidpp = lambda x: math.exp(-x)*(math.exp(-x)-1.0)/(1.0+math.exp(-x))**3 ap = 1.0 xp = 4.5 bp = 3.0 penalty = lambda x: ap * (0.5 * (math.tanh(bp * (x - xp)) - math.tanh( bp * (x + xp))) + 1.0) penaltyp = lambda x: ap * 0.5 * bp * ((1 / math.cosh(bp * (x - xp)))**2 - (1.0 / math.cosh(bp * (x + xp)))**2) sigmoid = lambda x: 0.5 * (math.tanh(beta * x) + 1.0) sigmoidp = lambda x: 0.5 * beta * (1.0 / math.cosh(beta * x))**2 sigmoidpp = lambda x: 0.5 * (-2.0) * beta * beta * math.tanh(beta * x) * ( 1.0 / math.sech(beta * x))**2 xmoid1 = lambda x: x xmoidp1 = lambda x: 1.0 xmoidpp1 = lambda x: 0.0 #bias = [[0.01],[0.01],[0.001],[0.0001],[0.00001],[0.0000001]] machine = MyFeedForward([3, 2, 3, 2], [xmoid1, sigmoid, sigmoid, xmoid1], [xmoidp1, sigmoidp, sigmoidp, xmoidp1], [xmoidpp1, sigmoidpp, sigmoidpp, xmoidpp1]) weights = machine.initial_w() nw = len(weights) xs = -0.567 ys = 0.67 zs = -0.17 es = 0.73839 ws = 0.03839 gg = machine.w_energy_gradient([xs, ys, zs], [es, ws], weights) error = gg[0] g1 = gg[1] print("error=", error) es1 = machine.evaluate([xs, ys, zs], weights) print("es1=", es1) print("xs,ys,zs,es,es1=", xs, ys, zs, es, ws, es1, (es - es1[0])**2 + (ws - es1[1])**2) print("len(g1)=", len(g1), 7 + 2 + 6 + 3) print("gb=", g1[:7]) print("gw=", g1[7:]) gg0 = machine.w_energy_gradient([xs, ys, zs], [es, ws], weights) delta = 0.00001 for ii in range(len(g1)): weights[ii] += delta gg1 = machine.w_energy_gradient([xs, ys, zs], [es, ws], weights) weights[ii] -= 2 * delta gg2 = machine.w_energy_gradient([xs, ys, zs], [es, ws], weights) print("gradient=", ii, weights[ii], (gg1[0] - gg2[0]) / (2 * delta), gg0[1][ii])
def __init__(self, nsweeps, nlayers, parameterfilename, feidatafile, energygradient0): self.energygradient0 = energygradient0 parameters = [] with open(parameterfilename, 'r') as ff: data = ff.read() for line in data.split('\n'): ss = line.split() if (len(ss) > 1): p = [] for s in ss: if s.isdigit(): p += [int(s)] elif isFloat(s): p += [float(s)] ok = p[0] in range(0, 6) if (p[0] == 0): ok = ok and (len(p) == 2) if (p[0] == 1): ok = ok and (len(p) == 2) if (p[0] == 2): ok = ok and (len(p) == 4) if (p[0] == 3): ok = ok and (len(p) == 3) if (p[0] == 4): ok = ok and (len(p) == 5) if (p[0] == 5): ok = ok and (len(p) == 5) if ok: parameters.append(p) #### define Atomic Feature Mapping #### nparameters = len(parameters) #self.nparameters = nparameters self.afm = atomfm.AtomsFM(parameters) #### define NN machine #### self.nlayers = nlayers #sigmoid = lambda x: 1.0/(1.0+math.exp(-x)) #sigmoidp = lambda x: math.exp(-x)/(math.exp(-x)+1.0)**2 #sigmoidpp = lambda x: 2*math.exp(-2*x)/(math.exp(-x)+1.0)**3 - math.exp(-x)/(math.exp(-x)+1.0)**2 sigmoid = lambda x: math.tanh(x) sigmoidp = lambda x: (1.0 / math.cosh(x))**2 sigmoidpp = lambda x: -2.0 * math.tanh(x) * (1.0 / math.sech(x))**2 #anparameters = [nparameters] #asigmoid = [lambda x: x] #asigmoidp = [lambda x: 1] #asigmoidpp = [lambda x: 0] anparameters = [] asigmoid = [] asigmoidp = [] asigmoidpp = [] for i in range(nlayers): anparameters.append(nparameters) asigmoid.append(sigmoid) asigmoidp.append(sigmoidp) asigmoidpp.append(sigmoidpp) anparameters.append(1) asigmoid.append(lambda x: x) asigmoidp.append(lambda x: 1) asigmoidpp.append(lambda x: 0) self.nn_machine = myfeedforward.MyFeedForward(anparameters, asigmoid, asigmoidp, asigmoidpp) #### read in number of atoms #### with open(feidatafile, 'r') as ff: feidata = ff.readline() nion = int(feidata) self.nion = nion print("nion=", nion) #### define NN machine weights for all atoms #### self.nn_weights = [] for ii in range(nion): self.nn_weights += self.nn_machine.initial_w() self.nw = len(self.nn_weights) / nion alpha = 0.05 for (symbols, rions, fions, energy) in read_fei_file(feidatafile): #aalpha = alpha*random.random() aalpha = alpha * random.random() etmp = [] dedw = [] for ii in range(nion): fm = self.afm(rions, ii) print "fm=", fm eee = self.nn_machine.dyoutdw_gradient( fm, self.nn_weights[ii * self.nw:(ii + 1) * self.nw]) etmp += eee[0] dedw += eee[1] #print "etmp=",ii,eee[0],energy error = math.sqrt((sum(etmp) - energy)**2) derror1detmp = 2.0 * (sum(etmp) - energy) for i in range(len(self.nn_weights)): self.nn_weights[i] -= aalpha * derror1detmp * dedw[i] self.nn_machine.print_w(self.nn_weights[0]) print("Checking Energies and Forces") #nion3 = 3*nion frame = 1 sumerror = 0.0 maxerror = 0.0 for (symbols, rions, fions, energy) in read_fei_file(feidatafile): etmp0 = [] #force3 = [0.0]*nion3 for ii in range(nion): fm = self.afm(rions, ii) ee0 = self.nn_machine.evaluate( fm, self.nn_weights[ii * self.nw:(ii + 1) * self.nw]) etmp0 += ee0 #fafm = self.afm.Egradients(rions,ii) #eee = self.nn_machine.evaluate(fafm[0],self.nn_weights[ii*self.nw:(ii+1)*self.nw]) #fff = self.nn_machine.gradients_evaluate(fafm[0],self.nn_weights[ii*self.nw:(ii+1)*self.nw]) #esum += eee[0] #for jj in range(nion3): # for k in range(nparameters): # force3[jj] -= fafm[1][jj + k*nion3]*fff[k] error = math.sqrt((sum(etmp0) - energy)**2) print frame, sum(etmp0), energy, error sumerror += error if (error > maxerror): maxerror = error frame += 1 print(" - average error=", sumerror / frame, " maxerror=", maxerror)
def d_tan_hyperbolic(input_stimulus): # Returns the derivative of the tan_hyperbolic fxn. d_hyp_value = (math.sech(input_stimulus)) ** 2 return d_hype_value
beta2 = 0.999 eps = 1e-8 beta = 2.0 #sigmoid = lambda x: 1.0/(1.0+math.exp(-x)) #sigmoidp = lambda x: math.exp(-x)/(1.0+math.exp(-x))**2 #sigmoidpp = lambda x: math.exp(-x)*(math.exp(-x)-1.0)/(1.0+math.exp(-x))**3 ap = 1.0 xp = 4.5 bp = 3.0 penalty = lambda x: ap*(0.5*(math.tanh(bp*(x-xp)) - math.tanh(bp*(x+xp))) + 1.0) penaltyp = lambda x: ap*0.5*bp*( (1/math.cosh(bp*(x-xp)))**2 - (1.0/math.cosh(bp*(x+xp)))**2) sigmoid = lambda x: 0.5*(math.tanh(beta*x)+1.0) sigmoidp = lambda x: 0.5*beta*(1.0/math.cosh(beta*x))**2 sigmoidpp = lambda x: 0.5*(-2.0)*beta*beta*math.tanh(beta*x)*(1.0/math.sech(beta*x))**2 xmoid1 = lambda x: x xmoidp1 = lambda x: 1.0 xmoidpp1 = lambda x: 0.0 #bias = [[0.01],[0.01],[0.001],[0.0001],[0.00001],[0.0000001]] bias = [[0.01],[0.01],[0.001]] bias = [] machine = myfeedforward.MyFeedForward([3,60,120,1],[xmoid1,sigmoid,sigmoid,xmoid1],[xmoidp1,sigmoidp,sigmoidp,xmoidp1],[xmoidpp1,sigmoidpp,sigmoidpp,xmoidpp1],bias) if os.path.isfile("tutorial5.weights"): with open("tutorial5.weights",'r') as ff: weights = pickle.loads(ff.read()) else: weights = machine.initial_w() for i in range(len(weights)):
plot.resetwindow(-1.0, ymin - dy, 1.0, ymax + dy, "Spring Energies") plot.plot(x1, y, "blue") plot.plot(x1, y1, "red") alpha = 0.005 A = 0.2 beta = 3.0 #sigmoid = lambda x: 1.0/(1.0+math.exp(-x)) #sigmoidp = lambda x: math.exp(-x)/(1.0+math.exp(-x))**2 #sigmoidpp = lambda x: math.exp(-x)*(math.exp(-x)-1.0)/(1.0+math.exp(-x))**3 sigmoid = lambda x: 0.5 * (math.tanh(beta * x) + 1.0) sigmoidp = lambda x: 0.5 * beta * (1.0 / math.cosh(beta * x))**2 sigmoidpp = lambda x: 0.5 * (-2.0) * beta * beta * math.tanh(beta * x) * ( 1.0 / math.sech(beta * x))**2 xmoid1 = lambda x: x xmoidp1 = lambda x: 1.0 xmoidpp1 = lambda x: 0.0 #xmoid2 = lambda x: x*x * 0.5 #xmoidp2 = lambda x: x #xmoidpp2 = lambda x: 1.0 #xmoid3 = lambda x: x*x*x * (1.0/6.0) #xmoidp3 = lambda x: 3*x*x * (1.0/6.0) #xmoidpp3 = lambda x: 6*x * (1.0/6.0) # #xmoid4 = lambda x: x*x*x*x * (1.0/24.0) #xmoidp4 = lambda x: 4*x*x*x * (1.0/24.0) #xmoidpp4 = lambda x: 12*x*x * (1/0/24.0) #bias = [[0.01],[0.01],[0.001],[0.0001],[0.00001],[0.0000001]]
import matplotlib.pyplot as plt import matplotlib from types import NoneType from utilities import DataCleaner, comparePrediction #progress bar formatting widgets = [Percentage(), " ", Bar('|'), ' ', ETA()] # TODO softmax and possibly linear activationFunctions = { # activation functions 'logistic': lambda x: 1 / (1 + math.exp(-x)), 'tanh': lambda x: math.tanh(x), } activationFunctionsD = { # their derivatives 'logistic': lambda x: math.exp(x) / ((math.exp(x) + 1) ** 2), 'tanh': lambda x: (math.sech(x)) ** 2, } class Layer(): """ Represents each layer in a multilayer preceptron. Contains weights, inputs, outputs for each Layer """ def __init__(self, rows, columns, name="Hidden Layer", activation="logistic", weightsMultiplier=1): """ columns - neurons in current layer, rows - size of the previous layer """