import os, math, torch, kora.install.rdkit, pandas as pd from model.parsing import parse_train_args from data_model.data import construct_loader from util import Standardizer, create_logger, get_loss_func from model.main import GNN import csv import numpy as np from model.training import * args = parse_train_args() torch.manual_seed(args.seed) train_loader, val_loader = construct_loader(args) mean = train_loader.dataset.mean std = train_loader.dataset.std stdzer = Standardizer(mean, std, args.task) loss = get_loss_func(args) # load model model = GNN(args, train_loader.dataset.num_node_features, train_loader.dataset.num_edge_features).to(args.device) print('Model architecture: ', model) state_dict = torch.load(args.model_path, map_location=args.device ) if 'best_model' in args.model_path else torch.load( args.model_path, map_location=args.device)['model_state_dict'] model.load_state_dict(state_dict) def train_and_save_predictions(loader, preds_path, viz_dir=None, viz_ids=None): # predict on train data
def train(self, X, T, **params): verbose = params.pop('verbose', False) # training parameters _lambda = params.pop('Lambda', 0.) #parameters for scg niter = params.pop('niter', 1000) wprecision = params.pop('wprecision', 1e-10) fprecision = params.pop('fprecision', 1e-10) wtracep = params.pop('wtracep', False) ftracep = params.pop('ftracep', False) # optimization optim = params.pop('optim', 'scg') if self.stdX == None: explore = params.pop('explore', False) self.stdX = Standardizer(X, explore) Xs = self.stdX.standardize(X) if self.stdT == None and self.stdTarget: self.stdT = Standardizer(T) T = self.stdT.standardize(T) def gradientf(weights): self.unpack(weights) Y, Z = self.forward(Xs) error = self._errorf(T, Y) return self.backward(error, Z, T, _lambda) def optimtargetf(weights): """ optimization target function : MSE """ self.unpack(weights) #self._weights[:] = weights[:] # unpack Y, _ = self.forward(Xs) Wnb = np.array([]) for i in range(self._nLayers): if len(Wnb) == 0: Wnb = self._W[i][1:, ].reshape( self._W[i].size - self._W[i][0, ].size, 1) else: Wnb = np.vstack((Wnb, self._W[i][1:, ].reshape( self._W[i].size - self._W[i][0, ].size, 1))) wpenalty = _lambda * np.dot(Wnb.flat, Wnb.flat) return self._objectf(T, Y, wpenalty) if optim == 'scg': result = scg(self.cp_weight(), gradientf, optimtargetf, wPrecision=wprecision, fPrecision=fprecision, nIterations=niter, wtracep=wtracep, ftracep=ftracep, verbose=False) self.unpack(result['w'][:]) self.f = result['f'] elif optim == 'steepest': result = steepest(self.cp_weight(), gradientf, optimtargetf, nIterations=niter, xPrecision=wprecision, fPrecision=fprecision, xtracep=wtracep, ftracep=ftracep) self.unpack(result['w'][:]) if ftracep: self.ftrace = result['ftrace'] if 'reason' in result.keys() and verbose: print(result['reason']) return result
class NeuralNet: """ neural network class for regression Parameters ---------- nunits: list the number of inputs, hidden units, and outputs Methods ------- set_hunit update/initiate weights pack pack multiple weights of each layer into one vector forward forward processing of neural network backward back-propagation of neural network train train the neural network use appply the trained network for prediction Attributes ---------- _nLayers the number of hidden unit layers rho learning rate _W weights _weights weights in one dimension (_W is referencing _weight) stdX standardization class for data stdT standardization class for target Notes ----- """ def __init__(self, nunits): self._nLayers = len(nunits) - 1 self.rho = [1] * self._nLayers self._W = [] wdims = [] lenweights = 0 for i in range(self._nLayers): nwr = nunits[i] + 1 nwc = nunits[i + 1] wdims.append((nwr, nwc)) lenweights = lenweights + nwr * nwc self._weights = np.random.uniform(-0.1, 0.1, lenweights) start = 0 # fixed index error 20110107 for i in range(self._nLayers): end = start + wdims[i][0] * wdims[i][1] self._W.append(self._weights[start:end]) self._W[i].resize(wdims[i]) start = end self.stdX = None self.stdT = None self.stdTarget = True def add_ones(self, w): return np.hstack((np.ones((w.shape[0], 1)), w)) def get_nlayers(self): return self._nLayers def set_hunit(self, w): for i in range(self._nLayers - 1): if w[i].shape != self._W[i].shape: print("set_hunit: shapes do not match!") break else: self._W[i][:] = w[i][:] def pack(self, w): return np.hstack(map(np.ravel, w)) def unpack(self, weights): self._weights[:] = weights[:] # unpack def cp_weight(self): return copy(self._weights) def RBF(self, X, m=None, s=None): if m is None: m = np.mean(X) if s is None: s = 2 #np.std(X) r = 1. / (np.sqrt(2 * np.pi) * s) return r * np.exp(-(X - m)**2 / (2 * s**2)) def forward(self, X): t = X Z = [] for i in range(self._nLayers): Z.append(t) if i == self._nLayers - 1: t = np.dot(self.add_ones(t), self._W[i]) else: t = np.tanh(np.dot(self.add_ones(t), self._W[i])) #t = self.RBF(np.dot(np.hstack((np.ones((t.shape[0],1)),t)),self._W[i])) return (t, Z) def backward(self, error, Z, T, lmb=0): delta = error N = T.size dws = [] for i in range(self._nLayers - 1, -1, -1): rh = float(self.rho[i]) / N if i == 0: lmbterm = 0 else: lmbterm = lmb * np.vstack((np.zeros( (1, self._W[i].shape[1])), self._W[i][1:, ])) dws.insert(0, (-rh * np.dot(self.add_ones(Z[i]).T, delta) + lmbterm)) if i != 0: delta = np.dot(delta, self._W[i][1:, :].T) * (1 - Z[i]**2) return self.pack(dws) def _errorf(self, T, Y): return T - Y def _objectf(self, T, Y, wpenalty): return 0.5 * np.mean(np.square(T - Y)) + wpenalty def train(self, X, T, **params): verbose = params.pop('verbose', False) # training parameters _lambda = params.pop('Lambda', 0.) #parameters for scg niter = params.pop('niter', 1000) wprecision = params.pop('wprecision', 1e-10) fprecision = params.pop('fprecision', 1e-10) wtracep = params.pop('wtracep', False) ftracep = params.pop('ftracep', False) # optimization optim = params.pop('optim', 'scg') if self.stdX == None: explore = params.pop('explore', False) self.stdX = Standardizer(X, explore) Xs = self.stdX.standardize(X) if self.stdT == None and self.stdTarget: self.stdT = Standardizer(T) T = self.stdT.standardize(T) def gradientf(weights): self.unpack(weights) Y, Z = self.forward(Xs) error = self._errorf(T, Y) return self.backward(error, Z, T, _lambda) def optimtargetf(weights): """ optimization target function : MSE """ self.unpack(weights) #self._weights[:] = weights[:] # unpack Y, _ = self.forward(Xs) Wnb = np.array([]) for i in range(self._nLayers): if len(Wnb) == 0: Wnb = self._W[i][1:, ].reshape( self._W[i].size - self._W[i][0, ].size, 1) else: Wnb = np.vstack((Wnb, self._W[i][1:, ].reshape( self._W[i].size - self._W[i][0, ].size, 1))) wpenalty = _lambda * np.dot(Wnb.flat, Wnb.flat) return self._objectf(T, Y, wpenalty) if optim == 'scg': result = scg(self.cp_weight(), gradientf, optimtargetf, wPrecision=wprecision, fPrecision=fprecision, nIterations=niter, wtracep=wtracep, ftracep=ftracep, verbose=False) self.unpack(result['w'][:]) self.f = result['f'] elif optim == 'steepest': result = steepest(self.cp_weight(), gradientf, optimtargetf, nIterations=niter, xPrecision=wprecision, fPrecision=fprecision, xtracep=wtracep, ftracep=ftracep) self.unpack(result['w'][:]) if ftracep: self.ftrace = result['ftrace'] if 'reason' in result.keys() and verbose: print(result['reason']) return result def use(self, X, retZ=False): if self.stdX: Xs = self.stdX.standardize(X) else: Xs = X Y, Z = self.forward(Xs) if self.stdT is not None: Y = self.stdT.unstandardize(Y) if retZ: return Y, Z return Y
class NeuralNetLog: def __init__(self, nunits): self._nLayers = len(nunits) - 1 self.rho = [1] * self._nLayers self._W = [] wdims = [] lenweights = 0 for i in range(self._nLayers): nwr = nunits[i] + 1 nwc = nunits[i + 1] wdims.append((nwr, nwc)) lenweights = lenweights + nwr * nwc self._weights = np.random.uniform(-0.1, 0.1, lenweights) start = 0 # fixed index error 20110107 for i in range(self._nLayers): end = start + wdims[i][0] * wdims[i][1] self._W.append(self._weights[start:end]) self._W[i].resize(wdims[i]) start = end self.stdX = None self.stdT = None self.stdTarget = True def add_ones(self, w): return np.hstack((np.ones((w.shape[0], 1)), w)) def get_nlayers(self): return self._nLayers def set_hunit(self, w): for i in range(self._nLayers - 1): if w[i].shape != self._W[i].shape: print("set_hunit: shapes do not match!") break else: self._W[i][:] = w[i][:] def pack(self, w): return np.hstack(map(np.ravel, w)) def unpack(self, weights): self._weights[:] = weights[:] # unpack def cp_weight(self): return copy.copy(self._weights) def RBF(self, X, m=None, s=None): if m is None: m = np.mean(X) if s is None: s = 2 #np.std(X) r = 1. / (np.sqrt(2 * np.pi) * s) return r * np.exp(-(X - m)**2 / (2 * s**2)) def forward(self, X): t = X Z = [] for i in range(self._nLayers): Z.append(t) if i == self._nLayers - 1: #t = np.tanh(np.dot(self.add_ones(t), self._W[i])) #t = np.dot(self.add_ones(t), self._W[i]) #expmat = np.exp(np.dot(self.add_ones(t), self._W[i])) #print(expmat.shape) #denom = np.sum(expmat,axis=0) #t = expmat/denom t = 1 / (1 + np.exp(-np.dot(self.add_ones(t), self._W[i]))) #print(t) else: t = np.tanh(np.dot(self.add_ones(t), self._W[i])) #t = self.RBF(np.dot(np.hstack((np.ones((t.shape[0],1)),t)),self._W[i])) return (t, Z) def backward(self, error, Z, T, lmb=0): delta = error N = T.size dws = [] for i in range(self._nLayers - 1, -1, -1): rh = float(self.rho[i]) / N if i == 0: lmbterm = 0 else: lmbterm = lmb * np.vstack((np.zeros( (1, self._W[i].shape[1])), self._W[i][1:, ])) dws.insert(0, (-rh * np.dot(self.add_ones(Z[i]).T, delta) + lmbterm)) if i != 0: #print(delta) #print("p2") #print(Z) #print("p3") #print(self._W[i][1:, :].T) delta = np.dot(delta, self._W[i][1:, :].T) * (1 - Z[i]**2) return self.pack(dws) def _errorf(self, T, Y): return T - Y def _objectf(self, T, Y, wpenalty): return -(np.sum(np.sum((T * np.log(Y)), axis=1), axis=0)) + wpenalty def train(self, X, T, **params): verbose = params.pop('verbose', False) # training parameters _lambda = params.pop('Lambda', 0) #parameters for scg niter = params.pop('niter', 1000) wprecision = params.pop('wprecision', 1e-10) fprecision = params.pop('fprecision', 1e-10) wtracep = params.pop('wtracep', False) ftracep = params.pop('ftracep', False) # optimization optim = params.pop('optim', 'scg') if self.stdX == None: explore = params.pop('explore', False) self.stdX = Standardizer(X, explore) Xs = self.stdX.standardize(X) if self.stdT == None and self.stdTarget and False: self.stdT = Standardizer(T) T = self.stdT.standardize(T) def gradientf(weights): self.unpack(weights) Y, Z = self.forward(Xs) error = self._errorf(T, Y) return self.backward(error, Z, T, _lambda) def optimtargetf(weights): """ optimization target function : MSE """ self.unpack(weights) #self._weights[:] = weights[:] # unpack Y, _ = self.forward(Xs) Wnb = np.array([]) for i in range(self._nLayers): if len(Wnb) == 0: Wnb = self._W[i][1:, ].reshape( self._W[i].size - self._W[i][0, ].size, 1) else: Wnb = np.vstack((Wnb, self._W[i][1:, ].reshape( self._W[i].size - self._W[i][0, ].size, 1))) wpenalty = _lambda * np.dot(Wnb.flat, Wnb.flat) return self._objectf(T, Y, wpenalty) if optim == 'scg': result = scg(self.cp_weight(), gradientf, optimtargetf, wPrecision=wprecision, fPrecision=fprecision, nIterations=niter, wtracep=wtracep, ftracep=ftracep, verbose=False) self.unpack(result['w'][:]) self.f = result['f'] elif optim == 'steepest': result = steepest(self.cp_weight(), gradientf, optimtargetf, nIterations=niter, xPrecision=wprecision, fPrecision=fprecision, xtracep=wtracep, ftracep=ftracep) self.unpack(result['w'][:]) if ftracep: self.ftrace = result['ftrace'] if 'reason' in result.keys() and verbose: print(result['reason']) return result def use(self, X, retZ=False): if self.stdX: Xs = self.stdX.standardize(X) else: Xs = X Y, Z = self.forward(Xs) if self.stdT is not None: Y = self.stdT.unstandardize(Y) if retZ: return Y, Z return Y