def objective(trial): ################################ #### INIT OPTIM AND MODEL ###### ################################ learningRate = trial.suggest_loguniform('learning_rate', 1.5e-3, 1e-2) #doLearningRateDecay = True # Learning rate decay #learningRateDecayRate = 0.9 # Rate #learningRateDecayPeriod = 1 # How many epochs after which update the lr trainingOptions['learningRateDecayRate'] = trial.suggest_uniform('learningRateDecayRate', 0.8, 0.999) trainingOptions['learningRateDecayPeriod'] = 1#trial.suggest_int('learningRateDecayPeriod', 1, 10) beta1 = trial.suggest_uniform('beta1', 0.89, 0.91) beta2 = trial.suggest_uniform('beta2', 0.99, 0.9999) trainingOptions['trial'] = trial #\\\ Optimizer thisOptim = optim.Adam(arch.parameters(), lr = learningRate, betas = (beta1,beta2)) #\\\ Model trainable_model = model.Model(arch.to(device), lossFunction(), thisOptim, trainer, evaluator, device, args.model_name, '') print("Training model %s..." % trainable_model, end = ' ', flush = True) thisTrainVars = trainable_model.train(data, nEpochs, batchSize, **trainingOptions) # Save the variables lossTrain = thisTrainVars['lossTrain'] costTrain = thisTrainVars['costTrain'] lossValid = thisTrainVars['lossValid'] costValid = thisTrainVars['costValid'] print("OK", flush = True) ####################################### ####### EVALUATION ############## ####################################### thisEvalVars = trainable_model.evaluate(data) costBest = thisEvalVars['costBest'] costLast = thisEvalVars['costLast'] print("Last Evaluation \t%s: %6.2f%% [Best] %6.2f%% [Last]" % (trainable_model, costBest * 100, costLast * 100)) return costBest
thisOptim = optim.RMSprop(thisArchit.parameters(), lr=learningRate, alpha=beta1) ######## # LOSS # ######## thisLossFunction = lossFunction # (if different from default, change # it here) ######### # MODEL # ######### SelGNNnpl = model.Model(thisArchit, thisLossFunction, thisOptim, thisName, saveDir, order) modelsGNN[thisName] = SelGNNnpl writeVarValues( varsFile, { 'name': thisName, 'thisTrainer': thisTrainer, 'thisLearningRate': thisLearningRate, 'thisBeta1': thisBeta1, 'thisBeta2': thisBeta2 }) if doPrint: print("OK.")
elif thisTrainer == 'RMSprop': thisOptim = optim.RMSprop(thisArchit.parameters(), lr=learningRate, alpha=beta1) ######## # LOSS # ######## thisLossFunction = lossFunction ######### # MODEL # ######### Polynomial = model.Model(thisArchit, thisLossFunction, thisOptim, thisName, saveDir, order) modelsGNN[thisName] = Polynomial writeVarValues( varsFile, { 'name': thisName, 'thisTrainer': thisTrainer, 'thisLearningRate': thisLearningRate, 'thisBeta1': thisBeta1, 'thisBeta2': thisBeta2 }) ################################################################### # # # TRAINING #
alpha=beta1) ######## # LOSS # ######## # Initialize the loss function thisLossFunction = loss.adaptExtraDimensionLoss(lossFunction) ######### # MODEL # ######### # Create the model modelCreated = model.Model(thisArchit, thisLossFunction, thisOptim, thisTrainer, thisEvaluator, thisDevice, thisName, saveDir) # Store it modelsGNN[thisName] = modelCreated # Write the main hyperparameters writeVarValues( varsFile, { 'name': thisName, 'thisOptimizationAlgorithm': thisOptimAlg, 'thisTrainer': thisTrainer, 'thisEvaluator': thisEvaluator, 'thisLearningRate': thisLearningRate, 'thisBeta1': thisBeta1, 'thisBeta2': thisBeta2
#\\\\\\\\\\\\ GIN = archit.SelectionGNN(hParamsGIN['F'], hParamsGIN['K'], hParamsGIN['bias'], hParamsGIN['sigma'], hParamsGIN['N'], hParamsGIN['rho'], hParamsGIN['alpha'], hParamsGIN['dimLayersMLP'], S) GIN.to(device) # Move architecture to the selected device # Optimizer optimizer = optim.Adam(GIN.parameters(), lr=learningRate, betas=(beta1, beta2)) # Loss function chosenLoss = lossFunction() # Model modelBind = model.Model(GIN, chosenLoss, optimizer, hParamsGIN['name'], saveDir, order) ############ # TRAINING # (and VALIDATION) ############ if doPrint: print("\t%15s: Training..." % GINname, end=' ', flush=True) modelBind.train(data, nEpochs, batchSize, **trainingOptions) if doPrint: print("OK", flush=True) ############## # EVALUATION #
thisOptim = optim.SGD(thisArchit.parameters(), lr=learningRate) elif thisTrainer == 'RMSprop': thisOptim = optim.RMSprop(thisArchit.parameters(), lr=learningRate, alpha=beta1) ######## # LOSS # ######## thisLossFunction = lossFunction ######### # MODEL # ######### EdgeVariant = model.Model(thisArchit, thisLossFunction, thisOptim, thisName, saveDir, order) modelsGNN[thisName] = EdgeVariant writeVarValues(varsFile, {'name': thisName, 'thisTrainer': thisTrainer, 'thisLearningRate': thisLearningRate, 'thisBeta1': thisBeta1, 'thisBeta2': thisBeta2}) ################################################################### # # # TRAINING # # # #####################################################################
elif thisTrainer == 'RMSprop': thisOptim = optim.RMSprop(thisArchit.parameters(), lr=learningRate, alpha=beta1) ######## # LOSS # ######## thisLossFunction = loss.adaptExtraDimensionLoss(lossFunction) ######### # MODEL # ######### modelCreated = model.Model(thisArchit, thisLossFunction, thisOptim, thisName, saveDir) modelsGNN[thisName] = modelCreated writeVarValues( varsFile, { 'name': thisName, 'thisTrainer': thisTrainer, 'thisLearningRate': thisLearningRate, 'thisBeta1': thisBeta1, 'thisBeta2': thisBeta2 }) if doPrint: print("OK")
nn.ReLU, [N], gml.NoPool, [1], [1], S, batchnorm=True), torch.nn.Linear(N, 1, bias=True)) ################################ #### INIT OPTIM AND MODEL ###### ################################ #\\\ Optimizer thisOptim = optim.Adam(arch.parameters(), lr=learningRate, betas=(beta1, beta2)) #\\\ Model trainable_model = model.Model(arch.to(device), lossFunction(), thisOptim, trainer, evaluator, device, args.model_name, '') ####################################### ####### TRAINING DE VERDURA ########### ####################################### print("Training model %s..." % trainable_model, end=' ', flush=True) #Train trainingOptions = {} trainingOptions['saveDir'] = '' trainingOptions['printInterval'] = 1 trainingOptions['validationInterval'] = validationInterval trainingOptions['doEarlyStopping'] = True trainingOptions['earlyStoppingLag'] = args.early_stopping trainingOptions['doLearningRateDecay'] = doLearningRateDecay
def train_net(data, h_parameters, phi=None): # Now, we are in position to know the number of nodes (for now; this might # change later on when the graph is created and the options on whether to # make it connected, etc., come into effect) nNodes = data.selectedAuthor['all']['wordFreq'].shape[1] ######### # GRAPH # ######### # Create graph nodesToKeep = [] # here we store the list of nodes kept after all # modifications to the graph, so we can then update the data samples # accordingly; since lists are passed as pointers (mutable objects) # we can store the node list without necessary getting an output to the # function G = graphTools.Graph('fuseEdges', nNodes, data.selectedAuthor['train']['WAN'], 'sum', graphNormalizationType, keepIsolatedNodes, forceUndirected, forceConnected, nodesToKeep) G.computeGFT() # Compute the GFT of the stored GSO # And re-update the number of nodes for changes in the graph (due to # enforced connectedness, for instance) if phi is None: nNodes = G.N nodesToKeep = np.array(nodesToKeep) # And re-update the data (keep only the nodes that are kept after isolated # nodes or nodes to make the graph connected have been removed) data.samples['train']['signals'] = \ data.samples['train']['signals'][:, nodesToKeep] data.samples['valid']['signals'] = \ data.samples['valid']['signals'][:, nodesToKeep] data.samples['test']['signals'] = \ data.samples['test']['signals'][:, nodesToKeep] else: nNodes = phi.shape[0] # Once data is completely formatted and in appropriate fashion, change its # type to torch and move it to the appropriate device data.astype(torch.float64) data.to(device) ################################################################## # # # MODELS INITIALIZATION # # # ##################################################################### # Override parameters with grid parameters. hParamsPolynomial['F'] = h_parameters[0] hParamsPolynomial['K'] = h_parameters[1] # This is the dictionary where we store the models (in a model.Model # class, that is then passed to training). modelsGNN = {} # If a new model is to be created, it should be called for here. # \\\\\\\\\\ # \\\ MODEL 2: Polynomial GNN # \\\\\\\\\\\\ thisName = hParamsPolynomial['name'] ############## # PARAMETERS # ############## # \\\ Optimizer options # (If different from the default ones, change here.) thisTrainer = trainer thisLearningRate = learningRate thisBeta1 = beta1 thisBeta2 = beta2 if phi is None: # \\\ Ordering S, order = graphTools.permIdentity(G.S / np.max(np.diag(G.E))) # order is an np.array with the ordering of the nodes with respect # to the original GSO (the original GSO is kept in G.S). else: # compute the Eigenvalues of matrix e, V = np.linalg.eig(phi) # \\\ Ordering highest_eig_val = np.max(np.diag(e)).real if highest_eig_val == 0: S, order = graphTools.permIdentity(phi) else: S, order = graphTools.permIdentity(phi / highest_eig_val) # order is an np.array with the ordering of the nodes with respect # to the original GSO (the original GSO is kept in G.S). ################ # ARCHITECTURE # ################ hParamsPolynomial['N'] = [nNodes] if doPrint: print('') print('COMBINATION {0}, {1}'.format(str(hParamsPolynomial['F']), str(hParamsPolynomial['K']))) thisArchit = archit.SelectionGNN( # Graph filtering hParamsPolynomial['F'], hParamsPolynomial['K'], hParamsPolynomial['bias'], # Nonlinearity hParamsPolynomial['sigma'], # Pooling hParamsPolynomial['N'], hParamsPolynomial['rho'], hParamsPolynomial['alpha'], # MLP hParamsPolynomial['dimLayersMLP'], # Structure S) # This is necessary to move all the learnable parameters to be # stored in the device (mostly, if it's a GPU) thisArchit.to(device) ############# # OPTIMIZER # ############# if thisTrainer == 'ADAM': thisOptim = optim.Adam(thisArchit.parameters(), lr=learningRate, betas=(beta1, beta2)) elif thisTrainer == 'SGD': thisOptim = optim.SGD(thisArchit.parameters(), lr=learningRate) elif thisTrainer == 'RMSprop': thisOptim = optim.RMSprop(thisArchit.parameters(), lr=learningRate, alpha=beta1) ######## # LOSS # ######## thisLossFunction = lossFunction ######### # MODEL # ######### Polynomial = model.Model(thisArchit, thisLossFunction, thisOptim, thisName, saveDir, order) modelsGNN[thisName] = Polynomial ################################################################### # # # TRAINING # # # ##################################################################### ############ # TRAINING # ############ # On top of the rest of the training options, we pass the identification # of this specific data split realization. # This is the function that trains the models detailed in the dictionary # modelsGNN using the data data, with the specified training options. train.MultipleModels(modelsGNN, data, nEpochs=nEpochs, batchSize=batchSize, **trainingOptions) return modelsGNN['PolynomiGNN']