def train_from_graphs(self, train_graphs, train_targets, validation_graphs=None, validation_targets=None, epochs=1000, batch_size=128, verbose=1, callbacks=None, prev_model=None, lr_scaling_factor=0.5, patience=500, **kwargs): # load from saved model if prev_model: self.load_weights(prev_model) # is_classification = 'entropy' in self.model.loss # monitor = 'val_acc' if is_classification else 'val_mae' # mode = 'max' if is_classification else 'min' dirname = kwargs.pop('dirname', 'callback') if not os.path.isdir(dirname): os.makedirs(dirname) if callbacks is None: # with this call back you can stop the model training by `touch STOP` callbacks = [ManualStop()] train_nb_atoms = [len(i['atom']) for i in train_graphs] train_targets = [ self.target_scaler.transform(i, j) for i, j in zip(train_targets, train_nb_atoms) ] if validation_graphs is not None: # filepath = os.path.join(dirname, '%s_{epoch:05d}_{%s:.6f}.hdf5' % (monitor, monitor)) val_nb_atoms = [len(i['atom']) for i in validation_graphs] validation_targets = [ self.target_scaler.transform(i, j) for i, j in zip(validation_targets, val_nb_atoms) ] val_inputs = self.graph_converter.get_flat_data( validation_graphs, validation_targets) val_generator = self._create_generator(*val_inputs, batch_size=batch_size) steps_per_val = int(np.ceil(len(validation_graphs) / batch_size)) # callbacks.extend([ReduceLRUponNan(filepath=filepath, # monitor=monitor, # mode=mode, # factor=lr_scaling_factor, # patience=patience, # )]) # callbacks.extend([ModelCheckpointMAE(filepath=filepath, # monitor=monitor, # mode=mode, # save_best_only=True, # save_weights_only=False, # val_gen=val_generator, # steps_per_val=steps_per_val, # target_scaler=self.target_scaler)]) else: val_generator = None steps_per_val = None train_inputs = self.graph_converter.get_flat_data( train_graphs, train_targets) # check dimension match self.check_dimension(train_graphs[0]) train_generator = self._create_generator(*train_inputs, batch_size=batch_size) steps_per_train = int(np.ceil(len(train_graphs) / batch_size)) self.fit_generator(train_generator, steps_per_epoch=steps_per_train, validation_data=val_generator, validation_steps=steps_per_val, epochs=epochs, verbose=verbose, callbacks=callbacks, **kwargs)
Args: ids (List): list of ids Returns: list of graphs and list of target values """ ids = [i for i in ids if i in final_graphs] return [final_graphs[i] for i in ids], [final_targets[i] for i in ids] train_graphs, train_targets = get_graphs_targets(train_ids) val_graphs, val_targets = get_graphs_targets(val_ids) # 5. Model training callbacks = [ReduceLRUponNan(patience=500), ManualStop()] model.train_from_graphs(train_graphs, train_targets, val_graphs, val_targets, epochs=EPOCHS, verbose=2, initial_epoch=0, callbacks=callbacks) # 6. Model testing ## load the best model with lowest validation error files = glob('./callback/*.hdf5') best_model = sorted(files, key=os.path.getctime)[-1]