Exemplo n.º 1
0
    def test_callback(self):
        callbacks = [GeneratorLog(self.train_gen, steps_per_train=1, val_gen=self.train_gen, steps_per_val=1,
                                  n_every=1, val_names=['conductivity'], val_units=['S/cm']),
                     ModelCheckpointMAE(filepath='./val_mae_{epoch:05d}_{val_mae:.6f}.hdf5', val_gen=self.train_gen,
                                        steps_per_val=1),
                     ]
        before_fit_file = glob.glob("./val_mae*.hdf5")
        self.model.fit_generator(self.train_gen, steps_per_epoch=1, epochs=1, callbacks=callbacks, verbose=0)
        after_fit_file = glob.glob("./val_mae*.hdf5")

        self.assertEqual(len(before_fit_file), 0)
        self.assertEqual(len(after_fit_file), 1)
        os.remove(after_fit_file[0])

        callback_mae = ModelCheckpointMAE(filepath='./val_mae_{epoch:05d}_{val_mae:.6f}.hdf5', val_gen=self.train_gen,
                                          steps_per_val=1, target_scaler=StandardScaler(1, 1, is_intensive=True))

        dummy_target = np.array([[1, 1], [2, 2]])
        dummy_nb_atoms = np.array([[2], [3]])
        transformed = callback_mae.target_scaler.inverse_transform(dummy_target, dummy_nb_atoms)
        self.assertTrue(np.allclose(transformed, np.array([[2, 2], [3, 3]])))

        callback_mae = ModelCheckpointMAE(filepath='./val_mae_{epoch:05d}_{val_mae:.6f}.hdf5', val_gen=self.train_gen,
                                          steps_per_val=1, target_scaler=StandardScaler(1, 1, is_intensive=False))

        dummy_target = np.array([[1, 1], [2, 2]])
        dummy_nb_atoms = np.array([[2], [3]])
        transformed = callback_mae.target_scaler.inverse_transform(dummy_target, dummy_nb_atoms)
        self.assertTrue(np.allclose(transformed, np.array([[4, 4], [9, 9]])))
Exemplo n.º 2
0
    def test_crystal_model(self):
        callbacks = [
            ModelCheckpointMAE(
                filepath='./val_mae_{epoch:05d}_{val_mae:.6f}.hdf5',
                save_best_only=True,
                val_gen=self.train_gen_crystal,
                steps_per_val=1,
                is_pa=False),
            GeneratorLog(self.train_gen_crystal,
                         1,
                         self.train_gen_crystal,
                         1,
                         val_names=['Ef'],
                         val_units=['eV/atom']),
            ManualStop()
        ]

        self.model.fit_generator(generator=self.train_gen_crystal,
                                 steps_per_epoch=1,
                                 epochs=2,
                                 verbose=1,
                                 callbacks=callbacks)
        model_files = glob('val_mae*.hdf5')
        self.assertGreater(len(model_files), 0)
        for i in model_files:
            os.remove(i)
Exemplo n.º 3
0
    def test_callback(self):
        callbacks = [
            GeneratorLog(self.train_gen,
                         steps_per_train=1,
                         val_gen=self.train_gen,
                         steps_per_val=1,
                         n_every=1,
                         val_names=['conductivity'],
                         val_units=['S/cm']),
            ModelCheckpointMAE(
                filepath='./val_mae_{epoch:05d}_{val_mae:.6f}.hdf5',
                val_gen=self.train_gen,
                steps_per_val=1),
        ]
        captured_output = StringIO()
        sys.stdout = captured_output

        before_fit_file = glob.glob("./val_mae*.hdf5")
        self.model.fit_generator(self.train_gen,
                                 steps_per_epoch=1,
                                 epochs=1,
                                 callbacks=callbacks,
                                 verbose=0)
        after_fit_file = glob.glob("./val_mae*.hdf5")
        sys.stdout = sys.__stdout__
        result = captured_output.getvalue()
        self.assertRegex(result,
                         "Train MAE:\nconductivity: [-+]?\d*\.\d+|\d+ S/cm")
        self.assertRegex(result,
                         "Test MAE:\nconductivity: [-+]?\d*\.\d+|\d+ S/cm")

        self.assertEqual(len(before_fit_file), 0)
        self.assertEqual(len(after_fit_file), 1)
        os.remove(after_fit_file[0])
Exemplo n.º 4
0
    def test_crystal_model(self):
        model = set2set_with_embedding_mp(self.n_bond_features,
                                          self.n_global_features,
                                          n_blocks=1,
                                          lr=1e-2,
                                          n1=4,
                                          n2=4,
                                          n3=4,
                                          n_pass=1,
                                          n_target=1)
        callbacks = [
            ModelCheckpointMAE(
                filepath='./val_mae_{epoch:05d}_{val_mae:.6f}.hdf5',
                save_best_only=True,
                val_gen=self.train_gen_crystal,
                steps_per_val=1,
                is_pa=False),
            GeneratorLog(self.train_gen_crystal,
                         1,
                         self.train_gen_crystal,
                         1,
                         val_names=['Ef'],
                         val_units=['eV/atom']),
            ManualStop()
        ]

        model.fit_generator(generator=self.train_gen_crystal,
                            steps_per_epoch=1,
                            epochs=2,
                            verbose=1,
                            callbacks=callbacks)
        model_files = glob('val_mae*.hdf5')
        self.assertGreater(len(model_files), 0)
        for i in model_files:
            os.remove(i)
Exemplo n.º 5
0
    def train_from_graphs(self,
                          train_graphs,
                          train_targets,
                          validation_graphs=None,
                          validation_targets=None,
                          epochs=1000,
                          batch_size=128,
                          verbose=1,
                          callbacks=None,
                          prev_model=None,
                          **kwargs
                          ):

        # load from saved model
        if prev_model:
            self.load_weights(prev_model)
        is_classification = 'entropy' in self.model.loss
        monitor = 'val_acc' if is_classification else 'val_mae'
        mode = 'max' if is_classification else 'min'
        dirname = kwargs.pop('dirname', 'callback')
        if not os.path.isdir(dirname):
            os.makedirs(dirname)
        if callbacks is None:
            # with this call back you can stop the model training by `touch STOP`
            callbacks = [ManualStop()]
        callbacks.append(ReduceLRUponNan())
        train_nb_atoms = [len(i['atom']) for i in train_graphs]
        train_targets = [self.target_scaler.transform(i, j) for i, j in zip(train_targets, train_nb_atoms)]
        train_targets = np.array(train_targets).ravel()

        if validation_graphs is not None:
            filepath = pjoin(dirname, 'val_mae_{epoch:05d}_{%s:.6f}.hdf5' % monitor)
            val_nb_atoms = [len(i['atom']) for i in validation_graphs]
            validation_targets = [self.target_scaler.transform(i, j) for i, j in zip(validation_targets, val_nb_atoms)]
            validation_targets = np.array(validation_targets).ravel()
            val_inputs = self.graph_convertor.get_flat_data(validation_graphs, validation_targets)

            val_generator = self._create_generator(*val_inputs,
                                                   batch_size=batch_size)
            steps_per_val = int(np.ceil(len(validation_graphs) / batch_size))
            callbacks.extend([ModelCheckpointMAE(filepath=filepath,
                                                 monitor=monitor,
                                                 mode=mode,
                                                 save_best_only=True,
                                                 save_weights_only=False,
                                                 val_gen=val_generator,
                                                 steps_per_val=steps_per_val,
                                                 y_scaler=None)])
        else:
            val_generator = None
            steps_per_val = None
        train_inputs = self.graph_convertor.get_flat_data(train_graphs, train_targets)
        train_generator = self._create_generator(*train_inputs, batch_size=batch_size)
        steps_per_train = int(np.ceil(len(train_graphs) / batch_size))
        self.fit_generator(train_generator, steps_per_epoch=steps_per_train,
                           validation_data=val_generator, validation_steps=steps_per_val,
                           epochs=epochs, verbose=verbose, callbacks=callbacks, **kwargs)
Exemplo n.º 6
0
    def train_from_graphs(self,
                          train_graphs,
                          train_targets,
                          validation_graphs=None,
                          validation_targets=None,
                          epochs=1000,
                          batch_size=128,
                          verbose=1,
                          callbacks=None,
                          prev_model=None,
                          **kwargs):

        # load from saved model
        if prev_model:
            self.load_weights(prev_model)
        dirname = kwargs.get('dirname', 'callback')
        if not os.path.isdir(dirname):
            os.makedirs(dirname)
        if callbacks is None:
            callbacks = [ManualStop()]
        train_targets = self.yscaler.fit_transform(
            np.array(train_targets).reshape((-1, 1))).ravel()
        if validation_graphs is not None:
            filepath = pjoin(dirname, 'val_mae_{epoch:05d}_{val_mae:.6f}.hdf5')
            validation_targets = self.yscaler.transform(
                np.array(validation_targets).reshape((-1, 1))).ravel()
            val_inputs = graphs2inputs(validation_graphs, validation_targets)

            val_generator = self._create_generator(*val_inputs,
                                                   batch_size=batch_size)
            steps_per_val = int(np.ceil(len(validation_graphs) / batch_size))
            # print(steps_per_val)
            callbacks.extend([
                ModelCheckpointMAE(filepath=filepath,
                                   save_best_only=True,
                                   save_weights_only=False,
                                   val_gen=val_generator,
                                   steps_per_val=steps_per_val,
                                   y_scaler=self.yscaler)
            ])
        else:
            val_generator = None
            steps_per_val = None
        train_inputs = graphs2inputs(train_graphs, train_targets)
        train_generator = self._create_generator(*train_inputs,
                                                 batch_size=batch_size)
        steps_per_train = int(np.ceil(len(train_graphs) / batch_size))
        self.fit_generator(train_generator,
                           steps_per_epoch=steps_per_train,
                           validation_data=val_generator,
                           validation_steps=steps_per_val,
                           epochs=epochs,
                           verbose=verbose,
                           callbacks=callbacks)
Exemplo n.º 7
0
    def test_reduce_lr_upon_nan(self):
        with ScratchDir('.'):
            callbacks = [ReduceLRUponNan(patience=100)]
            self.assertAlmostEqual(float(kb.get_value(self.model.optimizer.lr)), 1e-3)
            gen = Generator(self.x, np.array([1, np.nan]).reshape((1, 2, 1)))
            self.model.fit_generator(gen, steps_per_epoch=1, epochs=1, callbacks=callbacks, verbose=0)
            self.assertAlmostEqual(float(kb.get_value(self.model.optimizer.lr)), 0.5e-3)

            inp = [
                Input(shape=(None, self.n_feature)),
                Input(shape=(None, self.n_bond_features)),
                Input(shape=(None, self.n_global_features)),
                Input(shape=(None,), dtype='int32'),
                Input(shape=(None,), dtype='int32'),
                Input(shape=(None,), dtype='int32'),
                Input(shape=(None,), dtype='int32'),
            ]
            units_v = [2, 2]
            units_e = [2, 2]
            units_u = [2, ]
            layer = MEGNetLayer(units_v, units_e, units_u)
            out = layer(inp)
            out = Dense(1)(out[2])
            model = Model(inputs=inp, outputs=out)
            model.compile(loss='mse', optimizer='adam')
            x = [np.random.normal(size=(1, 4, self.n_feature)),
                     np.random.normal(size=(1, 6, self.n_bond_features)),
                     np.random.normal(size=(1, 2, self.n_global_features)),
                     np.array([[0, 0, 1, 1, 2, 3]]),
                     np.array([[1, 1, 0, 0, 3, 2]]),
                     np.array([[0, 0, 1, 1]]),
                     np.array([[0, 0, 0, 0, 1, 1]]),
                     ]
            y = np.random.normal(size=(1, 2, 1))
            train_gen = Generator(x, y)

            callbacks = [ReduceLRUponNan(filepath='./val_mae_{epoch:05d}_{val_mae:.6f}.hdf5', patience=100),
                         ModelCheckpointMAE(filepath='./val_mae_{epoch:05d}_{val_mae:.6f}.hdf5', val_gen=train_gen,
                                            steps_per_val=1)
                         ]
            # 1. involve training and saving
            model.fit_generator(train_gen, steps_per_epoch=1, epochs=2, callbacks=callbacks, verbose=1)
            # 2. throw nan loss, trigger ReduceLRUponNan
            model.fit_generator(gen, steps_per_epoch=1, epochs=2, callbacks=callbacks, verbose=1)
            # 3. Normal training, recover saved model from 1
            model.fit_generator(train_gen, steps_per_epoch=1, epochs=2, callbacks=callbacks, verbose=1)
            self.assertAlmostEqual(float(kb.get_value(model.optimizer.lr)), 0.25e-3)
Exemplo n.º 8
0
 def test_crystal_model(self):
     callbacks = [
         ModelCheckpointMAE(
             filepath='./val_mae_{epoch:05d}_{val_mae:.6f}.hdf5',
             save_best_only=True,
             val_gen=self.train_gen_crystal,
             steps_per_val=1),
         ManualStop()
     ]
     with ScratchDir('.'):
         self.model.fit(self.train_gen_crystal,
                        steps_per_epoch=1,
                        epochs=2,
                        verbose=1,
                        callbacks=callbacks)
         model_files = glob('val_mae*.hdf5')
         self.assertGreater(len(model_files), 0)
         for i in model_files:
             os.remove(i)
Exemplo n.º 9
0
    def train_from_graphs(self,
                          train_graphs: List[Dict],
                          train_targets: List[float],
                          validation_graphs: List[Dict] = None,
                          validation_targets: List[float] = None,
                          sample_weights: List[float] = None,
                          epochs: int = 1000,
                          batch_size: int = 128,
                          verbose: int = 1,
                          pad_string: str = None,
                          callbacks: List[Callback] = None,
                          prev_model: str = None,
                          lr_scaling_factor: float = 0.5,
                          patience: int = 500,
                          save_checkpoint: bool = True,
                          automatic_correction: bool = True,
                          **kwargs) -> "GraphModel":
        """
        Args:
            train_graphs: (list) list of graph dictionaries
            train_targets: (list) list of target values
            validation_graphs: (list) list of graphs as validation
            validation_targets: (list) list of validation targets
            sample_weights: (list) list of sample weights
            epochs: (int) number of epochs
            batch_size: (int) training batch size
            verbose: (int) keras fit verbose, 0 no progress bar, 1 only at the epoch end and 2 every batch
            pad_string: (str) string to add in front of callback filepath
            callbacks: (list) megnet or keras callback functions for training
            prev_model: (str) file name for previously saved model
            lr_scaling_factor: (float, less than 1) scale the learning rate down when nan loss encountered
            patience: (int) patience for early stopping
            save_checkpoint: (bool) whether to save checkpoint
            automatic_correction: (bool) correct nan errors
            **kwargs:
        """
        # load from saved model
        if prev_model:
            self.load_weights(prev_model)
        is_classification = 'entropy' in str(self.model.loss)
        monitor = 'val_acc' if is_classification else 'val_mae'
        mode = 'max' if is_classification else 'min'
        dirname = kwargs.pop('dirname', 'callback')
        has_sample_weights = sample_weights is not None
        if not os.path.isdir(dirname):
            os.makedirs(dirname)
        if callbacks is None:
            # with this call back you can stop the model training by `touch STOP`
            callbacks = [ManualStop()]
        train_nb_atoms = [len(i['atom']) for i in train_graphs]
        train_targets = [
            self.target_scaler.transform(i, j)
            for i, j in zip(train_targets, train_nb_atoms)
        ]
        if (validation_graphs is not None) and (validation_targets
                                                is not None):
            filepath = ('%s_{epoch:05d}_{%s:.6f}.hdf5' % (monitor, monitor))
            val_nb_atoms = [len(i['atom']) for i in validation_graphs]
            validation_targets = [
                self.target_scaler.transform(i, j)
                for i, j in zip(validation_targets, val_nb_atoms)
            ]
            val_inputs = self.graph_converter.get_flat_data(
                validation_graphs, validation_targets)

            val_generator = self._create_generator(*val_inputs,
                                                   batch_size=batch_size)
            steps_per_val = int(np.ceil(len(validation_graphs) / batch_size))
            if save_checkpoint:
                callbacks.extend([
                    ModelCheckpointMAE(pad_string=pad_string,
                                       filepath=filepath,
                                       monitor=monitor,
                                       mode=mode,
                                       save_best_only=True,
                                       save_weights_only=False,
                                       val_gen=val_generator,
                                       steps_per_val=steps_per_val,
                                       target_scaler=self.target_scaler)
                ])
                # avoid running validation twice in an epoch
                # val_generator = None  # type: ignore
                # steps_per_val = None  # type: ignore

            if automatic_correction:
                callbacks.extend([
                    ReduceLRUponNan(filepath=filepath,
                                    monitor=monitor,
                                    mode=mode,
                                    factor=lr_scaling_factor,
                                    patience=patience,
                                    has_sample_weights=has_sample_weights)
                ])
        else:
            val_generator = None  # type: ignore
            steps_per_val = None  # type: ignore

        train_inputs = self.graph_converter.get_flat_data(
            train_graphs, train_targets)
        # check dimension match
        self.check_dimension(train_graphs[0])
        train_generator = self._create_generator(*train_inputs,
                                                 sample_weights=sample_weights,
                                                 batch_size=batch_size)
        steps_per_train = int(np.ceil(len(train_graphs) / batch_size))
        self.fit(train_generator,
                 steps_per_epoch=steps_per_train,
                 validation_data=val_generator,
                 validation_steps=steps_per_val,
                 epochs=epochs,
                 verbose=verbose,
                 callbacks=callbacks,
                 **kwargs)
        return self
Exemplo n.º 10
0
    def train_from_graphs(self,
                          train_graphs: List[Dict],
                          train_targets: List[float],
                          validation_graphs: List[Dict] = None,
                          validation_targets: List[float] = None,
                          epochs: int = 1000,
                          batch_size: int = 128,
                          verbose: int = 1,
                          callbacks: List[Callback] = None,
                          prev_model: str = None,
                          lr_scaling_factor: float = 0.5,
                          patience: int = 500,
                          save_checkpoint: bool = True,
                          automatic_correction: bool = True,
                          **kwargs) -> None:
        """
        # TODO write doc...
        :param train_graphs:
        :param train_targets:
        :param validation_graphs:
        :param validation_targets:
        :param epochs:
        :param batch_size:
        :param verbose:
        :param callbacks:
        :param prev_model:
        :param lr_scaling_factor:
        :param patience:
        :param save_checkpoint:
        :param automatic_correction:
        :param kwargs:
        :return:
        """
        # load from saved model
        if prev_model:
            self.load_weights(prev_model)
        is_classification = 'entropy' in self.model.loss
        monitor = 'val_acc' if is_classification else 'val_mae'
        mode = 'max' if is_classification else 'min'
        dirname = kwargs.pop('dirname', 'callback')
        if not os.path.isdir(dirname):
            os.makedirs(dirname)
        if callbacks is None:
            # with this call back you can stop the model training by `touch STOP`
            callbacks = [ManualStop()]
        train_nb_atoms = [len(i['atom']) for i in train_graphs]
        train_targets = [
            self.target_scaler.transform(i, j)
            for i, j in zip(train_targets, train_nb_atoms)
        ]

        if validation_graphs is not None:
            filepath = os.path.join(
                dirname, '%s_{epoch:05d}_{%s:.6f}.hdf5' % (monitor, monitor))
            val_nb_atoms = [len(i['atom']) for i in validation_graphs]
            validation_targets = [
                self.target_scaler.transform(i, j)
                for i, j in zip(validation_targets, val_nb_atoms)
            ]
            val_inputs = self.graph_converter.get_flat_data(
                validation_graphs, validation_targets)

            val_generator = self._create_generator(*val_inputs,
                                                   batch_size=batch_size)
            steps_per_val = int(np.ceil(len(validation_graphs) / batch_size))
            if automatic_correction:
                callbacks.extend([
                    ReduceLRUponNan(
                        filepath=filepath,
                        monitor=monitor,
                        mode=mode,
                        factor=lr_scaling_factor,
                        patience=patience,
                    )
                ])
            if save_checkpoint:
                callbacks.extend([
                    ModelCheckpointMAE(filepath=filepath,
                                       monitor=monitor,
                                       mode=mode,
                                       save_best_only=True,
                                       save_weights_only=False,
                                       val_gen=val_generator,
                                       steps_per_val=steps_per_val,
                                       target_scaler=self.target_scaler)
                ])
        else:
            val_generator = None
            steps_per_val = None
        train_inputs = self.graph_converter.get_flat_data(
            train_graphs, train_targets)
        # check dimension match
        self.check_dimension(train_graphs[0])
        train_generator = self._create_generator(*train_inputs,
                                                 batch_size=batch_size)
        steps_per_train = int(np.ceil(len(train_graphs) / batch_size))
        self.fit_generator(train_generator,
                           steps_per_epoch=steps_per_train,
                           validation_data=val_generator,
                           validation_steps=steps_per_val,
                           epochs=epochs,
                           verbose=verbose,
                           callbacks=callbacks,
                           **kwargs)