Esempio n. 1
0
    def set_state(self, state):
        """Sets the values of Theano shared variables.

        Requires that ``state`` contains values for all the optimization
        parameters.

        :type state: h5py.File
        :param state: HDF5 file that contains the optimization parameters
        """

        if not 'optimizer' in state:
            raise IncompatibleStateError("Optimizer state is missing.")
        h5_optimizer = state['optimizer']

        if not 'learning_rate' in h5_optimizer.attrs:
            raise IncompatibleStateError("Learning rate is missing from "
                                         "optimizer state.")
        self.learning_rate = h5_optimizer.attrs['learning_rate']

        for name, param in self.params.items():
            if not name in state:
                raise IncompatibleStateError("Parameter %s is missing from "
                                             "training state." % name)
            new_value = state[name].value
            param.set_value(new_value)
            if len(new_value.shape) == 0:
                logging.debug("%s <- %s", name, str(new_value))
            else:
                logging.debug("%s <- array%s", name, str(new_value.shape))
Esempio n. 2
0
    def _check_h5_dict(h5_group, variables):
        """Checks that a dictionary matches a HDF5 group and raises an
        ``IncompatibleStateError`` if not.

        The dictionary can store strings, which are represented as HDF5
        attributes, and lists of strings, which are represented as attributes of
        HDF5 subgroups.

        :type h5_group: h5py.Group
        :param h5_group: The HD5 group that will store a dictionary of strings
                         in its attributes

        :type variables: dict
        :param variables: a dictionary that may contain strings and lists of strings
        """

        for variable, values in variables.items():
            if isinstance(values, list):
                h5_values = h5_group[variable]
                for value_id, value in enumerate(values):
                    h5_value = h5_values.attrs[str(value_id)]
                    if value != h5_value:
                        raise IncompatibleStateError(
                            "Neural network state has {0}={2}, while this "
                            "architecture has {0}={1}.".format(
                                variable, value, h5_value))
            else:
                h5_value = h5_group.attrs[variable]
                if values != h5_value:
                    raise IncompatibleStateError(
                        "Neural network state has {0}={2}, while this "
                        "architecture has {0}={1}.".format(
                            variable, value, h5_value))
Esempio n. 3
0
    def set_state(self, state):
        """Sets the values of Theano shared variables.

        Requires that ``state`` contains values for all the neural network
        parameters.

        :type state: h5py.File
        :param state: HDF5 file that contains the neural network parameters
        """

        for name, param in self.params.items():
            if not name in state:
                raise IncompatibleStateError(
                    "Parameter %s is missing from neural network state." %
                    name)
            new_value = state[name].value
            param.set_value(new_value)
            if len(new_value.shape) == 0:
                logging.debug("%s <- %s", name, str(new_value))
            else:
                logging.debug("%s <- array%s", name, str(new_value.shape))
        try:
            self.architecture.check_state(state)
        except IncompatibleStateError as error:
            raise IncompatibleStateError(
                "Attempting to restore state of a network that is incompatible "
                "with this architecture. " + str(error))
Esempio n. 4
0
    def from_state(cls, state):
        """Reads the vocabulary from a network state.

        :type state: hdf5.File
        :param state: HDF5 file that contains the architecture parameters
        """

        if 'vocabulary' not in state:
            raise IncompatibleStateError(
                "Vocabulary is missing from neural network state.")
        h5_vocabulary = state['vocabulary']

        if 'words' not in h5_vocabulary:
            raise IncompatibleStateError(
                "Vocabulary parameter 'words' is missing from neural network "
                "state.")
        id_to_word = h5_vocabulary['words'].value

        if 'classes' not in h5_vocabulary:
            raise IncompatibleStateError(
                "Vocabulary parameter 'classes' is missing from neural network "
                "state.")
        word_id_to_class_id = h5_vocabulary['classes'].value

        if 'probs' not in h5_vocabulary:
            raise IncompatibleStateError(
                "Vocabulary parameter 'probs' is missing from neural network "
                "state.")
        num_classes = word_id_to_class_id.max() + 1
        word_classes = [None] * num_classes
        h5_probs = h5_vocabulary['probs'].value
        for word_id, prob in enumerate(h5_probs):
            class_id = word_id_to_class_id[word_id]
            if word_classes[class_id] is None:
                word_class = WordClass(class_id, word_id, prob)
                word_classes[class_id] = word_class
            else:
                word_classes[class_id].add(word_id, prob)

        result = cls(id_to_word.tolist(), word_id_to_class_id.tolist(),
                     word_classes)

        if 'unigram_probs' in h5_vocabulary:
            result._unigram_probs = h5_vocabulary['unigram_probs'].value
            if len(result._unigram_probs) != result.num_words():
                raise IncompatibleStateError(
                    "Incorrect number of word unigram probabilities in neural "
                    "network state.")
            oos_probs = result._unigram_probs[result.num_shortlist_words():]
            if oos_probs.size:
                logging.debug(
                    "Out-of-shortlist word log probabilities are in "
                    "the range [%f, %f].", numpy.log(oos_probs.min()),
                    numpy.log(oos_probs.max()))
        else:
            logging.debug("Word unigram probabilities are missing from state.")

        return result
Esempio n. 5
0
    def _reset_state(self):
        """Resets the values of Theano shared variables to the current candidate
         state.

        Sets candidate state index point to the last element in the loaded cost
        history.

        Requires that if ``state`` is set, it contains values for all the
        training parameters.

        :type state: h5py.File
        :param state: if a HDF5 file is given, reads the the training parameters
                      from this file, and assumes this is the state of minimum
                      cost found so far
        """

        self._network.set_state(self._candidate_state)

        if 'trainer' not in self._candidate_state:
            raise IncompatibleStateError("Training state is missing.")
        h5_trainer = self._candidate_state['trainer']

        if 'epoch_number' not in h5_trainer.attrs:
            raise IncompatibleStateError(
                "Current epoch number is missing from "
                "training state.")
        self.epoch_number = int(h5_trainer.attrs['epoch_number'])

        if 'update_number' not in h5_trainer.attrs:
            raise IncompatibleStateError("Current update number is missing "
                                         "from training state.")
        self.update_number = int(h5_trainer.attrs['update_number'])

        logging.info("[%d] (%.2f %%) of epoch %d", self.update_number,
                     self.update_number / self._updates_per_epoch * 100,
                     self.epoch_number)

        if 'cost_history' in h5_trainer:
            self._cost_history = h5_trainer['cost_history'].value
            if self._cost_history.size == 0:
                print(
                    "Validation set cost history is empty in the training state."
                )
                self._candidate_index = None
            else:
                self._candidate_index = self._cost_history.size - 1
                self._log_validation()
        else:
            print("Warning: Validation set cost history is missing from "
                  "training state. Initializing to empty cost history.")
            self._cost_history = numpy.asarray([], dtype=theano.config.floatX)
            self._candidate_index = None

        self._training_iter.set_state(self._candidate_state)
        self._optimizer.set_state(self._candidate_state)
Esempio n. 6
0
    def _check_h5_dict(h5_group, variables):
        """Checks that a dictionary matches a HDF5 group and raises an
        ``IncompatibleStateError`` if not.

        The dictionary can store strings, which are represented as HDF5
        attributes, and lists of strings, which are represented as attributes of
        HDF5 subgroups.

        :type h5_group: h5py.Group
        :param h5_group: The HD5 group that will store a dictionary of strings
                         in its attributes

        :type variables: dict
        :param variables: a dictionary that may contain strings and lists of
                          strings
        """

        for variable, values in variables.items():
            if isinstance(values, list):
                # For backward compatibility. Remove at some point.
                if (variable == 'devices') and (not variable in h5_group):
                    if not values:
                        continue
                    else:
                        raise IncompatibleStateError(
                            "Neural network state does not specify devices, "
                            "while this architecture uses {}.".format(
                                ', '.join(str(x) for x in values)))
                h5_values = h5_group[variable]
                for value_id, value in enumerate(values):
                    h5_value = h5_values.attrs[str(value_id)]
                    if value != h5_value:
                        raise IncompatibleStateError(
                            "Neural network state has {0}={2}, while this "
                            "architecture has {0}={1}.".format(
                                variable, value, h5_value))
            else:
                h5_value = h5_group.attrs[variable]
                if values != h5_value:
                    raise IncompatibleStateError(
                        "Neural network state has {0}={2}, while this "
                        "architecture has {0}={1}.".format(
                            variable, values, h5_value))
Esempio n. 7
0
    def check_state(self, state):
        """Checks that the architecture stored in a state matches this
        network architecture, and raises an ``IncompatibleStateError``
        if not.

        :type state: h5py.File
        :param state: HDF5 file that contains the architecture parameters
        """

        if not 'architecture' in state:
            raise IncompatibleStateError(
                "Architecture is missing from neural network state.")
        h5_arch = state['architecture']

        if not 'inputs' in h5_arch:
            raise IncompatibleStateError(
                "Architecture parameter 'inputs' is missing from neural "
                "network state.")
        h5_inputs = h5_arch['inputs']
        for input_id, input in enumerate(self.inputs):
            h5_input = h5_inputs[str(input_id)]
            self._check_h5_dict(h5_input, input)

        if not 'layers' in h5_arch:
            raise IncompatibleStateError(
                "Architecture parameter 'layers' is missing from neural "
                "network state.")
        h5_layers = h5_arch['layers']
        for layer_id, layer in enumerate(self.layers):
            h5_layer = h5_layers[str(layer_id)]
            self._check_h5_dict(h5_layer, layer)

        if not 'output_layer' in h5_arch.attrs:
            raise IncompatibleStateError(
                "Architecture parameter 'output_layer' is missing from "
                "neural network state.")
        h5_value = h5_arch.attrs['output_layer']
        if self.output_layer != h5_value:
            raise IncompatibleStateError(
                "Neural network state has output_layer={1}, while "
                "this architecture has output_layer={0}.".format(
                    self.output_layer, h5_value))
Esempio n. 8
0
    def from_state(classname, state):
        """Reads the vocabulary from a network state.

        :type state: hdf5.File
        :param state: HDF5 file that contains the architecture parameters
        """

        if not 'vocabulary' in state:
            raise IncompatibleStateError(
                "Vocabulary is missing from neural network state.")
        h5_vocabulary = state['vocabulary']

        if not 'words' in h5_vocabulary:
            raise IncompatibleStateError(
                "Vocabulary parameter 'words' is missing from neural network "
                "state.")
        id_to_word = h5_vocabulary['words'].value

        if not 'classes' in h5_vocabulary:
            raise IncompatibleStateError(
                "Vocabulary parameter 'classes' is missing from neural network "
                "state.")
        word_id_to_class_id = h5_vocabulary['classes'].value

        if not 'probs' in h5_vocabulary:
            raise IncompatibleStateError(
                "Vocabulary parameter 'probs' is missing from neural network "
                "state.")
        num_classes = word_id_to_class_id.max() + 1
        word_classes = [None] * num_classes
        h5_probs = h5_vocabulary['probs'].value
        for word_id, prob in enumerate(h5_probs):
            class_id = word_id_to_class_id[word_id]
            if word_classes[class_id] is None:
                word_class = Vocabulary.WordClass(class_id, word_id, prob)
                word_classes[class_id] = word_class
            else:
                word_classes[class_id].add(word_id, prob)

        return classname(id_to_word.tolist(),
                         word_id_to_class_id.tolist(),
                         word_classes)
Esempio n. 9
0
    def set_state(self, state):
        """Sets the values of Theano shared variables.

        Requires that ``state`` contains values for all the optimization
        parameters.

        :type state: h5py.File
        :param state: HDF5 file that contains the optimization parameters
        """

        if 'optimizer' not in state:
            raise IncompatibleStateError("Optimizer state is missing.")
        h5_optimizer = state['optimizer']

        if 'learning_rate' not in h5_optimizer.attrs:
            raise IncompatibleStateError("Learning rate is missing from "
                                         "optimizer state.")
        self.learning_rate = h5_optimizer.attrs['learning_rate']

        self._params.set_state(state)
Esempio n. 10
0
    def from_state(classname, state):
        """Constructs a description of the network architecture stored in a
        state.

        :type state: hdf5.File
        :param state: HDF5 file that contains the architecture parameters
        """

        if not 'architecture' in state:
            raise IncompatibleStateError(
                "Architecture is missing from neural network state.")
        h5_arch = state['architecture']

        if not 'inputs' in h5_arch:
            raise IncompatibleStateError(
                "Architecture parameter 'inputs' is missing from neural "
                "network state.")
        h5_inputs = h5_arch['inputs']
        inputs = []
        for input_id in sorted(h5_inputs.keys()):
            h5_input = h5_inputs[input_id]
            inputs.append(classname._read_h5_dict(h5_input))

        if not 'layers' in h5_arch:
            raise IncompatibleStateError(
                "Architecture parameter 'layers' is missing from neural "
                "network state.")
        h5_layers = h5_arch['layers']
        layers = []
        layer_ids = [int(x) for x in h5_layers.keys()]
        for layer_id in sorted(layer_ids):
            h5_layer = h5_layers[str(layer_id)]
            layers.append(classname._read_h5_dict(h5_layer))

        if not 'output_layer' in h5_arch.attrs:
            raise IncompatibleStateError(
                "Architecture parameter 'output_layer' is missing from "
                "neural network state.")
        output_layer = h5_arch.attrs['output_layer']

        return classname(inputs, layers, output_layer)
Esempio n. 11
0
    def set_state(self, state):
        """Sets the values of the shared variables.

        Requires that ``state`` contains values for all the parameters.

        :type state: h5py.File
        :param state: HDF5 file that contains the parameters
        """

        for path, param in self._vars.items():
            if path not in state:
                raise IncompatibleStateError(
                    "Parameter `%s' is missing from state." % path)
            new_value = state[path].value
            param.set_value(new_value)
            if len(new_value.shape) == 0:
                logging.debug("%s <- %s", path, str(new_value))
            else:
                logging.debug("%s <- array%s", path, str(new_value.shape))
Esempio n. 12
0
    def set_state(self, state):
        """Sets the values of Theano shared variables.

        Requires that ``state`` contains values for all the neural network
        parameters.

        :type state: h5py.File
        :param state: HDF5 file that contains the neural network parameters
        """

        for layer in self.layers.values():
            layer.set_state(state)

        try:
            self.architecture.check_state(state)
        except IncompatibleStateError as error:
            raise IncompatibleStateError(
                "Attempting to restore state of a network that is incompatible "
                "with this architecture. " + str(error))