def set_state(self, state):
"""Sets the values of Theano shared variables.
Requires that ``state`` contains values for all the optimization
parameters.
:type state: h5py.File
:param state: HDF5 file that contains the optimization parameters
"""
if not 'optimizer' in state:
raise IncompatibleStateError("Optimizer state is missing.")
h5_optimizer = state['optimizer']
if not 'learning_rate' in h5_optimizer.attrs:
raise IncompatibleStateError("Learning rate is missing from "
"optimizer state.")
self.learning_rate = h5_optimizer.attrs['learning_rate']
for name, param in self.params.items():
if not name in state:
raise IncompatibleStateError("Parameter %s is missing from "
"training state." % name)
new_value = state[name].value
param.set_value(new_value)
if len(new_value.shape) == 0:
logging.debug("%s <- %s", name, str(new_value))
else:
logging.debug("%s <- array%s", name, str(new_value.shape))
def _check_h5_dict(h5_group, variables):
"""Checks that a dictionary matches a HDF5 group and raises an
``IncompatibleStateError`` if not.
The dictionary can store strings, which are represented as HDF5
attributes, and lists of strings, which are represented as attributes of
HDF5 subgroups.
:type h5_group: h5py.Group
:param h5_group: The HD5 group that will store a dictionary of strings
in its attributes
:type variables: dict
:param variables: a dictionary that may contain strings and lists of strings
"""
for variable, values in variables.items():
if isinstance(values, list):
h5_values = h5_group[variable]
for value_id, value in enumerate(values):
h5_value = h5_values.attrs[str(value_id)]
if value != h5_value:
raise IncompatibleStateError(
"Neural network state has {0}={2}, while this "
"architecture has {0}={1}.".format(
variable, value, h5_value))
else:
h5_value = h5_group.attrs[variable]
if values != h5_value:
raise IncompatibleStateError(
"Neural network state has {0}={2}, while this "
"architecture has {0}={1}.".format(
variable, value, h5_value))
def set_state(self, state):
"""Sets the values of Theano shared variables.
Requires that ``state`` contains values for all the neural network
parameters.
:type state: h5py.File
:param state: HDF5 file that contains the neural network parameters
"""
for name, param in self.params.items():
if not name in state:
raise IncompatibleStateError(
"Parameter %s is missing from neural network state." %
name)
new_value = state[name].value
param.set_value(new_value)
if len(new_value.shape) == 0:
logging.debug("%s <- %s", name, str(new_value))
else:
logging.debug("%s <- array%s", name, str(new_value.shape))
try:
self.architecture.check_state(state)
except IncompatibleStateError as error:
raise IncompatibleStateError(
"Attempting to restore state of a network that is incompatible "
"with this architecture. " + str(error))
def from_state(cls, state):
"""Reads the vocabulary from a network state.
:type state: hdf5.File
:param state: HDF5 file that contains the architecture parameters
"""
if 'vocabulary' not in state:
raise IncompatibleStateError(
"Vocabulary is missing from neural network state.")
h5_vocabulary = state['vocabulary']
if 'words' not in h5_vocabulary:
raise IncompatibleStateError(
"Vocabulary parameter 'words' is missing from neural network "
"state.")
id_to_word = h5_vocabulary['words'].value
if 'classes' not in h5_vocabulary:
raise IncompatibleStateError(
"Vocabulary parameter 'classes' is missing from neural network "
"state.")
word_id_to_class_id = h5_vocabulary['classes'].value
if 'probs' not in h5_vocabulary:
raise IncompatibleStateError(
"Vocabulary parameter 'probs' is missing from neural network "
"state.")
num_classes = word_id_to_class_id.max() + 1
word_classes = [None] * num_classes
h5_probs = h5_vocabulary['probs'].value
for word_id, prob in enumerate(h5_probs):
class_id = word_id_to_class_id[word_id]
if word_classes[class_id] is None:
word_class = WordClass(class_id, word_id, prob)
word_classes[class_id] = word_class
else:
word_classes[class_id].add(word_id, prob)
result = cls(id_to_word.tolist(), word_id_to_class_id.tolist(),
word_classes)
if 'unigram_probs' in h5_vocabulary:
result._unigram_probs = h5_vocabulary['unigram_probs'].value
if len(result._unigram_probs) != result.num_words():
raise IncompatibleStateError(
"Incorrect number of word unigram probabilities in neural "
"network state.")
oos_probs = result._unigram_probs[result.num_shortlist_words():]
if oos_probs.size:
logging.debug(
"Out-of-shortlist word log probabilities are in "
"the range [%f, %f].", numpy.log(oos_probs.min()),
numpy.log(oos_probs.max()))
else:
logging.debug("Word unigram probabilities are missing from state.")
return result
def _reset_state(self):
"""Resets the values of Theano shared variables to the current candidate
state.
Sets candidate state index point to the last element in the loaded cost
history.
Requires that if ``state`` is set, it contains values for all the
training parameters.
:type state: h5py.File
:param state: if a HDF5 file is given, reads the the training parameters
from this file, and assumes this is the state of minimum
cost found so far
"""
self._network.set_state(self._candidate_state)
if 'trainer' not in self._candidate_state:
raise IncompatibleStateError("Training state is missing.")
h5_trainer = self._candidate_state['trainer']
if 'epoch_number' not in h5_trainer.attrs:
raise IncompatibleStateError(
"Current epoch number is missing from "
"training state.")
self.epoch_number = int(h5_trainer.attrs['epoch_number'])
if 'update_number' not in h5_trainer.attrs:
raise IncompatibleStateError("Current update number is missing "
"from training state.")
self.update_number = int(h5_trainer.attrs['update_number'])
logging.info("[%d] (%.2f %%) of epoch %d", self.update_number,
self.update_number / self._updates_per_epoch * 100,
self.epoch_number)
if 'cost_history' in h5_trainer:
self._cost_history = h5_trainer['cost_history'].value
if self._cost_history.size == 0:
print(
"Validation set cost history is empty in the training state."
)
self._candidate_index = None
else:
self._candidate_index = self._cost_history.size - 1
self._log_validation()
else:
print("Warning: Validation set cost history is missing from "
"training state. Initializing to empty cost history.")
self._cost_history = numpy.asarray([], dtype=theano.config.floatX)
self._candidate_index = None
self._training_iter.set_state(self._candidate_state)
self._optimizer.set_state(self._candidate_state)
def _check_h5_dict(h5_group, variables):
"""Checks that a dictionary matches a HDF5 group and raises an
``IncompatibleStateError`` if not.
The dictionary can store strings, which are represented as HDF5
attributes, and lists of strings, which are represented as attributes of
HDF5 subgroups.
:type h5_group: h5py.Group
:param h5_group: The HD5 group that will store a dictionary of strings
in its attributes
:type variables: dict
:param variables: a dictionary that may contain strings and lists of
strings
"""
for variable, values in variables.items():
if isinstance(values, list):
# For backward compatibility. Remove at some point.
if (variable == 'devices') and (not variable in h5_group):
if not values:
continue
else:
raise IncompatibleStateError(
"Neural network state does not specify devices, "
"while this architecture uses {}.".format(
', '.join(str(x) for x in values)))
h5_values = h5_group[variable]
for value_id, value in enumerate(values):
h5_value = h5_values.attrs[str(value_id)]
if value != h5_value:
raise IncompatibleStateError(
"Neural network state has {0}={2}, while this "
"architecture has {0}={1}.".format(
variable, value, h5_value))
else:
h5_value = h5_group.attrs[variable]
if values != h5_value:
raise IncompatibleStateError(
"Neural network state has {0}={2}, while this "
"architecture has {0}={1}.".format(
variable, values, h5_value))
def check_state(self, state):
"""Checks that the architecture stored in a state matches this
network architecture, and raises an ``IncompatibleStateError``
if not.
:type state: h5py.File
:param state: HDF5 file that contains the architecture parameters
"""
if not 'architecture' in state:
raise IncompatibleStateError(
"Architecture is missing from neural network state.")
h5_arch = state['architecture']
if not 'inputs' in h5_arch:
raise IncompatibleStateError(
"Architecture parameter 'inputs' is missing from neural "
"network state.")
h5_inputs = h5_arch['inputs']
for input_id, input in enumerate(self.inputs):
h5_input = h5_inputs[str(input_id)]
self._check_h5_dict(h5_input, input)
if not 'layers' in h5_arch:
raise IncompatibleStateError(
"Architecture parameter 'layers' is missing from neural "
"network state.")
h5_layers = h5_arch['layers']
for layer_id, layer in enumerate(self.layers):
h5_layer = h5_layers[str(layer_id)]
self._check_h5_dict(h5_layer, layer)
if not 'output_layer' in h5_arch.attrs:
raise IncompatibleStateError(
"Architecture parameter 'output_layer' is missing from "
"neural network state.")
h5_value = h5_arch.attrs['output_layer']
if self.output_layer != h5_value:
raise IncompatibleStateError(
"Neural network state has output_layer={1}, while "
"this architecture has output_layer={0}.".format(
self.output_layer, h5_value))
def from_state(classname, state):
"""Reads the vocabulary from a network state.
:type state: hdf5.File
:param state: HDF5 file that contains the architecture parameters
"""
if not 'vocabulary' in state:
raise IncompatibleStateError(
"Vocabulary is missing from neural network state.")
h5_vocabulary = state['vocabulary']
if not 'words' in h5_vocabulary:
raise IncompatibleStateError(
"Vocabulary parameter 'words' is missing from neural network "
"state.")
id_to_word = h5_vocabulary['words'].value
if not 'classes' in h5_vocabulary:
raise IncompatibleStateError(
"Vocabulary parameter 'classes' is missing from neural network "
"state.")
word_id_to_class_id = h5_vocabulary['classes'].value
if not 'probs' in h5_vocabulary:
raise IncompatibleStateError(
"Vocabulary parameter 'probs' is missing from neural network "
"state.")
num_classes = word_id_to_class_id.max() + 1
word_classes = [None] * num_classes
h5_probs = h5_vocabulary['probs'].value
for word_id, prob in enumerate(h5_probs):
class_id = word_id_to_class_id[word_id]
if word_classes[class_id] is None:
word_class = Vocabulary.WordClass(class_id, word_id, prob)
word_classes[class_id] = word_class
else:
word_classes[class_id].add(word_id, prob)
return classname(id_to_word.tolist(),
word_id_to_class_id.tolist(),
word_classes)
def set_state(self, state):
"""Sets the values of Theano shared variables.
Requires that ``state`` contains values for all the optimization
parameters.
:type state: h5py.File
:param state: HDF5 file that contains the optimization parameters
"""
if 'optimizer' not in state:
raise IncompatibleStateError("Optimizer state is missing.")
h5_optimizer = state['optimizer']
if 'learning_rate' not in h5_optimizer.attrs:
raise IncompatibleStateError("Learning rate is missing from "
"optimizer state.")
self.learning_rate = h5_optimizer.attrs['learning_rate']
self._params.set_state(state)
def from_state(classname, state):
"""Constructs a description of the network architecture stored in a
state.
:type state: hdf5.File
:param state: HDF5 file that contains the architecture parameters
"""
if not 'architecture' in state:
raise IncompatibleStateError(
"Architecture is missing from neural network state.")
h5_arch = state['architecture']
if not 'inputs' in h5_arch:
raise IncompatibleStateError(
"Architecture parameter 'inputs' is missing from neural "
"network state.")
h5_inputs = h5_arch['inputs']
inputs = []
for input_id in sorted(h5_inputs.keys()):
h5_input = h5_inputs[input_id]
inputs.append(classname._read_h5_dict(h5_input))
if not 'layers' in h5_arch:
raise IncompatibleStateError(
"Architecture parameter 'layers' is missing from neural "
"network state.")
h5_layers = h5_arch['layers']
layers = []
layer_ids = [int(x) for x in h5_layers.keys()]
for layer_id in sorted(layer_ids):
h5_layer = h5_layers[str(layer_id)]
layers.append(classname._read_h5_dict(h5_layer))
if not 'output_layer' in h5_arch.attrs:
raise IncompatibleStateError(
"Architecture parameter 'output_layer' is missing from "
"neural network state.")
output_layer = h5_arch.attrs['output_layer']
return classname(inputs, layers, output_layer)
def set_state(self, state):
"""Sets the values of the shared variables.
Requires that ``state`` contains values for all the parameters.
:type state: h5py.File
:param state: HDF5 file that contains the parameters
"""
for path, param in self._vars.items():
if path not in state:
raise IncompatibleStateError(
"Parameter `%s' is missing from state." % path)
new_value = state[path].value
param.set_value(new_value)
if len(new_value.shape) == 0:
logging.debug("%s <- %s", path, str(new_value))
else:
logging.debug("%s <- array%s", path, str(new_value.shape))
def set_state(self, state):
"""Sets the values of Theano shared variables.
Requires that ``state`` contains values for all the neural network
parameters.
:type state: h5py.File
:param state: HDF5 file that contains the neural network parameters
"""
for layer in self.layers.values():
layer.set_state(state)
try:
self.architecture.check_state(state)
except IncompatibleStateError as error:
raise IncompatibleStateError(
"Attempting to restore state of a network that is incompatible "
"with this architecture. " + str(error))