def saveCrnnNetwork(epoch, layers):
"""
:type epoch: int
:type layers: list[(numpy.ndarray, numpy.ndarray)]
"""
print("Loading Crnn")
from Network import LayerNetwork
from NetworkHiddenLayer import ForwardLayer
from NetworkOutputLayer import OutputLayer
from Pretrain import pretrain_from_config
from EngineBase import EngineBase
pretrain = pretrain_from_config(config)
is_pretrain_epoch = pretrain and epoch <= pretrain.get_train_num_epochs()
modelFilename = config.value("model", None)
assert modelFilename, "need 'model' in config"
filename = EngineBase.epoch_model_filename(modelFilename, epoch,
is_pretrain_epoch)
assert not os.path.exists(filename), "already exists"
if is_pretrain_epoch:
network = pretrain.get_network_for_epoch(epoch)
else:
network = LayerNetwork.from_config_topology(config)
nHiddenLayers = len(network.hidden)
# print network topology
print("Crnn Network layer topology:")
print("input dim:", network.n_in)
print("hidden layer count:", nHiddenLayers)
print("output dim:", network.n_out["classes"])
print("net weights #:", network.num_params())
print("net params:", network.train_params_vars)
print("net output:", network.output["output"])
assert network.n_in == inputDim
#assert network.n_out == outputDim
assert nHiddenLayers + 1 == layerCount # hidden + output layer
assert len(layers) == layerCount
for i, (layerName, hidden) in enumerate(sorted(network.hidden.items())):
# Some checks whether this is a forward-layer.
assert isinstance(hidden, ForwardLayer)
saveCrnnLayer(hidden, *layers[i])
assert isinstance(network.output["output"], OutputLayer)
saveCrnnLayer(network.output["output"], *layers[len(layers) - 1])
import h5py
print(("Save Crnn model under %s" % filename))
model = h5py.File(filename, "w")
network.save_hdf(model, epoch)
model.close()
def saveCrnnNetwork(epoch, layers):
"""
:type epoch: int
:type layers: list[(numpy.ndarray, numpy.ndarray)]
"""
print("Loading Crnn")
from Network import LayerNetwork
from NetworkHiddenLayer import ForwardLayer
from NetworkOutputLayer import OutputLayer
from Pretrain import pretrain_from_config
from EngineBase import EngineBase
pretrain = pretrain_from_config(config)
is_pretrain_epoch = pretrain and epoch <= pretrain.get_train_num_epochs()
modelFilename = config.value("model", None)
assert modelFilename, "need 'model' in config"
filename = EngineBase.epoch_model_filename(modelFilename, epoch, is_pretrain_epoch)
assert not os.path.exists(filename), "already exists"
if is_pretrain_epoch:
network = pretrain.get_network_for_epoch(epoch)
else:
network = LayerNetwork.from_config_topology(config)
nHiddenLayers = len(network.hidden)
# print network topology
print("Crnn Network layer topology:")
print("input dim:", network.n_in)
print("hidden layer count:", nHiddenLayers)
print("output dim:", network.n_out["classes"])
print("net weights #:", network.num_params())
print("net params:", network.train_params_vars)
print("net output:", network.output["output"])
assert network.n_in == inputDim
#assert network.n_out == outputDim
assert nHiddenLayers + 1 == layerCount # hidden + output layer
assert len(layers) == layerCount
for i, (layerName, hidden) in enumerate(sorted(network.hidden.items())):
# Some checks whether this is a forward-layer.
assert isinstance(hidden, ForwardLayer)
saveCrnnLayer(hidden, *layers[i])
assert isinstance(network.output["output"], OutputLayer)
saveCrnnLayer(network.output["output"], *layers[len(layers) - 1])
import h5py
print(("Save Crnn model under %s" % filename))
model = h5py.File(filename, "w")
network.save_hdf(model, epoch)
model.close()
def test_config1():
config = Config()
config.update(config1_dict)
pretrain = pretrain_from_config(config)
assert_equal(pretrain.get_train_num_epochs(), 2)
net1_json = pretrain.get_network_json_for_epoch(1)
net2_json = pretrain.get_network_json_for_epoch(2)
net3_json = pretrain.get_network_json_for_epoch(3)
assert_in("hidden_0", net1_json)
assert_not_in("hidden_1", net1_json)
assert_in("hidden_0", net2_json)
assert_in("hidden_1", net2_json)
assert_equal(net2_json, net3_json)
def test_config1():
config = Config()
config.update(config1_dict)
pretrain = pretrain_from_config(config)
assert_equal(pretrain.get_train_num_epochs(), 2)
net1_json = pretrain.get_network_json_for_epoch(1)
net2_json = pretrain.get_network_json_for_epoch(2)
net3_json = pretrain.get_network_json_for_epoch(3)
assert_in("hidden_0", net1_json)
assert_not_in("hidden_1", net1_json)
assert_in("hidden_0", net2_json)
assert_in("hidden_1", net2_json)
assert_equal(net2_json, net3_json)
def from_config_topology(cls, config, mask=None, **kwargs):
"""
:type config: Config.Config
:param str mask: e.g. "unity" or None ("dropout"). "unity" is for testing.
:rtype: LayerNetwork
"""
json_content = cls.json_from_config(config, mask=mask)
from Pretrain import find_pretrain_wrap_values, pretrain_from_config
if find_pretrain_wrap_values(json_content):
pretrain = pretrain_from_config(config=config)
assert pretrain, "found Pretrain WrapEpochValue but no pretrain configured"
json_content = pretrain.get_final_network_json()
return cls.from_json_and_config(json_content, config, mask=mask, **kwargs)
def main(argv):
argparser = argparse.ArgumentParser(description='Dump network as JSON.')
argparser.add_argument('crnn_config_file')
argparser.add_argument('--epoch', default=1, type=int)
argparser.add_argument('--out', default="/dev/stdout")
args = argparser.parse_args(argv[1:])
init(configFilename=args.crnn_config_file, commandLineOptions=[])
pretrain = pretrain_from_config(config)
if pretrain:
network = pretrain.get_network_for_epoch(args.epoch)
else:
network = LayerNetwork.from_config_topology(config)
json_data = network.to_json_content()
f = open(args.out, 'w')
print(json.dumps(json_data, indent=2, sort_keys=True), file=f)
f.close()
rnn.finalize()
def init(config_str, config_dataset, use_pretrain, epoch, verbosity):
"""
:param str config_str: either filename to config-file, or dict for dataset
:param str|None config_dataset:
:param bool use_pretrain: might overwrite config options, or even the dataset
:param int epoch:
:param int verbosity:
"""
rnn.init_better_exchook()
rnn.init_thread_join_hack()
dataset_opts = None
config_filename = None
if config_str.strip().startswith("{"):
print("Using dataset %s." % config_str)
dataset_opts = eval(config_str.strip())
elif config_str.endswith(".hdf"):
dataset_opts = {"class": "HDFDataset", "files": [config_str]}
print("Using dataset %r." % dataset_opts)
assert os.path.exists(config_str)
else:
config_filename = config_str
print("Using config file %r." % config_filename)
assert os.path.exists(config_filename)
rnn.init_config(config_filename=config_filename, default_config={"cache_size": "0"})
global config
config = rnn.config
config.set("log", None)
config.set("log_verbosity", verbosity)
rnn.init_log()
print("Returnn %s starting up." % __file__, file=log.v2)
rnn.returnn_greeting()
rnn.init_faulthandler()
rnn.init_config_json_network()
Util.BackendEngine.select_engine(config=config)
if not dataset_opts:
if config_dataset:
dataset_opts = "config:%s" % config_dataset
else:
dataset_opts = "config:train"
if use_pretrain:
from Pretrain import pretrain_from_config
pretrain = pretrain_from_config(config)
if pretrain:
print("Using pretrain %s, epoch %i" % (pretrain, epoch), file=log.v2)
net_dict = pretrain.get_network_json_for_epoch(epoch=epoch)
if "#config" in net_dict:
config_overwrites = net_dict["#config"]
print("Pretrain overwrites these config options:", file=log.v2)
assert isinstance(config_overwrites, dict)
for key, value in sorted(config_overwrites.items()):
assert isinstance(key, str)
orig_value = config.typed_dict.get(key, None)
if isinstance(orig_value, dict) and isinstance(value, dict):
diff_str = "\n" + Util.dict_diff_str(orig_value, value)
elif isinstance(value, dict):
diff_str = "\n%r ->\n%s" % (orig_value, pformat(value))
else:
diff_str = " %r -> %r" % (orig_value, value)
print("Config key %r for epoch %i:%s" % (key, epoch, diff_str), file=log.v2)
config.set(key, value)
else:
print("No config overwrites for this epoch.", file=log.v2)
else:
print("No pretraining used.", file=log.v2)
elif config.typed_dict.get("pretrain", None):
print("Not using pretrain.", file=log.v2)
dataset_default_opts = {}
Dataset.kwargs_update_from_config(config, dataset_default_opts)
print("Using dataset:", dataset_opts, file=log.v2)
global dataset
dataset = init_dataset(dataset_opts, default_kwargs=dataset_default_opts)
assert isinstance(dataset, Dataset)
dataset.init_seq_order(epoch=epoch)
def test_config3():
config = Config()
config.update(config3_dict)
config.network_topology_json = config3_json
pretrain = pretrain_from_config(config)
assert_equal(pretrain.get_train_num_epochs(), 3)
def test_config2():
config = Config()
config.update(config2_dict)
pretrain = pretrain_from_config(config)
assert_equal(pretrain.get_train_num_epochs(), 3)
def init_network_from_config(self, config):
self.pretrain = pretrain_from_config(config)
self.max_seqs = config.int('max_seqs', -1)
self.max_seq_length_eval = config.int('max_seq_length_eval', 2e31)
self.compression = config.bool('compression', False)
epoch, model_epoch_filename = self.get_epoch_model(config)
assert model_epoch_filename or self.start_epoch
self.epoch = epoch or self.start_epoch
if model_epoch_filename:
print("loading weights from", model_epoch_filename, file=log.v2)
last_model_hdf = h5py.File(model_epoch_filename, "r")
else:
last_model_hdf = None
if config.bool('initialize_from_model', False):
# That's only about the topology, not the params.
print("initializing network topology from model", file=log.v5)
assert last_model_hdf, "last model not specified. use 'load' in config. or don't use 'initialize_from_model'"
network = LayerNetwork.from_hdf_model_topology(last_model_hdf)
else:
if self.pretrain:
# This would be obsolete if we don't want to load an existing model.
# In self.init_train_epoch(), we initialize a new model.
network = self.pretrain.get_network_for_epoch(epoch or self.start_epoch)
else:
network = LayerNetwork.from_config_topology(config)
# We have the parameters randomly initialized at this point.
# In training, as an initialization, we can copy over the params of an imported model,
# where our topology might slightly differ from the imported model.
if config.value('import_model_train_epoch1', '') and self.start_epoch == 1:
assert last_model_hdf
old_network = LayerNetwork.from_hdf_model_topology(last_model_hdf)
old_network.load_hdf(last_model_hdf)
last_model_hdf.close()
# Copy params to new network.
from NetworkCopyUtils import intelli_copy_layer
# network.hidden are the input + all hidden layers.
for layer_name, layer in sorted(old_network.hidden.items()):
print("Copy hidden layer %s" % layer_name, file=log.v3)
intelli_copy_layer(layer, network.hidden[layer_name])
for layer_name, layer in sorted(old_network.output.items()):
if layer_name in network.output:
print("Copy output layer %s" % layer_name, file=log.v3)
intelli_copy_layer(layer, network.output[layer_name])
else:
print("Did not copy output layer %s" % layer_name, file=log.v3)
print("Not copied hidden: %s" % sorted(set(network.hidden.keys()).symmetric_difference(old_network.hidden.keys())), file=log.v3)
print("Not copied output: %s" % sorted(set(network.output.keys()).symmetric_difference(old_network.output.keys())), file=log.v3)
# Maybe load existing model parameters.
elif last_model_hdf:
network.load_hdf(last_model_hdf)
last_model_hdf.close()
EngineUtil.maybe_subtract_priors(network, self.train_data, config)
self.network = network
if config.has('dump_json'):
self.network_dump_json(config.value('dump_json', ''))
self.print_network_info()
def test_config3(): config = Config() config.update(config3_dict) config.network_topology_json = config3_json pretrain = pretrain_from_config(config) assert_equal(pretrain.get_train_num_epochs(), 3)
def test_config2(): config = Config() config.update(config2_dict) pretrain = pretrain_from_config(config) assert_equal(pretrain.get_train_num_epochs(), 3)
def test_init_config1(): config = Config() config.update(config1_dict) pretrain = pretrain_from_config(config) assert_true(pretrain)
def init_network_from_config(self, config):
self.pretrain = pretrain_from_config(config)
self.max_seqs = config.int('max_seqs', -1)
self.max_seq_length_eval = config.int('max_seq_length_eval', 2e31)
self.compression = config.bool('compression', False)
epoch, model_epoch_filename = self.get_epoch_model(config)
assert model_epoch_filename or self.start_epoch
self.epoch = epoch or self.start_epoch
if model_epoch_filename:
print("loading weights from", model_epoch_filename, file=log.v2)
last_model_hdf = h5py.File(model_epoch_filename, "r")
else:
last_model_hdf = None
if config.bool('initialize_from_model', False):
# That's only about the topology, not the params.
print("initializing network topology from model", file=log.v5)
assert last_model_hdf, "last model not specified. use 'load' in config. or don't use 'initialize_from_model'"
network = LayerNetwork.from_hdf_model_topology(last_model_hdf)
else:
if self.pretrain:
# This would be obsolete if we don't want to load an existing model.
# In self.init_train_epoch(), we initialize a new model.
network = self.pretrain.get_network_for_epoch(
epoch or self.start_epoch)
else:
network = LayerNetwork.from_config_topology(config)
# We have the parameters randomly initialized at this point.
# In training, as an initialization, we can copy over the params of an imported model,
# where our topology might slightly differ from the imported model.
if config.value('import_model_train_epoch1',
'') and self.start_epoch == 1:
assert last_model_hdf
old_network = LayerNetwork.from_hdf_model_topology(last_model_hdf)
old_network.load_hdf(last_model_hdf)
last_model_hdf.close()
# Copy params to new network.
from NetworkCopyUtils import intelli_copy_layer
# network.hidden are the input + all hidden layers.
for layer_name, layer in sorted(old_network.hidden.items()):
print("Copy hidden layer %s" % layer_name, file=log.v3)
intelli_copy_layer(layer, network.hidden[layer_name])
for layer_name, layer in sorted(old_network.output.items()):
if layer_name in network.output:
print("Copy output layer %s" % layer_name, file=log.v3)
intelli_copy_layer(layer, network.output[layer_name])
else:
print("Did not copy output layer %s" % layer_name,
file=log.v3)
print("Not copied hidden: %s" % sorted(
set(network.hidden.keys()).symmetric_difference(
old_network.hidden.keys())),
file=log.v3)
print("Not copied output: %s" % sorted(
set(network.output.keys()).symmetric_difference(
old_network.output.keys())),
file=log.v3)
# Maybe load existing model parameters.
elif last_model_hdf:
network.load_hdf(last_model_hdf)
last_model_hdf.close()
EngineUtil.maybe_subtract_priors(network, self.train_data, config)
self.network = network
if config.has('dump_json'):
self.network_dump_json(config.value('dump_json', ''))
self.print_network_info()
def test_init_config1():
config = Config()
config.update(config1_dict)
pretrain = pretrain_from_config(config)
assert_true(pretrain)
def demo():
"""
Demo run. Given some learning rate file (with scores / existing lrs), will calculate how lrs would have been set,
given some config.
"""
import better_exchook
better_exchook.install()
import rnn
import sys
if len(sys.argv) <= 1:
print("usage: python %s [config] [other options] [++check_learning_rates 1]" % __file__)
print(
("example usage: "
"python %s ++learning_rate_control newbob ++learning_rate_file newbob.data ++learning_rate 0.001") % __file__)
rnn.init_config(command_line_options=sys.argv[1:])
# noinspection PyProtectedMember
rnn.config._hack_value_reading_debug()
rnn.config.update({"log": []})
rnn.init_log()
rnn.init_backend_engine()
check_lr = rnn.config.bool("check_learning_rates", False)
from Pretrain import pretrain_from_config
pretrain = pretrain_from_config(rnn.config)
first_non_pretrain_epoch = 1
pretrain_learning_rate = None
if pretrain:
first_non_pretrain_epoch = pretrain.get_train_num_epochs() + 1
log.initialize(verbosity=[5])
control = load_learning_rate_control_from_config(rnn.config)
print("LearningRateControl: %r" % control)
if not control.epoch_data:
print("No epoch data so far.")
return
first_epoch = min(control.epoch_data.keys())
if first_epoch != 1:
print("Strange, first epoch from epoch data is %i." % first_epoch)
print("Error key: %s from %r" % (control.get_error_key(epoch=first_epoch), control.epoch_data[first_epoch].error))
if pretrain:
pretrain_learning_rate = rnn.config.float('pretrain_learning_rate', control.default_learning_rate)
max_epoch = max(control.epoch_data.keys())
for epoch in range(1, max_epoch + 2): # all epochs [1..max_epoch+1]
old_learning_rate = None
if epoch in control.epoch_data:
old_learning_rate = control.epoch_data[epoch].learning_rate
if epoch < first_non_pretrain_epoch:
learning_rate = pretrain_learning_rate
s = "Pretrain epoch %i, fixed learning rate: %s (was: %s)" % (epoch, learning_rate, old_learning_rate)
elif 1 < first_non_pretrain_epoch == epoch:
learning_rate = control.default_learning_rate
s = "First epoch after pretrain, epoch %i, fixed learning rate: %s (was %s)" % (
epoch, learning_rate, old_learning_rate)
else:
learning_rate = control.calc_new_learning_rate_for_epoch(epoch)
s = "Calculated learning rate for epoch %i: %s (was: %s)" % (epoch, learning_rate, old_learning_rate)
if learning_rate < control.min_learning_rate:
learning_rate = control.min_learning_rate
s += ", clipped to %s" % learning_rate
s += ", previous relative error: %s" % control.calc_relative_error(epoch - 2, epoch - 1)
if hasattr(control, "_calc_recent_mean_relative_error"):
# noinspection PyProtectedMember
s += ", previous mean relative error: %s" % control._calc_recent_mean_relative_error(epoch)
print(s)
if check_lr and old_learning_rate is not None:
if old_learning_rate != learning_rate:
print("Learning rate is different in epoch %i!" % epoch)
sys.exit(1)
# Overwrite new learning rate so that the calculation for further learning rates stays consistent.
if epoch in control.epoch_data:
control.epoch_data[epoch].learning_rate = learning_rate
else:
control.epoch_data[epoch] = control.EpochData(learningRate=learning_rate)
print("Finished, last stored epoch was %i." % max_epoch)
def demo():
"""
Demo run. Given some learning rate file (with scores / existing lrs), will calculate how lrs would have been set,
given some config.
"""
import better_exchook
better_exchook.install()
import rnn
import sys
if len(sys.argv) <= 1:
print("usage: python %s [config] [other options] [++check_learning_rates 1]" % __file__)
print(
("example usage: "
"python %s ++learning_rate_control newbob ++learning_rate_file newbob.data ++learning_rate 0.001") % __file__)
rnn.init_config(command_line_options=sys.argv[1:])
# noinspection PyProtectedMember
rnn.config._hack_value_reading_debug()
rnn.config.update({"log": []})
rnn.init_log()
rnn.init_backend_engine()
check_lr = rnn.config.bool("check_learning_rates", False)
from Pretrain import pretrain_from_config
pretrain = pretrain_from_config(rnn.config)
first_non_pretrain_epoch = 1
pretrain_learning_rate = None
if pretrain:
first_non_pretrain_epoch = pretrain.get_train_num_epochs() + 1
log.initialize(verbosity=[5])
control = load_learning_rate_control_from_config(rnn.config)
print("LearningRateControl: %r" % control)
if not control.epoch_data:
print("No epoch data so far.")
return
first_epoch = min(control.epoch_data.keys())
if first_epoch != 1:
print("Strange, first epoch from epoch data is %i." % first_epoch)
print("Error key: %s from %r" % (control.get_error_key(epoch=first_epoch), control.epoch_data[first_epoch].error))
if pretrain:
pretrain_learning_rate = rnn.config.float('pretrain_learning_rate', control.default_learning_rate)
max_epoch = max(control.epoch_data.keys())
for epoch in range(1, max_epoch + 2): # all epochs [1..max_epoch+1]
old_learning_rate = None
if epoch in control.epoch_data:
old_learning_rate = control.epoch_data[epoch].learning_rate
if epoch < first_non_pretrain_epoch:
learning_rate = pretrain_learning_rate
s = "Pretrain epoch %i, fixed learning rate: %s (was: %s)" % (epoch, learning_rate, old_learning_rate)
elif 1 < first_non_pretrain_epoch == epoch:
learning_rate = control.default_learning_rate
s = "First epoch after pretrain, epoch %i, fixed learning rate: %s (was %s)" % (
epoch, learning_rate, old_learning_rate)
else:
learning_rate = control.calc_new_learning_rate_for_epoch(epoch)
s = "Calculated learning rate for epoch %i: %s (was: %s)" % (epoch, learning_rate, old_learning_rate)
if learning_rate < control.min_learning_rate:
learning_rate = control.min_learning_rate
s += ", clipped to %s" % learning_rate
s += ", previous relative error: %s" % control.calc_relative_error(epoch - 2, epoch - 1)
if hasattr(control, "_calc_recent_mean_relative_error"):
# noinspection PyProtectedMember
s += ", previous mean relative error: %s" % control._calc_recent_mean_relative_error(epoch)
print(s)
if check_lr and old_learning_rate is not None:
if old_learning_rate != learning_rate:
print("Learning rate is different in epoch %i!" % epoch)
sys.exit(1)
# Overwrite new learning rate so that the calculation for further learning rates stays consistent.
if epoch in control.epoch_data:
control.epoch_data[epoch].learning_rate = learning_rate
else:
control.epoch_data[epoch] = control.EpochData(learningRate=learning_rate)
print("Finished, last stored epoch was %i." % max_epoch)