def saveCrnnNetwork(epoch, layers): """ :type epoch: int :type layers: list[(numpy.ndarray, numpy.ndarray)] """ print("Loading Crnn") from Network import LayerNetwork from NetworkHiddenLayer import ForwardLayer from NetworkOutputLayer import OutputLayer from Pretrain import pretrain_from_config from EngineBase import EngineBase pretrain = pretrain_from_config(config) is_pretrain_epoch = pretrain and epoch <= pretrain.get_train_num_epochs() modelFilename = config.value("model", None) assert modelFilename, "need 'model' in config" filename = EngineBase.epoch_model_filename(modelFilename, epoch, is_pretrain_epoch) assert not os.path.exists(filename), "already exists" if is_pretrain_epoch: network = pretrain.get_network_for_epoch(epoch) else: network = LayerNetwork.from_config_topology(config) nHiddenLayers = len(network.hidden) # print network topology print("Crnn Network layer topology:") print("input dim:", network.n_in) print("hidden layer count:", nHiddenLayers) print("output dim:", network.n_out["classes"]) print("net weights #:", network.num_params()) print("net params:", network.train_params_vars) print("net output:", network.output["output"]) assert network.n_in == inputDim #assert network.n_out == outputDim assert nHiddenLayers + 1 == layerCount # hidden + output layer assert len(layers) == layerCount for i, (layerName, hidden) in enumerate(sorted(network.hidden.items())): # Some checks whether this is a forward-layer. assert isinstance(hidden, ForwardLayer) saveCrnnLayer(hidden, *layers[i]) assert isinstance(network.output["output"], OutputLayer) saveCrnnLayer(network.output["output"], *layers[len(layers) - 1]) import h5py print(("Save Crnn model under %s" % filename)) model = h5py.File(filename, "w") network.save_hdf(model, epoch) model.close()
def test_config1(): config = Config() config.update(config1_dict) pretrain = pretrain_from_config(config) assert_equal(pretrain.get_train_num_epochs(), 2) net1_json = pretrain.get_network_json_for_epoch(1) net2_json = pretrain.get_network_json_for_epoch(2) net3_json = pretrain.get_network_json_for_epoch(3) assert_in("hidden_0", net1_json) assert_not_in("hidden_1", net1_json) assert_in("hidden_0", net2_json) assert_in("hidden_1", net2_json) assert_equal(net2_json, net3_json)
def from_config_topology(cls, config, mask=None, **kwargs): """ :type config: Config.Config :param str mask: e.g. "unity" or None ("dropout"). "unity" is for testing. :rtype: LayerNetwork """ json_content = cls.json_from_config(config, mask=mask) from Pretrain import find_pretrain_wrap_values, pretrain_from_config if find_pretrain_wrap_values(json_content): pretrain = pretrain_from_config(config=config) assert pretrain, "found Pretrain WrapEpochValue but no pretrain configured" json_content = pretrain.get_final_network_json() return cls.from_json_and_config(json_content, config, mask=mask, **kwargs)
def main(argv): argparser = argparse.ArgumentParser(description='Dump network as JSON.') argparser.add_argument('crnn_config_file') argparser.add_argument('--epoch', default=1, type=int) argparser.add_argument('--out', default="/dev/stdout") args = argparser.parse_args(argv[1:]) init(configFilename=args.crnn_config_file, commandLineOptions=[]) pretrain = pretrain_from_config(config) if pretrain: network = pretrain.get_network_for_epoch(args.epoch) else: network = LayerNetwork.from_config_topology(config) json_data = network.to_json_content() f = open(args.out, 'w') print(json.dumps(json_data, indent=2, sort_keys=True), file=f) f.close() rnn.finalize()
def init(config_str, config_dataset, use_pretrain, epoch, verbosity): """ :param str config_str: either filename to config-file, or dict for dataset :param str|None config_dataset: :param bool use_pretrain: might overwrite config options, or even the dataset :param int epoch: :param int verbosity: """ rnn.init_better_exchook() rnn.init_thread_join_hack() dataset_opts = None config_filename = None if config_str.strip().startswith("{"): print("Using dataset %s." % config_str) dataset_opts = eval(config_str.strip()) elif config_str.endswith(".hdf"): dataset_opts = {"class": "HDFDataset", "files": [config_str]} print("Using dataset %r." % dataset_opts) assert os.path.exists(config_str) else: config_filename = config_str print("Using config file %r." % config_filename) assert os.path.exists(config_filename) rnn.init_config(config_filename=config_filename, default_config={"cache_size": "0"}) global config config = rnn.config config.set("log", None) config.set("log_verbosity", verbosity) rnn.init_log() print("Returnn %s starting up." % __file__, file=log.v2) rnn.returnn_greeting() rnn.init_faulthandler() rnn.init_config_json_network() Util.BackendEngine.select_engine(config=config) if not dataset_opts: if config_dataset: dataset_opts = "config:%s" % config_dataset else: dataset_opts = "config:train" if use_pretrain: from Pretrain import pretrain_from_config pretrain = pretrain_from_config(config) if pretrain: print("Using pretrain %s, epoch %i" % (pretrain, epoch), file=log.v2) net_dict = pretrain.get_network_json_for_epoch(epoch=epoch) if "#config" in net_dict: config_overwrites = net_dict["#config"] print("Pretrain overwrites these config options:", file=log.v2) assert isinstance(config_overwrites, dict) for key, value in sorted(config_overwrites.items()): assert isinstance(key, str) orig_value = config.typed_dict.get(key, None) if isinstance(orig_value, dict) and isinstance(value, dict): diff_str = "\n" + Util.dict_diff_str(orig_value, value) elif isinstance(value, dict): diff_str = "\n%r ->\n%s" % (orig_value, pformat(value)) else: diff_str = " %r -> %r" % (orig_value, value) print("Config key %r for epoch %i:%s" % (key, epoch, diff_str), file=log.v2) config.set(key, value) else: print("No config overwrites for this epoch.", file=log.v2) else: print("No pretraining used.", file=log.v2) elif config.typed_dict.get("pretrain", None): print("Not using pretrain.", file=log.v2) dataset_default_opts = {} Dataset.kwargs_update_from_config(config, dataset_default_opts) print("Using dataset:", dataset_opts, file=log.v2) global dataset dataset = init_dataset(dataset_opts, default_kwargs=dataset_default_opts) assert isinstance(dataset, Dataset) dataset.init_seq_order(epoch=epoch)
def test_config3(): config = Config() config.update(config3_dict) config.network_topology_json = config3_json pretrain = pretrain_from_config(config) assert_equal(pretrain.get_train_num_epochs(), 3)
def test_config2(): config = Config() config.update(config2_dict) pretrain = pretrain_from_config(config) assert_equal(pretrain.get_train_num_epochs(), 3)
def init_network_from_config(self, config): self.pretrain = pretrain_from_config(config) self.max_seqs = config.int('max_seqs', -1) self.max_seq_length_eval = config.int('max_seq_length_eval', 2e31) self.compression = config.bool('compression', False) epoch, model_epoch_filename = self.get_epoch_model(config) assert model_epoch_filename or self.start_epoch self.epoch = epoch or self.start_epoch if model_epoch_filename: print("loading weights from", model_epoch_filename, file=log.v2) last_model_hdf = h5py.File(model_epoch_filename, "r") else: last_model_hdf = None if config.bool('initialize_from_model', False): # That's only about the topology, not the params. print("initializing network topology from model", file=log.v5) assert last_model_hdf, "last model not specified. use 'load' in config. or don't use 'initialize_from_model'" network = LayerNetwork.from_hdf_model_topology(last_model_hdf) else: if self.pretrain: # This would be obsolete if we don't want to load an existing model. # In self.init_train_epoch(), we initialize a new model. network = self.pretrain.get_network_for_epoch(epoch or self.start_epoch) else: network = LayerNetwork.from_config_topology(config) # We have the parameters randomly initialized at this point. # In training, as an initialization, we can copy over the params of an imported model, # where our topology might slightly differ from the imported model. if config.value('import_model_train_epoch1', '') and self.start_epoch == 1: assert last_model_hdf old_network = LayerNetwork.from_hdf_model_topology(last_model_hdf) old_network.load_hdf(last_model_hdf) last_model_hdf.close() # Copy params to new network. from NetworkCopyUtils import intelli_copy_layer # network.hidden are the input + all hidden layers. for layer_name, layer in sorted(old_network.hidden.items()): print("Copy hidden layer %s" % layer_name, file=log.v3) intelli_copy_layer(layer, network.hidden[layer_name]) for layer_name, layer in sorted(old_network.output.items()): if layer_name in network.output: print("Copy output layer %s" % layer_name, file=log.v3) intelli_copy_layer(layer, network.output[layer_name]) else: print("Did not copy output layer %s" % layer_name, file=log.v3) print("Not copied hidden: %s" % sorted(set(network.hidden.keys()).symmetric_difference(old_network.hidden.keys())), file=log.v3) print("Not copied output: %s" % sorted(set(network.output.keys()).symmetric_difference(old_network.output.keys())), file=log.v3) # Maybe load existing model parameters. elif last_model_hdf: network.load_hdf(last_model_hdf) last_model_hdf.close() EngineUtil.maybe_subtract_priors(network, self.train_data, config) self.network = network if config.has('dump_json'): self.network_dump_json(config.value('dump_json', '')) self.print_network_info()
def test_init_config1(): config = Config() config.update(config1_dict) pretrain = pretrain_from_config(config) assert_true(pretrain)
def init_network_from_config(self, config): self.pretrain = pretrain_from_config(config) self.max_seqs = config.int('max_seqs', -1) self.max_seq_length_eval = config.int('max_seq_length_eval', 2e31) self.compression = config.bool('compression', False) epoch, model_epoch_filename = self.get_epoch_model(config) assert model_epoch_filename or self.start_epoch self.epoch = epoch or self.start_epoch if model_epoch_filename: print("loading weights from", model_epoch_filename, file=log.v2) last_model_hdf = h5py.File(model_epoch_filename, "r") else: last_model_hdf = None if config.bool('initialize_from_model', False): # That's only about the topology, not the params. print("initializing network topology from model", file=log.v5) assert last_model_hdf, "last model not specified. use 'load' in config. or don't use 'initialize_from_model'" network = LayerNetwork.from_hdf_model_topology(last_model_hdf) else: if self.pretrain: # This would be obsolete if we don't want to load an existing model. # In self.init_train_epoch(), we initialize a new model. network = self.pretrain.get_network_for_epoch( epoch or self.start_epoch) else: network = LayerNetwork.from_config_topology(config) # We have the parameters randomly initialized at this point. # In training, as an initialization, we can copy over the params of an imported model, # where our topology might slightly differ from the imported model. if config.value('import_model_train_epoch1', '') and self.start_epoch == 1: assert last_model_hdf old_network = LayerNetwork.from_hdf_model_topology(last_model_hdf) old_network.load_hdf(last_model_hdf) last_model_hdf.close() # Copy params to new network. from NetworkCopyUtils import intelli_copy_layer # network.hidden are the input + all hidden layers. for layer_name, layer in sorted(old_network.hidden.items()): print("Copy hidden layer %s" % layer_name, file=log.v3) intelli_copy_layer(layer, network.hidden[layer_name]) for layer_name, layer in sorted(old_network.output.items()): if layer_name in network.output: print("Copy output layer %s" % layer_name, file=log.v3) intelli_copy_layer(layer, network.output[layer_name]) else: print("Did not copy output layer %s" % layer_name, file=log.v3) print("Not copied hidden: %s" % sorted( set(network.hidden.keys()).symmetric_difference( old_network.hidden.keys())), file=log.v3) print("Not copied output: %s" % sorted( set(network.output.keys()).symmetric_difference( old_network.output.keys())), file=log.v3) # Maybe load existing model parameters. elif last_model_hdf: network.load_hdf(last_model_hdf) last_model_hdf.close() EngineUtil.maybe_subtract_priors(network, self.train_data, config) self.network = network if config.has('dump_json'): self.network_dump_json(config.value('dump_json', '')) self.print_network_info()
def demo(): """ Demo run. Given some learning rate file (with scores / existing lrs), will calculate how lrs would have been set, given some config. """ import better_exchook better_exchook.install() import rnn import sys if len(sys.argv) <= 1: print("usage: python %s [config] [other options] [++check_learning_rates 1]" % __file__) print( ("example usage: " "python %s ++learning_rate_control newbob ++learning_rate_file newbob.data ++learning_rate 0.001") % __file__) rnn.init_config(command_line_options=sys.argv[1:]) # noinspection PyProtectedMember rnn.config._hack_value_reading_debug() rnn.config.update({"log": []}) rnn.init_log() rnn.init_backend_engine() check_lr = rnn.config.bool("check_learning_rates", False) from Pretrain import pretrain_from_config pretrain = pretrain_from_config(rnn.config) first_non_pretrain_epoch = 1 pretrain_learning_rate = None if pretrain: first_non_pretrain_epoch = pretrain.get_train_num_epochs() + 1 log.initialize(verbosity=[5]) control = load_learning_rate_control_from_config(rnn.config) print("LearningRateControl: %r" % control) if not control.epoch_data: print("No epoch data so far.") return first_epoch = min(control.epoch_data.keys()) if first_epoch != 1: print("Strange, first epoch from epoch data is %i." % first_epoch) print("Error key: %s from %r" % (control.get_error_key(epoch=first_epoch), control.epoch_data[first_epoch].error)) if pretrain: pretrain_learning_rate = rnn.config.float('pretrain_learning_rate', control.default_learning_rate) max_epoch = max(control.epoch_data.keys()) for epoch in range(1, max_epoch + 2): # all epochs [1..max_epoch+1] old_learning_rate = None if epoch in control.epoch_data: old_learning_rate = control.epoch_data[epoch].learning_rate if epoch < first_non_pretrain_epoch: learning_rate = pretrain_learning_rate s = "Pretrain epoch %i, fixed learning rate: %s (was: %s)" % (epoch, learning_rate, old_learning_rate) elif 1 < first_non_pretrain_epoch == epoch: learning_rate = control.default_learning_rate s = "First epoch after pretrain, epoch %i, fixed learning rate: %s (was %s)" % ( epoch, learning_rate, old_learning_rate) else: learning_rate = control.calc_new_learning_rate_for_epoch(epoch) s = "Calculated learning rate for epoch %i: %s (was: %s)" % (epoch, learning_rate, old_learning_rate) if learning_rate < control.min_learning_rate: learning_rate = control.min_learning_rate s += ", clipped to %s" % learning_rate s += ", previous relative error: %s" % control.calc_relative_error(epoch - 2, epoch - 1) if hasattr(control, "_calc_recent_mean_relative_error"): # noinspection PyProtectedMember s += ", previous mean relative error: %s" % control._calc_recent_mean_relative_error(epoch) print(s) if check_lr and old_learning_rate is not None: if old_learning_rate != learning_rate: print("Learning rate is different in epoch %i!" % epoch) sys.exit(1) # Overwrite new learning rate so that the calculation for further learning rates stays consistent. if epoch in control.epoch_data: control.epoch_data[epoch].learning_rate = learning_rate else: control.epoch_data[epoch] = control.EpochData(learningRate=learning_rate) print("Finished, last stored epoch was %i." % max_epoch)