def copy_params_from_old_network(self, new_network, old_network): """ :type new_network: LayerNetwork :type old_network: LayerNetwork :returns the remaining hidden layer names which exist only in the new network. :rtype: set[str] """ # network.hidden are the input + all hidden layers. for layer_name, layer in old_network.hidden.items(): new_network.hidden[layer_name].set_params_by_dict(layer.get_params_dict()) # network.output is the remaining output layer. if self.copy_output_layer: from NetworkCopyUtils import intelli_copy_layer, LayerDoNotMatchForCopy for layer_name in new_network.output.keys(): assert layer_name in old_network.output try: intelli_copy_layer(old_network.output[layer_name], new_network.output[layer_name]) except LayerDoNotMatchForCopy: if self.copy_output_layer == "ifpossible": print("Pretrain: Can not copy output layer %s, will leave it randomly initialized" % layer_name, file=log.v4) else: raise else: print("Pretrain: Will not copy output layer", file=log.v4)
def copy_params_from_old_network(self, new_network, old_network): """ :type new_network: LayerNetwork :type old_network: LayerNetwork :returns the remaining hidden layer names which exist only in the new network. :rtype: set[str] """ # network.hidden are the input + all hidden layers. for layer_name, layer in old_network.hidden.items(): new_network.hidden[layer_name].set_params_by_dict( layer.get_params_dict()) # network.output is the remaining output layer. if self.copy_output_layer: for layer_name in new_network.output.keys(): assert layer_name in old_network.output try: intelli_copy_layer(old_network.output[layer_name], new_network.output[layer_name]) except LayerDoNotMatchForCopy: if self.copy_output_layer == "ifpossible": print( "Pretrain: Can not copy output layer %s, will leave it randomly initialized" % layer_name, file=log.v4) else: raise else: print("Pretrain: Will not copy output layer", file=log.v4)
def init_network_from_config(self, config): self.pretrain = pretrainFromConfig(config) self.max_seqs = config.int('max_seqs', -1) self.compression = config.bool('compression', False) epoch, model_epoch_filename = self.get_epoch_model(config) assert model_epoch_filename or self.start_epoch if model_epoch_filename: print("loading weights from", model_epoch_filename, file=log.v2) last_model_hdf = h5py.File(model_epoch_filename, "r") else: last_model_hdf = None if config.bool('initialize_from_model', False): # That's only about the topology, not the params. print("initializing network topology from model", file=log.v5) assert last_model_hdf, "last model not specified. use 'load' in config. or don't use 'initialize_from_model'" network = LayerNetwork.from_hdf_model_topology(last_model_hdf) else: if self.pretrain: # This would be obsolete if we don't want to load an existing model. # In self.init_train_epoch(), we initialize a new model. network = self.pretrain.get_network_for_epoch(epoch or self.start_epoch) else: network = LayerNetwork.from_config_topology(config) # We have the parameters randomly initialized at this point. # In training, as an initialization, we can copy over the params of an imported model, # where our topology might slightly differ from the imported model. if config.value('import_model_train_epoch1', '') and self.start_epoch == 1: assert last_model_hdf old_network = LayerNetwork.from_hdf_model_topology(last_model_hdf) old_network.load_hdf(last_model_hdf) last_model_hdf.close() # Copy params to new network. from NetworkCopyUtils import intelli_copy_layer # network.hidden are the input + all hidden layers. for layer_name, layer in sorted(old_network.hidden.items()): print("Copy hidden layer %s" % layer_name, file=log.v3) intelli_copy_layer(layer, network.hidden[layer_name]) for layer_name, layer in sorted(old_network.output.items()): print("Copy output layer %s" % layer_name, file=log.v3) intelli_copy_layer(layer, network.output[layer_name]) print("Not copied hidden: %s" % sorted(set(network.hidden.keys()).difference(old_network.hidden.keys())), file=log.v3) print("Not copied output: %s" % sorted(set(network.output.keys()).difference(old_network.output.keys())), file=log.v3) # Maybe load existing model parameters. elif last_model_hdf: network.load_hdf(last_model_hdf) last_model_hdf.close() EngineUtil.maybe_subtract_priors(network, self.train_data, config) self.network = network if config.has('dump_json'): self.network_dump_json(config.value('dump_json', '')) self.print_network_info()
def init_network_from_config(self, config): self.pretrain = pretrainFromConfig(config) self.max_seqs = config.int('max_seqs', -1) epoch, model_epoch_filename = self.get_epoch_model(config) assert model_epoch_filename or self.start_epoch if model_epoch_filename: print >> log.v2, "loading weights from", model_epoch_filename last_model_hdf = h5py.File(model_epoch_filename, "r") else: last_model_hdf = None if config.bool('initialize_from_model', False): # That's only about the topology, not the params. print >> log.v5, "initializing network topology from model" assert last_model_hdf, "last model not specified. use 'load' in config. or don't use 'initialize_from_model'" network = LayerNetwork.from_hdf_model_topology(last_model_hdf) else: if self.pretrain: # This would be obsolete if we don't want to load an existing model. # In self.init_train_epoch(), we initialize a new model. network = self.pretrain.get_network_for_epoch(epoch or self.start_epoch) else: network = LayerNetwork.from_config_topology(config) # We have the parameters randomly initialized at this point. # In training, as an initialization, we can copy over the params of an imported model, # where our topology might slightly differ from the imported model. if config.value('import_model_train_epoch1', '') and self.start_epoch == 1: assert last_model_hdf old_network = LayerNetwork.from_hdf_model_topology(last_model_hdf) old_network.load_hdf(last_model_hdf) last_model_hdf.close() # Copy params to new network. from NetworkCopyUtils import intelli_copy_layer # network.hidden are the input + all hidden layers. for layer_name, layer in sorted(old_network.hidden.items()): print >> log.v3, "Copy hidden layer %s" % layer_name intelli_copy_layer(layer, network.hidden[layer_name]) for layer_name, layer in sorted(old_network.output.items()): print >> log.v3, "Copy output layer %s" % layer_name intelli_copy_layer(layer, network.output[layer_name]) print >> log.v3, "Not copied hidden: %s" % sorted(set(network.hidden.keys()).difference(old_network.hidden.keys())) print >> log.v3, "Not copied output: %s" % sorted(set(network.output.keys()).difference(old_network.output.keys())) # Maybe load existing model parameters. elif last_model_hdf: network.load_hdf(last_model_hdf) last_model_hdf.close() EngineUtil.maybe_subtract_priors(network, self.train_data, config) self.network = network if config.has('dump_json'): self.network_dump_json(config.value('dump_json', '')) self.print_network_info()