コード例 #1
0
def test_NetworkDescription_to_json_config1():
  config = Config()
  config.update(config1_dict)
  desc = LayerNetworkDescription.from_config(config)
  desc_json_content = desc.to_json_content()
  pprint(desc_json_content)
  assert_in("hidden_0", desc_json_content)
  assert_equal(desc_json_content["hidden_0"]["class"], "forward")
  assert_in("hidden_1", desc_json_content)
  assert_in("output", desc_json_content)
  orig_network = LayerNetwork.from_description(desc)
  assert_in("hidden_0", orig_network.hidden)
  assert_in("hidden_1", orig_network.hidden)
  assert_equal(len(orig_network.hidden), 2)
  assert_is_instance(orig_network.hidden["hidden_0"], ForwardLayer)
  assert_equal(orig_network.hidden["hidden_0"].layer_class, "hidden")
  orig_json_content = orig_network.to_json_content()
  pprint(orig_json_content)
  assert_in("hidden_0", orig_json_content)
  assert_equal(orig_json_content["hidden_0"]["class"], "hidden")
  assert_in("hidden_1", orig_json_content)
  assert_in("output", orig_json_content)
  new_network = LayerNetwork.from_json(
    desc_json_content,
    config1_dict["num_inputs"],
    {"classes": (config1_dict["num_outputs"], 1)})
  new_json_content = new_network.to_json_content()
  if orig_json_content != new_json_content:
    print(dict_diff_str(orig_json_content, new_json_content))
    assert_equal(orig_json_content, new_network.to_json_content())
コード例 #2
0
ファイル: test_Network.py プロジェクト: rwth-i6/returnn
def test_enc_dec1_init():
  config = Config()
  config.load_file(StringIO(config_enc_dec1_json))

  network_json = LayerNetwork.json_from_config(config)
  assert_true(network_json)
  network = LayerNetwork.from_json_and_config(network_json, config)
  assert_true(network)
コード例 #3
0
ファイル: Engine.py プロジェクト: chagge/returnn
  def init_network_from_config(self, config):
    self.pretrain = pretrainFromConfig(config)
    self.max_seqs = config.int('max_seqs', -1)

    epoch, model_epoch_filename = self.get_epoch_model(config)
    assert model_epoch_filename or self.start_epoch

    if model_epoch_filename:
      print >> log.v2, "loading weights from", model_epoch_filename
      last_model_hdf = h5py.File(model_epoch_filename, "r")
    else:
      last_model_hdf = None

    if config.bool('initialize_from_model', False):
      # That's only about the topology, not the params.
      print >> log.v5, "initializing network topology from model"
      assert last_model_hdf, "last model not specified. use 'load' in config. or don't use 'initialize_from_model'"
      network = LayerNetwork.from_hdf_model_topology(last_model_hdf)
    else:
      if self.pretrain:
        # This would be obsolete if we don't want to load an existing model.
        # In self.init_train_epoch(), we initialize a new model.
        network = self.pretrain.get_network_for_epoch(epoch or self.start_epoch)
      else:
        network = LayerNetwork.from_config_topology(config)

    # We have the parameters randomly initialized at this point.
    # In training, as an initialization, we can copy over the params of an imported model,
    # where our topology might slightly differ from the imported model.
    if config.value('import_model_train_epoch1', '') and self.start_epoch == 1:
      assert last_model_hdf
      old_network = LayerNetwork.from_hdf_model_topology(last_model_hdf)
      old_network.load_hdf(last_model_hdf)
      last_model_hdf.close()
      # Copy params to new network.
      from NetworkCopyUtils import intelli_copy_layer
      # network.hidden are the input + all hidden layers.
      for layer_name, layer in sorted(old_network.hidden.items()):
        print >> log.v3, "Copy hidden layer %s" % layer_name
        intelli_copy_layer(layer, network.hidden[layer_name])
      for layer_name, layer in sorted(old_network.output.items()):
        print >> log.v3, "Copy output layer %s" % layer_name
        intelli_copy_layer(layer, network.output[layer_name])
      print >> log.v3, "Not copied hidden: %s" % sorted(set(network.hidden.keys()).difference(old_network.hidden.keys()))
      print >> log.v3, "Not copied output: %s" % sorted(set(network.output.keys()).difference(old_network.output.keys()))

    # Maybe load existing model parameters.
    elif last_model_hdf:
      network.load_hdf(last_model_hdf)
      last_model_hdf.close()
      EngineUtil.maybe_subtract_priors(network, self.train_data, config)

    self.network = network

    if config.has('dump_json'):
      self.network_dump_json(config.value('dump_json', ''))

    self.print_network_info()
コード例 #4
0
def test_config1_to_json_network_copy():
  config = Config()
  config.update(config1_dict)
  orig_network = LayerNetwork.from_config_topology(config)
  orig_json_content = orig_network.to_json_content()
  pprint(orig_json_content)
  new_network = LayerNetwork.from_json(orig_json_content, orig_network.n_in, orig_network.n_out)
  assert_equal(orig_network.n_in, new_network.n_in)
  assert_equal(orig_network.n_out, new_network.n_out)
  new_json_content = new_network.to_json_content()
  if orig_json_content != new_json_content:
    print(dict_diff_str(orig_json_content, new_json_content))
    assert_equal(orig_json_content, new_network.to_json_content())
コード例 #5
0
ファイル: Pretrain.py プロジェクト: rwth-i6/returnn
def pretrain_from_config(config):
  """
  :type config: Config.Config
  :rtype: Pretrain | None
  """
  import Util
  from Config import network_json_from_config
  pretrain_type = config.bool_or_other("pretrain", None)
  if pretrain_type == "default" or (isinstance(pretrain_type, dict) and pretrain_type) or pretrain_type is True:
    if Util.BackendEngine.is_theano_selected():
      from Network import LayerNetwork
      network_init_args = LayerNetwork.init_args_from_config(config)
    else:
      network_init_args = None
    original_network_json = network_json_from_config(config)
    opts = config.get_of_type("pretrain", dict, {})
    if config.has("pretrain_copy_output_layer"):
      opts.setdefault("copy_output_layer", config.bool_or_other("pretrain_copy_output_layer", "ifpossible"))
    if config.has("pretrain_greedy"):
      opts.setdefault("greedy", config.bool("pretrain_greedy", None))
    if config.has("pretrain_repetitions"):
      if config.is_typed("pretrain_repetitions"):
        opts.setdefault("repetitions", config.typed_value("pretrain_repetitions"))
      else:
        opts.setdefault("repetitions", config.int_list("pretrain_repetitions", None))
    if config.has("pretrain_construction_algo"):
      opts.setdefault("construction_algo", config.value("pretrain_construction_algo", None))
    return Pretrain(original_network_json=original_network_json, network_init_args=network_init_args, **opts)
  elif not pretrain_type:
    return None
  else:
    raise Exception("unknown pretrain type: %s" % pretrain_type)
コード例 #6
0
ファイル: Pretrain.py プロジェクト: atuxhe/returnn
 def get_network_for_epoch(self, epoch, mask=None):
   """
   :type epoch: int
   :rtype: Network.LayerNetwork
   """
   json_content = self.get_network_json_for_epoch(epoch)
   Layer.rng_seed = epoch
   return LayerNetwork.from_json(json_content, mask=mask, **self.network_init_args)
コード例 #7
0
ファイル: Pretrain.py プロジェクト: rwth-i6/returnn
 def get_network_for_epoch(self, epoch, mask=None):
   """
   :type epoch: int
   :param mask:
   :rtype: Network.LayerNetwork
   """
   from Network import LayerNetwork
   from NetworkBaseLayer import Layer
   json_content = self.get_network_json_for_epoch(epoch)
   Layer.rng_seed = epoch
   return LayerNetwork.from_json(json_content, mask=mask, **self.network_init_args)
コード例 #8
0
ファイル: import-sprint-nn.py プロジェクト: rwth-i6/returnn
def saveCrnnNetwork(epoch, layers):
  """
  :type epoch: int
  :type layers: list[(numpy.ndarray, numpy.ndarray)]
  """
  print("Loading Crnn")

  from Network import LayerNetwork
  from NetworkHiddenLayer import ForwardLayer
  from NetworkOutputLayer import OutputLayer
  from Pretrain import pretrain_from_config
  from EngineBase import EngineBase

  pretrain = pretrain_from_config(config)
  is_pretrain_epoch = pretrain and epoch <= pretrain.get_train_num_epochs()
  modelFilename = config.value("model", None)
  assert modelFilename, "need 'model' in config"
  filename = EngineBase.epoch_model_filename(modelFilename, epoch, is_pretrain_epoch)
  assert not os.path.exists(filename), "already exists"
  if is_pretrain_epoch:
    network = pretrain.get_network_for_epoch(epoch)
  else:
    network = LayerNetwork.from_config_topology(config)
  nHiddenLayers = len(network.hidden)

  # print network topology
  print("Crnn Network layer topology:")
  print("input dim:", network.n_in)
  print("hidden layer count:", nHiddenLayers)
  print("output dim:", network.n_out["classes"])
  print("net weights #:", network.num_params())
  print("net params:", network.train_params_vars)
  print("net output:", network.output["output"])

  assert network.n_in == inputDim
  #assert network.n_out == outputDim
  assert nHiddenLayers + 1 == layerCount  # hidden + output layer
  assert len(layers) == layerCount
  for i, (layerName, hidden) in enumerate(sorted(network.hidden.items())):
    # Some checks whether this is a forward-layer.
    assert isinstance(hidden, ForwardLayer)

    saveCrnnLayer(hidden, *layers[i])

  assert isinstance(network.output["output"], OutputLayer)
  saveCrnnLayer(network.output["output"], *layers[len(layers) - 1])

  import h5py
  print(("Save Crnn model under %s" % filename))
  model = h5py.File(filename, "w")
  network.save_hdf(model, epoch)
  model.close()
コード例 #9
0
ファイル: test_Network.py プロジェクト: rwth-i6/returnn
def test_enc_dec1_hdf():
  filename = tempfile.mktemp(prefix="crnn-model-test")
  model = h5py.File(filename, "w")

  config = Config()
  config.load_file(StringIO(config_enc_dec1_json))
  network_json = LayerNetwork.json_from_config(config)
  assert_true(network_json)
  network = LayerNetwork.from_json_and_config(network_json, config)
  assert_true(network)

  network.save_hdf(model, epoch=42)
  model.close()

  loaded_model = h5py.File(filename, "r")
  loaded_net = LayerNetwork.from_hdf_model_topology(loaded_model)
  assert_true(loaded_net)
  assert_equal(sorted(network.hidden.keys()), sorted(loaded_net.hidden.keys()))
  assert_equal(sorted(network.y.keys()), sorted(loaded_net.y.keys()))
  assert_equal(sorted(network.j.keys()), sorted(loaded_net.j.keys()))

  os.remove(filename)
コード例 #10
0
ファイル: Pretrain.py プロジェクト: atuxhe/returnn
def pretrainFromConfig(config):
  """
  :type config: Config.Config
  :rtype: Pretrain | None
  """
  pretrainType = config.value("pretrain", "")
  if pretrainType == "default":
    network_init_args = LayerNetwork.init_args_from_config(config)
    original_network_json = LayerNetwork.json_from_config(config)
    copy_output_layer = config.bool_or_other("pretrain_copy_output_layer", "ifpossible")
    greedy = config.bool("pretrain_greedy", None)
    repetitions = config.int_list("pretrain_repetitions", None)
    construction_algo = config.value("pretrain_construction_algo", None)
    return Pretrain(original_network_json=original_network_json,
                    network_init_args=network_init_args,
                    copy_output_layer=copy_output_layer,
                    greedy=greedy, repetitions=repetitions,
                    construction_algo=construction_algo)
  elif pretrainType == "":
    return None
  else:
    raise Exception("unknown pretrain type: %s" % pretrainType)
コード例 #11
0
def test_network_config1_init():
  config = Config()
  config.update(config1_dict)
  network = LayerNetwork.from_config_topology(config)
  assert_in("hidden_0", network.hidden)
  assert_in("hidden_1", network.hidden)
  assert_equal(len(network.hidden), 2)
  assert_is_instance(network.hidden["hidden_0"], ForwardLayer)
  assert_equal(network.hidden["hidden_0"].layer_class, "hidden")
  assert_false(network.recurrent)

  json_content = network.to_json_content()
  pprint(json_content)
  assert_in("hidden_0", json_content)
  assert_equal(json_content["hidden_0"]["class"], "hidden")
  assert_in("hidden_1", json_content)
  assert_in("output", json_content)
コード例 #12
0
def test_config2_bidirect_lstm():
  config = Config()
  config.update(config2_dict)
  desc = LayerNetworkDescription.from_config(config)
  assert_true(desc.bidirectional)
  network = LayerNetwork.from_config_topology(config)
  net_json = network.to_json_content()
  pprint(net_json)
  assert_in("output", net_json)
  assert_in("hidden_0_fw", net_json)
  assert_in("hidden_0_bw", net_json)
  assert_in("hidden_1_fw", net_json)
  assert_in("hidden_1_bw", net_json)
  assert_in("hidden_2_fw", net_json)
  assert_in("hidden_2_bw", net_json)
  assert_equal(net_json["output"]["from"], ["hidden_2_fw", "hidden_2_bw"])
  assert_equal(len(net_json), 7)
コード例 #13
0
ファイル: dump-network-json.py プロジェクト: atuxhe/returnn
def main(argv):
  argparser = argparse.ArgumentParser(description='Dump network as JSON.')
  argparser.add_argument('crnn_config_file')
  argparser.add_argument('--epoch', default=1, type=int)
  argparser.add_argument('--out', default="/dev/stdout")
  args = argparser.parse_args(argv[1:])
  init(configFilename=args.crnn_config_file, commandLineOptions=[])

  pretrain = pretrainFromConfig(config)
  if pretrain:
    network = pretrain.get_network_for_epoch(args.epoch)
  else:
    network = LayerNetwork.from_config_topology(config)

  json_data = network.to_json_content()
  f = open(args.out, 'w')
  print >> f, json.dumps(json_data, indent=2, sort_keys=True)
  f.close()

  rnn.finalize()
コード例 #14
0
ファイル: NetworkDescription.py プロジェクト: rwth-i6/returnn
 def num_inputs_outputs_from_config(cls, config):
   """
   :type config: Config.Config
   :returns (num_inputs, num_outputs),
      where num_inputs is like num_outputs["data"][0],
      and num_outputs is a dict of data_key -> (dim, ndim),
        where data_key is e.g. "classes" or "data",
        dim is the feature dimension or the number of classes,
        and ndim is the ndim counted without batch-dim,
        i.e. ndim=1 means usually sparse data and ndim=2 means dense data.
   :rtype: (int,dict[str,(int,int)])
   """
   from Util import BackendEngine
   num_inputs = config.int('num_inputs', 0)
   target = config.value('target', 'classes')
   if config.is_typed('num_outputs'):
     num_outputs = config.typed_value('num_outputs')
     if not isinstance(num_outputs, dict):
       num_outputs = {target: num_outputs}
     num_outputs = num_outputs.copy()
     from Dataset import convert_data_dims
     num_outputs = convert_data_dims(num_outputs, leave_dict_as_is=BackendEngine.is_tensorflow_selected())
     if "data" in num_outputs:
       num_inputs = num_outputs["data"]
       if isinstance(num_inputs, (list, tuple)):
         num_inputs = num_inputs[0]
       elif isinstance(num_inputs, dict):
         if "dim" in num_inputs:
           num_inputs = num_inputs["dim"]
         else:
           num_inputs = num_inputs["shape"][-1]
       else:
         raise TypeError("data key %r" % num_inputs)
   elif config.has('num_outputs'):
     num_outputs = {target: [config.int('num_outputs', 0), 1]}
   else:
     num_outputs = None
   dataset = None
   if config.list('train') and ":" not in config.value('train', ''):
     dataset = config.list('train')[0]
   if not config.is_typed('num_outputs') and dataset:
     # noinspection PyBroadException
     try:
       _num_inputs = hdf5_dimension(dataset, 'inputCodeSize') * config.int('window', 1)
     except Exception:
       _num_inputs = hdf5_dimension(dataset, 'inputPattSize') * config.int('window', 1)
     # noinspection PyBroadException
     try:
       _num_outputs = {target: [hdf5_dimension(dataset, 'numLabels'), 1]}
     except Exception:
       _num_outputs = hdf5_group(dataset, 'targets/size')
       for k in _num_outputs:
         _num_outputs[k] = [_num_outputs[k], len(hdf5_shape(dataset, 'targets/data/' + k))]
     if num_inputs:
       assert num_inputs == _num_inputs
     if num_outputs:
       assert num_outputs == _num_outputs
     num_inputs = _num_inputs
     num_outputs = _num_outputs
   if not num_inputs and not num_outputs and config.has("load") and BackendEngine.is_theano_selected():
     from Network import LayerNetwork
     import h5py
     model = h5py.File(config.value("load", ""), "r")
     # noinspection PyProtectedMember
     num_inputs, num_outputs = LayerNetwork._n_in_out_from_hdf_model(model)
   assert num_inputs and num_outputs, "provide num_inputs/num_outputs directly or via train"
   return num_inputs, num_outputs
コード例 #15
0
ファイル: DebugHelpers.py プロジェクト: rwth-i6/returnn
 def __init__(self, filename):
   self.network = LayerNetwork.from_hdf(filename, mask="unity", train_flag=False, eval_flag=True)
   self.f_forwarder = None