Пример #1
0
def init(config_filename, cmd_line_opts, dataset_config_str):
    """
  :param str config_filename: global config for CRNN
  :param list[str] cmd_line_opts: options for initConfig method
  :param str dataset_config_str: dataset via init_dataset_via_str()
  """
    rnn.initBetterExchook()
    rnn.initThreadJoinHack()
    if config_filename:
        rnn.initConfig(config_filename, cmd_line_opts)
        rnn.initLog()
    else:
        log.initialize(verbosity=[5])
    print("Returnn hdf_dump starting up.", file=log.v3)
    rnn.initFaulthandler()
    if config_filename:
        rnn.initData()
        rnn.printTaskProperties()
        assert isinstance(rnn.train_data, Dataset)
        return rnn.train_data
    else:
        assert dataset_config_str
        dataset = init_dataset_via_str(dataset_config_str)
        print("Source dataset:", dataset.len_info(), file=log.v3)
        return dataset
Пример #2
0
def init(config_str):
    """
  :param str config_str: either filename to config-file, or dict for dataset
  """
    rnn.initBetterExchook()
    rnn.initThreadJoinHack()
    if config_str.startswith("{"):
        print("Using dataset %s." % config_str)
        datasetDict = eval(config_str)
        configFilename = None
    else:
        datasetDict = None
        configFilename = config_str
        print("Using config file %r." % configFilename)
        assert os.path.exists(configFilename)
    rnn.initConfig(configFilename=configFilename, commandLineOptions=[])
    global config
    config = rnn.config
    config.set("log", None)
    if datasetDict:
        config.set("train", datasetDict)
    rnn.initLog()
    print("CRNN dump-dataset starting up.", file=log.v1)
    rnn.initFaulthandler()
    rnn.initConfigJsonNetwork()
    rnn.initData()
    rnn.printTaskProperties()
Пример #3
0
def init(config_filename, cmd_line_opts, dataset_config_str):
  """
  :param str config_filename: global config for CRNN
  :param list[str] cmd_line_opts: options for initConfig method
  :param str dataset_config_str: dataset via init_dataset_via_str()
  """
  rnn.initBetterExchook()
  rnn.initThreadJoinHack()
  if config_filename:
    rnn.initConfig(config_filename, cmd_line_opts)
    rnn.initLog()
  else:
    log.initialize(verbosity=[5])
  print >> log.v3, "CRNN dump-dataset starting up."
  rnn.initFaulthandler()
  rnn.initConfigJsonNetwork()
  if config_filename:
    rnn.initData()
    rnn.printTaskProperties()
    assert isinstance(rnn.train_data, Dataset)
    return rnn.train_data
  else:
    assert dataset_config_str
    dataset = init_dataset_via_str(dataset_config_str)
    print >> log.v3, "Source dataset:", dataset.len_info()
    return dataset
Пример #4
0
def init(config_filename, log_verbosity):
    """
  :param str config_filename: filename to config-file
  :param int log_verbosity:
  """
    rnn.initBetterExchook()
    rnn.initThreadJoinHack()
    if config_filename:
        print("Using config file %r." % config_filename)
        assert os.path.exists(config_filename)
    rnn.initConfig(configFilename=config_filename, commandLineOptions=[])
    global config
    config = rnn.config
    config.set("log", None)
    config.set("log_verbosity", log_verbosity)
    config.set("use_tensorflow", True)
    rnn.initLog()
    print("Returnn compile-native-op starting up.", file=log.v1)
    rnn.returnnGreeting()
    rnn.initBackendEngine()
    assert Util.BackendEngine.is_tensorflow_selected(
    ), "this is only for TensorFlow"
    rnn.initFaulthandler()
    rnn.initConfigJsonNetwork()
    if 'network' in config.typed_dict:
        print("Loading network")
        from TFNetwork import TFNetwork
        network = TFNetwork(name="root",
                            config=config,
                            rnd_seed=1,
                            train_flag=False,
                            eval_flag=True,
                            search_flag=False)
        network.construct_from_dict(config.typed_dict["network"])
def init(config_filename, log_verbosity):
    """
  :param str config_filename: filename to config-file
  :param int log_verbosity:
  """
    rnn.initBetterExchook()
    rnn.initThreadJoinHack()
    if config_filename:
        print("Using config file %r." % config_filename)
        assert os.path.exists(config_filename)
    rnn.initConfig(configFilename=config_filename, commandLineOptions=[])
    global config
    config = rnn.config
    config.set("task", "calculate_wer")
    config.set("log", None)
    config.set("log_verbosity", log_verbosity)
    config.set("use_tensorflow", True)
    rnn.initLog()
    print("Returnn calculate-word-error-rate starting up.", file=log.v1)
    rnn.returnnGreeting()
    rnn.initBackendEngine()
    assert Util.BackendEngine.is_tensorflow_selected(
    ), "this is only for TensorFlow"
    rnn.initFaulthandler()
    rnn.initConfigJsonNetwork()
    rnn.printTaskProperties()
Пример #6
0
def init(config_str, verbosity):
    """
  :param str config_str: either filename to config-file, or dict for dataset
  :param int verbosity:
  """
    rnn.initBetterExchook()
    rnn.initThreadJoinHack()
    datasetDict = None
    configFilename = None
    if config_str.strip().startswith("{"):
        print("Using dataset %s." % config_str)
        datasetDict = eval(config_str.strip())
    elif config_str.endswith(".hdf"):
        datasetDict = {"class": "HDFDataset", "files": [config_str]}
        print("Using dataset %r." % datasetDict)
        assert os.path.exists(config_str)
    else:
        configFilename = config_str
        print("Using config file %r." % configFilename)
        assert os.path.exists(configFilename)
    rnn.initConfig(configFilename=configFilename,
                   default_config={"cache_size": "0"})
    global config
    config = rnn.config
    config.set("log", None)
    config.set("log_verbosity", verbosity)
    if datasetDict:
        config.set("train", datasetDict)
    rnn.initLog()
    print("Returnn dump-dataset starting up.", file=log.v2)
    rnn.returnnGreeting()
    rnn.initFaulthandler()
    rnn.initConfigJsonNetwork()
    rnn.initData()
    rnn.printTaskProperties()
Пример #7
0
def init(configFilename, commandLineOptions):
    rnn.initBetterExchook()
    rnn.initThreadJoinHack()
    rnn.initConfig(configFilename, commandLineOptions)
    global config
    config = rnn.config
    rnn.initLog()
    print >> log.v3, "CRNN demo-dataset starting up"
    rnn.initFaulthandler()
    rnn.initConfigJsonNetwork()
    rnn.initData()
    rnn.printTaskProperties()
Пример #8
0
def init(configFilename, commandLineOptions):
  rnn.initBetterExchook()
  rnn.initThreadJoinHack()
  rnn.initConfig(configFilename, commandLineOptions)
  global config
  config = rnn.config
  rnn.initLog()
  print >> log.v3, "CRNN demo-dataset starting up"
  rnn.initFaulthandler()
  rnn.initConfigJsonNetwork()
  rnn.initData()
  rnn.printTaskProperties()
Пример #9
0
def init(configFilename, commandLineOptions):
    rnn.initBetterExchook()
    rnn.initThreadJoinHack()
    rnn.initConfig(configFilename, commandLineOptions)
    global config
    config = rnn.config
    config.set("log", None)
    rnn.initLog()
    print("CRNN dump-dataset starting up.", file=log.v1)
    rnn.initFaulthandler()
    rnn.initConfigJsonNetwork()
    rnn.initData()
    rnn.printTaskProperties()
Пример #10
0
def init(configFilename=None):
    rnn.initBetterExchook()
    rnn.initThreadJoinHack()
    if configFilename:
        rnn.initConfig(configFilename, commandLineOptions=[])
        rnn.initLog()
    else:
        log.initialize()
    print("CRNN collect-orth-symbols starting up.", file=log.v3)
    rnn.initFaulthandler()
    if configFilename:
        rnn.initConfigJsonNetwork()
        rnn.initData()
        rnn.printTaskProperties()
Пример #11
0
def init(configFilename=None):
  rnn.initBetterExchook()
  rnn.initThreadJoinHack()
  if configFilename:
    rnn.initConfig(configFilename, commandLineOptions=[])
    rnn.initLog()
  else:
    log.initialize()
  print >> log.v3, "CRNN collect-orth-symbols starting up."
  rnn.initFaulthandler()
  if configFilename:
    rnn.initConfigJsonNetwork()
    rnn.initData()
    rnn.printTaskProperties()
Пример #12
0
def initBase(configfile=None, targetMode=None, epoch=None):
  """
  :type configfile: str | None
  """

  global isInitialized
  isInitialized = True
  # Run through in any case. Maybe just to set targetMode.

  global config
  if not config:
    if configfile is None:
      configfile = DefaultSprintCrnnConfig
    assert os.path.exists(configfile)

    rnn.initThreadJoinHack()
    rnn.initConfig(configfile, [])
    config = rnn.config
    rnn.initLog()
    rnn.initConfigJsonNetwork()

  if targetMode:
    setTargetMode(targetMode)

  initDataset()

  if targetMode and targetMode == "forward" and epoch:
    model_filename = config.value('model', '')
    fns = [Engine.epoch_model_filename(model_filename, epoch, is_pretrain) for is_pretrain in [False, True]]
    fns_existing = [fn for fn in fns if os.path.exists(fn)]
    assert len(fns_existing) == 1, "%s not found" % fns
    model_epoch_filename = fns_existing[0]
    config.set('load', model_epoch_filename)
    assert Engine.get_epoch_model(config)[1] == model_epoch_filename

  global engine
  if not engine:
    devices = rnn.initDevices()
    rnn.printTaskProperties(devices)
    rnn.initEngine(devices)
    engine = rnn.engine
    assert isinstance(engine, Engine)
Пример #13
0
def init(config_filename, log_verbosity):
    """
  :param str config_filename: filename to config-file
  :param int log_verbosity:
  """
    rnn.initBetterExchook()
    rnn.initThreadJoinHack()
    if config_filename:
        print("Using config file %r." % config_filename)
        assert os.path.exists(config_filename)
    rnn.initConfig(configFilename=config_filename, commandLineOptions=[])
    global config
    config = rnn.config
    config.set("task", "dump")
    config.set("log", None)
    config.set("log_verbosity", log_verbosity)
    rnn.initLog()
    print("Returnn dump-dataset-raw-strings starting up.", file=log.v1)
    rnn.returnnGreeting()
    rnn.initFaulthandler()
Пример #14
0
from threading import Condition

InitTypes = set()
Verbose = False  # disables all per-segment log messages
Quiet = False # disables all but error messages

def print(str):
  import __builtin__
  if not Quiet:
    __builtin__.print(str)

print("CRNN SprintControl[pid %i] Python module load" % os.getpid())

rnn.initBetterExchook()
Debug.initFaulthandler(sigusr1_chain=True)  # Sprint also handles SIGUSR1.
rnn.initThreadJoinHack()

# Start Sprint PythonControl interface. {

def init(name, reference, config, sprint_unit=None, version_number=None, callback=None, **kwargs):
  """
  This will be called by Sprint PythonControl.
  But we also call it ourselves e.g. in getSegmentList() and SprintNnPythonLayer.
  :param str name: this specifies the caller. e.g. "Sprint.PythonControl"
  :param reference: this is any object to identify the specific instance of the caller, if there are multiple.
  :param str config: this will be passed over from Sprint. you can configure that via --*.pymod-config.
  :param str sprint_unit: if this is called by Sprint PythonControl, this will specify which specific part
    of Sprint is using this PythonControl, because there can be multiple parts.
    E.g. there is "FeedForwardTrainer", "SegmentwiseNnTrainer" and "NnTrainer.pythonControl".
  :param int|None version_number: if this is called by Sprint PythonControl, this will set the version number.
    only newer Sprint versions will set this.
Пример #15
0
def initBase(configfile=None, targetMode=None, epoch=None):
    """
  :param str|None configfile: filename, via init(), this is set
  :param str|None targetMode: "forward" or so. via init(), this is set
  :param int epoch: via init(), this is set
  """

    global isInitialized
    isInitialized = True
    # Run through in any case. Maybe just to set targetMode.

    if not getattr(sys, "argv", None):
        # Set some dummy. Some code might want this (e.g. TensorFlow).
        sys.argv = [__file__]

    global config
    if not config:
        # Some subset of what we do in rnn.init().

        rnn.initBetterExchook()
        rnn.initThreadJoinHack()

        if configfile is None:
            configfile = DefaultSprintCrnnConfig
        assert os.path.exists(configfile)
        rnn.initConfig(configFilename=configfile)
        config = rnn.config

        rnn.initLog()
        rnn.returnnGreeting(configFilename=configfile)
        rnn.initBackendEngine()
        rnn.initFaulthandler(sigusr1_chain=True)
        rnn.initConfigJsonNetwork()

        if BackendEngine.is_tensorflow_selected():
            # Use TFEngine.Engine class instead of Engine.Engine.
            import TFEngine
            global Engine
            Engine = TFEngine.Engine

        import atexit
        atexit.register(_at_exit_handler)

    if targetMode:
        setTargetMode(targetMode)

    initDataset()

    if targetMode and targetMode == "forward" and epoch:
        model_filename = config.value('model', '')
        fns = [
            Engine.epoch_model_filename(model_filename, epoch, is_pretrain)
            for is_pretrain in [False, True]
        ]
        fn_postfix = ""
        if BackendEngine.is_tensorflow_selected():
            fn_postfix += ".meta"
        fns_existing = [fn for fn in fns if os.path.exists(fn + fn_postfix)]
        assert len(fns_existing) == 1, "%s not found" % fns
        model_epoch_filename = fns_existing[0]
        config.set('load', model_epoch_filename)
        assert Engine.get_epoch_model(config)[1] == model_epoch_filename, \
          "%r != %r" % (Engine.get_epoch_model(config), model_epoch_filename)

    global engine
    if not engine:
        devices = rnn.initDevices()
        rnn.printTaskProperties(devices)
        rnn.initEngine(devices)
        engine = rnn.engine
        assert isinstance(engine, Engine)
Пример #16
0
from threading import Condition

InitTypes = set()
Verbose = False  # disables all per-segment log messages
Quiet = False # disables all but error messages

_orig_print = print
def print(*args, **kwargs):
  if not Quiet:
    _orig_print(*args, **kwargs)

print("CRNN SprintControl[pid %i] Python module load" % os.getpid())

rnn.initBetterExchook()
Debug.initFaulthandler(sigusr1_chain=True)  # Sprint also handles SIGUSR1.
rnn.initThreadJoinHack()

# Start Sprint PythonControl interface. {

def init(name, reference, config, sprint_unit=None, version_number=None, callback=None, **kwargs):
  """
  This will be called by Sprint PythonControl.
  But we also call it ourselves e.g. in getSegmentList() and SprintNnPythonLayer.
  :param str name: this specifies the caller. e.g. "Sprint.PythonControl"
  :param reference: this is any object to identify the specific instance of the caller, if there are multiple.
  :param str config: this will be passed over from Sprint. you can configure that via --*.pymod-config.
  :param str sprint_unit: if this is called by Sprint PythonControl, this will specify which specific part
    of Sprint is using this PythonControl, because there can be multiple parts.
    E.g. there is "FeedForwardTrainer", "SegmentwiseNnTrainer" and "NnTrainer.pythonControl".
  :param int|None version_number: if this is called by Sprint PythonControl, this will set the version number.
    only newer Sprint versions will set this.