Beispiel #1
0
def init(config_filename, log_verbosity):
    """
  :param str config_filename: filename to config-file
  :param int log_verbosity:
  """
    rnn.initBetterExchook()
    rnn.initThreadJoinHack()
    if config_filename:
        print("Using config file %r." % config_filename)
        assert os.path.exists(config_filename)
    rnn.initConfig(configFilename=config_filename, commandLineOptions=[])
    global config
    config = rnn.config
    config.set("log", None)
    config.set("log_verbosity", log_verbosity)
    config.set("use_tensorflow", True)
    rnn.initLog()
    print("Returnn compile-native-op starting up.", file=log.v1)
    rnn.returnnGreeting()
    rnn.initBackendEngine()
    assert Util.BackendEngine.is_tensorflow_selected(
    ), "this is only for TensorFlow"
    rnn.initFaulthandler()
    rnn.initConfigJsonNetwork()
    if 'network' in config.typed_dict:
        print("Loading network")
        from TFNetwork import TFNetwork
        network = TFNetwork(name="root",
                            config=config,
                            rnd_seed=1,
                            train_flag=False,
                            eval_flag=True,
                            search_flag=False)
        network.construct_from_dict(config.typed_dict["network"])
Beispiel #2
0
def init(config_str):
    """
  :param str config_str: either filename to config-file, or dict for dataset
  """
    rnn.initBetterExchook()
    rnn.initThreadJoinHack()
    if config_str.startswith("{"):
        print("Using dataset %s." % config_str)
        datasetDict = eval(config_str)
        configFilename = None
    else:
        datasetDict = None
        configFilename = config_str
        print("Using config file %r." % configFilename)
        assert os.path.exists(configFilename)
    rnn.initConfig(configFilename=configFilename, commandLineOptions=[])
    global config
    config = rnn.config
    config.set("log", None)
    if datasetDict:
        config.set("train", datasetDict)
    rnn.initLog()
    print("CRNN dump-dataset starting up.", file=log.v1)
    rnn.initFaulthandler()
    rnn.initConfigJsonNetwork()
    rnn.initData()
    rnn.printTaskProperties()
def init(config_filename, log_verbosity):
    """
  :param str config_filename: filename to config-file
  :param int log_verbosity:
  """
    rnn.initBetterExchook()
    rnn.initThreadJoinHack()
    if config_filename:
        print("Using config file %r." % config_filename)
        assert os.path.exists(config_filename)
    rnn.initConfig(configFilename=config_filename, commandLineOptions=[])
    global config
    config = rnn.config
    config.set("task", "calculate_wer")
    config.set("log", None)
    config.set("log_verbosity", log_verbosity)
    config.set("use_tensorflow", True)
    rnn.initLog()
    print("Returnn calculate-word-error-rate starting up.", file=log.v1)
    rnn.returnnGreeting()
    rnn.initBackendEngine()
    assert Util.BackendEngine.is_tensorflow_selected(
    ), "this is only for TensorFlow"
    rnn.initFaulthandler()
    rnn.initConfigJsonNetwork()
    rnn.printTaskProperties()
Beispiel #4
0
def init(config_filename, cmd_line_opts, dataset_config_str):
  """
  :param str config_filename: global config for CRNN
  :param list[str] cmd_line_opts: options for initConfig method
  :param str dataset_config_str: dataset via init_dataset_via_str()
  """
  rnn.initBetterExchook()
  rnn.initThreadJoinHack()
  if config_filename:
    rnn.initConfig(config_filename, cmd_line_opts)
    rnn.initLog()
  else:
    log.initialize(verbosity=[5])
  print >> log.v3, "CRNN dump-dataset starting up."
  rnn.initFaulthandler()
  rnn.initConfigJsonNetwork()
  if config_filename:
    rnn.initData()
    rnn.printTaskProperties()
    assert isinstance(rnn.train_data, Dataset)
    return rnn.train_data
  else:
    assert dataset_config_str
    dataset = init_dataset_via_str(dataset_config_str)
    print >> log.v3, "Source dataset:", dataset.len_info()
    return dataset
Beispiel #5
0
def init(config_str, verbosity):
    """
  :param str config_str: either filename to config-file, or dict for dataset
  :param int verbosity:
  """
    rnn.initBetterExchook()
    rnn.initThreadJoinHack()
    datasetDict = None
    configFilename = None
    if config_str.strip().startswith("{"):
        print("Using dataset %s." % config_str)
        datasetDict = eval(config_str.strip())
    elif config_str.endswith(".hdf"):
        datasetDict = {"class": "HDFDataset", "files": [config_str]}
        print("Using dataset %r." % datasetDict)
        assert os.path.exists(config_str)
    else:
        configFilename = config_str
        print("Using config file %r." % configFilename)
        assert os.path.exists(configFilename)
    rnn.initConfig(configFilename=configFilename,
                   default_config={"cache_size": "0"})
    global config
    config = rnn.config
    config.set("log", None)
    config.set("log_verbosity", verbosity)
    if datasetDict:
        config.set("train", datasetDict)
    rnn.initLog()
    print("Returnn dump-dataset starting up.", file=log.v2)
    rnn.returnnGreeting()
    rnn.initFaulthandler()
    rnn.initConfigJsonNetwork()
    rnn.initData()
    rnn.printTaskProperties()
Beispiel #6
0
def init(config_filename, cmd_line_opts, dataset_config_str):
    """
  :param str config_filename: global config for CRNN
  :param list[str] cmd_line_opts: options for initConfig method
  :param str dataset_config_str: dataset via init_dataset_via_str()
  """
    rnn.initBetterExchook()
    rnn.initThreadJoinHack()
    if config_filename:
        rnn.initConfig(config_filename, cmd_line_opts)
        rnn.initLog()
    else:
        log.initialize(verbosity=[5])
    print("Returnn hdf_dump starting up.", file=log.v3)
    rnn.initFaulthandler()
    if config_filename:
        rnn.initData()
        rnn.printTaskProperties()
        assert isinstance(rnn.train_data, Dataset)
        return rnn.train_data
    else:
        assert dataset_config_str
        dataset = init_dataset_via_str(dataset_config_str)
        print("Source dataset:", dataset.len_info(), file=log.v3)
        return dataset
Beispiel #7
0
def init(configFilename, commandLineOptions):
    rnn.initBetterExchook()
    rnn.initConfig(configFilename, commandLineOptions)
    global config
    config = rnn.config
    config.set("log", [])
    rnn.initLog()
    print("CRNN dump-dataset starting up.", file=log.v3)
    rnn.initConfigJsonNetwork()
Beispiel #8
0
def init(configFilename, commandLineOptions):
  rnn.initBetterExchook()
  rnn.initConfig(configFilename, commandLineOptions)
  global config
  config = rnn.config
  config.set("log", [])
  rnn.initLog()
  print >> log.v3, "CRNN dump-dataset starting up."
  rnn.initConfigJsonNetwork()
Beispiel #9
0
def init(configFilename, commandLineOptions):
    rnn.initBetterExchook()
    rnn.initThreadJoinHack()
    rnn.initConfig(configFilename, commandLineOptions)
    global config
    config = rnn.config
    rnn.initLog()
    print >> log.v3, "CRNN demo-dataset starting up"
    rnn.initFaulthandler()
    rnn.initConfigJsonNetwork()
    rnn.initData()
    rnn.printTaskProperties()
Beispiel #10
0
def init(configFilename, commandLineOptions):
  rnn.initBetterExchook()
  rnn.initThreadJoinHack()
  rnn.initConfig(configFilename, commandLineOptions)
  global config
  config = rnn.config
  rnn.initLog()
  print >> log.v3, "CRNN demo-dataset starting up"
  rnn.initFaulthandler()
  rnn.initConfigJsonNetwork()
  rnn.initData()
  rnn.printTaskProperties()
Beispiel #11
0
def init(configFilename, commandLineOptions):
    rnn.initBetterExchook()
    rnn.initThreadJoinHack()
    rnn.initConfig(configFilename, commandLineOptions)
    global config
    config = rnn.config
    config.set("log", None)
    rnn.initLog()
    print("CRNN dump-dataset starting up.", file=log.v1)
    rnn.initFaulthandler()
    rnn.initConfigJsonNetwork()
    rnn.initData()
    rnn.printTaskProperties()
def init(configFilename=None):
    rnn.initBetterExchook()
    rnn.initThreadJoinHack()
    if configFilename:
        rnn.initConfig(configFilename, commandLineOptions=[])
        rnn.initLog()
    else:
        log.initialize()
    print("CRNN collect-orth-symbols starting up.", file=log.v3)
    rnn.initFaulthandler()
    if configFilename:
        rnn.initConfigJsonNetwork()
        rnn.initData()
        rnn.printTaskProperties()
Beispiel #13
0
def init(configFilename=None):
  rnn.initBetterExchook()
  rnn.initThreadJoinHack()
  if configFilename:
    rnn.initConfig(configFilename, commandLineOptions=[])
    rnn.initLog()
  else:
    log.initialize()
  print >> log.v3, "CRNN collect-orth-symbols starting up."
  rnn.initFaulthandler()
  if configFilename:
    rnn.initConfigJsonNetwork()
    rnn.initData()
    rnn.printTaskProperties()
Beispiel #14
0
def init(config_filename, log_verbosity):
    """
  :param str config_filename: filename to config-file
  :param int log_verbosity:
  """
    rnn.initBetterExchook()
    rnn.initThreadJoinHack()
    if config_filename:
        print("Using config file %r." % config_filename)
        assert os.path.exists(config_filename)
    rnn.initConfig(configFilename=config_filename, commandLineOptions=[])
    global config
    config = rnn.config
    config.set("task", "dump")
    config.set("log", None)
    config.set("log_verbosity", log_verbosity)
    rnn.initLog()
    print("Returnn dump-dataset-raw-strings starting up.", file=log.v1)
    rnn.returnnGreeting()
    rnn.initFaulthandler()
Beispiel #15
0
def init(configFilename, commandLineOptions, args):
    rnn.initBetterExchook()
    config_updates = {
        "log": None,
        "task": "eval",
        "eval": "config:get_dataset(%r)" % args.data,
        "train": None,
        "dev": None,
        "need_data": True,
    }
    if args.epoch:
        config_updates["load_epoch"] = args.epoch
    if args.do_search:
        config_updates.update({
            "task": "search",
            "search_data": "config:get_dataset(%r)" % args.data,
            "search_do_eval": False,
            "beam_size": int(args.beam_size),
            "max_seq_length": 0,
        })

    rnn.init(configFilename=configFilename,
             commandLineOptions=commandLineOptions,
             config_updates=config_updates,
             extra_greeting="CRNN dump-forward starting up.")
    rnn.engine.init_train_from_config(config=rnn.config, train_data=None)

    if rnn.engine.pretrain:
        new_network_desc = rnn.engine.pretrain.get_network_json_for_epoch(
            rnn.engine.epoch)
        rnn.engine.maybe_init_new_network(new_network_desc)
    global config
    config = rnn.config
    config.set("log", [])
    rnn.initLog()
    print("CRNN get-attention-weights starting up.", file=log.v3)
Beispiel #16
0
def init_returnn(config_fn, cmd_line_opts, args):
    """
  :param str config_fn:
  :param list[str] cmd_line_opts:
  :param args: arg_parse object
  """
    rnn.initBetterExchook()
    config_updates = {"log": [], "task": "eval", "need_data": False}
    if args.epoch:
        config_updates["load_epoch"] = args.epoch
    if args.do_search:
        config_updates.update({
            "task": "search",
            "search_do_eval": False,
            "beam_size": args.beam_size,
            "max_seq_length": 0,
        })

    rnn.init(configFilename=config_fn,
             commandLineOptions=cmd_line_opts,
             config_updates=config_updates,
             extra_greeting="RETURNN get-attention-weights starting up.")
    global config
    config = rnn.config
Beispiel #17
0
 def control_loop():
   rnn.initBetterExchook()
   self.run_control_loop(self.own_threaded_callback)
Beispiel #18
0
from TaskSystem import Pickler, Unpickler, numpy_set_unused
from Util import to_bool
from threading import Condition

InitTypes = set()
Verbose = False  # disables all per-segment log messages
Quiet = False # disables all but error messages

def print(str):
  import __builtin__
  if not Quiet:
    __builtin__.print(str)

print("CRNN SprintControl[pid %i] Python module load" % os.getpid())

rnn.initBetterExchook()
Debug.initFaulthandler(sigusr1_chain=True)  # Sprint also handles SIGUSR1.
rnn.initThreadJoinHack()

# Start Sprint PythonControl interface. {

def init(name, reference, config, sprint_unit=None, version_number=None, callback=None, **kwargs):
  """
  This will be called by Sprint PythonControl.
  But we also call it ourselves e.g. in getSegmentList() and SprintNnPythonLayer.
  :param str name: this specifies the caller. e.g. "Sprint.PythonControl"
  :param reference: this is any object to identify the specific instance of the caller, if there are multiple.
  :param str config: this will be passed over from Sprint. you can configure that via --*.pymod-config.
  :param str sprint_unit: if this is called by Sprint PythonControl, this will specify which specific part
    of Sprint is using this PythonControl, because there can be multiple parts.
    E.g. there is "FeedForwardTrainer", "SegmentwiseNnTrainer" and "NnTrainer.pythonControl".
Beispiel #19
0
def initBase(configfile=None, targetMode=None, epoch=None):
    """
  :param str|None configfile: filename, via init(), this is set
  :param str|None targetMode: "forward" or so. via init(), this is set
  :param int epoch: via init(), this is set
  """

    global isInitialized
    isInitialized = True
    # Run through in any case. Maybe just to set targetMode.

    if not getattr(sys, "argv", None):
        # Set some dummy. Some code might want this (e.g. TensorFlow).
        sys.argv = [__file__]

    global config
    if not config:
        # Some subset of what we do in rnn.init().

        rnn.initBetterExchook()
        rnn.initThreadJoinHack()

        if configfile is None:
            configfile = DefaultSprintCrnnConfig
        assert os.path.exists(configfile)
        rnn.initConfig(configFilename=configfile)
        config = rnn.config

        rnn.initLog()
        rnn.returnnGreeting(configFilename=configfile)
        rnn.initBackendEngine()
        rnn.initFaulthandler(sigusr1_chain=True)
        rnn.initConfigJsonNetwork()

        if BackendEngine.is_tensorflow_selected():
            # Use TFEngine.Engine class instead of Engine.Engine.
            import TFEngine
            global Engine
            Engine = TFEngine.Engine

        import atexit
        atexit.register(_at_exit_handler)

    if targetMode:
        setTargetMode(targetMode)

    initDataset()

    if targetMode and targetMode == "forward" and epoch:
        model_filename = config.value('model', '')
        fns = [
            Engine.epoch_model_filename(model_filename, epoch, is_pretrain)
            for is_pretrain in [False, True]
        ]
        fn_postfix = ""
        if BackendEngine.is_tensorflow_selected():
            fn_postfix += ".meta"
        fns_existing = [fn for fn in fns if os.path.exists(fn + fn_postfix)]
        assert len(fns_existing) == 1, "%s not found" % fns
        model_epoch_filename = fns_existing[0]
        config.set('load', model_epoch_filename)
        assert Engine.get_epoch_model(config)[1] == model_epoch_filename, \
          "%r != %r" % (Engine.get_epoch_model(config), model_epoch_filename)

    global engine
    if not engine:
        devices = rnn.initDevices()
        rnn.printTaskProperties(devices)
        rnn.initEngine(devices)
        engine = rnn.engine
        assert isinstance(engine, Engine)
Beispiel #20
0
 def control_loop():
   rnn.initBetterExchook()
   self.run_control_loop(self.own_threaded_callback)
Beispiel #21
0
from TaskSystem import Pickler, Unpickler, numpy_set_unused
from Util import to_bool
from threading import Condition

InitTypes = set()
Verbose = False  # disables all per-segment log messages
Quiet = False # disables all but error messages

_orig_print = print
def print(*args, **kwargs):
  if not Quiet:
    _orig_print(*args, **kwargs)

print("CRNN SprintControl[pid %i] Python module load" % os.getpid())

rnn.initBetterExchook()
Debug.initFaulthandler(sigusr1_chain=True)  # Sprint also handles SIGUSR1.
rnn.initThreadJoinHack()

# Start Sprint PythonControl interface. {

def init(name, reference, config, sprint_unit=None, version_number=None, callback=None, **kwargs):
  """
  This will be called by Sprint PythonControl.
  But we also call it ourselves e.g. in getSegmentList() and SprintNnPythonLayer.
  :param str name: this specifies the caller. e.g. "Sprint.PythonControl"
  :param reference: this is any object to identify the specific instance of the caller, if there are multiple.
  :param str config: this will be passed over from Sprint. you can configure that via --*.pymod-config.
  :param str sprint_unit: if this is called by Sprint PythonControl, this will specify which specific part
    of Sprint is using this PythonControl, because there can be multiple parts.
    E.g. there is "FeedForwardTrainer", "SegmentwiseNnTrainer" and "NnTrainer.pythonControl".