def init(config_str): """ :param str config_str: either filename to config-file, or dict for dataset """ rnn.initBetterExchook() rnn.initThreadJoinHack() if config_str.startswith("{"): print("Using dataset %s." % config_str) datasetDict = eval(config_str) configFilename = None else: datasetDict = None configFilename = config_str print("Using config file %r." % configFilename) assert os.path.exists(configFilename) rnn.initConfig(configFilename=configFilename, commandLineOptions=[]) global config config = rnn.config config.set("log", None) if datasetDict: config.set("train", datasetDict) rnn.initLog() print("CRNN dump-dataset starting up.", file=log.v1) rnn.initFaulthandler() rnn.initConfigJsonNetwork() rnn.initData() rnn.printTaskProperties()
def init(config_filename, cmd_line_opts, dataset_config_str): """ :param str config_filename: global config for CRNN :param list[str] cmd_line_opts: options for initConfig method :param str dataset_config_str: dataset via init_dataset_via_str() """ rnn.initBetterExchook() rnn.initThreadJoinHack() if config_filename: rnn.initConfig(config_filename, cmd_line_opts) rnn.initLog() else: log.initialize(verbosity=[5]) print >> log.v3, "CRNN dump-dataset starting up." rnn.initFaulthandler() rnn.initConfigJsonNetwork() if config_filename: rnn.initData() rnn.printTaskProperties() assert isinstance(rnn.train_data, Dataset) return rnn.train_data else: assert dataset_config_str dataset = init_dataset_via_str(dataset_config_str) print >> log.v3, "Source dataset:", dataset.len_info() return dataset
def init(config_filename, log_verbosity): """ :param str config_filename: filename to config-file :param int log_verbosity: """ rnn.initBetterExchook() rnn.initThreadJoinHack() if config_filename: print("Using config file %r." % config_filename) assert os.path.exists(config_filename) rnn.initConfig(configFilename=config_filename, commandLineOptions=[]) global config config = rnn.config config.set("log", None) config.set("log_verbosity", log_verbosity) config.set("use_tensorflow", True) rnn.initLog() print("Returnn compile-native-op starting up.", file=log.v1) rnn.returnnGreeting() rnn.initBackendEngine() assert Util.BackendEngine.is_tensorflow_selected( ), "this is only for TensorFlow" rnn.initFaulthandler() rnn.initConfigJsonNetwork() if 'network' in config.typed_dict: print("Loading network") from TFNetwork import TFNetwork network = TFNetwork(name="root", config=config, rnd_seed=1, train_flag=False, eval_flag=True, search_flag=False) network.construct_from_dict(config.typed_dict["network"])
def init(config_filename, cmd_line_opts, dataset_config_str): """ :param str config_filename: global config for CRNN :param list[str] cmd_line_opts: options for initConfig method :param str dataset_config_str: dataset via init_dataset_via_str() """ rnn.initBetterExchook() rnn.initThreadJoinHack() if config_filename: rnn.initConfig(config_filename, cmd_line_opts) rnn.initLog() else: log.initialize(verbosity=[5]) print("CRNN dump-dataset starting up.", file=log.v3) rnn.initFaulthandler() rnn.initConfigJsonNetwork() if config_filename: rnn.initData() rnn.printTaskProperties() assert isinstance(rnn.train_data, Dataset) return rnn.train_data else: assert dataset_config_str dataset = init_dataset_via_str(dataset_config_str) print("Source dataset:", dataset.len_info(), file=log.v3) return dataset
def init(config_filename, log_verbosity): """ :param str config_filename: filename to config-file :param int log_verbosity: """ rnn.initBetterExchook() rnn.initThreadJoinHack() if config_filename: print("Using config file %r." % config_filename) assert os.path.exists(config_filename) rnn.initConfig(configFilename=config_filename, commandLineOptions=[]) global config config = rnn.config config.set("task", "calculate_wer") config.set("log", None) config.set("log_verbosity", log_verbosity) config.set("use_tensorflow", True) rnn.initLog() print("Returnn calculate-word-error-rate starting up.", file=log.v1) rnn.returnnGreeting() rnn.initBackendEngine() assert Util.BackendEngine.is_tensorflow_selected( ), "this is only for TensorFlow" rnn.initFaulthandler() rnn.initConfigJsonNetwork() rnn.printTaskProperties()
def init(config_str, verbosity): """ :param str config_str: either filename to config-file, or dict for dataset :param int verbosity: """ rnn.initBetterExchook() rnn.initThreadJoinHack() datasetDict = None configFilename = None if config_str.strip().startswith("{"): print("Using dataset %s." % config_str) datasetDict = eval(config_str.strip()) elif config_str.endswith(".hdf"): datasetDict = {"class": "HDFDataset", "files": [config_str]} print("Using dataset %r." % datasetDict) assert os.path.exists(config_str) else: configFilename = config_str print("Using config file %r." % configFilename) assert os.path.exists(configFilename) rnn.initConfig(configFilename=configFilename, default_config={"cache_size": "0"}) global config config = rnn.config config.set("log", None) config.set("log_verbosity", verbosity) if datasetDict: config.set("train", datasetDict) rnn.initLog() print("Returnn dump-dataset starting up.", file=log.v2) rnn.returnnGreeting() rnn.initFaulthandler() rnn.initConfigJsonNetwork() rnn.initData() rnn.printTaskProperties()
def init(configFilename, commandLineOptions): rnn.initBetterExchook() rnn.initConfig(configFilename, commandLineOptions) global config config = rnn.config config.set("log", []) rnn.initLog() print("CRNN dump-dataset starting up.", file=log.v3) rnn.initConfigJsonNetwork()
def init(configFilename, commandLineOptions): rnn.initBetterExchook() rnn.initConfig(configFilename, commandLineOptions) global config config = rnn.config config.set("log", []) rnn.initLog() print >> log.v3, "CRNN dump-dataset starting up." rnn.initConfigJsonNetwork()
def init(configFilename, commandLineOptions): rnn.initBetterExchook() rnn.initThreadJoinHack() rnn.initConfig(configFilename, commandLineOptions) global config config = rnn.config rnn.initLog() print >> log.v3, "CRNN demo-dataset starting up" rnn.initFaulthandler() rnn.initConfigJsonNetwork() rnn.initData() rnn.printTaskProperties()
def init(configFilename, commandLineOptions): rnn.initBetterExchook() rnn.initThreadJoinHack() rnn.initConfig(configFilename, commandLineOptions) global config config = rnn.config config.set("log", None) rnn.initLog() print("CRNN dump-dataset starting up.", file=log.v1) rnn.initFaulthandler() rnn.initConfigJsonNetwork() rnn.initData() rnn.printTaskProperties()
def init(configFilename=None): rnn.initBetterExchook() rnn.initThreadJoinHack() if configFilename: rnn.initConfig(configFilename, commandLineOptions=[]) rnn.initLog() else: log.initialize() print("CRNN collect-orth-symbols starting up.", file=log.v3) rnn.initFaulthandler() if configFilename: rnn.initConfigJsonNetwork() rnn.initData() rnn.printTaskProperties()
def init(configFilename=None): rnn.initBetterExchook() rnn.initThreadJoinHack() if configFilename: rnn.initConfig(configFilename, commandLineOptions=[]) rnn.initLog() else: log.initialize() print >> log.v3, "CRNN collect-orth-symbols starting up." rnn.initFaulthandler() if configFilename: rnn.initConfigJsonNetwork() rnn.initData() rnn.printTaskProperties()
def initBase(configfile=None, targetMode=None, epoch=None): """ :type configfile: str | None """ global isInitialized isInitialized = True # Run through in any case. Maybe just to set targetMode. global config if not config: if configfile is None: configfile = DefaultSprintCrnnConfig assert os.path.exists(configfile) rnn.initThreadJoinHack() rnn.initConfig(configfile, []) config = rnn.config rnn.initLog() rnn.initConfigJsonNetwork() if targetMode: setTargetMode(targetMode) initDataset() if targetMode and targetMode == "forward" and epoch: model_filename = config.value('model', '') fns = [Engine.epoch_model_filename(model_filename, epoch, is_pretrain) for is_pretrain in [False, True]] fns_existing = [fn for fn in fns if os.path.exists(fn)] assert len(fns_existing) == 1, "%s not found" % fns model_epoch_filename = fns_existing[0] config.set('load', model_epoch_filename) assert Engine.get_epoch_model(config)[1] == model_epoch_filename global engine if not engine: devices = rnn.initDevices() rnn.printTaskProperties(devices) rnn.initEngine(devices) engine = rnn.engine assert isinstance(engine, Engine)
def initBase(configfile=None, targetMode=None, epoch=None): """ :param str|None configfile: filename, via init(), this is set :param str|None targetMode: "forward" or so. via init(), this is set :param int epoch: via init(), this is set """ global isInitialized isInitialized = True # Run through in any case. Maybe just to set targetMode. if not getattr(sys, "argv", None): # Set some dummy. Some code might want this (e.g. TensorFlow). sys.argv = [__file__] global config if not config: # Some subset of what we do in rnn.init(). rnn.initBetterExchook() rnn.initThreadJoinHack() if configfile is None: configfile = DefaultSprintCrnnConfig assert os.path.exists(configfile) rnn.initConfig(configFilename=configfile) config = rnn.config rnn.initLog() rnn.returnnGreeting(configFilename=configfile) rnn.initBackendEngine() rnn.initFaulthandler(sigusr1_chain=True) rnn.initConfigJsonNetwork() if BackendEngine.is_tensorflow_selected(): # Use TFEngine.Engine class instead of Engine.Engine. import TFEngine global Engine Engine = TFEngine.Engine import atexit atexit.register(_at_exit_handler) if targetMode: setTargetMode(targetMode) initDataset() if targetMode and targetMode == "forward" and epoch: model_filename = config.value('model', '') fns = [ Engine.epoch_model_filename(model_filename, epoch, is_pretrain) for is_pretrain in [False, True] ] fn_postfix = "" if BackendEngine.is_tensorflow_selected(): fn_postfix += ".meta" fns_existing = [fn for fn in fns if os.path.exists(fn + fn_postfix)] assert len(fns_existing) == 1, "%s not found" % fns model_epoch_filename = fns_existing[0] config.set('load', model_epoch_filename) assert Engine.get_epoch_model(config)[1] == model_epoch_filename, \ "%r != %r" % (Engine.get_epoch_model(config), model_epoch_filename) global engine if not engine: devices = rnn.initDevices() rnn.printTaskProperties(devices) rnn.initEngine(devices) engine = rnn.engine assert isinstance(engine, Engine)