def init(config_filename, log_verbosity):
"""
:param str config_filename: filename to config-file
:param int log_verbosity:
"""
rnn.initBetterExchook()
rnn.initThreadJoinHack()
if config_filename:
print("Using config file %r." % config_filename)
assert os.path.exists(config_filename)
rnn.initConfig(configFilename=config_filename, commandLineOptions=[])
global config
config = rnn.config
config.set("log", None)
config.set("log_verbosity", log_verbosity)
config.set("use_tensorflow", True)
rnn.initLog()
print("Returnn compile-native-op starting up.", file=log.v1)
rnn.returnnGreeting()
rnn.initBackendEngine()
assert Util.BackendEngine.is_tensorflow_selected(
), "this is only for TensorFlow"
rnn.initFaulthandler()
rnn.initConfigJsonNetwork()
if 'network' in config.typed_dict:
print("Loading network")
from TFNetwork import TFNetwork
network = TFNetwork(name="root",
config=config,
rnd_seed=1,
train_flag=False,
eval_flag=True,
search_flag=False)
network.construct_from_dict(config.typed_dict["network"])
def init(config_str):
"""
:param str config_str: either filename to config-file, or dict for dataset
"""
rnn.initBetterExchook()
rnn.initThreadJoinHack()
if config_str.startswith("{"):
print("Using dataset %s." % config_str)
datasetDict = eval(config_str)
configFilename = None
else:
datasetDict = None
configFilename = config_str
print("Using config file %r." % configFilename)
assert os.path.exists(configFilename)
rnn.initConfig(configFilename=configFilename, commandLineOptions=[])
global config
config = rnn.config
config.set("log", None)
if datasetDict:
config.set("train", datasetDict)
rnn.initLog()
print("CRNN dump-dataset starting up.", file=log.v1)
rnn.initFaulthandler()
rnn.initConfigJsonNetwork()
rnn.initData()
rnn.printTaskProperties()
def init(config_filename, log_verbosity):
"""
:param str config_filename: filename to config-file
:param int log_verbosity:
"""
rnn.initBetterExchook()
rnn.initThreadJoinHack()
if config_filename:
print("Using config file %r." % config_filename)
assert os.path.exists(config_filename)
rnn.initConfig(configFilename=config_filename, commandLineOptions=[])
global config
config = rnn.config
config.set("task", "calculate_wer")
config.set("log", None)
config.set("log_verbosity", log_verbosity)
config.set("use_tensorflow", True)
rnn.initLog()
print("Returnn calculate-word-error-rate starting up.", file=log.v1)
rnn.returnnGreeting()
rnn.initBackendEngine()
assert Util.BackendEngine.is_tensorflow_selected(
), "this is only for TensorFlow"
rnn.initFaulthandler()
rnn.initConfigJsonNetwork()
rnn.printTaskProperties()
def init(config_filename, cmd_line_opts, dataset_config_str):
"""
:param str config_filename: global config for CRNN
:param list[str] cmd_line_opts: options for initConfig method
:param str dataset_config_str: dataset via init_dataset_via_str()
"""
rnn.initBetterExchook()
rnn.initThreadJoinHack()
if config_filename:
rnn.initConfig(config_filename, cmd_line_opts)
rnn.initLog()
else:
log.initialize(verbosity=[5])
print >> log.v3, "CRNN dump-dataset starting up."
rnn.initFaulthandler()
rnn.initConfigJsonNetwork()
if config_filename:
rnn.initData()
rnn.printTaskProperties()
assert isinstance(rnn.train_data, Dataset)
return rnn.train_data
else:
assert dataset_config_str
dataset = init_dataset_via_str(dataset_config_str)
print >> log.v3, "Source dataset:", dataset.len_info()
return dataset
def init(config_str, verbosity):
"""
:param str config_str: either filename to config-file, or dict for dataset
:param int verbosity:
"""
rnn.initBetterExchook()
rnn.initThreadJoinHack()
datasetDict = None
configFilename = None
if config_str.strip().startswith("{"):
print("Using dataset %s." % config_str)
datasetDict = eval(config_str.strip())
elif config_str.endswith(".hdf"):
datasetDict = {"class": "HDFDataset", "files": [config_str]}
print("Using dataset %r." % datasetDict)
assert os.path.exists(config_str)
else:
configFilename = config_str
print("Using config file %r." % configFilename)
assert os.path.exists(configFilename)
rnn.initConfig(configFilename=configFilename,
default_config={"cache_size": "0"})
global config
config = rnn.config
config.set("log", None)
config.set("log_verbosity", verbosity)
if datasetDict:
config.set("train", datasetDict)
rnn.initLog()
print("Returnn dump-dataset starting up.", file=log.v2)
rnn.returnnGreeting()
rnn.initFaulthandler()
rnn.initConfigJsonNetwork()
rnn.initData()
rnn.printTaskProperties()
def init(config_filename, cmd_line_opts, dataset_config_str):
"""
:param str config_filename: global config for CRNN
:param list[str] cmd_line_opts: options for initConfig method
:param str dataset_config_str: dataset via init_dataset_via_str()
"""
rnn.initBetterExchook()
rnn.initThreadJoinHack()
if config_filename:
rnn.initConfig(config_filename, cmd_line_opts)
rnn.initLog()
else:
log.initialize(verbosity=[5])
print("Returnn hdf_dump starting up.", file=log.v3)
rnn.initFaulthandler()
if config_filename:
rnn.initData()
rnn.printTaskProperties()
assert isinstance(rnn.train_data, Dataset)
return rnn.train_data
else:
assert dataset_config_str
dataset = init_dataset_via_str(dataset_config_str)
print("Source dataset:", dataset.len_info(), file=log.v3)
return dataset
def init(configFilename, commandLineOptions):
rnn.initBetterExchook()
rnn.initConfig(configFilename, commandLineOptions)
global config
config = rnn.config
config.set("log", [])
rnn.initLog()
print("CRNN dump-dataset starting up.", file=log.v3)
rnn.initConfigJsonNetwork()
def init(configFilename, commandLineOptions):
rnn.initBetterExchook()
rnn.initConfig(configFilename, commandLineOptions)
global config
config = rnn.config
config.set("log", [])
rnn.initLog()
print >> log.v3, "CRNN dump-dataset starting up."
rnn.initConfigJsonNetwork()
def init(configFilename, commandLineOptions):
rnn.initBetterExchook()
rnn.initThreadJoinHack()
rnn.initConfig(configFilename, commandLineOptions)
global config
config = rnn.config
rnn.initLog()
print >> log.v3, "CRNN demo-dataset starting up"
rnn.initFaulthandler()
rnn.initConfigJsonNetwork()
rnn.initData()
rnn.printTaskProperties()
def init(configFilename, commandLineOptions): rnn.initBetterExchook() rnn.initThreadJoinHack() rnn.initConfig(configFilename, commandLineOptions) global config config = rnn.config rnn.initLog() print >> log.v3, "CRNN demo-dataset starting up" rnn.initFaulthandler() rnn.initConfigJsonNetwork() rnn.initData() rnn.printTaskProperties()
def init(configFilename, commandLineOptions):
rnn.initBetterExchook()
rnn.initThreadJoinHack()
rnn.initConfig(configFilename, commandLineOptions)
global config
config = rnn.config
config.set("log", None)
rnn.initLog()
print("CRNN dump-dataset starting up.", file=log.v1)
rnn.initFaulthandler()
rnn.initConfigJsonNetwork()
rnn.initData()
rnn.printTaskProperties()
def init(configFilename=None):
rnn.initBetterExchook()
rnn.initThreadJoinHack()
if configFilename:
rnn.initConfig(configFilename, commandLineOptions=[])
rnn.initLog()
else:
log.initialize()
print("CRNN collect-orth-symbols starting up.", file=log.v3)
rnn.initFaulthandler()
if configFilename:
rnn.initConfigJsonNetwork()
rnn.initData()
rnn.printTaskProperties()
def init(configFilename=None):
rnn.initBetterExchook()
rnn.initThreadJoinHack()
if configFilename:
rnn.initConfig(configFilename, commandLineOptions=[])
rnn.initLog()
else:
log.initialize()
print >> log.v3, "CRNN collect-orth-symbols starting up."
rnn.initFaulthandler()
if configFilename:
rnn.initConfigJsonNetwork()
rnn.initData()
rnn.printTaskProperties()
def init(config_filename, log_verbosity):
"""
:param str config_filename: filename to config-file
:param int log_verbosity:
"""
rnn.initBetterExchook()
rnn.initThreadJoinHack()
if config_filename:
print("Using config file %r." % config_filename)
assert os.path.exists(config_filename)
rnn.initConfig(configFilename=config_filename, commandLineOptions=[])
global config
config = rnn.config
config.set("task", "dump")
config.set("log", None)
config.set("log_verbosity", log_verbosity)
rnn.initLog()
print("Returnn dump-dataset-raw-strings starting up.", file=log.v1)
rnn.returnnGreeting()
rnn.initFaulthandler()
def init(configFilename, commandLineOptions, args):
rnn.initBetterExchook()
config_updates = {
"log": None,
"task": "eval",
"eval": "config:get_dataset(%r)" % args.data,
"train": None,
"dev": None,
"need_data": True,
}
if args.epoch:
config_updates["load_epoch"] = args.epoch
if args.do_search:
config_updates.update({
"task": "search",
"search_data": "config:get_dataset(%r)" % args.data,
"search_do_eval": False,
"beam_size": int(args.beam_size),
"max_seq_length": 0,
})
rnn.init(configFilename=configFilename,
commandLineOptions=commandLineOptions,
config_updates=config_updates,
extra_greeting="CRNN dump-forward starting up.")
rnn.engine.init_train_from_config(config=rnn.config, train_data=None)
if rnn.engine.pretrain:
new_network_desc = rnn.engine.pretrain.get_network_json_for_epoch(
rnn.engine.epoch)
rnn.engine.maybe_init_new_network(new_network_desc)
global config
config = rnn.config
config.set("log", [])
rnn.initLog()
print("CRNN get-attention-weights starting up.", file=log.v3)
def init_returnn(config_fn, cmd_line_opts, args):
"""
:param str config_fn:
:param list[str] cmd_line_opts:
:param args: arg_parse object
"""
rnn.initBetterExchook()
config_updates = {"log": [], "task": "eval", "need_data": False}
if args.epoch:
config_updates["load_epoch"] = args.epoch
if args.do_search:
config_updates.update({
"task": "search",
"search_do_eval": False,
"beam_size": args.beam_size,
"max_seq_length": 0,
})
rnn.init(configFilename=config_fn,
commandLineOptions=cmd_line_opts,
config_updates=config_updates,
extra_greeting="RETURNN get-attention-weights starting up.")
global config
config = rnn.config
def control_loop(): rnn.initBetterExchook() self.run_control_loop(self.own_threaded_callback)
from TaskSystem import Pickler, Unpickler, numpy_set_unused
from Util import to_bool
from threading import Condition
InitTypes = set()
Verbose = False # disables all per-segment log messages
Quiet = False # disables all but error messages
def print(str):
import __builtin__
if not Quiet:
__builtin__.print(str)
print("CRNN SprintControl[pid %i] Python module load" % os.getpid())
rnn.initBetterExchook()
Debug.initFaulthandler(sigusr1_chain=True) # Sprint also handles SIGUSR1.
rnn.initThreadJoinHack()
# Start Sprint PythonControl interface. {
def init(name, reference, config, sprint_unit=None, version_number=None, callback=None, **kwargs):
"""
This will be called by Sprint PythonControl.
But we also call it ourselves e.g. in getSegmentList() and SprintNnPythonLayer.
:param str name: this specifies the caller. e.g. "Sprint.PythonControl"
:param reference: this is any object to identify the specific instance of the caller, if there are multiple.
:param str config: this will be passed over from Sprint. you can configure that via --*.pymod-config.
:param str sprint_unit: if this is called by Sprint PythonControl, this will specify which specific part
of Sprint is using this PythonControl, because there can be multiple parts.
E.g. there is "FeedForwardTrainer", "SegmentwiseNnTrainer" and "NnTrainer.pythonControl".
def initBase(configfile=None, targetMode=None, epoch=None):
"""
:param str|None configfile: filename, via init(), this is set
:param str|None targetMode: "forward" or so. via init(), this is set
:param int epoch: via init(), this is set
"""
global isInitialized
isInitialized = True
# Run through in any case. Maybe just to set targetMode.
if not getattr(sys, "argv", None):
# Set some dummy. Some code might want this (e.g. TensorFlow).
sys.argv = [__file__]
global config
if not config:
# Some subset of what we do in rnn.init().
rnn.initBetterExchook()
rnn.initThreadJoinHack()
if configfile is None:
configfile = DefaultSprintCrnnConfig
assert os.path.exists(configfile)
rnn.initConfig(configFilename=configfile)
config = rnn.config
rnn.initLog()
rnn.returnnGreeting(configFilename=configfile)
rnn.initBackendEngine()
rnn.initFaulthandler(sigusr1_chain=True)
rnn.initConfigJsonNetwork()
if BackendEngine.is_tensorflow_selected():
# Use TFEngine.Engine class instead of Engine.Engine.
import TFEngine
global Engine
Engine = TFEngine.Engine
import atexit
atexit.register(_at_exit_handler)
if targetMode:
setTargetMode(targetMode)
initDataset()
if targetMode and targetMode == "forward" and epoch:
model_filename = config.value('model', '')
fns = [
Engine.epoch_model_filename(model_filename, epoch, is_pretrain)
for is_pretrain in [False, True]
]
fn_postfix = ""
if BackendEngine.is_tensorflow_selected():
fn_postfix += ".meta"
fns_existing = [fn for fn in fns if os.path.exists(fn + fn_postfix)]
assert len(fns_existing) == 1, "%s not found" % fns
model_epoch_filename = fns_existing[0]
config.set('load', model_epoch_filename)
assert Engine.get_epoch_model(config)[1] == model_epoch_filename, \
"%r != %r" % (Engine.get_epoch_model(config), model_epoch_filename)
global engine
if not engine:
devices = rnn.initDevices()
rnn.printTaskProperties(devices)
rnn.initEngine(devices)
engine = rnn.engine
assert isinstance(engine, Engine)
def control_loop(): rnn.initBetterExchook() self.run_control_loop(self.own_threaded_callback)
from TaskSystem import Pickler, Unpickler, numpy_set_unused
from Util import to_bool
from threading import Condition
InitTypes = set()
Verbose = False # disables all per-segment log messages
Quiet = False # disables all but error messages
_orig_print = print
def print(*args, **kwargs):
if not Quiet:
_orig_print(*args, **kwargs)
print("CRNN SprintControl[pid %i] Python module load" % os.getpid())
rnn.initBetterExchook()
Debug.initFaulthandler(sigusr1_chain=True) # Sprint also handles SIGUSR1.
rnn.initThreadJoinHack()
# Start Sprint PythonControl interface. {
def init(name, reference, config, sprint_unit=None, version_number=None, callback=None, **kwargs):
"""
This will be called by Sprint PythonControl.
But we also call it ourselves e.g. in getSegmentList() and SprintNnPythonLayer.
:param str name: this specifies the caller. e.g. "Sprint.PythonControl"
:param reference: this is any object to identify the specific instance of the caller, if there are multiple.
:param str config: this will be passed over from Sprint. you can configure that via --*.pymod-config.
:param str sprint_unit: if this is called by Sprint PythonControl, this will specify which specific part
of Sprint is using this PythonControl, because there can be multiple parts.
E.g. there is "FeedForwardTrainer", "SegmentwiseNnTrainer" and "NnTrainer.pythonControl".