def init(config_filename, cmd_line_opts, dataset_config_str):
"""
:param str config_filename: global config for CRNN
:param list[str] cmd_line_opts: options for init_config method
:param str dataset_config_str: dataset via init_dataset_via_str()
"""
rnn.init_better_exchook()
rnn.init_thread_join_hack()
if config_filename:
rnn.init_config(config_filename, cmd_line_opts)
rnn.init_log()
else:
log.initialize(verbosity=[5])
print("Returnn hdf_dump starting up.", file=log.v3)
rnn.init_faulthandler()
if config_filename:
rnn.init_data()
rnn.print_task_properties()
assert isinstance(rnn.train_data, Dataset)
dataset = rnn.train_data
else:
assert dataset_config_str
dataset = init_dataset(dataset_config_str)
print("Source dataset:", dataset.len_info(), file=log.v3)
return dataset
def init(config_filename, cmd_line_opts, dataset_config_str):
"""
:param str config_filename: global config for CRNN
:param list[str] cmd_line_opts: options for init_config method
:param str dataset_config_str: dataset via init_dataset_via_str()
"""
rnn.init_better_exchook()
rnn.init_thread_join_hack()
if config_filename:
rnn.init_config(config_filename, cmd_line_opts)
rnn.init_log()
else:
log.initialize(verbosity=[5])
print("Returnn hdf_dump starting up.", file=log.v3)
rnn.init_faulthandler()
if config_filename:
rnn.init_data()
rnn.print_task_properties()
assert isinstance(rnn.train_data, Dataset)
dataset = rnn.train_data
else:
assert dataset_config_str
dataset = init_dataset(dataset_config_str)
print("Source dataset:", dataset.len_info(), file=log.v3)
return dataset
def init(config_str, verbosity):
"""
:param str config_str: either filename to config-file, or dict for dataset
:param int verbosity:
"""
rnn.init_better_exchook()
rnn.init_thread_join_hack()
datasetDict = None
configFilename = None
if config_str.strip().startswith("{"):
print("Using dataset %s." % config_str)
datasetDict = eval(config_str.strip())
elif config_str.endswith(".hdf"):
datasetDict = {"class": "HDFDataset", "files": [config_str]}
print("Using dataset %r." % datasetDict)
assert os.path.exists(config_str)
else:
configFilename = config_str
print("Using config file %r." % configFilename)
assert os.path.exists(configFilename)
rnn.init_config(config_filename=configFilename,
default_config={"cache_size": "0"})
global config
config = rnn.config
config.set("log", None)
config.set("log_verbosity", verbosity)
if datasetDict:
config.set("train", datasetDict)
rnn.init_log()
print("Returnn dump-dataset starting up.", file=log.v2)
rnn.returnn_greeting()
rnn.init_faulthandler()
rnn.init_config_json_network()
rnn.init_data()
rnn.print_task_properties()
def init(config_str, verbosity):
"""
:param str config_str: either filename to config-file, or dict for dataset
:param int verbosity:
"""
rnn.init_better_exchook()
rnn.init_thread_join_hack()
datasetDict = None
configFilename = None
if config_str.strip().startswith("{"):
print("Using dataset %s." % config_str)
datasetDict = eval(config_str.strip())
elif config_str.endswith(".hdf"):
datasetDict = {"class": "HDFDataset", "files": [config_str]}
print("Using dataset %r." % datasetDict)
assert os.path.exists(config_str)
else:
configFilename = config_str
print("Using config file %r." % configFilename)
assert os.path.exists(configFilename)
rnn.init_config(config_filename=configFilename, default_config={"cache_size": "0"})
global config
config = rnn.config
config.set("log", None)
config.set("log_verbosity", verbosity)
if datasetDict:
config.set("train", datasetDict)
rnn.init_log()
print("Returnn dump-dataset starting up.", file=log.v2)
rnn.returnn_greeting()
rnn.init_faulthandler()
rnn.init_config_json_network()
rnn.init_data()
rnn.print_task_properties()
def init(configFilename, commandLineOptions):
rnn.init_better_exchook()
rnn.init_thread_join_hack()
rnn.init_config(configFilename, commandLineOptions)
global config
config = rnn.config
rnn.init_log()
print("CRNN demo-dataset starting up", file=log.v3)
rnn.init_faulthandler()
rnn.init_config_json_network()
rnn.init_data()
rnn.print_task_properties()
def test_rnn_initData():
hdf_fn = generate_hdf_from_dummy()
from Config import Config
config = Config({"cache_size": "0", "train": hdf_fn, "dev": hdf_fn})
import rnn
rnn.config = config
rnn.init_data()
train, dev = rnn.train_data, rnn.dev_data
assert train and dev
assert isinstance(train, HDFDataset)
assert isinstance(dev, HDFDataset)
assert train.cache_byte_size_total_limit == dev.cache_byte_size_total_limit == 0
assert train.cache_byte_size_limit_at_start == dev.cache_byte_size_limit_at_start == 0
def test_rnn_initData():
hdf_fn = generate_hdf_from_dummy()
from Config import Config
config = Config({"cache_size": "0", "train": hdf_fn, "dev": hdf_fn})
import rnn
rnn.config = config
rnn.init_data()
train, dev = rnn.train_data, rnn.dev_data
assert train and dev
assert isinstance(train, HDFDataset)
assert isinstance(dev, HDFDataset)
assert train.cache_byte_size_total_limit == dev.cache_byte_size_total_limit == 0
assert train.cache_byte_size_limit_at_start == dev.cache_byte_size_limit_at_start == 0
def init(configFilename=None):
rnn.init_better_exchook()
rnn.init_thread_join_hack()
if configFilename:
rnn.init_config(configFilename, command_line_options=[])
rnn.init_log()
else:
log.initialize()
print("Returnn collect-words starting up.", file=log.v3)
rnn.init_faulthandler()
if configFilename:
rnn.init_config_json_network()
rnn.init_data()
rnn.print_task_properties()
def init(configFilename=None):
rnn.init_better_exchook()
rnn.init_thread_join_hack()
if configFilename:
rnn.init_config(configFilename, command_line_options=[])
rnn.init_log()
else:
log.initialize()
print("Returnn collect-words starting up.", file=log.v3)
rnn.init_faulthandler()
if configFilename:
rnn.init_config_json_network()
rnn.init_data()
rnn.print_task_properties()
