def init(config_filename=None,
command_line_options=(),
config_updates=None,
extra_greeting=None):
"""
:param str|None config_filename:
:param tuple[str]|list[str]|None command_line_options: e.g. sys.argv[1:]
:param dict[str]|None config_updates: see :func:`init_config`
:param str|None extra_greeting:
"""
init_better_exchook()
init_thread_join_hack()
init_config(config_filename=config_filename,
command_line_options=command_line_options,
extra_updates=config_updates)
if config.bool("patch_atfork", False):
from Util import maybe_restart_returnn_with_atfork_patch
maybe_restart_returnn_with_atfork_patch()
init_log()
if extra_greeting:
print(extra_greeting, file=log.v1)
returnn_greeting(config_filename=config_filename,
command_line_options=command_line_options)
init_faulthandler()
init_backend_engine()
if BackendEngine.is_theano_selected():
if config.value('task', 'train') == "theano_graph":
config.set("multiprocessing", False)
if config.bool('multiprocessing', True):
init_cuda_not_in_main_proc_check()
if config.bool('ipython', False):
init_ipython_kernel()
init_config_json_network()
devices = init_theano_devices()
if need_data():
init_data()
print_task_properties(devices)
if config.value('task', 'train') == 'server':
import Server
global server
server = Server.Server(config)
else:
init_engine(devices)
def init(config_filename=None, command_line_options=(), config_updates=None, extra_greeting=None):
"""
:param str|None config_filename:
:param tuple[str]|list[str]|None command_line_options: e.g. sys.argv[1:]
:param dict[str]|None config_updates: see :func:`init_config`
:param str|None extra_greeting:
"""
init_better_exchook()
init_thread_join_hack()
init_config(config_filename=config_filename, command_line_options=command_line_options, extra_updates=config_updates)
if config.bool("patch_atfork", False):
from Util import maybe_restart_returnn_with_atfork_patch
maybe_restart_returnn_with_atfork_patch()
init_log()
if extra_greeting:
print(extra_greeting, file=log.v1)
returnn_greeting(config_filename=config_filename, command_line_options=command_line_options)
init_faulthandler()
init_backend_engine()
if BackendEngine.is_theano_selected():
if config.value('task', 'train') == "theano_graph":
config.set("multiprocessing", False)
if config.bool('multiprocessing', True):
init_cuda_not_in_main_proc_check()
if config.bool('ipython', False):
init_ipython_kernel()
init_config_json_network()
devices = init_theano_devices()
if need_data():
init_data()
print_task_properties(devices)
if config.value('task', 'train') == 'server':
import Server
global server
server = Server.Server(config)
else:
init_engine(devices)
from nose.tools import assert_equal
from nose.tools import assert_not_equal
from nose.tools import assert_raises
from nose.tools import raises
sys.path += ["."] # Python 3 hack
sys.path += [os.path.dirname(os.path.abspath(__file__)) + "/.."]
import better_exchook
from LmDataset import TranslationDataset, TranslationFactorsDataset
from Util import init_thread_join_hack
better_exchook.install()
better_exchook.replace_traceback_format_tb()
init_thread_join_hack()
dummy_source_text = ("This is some example text.\n"
"It is used to test the translation dataset.\n"
"We will write it into a temporary file.\n")
dummy_target_text = ("Das ist ein Beispieltext.\n"
"Er wird zum Testen des TranslationDatasets benutzt.\n"
"Wir werden ihn in eine temporäre Datei schreiben.\n")
def create_vocabulary(text):
"""
:param str text: any natural text
:return: mapping of words in the text to ids, as well as the inverse mapping
:rtype: (dict[str, int], dict[int, str])