def setup():
"""
Calls necessary setups.
"""
# Disable extensive TF debug verbosity. Must come before the first TF import.
import logging
logging.getLogger('tensorflow').disabled = True
# os.environ['TF_CPP_MIN_LOG_LEVEL'] = '0'
# logging.getLogger("tensorflow").setLevel(logging.INFO)
# Get us some further useful debug messages (in some cases, e.g. CUDA).
# For example: https://github.com/tensorflow/tensorflow/issues/24496
# os.environ["CUDNN_LOGINFO_DBG"] = "1"
# os.environ["CUDNN_LOGDEST_DBG"] = "stdout"
# The following might fix (workaround): Could not create cudnn handle: CUDNN_STATUS_INTERNAL_ERROR
# (https://github.com/tensorflow/tensorflow/issues/24496).
# os.environ["TF_FORCE_GPU_ALLOW_GROWTH"] = "true"
import _setup_returnn_env # noqa
import returnn.util.basic as util
util.init_thread_join_hack()
from returnn.util import better_exchook
better_exchook.install()
better_exchook.replace_traceback_format_tb()
from returnn.log import log
log.initialize(verbosity=[5])
# TF is optional.
# Note that importing TF still has a small side effect:
# BackendEngine._get_default_engine() will return TF by default, if TF is already loaded.
# For most tests, this does not matter.
try:
import tensorflow as tf
except ImportError:
tf = None
if tf:
import returnn.tf.util.basic as tf_util
tf_util.debug_register_better_repr()
import returnn.util.debug as debug
debug.install_lib_sig_segfault()
try:
import faulthandler
# Enable after libSigSegfault, so that we have both,
# because faulthandler will also call the original sig handler.
faulthandler.enable()
except ImportError:
print("no faulthandler")
def init(config_filename=None,
command_line_options=(),
config_updates=None,
extra_greeting=None):
"""
:param str|None config_filename:
:param tuple[str]|list[str]|None command_line_options: e.g. sys.argv[1:]
:param dict[str]|None config_updates: see :func:`init_config`
:param str|None extra_greeting:
"""
init_better_exchook()
init_thread_join_hack()
init_config(config_filename=config_filename,
command_line_options=command_line_options,
extra_updates=config_updates)
if config.bool("patch_atfork", False):
from returnn.util.basic import maybe_restart_returnn_with_atfork_patch
maybe_restart_returnn_with_atfork_patch()
init_log()
if extra_greeting:
print(extra_greeting, file=log.v1)
returnn_greeting(config_filename=config_filename,
command_line_options=command_line_options)
init_faulthandler()
init_backend_engine()
if BackendEngine.is_theano_selected():
if config.value('task', 'train') == "theano_graph":
config.set("multiprocessing", False)
if config.bool('multiprocessing', True):
init_cuda_not_in_main_proc_check()
if config.bool('ipython', False):
init_ipython_kernel()
init_config_json_network()
devices = init_theano_devices()
if need_data():
init_data()
print_task_properties(devices)
if config.value('task', 'train') == 'server':
from returnn.theano.server import Server
global server
server = Server(config)
else:
init_engine(devices)
import gzip
import pickle
from nose.tools import assert_equal
from nose.tools import assert_not_equal
from nose.tools import assert_raises
from nose.tools import raises
import _setup_test_env # noqa
from returnn.util import better_exchook
from returnn.datasets.lm import TranslationDataset, TranslationFactorsDataset
from returnn.util.basic import init_thread_join_hack
better_exchook.install()
better_exchook.replace_traceback_format_tb()
init_thread_join_hack()
dummy_source_text = ("This is some example text.\n"
"It is used to test the translation dataset.\n"
"We will write it into a temporary file.\n")
dummy_target_text = ("Das ist ein Beispieltext.\n"
"Er wird zum Testen des TranslationDatasets benutzt.\n"
"Wir werden ihn in eine temporäre Datei schreiben.\n")
def create_vocabulary(text):
"""
:param str text: any natural text
:return: mapping of words in the text to ids, as well as the inverse mapping
:rtype: (dict[str, int], dict[int, str])