def __init__(self, sprintExecPath, minPythonControlVersion=2, sprintConfigStr="", sprintControlConfig=None,
usePythonSegmentOrder=True):
"""
:param str sprintExecPath: this executable will be called for the sub proc.
:param int minPythonControlVersion: will be checked in the subprocess. via Sprint PythonControl
:param str sprintConfigStr: passed to Sprint as command line args.
can have "config:" prefix - in that case, looked up in config.
handled via eval_shell_str(), can thus have lazy content (if it is callable, will be called).
:param dict[str]|None sprintControlConfig: passed to SprintControl.init().
"""
assert os.path.exists(sprintExecPath)
self.sprintExecPath = sprintExecPath
self.minPythonControlVersion = minPythonControlVersion
if sprintConfigStr.startswith("config:"):
from returnn.config import get_global_config
config = get_global_config()
assert config
sprintConfigStr = config.typed_dict[sprintConfigStr[len("config:"):]]
self.sprintConfig = eval_shell_str(sprintConfigStr)
self.sprintControlConfig = sprintControlConfig
self.usePythonSegmentOrder = usePythonSegmentOrder
self.child_pid = None # type: typing.Optional[int]
self.parent_pid = os.getpid()
# There is no generic way to see whether Python is exiting.
# This is our workaround. We check for it in self.run_inner().
self.python_exit = False
atexit.register(self.exit_handler)
self._cur_seg_name = None
self._cur_posteriors_shape = None
self.is_calculating = False
self.init()
def create_bias(self, n, prefix='b', name="", init_eval_str=None):
"""
:param int n: output dimension
:rtype: theano.shared
"""
if not name:
name = "%s_%s" % (prefix, self.name)
if name in self.params:
name += "_%i" % len(self.params)
if not init_eval_str:
init_eval_str = self.bias_init
if self.depth > 1:
size = (self.depth, n)
else:
size = (n, )
def random_normal(scale, loc=0.0):
return self.rng.normal(loc=loc, scale=scale, size=size)
def random_uniform(l, loc=0.0):
return self.rng.uniform(low=-l + loc, high=l + loc, size=size)
import returnn.config as config_mod
import returnn.util.basic as util
try:
config = config_mod.get_global_config()
except Exception:
config = None
else:
config = util.DictAsObj(config.typed_dict)
eval_locals = {
"numpy": numpy,
"rng": self.rng,
"config": config,
"self": self,
"n": n,
"name": name,
"sqrt": numpy.sqrt,
"log": numpy.log,
"zeros": (lambda: numpy.zeros(size, dtype=theano.config.floatX)),
"random_normal": random_normal,
"random_uniform": random_uniform
}
values = eval(init_eval_str, eval_locals)
values = numpy.asarray(values, dtype=theano.config.floatX)
assert values.shape == (n, )
return self.shared(values, name)
def get_ctx(config=None):
"""
:param Config|None config:
:returns: the global context if Horovod is enabled, or None otherwise.
If we did not setup the context yet, it will automatically create it.
:rtype: HorovodContext|None
"""
global _is_set_up, _ctx
if _is_set_up:
return _ctx
if not config:
from returnn.config import get_global_config
config = get_global_config()
_is_set_up = True
if not config.is_true("use_horovod"):
return None
_ctx = HorovodContext(config=config)
return _ctx
def _create_eval_weights(self, n, m, name, default_name_prefix, init_eval_str):
"""
:param int n: input dimension
:param int m: output dimension
:param str|None name: layer name
:rtype: theano.shared
"""
if not name: name = "%s_%s_%i" % (default_name_prefix, self.name, len(self.params))
from returnn.config import get_global_config
import returnn.util.basic as util
try:
config = get_global_config()
except Exception:
config = None
else:
config = util.DictAsObj(config.typed_dict)
eval_locals = {
"numpy": numpy,
"theano": theano,
"rng": self.rng,
"config": config,
"self": self,
"n": n,
"m": m,
"name": name,
"sqrt": numpy.sqrt,
"eye": (lambda N=n, M=m: numpy.eye(N, M, dtype=theano.config.floatX)),
"random_normal": (
lambda scale=None, **kwargs: self.create_random_normal_weights(n, m, scale=scale, name=name, **kwargs)),
"random_uniform": (
lambda l=None, p=None, **kwargs: self.create_random_uniform_weights(n, m, p=p, l=l, name=name, **kwargs)),
"random_unitary": (lambda **kwargs: self.create_random_unitary_weights(n, m, name=name, **kwargs)),
"random_unitary_tiled": (lambda **kwargs: self.create_random_unitary_tiled_weights(n, m, name=name, **kwargs))
}
v = eval(init_eval_str, eval_locals)
if isinstance(v, numpy.ndarray):
v = numpy.asarray(v, dtype=theano.config.floatX)
v = self.shared(v, name)
assert isinstance(v, theano.compile.SharedVariable)
assert v.ndim == 2
vshape = v.get_value(borrow=True, return_internal_type=True).shape
assert vshape == (n, m)
return v
def perform(self, node, inputs, output_storage, params=None):
"""
:param node:
:param inputs:
:param output_storage:
:param params:
"""
start_time = time.time()
log_posteriors, seq_lengths = inputs
if numpy.isnan(log_posteriors).any():
print('SprintErrorSigOp: log_posteriors contain NaN!', file=log.v1)
if numpy.isinf(log_posteriors).any():
print('SprintErrorSigOp: log_posteriors contain Inf!', file=log.v1)
print('SprintErrorSigOp: log_posteriors:', log_posteriors, file=log.v1)
if self.sprint_instance_pool is None:
print("SprintErrorSigOp: Starting Sprint %r" % self.sprint_opts, file=log.v3)
self.sprint_instance_pool = SprintInstancePool.get_global_instance(sprint_opts=self.sprint_opts)
assert isinstance(self.sprint_instance_pool, SprintInstancePool) # PyCharm confused otherwise
loss, errsig = self.sprint_instance_pool.get_batch_loss_and_error_signal(log_posteriors, seq_lengths)
output_storage[0][0] = loss
output_storage[1][0] = errsig
print('SprintErrorSigOp: avg frame loss for segments:', loss.sum() / seq_lengths.sum(), file=log.v5)
end_time = time.time()
if self.debug_perform_time is None:
from returnn.config import get_global_config
config = get_global_config()
self.debug_perform_time = config.bool("debug_SprintErrorSigOp_perform_time", False)
if self.debug_perform_time:
print("SprintErrorSigOp perform time:", end_time - start_time, file=log.v1)
from returnn.theano.device import deviceInstance
assert deviceInstance.is_device_proc()
forward_time = start_time - deviceInstance.compute_start_time
print("SprintErrorSigOp forward time:", forward_time, file=log.v1)
def get_layer_class(name, raise_exception=True):
"""
:type name: str
:rtype: type(NetworkHiddenLayer.HiddenLayer)
"""
if name in LayerClasses:
return LayerClasses[name]
if name.startswith("config."):
from returnn.config import get_global_config
config = get_global_config()
cls = config.typed_value(name[len("config."):])
import inspect
if not inspect.isclass(cls):
if raise_exception:
raise Exception("get_layer_class: %s not found" % name)
else:
return None
if cls.layer_class is None:
# Will make Layer.save() (to HDF) work correctly.
cls.layer_class = name
return cls
if raise_exception:
raise Exception("get_layer_class: invalid layer type: %s" % name)
return None
import returnn # pip install returnn
from returnn.tf.engine import Engine
from returnn.datasets import init_dataset
from returnn.config import get_global_config
from returnn.util.basic import get_login_username
from returnn.util import better_exchook
print("TF version:", tf.__version__)
print("RETURNN imported from:", returnn.__file__)
print("RETURNN version:", returnn.__version__)
print("RETURNN long version:", returnn.__long_version__)
better_exchook.install()
config = get_global_config(auto_create=True)
config.update(
dict(
batching="random",
batch_size=5000,
max_seqs=10,
chunking="0",
network={
"fw0": {
"class": "rec",
"unit": "NativeLstm2",
"dropout": 0.1,
"n_out": 10,
"from": "data:data"
},
"output": {