def __init__(self, sprintExecPath, minPythonControlVersion=2, sprintConfigStr="", sprintControlConfig=None, usePythonSegmentOrder=True):
"""
:param str sprintExecPath: this executable will be called for the sub proc.
:param int minPythonControlVersion: will be checked in the subprocess. via Sprint PythonControl
:param str sprintConfigStr: passed to Sprint as command line args.
can have "config:" prefix - in that case, looked up in config.
handled via eval_shell_str(), can thus have lazy content (if it is callable, will be called).
:param dict[str]|None sprintControlConfig: passed to SprintControl.init().
"""
assert os.path.exists(sprintExecPath)
self.sprintExecPath = sprintExecPath
self.minPythonControlVersion = minPythonControlVersion
if sprintConfigStr.startswith("config:"):
from Config import get_global_config
config = get_global_config()
assert config
sprintConfigStr = config.typed_dict[sprintConfigStr[len("config:"):]]
self.sprintConfig = eval_shell_str(sprintConfigStr)
self.sprintControlConfig = sprintControlConfig
self.usePythonSegmentOrder = usePythonSegmentOrder
self.child_pid = None
self.parent_pid = os.getpid()
# There is no generic way to see whether Python is exiting.
# This is our workaround. We check for it in self.run_inner().
self.python_exit = False
atexit.register(self.exit_handler)
self._cur_seg_name = None
self._cur_posteriors_shape = None
self.is_calculating = False
self.init()
def _build_sprint_args(self):
config_str = "action:ExternSprintDataset,c2p_fd:%i,p2c_fd:%i" % (
self.pipe_c2p[1].fileno(), self.pipe_p2c[0].fileno())
if TaskSystem.SharedMemNumpyConfig["enabled"]:
config_str += ",EnableAutoNumpySharedMemPickling:True"
epoch = self.crnnEpoch or 1
args = [
self.sprintTrainerExecPath,
"--*.seed=%i" % (epoch // self.partitionEpoch)]
if self.partitionEpoch > 1:
args += [
"--*.corpus.partition=%i" % self.partitionEpoch,
"--*.corpus.select-partition=%i" % (epoch % self.partitionEpoch)]
args += [
"--*.python-segment-order=true",
"--*.python-segment-order-pymod-path=%s" % self._my_python_mod_path,
"--*.python-segment-order-pymod-name=SprintExternInterface",
"--*.use-data-source=false",
"--*.trainer=python-trainer",
"--*.pymod-path=%s" % self._my_python_mod_path,
"--*.pymod-name=SprintExternInterface",
"--*.pymod-config=%s" % config_str]
if self.predefined_seq_list_order:
import tempfile
self.seq_list_file = tempfile.mktemp(prefix="crnn-sprint-predefined-seq-list")
with open(self.seq_list_file, "w") as f:
for tag in self.predefined_seq_list_order:
f.write(tag)
f.write("\n")
f.close()
args += ["--*.corpus.segments.file=%s" % self.seq_list_file]
args += eval_shell_str(self.sprintConfig)
return args
def _build_sprint_args(self):
"""
:rtype: list[str]
"""
config_str = "action:ExternSprintDataset,c2p_fd:%i,p2c_fd:%i" % (
self.pipe_c2p[1].fileno(), self.pipe_p2c[0].fileno())
if TaskSystem.SharedMemNumpyConfig["enabled"]:
config_str += ",EnableAutoNumpySharedMemPickling:True"
epoch = self.crnnEpoch or 1
assert epoch >= 1
if isinstance(self.sprint_trainer_exec_path, (list, tuple)):
args = list(self.sprint_trainer_exec_path)
else:
args = [self.sprint_trainer_exec_path]
# First the user options. Usually also involves loading some config.
args += eval_shell_str(self.sprint_config)
# Now our options. They might overwrite some of the config settings. (That is why we do it after the user opts.)
args += [
"--*.seed=%i" % (self._get_random_seed_for_epoch(epoch=epoch) - 1)]
if self.partition_epoch > 1:
args += [
"--*.corpus.partition=%i" % self.partition_epoch,
"--*.corpus.select-partition=%i" % ((epoch - 1) % self.partition_epoch)]
args += [
"--*.python-segment-order=true",
"--*.python-segment-order-pymod-path=%s" % self._my_python_mod_path,
"--*.python-segment-order-pymod-name=SprintExternInterface",
"--*.use-data-source=false",
"--*.trainer=python-trainer",
"--*.pymod-path=%s" % self._my_python_mod_path,
"--*.pymod-name=SprintExternInterface",
"--*.pymod-config=%s" % config_str]
if self.predefined_seq_list_order:
import tempfile
self.seq_list_file = tempfile.mktemp(prefix="crnn-sprint-predefined-seq-list")
with open(self.seq_list_file, "w") as f:
for tag in self.predefined_seq_list_order:
f.write(tag)
f.write("\n")
f.close()
args += [
"--*.corpus.segment-order-shuffle=false",
"--*.corpus.segments.file=%s" % self.seq_list_file,
"--*.corpus.segment-order=%s" % self.seq_list_file]
if self.seq_tags_filter is not None:
assert not self.predefined_seq_list_order
import tempfile
self.seq_list_file = tempfile.mktemp(prefix="crnn-sprint-predefined-seq-filter")
with open(self.seq_list_file, "w") as f:
for tag in self.seq_tags_filter:
f.write(tag)
f.write("\n")
f.close()
args += ["--*.corpus.segments.file=%s" % self.seq_list_file]
return args
def _build_sprint_args(self):
"""
:rtype: list[str]
"""
config_str = "action:ExternSprintDataset,c2p_fd:%i,p2c_fd:%i" % (
self.pipe_c2p[1].fileno(), self.pipe_p2c[0].fileno())
if TaskSystem.SharedMemNumpyConfig["enabled"]:
config_str += ",EnableAutoNumpySharedMemPickling:True"
epoch = self.crnnEpoch or 1
assert epoch >= 1
if isinstance(self.sprint_trainer_exec_path, (list, tuple)):
args = list(self.sprint_trainer_exec_path)
else:
args = [self.sprint_trainer_exec_path]
# First the user options. Usually also involves loading some config.
args += eval_shell_str(self.sprint_config)
# Now our options. They might overwrite some of the config settings. (That is why we do it after the user opts.)
args += [
"--*.seed=%i" % ((epoch - 1) // self.partition_epoch)]
if self.partition_epoch > 1:
args += [
"--*.corpus.partition=%i" % self.partition_epoch,
"--*.corpus.select-partition=%i" % ((epoch - 1) % self.partition_epoch)]
args += [
"--*.python-segment-order=true",
"--*.python-segment-order-pymod-path=%s" % self._my_python_mod_path,
"--*.python-segment-order-pymod-name=SprintExternInterface",
"--*.use-data-source=false",
"--*.trainer=python-trainer",
"--*.pymod-path=%s" % self._my_python_mod_path,
"--*.pymod-name=SprintExternInterface",
"--*.pymod-config=%s" % config_str]
if self.predefined_seq_list_order:
import tempfile
self.seq_list_file = tempfile.mktemp(prefix="crnn-sprint-predefined-seq-list")
with open(self.seq_list_file, "w") as f:
for tag in self.predefined_seq_list_order:
f.write(tag)
f.write("\n")
f.close()
args += [
"--*.corpus.segment-order-shuffle=false",
"--*.corpus.segments.file=%s" % self.seq_list_file,
"--*.corpus.segment-order=%s" % self.seq_list_file]
return args
def __init__(self,
sprintExecPath,
minPythonControlVersion=2,
sprintConfigStr="",
sprintControlConfig=None,
usePythonSegmentOrder=True):
"""
:param str sprintExecPath: this executable will be called for the sub proc.
:param int minPythonControlVersion: will be checked in the subprocess. via Sprint PythonControl
:param str sprintConfigStr: passed to Sprint as command line args.
can have "config:" prefix - in that case, looked up in config.
handled via eval_shell_str(), can thus have lazy content (if it is callable, will be called).
:param dict[str]|None sprintControlConfig: passed to SprintControl.init().
"""
assert os.path.exists(sprintExecPath)
self.sprintExecPath = sprintExecPath
self.minPythonControlVersion = minPythonControlVersion
if sprintConfigStr.startswith("config:"):
from Config import get_global_config
config = get_global_config()
assert config
sprintConfigStr = config.typed_dict[
sprintConfigStr[len("config:"):]]
self.sprintConfig = eval_shell_str(sprintConfigStr)
self.sprintControlConfig = sprintControlConfig
self.usePythonSegmentOrder = usePythonSegmentOrder
self.child_pid = None
self.parent_pid = os.getpid()
# There is no generic way to see whether Python is exiting.
# This is our workaround. We check for it in self.run_inner().
self.python_exit = False
atexit.register(self.exit_handler)
self._cur_seg_name = None
self._cur_posteriors_shape = None
self.is_calculating = False
self.init()