def run(self):
better_exchook.install()
while True:
func = workerQueue.get()
Logging.log("Next work item: %s" % func,
"remaining: %i" % workerQueue.qsize())
with self.lock:
self.cur_item = func
try:
func()
except SystemExit:
return
except Exception:
Logging.log_exception("Worker", *sys.exc_info())
finally:
# Note, this is buggy:
# In case that func() adds itself back to the work-queue,
# we would remove it here and then sometime later when we
# execute it again, it's not in the set anymore.
# Note that some other code also does not expect this.
# TODO fix this
if func in currentWork:
currentWork.remove(func)
# noinspection PyUnresolvedReferences
currentWork.save()
def demo():
"""
Will print out the different network topologies of the specified pretraining scheme.
"""
import better_exchook
better_exchook.install()
import rnn
import sys
if len(sys.argv) <= 1:
print("usage: python %s [config] [other options]" % __file__)
print(
"example usage: python %s ++pretrain default ++pretrain_construction_algo from_input"
% __file__)
rnn.init_config(command_line_options=sys.argv[1:])
# noinspection PyProtectedMember
rnn.config._hack_value_reading_debug()
rnn.config.update({"log": []})
rnn.init_log()
rnn.init_backend_engine()
if not rnn.config.value("pretrain", ""):
print(
"config option 'pretrain' not set, will set it for this demo to 'default'"
)
rnn.config.set("pretrain", "default")
pretrain = pretrain_from_config(rnn.config)
print("pretrain: %s" % pretrain)
num_pretrain_epochs = pretrain.get_train_num_epochs()
from pprint import pprint
for epoch in range(1, 1 + num_pretrain_epochs):
print("epoch %i (of %i) network json:" % (epoch, num_pretrain_epochs))
net_json = pretrain.get_network_json_for_epoch(epoch)
pprint(net_json)
print("done.")
def main(mod):
"""
:param dict[str] mod:
"""
import unittest
better_exchook.install()
if len(sys.argv) <= 1:
for k, v in sorted(mod.items()):
if k.startswith("test_"):
print("-" * 40)
print("Executing: %s" % k)
try:
v()
except unittest.SkipTest as exc:
print("SkipTest:", exc)
print("-" * 40)
print("Finished all tests.")
else:
assert len(sys.argv) >= 2
for arg in sys.argv[1:]:
print("Executing: %s" % arg)
if arg in mod:
mod[arg]() # assume function and execute
else:
eval(arg) # assume Python code and execute
def demo():
import better_exchook
better_exchook.install()
import sys
dsclazzeval = sys.argv[1]
dataset = eval(dsclazzeval)
assert isinstance(dataset, GeneratingDataset)
assert dataset._input_classes and dataset._output_classes
assert dataset.num_outputs["data"][1] == 2 # expect 1-hot
assert dataset.num_outputs["classes"][1] == 1 # expect sparse
for i in range(10):
print("Seq idx %i:" % i)
s = dataset.generate_seq(i)
assert isinstance(s, DatasetSeq)
features = s.features
output_seq = s.targets["classes"]
assert features.ndim == 2
assert output_seq.ndim == 1
input_seq = numpy.argmax(features, axis=1)
input_seq_str = "".join([dataset._input_classes[i] for i in input_seq])
output_seq_str = "".join(
[dataset._output_classes[i] for i in output_seq])
print(" %r" % input_seq_str)
print(" %r" % output_seq_str)
assert features.shape[1] == dataset.num_outputs["data"][0]
assert features.shape[0] == output_seq.shape[0]
def test():
import better_exchook
better_exchook.install()
state = parse("/Library/Frameworks/SDL.framework/Headers/SDL.h")
return state
def demo():
import better_exchook
better_exchook.install()
import rnn
import sys
if len(sys.argv) <= 1:
print("usage: python %s [config] [other options]" % __file__)
print("example usage: python %s ++pretrain default ++pretrain_construction_algo from_input" % __file__)
rnn.initConfig(commandLineOptions=sys.argv[1:])
rnn.config._hack_value_reading_debug()
rnn.config.update({"log": []})
rnn.initLog()
rnn.initBackendEngine()
if not rnn.config.value("pretrain", ""):
print("config option 'pretrain' not set, will set it for this demo to 'default'")
rnn.config.set("pretrain", "default")
pretrain = pretrainFromConfig(rnn.config)
print("pretrain: %s" % pretrain)
num_pretrain_epochs = pretrain.get_train_num_epochs()
from pprint import pprint
for epoch in range(1, 1 + num_pretrain_epochs):
print("epoch %i (of %i) network json:" % (epoch, num_pretrain_epochs))
net_json = pretrain.get_network_json_for_epoch(epoch)
pprint(net_json)
print("done.")
def _demo():
import better_exchook
better_exchook.install()
from argparse import ArgumentParser
arg_parser = ArgumentParser()
arg_parser.add_argument("--ogg")
arg_parser.add_argument("--opts")
arg_parser.add_argument("--kind")
arg_parser.add_argument("--dim", type=int)
arg_parser.add_argument("--end_frame",
type=int,
default=None,
help="e.g. 200, better for plotting")
args = arg_parser.parse_args()
lib_fn = get_auto_compiled_lib_filename(verbose=True)
print("Lib filename:", lib_fn)
lib = ParseOggVorbisLib()
if args.ogg:
raw_bytes = open(args.ogg, "rb").read()
if args.opts:
opts = eval(args.opts)
else:
opts = {}
features = lib.get_features_from_raw_bytes(raw_bytes=raw_bytes,
kind=args.kind,
output_dim=args.dim,
**opts)
_plot(features, end_frame=args.end_frame)
def demo():
"""
Some demo.
"""
# some examples
# this code produces this output: https://gist.github.com/922622
try:
x = {1: 2, "a": "b"}
# noinspection PyMissingOrEmptyDocstring
def f():
y = "foo"
# noinspection PyUnresolvedReferences,PyStatementEffect
x, 42, sys.stdin.__class__, sys.exc_info, y, z
f()
except Exception:
better_exchook(*sys.exc_info())
try:
# noinspection PyArgumentList
(lambda _x: None)(__name__, 42) # multiline
except Exception:
better_exchook(*sys.exc_info())
try:
class Obj:
def __repr__(self):
return ("<Obj multi-\n" + " line repr>")
obj = Obj()
assert not obj
except Exception:
better_exchook(*sys.exc_info())
# noinspection PyMissingOrEmptyDocstring
def f1(a):
f2(a + 1, 2)
# noinspection PyMissingOrEmptyDocstring
def f2(a, b):
f3(a + b)
# noinspection PyMissingOrEmptyDocstring
def f3(a):
b = ("abc" * 100) + "-interesting" # some long demo str
a(b) # error, not callable
try:
f1(13)
except Exception:
better_exchook(*sys.exc_info())
# use this to overwrite the global exception handler
install()
# and fail
# noinspection PyUnresolvedReferences
finalfail(sys)
def thread_main(self):
try:
import better_exchook
better_exchook.install()
while self.batches.has_more() and not self.coord.should_stop():
enqueue_args = self.get_next_batch()
if self.queue:
self.queue.put(enqueue_args)
else:
self.tf_session.run(self.tf_queue.enqueue(enqueue_args)) # TODO cache op
with self.state_change_cond:
self.state_change_cond.notifyAll()
self.batches.advance(1)
self.reached_end = not self.batches.has_more()
except Exception as exc:
print("Exception in DataProvider thread: %r" % exc)
sys.excepthook(*sys.exc_info())
finally:
with self.state_change_cond:
self.thread_finished = True
self.state_change_cond.notifyAll()
def parse_file():
import better_exchook
better_exchook.install()
state = State()
state.autoSetupSystemMacros()
filename = "/Library/Frameworks/SDL.framework/Headers/SDL.h"
preprocessed = state.preprocess_file(filename, local=True)
tokens = cpre2_parse(state, preprocessed)
token_list = []
def copy_hook(input, output):
for x in input:
output.append(x)
yield x
tokens = copy_hook(tokens, token_list)
cpre3_parse(state, tokens)
print("tokens:")
pprint(token_list)
print("parse errors:")
pprint(state._errors)
assert not state._errors
return state, token_list
def run(self):
better_exchook.install()
while True:
with lock:
func = workerQueue.get()
if func in workerQueueSet:
workerQueueSet.remove(func)
Logging.log("Next work item: %s" % func,
"remaining: %i" % workerQueue.qsize())
with self.lock:
self.cur_item = func
try:
func()
except SystemExit:
return
except Exception:
Logging.log_exception("Worker", *sys.exc_info())
finally:
# Note: func() can add itself back to the work-queue.
with lock:
if func not in workerQueueSet:
if func in currentWorkSet:
currentWorkSet.remove(func)
# noinspection PyUnresolvedReferences
currentWorkSet.save()
def main():
import better_exchook
better_exchook.install()
from argparse import ArgumentParser
from pprint import pprint
import time
arg_parser = ArgumentParser()
arg_parser.add_argument("--path", required=True, help="path of db")
arg_parser.add_argument("--rev")
args = arg_parser.parse_args()
cur_db = Db(path=args.path)
old_db = HistoricDb(path=args.path, git_revision=args.rev)
print("Old DB commit:", old_db.git_commit.hexsha[:8], ",",
time.asctime(time.localtime(old_db.git_commit.authored_date)), ",",
old_db.git_commit.message.strip())
cur_drinkers = set(cur_db.get_drinker_names())
old_drinkers = set(old_db.get_drinker_names())
print("New drinkers:", cur_drinkers.difference(old_drinkers))
print("Removed drinkers:", old_drinkers.difference(cur_drinkers))
cur_total_buy_item_counts = cur_db.get_total_buy_item_counts()
old_total_buy_item_counts = old_db.get_total_buy_item_counts()
print("Current DB total buy items counts:")
pprint(cur_total_buy_item_counts)
print("Old DB total buy items counts:")
pprint(old_total_buy_item_counts)
keys = set(cur_total_buy_item_counts.keys()).union(
set(old_total_buy_item_counts.keys()))
diff_total_buy_item_counts = {
key: cur_total_buy_item_counts.get(key, 0) -
old_total_buy_item_counts.get(key, 0)
for key in keys
}
print("Diff total buy items counts:")
pprint(diff_total_buy_item_counts)
def _main(argv):
import better_exchook
better_exchook.install()
log.initialize(verbosity=[5])
print("LmDataset demo startup")
kwargs = eval(argv[0])
print("Creating LmDataset with kwargs=%r ..." % kwargs)
dataset = LmDataset(**kwargs)
print("init_seq_order ...")
dataset.init_seq_order(epoch=1)
seq_idx = 0
last_log_time = time.time()
print("start iterating through seqs ...")
while dataset.is_less_than_num_seqs(seq_idx):
if seq_idx == 0:
print("load_seqs with seq_idx=%i ...." % seq_idx)
dataset.load_seqs(seq_idx, seq_idx + 1)
if time.time() - last_log_time > 2.0:
last_log_time = time.time()
print(
"Loading %s progress, %i/%i (%.0f%%) seqs loaded (%.0f%% skipped), (%.0f%% unknown) total syms %i ..."
% (dataset.__class__.__name__, dataset.next_orth_idx,
dataset.estimated_num_seqs, 100.0 * dataset.next_orth_idx /
dataset.estimated_num_seqs, 100.0 * dataset.num_skipped /
(dataset.next_orth_idx or 1), 100.0 * dataset.num_unknown /
dataset._num_timesteps_accumulated["data"],
dataset._num_timesteps_accumulated["data"]))
seq_idx += 1
print("finished iterating, num seqs: %i" % seq_idx)
print("dataset len:", dataset.len_info())
def demo():
import better_exchook
better_exchook.install()
import sys
dsclazzeval = sys.argv[1]
dataset = eval(dsclazzeval)
assert isinstance(dataset, GeneratingDataset)
assert dataset._input_classes and dataset._output_classes
assert dataset.num_outputs["data"][1] == 2 # expect 1-hot
assert dataset.num_outputs["classes"][1] == 1 # expect sparse
for i in range(10):
print "Seq idx", i, ":"
s = dataset.generate_seq(i)
assert isinstance(s, DatasetSeq)
features = s.features
output_seq = s.targets["classes"]
assert features.ndim == 2
assert output_seq.ndim == 1
input_seq = numpy.argmax(features, axis=1)
input_seq_str = "".join([dataset._input_classes[i] for i in input_seq])
output_seq_str = "".join([dataset._output_classes[i] for i in output_seq])
print " ", repr(input_seq_str)
print " ", repr(output_seq_str)
assert features.shape[1] == dataset.num_outputs["data"][0]
assert features.shape[0] == output_seq.shape[0]
def checkExec():
if "--forkExecProc" in sys.argv:
try:
import better_exchook
except ImportError:
pass # Doesn't matter.
else:
better_exchook.install()
argidx = sys.argv.index("--forkExecProc")
writeFileNo = int(sys.argv[argidx + 1])
readFileNo = int(sys.argv[argidx + 2])
readend = os.fdopen(readFileNo, "r")
writeend = os.fdopen(writeFileNo, "w")
unpickler = Unpickler(readend)
name = unpickler.load()
if ExecingProcess.Verbose: print "ExecingProcess child %s (pid %i)" % (name, os.getpid())
try:
target = unpickler.load()
args = unpickler.load()
except EOFError:
print "Error: unpickle incomplete"
raise SystemExit
ret = target(*args)
# IOError is probably broken pipe. That probably means that the parent died.
try: Pickler(writeend).dump(ret)
except IOError: pass
try: readend.close()
except IOError: pass
try: writeend.close()
except IOError: pass
if ExecingProcess.Verbose: print "ExecingProcess child %s (pid %i) finished" % (name, os.getpid())
raise SystemExit
def threadMain(self):
better_exchook.install()
thread = currentThread()
setCurThreadName("PyMod %s" % self.name)
while True:
if self.module:
try:
reload(self.module)
except Exception:
print "couldn't reload module", self.module
sys.excepthook(*sys.exc_info())
# continue anyway, maybe it still works and maybe the mainFunc does sth good/important
else:
self.module = __import__(self.moduleName)
mainFunc = getattr(self.module, self.mainFuncName)
try:
mainFunc()
except KeyboardInterrupt:
break
except Exception:
print "Exception in module", self.name
sys.excepthook(*sys.exc_info())
if not thread.reload: break
sys.stdout.write("reloading module %s\n" % self.name)
thread.cancel = False
thread.reload = False
thread.waitQueue = None
def parse_file(): import better_exchook better_exchook.install() state = State() state.autoSetupSystemMacros() filename = "/Library/Frameworks/SDL.framework/Headers/SDL.h" preprocessed = state.preprocess_file(filename, local=True) tokens = cpre2_parse(state, preprocessed) token_list = [] def copy_hook(input, output): for x in input: output.append(x) yield x tokens = copy_hook(tokens, token_list) cpre3_parse(state, tokens) print "tokens:" pprint(token_list) print "parse errors:" pprint(state._errors) assert not state._errors return state, token_list
def demo():
"""
Will print out the different network topologies of the specified pretraining scheme.
"""
import better_exchook
better_exchook.install()
import rnn
import sys
if len(sys.argv) <= 1:
print("usage: python %s [config] [other options]" % __file__)
print("example usage: python %s ++pretrain default ++pretrain_construction_algo from_input" % __file__)
rnn.init_config(command_line_options=sys.argv[1:])
# noinspection PyProtectedMember
rnn.config._hack_value_reading_debug()
rnn.config.update({"log": []})
rnn.init_log()
rnn.init_backend_engine()
if not rnn.config.value("pretrain", ""):
print("config option 'pretrain' not set, will set it for this demo to 'default'")
rnn.config.set("pretrain", "default")
pretrain = pretrain_from_config(rnn.config)
print("pretrain: %s" % pretrain)
num_pretrain_epochs = pretrain.get_train_num_epochs()
from pprint import pprint
for epoch in range(1, 1 + num_pretrain_epochs):
print("epoch %i (of %i) network json:" % (epoch, num_pretrain_epochs))
net_json = pretrain.get_network_json_for_epoch(epoch)
pprint(net_json)
print("done.")
def _thread_main(self):
try:
import better_exchook
better_exchook.install()
while self.batches.has_more() and not self.coord.should_stop():
enqueue_args = self.get_next_batch(consider_batch_slice=True)
if enqueue_args is not None:
if self.queue:
self.queue.put(enqueue_args)
else:
self.tf_queue.enqueue(tf_session=self.tf_session,
data=enqueue_args)
with self.state_change_cond:
self.state_change_cond.notifyAll()
self.batches.advance(1)
self.reached_end = not self.batches.has_more()
except Exception as exc:
print("Exception in DataProvider thread: %r" % exc, file=log.v1)
sys.excepthook(*sys.exc_info())
finally:
with self.state_change_cond:
self.thread_finished = True
self.state_change_cond.notifyAll()
def _start_child(self, epoch):
assert self.child_pid is None
assert self.reader_thread is None
self.pipe_c2p = self._pipe_open()
self.pipe_p2c = self._pipe_open()
args = self._build_sprint_args()
print("%s: epoch" % self, epoch, "exec", args, file=log.v5)
pid = os.fork()
if pid == 0: # child
# In case we are in some test environment or so, recover the original stdout/stderr.
sys.stdin = sys.__stdin__
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
import better_exchook
better_exchook.install()
try:
sys.stdin.close() # Force no tty stdin.
self.pipe_c2p[0].close()
self.pipe_p2c[1].close()
os.execv(args[0], args) # Does not return if successful.
print("%s child exec failed." % self)
except BaseException:
print("%s child: Error when starting Sprint %r." %
(self, args))
sys.excepthook(*sys.exc_info())
finally:
print("%s child: exit" % self)
os._exit(1)
return # Not reached.
# parent
self.pipe_c2p[1].close()
self.pipe_p2c[0].close()
self.child_pid = pid
try:
initSignal, (inputDim, outputDim,
num_segments) = self._read_next_raw()
assert initSignal == b"init"
assert isinstance(inputDim, int) and isinstance(outputDim, int)
# Ignore num_segments. It can be totally different than the real number of sequences.
self.setDimensions(inputDim, outputDim)
except Exception:
print("%s: Sprint child process (%r) caused an exception." %
(self, args),
file=log.v1)
sys.excepthook(*sys.exc_info())
self._exit_child(wait_thread=False)
raise Exception("%s Sprint init failed" % self)
self.reader_thread = Thread(target=self.reader_thread_proc,
args=(
pid,
epoch,
),
name="%s reader thread" % self)
self.reader_thread.daemon = True
self.reader_thread.start()
def setup():
"""
Calls necessary setups.
"""
better_exchook.install()
better_exchook.replace_traceback_format_tb()
import _setup_sleepy_env # noqa
def init(**kwargs):
import better_exchook
better_exchook.install()
# This module can also be used for Sprint PythonControl, which will also call init().
# We need to catch these cases.
if "name" in kwargs and kwargs["name"] == "Sprint.PythonControl":
return PythonControl.init(**kwargs)
return init_PythonTrainer(**kwargs)
def init(**kwargs):
import better_exchook
better_exchook.install()
# This module can also be used for Sprint PythonControl, which will also call init().
# We need to catch these cases.
if "name" in kwargs and kwargs["name"] == "Sprint.PythonControl":
return PythonControl.init(**kwargs)
return init_PythonTrainer(**kwargs)
def _start_child(self, epoch):
"""
:param epoch:
:return:
"""
assert self.child_pid is None
assert self.reader_thread is None
self.pipe_c2p = self._pipe_open()
self.pipe_p2c = self._pipe_open()
args = self._build_sprint_args()
print("%s: epoch" % self, epoch, "exec", args, file=log.v5)
pid = os.fork()
if pid == 0: # child
# In case we are in some test environment or so, recover the original stdout/stderr.
sys.stdin = sys.__stdin__
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
import better_exchook
better_exchook.install()
# noinspection PyBroadException
try:
sys.stdin.close() # Force no tty stdin.
self.pipe_c2p[0].close()
self.pipe_p2c[1].close()
os.execv(args[0], args) # Does not return if successful.
print("%s child exec failed." % self)
except BaseException:
print("%s child: Error when starting Sprint %r." % (self, args))
sys.excepthook(*sys.exc_info())
finally:
print("%s child: exit" % self)
# noinspection PyProtectedMember
os._exit(1)
return # Not reached.
# parent
self.pipe_c2p[1].close()
self.pipe_p2c[0].close()
self.child_pid = pid
try:
init_signal, (input_dim, output_dim, num_segments) = self._read_next_raw()
assert init_signal == b"init"
assert isinstance(input_dim, int) and isinstance(output_dim, int)
# Ignore num_segments. It can be totally different than the real number of sequences.
self.set_dimensions(input_dim, output_dim)
except Exception:
print("%s: Sprint child process (%r) caused an exception." % (self, args), file=log.v1)
sys.excepthook(*sys.exc_info())
self._exit_child(wait_thread=False)
raise Exception("%s Sprint init failed" % self)
self.reader_thread = Thread(target=self._reader_thread_proc, args=(pid, epoch,),
name="%s reader thread" % self)
self.reader_thread.daemon = True
self.reader_thread.start()
def demo():
import better_exchook
better_exchook.install()
import rnn
import sys
if len(sys.argv) <= 1:
print("usage: python %s [config] [other options]" % __file__)
print("example usage: python %s ++learning_rate_control newbob ++learning_rate_file newbob.data ++learning_rate 0.001" % __file__)
rnn.initConfig(commandLineOptions=sys.argv[1:])
rnn.config._hack_value_reading_debug()
from Pretrain import pretrainFromConfig
pretrain = pretrainFromConfig(rnn.config)
first_non_pretrain_epoch = 1
pretrain_learning_rate = None
if pretrain:
first_non_pretrain_epoch = pretrain.get_train_num_epochs() + 1
log.initialize(verbosity=[5])
control = loadLearningRateControlFromConfig(rnn.config)
print("LearningRateControl: %r" % control)
if not control.epochData:
print("No epoch data so far.")
return
firstEpoch = min(control.epochData.keys())
if firstEpoch != 1:
print("Strange, first epoch from epoch data is %i." % firstEpoch)
print("Error key: %s from %r" % (control.getErrorKey(epoch=firstEpoch), control.epochData[firstEpoch].error))
if pretrain:
pretrain_learning_rate = rnn.config.float('pretrain_learning_rate', control.defaultLearningRate)
maxEpoch = max(control.epochData.keys())
for epoch in range(1, maxEpoch + 2): # all epochs [1..maxEpoch+1]
oldLearningRate = None
if epoch in control.epochData:
oldLearningRate = control.epochData[epoch].learningRate
if epoch < first_non_pretrain_epoch:
learningRate = pretrain_learning_rate
s = "Pretrain epoch %i, fixed learning rate: %s (was: %s)" % (epoch, learningRate, oldLearningRate)
elif first_non_pretrain_epoch > 1 and epoch == first_non_pretrain_epoch:
learningRate = control.defaultLearningRate
s = "First epoch after pretrain, epoch %i, fixed learning rate: %s (was %s)" % (epoch, learningRate, oldLearningRate)
else:
learningRate = control.calcNewLearnignRateForEpoch(epoch)
s = "Calculated learning rate for epoch %i: %s (was: %s)" % (epoch, learningRate, oldLearningRate)
if learningRate < control.minLearningRate:
learningRate = control.minLearningRate
s += ", clipped to %s" % learningRate
s += ", previous relative error: %s" % control.calcRelativeError(epoch - 2, epoch - 1)
if hasattr(control, "_calcRecentMeanRelativeError"):
s += ", previous mean relative error: %s" % control._calcRecentMeanRelativeError(epoch)
print(s)
# Overwrite new learning rate so that the calculation for further learning rates stays consistent.
if epoch in control.epochData:
control.epochData[epoch].learningRate = learningRate
else:
control.epochData[epoch] = control.EpochData(learningRate=learningRate)
print("Finished, last stored epoch was %i." % maxEpoch)
def main():
global LstmCellTypes
print("Benchmarking LSTMs.")
better_exchook.install()
print("Args:", " ".join(sys.argv))
arg_parser = ArgumentParser()
arg_parser.add_argument("cfg", nargs="*", help="opt=value, opt in %r" % sorted(base_settings.keys()))
arg_parser.add_argument("--no-cpu", action="store_true")
arg_parser.add_argument("--no-gpu", action="store_true")
arg_parser.add_argument("--selected", help="comma-separated list from %r" % LstmCellTypes)
arg_parser.add_argument("--no-setup-tf-thread-pools", action="store_true")
args = arg_parser.parse_args()
for opt in args.cfg:
key, value = opt.split("=", 1)
assert key in base_settings
value_type = type(base_settings[key])
base_settings[key] = value_type(value)
print("Settings:")
pprint(base_settings)
log.initialize(verbosity=[4])
print("Returnn:", describe_crnn_version(), file=log.v3)
print("TensorFlow:", describe_tensorflow_version(), file=log.v3)
print("Python:", sys.version.replace("\n", ""), sys.platform)
if not args.no_setup_tf_thread_pools:
setup_tf_thread_pools(log_file=log.v2)
else:
print("Not setting up the TF thread pools. Will be done automatically by TF to number of CPU cores.")
if args.no_gpu:
print("GPU will not be used.")
else:
print("GPU available: %r" % is_gpu_available())
print_available_devices()
if args.selected:
LstmCellTypes = args.selected.split(",")
benchmarks = {}
if not args.no_gpu and is_gpu_available():
for lstm_unit in LstmCellTypes:
benchmarks["GPU:" + lstm_unit] = benchmark(lstm_unit=lstm_unit, use_gpu=True)
if not args.no_cpu:
for lstm_unit in LstmCellTypes:
if lstm_unit in GpuOnlyCellTypes:
continue
benchmarks["CPU:" + lstm_unit] = benchmark(lstm_unit=lstm_unit, use_gpu=False)
print("-" * 20)
print("Settings:")
pprint(base_settings)
print("Final results:")
for t, lstm_unit in sorted([(t, lstm_unit) for (lstm_unit, t) in sorted(benchmarks.items())]):
print(" %s: %s" % (lstm_unit, hms_fraction(t)))
print("Done.")
def main():
global LstmCellTypes
print("Benchmarking LSTMs.")
better_exchook.install()
print("Args:", " ".join(sys.argv))
arg_parser = ArgumentParser()
arg_parser.add_argument("cfg", nargs="*", help="opt=value, opt in %r" % sorted(base_settings.keys()))
arg_parser.add_argument("--no-cpu", action="store_true")
arg_parser.add_argument("--no-gpu", action="store_true")
arg_parser.add_argument("--selected", help="comma-separated list from %r" % LstmCellTypes)
arg_parser.add_argument("--no-setup-tf-thread-pools", action="store_true")
args = arg_parser.parse_args()
for opt in args.cfg:
key, value = opt.split("=", 1)
assert key in base_settings
value_type = type(base_settings[key])
base_settings[key] = value_type(value)
print("Settings:")
pprint(base_settings)
log.initialize(verbosity=[4])
print("Returnn:", describe_returnn_version(), file=log.v3)
print("TensorFlow:", describe_tensorflow_version(), file=log.v3)
print("Python:", sys.version.replace("\n", ""), sys.platform)
if not args.no_setup_tf_thread_pools:
setup_tf_thread_pools(log_file=log.v2)
else:
print("Not setting up the TF thread pools. Will be done automatically by TF to number of CPU cores.")
if args.no_gpu:
print("GPU will not be used.")
else:
print("GPU available: %r" % is_gpu_available())
print_available_devices()
if args.selected:
LstmCellTypes = args.selected.split(",")
benchmarks = {}
if not args.no_gpu and is_gpu_available():
for lstm_unit in LstmCellTypes:
benchmarks["GPU:" + lstm_unit] = benchmark(lstm_unit=lstm_unit, use_gpu=True)
if not args.no_cpu:
for lstm_unit in LstmCellTypes:
if lstm_unit in GpuOnlyCellTypes:
continue
benchmarks["CPU:" + lstm_unit] = benchmark(lstm_unit=lstm_unit, use_gpu=False)
print("-" * 20)
print("Settings:")
pprint(base_settings)
print("Final results:")
for t, lstm_unit in sorted([(t, lstm_unit) for (lstm_unit, t) in sorted(benchmarks.items())]):
print(" %s: %s" % (lstm_unit, hms_fraction(t)))
print("Done.")
def demo():
import better_exchook
better_exchook.install()
import rnn
import sys
if len(sys.argv) <= 1:
print("usage: python %s [config] [other options]" % __file__)
print("example usage: python %s ++learning_rate_control newbob ++learning_rate_file newbob.data ++learning_rate 0.001" % __file__)
rnn.initConfig(commandLineOptions=sys.argv[1:])
from Pretrain import pretrainFromConfig
pretrain = pretrainFromConfig(rnn.config)
first_non_pretrain_epoch = 1
pretrain_learning_rate = None
if pretrain:
first_non_pretrain_epoch = pretrain.get_train_num_epochs() + 1
rnn.config._hack_value_reading_debug()
log.initialize(verbosity=[5])
control = loadLearningRateControlFromConfig(rnn.config)
print("LearningRateControl: %r" % control)
if not control.epochData:
print("No epoch data so far.")
return
if pretrain:
pretrain_learning_rate = rnn.config.float('pretrain_learning_rate', control.defaultLearningRate)
maxEpoch = max(control.epochData.keys())
for epoch in range(1, maxEpoch + 2): # all epochs [1..maxEpoch+1]
oldLearningRate = None
if epoch in control.epochData:
oldLearningRate = control.epochData[epoch].learningRate
if epoch < first_non_pretrain_epoch:
learningRate = pretrain_learning_rate
s = "Pretrain epoch %i, fixed learning rate: %s (was: %s)" % (epoch, learningRate, oldLearningRate)
elif first_non_pretrain_epoch > 1 and epoch == first_non_pretrain_epoch:
learningRate = control.defaultLearningRate
s = "First epoch after pretrain, epoch %i, fixed learning rate: %s (was %s)" % (epoch, learningRate, oldLearningRate)
else:
learningRate = control.calcLearningRateForEpoch(epoch)
s = "Calculated learning rate for epoch %i: %s (was: %s)" % (epoch, learningRate, oldLearningRate)
if learningRate < control.minLearningRate:
learningRate = control.minLearningRate
s += ", clipped to %s" % learningRate
s += ", previous relative error: %s" % control.calcRelativeError(epoch - 2, epoch - 1)
print(s)
# Overwrite new learning rate so that the calculation for further learning rates stays consistent.
if epoch in control.epochData:
control.epochData[epoch].learningRate = learningRate
else:
control.epochData[epoch] = control.EpochData(learningRate=learningRate)
print("Finished, last stored epoch was %i." % maxEpoch)
def _main():
import better_exchook
better_exchook.install()
from argparse import ArgumentParser
arg_parser = ArgumentParser()
arg_parser.add_argument(
"lm_dataset", help="Python eval string, should eval to dict" +
", or otherwise filename, and will just dump")
arg_parser.add_argument("--post_processor", nargs="*")
args = arg_parser.parse_args()
if not args.lm_dataset.startswith("{") and os.path.isfile(args.lm_dataset):
callback = print
if args.post_processor:
pp = get_post_processor_function(args.post_processor)
callback = lambda text: print(pp(text))
iter_corpus(args.lm_dataset, callback)
sys.exit(0)
log.initialize(verbosity=[5])
print("LmDataset demo startup")
kwargs = eval(args.lm_dataset)
assert isinstance(kwargs, dict), "arg should be str of dict: %s" % args.lm_dataset
print("Creating LmDataset with kwargs=%r ..." % kwargs)
dataset = LmDataset(**kwargs)
print("init_seq_order ...")
dataset.init_seq_order(epoch=1)
seq_idx = 0
last_log_time = time.time()
print("start iterating through seqs ...")
while dataset.is_less_than_num_seqs(seq_idx):
if seq_idx == 0:
print("load_seqs with seq_idx=%i ...." % seq_idx)
dataset.load_seqs(seq_idx, seq_idx + 1)
if time.time() - last_log_time > 2.0:
last_log_time = time.time()
print("Loading %s progress, %i/%i (%.0f%%) seqs loaded (%.0f%% skipped), (%.0f%% unknown) total syms %i ..." % (
dataset.__class__.__name__, dataset.next_orth_idx, dataset.estimated_num_seqs,
100.0 * dataset.next_orth_idx / dataset.estimated_num_seqs,
100.0 * dataset.num_skipped / (dataset.next_orth_idx or 1),
100.0 * dataset.num_unknown / dataset._num_timesteps_accumulated["data"],
dataset._num_timesteps_accumulated["data"]))
seq_idx += 1
print("finished iterating, num seqs: %i" % seq_idx)
print("dataset len:", dataset.len_info())
def _thread_main(self):
from Util import interrupt_main
try:
import better_exchook
better_exchook.install()
from Util import AsyncThreadRun
# First iterate once over the data to get the data len as fast as possible.
data_len = 0
while True:
ls = self._data_files["data"].readlines(10**4)
data_len += len(ls)
if not ls:
break
with self._lock:
self._data_len = data_len
self._data_files["data"].seek(
0, os.SEEK_SET) # we will read it again below
# Now, read and use the vocab for a compact representation in memory.
keys_to_read = ["data", "classes"]
while True:
for k in list(keys_to_read):
data_strs = self._data_files[k].readlines(10**6)
if not data_strs:
assert len(self._data[k]) == self._data_len
keys_to_read.remove(k)
continue
assert len(
self._data[k]) + len(data_strs) <= self._data_len
vocab = self._vocabs[k]
data = [
self._data_str_to_numpy(
vocab,
s.decode("utf8").strip() + self._add_postfix[k])
for s in data_strs
]
with self._lock:
self._data[k].extend(data)
if not keys_to_read:
break
for k, f in list(self._data_files.items()):
f.close()
self._data_files[k] = None
except Exception:
sys.excepthook(*sys.exc_info())
interrupt_main()
def checkExec():
if "--forkExecProc" in sys.argv:
mod_path = os.path.realpath(
os.path.join(os.path.dirname(__file__), '..'))
if not any(mod_path == os.path.realpath(s) for s in sys.path):
sys.path.append(mod_path)
try:
import better_exchook
except ImportError:
pass # Doesn't matter.
else:
better_exchook.install()
argidx = sys.argv.index("--forkExecProc")
writeFileNo = int(sys.argv[argidx + 1])
readFileNo = int(sys.argv[argidx + 2])
readend = os.fdopen(readFileNo, "r")
writeend = os.fdopen(writeFileNo, "w")
unpickler = Unpickler(readend)
name = unpickler.load()
if ExecingProcess.Verbose:
print("ExecingProcess child %s (pid %i)" % (name, os.getpid()))
try:
target = unpickler.load()
args = unpickler.load()
except EOFError:
print("Error: unpickle incomplete")
raise SystemExit
ret = target(*args)
sys.exited = True
# IOError is probably broken pipe. That probably means that the parent died.
try:
Pickler(writeend).dump(ret)
except IOError:
pass
try:
readend.close()
except IOError:
pass
try:
writeend.close()
except IOError:
pass
if ExecingProcess.Verbose:
print("ExecingProcess child %s (pid %i) finished" %
(name, os.getpid()))
raise SystemExit
def demo():
"""
Will print out the different network topologies of the specified pretraining scheme.
"""
import better_exchook
better_exchook.install()
import rnn
import argparse
from Util import dict_diff_str
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument("config")
arg_parser.add_argument("--diff",
action="store_true",
help="show diff only")
arg_parser.add_argument('other_returnn_args',
nargs=argparse.REMAINDER,
help="config updates or so")
args = arg_parser.parse_args()
rnn.init_config(config_filename=args.config,
command_line_options=args.other_returnn_args,
extra_updates={"log": []})
# noinspection PyProtectedMember
rnn.config._hack_value_reading_debug()
rnn.init_log()
if not rnn.config.value("pretrain", ""):
print(
"config option 'pretrain' not set, will set it for this demo to 'default'"
)
rnn.config.set("pretrain", "default")
pretrain = pretrain_from_config(rnn.config)
print("pretrain: %s" % pretrain)
num_pretrain_epochs = pretrain.get_train_num_epochs()
last_net_json = None
from pprint import pprint
for epoch in range(1, 1 + num_pretrain_epochs):
print("epoch %i (of %i) network json:" % (epoch, num_pretrain_epochs))
net_json = pretrain.get_network_json_for_epoch(epoch)
if args.diff:
if last_net_json is not None:
print(dict_diff_str(last_net_json, net_json))
else:
print("(initial)")
else:
pprint(net_json)
last_net_json = net_json
print("done.")
def watcher_loop():
import main
import Action
better_exchook.install()
while not exitEvent.isSet():
if main.DownloadOnly:
if len(currentWorkSet) == 0:
queue_work(Action.CheckDownloadsFinished())
exitEvent.wait(1)
continue
if workerQueue.qsize() >= kMinQueuedActions:
exitEvent.wait(1)
continue
func = Action.get_new_action()
workerQueue.put(func)
def setup(*raw_arg_list):
print("RandomFtpGrabber startup.")
import better_exchook
better_exchook.install()
import Logging
better_exchook.output = Logging.log
arg_parser = ArgumentParser()
arg_parser.add_argument("--dir", default=os.getcwd())
arg_parser.add_argument("--numWorkers", type=int)
arg_parser.add_argument("--shell", action="store_true")
arg_parser.add_argument("--skipShodan", action="store_true")
arg_parser.add_argument("--downloadRemaining", action="store_true")
global Args
Args = arg_parser.parse_args(raw_arg_list)
if sys.version_info.major != 3:
Logging.log("Warning: This code was only tested with Python3.")
import time
time.sleep(10) # wait a bit to make sure the user sees this
start_stdin_handler_loop()
import main
main.RootDir = Args.dir
Logging.log("root dir: %s" % RootDir)
main.SkipShodan = Args.skipShodan
if not main.SkipShodan:
prepare_lists()
main.DownloadOnly = Args.downloadRemaining
if not main.DownloadOnly:
setup_lists()
import TaskSystem # important to be initially imported in the main thread
if Args.numWorkers:
TaskSystem.kNumWorkers = Args.numWorkers
TaskSystem.kMinQueuedActions = Args.numWorkers
TaskSystem.kSuggestedMaxQueuedActions = Args.numWorkers * 2
if Args.shell:
TaskSystem.kNumWorkers = 0
TaskSystem.setup()
def main():
better_exchook.install()
my_files = sorted(os.listdir(my_dir))
for fn in my_files:
if fn.startswith("test_") and fn.endswith(".py"):
print("=" * 40)
print("Python test file:", fn)
mod_name = os.path.splitext(fn)[0]
mod = importlib.import_module(mod_name)
for name in dir(mod):
if not name.startswith("test_"):
continue
value = getattr(mod, name)
if callable(value):
print("-" * 40)
print("Test:", mod_name, ".", name)
value()
print("-" * 40)
print("=" * 40)
def _demo():
def _make_int_list(s):
"""
:param str s:
:rtype: list[int]
"""
return [int(s_) for s_ in s.split(",")]
import better_exchook
better_exchook.install()
from argparse import ArgumentParser
arg_parser = ArgumentParser()
arg_parser.add_argument("--states", type=_make_int_list, default=[0])
arg_parser.add_argument("--inputs", type=_make_int_list, default=[0])
# Other example FST files can be found online, e.g.: https://github.com/placebokkk/gofst/tree/master/ex01
# Or: https://github.com/zh794390558/deeplearning/tree/master/kaldi/fst/data/test
arg_parser.add_argument("--fst",
default=returnn_dir + "/tests/lexicon_opt.fst")
args = arg_parser.parse_args()
# Try to compile now.
get_tf_mod(verbose=True)
# Some demo.
assert os.path.exists(args.fst)
fst_tf = get_fst(filename=args.fst)
states_tf = tf.compat.v1.placeholder(tf.int32, [None])
inputs_tf = tf.compat.v1.placeholder(tf.int32, [None])
output_tf = fst_transition(fst_handle=fst_tf,
states=states_tf,
inputs=inputs_tf)
with tf.compat.v1.Session() as session:
out_next_states, out_labels, out_scores = session.run(output_tf,
feed_dict={
states_tf:
args.states,
inputs_tf:
args.inputs
})
print("states:", args.states)
print("inputs:", args.inputs)
print("output next states:", out_next_states)
print("output labels:", out_labels)
print("output scores:", out_scores)
def demo():
import better_exchook
better_exchook.install()
import rnn
import sys
if len(sys.argv) <= 1:
print("usage: python %s [config] [other options]" % __file__)
print("example usage: python %s ++pretrain default ++pretrain_construction_algo from_input" % __file__)
rnn.initConfig(commandLineOptions=sys.argv[1:])
rnn.config._hack_value_reading_debug()
if not rnn.config.value("pretrain", ""):
print("config option 'pretrain' not set, will set it for this demo to 'default'")
rnn.config.set("pretrain", "default")
pretrain = pretrainFromConfig(rnn.config)
print("pretrain: %s" % pretrain)
num_pretrain_epochs = pretrain.get_train_num_epochs()
from pprint import pprint
for epoch in range(1, 1 + num_pretrain_epochs):
print("epoch %i (of %i) network json:" % (epoch, num_pretrain_epochs))
net_json = pretrain.get_network_json_for_epoch(epoch)
pprint(net_json)
print("done.")
def _main(argv):
import better_exchook
better_exchook.install()
log.initialize(verbosity=[5])
dataset = LmDataset(**eval(argv[0]))
dataset.init_seq_order(epoch=1)
seq_idx = 0
last_log_time = time.time()
while dataset.is_less_than_num_seqs(seq_idx):
dataset.load_seqs(seq_idx, seq_idx + 1)
if time.time() - last_log_time > 2.0:
last_log_time = time.time()
print >> log.v5, "Loading %s progress, %i/%i (%.0f%%) seqs loaded (%.0f%% skipped), total syms %i ..." % (
dataset.__class__.__name__, dataset.next_orth_idx, dataset.estimated_num_seqs,
100.0 * dataset.next_orth_idx / dataset.estimated_num_seqs,
100.0 * dataset.num_skipped / (dataset.next_orth_idx or 1),
dataset._num_timesteps_accumulated["data"])
seq_idx += 1
print >>log.v3, "dataset len:", dataset.len_info()
def threadMain(self):
better_exchook.install()
thread = currentThread()
while True:
if self.module:
try:
reload(self.module)
except Exception:
print "couldn't reload module", self.module
sys.excepthook(*sys.exc_info())
# continue anyway, maybe it still works and maybe the mainFunc does sth good/important
else:
self.module = __import__(self.moduleName)
mainFunc = getattr(self.module, self.mainFuncName)
try:
mainFunc()
except Exception:
print "Exception in thread", thread.name
sys.excepthook(*sys.exc_info())
if not thread.reload: break
sys.stdout.write("reloading module %s\n" % self.name)
thread.cancel = False
thread.reload = False
thread.waitQueue = None
def checkExec():
if "--forkExecProc" in sys.argv:
mod_path = os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))
if not any(mod_path == os.path.realpath(s) for s in sys.path):
sys.path.append(mod_path)
try:
import better_exchook
except ImportError:
pass # Doesn't matter.
else:
better_exchook.install()
argidx = sys.argv.index("--forkExecProc")
writeFileNo = int(sys.argv[argidx + 1])
readFileNo = int(sys.argv[argidx + 2])
readend = os.fdopen(readFileNo, "rb")
writeend = os.fdopen(writeFileNo, "wb")
unpickler = Unpickler(readend)
name = unpickler.load()
if ExecingProcess.Verbose: print("ExecingProcess child %s (pid %i)" % (name, os.getpid()))
try:
target = unpickler.load()
args = unpickler.load()
except EOFError:
print("Error: unpickle incomplete")
raise SystemExit
ret = target(*args)
sys.exited = True
# IOError is probably broken pipe. That probably means that the parent died.
try: Pickler(writeend).dump(ret)
except IOError: pass
try: readend.close()
except IOError: pass
try: writeend.close()
except IOError: pass
if ExecingProcess.Verbose: print("ExecingProcess child %s (pid %i) finished" % (name, os.getpid()))
raise SystemExit
import subprocess
# echo via subprocess, because this stdout as well as the other will always be visible.
subprocess.check_call(
["echo", "travis_fold:start:test_returnn_as_framework_TaskSystem"])
subprocess.check_call([
py,
os.path.abspath("tests/returnn-as-framework.py"),
"test_TaskSystem_Pickler()"
],
cwd="/")
subprocess.check_call(
["echo", "travis_fold:end:test_returnn_as_framework_TaskSystem"])
if __name__ == '__main__':
better_exchook.install()
TestDemos.setup_class()
tests = TestDemos()
import sys
fns = sys.argv[1:]
if not fns:
fns = [arg for arg in dir(tests) if arg.startswith("test_")]
for arg in fns:
f = getattr(tests, arg)
print("-" * 20)
print("Run:", f)
print("-" * 20)
f()
print("-" * 20)
print("Ok.")
print("-" * 20)
#!/usr/bin/python # MusicPlayer, https://github.com/albertz/music-player # Copyright (c) 2012, Albert Zeyer, www.az2000.de # All rights reserved. # This code is under the 2-clause BSD license, see License.txt in the root directory of this project. import better_exchook better_exchook.install() try: import faulthandler faulthandler.enable(all_threads=True) except ImportError: print "note: faulthandler module not available" # Do this early to do some option parsing and maybe special handling. import appinfo # This might do some init which might be important to be done in the main thread. import utils utils.ExecingProcess.checkExec() import sys, time print "MusicPlayer", appinfo.version, "from", appinfo.buildTime, "on", appinfo.platform, "(%s)" % sys.platform print "startup on", utils.formatDate(time.time()) from State import state, modules if __name__ == '__main__':
def main():
""" Parses command line arguments and executes commands """
# Setup recipe importer
sys.meta_path.append(RecipeFinder)
# Setup argument parser
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers()
parser.add_argument('--log_level', dest='log_level', metavar='LOG_LEVEL',
type=int, default=20, help='log level, 10 for debug messages, 50 for only critical,'
' default: 20, ')
parser.add_argument('--config', dest='config_files', action='append',
default=[], help='config file for setup, defining which jobs to run')
# Not fully supported at the moment
# parser.add_argument('--settings', dest='settings_file',
# default=gs.GLOBAL_SETTINGS_FILE_DEFAULT,
# help='settings file, aka how to run the jobs')
# parser.add_argument('-s', '--setting', dest='commandline_settings', action='append',
# default=[], help='overwrite global settings directly via commandline')
parser_manager = subparsers.add_parser('manager', aliases=['m'],
conflict_handler='resolve',
help="Load config files and start manager loop")
parser_manager.set_defaults(func=manager)
parser_manager.add_argument("-r", dest="run", default=False,
action='store_true', help="Run the given task")
parser_manager.add_argument("-co", dest="clear_once", action="store_true",
default=False,
help="Move jobs aside that are in an error "
"state when the manager runs the first time")
parser_manager.add_argument("--http", dest="http_port", default=None,
type=int, help="Enables http server, takes "
"port as argument")
parser_manager.add_argument("--fs", "--filesystem", dest="filesystem",
default=None,
help="Start filesystem in given directory")
parser_manager.add_argument('argv', metavar='ARGV', type=str,
nargs='*',
help='an additional way do '
'define config files')
parser_console = subparsers.add_parser('console', aliases=['c'],
usage='sis console [-h] [--load LOAD_SIS_GRAPH] [ARGV [ARGV ...]]\n\n'
'Open console to debug sisyphus graph or job',
help="Start console to interactively work on sis graph. Things like: "
"Rerunning tasks, cleaning up the work directory, and debugging "
"is best done here")
parser_console.add_argument("--load", dest="load", default=[], action='append',
help="load graph and start console")
parser_console.add_argument("--skip_config", dest="not_load_config", default=False, action='store_true',
help="do not load config files before starting the console")
parser_console.add_argument("-c", dest="commands", default=[], action='append',
help="Run commands after loading console")
parser_console.add_argument('argv', metavar='ARGV', type=str, nargs='*',
help='an additional way do define config files')
parser_console.set_defaults(func=console)
parser_worker = subparsers.add_parser(gs.CMD_WORKER, help='Start worker to compute job (for internally usage)')
parser_worker.set_defaults(func=worker)
parser_worker.add_argument('jobdir', metavar='JOBDIR', type=str,
help='Job directory of the executed function')
parser_worker.add_argument('task_name', metavar='TASK_NAME', type=str,
help='Task name')
parser_worker.add_argument('task_id', metavar='TASK_ID', type=int,
nargs='?', default=None,
help='Task id, if not set trying to '
'read it from environment variables')
parser_worker.add_argument('--force_resume', "--fr", default=False, action="store_true",
help='force resume of non resumable tasks, good for debugging')
parser_worker.add_argument('--engine', default='short', help='The engine running the Job')
# Currently disabled parser, maybe used again in the future
#
# Disabled, since it's not used for now
# parser_unittest = subparsers.add_parser('unittest', help='Run unittest tasks')
# parser_unittest.set_defaults(func=unittest)
# parser_unittest.add_argument("--do_not_run", dest="run", default=True, action='store_false',
# help="Do not run the given task")
# parser_unittest.add_argument('argv', metavar='ARGV', type=str, nargs='*',
# help='an additional way do define config files')
# Disabled since it's currently not working
# parser_notebook = subparsers.add_parser('notebook',
# usage='sis notebook [-h]\n\n'
# 'Open notebook session',
# help="Start notebook to interactively work on sis graph")
# parser_notebook.add_argument("--load", dest="load", default=[], action='append',
# help="load graph and start console")
# parser_notebook.add_argument("--file", "-f", dest="filename", default='default',
# help="load (and create if needed) this notebook file")
# parser_notebook.set_defaults(func=helper.notebook)
# Disabled since it's currently not working
# parser_connect = subparsers.add_parser('connect', usage='sis connect [-h] [connect_file]\n\n'
# 'Opens a console connected to given kernel',
# help='Opens a console connected with given kernel')
# parser_connect.add_argument('argv', metavar='ARGV', type=str, nargs='?',
# default=[], help='connection_file')
# parser_connect.set_defaults(func=helper.connect)
args = parser.parse_args()
if not hasattr(args, 'func'):
parser.print_help()
return
# add argv to config_files if manager or console is called
if args.func in [manager, console]:
args.config_files += args.argv
# Setup logging
logging.basicConfig(format='[%(asctime)s] %(levelname)s: %(message)s',
level=args.log_level)
# Changing settings via commandline is currently not supported
# Needs to ensure all parameters are passed correctly to worker, ignored since nobody requested it so far
# update_global_settings_from_file(args.settings_file)
# update_global_settings_from_list(args.commandline_settings)
update_global_settings_from_file(gs.GLOBAL_SETTINGS_FILE_DEFAULT)
if gs.USE_VERBOSE_TRACEBACK:
if gs.VERBOSE_TRACEBACK_TYPE == "ipython":
from IPython.core import ultratb
sys.excepthook = ultratb.VerboseTB()
elif gs.VERBOSE_TRACEBACK_TYPE == "better_exchook":
# noinspection PyPackageRequirements
import better_exchook
better_exchook.install()
better_exchook.replace_traceback_format_tb()
else:
raise Exception("invalid VERBOSE_TRACEBACK_TYPE %r" % gs.VERBOSE_TRACEBACK_TYPE)
if gs.USE_SIGNAL_HANDLERS:
from sisyphus.tools import maybe_install_signal_handers
maybe_install_signal_handers()
if args.func != manager:
gs.JOB_AUTO_CLEANUP = False
try:
args.func(args)
except BaseException as exc:
if not isinstance(exc, SystemExit):
logging.error("Main thread unhandled exception:")
sys.excepthook(*sys.exc_info())
import threading
non_daemon_threads = {
thread for thread in threading.enumerate()
if not thread.daemon and thread is not threading.main_thread()}
if non_daemon_threads:
logging.warning("Main thread exit. Still running non-daemon threads: %r" % non_daemon_threads)
def demo():
"""
Demo run. Given some learning rate file (with scores / existing lrs), will calculate how lrs would have been set,
given some config.
"""
import better_exchook
better_exchook.install()
import rnn
import sys
if len(sys.argv) <= 1:
print("usage: python %s [config] [other options] [++check_learning_rates 1]" % __file__)
print(
("example usage: "
"python %s ++learning_rate_control newbob ++learning_rate_file newbob.data ++learning_rate 0.001") % __file__)
rnn.init_config(command_line_options=sys.argv[1:])
# noinspection PyProtectedMember
rnn.config._hack_value_reading_debug()
rnn.config.update({"log": []})
rnn.init_log()
rnn.init_backend_engine()
check_lr = rnn.config.bool("check_learning_rates", False)
from Pretrain import pretrain_from_config
pretrain = pretrain_from_config(rnn.config)
first_non_pretrain_epoch = 1
pretrain_learning_rate = None
if pretrain:
first_non_pretrain_epoch = pretrain.get_train_num_epochs() + 1
log.initialize(verbosity=[5])
control = load_learning_rate_control_from_config(rnn.config)
print("LearningRateControl: %r" % control)
if not control.epoch_data:
print("No epoch data so far.")
return
first_epoch = min(control.epoch_data.keys())
if first_epoch != 1:
print("Strange, first epoch from epoch data is %i." % first_epoch)
print("Error key: %s from %r" % (control.get_error_key(epoch=first_epoch), control.epoch_data[first_epoch].error))
if pretrain:
pretrain_learning_rate = rnn.config.float('pretrain_learning_rate', control.default_learning_rate)
max_epoch = max(control.epoch_data.keys())
for epoch in range(1, max_epoch + 2): # all epochs [1..max_epoch+1]
old_learning_rate = None
if epoch in control.epoch_data:
old_learning_rate = control.epoch_data[epoch].learning_rate
if epoch < first_non_pretrain_epoch:
learning_rate = pretrain_learning_rate
s = "Pretrain epoch %i, fixed learning rate: %s (was: %s)" % (epoch, learning_rate, old_learning_rate)
elif 1 < first_non_pretrain_epoch == epoch:
learning_rate = control.default_learning_rate
s = "First epoch after pretrain, epoch %i, fixed learning rate: %s (was %s)" % (
epoch, learning_rate, old_learning_rate)
else:
learning_rate = control.calc_new_learning_rate_for_epoch(epoch)
s = "Calculated learning rate for epoch %i: %s (was: %s)" % (epoch, learning_rate, old_learning_rate)
if learning_rate < control.min_learning_rate:
learning_rate = control.min_learning_rate
s += ", clipped to %s" % learning_rate
s += ", previous relative error: %s" % control.calc_relative_error(epoch - 2, epoch - 1)
if hasattr(control, "_calc_recent_mean_relative_error"):
# noinspection PyProtectedMember
s += ", previous mean relative error: %s" % control._calc_recent_mean_relative_error(epoch)
print(s)
if check_lr and old_learning_rate is not None:
if old_learning_rate != learning_rate:
print("Learning rate is different in epoch %i!" % epoch)
sys.exit(1)
# Overwrite new learning rate so that the calculation for further learning rates stays consistent.
if epoch in control.epoch_data:
control.epoch_data[epoch].learning_rate = learning_rate
else:
control.epoch_data[epoch] = control.EpochData(learningRate=learning_rate)
print("Finished, last stored epoch was %i." % max_epoch)
