def __init__(self, name, outputLayer, output_expressions, additional_input_expressions = {}, updates = [], **kwargs) :
"""
:param str name: name of the function
:param Output outputLayer: the output layer the function should be applied to
:param list output_expressions: list of the symbolic expressions you want as output
:param dict additional_input_expressions: additional inputs needed to compute the expressions
:param list updates: list of tuples (shared variable, symbolic expression of the update to be applied to it)
:param dict **kwargs: additional arguments to passed to the real theano function underneath
"""
self.name = name
self.outputLayer = outputLayer
self.inputs = OrderedDict()
self.tmpInputs = OrderedDict()
for inp in self.outputLayer.network.inputs.itervalues() :
self.inputs[inp.name] = inp.outputs
self.inputs.update(additional_input_expressions)
for i in self.inputs :
self.tmpInputs[i] = None
self.additional_input_expressions = additional_input_expressions
self.outputs = output_expressions
self.updates = updates
self.theano_fct = theano.function(inputs = self.inputs.values(), outputs = self.outputs, updates = self.updates, **kwargs)
if any([x.__class__.__name__.lower().find("gpu") for x in self.theano_fct.maker.fgraph.toposort()]):
device = "GPU"
else:
device = "CPU"
if MSET.VERBOSE :
MCAN.friendly("Run device", "I will use the [-%s-] to run function '%s' of layer '%s'!" % (device, name, outputLayer.name))
def __init__(self, name, outputLayer, output_expressions, additional_input_expressions = {}, updates = [], **kwargs) :
"""
:param str name: name of the function
:param Output outputLayer: the output layer the function should be applied to
:param list output_expressions: list of tuples of symbolic expressions you want as output and the names you want to give to them: (name, expressions)
:param dict additional_input_expressions: additional inputs needed to compute the expressions
:param list updates: list of tuples (shared variable, symbolic expression of the update to be applied to it)
:param dict \*\*kwargs: additional arguments to passed to the real theano function underneath
"""
self.cast_warning_told = False
self.name = name
self.outputLayer = outputLayer
self.inputs = OrderedDict()
inpSet = set()
for inp in self.outputLayer.network.inputs.itervalues() :
inpOut = inp.inputs
if inpOut not in inpSet :
self.inputs[inp.name] = inpOut
inpSet.add(inpOut)
for k, v in additional_input_expressions.iteritems() :
if v not in inpSet :
self.inputs[k] = v
inpSet.add(v)
self.fctInputs = OrderedDict()
for i in self.inputs :
self.fctInputs[i] = None
self.additional_input_expressions = additional_input_expressions
self.output_expressions = OrderedDict()
for name, output_expr in output_expressions :
self.output_expressions[name] = output_expr
self.updates = updates
self.theano_fct = theano.function(inputs = self.inputs.values(), outputs = self.output_expressions.values(), updates = self.updates, **kwargs)
warningMsg = False
if DEVICE_IS_GPU :
device = "GPU"
msg = "I will use the [-%s-] to run function '%s' of layer '%s'!" % (device, name, outputLayer.name)
if str(self.getToposort()).find("float64") > -1:
warningMsg = True
msg += "\n\nBut there are some float64s that do not fit on the GPU and will slow down the computations.\nPlease consider:"
msg += "\n\t* Launching with THEANO_FLAGS=device=gpu,floatX=float32 python <your script>.py."
msg += "\n\t* If you have any dmatrix, dvector or dscalar in your code replace them with matrix, vector, scalar."
else:
device = "CPU"
msg = "I will use the [-%s-] to run function '%s' of layer '%s'!" % (device, name, outputLayer.name)
self.results = OrderedDict()
MCAN.friendly("Run device", msg, warning = warningMsg)
def __init__(self, name, applicationType, outputLayer, output_expressions, additional_input_expressions = {}, updates = [], **kwargs) :
"""
:param str name: name of the function
:param Output outputLayer: the output layer the function should be applied to
:param list output_expressions: list of the symbolic expressions you want as output
:param dict additional_input_expressions: additional inputs needed to compute the expressions
:param list updates: list of tuples (shared variable, symbolic expression of the update to be applied to it)
:param dict **kwargs: additional arguments to passed to the real theano function underneath
"""
self.cast_warning_told = False
self.name = name
self.outputLayer = outputLayer
self.applicationType = applicationType.lower()
self.inputs = OrderedDict()
inpSet = set()
for inp in self.outputLayer.network.inputs.itervalues() :
if self.applicationType == TYPE_TEST :
inpOut = inp.test_outputs
elif self.applicationType == TYPE_TRAIN :
inpOut = inp.outputs
else :
raise AttributeError('Unknow applicationType %s' % applicationType)
if inpOut not in inpSet :
self.inputs[inp.name] = inpOut
inpSet.add(inpOut)
for k, v in additional_input_expressions.iteritems() :
if v not in inpSet :
self.inputs[k] = v
inpSet.add(v)
self.fctInputs = OrderedDict()
for i in self.inputs :
self.fctInputs[i] = None
self.additional_input_expressions = additional_input_expressions
self.outputs = output_expressions
self.updates = updates
self.theano_fct = theano.function(inputs = self.inputs.values(), outputs = self.outputs, updates = self.updates, **kwargs)
if theano.config.device.find("gpu") > -1:
device = "GPU"
else:
device = "CPU"
MCAN.friendly("Run device", "I will use the [-%s-] to run the *%s* type function '%s' of layer '%s'!" % (device, self.applicationType, name, outputLayer.name))
def __init__(self, name, outputLayer, output_expressions, additional_input_expressions={}, updates=[], **kwargs):
"""
:param str name: name of the function
:param Output outputLayer: the output layer the function should be applied to
:param list output_expressions: list of the symbolic expressions you want as output
:param dict additional_input_expressions: additional inputs needed to compute the expressions
:param list updates: list of tuples (shared variable, symbolic expression of the update to be applied to it)
:param dict \*\*kwargs: additional arguments to passed to the real theano function underneath
"""
self.cast_warning_told = False
self.name = name
self.outputLayer = outputLayer
self.inputs = OrderedDict()
inpSet = set()
for inp in self.outputLayer.network.inputs.itervalues():
inpOut = inp.inputs
if inpOut not in inpSet:
self.inputs[inp.name] = inpOut
inpSet.add(inpOut)
for k, v in additional_input_expressions.iteritems():
if v not in inpSet:
self.inputs[k] = v
inpSet.add(v)
self.fctInputs = OrderedDict()
for i in self.inputs:
self.fctInputs[i] = None
self.additional_input_expressions = additional_input_expressions
self.outputs = output_expressions
self.updates = updates
self.theano_fct = theano.function(
inputs=self.inputs.values(), outputs=self.outputs, updates=self.updates, **kwargs
)
if DEVICE_IS_GPU > -1:
device = "GPU"
else:
device = "CPU"
MCAN.friendly(
"Run device", "I will use the [-%s-] to run function '%s' of layer '%s'!" % (device, name, outputLayer.name)
)
def start(self, runName, model, recorder, *args, **kwargs):
"""Starts the training and encapsulates it into a safe environement.
If the training stops because of an Exception or SIGTEM, the trainer
will save logs, the store, and the last version of the model.
"""
import simplejson, signal, cPickle
def _handler_sig_term(sig, frame):
_dieGracefully("SIGTERM", None)
sys.exit(sig)
def _dieGracefully(exType, tb=None):
if type(exType) is types.StringType:
exName = exType
else:
exName = exType.__name__
death_time = time.ctime().replace(' ', '_')
filename = "dx-xb_" + runName + "_death_by_" + exName + "_" + death_time
sys.stderr.write(
"\n===\nDying gracefully from %s, and saving myself to:\n...%s\n===\n"
% (exName, filename))
model.save(filename)
f = open(filename + ".traceback.log", 'w')
f.write(
"Mariana training Interruption\n=============================\n"
)
f.write("\nDetails\n-------\n")
f.write("Name: %s\n" % runName)
f.write("pid: %s\n" % os.getpid())
f.write("Killed by: %s\n" % str(exType))
f.write("Time of death: %s\n" % death_time)
f.write("Model saved to: %s\n" % filename)
sstore = str(self.store).replace("'", '"').replace(
"True", 'true').replace("False", 'false')
try:
f.write("store:\n%s" % json.dumps(json.loads(sstore),
sort_keys=True,
indent=4,
separators=(',', ': ')))
except Exception as e:
print "Warning: Couldn't format the store to json, saving it ugly."
print "Reason:", e
f.write(sstore)
if tb is not None:
f.write("\nTraceback\n---------\n")
f.write(
str(traceback.extract_tb(tb)).replace(
"), (", "),\n(").replace("[(",
"[\n(").replace(")]", ")\n]"))
f.flush()
f.close()
f = open(filename + ".store.pkl", "wb")
cPickle.dump(self.store, f)
f.close()
signal.signal(signal.SIGTERM, _handler_sig_term)
if MSET.VERBOSE:
print "\n" + "Training starts."
MCAN.friendly("Process id", "The pid of this run is: %d" % os.getpid())
if recorder == "default":
params = {"printRate": 1, "writeRate": 1}
recorder = MREC.GGPlot2(runName, **params)
MCAN.friendly(
"Default recorder",
"The trainer will recruit the default 'GGPlot2' recorder with the following arguments:\n\t %s.\nResults will be saved into '%s'."
% (params, recorder.filename))
try:
return self.run(runName, model, recorder, *args, **kwargs)
except MSTOP.EndOfTraining as e:
print e.message
death_time = time.ctime().replace(' ', '_')
filename = "finished_" + runName + "_" + death_time
f = open(filename + ".stopreason.txt", 'w')
f.write("Name: %s\n" % runName)
f.write("pid: %s\n" % os.getpid())
f.write("Time of death: %s\n" % death_time)
f.write("Epoch of death: %s\n" % self.store["runInfos"]["epoch"])
f.write("Stopped by: %s\n" % e.stopCriterion.name)
f.write("Reason: %s\n" % e.message)
sstore = str(self.store).replace("'", '"').replace(
"True", 'true').replace("False", 'false')
try:
f.write("store:\n%s" % json.dumps(json.loads(sstore),
sort_keys=True,
indent=4,
separators=(',', ': ')))
except Exception as e:
print "Warning: Couldn't format the store to json, saving it ugly."
print "Reason:", e
f.write(sstore)
f.flush()
f.close()
model.save(filename)
f = open(filename + ".store.pkl", "wb")
cPickle.dump(self.store, f)
f.close()
except KeyboardInterrupt:
if not self.saveIfMurdered:
raise
exType, ex, tb = sys.exc_info()
_dieGracefully(exType, tb)
raise
except:
if not self.saveIfMurdered:
raise
exType, ex, tb = sys.exc_info()
_dieGracefully(exType, tb)
raise
def start(self, runName, model, recorder, *args, **kwargs) :
"""Starts the training and encapsulates it into a safe environement.
If the training stops because of an Exception or SIGTEM, the trainer
will save logs, the store, and the last version of the model.
"""
import simplejson, signal, cPickle
def _handler_sig_term(sig, frame) :
_dieGracefully("SIGTERM", None)
sys.exit(sig)
def _dieGracefully(exType, tb = None) :
if type(exType) is types.StringType :
exName = exType
else :
exName = exType.__name__
death_time = time.ctime().replace(' ', '_')
filename = "dx-xb_" + runName + "_death_by_" + exName + "_" + death_time
sys.stderr.write("\n===\nDying gracefully from %s, and saving myself to:\n...%s\n===\n" % (exName, filename))
model.save(filename)
f = open(filename + ".traceback.log", 'w')
f.write("Mariana training Interruption\n=============================\n")
f.write("\nDetails\n-------\n")
f.write("Name: %s\n" % runName)
f.write("pid: %s\n" % os.getpid())
f.write("Killed by: %s\n" % str(exType))
f.write("Time of death: %s\n" % death_time)
f.write("Model saved to: %s\n" % filename)
sstore = str(self.store).replace("'", '"').replace("True", 'true').replace("False", 'false')
try :
f.write(
"store:\n%s" % json.dumps(
json.loads(sstore), sort_keys=True,
indent=4,
separators=(',', ': ')
)
)
except Exception as e:
print "Warning: Couldn't format the store to json, saving it ugly."
print "Reason:", e
f.write(sstore)
if tb is not None :
f.write("\nTraceback\n---------\n")
f.write(str(traceback.extract_tb(tb)).replace("), (", "),\n(").replace("[(","[\n(").replace(")]",")\n]"))
f.flush()
f.close()
f = open(filename + ".store.pkl", "wb")
cPickle.dump(self.store, f)
f.close()
signal.signal(signal.SIGTERM, _handler_sig_term)
if MSET.VERBOSE :
print "\n" + "Training starts."
MCAN.friendly("Process id", "The pid of this run is: %d" % os.getpid())
if recorder == "default" :
params = {
"printRate" : 1,
"writeRate" : 1
}
recorder = MREC.GGPlot2(runName, **params)
MCAN.friendly(
"Default recorder",
"The trainer will recruit the default 'GGPlot2' recorder with the following arguments:\n\t %s.\nResults will be saved into '%s'." % (params, recorder.filename)
)
try :
return self.run(runName, model, recorder, *args, **kwargs)
except MSTOP.EndOfTraining as e :
print e.message
death_time = time.ctime().replace(' ', '_')
filename = "finished_" + runName + "_" + death_time
f = open(filename + ".stopreason.txt", 'w')
f.write("Name: %s\n" % runName)
f.write("pid: %s\n" % os.getpid())
f.write("Time of death: %s\n" % death_time)
f.write("Epoch of death: %s\n" % self.store["runInfos"]["epoch"])
f.write("Stopped by: %s\n" % e.stopCriterion.name)
f.write("Reason: %s\n" % e.message)
sstore = str(self.store).replace("'", '"').replace("True", 'true').replace("False", 'false')
try :
f.write(
"store:\n%s" % json.dumps(
json.loads(sstore), sort_keys=True,
indent=4,
separators=(',', ': ')
)
)
except Exception as e:
print "Warning: Couldn't format the store to json, saving it ugly."
print "Reason:", e
f.write(sstore)
f.flush()
f.close()
model.save(filename)
f = open(filename + ".store.pkl", "wb")
cPickle.dump(self.store, f)
f.close()
except KeyboardInterrupt :
if not self.saveIfMurdered :
raise
exType, ex, tb = sys.exc_info()
_dieGracefully(exType, tb)
raise
except :
if not self.saveIfMurdered :
raise
exType, ex, tb = sys.exc_info()
_dieGracefully(exType, tb)
raise
def __init__(self,
name,
outputLayer,
output_expressions,
additional_input_expressions={},
updates=[],
**kwargs):
"""
:param str name: name of the function
:param Output outputLayer: the output layer the function should be applied to
:param list output_expressions: list of tuples of symbolic expressions you want as output and the names you want to give to them: (name, expressions)
:param dict additional_input_expressions: additional inputs needed to compute the expressions
:param list updates: list of tuples (shared variable, symbolic expression of the update to be applied to it)
:param dict \*\*kwargs: additional arguments to passed to the real theano function underneath
"""
def _bckTrckInputs(startLayer, inputs=OrderedDict(), inpSet=set()):
if len(startLayer.network.inConnections[startLayer]) == 0:
inpOut = startLayer.inputs
if inpOut not in inpSet:
inputs[startLayer.name] = inpOut
inpSet.add(inpOut)
else:
for layer in startLayer.network.inConnections[startLayer]:
_bckTrckInputs(layer, inputs, inpSet)
return inputs, inpSet
self.cast_warning_told = False
self.name = name
self.outputLayer = outputLayer
self.inputs, inpSet = _bckTrckInputs(outputLayer)
for k, v in additional_input_expressions.iteritems():
if v not in inpSet:
self.inputs[k] = v
inpSet.add(v)
self.fctInputs = OrderedDict()
for i in self.inputs:
self.fctInputs[i] = None
self.additional_input_expressions = additional_input_expressions
self.output_expressions = OrderedDict()
for name, output_expr in output_expressions:
self.output_expressions[name] = output_expr
self.updates = updates
self.theano_fct = theano.function(
inputs=self.inputs.values(),
outputs=self.output_expressions.values(),
updates=self.updates,
**kwargs)
warningMsg = False
if DEVICE_IS_GPU:
device = "GPU"
msg = "I will use the [-%s-] to run function '%s' of layer '%s'!" % (
device, name, outputLayer.name)
if str(self.getToposort()).find("float64") > -1:
warningMsg = True
msg += "\n\nBut there are some float64s that do not fit on the GPU and will slow down the computations.\nPlease consider:"
msg += "\n\t* Launching with THEANO_FLAGS=device=gpu,floatX=float32 python <your script>.py."
msg += "\n\t* If you have any dmatrix, dvector or dscalar in your code replace them with matrix, vector, scalar."
else:
device = "CPU"
msg = "I will use the [-%s-] to run function '%s' of layer '%s'!" % (
device, name, outputLayer.name)
self.results = OrderedDict()
MCAN.friendly("Run device", msg, warning=warningMsg)
def start(self, runName, saveIfMurdered=False, **kwargs):
"""Starts the training and encapsulates it into a safe environement.
If the training stops because of an Exception or SIGTEM, the trainer
will save logs, the store, and the last version of the model.
"""
import simplejson, signal, cPickle
def _handler_sig_term(sig, frame):
_dieGracefully("SIGTERM", None)
sys.exit(sig)
def _dieGracefully(exType, tb=None):
if type(exType) is types.StringType:
exName = exType
else:
exName = exType.__name__
death_time = time.ctime().replace(' ', '_')
filename = "dx-xb_" + runName + "_death_by_" + exName + "_" + death_time
sys.stderr.write(
"\n===\nDying gracefully from %s, and saving myself to:\n...%s\n===\n"
% (exName, filename))
self.model.save(filename)
f = open(filename + ".traceback.log", 'w')
f.write(
"Mariana training Interruption\n=============================\n"
)
f.write("\nDetails\n-------\n")
f.write("Name: %s\n" % runName)
f.write("pid: %s\n" % os.getpid())
f.write("Killed by: %s\n" % str(exType))
f.write("Time of death: %s\n" % death_time)
f.write("Model saved to: %s\n" % filename)
if tb is not None:
f.write("\nTraceback\n---------\n")
f.write(
str(traceback.extract_tb(tb)).replace(
"), (", "),\n(").replace("[(",
"[\n(").replace(")]", ")\n]"))
f.flush()
f.close()
f = open(filename + ".store.pkl", "wb")
cPickle.dump(self.store, f)
f.close()
signal.signal(signal.SIGTERM, _handler_sig_term)
if MSET.VERBOSE:
print "\n" + "Training starts."
MCAN.friendly("Process id", "The pid of this run is: %d" % os.getpid())
try:
return self.run(runName, **kwargs)
except MSTOP.EndOfTraining as e:
print e.message
death_time = time.ctime().replace(' ', '_')
filename = "finished_" + runName + "_" + death_time
f = open(filename + ".stopreason.txt", 'w')
f.write("Name: %s\n" % runName)
f.write("pid: %s\n" % os.getpid())
f.write("Time of death: %s\n" % death_time)
f.write("Epoch of death: %s\n" % self.store["runInfos"]["epoch"])
f.write("Stopped by: %s\n" % e.stopCriterion.name)
f.write("Reason: %s\n" % e.message)
f.flush()
f.close()
model.save(filename)
f = open(filename + ".store.pkl", "wb")
cPickle.dump(self.store, f)
f.close()
except KeyboardInterrupt:
if not saveIfMurdered:
raise
exType, ex, tb = sys.exc_info()
_dieGracefully(exType, tb)
raise
except:
if not saveIfMurdered:
raise
exType, ex, tb = sys.exc_info()
_dieGracefully(exType, tb)
raise
