def __init__(self, name, outputLayer, output_expressions, additional_input_expressions = {}, updates = [], **kwargs) : """ :param str name: name of the function :param Output outputLayer: the output layer the function should be applied to :param list output_expressions: list of the symbolic expressions you want as output :param dict additional_input_expressions: additional inputs needed to compute the expressions :param list updates: list of tuples (shared variable, symbolic expression of the update to be applied to it) :param dict **kwargs: additional arguments to passed to the real theano function underneath """ self.name = name self.outputLayer = outputLayer self.inputs = OrderedDict() self.tmpInputs = OrderedDict() for inp in self.outputLayer.network.inputs.itervalues() : self.inputs[inp.name] = inp.outputs self.inputs.update(additional_input_expressions) for i in self.inputs : self.tmpInputs[i] = None self.additional_input_expressions = additional_input_expressions self.outputs = output_expressions self.updates = updates self.theano_fct = theano.function(inputs = self.inputs.values(), outputs = self.outputs, updates = self.updates, **kwargs) if any([x.__class__.__name__.lower().find("gpu") for x in self.theano_fct.maker.fgraph.toposort()]): device = "GPU" else: device = "CPU" if MSET.VERBOSE : MCAN.friendly("Run device", "I will use the [-%s-] to run function '%s' of layer '%s'!" % (device, name, outputLayer.name))
def __init__(self, name, outputLayer, output_expressions, additional_input_expressions = {}, updates = [], **kwargs) : """ :param str name: name of the function :param Output outputLayer: the output layer the function should be applied to :param list output_expressions: list of tuples of symbolic expressions you want as output and the names you want to give to them: (name, expressions) :param dict additional_input_expressions: additional inputs needed to compute the expressions :param list updates: list of tuples (shared variable, symbolic expression of the update to be applied to it) :param dict \*\*kwargs: additional arguments to passed to the real theano function underneath """ self.cast_warning_told = False self.name = name self.outputLayer = outputLayer self.inputs = OrderedDict() inpSet = set() for inp in self.outputLayer.network.inputs.itervalues() : inpOut = inp.inputs if inpOut not in inpSet : self.inputs[inp.name] = inpOut inpSet.add(inpOut) for k, v in additional_input_expressions.iteritems() : if v not in inpSet : self.inputs[k] = v inpSet.add(v) self.fctInputs = OrderedDict() for i in self.inputs : self.fctInputs[i] = None self.additional_input_expressions = additional_input_expressions self.output_expressions = OrderedDict() for name, output_expr in output_expressions : self.output_expressions[name] = output_expr self.updates = updates self.theano_fct = theano.function(inputs = self.inputs.values(), outputs = self.output_expressions.values(), updates = self.updates, **kwargs) warningMsg = False if DEVICE_IS_GPU : device = "GPU" msg = "I will use the [-%s-] to run function '%s' of layer '%s'!" % (device, name, outputLayer.name) if str(self.getToposort()).find("float64") > -1: warningMsg = True msg += "\n\nBut there are some float64s that do not fit on the GPU and will slow down the computations.\nPlease consider:" msg += "\n\t* Launching with THEANO_FLAGS=device=gpu,floatX=float32 python <your script>.py." msg += "\n\t* If you have any dmatrix, dvector or dscalar in your code replace them with matrix, vector, scalar." else: device = "CPU" msg = "I will use the [-%s-] to run function '%s' of layer '%s'!" % (device, name, outputLayer.name) self.results = OrderedDict() MCAN.friendly("Run device", msg, warning = warningMsg)
def __init__(self, name, applicationType, outputLayer, output_expressions, additional_input_expressions = {}, updates = [], **kwargs) : """ :param str name: name of the function :param Output outputLayer: the output layer the function should be applied to :param list output_expressions: list of the symbolic expressions you want as output :param dict additional_input_expressions: additional inputs needed to compute the expressions :param list updates: list of tuples (shared variable, symbolic expression of the update to be applied to it) :param dict **kwargs: additional arguments to passed to the real theano function underneath """ self.cast_warning_told = False self.name = name self.outputLayer = outputLayer self.applicationType = applicationType.lower() self.inputs = OrderedDict() inpSet = set() for inp in self.outputLayer.network.inputs.itervalues() : if self.applicationType == TYPE_TEST : inpOut = inp.test_outputs elif self.applicationType == TYPE_TRAIN : inpOut = inp.outputs else : raise AttributeError('Unknow applicationType %s' % applicationType) if inpOut not in inpSet : self.inputs[inp.name] = inpOut inpSet.add(inpOut) for k, v in additional_input_expressions.iteritems() : if v not in inpSet : self.inputs[k] = v inpSet.add(v) self.fctInputs = OrderedDict() for i in self.inputs : self.fctInputs[i] = None self.additional_input_expressions = additional_input_expressions self.outputs = output_expressions self.updates = updates self.theano_fct = theano.function(inputs = self.inputs.values(), outputs = self.outputs, updates = self.updates, **kwargs) if theano.config.device.find("gpu") > -1: device = "GPU" else: device = "CPU" MCAN.friendly("Run device", "I will use the [-%s-] to run the *%s* type function '%s' of layer '%s'!" % (device, self.applicationType, name, outputLayer.name))
def __init__(self, name, outputLayer, output_expressions, additional_input_expressions={}, updates=[], **kwargs): """ :param str name: name of the function :param Output outputLayer: the output layer the function should be applied to :param list output_expressions: list of the symbolic expressions you want as output :param dict additional_input_expressions: additional inputs needed to compute the expressions :param list updates: list of tuples (shared variable, symbolic expression of the update to be applied to it) :param dict \*\*kwargs: additional arguments to passed to the real theano function underneath """ self.cast_warning_told = False self.name = name self.outputLayer = outputLayer self.inputs = OrderedDict() inpSet = set() for inp in self.outputLayer.network.inputs.itervalues(): inpOut = inp.inputs if inpOut not in inpSet: self.inputs[inp.name] = inpOut inpSet.add(inpOut) for k, v in additional_input_expressions.iteritems(): if v not in inpSet: self.inputs[k] = v inpSet.add(v) self.fctInputs = OrderedDict() for i in self.inputs: self.fctInputs[i] = None self.additional_input_expressions = additional_input_expressions self.outputs = output_expressions self.updates = updates self.theano_fct = theano.function( inputs=self.inputs.values(), outputs=self.outputs, updates=self.updates, **kwargs ) if DEVICE_IS_GPU > -1: device = "GPU" else: device = "CPU" MCAN.friendly( "Run device", "I will use the [-%s-] to run function '%s' of layer '%s'!" % (device, name, outputLayer.name) )
def start(self, runName, model, recorder, *args, **kwargs): """Starts the training and encapsulates it into a safe environement. If the training stops because of an Exception or SIGTEM, the trainer will save logs, the store, and the last version of the model. """ import simplejson, signal, cPickle def _handler_sig_term(sig, frame): _dieGracefully("SIGTERM", None) sys.exit(sig) def _dieGracefully(exType, tb=None): if type(exType) is types.StringType: exName = exType else: exName = exType.__name__ death_time = time.ctime().replace(' ', '_') filename = "dx-xb_" + runName + "_death_by_" + exName + "_" + death_time sys.stderr.write( "\n===\nDying gracefully from %s, and saving myself to:\n...%s\n===\n" % (exName, filename)) model.save(filename) f = open(filename + ".traceback.log", 'w') f.write( "Mariana training Interruption\n=============================\n" ) f.write("\nDetails\n-------\n") f.write("Name: %s\n" % runName) f.write("pid: %s\n" % os.getpid()) f.write("Killed by: %s\n" % str(exType)) f.write("Time of death: %s\n" % death_time) f.write("Model saved to: %s\n" % filename) sstore = str(self.store).replace("'", '"').replace( "True", 'true').replace("False", 'false') try: f.write("store:\n%s" % json.dumps(json.loads(sstore), sort_keys=True, indent=4, separators=(',', ': '))) except Exception as e: print "Warning: Couldn't format the store to json, saving it ugly." print "Reason:", e f.write(sstore) if tb is not None: f.write("\nTraceback\n---------\n") f.write( str(traceback.extract_tb(tb)).replace( "), (", "),\n(").replace("[(", "[\n(").replace(")]", ")\n]")) f.flush() f.close() f = open(filename + ".store.pkl", "wb") cPickle.dump(self.store, f) f.close() signal.signal(signal.SIGTERM, _handler_sig_term) if MSET.VERBOSE: print "\n" + "Training starts." MCAN.friendly("Process id", "The pid of this run is: %d" % os.getpid()) if recorder == "default": params = {"printRate": 1, "writeRate": 1} recorder = MREC.GGPlot2(runName, **params) MCAN.friendly( "Default recorder", "The trainer will recruit the default 'GGPlot2' recorder with the following arguments:\n\t %s.\nResults will be saved into '%s'." % (params, recorder.filename)) try: return self.run(runName, model, recorder, *args, **kwargs) except MSTOP.EndOfTraining as e: print e.message death_time = time.ctime().replace(' ', '_') filename = "finished_" + runName + "_" + death_time f = open(filename + ".stopreason.txt", 'w') f.write("Name: %s\n" % runName) f.write("pid: %s\n" % os.getpid()) f.write("Time of death: %s\n" % death_time) f.write("Epoch of death: %s\n" % self.store["runInfos"]["epoch"]) f.write("Stopped by: %s\n" % e.stopCriterion.name) f.write("Reason: %s\n" % e.message) sstore = str(self.store).replace("'", '"').replace( "True", 'true').replace("False", 'false') try: f.write("store:\n%s" % json.dumps(json.loads(sstore), sort_keys=True, indent=4, separators=(',', ': '))) except Exception as e: print "Warning: Couldn't format the store to json, saving it ugly." print "Reason:", e f.write(sstore) f.flush() f.close() model.save(filename) f = open(filename + ".store.pkl", "wb") cPickle.dump(self.store, f) f.close() except KeyboardInterrupt: if not self.saveIfMurdered: raise exType, ex, tb = sys.exc_info() _dieGracefully(exType, tb) raise except: if not self.saveIfMurdered: raise exType, ex, tb = sys.exc_info() _dieGracefully(exType, tb) raise
def start(self, runName, model, recorder, *args, **kwargs) : """Starts the training and encapsulates it into a safe environement. If the training stops because of an Exception or SIGTEM, the trainer will save logs, the store, and the last version of the model. """ import simplejson, signal, cPickle def _handler_sig_term(sig, frame) : _dieGracefully("SIGTERM", None) sys.exit(sig) def _dieGracefully(exType, tb = None) : if type(exType) is types.StringType : exName = exType else : exName = exType.__name__ death_time = time.ctime().replace(' ', '_') filename = "dx-xb_" + runName + "_death_by_" + exName + "_" + death_time sys.stderr.write("\n===\nDying gracefully from %s, and saving myself to:\n...%s\n===\n" % (exName, filename)) model.save(filename) f = open(filename + ".traceback.log", 'w') f.write("Mariana training Interruption\n=============================\n") f.write("\nDetails\n-------\n") f.write("Name: %s\n" % runName) f.write("pid: %s\n" % os.getpid()) f.write("Killed by: %s\n" % str(exType)) f.write("Time of death: %s\n" % death_time) f.write("Model saved to: %s\n" % filename) sstore = str(self.store).replace("'", '"').replace("True", 'true').replace("False", 'false') try : f.write( "store:\n%s" % json.dumps( json.loads(sstore), sort_keys=True, indent=4, separators=(',', ': ') ) ) except Exception as e: print "Warning: Couldn't format the store to json, saving it ugly." print "Reason:", e f.write(sstore) if tb is not None : f.write("\nTraceback\n---------\n") f.write(str(traceback.extract_tb(tb)).replace("), (", "),\n(").replace("[(","[\n(").replace(")]",")\n]")) f.flush() f.close() f = open(filename + ".store.pkl", "wb") cPickle.dump(self.store, f) f.close() signal.signal(signal.SIGTERM, _handler_sig_term) if MSET.VERBOSE : print "\n" + "Training starts." MCAN.friendly("Process id", "The pid of this run is: %d" % os.getpid()) if recorder == "default" : params = { "printRate" : 1, "writeRate" : 1 } recorder = MREC.GGPlot2(runName, **params) MCAN.friendly( "Default recorder", "The trainer will recruit the default 'GGPlot2' recorder with the following arguments:\n\t %s.\nResults will be saved into '%s'." % (params, recorder.filename) ) try : return self.run(runName, model, recorder, *args, **kwargs) except MSTOP.EndOfTraining as e : print e.message death_time = time.ctime().replace(' ', '_') filename = "finished_" + runName + "_" + death_time f = open(filename + ".stopreason.txt", 'w') f.write("Name: %s\n" % runName) f.write("pid: %s\n" % os.getpid()) f.write("Time of death: %s\n" % death_time) f.write("Epoch of death: %s\n" % self.store["runInfos"]["epoch"]) f.write("Stopped by: %s\n" % e.stopCriterion.name) f.write("Reason: %s\n" % e.message) sstore = str(self.store).replace("'", '"').replace("True", 'true').replace("False", 'false') try : f.write( "store:\n%s" % json.dumps( json.loads(sstore), sort_keys=True, indent=4, separators=(',', ': ') ) ) except Exception as e: print "Warning: Couldn't format the store to json, saving it ugly." print "Reason:", e f.write(sstore) f.flush() f.close() model.save(filename) f = open(filename + ".store.pkl", "wb") cPickle.dump(self.store, f) f.close() except KeyboardInterrupt : if not self.saveIfMurdered : raise exType, ex, tb = sys.exc_info() _dieGracefully(exType, tb) raise except : if not self.saveIfMurdered : raise exType, ex, tb = sys.exc_info() _dieGracefully(exType, tb) raise
def __init__(self, name, outputLayer, output_expressions, additional_input_expressions={}, updates=[], **kwargs): """ :param str name: name of the function :param Output outputLayer: the output layer the function should be applied to :param list output_expressions: list of tuples of symbolic expressions you want as output and the names you want to give to them: (name, expressions) :param dict additional_input_expressions: additional inputs needed to compute the expressions :param list updates: list of tuples (shared variable, symbolic expression of the update to be applied to it) :param dict \*\*kwargs: additional arguments to passed to the real theano function underneath """ def _bckTrckInputs(startLayer, inputs=OrderedDict(), inpSet=set()): if len(startLayer.network.inConnections[startLayer]) == 0: inpOut = startLayer.inputs if inpOut not in inpSet: inputs[startLayer.name] = inpOut inpSet.add(inpOut) else: for layer in startLayer.network.inConnections[startLayer]: _bckTrckInputs(layer, inputs, inpSet) return inputs, inpSet self.cast_warning_told = False self.name = name self.outputLayer = outputLayer self.inputs, inpSet = _bckTrckInputs(outputLayer) for k, v in additional_input_expressions.iteritems(): if v not in inpSet: self.inputs[k] = v inpSet.add(v) self.fctInputs = OrderedDict() for i in self.inputs: self.fctInputs[i] = None self.additional_input_expressions = additional_input_expressions self.output_expressions = OrderedDict() for name, output_expr in output_expressions: self.output_expressions[name] = output_expr self.updates = updates self.theano_fct = theano.function( inputs=self.inputs.values(), outputs=self.output_expressions.values(), updates=self.updates, **kwargs) warningMsg = False if DEVICE_IS_GPU: device = "GPU" msg = "I will use the [-%s-] to run function '%s' of layer '%s'!" % ( device, name, outputLayer.name) if str(self.getToposort()).find("float64") > -1: warningMsg = True msg += "\n\nBut there are some float64s that do not fit on the GPU and will slow down the computations.\nPlease consider:" msg += "\n\t* Launching with THEANO_FLAGS=device=gpu,floatX=float32 python <your script>.py." msg += "\n\t* If you have any dmatrix, dvector or dscalar in your code replace them with matrix, vector, scalar." else: device = "CPU" msg = "I will use the [-%s-] to run function '%s' of layer '%s'!" % ( device, name, outputLayer.name) self.results = OrderedDict() MCAN.friendly("Run device", msg, warning=warningMsg)
def start(self, runName, saveIfMurdered=False, **kwargs): """Starts the training and encapsulates it into a safe environement. If the training stops because of an Exception or SIGTEM, the trainer will save logs, the store, and the last version of the model. """ import simplejson, signal, cPickle def _handler_sig_term(sig, frame): _dieGracefully("SIGTERM", None) sys.exit(sig) def _dieGracefully(exType, tb=None): if type(exType) is types.StringType: exName = exType else: exName = exType.__name__ death_time = time.ctime().replace(' ', '_') filename = "dx-xb_" + runName + "_death_by_" + exName + "_" + death_time sys.stderr.write( "\n===\nDying gracefully from %s, and saving myself to:\n...%s\n===\n" % (exName, filename)) self.model.save(filename) f = open(filename + ".traceback.log", 'w') f.write( "Mariana training Interruption\n=============================\n" ) f.write("\nDetails\n-------\n") f.write("Name: %s\n" % runName) f.write("pid: %s\n" % os.getpid()) f.write("Killed by: %s\n" % str(exType)) f.write("Time of death: %s\n" % death_time) f.write("Model saved to: %s\n" % filename) if tb is not None: f.write("\nTraceback\n---------\n") f.write( str(traceback.extract_tb(tb)).replace( "), (", "),\n(").replace("[(", "[\n(").replace(")]", ")\n]")) f.flush() f.close() f = open(filename + ".store.pkl", "wb") cPickle.dump(self.store, f) f.close() signal.signal(signal.SIGTERM, _handler_sig_term) if MSET.VERBOSE: print "\n" + "Training starts." MCAN.friendly("Process id", "The pid of this run is: %d" % os.getpid()) try: return self.run(runName, **kwargs) except MSTOP.EndOfTraining as e: print e.message death_time = time.ctime().replace(' ', '_') filename = "finished_" + runName + "_" + death_time f = open(filename + ".stopreason.txt", 'w') f.write("Name: %s\n" % runName) f.write("pid: %s\n" % os.getpid()) f.write("Time of death: %s\n" % death_time) f.write("Epoch of death: %s\n" % self.store["runInfos"]["epoch"]) f.write("Stopped by: %s\n" % e.stopCriterion.name) f.write("Reason: %s\n" % e.message) f.flush() f.close() model.save(filename) f = open(filename + ".store.pkl", "wb") cPickle.dump(self.store, f) f.close() except KeyboardInterrupt: if not saveIfMurdered: raise exType, ex, tb = sys.exc_info() _dieGracefully(exType, tb) raise except: if not saveIfMurdered: raise exType, ex, tb = sys.exc_info() _dieGracefully(exType, tb) raise