def start(self, runName, model, recorder, *args, **kwargs): """Starts the training and encapsulates it into a safe environement. If the training stops because of an Exception or SIGTEM, the trainer will save logs, the store, and the last version of the model. """ import simplejson, signal, cPickle def _handler_sig_term(sig, frame): _dieGracefully("SIGTERM", None) sys.exit(sig) def _dieGracefully(exType, tb=None): if type(exType) is types.StringType: exName = exType else: exName = exType.__name__ death_time = time.ctime().replace(' ', '_') filename = "dx-xb_" + runName + "_death_by_" + exName + "_" + death_time sys.stderr.write( "\n===\nDying gracefully from %s, and saving myself to:\n...%s\n===\n" % (exName, filename)) model.save(filename) f = open(filename + ".traceback.log", 'w') f.write( "Mariana training Interruption\n=============================\n" ) f.write("\nDetails\n-------\n") f.write("Name: %s\n" % runName) f.write("pid: %s\n" % os.getpid()) f.write("Killed by: %s\n" % str(exType)) f.write("Time of death: %s\n" % death_time) f.write("Model saved to: %s\n" % filename) sstore = str(self.store).replace("'", '"').replace( "True", 'true').replace("False", 'false') try: f.write("store:\n%s" % json.dumps(json.loads(sstore), sort_keys=True, indent=4, separators=(',', ': '))) except Exception as e: print "Warning: Couldn't format the store to json, saving it ugly." print "Reason:", e f.write(sstore) if tb is not None: f.write("\nTraceback\n---------\n") f.write( str(traceback.extract_tb(tb)).replace( "), (", "),\n(").replace("[(", "[\n(").replace(")]", ")\n]")) f.flush() f.close() f = open(filename + ".store.pkl", "wb") cPickle.dump(self.store, f) f.close() signal.signal(signal.SIGTERM, _handler_sig_term) if MSET.VERBOSE: print "\n" + "Training starts." MCAN.friendly("Process id", "The pid of this run is: %d" % os.getpid()) if recorder == "default": params = {"printRate": 1, "writeRate": 1} recorder = MREC.GGPlot2(runName, **params) MCAN.friendly( "Default recorder", "The trainer will recruit the default 'GGPlot2' recorder with the following arguments:\n\t %s.\nResults will be saved into '%s'." % (params, recorder.filename)) try: return self.run(runName, model, recorder, *args, **kwargs) except MSTOP.EndOfTraining as e: print e.message death_time = time.ctime().replace(' ', '_') filename = "finished_" + runName + "_" + death_time f = open(filename + ".stopreason.txt", 'w') f.write("Name: %s\n" % runName) f.write("pid: %s\n" % os.getpid()) f.write("Time of death: %s\n" % death_time) f.write("Epoch of death: %s\n" % self.store["runInfos"]["epoch"]) f.write("Stopped by: %s\n" % e.stopCriterion.name) f.write("Reason: %s\n" % e.message) sstore = str(self.store).replace("'", '"').replace( "True", 'true').replace("False", 'false') try: f.write("store:\n%s" % json.dumps(json.loads(sstore), sort_keys=True, indent=4, separators=(',', ': '))) except Exception as e: print "Warning: Couldn't format the store to json, saving it ugly." print "Reason:", e f.write(sstore) f.flush() f.close() model.save(filename) f = open(filename + ".store.pkl", "wb") cPickle.dump(self.store, f) f.close() except KeyboardInterrupt: if not self.saveIfMurdered: raise exType, ex, tb = sys.exc_info() _dieGracefully(exType, tb) raise except: if not self.saveIfMurdered: raise exType, ex, tb = sys.exc_info() _dieGracefully(exType, tb) raise
testMaps.mapOutput(o, testData.numbers) validationData = MDM.Series(images=validation_set[0], numbers=validation_set[1]) validationMaps = MDM.DatasetMapper() validationMaps.mapInput(i, validationData.images) validationMaps.mapOutput(o, validationData.numbers) earlyStop = MSTOP.GeometricEarlyStopping(testMaps, patience=100, patienceIncreaseFactor=1.1, significantImprovement=0.00001, outputFunction="score", outputLayer=o) epochWall = MSTOP.EpochWall(1000) trainer = MT.DefaultTrainer(trainMaps=trainMaps, testMaps=testMaps, validationMaps=validationMaps, stopCriteria=[earlyStop, epochWall], testFunctionName="testAndAccuracy", validationFunctionName="testAndAccuracy", trainMiniBatchSize=20, saveIfMurdered=False) recorder = MREC.GGPlot2("MLP", whenToSave=[MREC.SaveMin("test", o.name, "score")], printRate=1, writeRate=1) trainer.start("MLP", mlp, recorder=recorder)