def commit(self, store, model): """Appends the current state of the store to the CSV. This one is meant to be called by the trainer""" def _fillLine(csvFile, score, minScore, maxScore, mapName, setLen, outputName, outputFunction, **csvValues): line = csvFile.newLine() for k, v in csvValues.iteritems(): line[k] = v line["score"] = score[0] line["min_score"] = minScore[0] line["min_score_commit"] = minScore[1] line["max_score"] = maxScore[0] line["max_score_commit"] = maxScore[1] line["set"] = "%s" % (mapName) line["set_size"] = "%s" % (setLen) line["output_layer"] = outputName line["output_function"] = outputFunction line.commit() self.length += 1 if self.csvLegend is None: self.csvLegend = store["hyperParameters"].keys() self.csvLegend.extend(store["runInfos"].keys()) self.csvLegend.extend([ "score", "min_score", "min_score_commit", "max_score", "max_score_commit", "set", "set_size", "output_layer", "output_function" ]) self.csvFile = CSVFile(legend=self.csvLegend, separator="\t") self.csvFile.streamToFile(self.filename, writeRate=self.writeRate) muchData = store["hyperParameters"] muchData.update(store["runInfos"]) self.scores.update(store["scores"], store["runInfos"]["epoch"]) for mapName, os in store["scores"].iteritems(): for outputName, fs in os.iteritems(): for functionName in fs: _fillLine( self.csvFile, self.scores.getScore(mapName, outputName, functionName), self.scores.getMinScore(mapName, outputName, functionName), self.scores.getMaxScore(mapName, outputName, functionName), mapName, store["setSizes"][mapName], outputName, functionName, **muchData) if self.printRate > 0 and (self.length % self.printRate) == 0: self.printCurrentState() for w in self.whenToSave: if w._shouldISave(self): model.save(w.getFilename(self))
def _importSNPs_AgnosticSNP(setName, species, genomeSource, snpsFile): "This function will also create an index on start->chromosomeNumber->setName. Warning : pyGeno wil interpret all positions as 0 based" printf('importing SNP set %s for species %s...' % (setName, species)) snpData = CSVFile() snpData.parse(snpsFile, separator="\t") AgnosticSNP.dropIndex(('start', 'chromosomeNumber', 'setName')) conf.db.beginTransaction() pBar = ProgressBar(len(snpData)) pLabel = '' currChrNumber = None for snpEntry in snpData: tmpChr = snpEntry['chromosomeNumber'] if tmpChr != currChrNumber: currChrNumber = tmpChr pLabel = 'Chr %s...' % currChrNumber snp = AgnosticSNP() snp.species = species snp.setName = setName for f in snp.getFields(): try: setattr(snp, f, snpEntry[f]) except KeyError: if f != 'species' and f != 'setName': printf("Warning filetype as no key %s", f) snp.quality = float(snp.quality) snp.start = int(snp.start) snp.end = int(snp.end) snp.save() pBar.update(label=pLabel) pBar.close() snpMaster = SNPMaster() snpMaster.set(setName=setName, SNPType='AgnosticSNP', species=species) snpMaster.save() printf('saving...') conf.db.endTransaction() printf('creating indexes...') AgnosticSNP.ensureGlobalIndex(('start', 'chromosomeNumber', 'setName')) printf('importation of SNP set %s for species %s done.' % (setName, species)) return True
def commit(self, store, model): """Appends the current state of the store to the CSV. This one is meant to be called by the trainer""" def _fillLine(csvFile, score, bestScore, setName, setLen, outputName, **csvValues): line = csvFile.newLine() for k, v in csvValues.iteritems(): line[k] = v line["score"] = score line["best_score"] = bestScore[0] line["best_score_commit"] = bestScore[1] line["set"] = "%s(%s)" % (setName, setLen) line["output"] = outputName line.commit() self.length += 1 if self.csvLegend is None: self.csvLegend = store["hyperParameters"].keys() self.csvLegend.extend(store["runInfos"].keys()) self.csvLegend.extend( ["score", "best_score", "best_score_commit", "set", "output"]) self.csvFile = CSVFile(legend=self.csvLegend) self.csvFile.streamToFile(self.filename, writeRate=self.writeRate) for theSet, scores in store["scores"].iteritems(): self.currentScores[theSet] = {} if theSet not in self.bestScores: self.bestScores[theSet] = {} for outputName, score in scores.iteritems(): self.currentScores[theSet][outputName] = score if outputName not in self.bestScores[ theSet] or score < self.bestScores[theSet][outputName][ 0]: self.bestScores[theSet][outputName] = (score, self.length) model.save(self.getBestModelFilename(outputName, theSet)) muchData = store["hyperParameters"] muchData.update(store["runInfos"]) _fillLine(self.csvFile, self.currentScores[theSet][outputName], self.bestScores[theSet][outputName], theSet, store["setSizes"][theSet], outputName, **muchData) if self.printRate > 0 and (self.length % self.printRate) == 0: self.printCurrentState()
def commit(self, trainer): """""" if self.csvFile is not None: line = self.csvFile.newLine() else: line = {} for log in self.loggers: for k, v in log.log(trainer): line[k] = v if self.csvFile is None: self.csvFile = CSVFile(legend=line.keys(), separator=self.separator) self.csvFile.streamToFile(self.filename, writeRate=self.writeRate) newLine = self.csvFile.newLine() for k, v in line.iteritems(): newLine[k] = v line = newLine self.nbCommits += 1 line.commit()