def commit(self, store, model):
"""Appends the current state of the store to the CSV. This one is meant to be called by the trainer"""
def _fillLine(csvFile, score, minScore, maxScore, mapName, setLen,
outputName, outputFunction, **csvValues):
line = csvFile.newLine()
for k, v in csvValues.iteritems():
line[k] = v
line["score"] = score[0]
line["min_score"] = minScore[0]
line["min_score_commit"] = minScore[1]
line["max_score"] = maxScore[0]
line["max_score_commit"] = maxScore[1]
line["set"] = "%s" % (mapName)
line["set_size"] = "%s" % (setLen)
line["output_layer"] = outputName
line["output_function"] = outputFunction
line.commit()
self.length += 1
if self.csvLegend is None:
self.csvLegend = store["hyperParameters"].keys()
self.csvLegend.extend(store["runInfos"].keys())
self.csvLegend.extend([
"score", "min_score", "min_score_commit", "max_score",
"max_score_commit", "set", "set_size", "output_layer",
"output_function"
])
self.csvFile = CSVFile(legend=self.csvLegend, separator="\t")
self.csvFile.streamToFile(self.filename, writeRate=self.writeRate)
muchData = store["hyperParameters"]
muchData.update(store["runInfos"])
self.scores.update(store["scores"], store["runInfos"]["epoch"])
for mapName, os in store["scores"].iteritems():
for outputName, fs in os.iteritems():
for functionName in fs:
_fillLine(
self.csvFile,
self.scores.getScore(mapName, outputName,
functionName),
self.scores.getMinScore(mapName, outputName,
functionName),
self.scores.getMaxScore(mapName, outputName,
functionName), mapName,
store["setSizes"][mapName], outputName, functionName,
**muchData)
if self.printRate > 0 and (self.length % self.printRate) == 0:
self.printCurrentState()
for w in self.whenToSave:
if w._shouldISave(self):
model.save(w.getFilename(self))
def _importSNPs_AgnosticSNP(setName, species, genomeSource, snpsFile):
"This function will also create an index on start->chromosomeNumber->setName. Warning : pyGeno wil interpret all positions as 0 based"
printf('importing SNP set %s for species %s...' % (setName, species))
snpData = CSVFile()
snpData.parse(snpsFile, separator="\t")
AgnosticSNP.dropIndex(('start', 'chromosomeNumber', 'setName'))
conf.db.beginTransaction()
pBar = ProgressBar(len(snpData))
pLabel = ''
currChrNumber = None
for snpEntry in snpData:
tmpChr = snpEntry['chromosomeNumber']
if tmpChr != currChrNumber:
currChrNumber = tmpChr
pLabel = 'Chr %s...' % currChrNumber
snp = AgnosticSNP()
snp.species = species
snp.setName = setName
for f in snp.getFields():
try:
setattr(snp, f, snpEntry[f])
except KeyError:
if f != 'species' and f != 'setName':
printf("Warning filetype as no key %s", f)
snp.quality = float(snp.quality)
snp.start = int(snp.start)
snp.end = int(snp.end)
snp.save()
pBar.update(label=pLabel)
pBar.close()
snpMaster = SNPMaster()
snpMaster.set(setName=setName, SNPType='AgnosticSNP', species=species)
snpMaster.save()
printf('saving...')
conf.db.endTransaction()
printf('creating indexes...')
AgnosticSNP.ensureGlobalIndex(('start', 'chromosomeNumber', 'setName'))
printf('importation of SNP set %s for species %s done.' %
(setName, species))
return True
def commit(self, store, model):
"""Appends the current state of the store to the CSV. This one is meant to be called by the trainer"""
def _fillLine(csvFile, score, bestScore, setName, setLen, outputName,
**csvValues):
line = csvFile.newLine()
for k, v in csvValues.iteritems():
line[k] = v
line["score"] = score
line["best_score"] = bestScore[0]
line["best_score_commit"] = bestScore[1]
line["set"] = "%s(%s)" % (setName, setLen)
line["output"] = outputName
line.commit()
self.length += 1
if self.csvLegend is None:
self.csvLegend = store["hyperParameters"].keys()
self.csvLegend.extend(store["runInfos"].keys())
self.csvLegend.extend(
["score", "best_score", "best_score_commit", "set", "output"])
self.csvFile = CSVFile(legend=self.csvLegend)
self.csvFile.streamToFile(self.filename, writeRate=self.writeRate)
for theSet, scores in store["scores"].iteritems():
self.currentScores[theSet] = {}
if theSet not in self.bestScores:
self.bestScores[theSet] = {}
for outputName, score in scores.iteritems():
self.currentScores[theSet][outputName] = score
if outputName not in self.bestScores[
theSet] or score < self.bestScores[theSet][outputName][
0]:
self.bestScores[theSet][outputName] = (score, self.length)
model.save(self.getBestModelFilename(outputName, theSet))
muchData = store["hyperParameters"]
muchData.update(store["runInfos"])
_fillLine(self.csvFile, self.currentScores[theSet][outputName],
self.bestScores[theSet][outputName], theSet,
store["setSizes"][theSet], outputName, **muchData)
if self.printRate > 0 and (self.length % self.printRate) == 0:
self.printCurrentState()
def commit(self, trainer):
""""""
if self.csvFile is not None:
line = self.csvFile.newLine()
else:
line = {}
for log in self.loggers:
for k, v in log.log(trainer):
line[k] = v
if self.csvFile is None:
self.csvFile = CSVFile(legend=line.keys(),
separator=self.separator)
self.csvFile.streamToFile(self.filename, writeRate=self.writeRate)
newLine = self.csvFile.newLine()
for k, v in line.iteritems():
newLine[k] = v
line = newLine
self.nbCommits += 1
line.commit()
