def procureModel(self):
if self.options.resume_from_checkpoint:
model = ModelTemplate.resume(self.options.resume_from_checkpoint)
self.sequitur = model.sequitur
elif self.options.modelFile:
if sys.version_info[:2] >= (3, 0):
model = pickle.load(open(self.options.modelFile, 'rb'), encoding='latin1')
else:
try:
model = pickle.load(open(self.options.modelFile, 'rb'))
except ValueError:
print('This error most likely occured because the loaded model was created in python3.\n', file=sys.stderr)
raise
self.sequitur = model.sequitur
else:
self.sequitur = Sequitur()
model = None
if self.options.shouldRampUp:
model.rampUp()
if self.options.trainSample:
model = self.trainModel(model)
if not model:
print('failed to estimate or load model', file=self.log)
return
if not model:
raise UsageError
# model.sequenceModel.showMostProbable(sys.stdout, model.sequitur.symbol, limit=250)
if self.options.shouldTranspose:
model.transpose()
if self.options.newModelFile:
oldSize, newSize = model.strip()
print('stripped number of multigrams from %d to %d' % (oldSize, newSize), file=self.log)
f = open(self.options.newModelFile, 'wb')
pickle.dump(model, f, pickle.HIGHEST_PROTOCOL)
f.close()
del f
if self.options.shouldSelfTest:
print('warning: --self-test does not treat pronunciation variants correctly', file=self.log)
if not self.develSample:
print('error: cannot do --self-test without --devel sample', file=self.log)
else:
translator = Translator(model)
evaluator = Evaluator()
evaluator.setSample(self.develSample)
evaluator.verboseLog = self.log
result = evaluator.evaluate(translator)
print(result, file=self.log)
return model
def procureModel(self):
if self.options.resume_from_checkpoint:
model = ModelTemplate.resume(self.options.resume_from_checkpoint)
self.sequitur = model.sequitur
elif self.options.modelFile:
model = pickle.load(open(self.options.modelFile, "rb"))
self.sequitur = model.sequitur
else:
self.sequitur = Sequitur()
model = None
if self.options.shouldRampUp:
model.rampUp()
if self.options.trainSample:
model = self.trainModel(model)
if not model:
print('failed to estimate or load model', file=self.log)
return
if not model:
raise UsageError
# model.sequenceModel.showMostProbable(sys.stdout, model.sequitur.symbol, limit=250)
if self.options.shouldTranspose:
model.transpose()
if self.options.newModelFile:
oldSize, newSize = model.strip()
print('stripped number of multigrams from %d to %d' %
(oldSize, newSize),
file=self.log)
f = open(self.options.newModelFile, 'wb')
pickle.dump(model, f)
f.close()
del f
if self.options.shouldSelfTest:
print(
'warning: --self-test does not treat pronunciation variants correctly',
file=self.log)
if not self.develSample:
print('error: cannot do --self-test without --devel sample',
file=self.log)
else:
translator = Translator(model)
evaluator = Evaluator()
evaluator.setSample(self.develSample)
evaluator.verboseLog = self.log
result = evaluator.evaluate(translator)
print(result, file=self.log)
return model
class OnlineTester(object):
def __init__(self, name, sample):
self.name = name
self.evaluator = Evaluator()
self.evaluator.setSample(sample)
def __call__(self, log, context, model):
translator = Translator(model)
result = self.evaluator.evaluate(translator)
print >> log, 'ER %s: string errors %s symbol errors %s' % (
self.name, result.stringError, result.symbolError)
class OnlineTester(object):
def __init__(self, name, sample):
self.name = name
self.evaluator = Evaluator()
self.evaluator.setSample(sample)
def __call__(self, log, context, model):
translator = Translator(model)
result = self.evaluator.evaluate(translator)
print('ER %s: string errors %s symbol errors %s' % (
self.name, result.stringError, result.symbolError), file=log)
def procureModel(self):
if self.options.resume_from_checkpoint:
model = ModelTemplate.resume(self.options.resume_from_checkpoint)
self.sequitur = model.sequitur
elif self.options.modelFile:
model = pickle.load(open(self.options.modelFile))
self.sequitur = model.sequitur
else:
self.sequitur = Sequitur()
model = None
if self.options.shouldRampUp:
model.rampUp()
if self.options.trainSample:
model = self.trainModel(model)
if not model:
print >> self.log, 'failed to estimate or load model'
return
if not model:
raise UsageError
# model.sequenceModel.showMostProbable(sys.stdout, model.sequitur.symbol, limit=250)
if self.options.shouldTranspose:
model.transpose()
if self.options.newModelFile:
oldSize, newSize = model.strip()
print >> self.log, 'stripped number of multigrams from %d to %d' % (oldSize, newSize)
f = open(self.options.newModelFile, 'w')
pickle.dump(model, f, pickle.HIGHEST_PROTOCOL)
f.close()
del f
if self.options.shouldSelfTest:
print >> self.log, 'warning: --self-test does not treat pronunciation variants correctly'
if not self.develSample:
print >> self.log, 'error: cannot do --self-test without --devel sample'
else:
translator = Translator(model)
evaluator = Evaluator()
evaluator.setSample(self.develSample)
evaluator.verboseLog = self.log
result = evaluator.evaluate(translator)
print >> self.log, result
return model
def mainTest(translator, testSample, options):
if options.shouldTranspose:
testSample = SequiturTool.transposeSample(testSample)
if options.testResult:
resultFile = gOpenOut(options.testResult, defaultEncoding)
else:
resultFile = None
from Evaluation import Evaluator
evaluator = Evaluator()
evaluator.setSample(testSample)
evaluator.resultFile = resultFile
evaluator.verboseLog = stdout
if options.test_segmental:
supraSegmental = set(['.', "'", '"'])
def removeSupraSegmental(phon):
return filter(lambda p: p not in supraSegmental, phon)
evaluator.compareFilter = removeSupraSegmental
result = evaluator.evaluate(translator)
print >> stdout, result
def procureModel(self):
#print self.options,type(self.options)
#print self.loadSample,type(self.loadSample)
#print self.log,type(self.log)
if self.options.resume_from_checkpoint:
model = ModelTemplate.resume(self.options.resume_from_checkpoint)
self.sequitur = model.sequitur
elif self.options.modelFile:
#print "loading",self.options.modelFile
f = open(self.options.modelFile)
#print "loaded",f
#print "type:",type(f)
#print pickle
class Model(object):
pass
model = pickle.load(f)
#print "loaded",self.options.modelFile
self.sequitur = model.sequitur
else:
self.sequitur = Sequitur()
model = None
if self.options.shouldRampUp:
model.rampUp()
if self.options.trainSample:
model = self.trainModel(model)
if not model:
print >> self.log, 'failed to estimate or load model'
return
if not model:
raise UsageError
# model.sequenceModel.showMostProbable(sys.stdout, model.sequitur.symbol, limit=250)
if self.options.shouldTranspose:
model.transpose()
if self.options.newModelFile:
oldSize, newSize = model.strip()
print >> self.log, 'stripped number of multigrams from %d to %d' % (
oldSize, newSize)
f = open(self.options.newModelFile, 'w')
pickle.dump(model, f, pickle.HIGHEST_PROTOCOL)
f.close()
del f
if self.options.shouldSelfTest:
print >> self.log, 'warning: --self-test does not treat pronunciation variants correctly'
if not self.develSample:
print >> self.log, 'error: cannot do --self-test without --devel sample'
else:
translator = Translator(model)
evaluator = Evaluator()
evaluator.setSample(self.develSample)
evaluator.verboseLog = self.log
result = evaluator.evaluate(translator)
print >> self.log, result
return model