예제 #1
0
    def procureModel(self):
        if self.options.resume_from_checkpoint:
            model = ModelTemplate.resume(self.options.resume_from_checkpoint)
            self.sequitur = model.sequitur
        elif self.options.modelFile:
            if sys.version_info[:2] >= (3, 0):
                model = pickle.load(open(self.options.modelFile, 'rb'), encoding='latin1')
            else:
                try:
                    model = pickle.load(open(self.options.modelFile, 'rb'))
                except ValueError:
                    print('This error most likely occured because the loaded model was created in python3.\n', file=sys.stderr)
                    raise
                
            self.sequitur = model.sequitur
        else:
            self.sequitur = Sequitur()
            model = None

        if self.options.shouldRampUp:
            model.rampUp()

        if self.options.trainSample:
            model = self.trainModel(model)
            if not model:
                print('failed to estimate or load model', file=self.log)
                return

        if not model:
            raise UsageError

#       model.sequenceModel.showMostProbable(sys.stdout, model.sequitur.symbol, limit=250)

        if self.options.shouldTranspose:
            model.transpose()

        if self.options.newModelFile:
            oldSize, newSize = model.strip()
            print('stripped number of multigrams from %d to %d' % (oldSize, newSize), file=self.log)
            f = open(self.options.newModelFile, 'wb')
            pickle.dump(model, f, pickle.HIGHEST_PROTOCOL)
            f.close()
            del f

        if self.options.shouldSelfTest:
            print('warning: --self-test does not treat pronunciation variants correctly', file=self.log)
            if not self.develSample:
                print('error: cannot do --self-test without --devel sample', file=self.log)
            else:
                translator = Translator(model)
                evaluator = Evaluator()
                evaluator.setSample(self.develSample)
                evaluator.verboseLog = self.log
                result = evaluator.evaluate(translator)
                print(result, file=self.log)

        return model
예제 #2
0
    def procureModel(self):
        if self.options.resume_from_checkpoint:
            model = ModelTemplate.resume(self.options.resume_from_checkpoint)
            self.sequitur = model.sequitur
        elif self.options.modelFile:
            model = pickle.load(open(self.options.modelFile, "rb"))
            self.sequitur = model.sequitur
        else:
            self.sequitur = Sequitur()
            model = None

        if self.options.shouldRampUp:
            model.rampUp()

        if self.options.trainSample:
            model = self.trainModel(model)
            if not model:
                print('failed to estimate or load model', file=self.log)
                return

        if not model:
            raise UsageError


#       model.sequenceModel.showMostProbable(sys.stdout, model.sequitur.symbol, limit=250)

        if self.options.shouldTranspose:
            model.transpose()

        if self.options.newModelFile:
            oldSize, newSize = model.strip()
            print('stripped number of multigrams from %d to %d' %
                  (oldSize, newSize),
                  file=self.log)
            f = open(self.options.newModelFile, 'wb')
            pickle.dump(model, f)
            f.close()
            del f

        if self.options.shouldSelfTest:
            print(
                'warning: --self-test does not treat pronunciation variants correctly',
                file=self.log)
            if not self.develSample:
                print('error: cannot do --self-test without --devel sample',
                      file=self.log)
            else:
                translator = Translator(model)
                evaluator = Evaluator()
                evaluator.setSample(self.develSample)
                evaluator.verboseLog = self.log
                result = evaluator.evaluate(translator)
                print(result, file=self.log)

        return model
예제 #3
0
class OnlineTester(object):
    def __init__(self, name, sample):
	self.name = name
	self.evaluator = Evaluator()
	self.evaluator.setSample(sample)

    def __call__(self, log, context, model):
	translator = Translator(model)
	result = self.evaluator.evaluate(translator)
	print >> log, 'ER %s: string errors %s    symbol errors %s' % (
	    self.name, result.stringError, result.symbolError)
예제 #4
0
class OnlineTester(object):
    def __init__(self, name, sample):
        self.name = name
        self.evaluator = Evaluator()
        self.evaluator.setSample(sample)

    def __call__(self, log, context, model):
        translator = Translator(model)
        result = self.evaluator.evaluate(translator)
        print('ER %s: string errors %s    symbol errors %s' % (
            self.name, result.stringError, result.symbolError), file=log)
예제 #5
0
    def procureModel(self):
	if self.options.resume_from_checkpoint:
	    model = ModelTemplate.resume(self.options.resume_from_checkpoint)
	    self.sequitur = model.sequitur
	elif self.options.modelFile:
	    model = pickle.load(open(self.options.modelFile))
	    self.sequitur = model.sequitur
	else:
	    self.sequitur = Sequitur()
	    model = None

	if self.options.shouldRampUp:
	    model.rampUp()

	if self.options.trainSample:
	    model = self.trainModel(model)
            if not model:
                print >> self.log, 'failed to estimate or load model'
                return

	if not model:
            raise UsageError

#       model.sequenceModel.showMostProbable(sys.stdout, model.sequitur.symbol, limit=250)

	if self.options.shouldTranspose:
	    model.transpose()

	if self.options.newModelFile:
	    oldSize, newSize = model.strip()
	    print >> self.log, 'stripped number of multigrams from %d to %d' % (oldSize, newSize)
	    f = open(self.options.newModelFile, 'w')
	    pickle.dump(model, f, pickle.HIGHEST_PROTOCOL)
	    f.close()
	    del f

	if self.options.shouldSelfTest:
	    print >> self.log, 'warning: --self-test does not treat pronunciation variants correctly'
	    if not self.develSample:
		print >> self.log, 'error: cannot do --self-test without --devel sample'
	    else:
		translator = Translator(model)
		evaluator = Evaluator()
		evaluator.setSample(self.develSample)
		evaluator.verboseLog = self.log
		result = evaluator.evaluate(translator)
		print >> self.log, result

	return model
예제 #6
0
def mainTest(translator, testSample, options):
    if options.shouldTranspose:
        testSample = SequiturTool.transposeSample(testSample)
    if options.testResult:
        resultFile = gOpenOut(options.testResult, defaultEncoding)
    else:
        resultFile = None
    from Evaluation import Evaluator
    evaluator = Evaluator()
    evaluator.setSample(testSample)
    evaluator.resultFile = resultFile
    evaluator.verboseLog = stdout
    if options.test_segmental:
        supraSegmental = set(['.', "'", '"'])
        def removeSupraSegmental(phon):
            return filter(lambda p: p not in supraSegmental, phon)
        evaluator.compareFilter = removeSupraSegmental
    result = evaluator.evaluate(translator)
    print >> stdout, result
예제 #7
0
    def procureModel(self):
        #print self.options,type(self.options)
        #print self.loadSample,type(self.loadSample)
        #print self.log,type(self.log)

        if self.options.resume_from_checkpoint:
            model = ModelTemplate.resume(self.options.resume_from_checkpoint)
            self.sequitur = model.sequitur
        elif self.options.modelFile:
            #print "loading",self.options.modelFile
            f = open(self.options.modelFile)

            #print "loaded",f
            #print "type:",type(f)
            #print pickle
            class Model(object):
                pass

            model = pickle.load(f)
            #print "loaded",self.options.modelFile
            self.sequitur = model.sequitur
        else:
            self.sequitur = Sequitur()
            model = None

        if self.options.shouldRampUp:
            model.rampUp()

        if self.options.trainSample:
            model = self.trainModel(model)
            if not model:
                print >> self.log, 'failed to estimate or load model'
                return

        if not model:
            raise UsageError


#       model.sequenceModel.showMostProbable(sys.stdout, model.sequitur.symbol, limit=250)

        if self.options.shouldTranspose:
            model.transpose()

        if self.options.newModelFile:
            oldSize, newSize = model.strip()
            print >> self.log, 'stripped number of multigrams from %d to %d' % (
                oldSize, newSize)
            f = open(self.options.newModelFile, 'w')
            pickle.dump(model, f, pickle.HIGHEST_PROTOCOL)
            f.close()
            del f

        if self.options.shouldSelfTest:
            print >> self.log, 'warning: --self-test does not treat pronunciation variants correctly'
            if not self.develSample:
                print >> self.log, 'error: cannot do --self-test without --devel sample'
            else:
                translator = Translator(model)
                evaluator = Evaluator()
                evaluator.setSample(self.develSample)
                evaluator.verboseLog = self.log
                result = evaluator.evaluate(translator)
                print >> self.log, result

        return model