Beispiel #1
0
    def make(self, vocabulary, counts, order):
	self.setVocabulary(vocabulary)
	self.setHighestOrder(order)
	coc = [ mGramCounts.countsOfCounts(mGramCounts.mGramReduceToOrder(counts, order))
		for order in range(order + 1) ]
	self.estimateDiscounts(coc)
	result = Lm(order)
	counts = store(contract(counts))
	self.build(counts, result)
	return result
Beispiel #2
0
    def make(self, vocabulary, counts, order):
	self.setVocabulary(vocabulary)
	self.setHighestOrder(order)
	coc = [ mGramCounts.countsOfCounts(mGramCounts.mGramReduceToOrder(counts, order))
		for order in range(order + 1) ]
	self.estimateDiscounts(coc)
	result = Lm(order)
	counts = store(contract(counts))
	self.build(counts, result)
	return result
Beispiel #3
0
def main(options, args):
    builder = LanguageModelBuilder()
    builder.setLogFile(sys.stdout)

    vocabulary = loadVocabulary(options.vocabulary)
    builder.setVocabulary(vocabulary)

    builder.setHighestOrder(options.order - 1)

    if options.count_cutoffs:
        cutoffs = list(map(int, options.count_cutoffs.split()))
        builder.setCountCutoffs(cutoffs)

    binaryCountFile = options.read + '.bin'
    if os.path.isfile(binaryCountFile):
        counts = StoredCounts(binaryCountFile)
    else:
        counts = loadCounts(options.read, vocabulary, binaryCountFile)

    if options.counts_of_counts:
        coc = eval(gOpenIn(options.counts_of_counts).read())
    else:
        coc = [
            mGramCounts.countsOfCounts(
                mGramCounts.mGramReduceToOrder(counts, order))
            for order in range(options.order)
        ]

    maximumOrder = maximumCountsOrder(coc)
    if builder.highestOrder > maximumOrder:
        print('warning: no counts for orders above %d' % (maximumOrder + 1))
        builder.setHighestOrder(maximumOrder)

    builder.estimateDiscounts(coc)

    if options.lm:
        lm = makeLmWriter(options)
    else:
        lm = LmDummy()

    builder.build(counts, lm)

    if __debug__ and False:  ### TESTING
        print('verifying normalization ...', file=sys.stdout)
        lm2 = Lm(lm)
        lm2.checkNormalisation()
Beispiel #4
0
def main(options, args):
    builder = LanguageModelBuilder()
    builder.setLogFile(sys.stdout)

    vocabulary = loadVocabulary(options.vocabulary)
    builder.setVocabulary(vocabulary)

    builder.setHighestOrder(options.order - 1)

    if options.count_cutoffs:
	cutoffs = map(int, options.count_cutoffs.split())
	builder.setCountCutoffs(cutoffs)

    binaryCountFile = options.read + '.bin'
    if os.path.isfile(binaryCountFile):
	counts = StoredCounts(binaryCountFile)
    else:
	counts = loadCounts(options.read, vocabulary, binaryCountFile)

    if options.counts_of_counts:
	coc = eval(gOpenIn(options.counts_of_counts).read())
    else:
	coc = [ mGramCounts.countsOfCounts(mGramCounts.mGramReduceToOrder(counts, order))
		for order in range(options.order) ]

    maximumOrder = maximumCountsOrder(coc)
    if builder.highestOrder > maximumOrder:
	print 'warning: no counts for orders above %d' % (maximumOrder+1)
	builder.setHighestOrder(maximumOrder)

    builder.estimateDiscounts(coc)

    if options.lm:
	lm = makeLmWriter(options)
    else:
	lm = LmDummy()

    builder.build(counts, lm)

    if __debug__ and False: ### TESTING
	print >> sys.stdout, 'verifying normalization ...'
	lm2 = Lm(lm)
	lm2.checkNormalisation()