def __init__(self, order=3, smoothing=None, tokenize=None):
if order < 1:
print("Invalid order: " + str(order)),
print("Defaulting to order 3.")
self.order = 3
self.order = order
self.counts = dict()
self.tokenize = tokenize
if not self.tokenize:
self.tokenize = DEFAULT_TOKENIZE
self.Starter = '<S>'
self.Ender = '</S>'
self.OOV = '!OOV!'
self.probEst = smoothing # might be None
if not self.probEst:
from probest import MLE
self.probEst = MLE()
