def __init__(self, dictpath):

self.tokenizer = Tokenizer.Tokenizer()

self.documents = defaultdict(Document.Document)

self.statistics = defaultdict(int)

self.dictionary = Dictionary.Dictionary(dictpath)

self.dictionary.load('syllable')

self.dictionary.load('token')

self.dictionary.load('type')

self.dictionary.load('length')

self.model = MaxentModel()

self.threshold = 0.0

def set(self, modelname=None, threshold=0.0, filename=None):

assert(modelname != None)

assert(modelname.strip() != '')

assert(filename != None)

assert(filename.strip() != '')

try:

util.Logger.debug("Started to load model...")

self.model.load(modelname)

self.threshold = threshold

util.Logger.debug("Completed to load model '%s'" % modelname)

except:

raise

try:

util.Logger.debug("Started to load document...")

document = Document.Document()

file = open(filename)

for token in self.tokenizer.tokenize(file):

document.add(token)

file.close()

self.documents[filename] = document

util.Logger.debug("Competed to load document '%s'" % filename)

except:

raise

def get(self, filename=None):

assert(filename != None)

assert(filename.strip() != '')

if filename in self.documents:

return self.documents[filename]

else:

return Document.Document()

def eos(self, context):

label = 'yes'

prob = self.model.eval(context, label)

buf = ''

if prob >= self.threshold:

return True

else:

return False

# append property into list-buf

def append_maxent_parameter(self, list, i, property):

i += 1

list.append(str(i) + ':' + str(property))

return i

# FIXME: code duplicattion with sbd.detector.Probabilistic.py

def eval(self, document, id, prevToken, currToken, nextToken, syllable_length=0, merged_use=False):

dict = self.dictionary

common = util.Common()

# default token value

default = '_'

# { pos-type, pos-name }

current_pos_type = common.name_of_type(currToken)

current_pos_name = common.name_of_pos(currToken)

prefix_pos_type = common.name_of_type(prevToken)

prefix_pos_name = common.name_of_pos(prevToken)

suffix_pos_type = common.name_of_type(nextToken)

suffix_pos_name = common.name_of_pos(nextToken)

# { syllables }

prefix_syllable_name = []

prefix_syllable_prob = []

suffix_syllable_name = []

suffix_syllable_prob = []

merged_syllable_name = []

merged_syllable_prob = []

for length in xrange(syllable_length):

if prevToken.length == 0: prefixName = default * syllable_length

else: prefixName = prevToken.syllable(-1*(length+1))

prefix_syllable_name.append(prefixName)

prefix_syllable_prob.append(dict.getPrefixSyllableProb(prefixName))

if nextToken.length == 0: suffixName = default * syllable_length

else: suffixName = nextToken.syllable(length+1)

suffix_syllable_name.append(suffixName)

suffix_syllable_prob.append(dict.getSuffixSyllableProb(suffixName))

if merged_use:

mergedName = prefixName + '_' + suffixName

merged_syllable_name.append(mergedName)

merged_syllable_prob.append(dict.getMergedSyllableProb(mergedName))

# { token-name, token-prob }

if currToken.length == 0: current_token_name = default

else: current_token_name = currToken.value

current_token_prob = dict.getCurrentTokenProb(current_token_name)

if prevToken.length == 0: prefix_token_name = default

else: prefix_token_name = prevToken.value

prefix_token_prob = dict.getPrefixTokenProb(prefix_token_name)

if nextToken.length == 0: suffix_token_name = default

else: suffix_token_name = nextToken.value

suffix_token_prob = dict.getSuffixTokenProb(suffix_token_name)

# { candidate-distance }

prefix_candidate_dist = document.prevCandidateDist(id)

suffix_candidate_dist = document.nextCandidateDist(id)

# { punctuation-distance }

prefix_punctuation_dist = document.prevPunctuationDist(id)

suffix_punctuation_dist = document.nextPunctuationDist(id)

# { token-length }

current_token_length = currToken.length

prefix_token_length = prevToken.length

suffix_token_length = nextToken.length

# { end-of-sentence }

end_of_sentence = 'no'

if currToken.end_of_sentence:

end_of_sentence = 'yes'

context = [end_of_sentence]

i = 0

# { building instances }

i = self.append_maxent_parameter(context, i, current_pos_type)

i = self.append_maxent_parameter(context, i, current_pos_name)

i = self.append_maxent_parameter(context, i, prefix_pos_type)

i = self.append_maxent_parameter(context, i, prefix_pos_name)

i = self.append_maxent_parameter(context, i, suffix_pos_type)

i = self.append_maxent_parameter(context, i, suffix_pos_name)

# XXX: maxent use NAME instead of PROBABILITY

for length in xrange(syllable_length):

i = self.append_maxent_parameter(context, i, prefix_syllable_name[length])

i = self.append_maxent_parameter(context, i, suffix_syllable_name[length])

if merged_use:

i = self.append_maxent_parameter(context, i, merged_syllable_name[length])

i = self.append_maxent_parameter(context, i, current_token_name)

i = self.append_maxent_parameter(context, i, prefix_token_name)

i = self.append_maxent_parameter(context, i, suffix_token_name)

i = self.append_maxent_parameter(context, i, str(current_token_length))

i = self.append_maxent_parameter(context, i, str(prefix_token_length))

i = self.append_maxent_parameter(context, i, str(suffix_token_length))

eos = self.eos(context)

return eos

def calc(self, answer, rule):

if answer == True and rule == True:

result = 'TP'

elif answer == True and rule == False:

result = 'TN'

elif answer == False and rule == True:

result = 'FP'

else:

result = 'FN'

self.statistics[result] += 1

def summary(self):

precision = 0.0

recall = 0.0

fscore = 0.0

tp = self.statistics['TP']

tn = self.statistics['TN']

fp = self.statistics['FP']

util.Logger.info("tp:", tp, "tn:", tn, "fp:", fp)

if (tp + tn) > 0:

precision = tp * 1.0 / (tp + tn)

if (tp + fp) > 0:

recall = tp * 1.0 / (tp + fp)

if (precision+recall) > 0:

fscore = (2*precision*recall) / (precision+recall)

util.Logger.info("Precision:\t%0.3f%%" % (precision * 100.0))

util.Logger.info("Recall:\t\t%0.3f%%" % (recall * 100.0))

buff = []

token = document.prev(id)

i = id-1

assert(i > 0)

while (not token.isEos()):

token = document.prev(i)

i = i-1

if (i > 0):

return ''

assert(i > 0)

for j in xrange(id-i-1):

curr = document.next(i+j)

buff.append(curr.value)

if curr.isEoe():

buff.append(' ')

print "python MaxentBasedSBD.py [dictpath] [modelname] [syllable_length] [merged_yn] [ignore_case_yn] [threshold] [filename] [eval|seg]"

print "python MaxentBasedSBD.py dict/sejong model/maxent.model 1 yes no 0.8 corpus/sejong/sample.txt eval"

print "python MaxentBasedSBD.py dict/sejong model/maxent.model 1 yes no 0.8 corpus/sejong/sample.txt seg"

print "python MaxentBasedSBD.py dict/pentree model/maxent.model 0 no yes 0.8 corpus/pentree/sample.txt eval"