INSERT_NGRAM_DML = '''
INSERT INTO ngram(words, freq) VALUES(?, 1);
'''
UPDATE_NGRAM_DML = '''
UPDATE ngram SET freq = freq + 1 WHERE words = ?;
'''
PRUNE_NGRAM_DML = '''
DELETE FROM ngram WHERE freq <= ?;
'''
config = MyConfig()
#maximum combine number
N = config.getMaximumCombineNumber()
#change cwd to the word recognizer directory
words_dir = config.getWordRecognizerDir()
os.chdir(words_dir)
#chdir done
def handleOneDocument(infile, cur, length):
print(infile, length)
infilestatuspath = infile + config.getStatusPostfix()
infilestatus = utils.load_status(infilestatuspath)
if not utils.check_epoch(infilestatus, 'Segment'):
raise utils.EpochError('Please segment first.\n')
if utils.check_epoch(infilestatus, 'Populate'):
INSERT_NGRAM_DML = '''
INSERT INTO ngram(words, freq) VALUES(?, 1);
'''
UPDATE_NGRAM_DML = '''
UPDATE ngram SET freq = freq + 1 WHERE words = ?;
'''
PRUNE_NGRAM_DML = '''
DELETE FROM ngram WHERE freq <= ?;
'''
config = MyConfig()
#maximum combine number
N = config.getMaximumCombineNumber()
#change cwd to the word recognizer directory
words_dir = config.getWordRecognizerDir()
os.chdir(words_dir)
#chdir done
def handleOneDocument(infile, cur, length):
print(infile, length)
infilestatuspath = infile + config.getStatusPostfix()
infilestatus = utils.load_status(infilestatuspath)
if not utils.check_epoch(infilestatus, 'Segment'):
raise utils.EpochError('Please segment first.\n')
if utils.check_epoch(infilestatus, 'Populate'):
