from subprocess import Popen, PIPE
from argparse import ArgumentParser
import utils
from myconfig import MyConfig
#Please `make -f Makefile.data prepare` first
config = MyConfig()
#change cwd to the libpinyin data directory
libpinyin_dir = config.getEvalsDir()
libpinyin_sub_dir = os.path.join(libpinyin_dir, 'data')
os.chdir(libpinyin_sub_dir)
#chdir done
datafiles = [config.getFinalModelFileName(), \
config.getEvalsTextFileName(), \
'deleted_bigram.db']
def checkData():
#check data files
for onefile in datafiles:
if not os.access(onefile, os.F_OK):
sys.exit('missing one data file:' + onefile)
def cleanUpData():
#begin processing
cmdline = ['/usr/bin/make', 'clean']
subprocess = Popen(cmdline, shell=False, close_fds=True)
else:
#backup merged model
utils.copyfile(mergedmodel, prunedmodel)
pruneModel(prunedmodel, args.k, args.CDF)
#validate pruned model
print('validating')
validateModel(prunedmodel)
#export textual format
print('exporting')
exportfile = os.path.join(trydir, 'kmm_pruned.text')
exportModel(prunedmodel, exportfile)
#convert to interpolation
print('converting')
kmm_model = exportfile
inter_model = os.path.join(trydir, config.getFinalModelFileName())
convertModel(kmm_model, inter_model)
modelsize = utils.get_file_length(inter_model)
cwdstatus['PruneModelSize'] = modelsize
utils.store_status(cwdstatuspath, cwdstatus)
print('final model size:', modelsize)
#sign status epoch
utils.sign_epoch(cwdstatus, 'Prune')
utils.store_status(cwdstatuspath, cwdstatus)
print('done')
from argparse import ArgumentParser
import utils
from myconfig import MyConfig
#Please `make -f Makefile.data prepare` first
config = MyConfig()
#change cwd to the libpinyin data directory
libpinyin_dir = config.getEvalsDir()
libpinyin_sub_dir = os.path.join(libpinyin_dir, 'data')
os.chdir(libpinyin_sub_dir)
#chdir done
datafiles = [config.getFinalModelFileName(), \
config.getEvalsTextFileName(), \
'deleted_bigram.db']
def checkData():
#check data files
for onefile in datafiles:
if not os.access(onefile, os.F_OK):
sys.exit('missing one data file:' + onefile)
def cleanUpData():
#begin processing
cmdline = ['/usr/bin/make', 'clean']
subprocess = Popen(cmdline, shell=False, close_fds=True)
else:
#backup merged model
utils.copyfile(mergedmodel, prunedmodel)
pruneModel(prunedmodel, args.k, args.CDF)
#validate pruned model
print('validating')
validateModel(prunedmodel)
#export textual format
print('exporting')
exportfile = os.path.join(trydir, 'kmm_pruned.text')
exportModel(prunedmodel, exportfile)
#convert to interpolation
print('converting')
kmm_model = exportfile
inter_model = os.path.join(trydir, config.getFinalModelFileName())
convertModel(kmm_model, inter_model)
modelsize = utils.get_file_length(inter_model)
cwdstatus['PruneModelSize'] = modelsize
utils.store_status(cwdstatuspath, cwdstatus)
print('final model size:', modelsize)
#sign status epoch
utils.sign_epoch(cwdstatus, 'Prune')
utils.store_status(cwdstatuspath, cwdstatus)
print('done')
