from subprocess import Popen, PIPE from argparse import ArgumentParser import utils from myconfig import MyConfig #Please `make -f Makefile.data prepare` first config = MyConfig() #change cwd to the libpinyin data directory libpinyin_dir = config.getEvalsDir() libpinyin_sub_dir = os.path.join(libpinyin_dir, 'data') os.chdir(libpinyin_sub_dir) #chdir done datafiles = [config.getFinalModelFileName(), \ config.getEvalsTextFileName(), \ 'deleted_bigram.db'] def checkData(): #check data files for onefile in datafiles: if not os.access(onefile, os.F_OK): sys.exit('missing one data file:' + onefile) def cleanUpData(): #begin processing cmdline = ['/usr/bin/make', 'clean'] subprocess = Popen(cmdline, shell=False, close_fds=True)
else: #backup merged model utils.copyfile(mergedmodel, prunedmodel) pruneModel(prunedmodel, args.k, args.CDF) #validate pruned model print('validating') validateModel(prunedmodel) #export textual format print('exporting') exportfile = os.path.join(trydir, 'kmm_pruned.text') exportModel(prunedmodel, exportfile) #convert to interpolation print('converting') kmm_model = exportfile inter_model = os.path.join(trydir, config.getFinalModelFileName()) convertModel(kmm_model, inter_model) modelsize = utils.get_file_length(inter_model) cwdstatus['PruneModelSize'] = modelsize utils.store_status(cwdstatuspath, cwdstatus) print('final model size:', modelsize) #sign status epoch utils.sign_epoch(cwdstatus, 'Prune') utils.store_status(cwdstatuspath, cwdstatus) print('done')
from argparse import ArgumentParser import utils from myconfig import MyConfig #Please `make -f Makefile.data prepare` first config = MyConfig() #change cwd to the libpinyin data directory libpinyin_dir = config.getEvalsDir() libpinyin_sub_dir = os.path.join(libpinyin_dir, 'data') os.chdir(libpinyin_sub_dir) #chdir done datafiles = [config.getFinalModelFileName(), \ config.getEvalsTextFileName(), \ 'deleted_bigram.db'] def checkData(): #check data files for onefile in datafiles: if not os.access(onefile, os.F_OK): sys.exit('missing one data file:' + onefile) def cleanUpData(): #begin processing cmdline = ['/usr/bin/make', 'clean'] subprocess = Popen(cmdline, shell=False, close_fds=True)
else: #backup merged model utils.copyfile(mergedmodel, prunedmodel) pruneModel(prunedmodel, args.k, args.CDF) #validate pruned model print('validating') validateModel(prunedmodel) #export textual format print('exporting') exportfile = os.path.join(trydir, 'kmm_pruned.text') exportModel(prunedmodel, exportfile) #convert to interpolation print('converting') kmm_model = exportfile inter_model = os.path.join(trydir, config.getFinalModelFileName()) convertModel(kmm_model, inter_model) modelsize = utils.get_file_length(inter_model) cwdstatus['PruneModelSize'] = modelsize utils.store_status(cwdstatuspath, cwdstatus) print('final model size:', modelsize) #sign status epoch utils.sign_epoch(cwdstatus, 'Prune') utils.store_status(cwdstatuspath, cwdstatus) print('done')