shmfilepath = shmdir + os.sep + filename
utils.copyfile(filepath, shmfilepath)
handleOnePass(indexpath, shmdir, i)
pruneNgramTable(indexpath, shmdir, i)
utils.copyfile(shmfilepath, filepath)
os.unlink(shmfilepath)
else:
handleOnePass(indexpath, workdir, i)
pruneNgramTable(indexpath, workdir, i)
#sign epoch
utils.sign_epoch(indexstatus, 'Populate')
utils.store_status(indexstatuspath, indexstatus)
if __name__ == '__main__':
parser = ArgumentParser(description='Populate n-gram.')
parser.add_argument('--indexdir', action='store', \
help='index directory', \
default=config.getTextIndexDir())
parser.add_argument('--fast', action='store_const', \
help='Use /dev/shm to speed up populate', \
const=True, default=False)
args = parser.parse_args()
print(args)
walkIndexFast(handleOneIndex, args.indexdir, args.fast)
print('done')
nexttextnum = i + 1
storeModelStatus(modelfile, textnum, nexttextnum)
indexfile.close()
#end processing
#save current progress in status file
modelnum += 1
indexstatus['GenerateTextEnd'] = nexttextnum
indexstatus['GenerateModelEnd'] = modelnum
utils.sign_epoch(indexstatus, 'Generate')
utils.store_status(indexstatuspath, indexstatus)
if __name__ == '__main__':
parser = ArgumentParser(description='Generate model candidates.')
parser.add_argument('--indexdir', action='store', \
help='index directory', \
default=config.getTextIndexDir())
parser.add_argument('--fast', action='store_const', \
help='Use in-memory filesystem to speed up generate', \
const=True, default=False)
args = parser.parse_args()
print(args)
walkIndexFast(handleOneIndex, args.indexdir, args.fast)
print('done')
