def binarise_model(working_path, l_lang, l_direct, l_orient, l_model, tconf, d):
'''This function binarises the phrase and reoridering tables.
Binarising them speeds up loading the decoder, though doesn't actually speed up decoding sentences
:param string working_path: the path to the working directory
:param string l_lang: reordering language setting, either f or fe
:param string l_direct: reordering directionality setting, either forward, backward, or bidirectional
:param string l_orient: reordering orientation setting, either mslr, msd, monotonicity, leftright
:param string l_model: reordering modeltype setting, either wbe, phrase, or hier
:param config tconf: translate configuration
:param dict d: output dictionary
'''
with Timer(d, 'binarise', lg=logger):
binarised_model_path = os.path.join(working_path, 'binarised-model')
os.makedirs(binarised_model_path)
command("{0}/bin/processPhraseTable -ttable 0 0 {1}/train/model/{2}.gz -nscores 5 -out {1}/binarised-model/phrase-table".format(tconf.paths.moses, working_path, tconf.settings.phrase_table_name), logger=logger, capture=True)
command("{0}/bin/processLexicalTable -in {1}/train/model/{6}.{2}-{3}-{4}-{5}.gz -out {1}/binarised-model/reordering-table".format(tconf.paths.moses, working_path, l_model, l_orient, l_direct, l_lang, tconf.settings.reordering_name), logger=logger, capture=True)
copy_always(os.path.join(working_path, 'mert-work', 'moses.ini'),
os.path.join(binarised_model_path, 'moses.ini'))
sub_dict = (re.compile(r'PhraseDictionaryMemory'), 'PhraseDictionaryBinary')
mosesini = os.path.join(working_path, 'binarised-model', 'moses.ini')
logger.info(mosesini)
munge_page(mosesini, sub_dict)
phrase_table_path = os.path.join('train', 'model', tconf.settings.phrase_table_name) + '.gz'
sub_table = (re.compile(phrase_table_path), 'binarised-model/phrase-table')
munge_page(mosesini, sub_table)
def binarise_model(working_path, l_lang, l_direct, l_orient, l_model, tconf, d):
'''This function binarises the phrase and reoridering tables.
Binarising them speeds up loading the decoder, though doesn't actually speed up decoding sentences
:param string working_path: the path to the working directory
:param string l_lang: reordering language setting, either f or fe
:param string l_direct: reordering directionality setting, either forward, backward, or bidirectional
:param string l_orient: reordering orientation setting, either mslr, msd, monotonicity, leftright
:param string l_model: reordering modeltype setting, either wbe, phrase, or hier
:param config tconf: translate configuration
:param dict d: output dictionary
'''
with Timer(d, 'binarise', lg=logger):
binarised_model_path = os.path.join(working_path, 'binarised-model')
os.makedirs(binarised_model_path)
command("{0}/bin/processPhraseTable -ttable 0 0 {1}/train/model/{2}.gz -nscores 5 -out {1}/binarised-model/phrase-table".format(tconf.paths.moses, working_path, tconf.settings.phrase_table_name), logger=logger, capture=True)
command("{0}/bin/processLexicalTable -in {1}/train/model/{6}.{2}-{3}-{4}-{5}.gz -out {1}/binarised-model/reordering-table".format(tconf.paths.moses, working_path, l_model, l_orient, l_direct, l_lang, tconf.settings.reordering_name), logger=logger, capture=True)
copy_always(os.path.join(working_path, 'mert-work', 'moses.ini'),
os.path.join(binarised_model_path, 'moses.ini'))
sub_dict = (re.compile(r'PhraseDictionaryMemory'), 'PhraseDictionaryBinary')
mosesini = os.path.join(working_path, 'binarised-model', 'moses.ini')
logger.info(mosesini)
munge_page(mosesini, sub_dict)
phrase_table_path = os.path.join('train', 'model', tconf.settings.phrase_table_name) + '.gz'
sub_table = (re.compile(phrase_table_path), 'binarised-model/phrase-table')
munge_page(mosesini, sub_table)
def error_pages(sconf, conf):
builder = sconf.builder
if "errors" not in conf.system.files.data:
return None
else:
sub = (re.compile(r"\.\./\.\./"), conf.project.url + r"/" + conf.project.tag + r"/")
for idx, error in enumerate(conf.system.files.data.errors):
page = os.path.join(conf.paths.projectroot, conf.paths.branch_output, builder, "meta", error, "index.html")
munge_page(fn=page, regex=sub, tag="error-pages")
logger.info("error-pages: rendered {0} error pages".format(idx))
def error_pages(sconf, conf):
builder = sconf.builder
if 'errors' not in conf.system.files.data:
return None
else:
sub = (re.compile(r'\.\./\.\./'), conf.project.url + r'/' + conf.project.tag + r'/')
for idx, error in enumerate(conf.system.files.data.errors):
page = os.path.join(conf.paths.projectroot,
conf.paths.branch_output, builder,
'meta', error, 'index.html')
munge_page(fn=page, regex=sub, tag='error-pages')
logging.info('error-pages: rendered {0} error pages'.format(idx))
def error_pages(sconf, conf):
builder = sconf.builder
if 'errors' not in conf.system.files.data:
return None
else:
sub = (re.compile(r'\.\./\.\./'),
conf.project.url + r'/' + conf.project.tag + r'/')
for idx, error in enumerate(conf.system.files.data.errors):
page = os.path.join(conf.paths.projectroot,
conf.paths.branch_output, builder, 'meta',
error, 'index.html')
munge_page(fn=page, regex=sub, tag='error-pages')
logger.info('error-pages: rendered {0} error pages'.format(idx))
def error_pages(sconf, conf):
builder = sconf.builder
if 'errors' not in conf.system.files.data:
return None
regex = re.compile(r'\.\./\.\./')
segments = [conf.project.url, conf.project.tag]
if conf.project.error_prefix:
segments.append(conf.project.error_prefix)
sub = (regex, '/'.join(segments).rstrip('/') + '/')
for idx, error in enumerate(conf.system.files.data.errors):
page = os.path.join(conf.paths.projectroot, conf.paths.branch_output,
builder, 'meta', error, 'index.html')
munge_page(fn=page, regex=sub, tag='error-pages')
logger.info('error-pages: rendered {0} error pages'.format(idx))
def _clean_sphinx_latex(fn, regexes):
munge_page(fn, regexes, tag='pdf')