Ejemplo n.º 1
0
def binarise_model(working_path, l_lang, l_direct, l_orient, l_model, tconf, d):
    '''This function binarises the phrase and reoridering tables.
    Binarising them speeds up loading the decoder, though doesn't actually speed up decoding sentences

    :param string working_path: the path to the working directory
    :param string l_lang: reordering language setting, either f or fe
    :param string l_direct: reordering directionality setting, either forward, backward, or bidirectional
    :param string l_orient: reordering orientation setting, either mslr, msd, monotonicity, leftright
    :param string l_model: reordering modeltype setting, either wbe, phrase, or hier
    :param config tconf: translate configuration
    :param dict d: output dictionary
    '''

    with Timer(d, 'binarise', lg=logger):
        binarised_model_path = os.path.join(working_path, 'binarised-model')
        os.makedirs(binarised_model_path)
        command("{0}/bin/processPhraseTable  -ttable 0 0 {1}/train/model/{2}.gz -nscores 5 -out {1}/binarised-model/phrase-table".format(tconf.paths.moses, working_path, tconf.settings.phrase_table_name), logger=logger, capture=True)
        command("{0}/bin/processLexicalTable -in {1}/train/model/{6}.{2}-{3}-{4}-{5}.gz -out {1}/binarised-model/reordering-table".format(tconf.paths.moses, working_path, l_model, l_orient, l_direct, l_lang, tconf.settings.reordering_name), logger=logger, capture=True)

        copy_always(os.path.join(working_path, 'mert-work', 'moses.ini'),
                    os.path.join(binarised_model_path, 'moses.ini'))

        sub_dict = (re.compile(r'PhraseDictionaryMemory'), 'PhraseDictionaryBinary')
        mosesini = os.path.join(working_path, 'binarised-model', 'moses.ini')
        logger.info(mosesini)
        munge_page(mosesini, sub_dict)
        phrase_table_path = os.path.join('train', 'model', tconf.settings.phrase_table_name) + '.gz'
        sub_table = (re.compile(phrase_table_path), 'binarised-model/phrase-table')
        munge_page(mosesini, sub_table)
Ejemplo n.º 2
0
def binarise_model(working_path, l_lang, l_direct, l_orient, l_model, tconf, d):
    '''This function binarises the phrase and reoridering tables.
    Binarising them speeds up loading the decoder, though doesn't actually speed up decoding sentences

    :param string working_path: the path to the working directory
    :param string l_lang: reordering language setting, either f or fe
    :param string l_direct: reordering directionality setting, either forward, backward, or bidirectional
    :param string l_orient: reordering orientation setting, either mslr, msd, monotonicity, leftright
    :param string l_model: reordering modeltype setting, either wbe, phrase, or hier
    :param config tconf: translate configuration
    :param dict d: output dictionary
    '''

    with Timer(d, 'binarise', lg=logger):
        binarised_model_path = os.path.join(working_path, 'binarised-model')
        os.makedirs(binarised_model_path)
        command("{0}/bin/processPhraseTable  -ttable 0 0 {1}/train/model/{2}.gz -nscores 5 -out {1}/binarised-model/phrase-table".format(tconf.paths.moses, working_path, tconf.settings.phrase_table_name), logger=logger, capture=True)
        command("{0}/bin/processLexicalTable -in {1}/train/model/{6}.{2}-{3}-{4}-{5}.gz -out {1}/binarised-model/reordering-table".format(tconf.paths.moses, working_path, l_model, l_orient, l_direct, l_lang, tconf.settings.reordering_name), logger=logger, capture=True)

        copy_always(os.path.join(working_path, 'mert-work', 'moses.ini'),
                    os.path.join(binarised_model_path, 'moses.ini'))

        sub_dict = (re.compile(r'PhraseDictionaryMemory'), 'PhraseDictionaryBinary')
        mosesini = os.path.join(working_path, 'binarised-model', 'moses.ini')
        logger.info(mosesini)
        munge_page(mosesini, sub_dict)
        phrase_table_path = os.path.join('train', 'model', tconf.settings.phrase_table_name) + '.gz'
        sub_table = (re.compile(phrase_table_path), 'binarised-model/phrase-table')
        munge_page(mosesini, sub_table)
Ejemplo n.º 3
0
def error_pages(sconf, conf):
    builder = sconf.builder

    if "errors" not in conf.system.files.data:
        return None
    else:
        sub = (re.compile(r"\.\./\.\./"), conf.project.url + r"/" + conf.project.tag + r"/")

        for idx, error in enumerate(conf.system.files.data.errors):
            page = os.path.join(conf.paths.projectroot, conf.paths.branch_output, builder, "meta", error, "index.html")
            munge_page(fn=page, regex=sub, tag="error-pages")

        logger.info("error-pages: rendered {0} error pages".format(idx))
Ejemplo n.º 4
0
def error_pages(sconf, conf):
    builder = sconf.builder

    if 'errors' not in conf.system.files.data:
        return None
    else:
        sub = (re.compile(r'\.\./\.\./'), conf.project.url + r'/' + conf.project.tag + r'/')

        for idx, error in enumerate(conf.system.files.data.errors):
            page = os.path.join(conf.paths.projectroot,
                                conf.paths.branch_output, builder,
                                'meta', error, 'index.html')
            munge_page(fn=page, regex=sub, tag='error-pages')

        logging.info('error-pages: rendered {0} error pages'.format(idx))
Ejemplo n.º 5
0
def error_pages(sconf, conf):
    builder = sconf.builder

    if 'errors' not in conf.system.files.data:
        return None
    else:
        sub = (re.compile(r'\.\./\.\./'),
               conf.project.url + r'/' + conf.project.tag + r'/')

        for idx, error in enumerate(conf.system.files.data.errors):
            page = os.path.join(conf.paths.projectroot,
                                conf.paths.branch_output, builder, 'meta',
                                error, 'index.html')
            munge_page(fn=page, regex=sub, tag='error-pages')

        logger.info('error-pages: rendered {0} error pages'.format(idx))
Ejemplo n.º 6
0
def error_pages(sconf, conf):
    builder = sconf.builder

    if 'errors' not in conf.system.files.data:
        return None

    regex = re.compile(r'\.\./\.\./')
    segments = [conf.project.url, conf.project.tag]
    if conf.project.error_prefix:
        segments.append(conf.project.error_prefix)

    sub = (regex, '/'.join(segments).rstrip('/') + '/')

    for idx, error in enumerate(conf.system.files.data.errors):
        page = os.path.join(conf.paths.projectroot, conf.paths.branch_output,
                            builder, 'meta', error, 'index.html')
        munge_page(fn=page, regex=sub, tag='error-pages')

    logger.info('error-pages: rendered {0} error pages'.format(idx))
Ejemplo n.º 7
0
def _clean_sphinx_latex(fn, regexes):
    munge_page(fn, regexes, tag='pdf')