Python sbc4_tag_lm Examples

Example #1

File: lm_tm_luigi.py Project: d2207197/smttoktag

# -*- coding: utf-8 -*-

import luigi
import tools
import gentask
from sbc4_tm_lm_tasks import sbc4_zh_to_tok_tag_phrasetable, sbc4_tag_lm


class test_zh_data(luigi.ExternalTask):

    def output(self):
        return luigi.LocalTarget('data/testzh.txt')


gentask.zhtoktag('test_zh_tok', test_zh_data(), 'tt',
                 tm=sbc4_zh_to_tok_tag_phrasetable(), lm=sbc4_tag_lm())


class oxford_np_ench(luigi.ExternalTask):

    def output(self):
        return luigi.LocalTarget('data/oxford.np.ench.txt')


class oxford_np_ch(luigi.Task):

    def requires(self):
        return oxford_np_ench()

    def output(self):
        return luigi.LocalTarget('data/oxford.np.ch.txt')

Example #2

File: movie_sub_task.py Project: d2207197/smttoktag

# 'en_patterns_pretty', en_patterns(), target_dir / 'en.patterns.json')

# patterns_allline_task = gentask_pattern.pipeline_allline_task(
# 'moviesub_en_patterns_allline', en_truecase())

filtered_patterns = gentask_pattern.filtered_patterns_from_sentences(
    'moviesub_en_filtered_patterns', en())

# ch = gentask.slice_lines_grouped_by_n('ch', ench(), target_dir / 'ch.txt',
# n=3,
# s=1)
ch_untok = gentask.untok('ch_untok', ch(), target_dir / 'ch.untok.txt')
ch_toktag = gentask.zhtoktag('ch_toktag', ch_untok(),
                             target_dir / 'ch.toktag.txt',
                             tm=sbc4_tok_tag_tm(),
                             lm=sbc4_tag_lm())

ch_tok = gentask.remove_slashtag('ch_tok', ch_toktag(),
                                 target_dir / 'ch.tok.txt')

en_chtok = gentask.parallel_lines_merge('en_chtok', en(), ch_tok(),
                                        target_dir / 'en_chtok.txt')

# giza_task = gentask_giza.giza(inputf=str(target_dir / 'en_chtok.txt'),
#                               outputd=str(target_dir / 'giza/'))

unpack_singleline_patterns = gentask.localtarget_task(
    target_dir / 'en.gt.hiih.patterns.pretty.unpack_singleline.json')

phrasetable = gentask.localtarget_task(target_dir / 'phrase-table.10000.gz')