Python lexer 예제들

예제 #1

파일: translation_pipeline.py 프로젝트: salamynder/GF

def pipelineParsing(grammar, language, sentences, K=20):
    #buf = [sent for sent in sentences];
    buf, sentences = itertools.tee(sentences, 2);
    sentences = itertools.imap(gf_utils.lexer(lang=language), sentences);
    parser = gf_utils.getKBestParses(grammar, language, K, callbacks=[("PN", parseNames(grammar, language)), ("Symb", parseUnknown(grammar, language))]);
    for sent, (time, parsesBlock) in itertools.izip(buf, itertools.imap(parser, sentences)):
	yield (sent, parsesBlock);

예제 #2

파일: translation_pipeline.py 프로젝트: livyreal/GF

def pipelineParsing(grammar, language, sentences, K=20):
    #buf = [sent for sent in sentences];
    buf, sentences = itertools.tee(sentences, 2);
    sentences = itertools.imap(gf_utils.lexer(lang=language), sentences);
    parser = gf_utils.getKBestParses(grammar, language, K);
    for sent, (time, parsesBlock) in itertools.izip(buf, itertools.imap(parser, sentences)):
	yield (sent, parsesBlock);