Exemple #1
0
def compute_langlet_expr(langlet, start_symbol=None):
    running_cycle = set()
    state_traces = {}
    for s, nfa in langlet.parse_nfa.nfas.items():
        if s is start_symbol or s not in start_symbols:
            state_traces[s] = compute_state_traces(nfa)
    _, start, _ = langlet.parse_nfa.nfas[start_symbol]

    segtree = SegmentTree(langlet)
    segtree.create()
    supertrace = compute_super_tr(langlet, start, state_traces, segtree, running_cycle, start_symbols)
    flat_traces = compute_flat_tr(langlet)
    langlet_trace = []
    for t in supertrace:
        if is_symbol(t[0]):
            langlet_trace.extend(flat_traces[t[0]])
        else:
            langlet_trace.append(t[0])
    # for item in langlet_trace:
    #    print item, langlet.get_node_name(item)
    tgen = TokenGenerator(langlet, stdlen=1)
    tokstream = []
    letters = "abcdefg"
    i = 0
    for tid in langlet_trace:
        if tid == langlet.token.NAME:
            tokstream.append([tid, letters[i % len(letters)]])
            i += 1
        elif is_keyword(tid):
            tokstream.append([tid, langlet.get_node_name(tid)[4:]])
        else:
            tokstream.append([tid, tgen.gen_token_string(tid + SYMBOL_OFFSET)])
    return langlet.unparse([1000] + tokstream)
Exemple #2
0
def compute_langlet_expr(langlet, start_symbol=None):
    running_cycle = set()
    state_traces = {}
    for s, nfa in langlet.parse_nfa.nfas.items():
        if s is start_symbol or s not in start_symbols:
            state_traces[s] = compute_state_traces(nfa)
    _, start, _ = langlet.parse_nfa.nfas[start_symbol]

    segtree = SegmentTree(langlet)
    segtree.create()
    supertrace = compute_super_tr(langlet, start, state_traces, segtree,
                                  running_cycle, start_symbols)
    flat_traces = compute_flat_tr(langlet)
    langlet_trace = []
    for t in supertrace:
        if is_symbol(t[0]):
            langlet_trace.extend(flat_traces[t[0]])
        else:
            langlet_trace.append(t[0])
    # for item in langlet_trace:
    #    print item, langlet.get_node_name(item)
    tgen = TokenGenerator(langlet, stdlen=1)
    tokstream = []
    letters = "abcdefg"
    i = 0
    for tid in langlet_trace:
        if tid == langlet.token.NAME:
            tokstream.append([tid, letters[i % len(letters)]])
            i += 1
        elif is_keyword(tid):
            tokstream.append([tid, langlet.get_node_name(tid)[4:]])
        else:
            tokstream.append([tid, tgen.gen_token_string(tid + SYMBOL_OFFSET)])
    return langlet.unparse([1000] + tokstream)
Exemple #3
0
def _compute_embedd_tr(langlet, tr, r, nid, segtree, start_symbols):
    nfas = langlet.parse_nfa.nfas
    for X in nfas:
        if X in start_symbols:
            continue
        if r in langlet.parse_nfa.symbols_of[X]:
            (start_X, _, _) = nfa_X = nfas[X]
            for _tr_X in compute_state_traces(nfa_X):
                for i, S in enumerate(_tr_X):
                    if S[0] == r:
                        tr_X = _tr_X[:i] + tr + _tr_X[i + 1 :]
            segment = segtree[nid:X]
            if segment:
                sg = proj_segment(segment)[:]
                for j, item in enumerate(sg):
                    if not is_symbol(item):
                        sg[j] = (item, 1, nid)
                i = sg.index(X)
                return sg[:i] + tr_X + sg[i + 1 :]
            else:
                trace = _compute_embedd_tr(langlet, tr_X, X, nid, segtree, start_symbols)
                return trace
    return []
Exemple #4
0
def _compute_embedd_tr(langlet, tr, r, nid, segtree, start_symbols):
    nfas = langlet.parse_nfa.nfas
    for X in nfas:
        if X in start_symbols:
            continue
        if r in langlet.parse_nfa.symbols_of[X]:
            (start_X, _, _) = nfa_X = nfas[X]
            for _tr_X in compute_state_traces(nfa_X):
                for i, S in enumerate(_tr_X):
                    if S[0] == r:
                        tr_X = _tr_X[:i] + tr + _tr_X[i + 1:]
            segment = segtree[nid:X]
            if segment:
                sg = proj_segment(segment)[:]
                for j, item in enumerate(sg):
                    if not is_symbol(item):
                        sg[j] = (item, 1, nid)
                i = sg.index(X)
                return sg[:i] + tr_X + sg[i + 1:]
            else:
                trace = _compute_embedd_tr(langlet, tr_X, X, nid, segtree,
                                           start_symbols)
                return trace
    return []