def compute_langlet_expr(langlet, start_symbol=None):
running_cycle = set()
state_traces = {}
for s, nfa in langlet.parse_nfa.nfas.items():
if s is start_symbol or s not in start_symbols:
state_traces[s] = compute_state_traces(nfa)
_, start, _ = langlet.parse_nfa.nfas[start_symbol]
segtree = SegmentTree(langlet)
segtree.create()
supertrace = compute_super_tr(langlet, start, state_traces, segtree, running_cycle, start_symbols)
flat_traces = compute_flat_tr(langlet)
langlet_trace = []
for t in supertrace:
if is_symbol(t[0]):
langlet_trace.extend(flat_traces[t[0]])
else:
langlet_trace.append(t[0])
# for item in langlet_trace:
# print item, langlet.get_node_name(item)
tgen = TokenGenerator(langlet, stdlen=1)
tokstream = []
letters = "abcdefg"
i = 0
for tid in langlet_trace:
if tid == langlet.token.NAME:
tokstream.append([tid, letters[i % len(letters)]])
i += 1
elif is_keyword(tid):
tokstream.append([tid, langlet.get_node_name(tid)[4:]])
else:
tokstream.append([tid, tgen.gen_token_string(tid + SYMBOL_OFFSET)])
return langlet.unparse([1000] + tokstream)
def compute_langlet_expr(langlet, start_symbol=None):
running_cycle = set()
state_traces = {}
for s, nfa in langlet.parse_nfa.nfas.items():
if s is start_symbol or s not in start_symbols:
state_traces[s] = compute_state_traces(nfa)
_, start, _ = langlet.parse_nfa.nfas[start_symbol]
segtree = SegmentTree(langlet)
segtree.create()
supertrace = compute_super_tr(langlet, start, state_traces, segtree,
running_cycle, start_symbols)
flat_traces = compute_flat_tr(langlet)
langlet_trace = []
for t in supertrace:
if is_symbol(t[0]):
langlet_trace.extend(flat_traces[t[0]])
else:
langlet_trace.append(t[0])
# for item in langlet_trace:
# print item, langlet.get_node_name(item)
tgen = TokenGenerator(langlet, stdlen=1)
tokstream = []
letters = "abcdefg"
i = 0
for tid in langlet_trace:
if tid == langlet.token.NAME:
tokstream.append([tid, letters[i % len(letters)]])
i += 1
elif is_keyword(tid):
tokstream.append([tid, langlet.get_node_name(tid)[4:]])
else:
tokstream.append([tid, tgen.gen_token_string(tid + SYMBOL_OFFSET)])
return langlet.unparse([1000] + tokstream)
def _compute_embedd_tr(langlet, tr, r, nid, segtree, start_symbols):
nfas = langlet.parse_nfa.nfas
for X in nfas:
if X in start_symbols:
continue
if r in langlet.parse_nfa.symbols_of[X]:
(start_X, _, _) = nfa_X = nfas[X]
for _tr_X in compute_state_traces(nfa_X):
for i, S in enumerate(_tr_X):
if S[0] == r:
tr_X = _tr_X[:i] + tr + _tr_X[i + 1 :]
segment = segtree[nid:X]
if segment:
sg = proj_segment(segment)[:]
for j, item in enumerate(sg):
if not is_symbol(item):
sg[j] = (item, 1, nid)
i = sg.index(X)
return sg[:i] + tr_X + sg[i + 1 :]
else:
trace = _compute_embedd_tr(langlet, tr_X, X, nid, segtree, start_symbols)
return trace
return []
def _compute_embedd_tr(langlet, tr, r, nid, segtree, start_symbols):
nfas = langlet.parse_nfa.nfas
for X in nfas:
if X in start_symbols:
continue
if r in langlet.parse_nfa.symbols_of[X]:
(start_X, _, _) = nfa_X = nfas[X]
for _tr_X in compute_state_traces(nfa_X):
for i, S in enumerate(_tr_X):
if S[0] == r:
tr_X = _tr_X[:i] + tr + _tr_X[i + 1:]
segment = segtree[nid:X]
if segment:
sg = proj_segment(segment)[:]
for j, item in enumerate(sg):
if not is_symbol(item):
sg[j] = (item, 1, nid)
i = sg.index(X)
return sg[:i] + tr_X + sg[i + 1:]
else:
trace = _compute_embedd_tr(langlet, tr_X, X, nid, segtree,
start_symbols)
return trace
return []