def enhance_grammar(g):
# convert '[__DIGIT__]' to '<__DIGIT__>'
# convert '[__DIGIT__]+' to '<__DIGIT_s__>'
# add both definitions.
g_ = {}
added = set()
for key in g:
new_rules = []
g_[key] = new_rules
for rule in g[key]:
new_rule = []
new_rules.append(new_rule)
for token in rule:
if G.is_nt(token):
new_rule.append(token)
elif token in F.ASCII_MAP or token[0:-1] in F.ASCII_MAP:
rep, nk = asciimap_to_nt(token)
new_rule.append(nk)
added.add((rep, nk, token))
else:
new_rule.append(token)
for rep, nk, token in added:
if rep == '+':
r, key = asciimap_to_nt(token[0:-1])
g_[nk] = [[key], [key, nk]]
g_[key] = [[k] for k in F.ASCII_MAP[token[0:-1]]]
else:
g_[nk] = [[k] for k in F.ASCII_MAP[token]]
return g_
def is_token_rule(key, grammar):
has_generalizable_token = False
for rule in grammar[key]:
for t in rule:
if grammartools.is_nt(t): return False
if (t[0], t[-1]) == ('[', ']'):
has_generalizable_token = True
return has_generalizable_token
def get_def(t):
if t in ASCII_MAP:
return [random.choice(ASCII_MAP[t]), []]
elif t and t[-1] == '+' and t[0:-1] in ASCII_MAP:
num = random.randrange(FUZZRANGE) + 1
val = [random.choice(ASCII_MAP[t[0:-1]]) for i in range(num)]
return [''.join(val), []]
elif G.is_nt(t):
return [t, None]
else:
return [t, []]
def tree_to_pstr(tree, op_='', _cl=''):
expanded = []
to_expand = [tree]
while to_expand:
(key, children), *to_expand = to_expand
if G.is_nt(key):
expanded.append(op_)
to_expand = children + [(_cl, [])] + to_expand
else:
assert not children
expanded.append(key)
return ''.join(expanded)
def tree_to_str(tree):
expanded = []
to_expand = [tree]
while to_expand:
(key, children, *rest), *to_expand = to_expand
if G.is_nt(key):
#assert children # not necessary
to_expand = children + to_expand
else:
assert not children
expanded.append(key)
return ''.join(expanded)
def nonterminals(self, rule):
return [t for t in rule if G.is_nt(t)]
