def mine_grammar(self):
grammar = extend_grammar(self.QUERY_GRAMMAR)
grammar["<action>"] = [self.action]
query = ""
for field in self.fields:
field_symbol = new_symbol(grammar, "<" + field + ">")
field_type = self.fields[field]
if query != "":
query += "&"
query += field_symbol
if isinstance(field_type, str):
field_type_symbol = "<" + field_type + ">"
grammar[field_symbol] = [field + "=" + field_type_symbol]
if field_type_symbol not in grammar:
# Unknown type
grammar[field_type_symbol] = ["<text>"]
else:
# List of values
value_symbol = new_symbol(grammar, "<" + field + "-value>")
grammar[field_symbol] = [field + "=" + value_symbol]
grammar[value_symbol] = field_type
grammar["<query>"] = [query]
# Remove unused parts
for nonterminal in unreachable_nonterminals(grammar):
del grammar[nonterminal]
assert is_valid_grammar(grammar)
return grammar
class HTMLGrammarMiner(HTMLGrammarMiner):
QUERY_GRAMMAR = extend_grammar(
CGI_GRAMMAR,
{
"<start>": ["<action>?<query>"],
"<text>": ["<string>"],
"<number>": ["<digits>"],
"<digits>": ["<digit>", "<digits><digit>"],
"<digit>": crange('0', '9'),
"<checkbox>": ["<_checkbox>"],
"<_checkbox>": ["on", "off"],
"<email>": ["<_email>"],
"<_email>": [cgi_encode("<string>@<string>", "<>")],
# Use a fixed password in case we need to repeat it
"<password>": ["<_password>"],
"<_password>": ["abcABC.123"],
# Stick to printable characters to avoid logging problems
"<percent>": ["%<hexdigit-1><hexdigit>"],
"<hexdigit-1>": srange("34567"),
# Submissions:
"<submit>": [""]
})
def update_new_state(self):
if self.log_gui_exploration:
print("In new state", unicode_escape(self.state_symbol), unicode_escape(repr(self.state)))
state_grammar = self.miner.mine_state_grammar(grammar=self.grammar,
state_symbol=self.state_symbol)
del state_grammar[START_SYMBOL]
del state_grammar[GUIGrammarMiner.START_STATE]
self.set_grammar(extend_grammar(self.grammar, state_grammar))
def list_grammar(object_grammar, list_object_symbol=None):
obj_list_grammar = extend_grammar(LIST_GRAMMAR)
if list_object_symbol is None:
# Default: Use the first expansion of <start> as list symbol
list_object_symbol = object_grammar[START_SYMBOL][0]
obj_list_grammar.update(object_grammar)
obj_list_grammar[START_SYMBOL] = ["<list>"]
obj_list_grammar["<list-object>"] = [list_object_symbol]
assert is_valid_grammar(obj_list_grammar)
return obj_list_grammar
def mine_state_grammar(self, grammar={}, state_symbol=None):
grammar = extend_grammar(self.GUI_GRAMMAR, grammar)
if state_symbol is None:
state_symbol = self.new_state_symbol(grammar)
grammar[state_symbol] = []
alternatives = []
form = ""
submit = None
for action in self.mine_state_actions():
if action.startswith("submit"):
submit = action
elif action.startswith("click"):
link_target = self.new_state_symbol(grammar)
grammar[link_target] = [self.UNEXPLORED_STATE]
alternatives.append(action + '\n' + link_target)
elif action.startswith("ignore"):
pass
else: # fill(), check() actions
if len(form) > 0:
form += '\n'
form += action
if submit is not None:
if len(form) > 0:
form += '\n'
form += submit
if len(form) > 0:
form_target = self.new_state_symbol(grammar)
grammar[form_target] = [self.UNEXPLORED_STATE]
alternatives.append(form + '\n' + form_target)
alternatives += [self.FINAL_STATE]
grammar[state_symbol] = alternatives
# Remove unused parts
for nonterminal in unreachable_nonterminals(grammar):
del grammar[nonterminal]
assert is_valid_grammar(grammar)
return grammar
def __init__(self, html_text, sql_payload):
super().__init__(html_text)
self.QUERY_GRAMMAR = extend_grammar(self.QUERY_GRAMMAR, {
"<text>": ["<string>", "<sql-injection-attack>"],
"<number>": ["<digits>", "<sql-injection-attack>"],
"<checkbox>": ["<_checkbox>", "<sql-injection-attack>"],
"<email>": ["<_email>", "<sql-injection-attack>"],
"<sql-injection-attack>": [
cgi_encode(attack, "<->") for attack in self.ATTACKS
],
"<sql-values>": ["", cgi_encode("<sql-values>, '<string>'", "<->")],
"<sql-payload>": [cgi_encode(sql_payload)],
"<sql-comment>": ["--", "#"],
})
def duplicate_context(grammar, symbol, expansion=None, depth=float('inf')):
"""Duplicate an expansion within a grammar.
In the given grammar, take the given expansion of the given symbol
(if expansion is omitted: all symbols), and replace it with a
new expansion referring to a duplicate of all originally referenced rules.
If depth is given, limit duplication to `depth` references (default: unlimited)
"""
orig_grammar = extend_grammar(grammar)
_duplicate_context(grammar, orig_grammar, symbol,
expansion, depth, seen={})
# After duplication, we may have unreachable rules; delete them
for nonterminal in unreachable_nonterminals(grammar):
del grammar[nonterminal]
# ### HTML Injection Attacks
if __name__ == "__main__":
print('\n### HTML Injection Attacks')
if __package__ is None or __package__ == "":
from Grammars import extend_grammar
else:
from .Grammars import extend_grammar
ORDER_GRAMMAR_WITH_HTML_INJECTION = extend_grammar(
ORDER_GRAMMAR, {
"<name>": [
cgi_encode('''
Jane Doe<p>
<strong><a href="www.lots.of.malware">Click here for cute cat pictures!</a></strong>
</p>
''')
],
})
if __name__ == "__main__":
html_injection_fuzzer = GrammarFuzzer(ORDER_GRAMMAR_WITH_HTML_INJECTION)
order_with_injected_html = html_injection_fuzzer.fuzz()
order_with_injected_html
if __name__ == "__main__":
HTML(webbrowser(urljoin(httpd_url, order_with_injected_html)))
if __name__ == "__main__":
print(db.execute("SELECT * FROM orders WHERE name LIKE '%<%'").fetchall())
def initial_grammar(self):
return extend_grammar({
START_SYMBOL: [self.CALL_SYMBOL],
self.CALL_SYMBOL: []
})
if __name__ == "__main__":
print('\n## Directed Fuzzing')
def set_prob(grammar, symbol, expansion, prob):
"""Set the probability of the given expansion of grammar[symbol]"""
set_opts(grammar, symbol, expansion, opts(prob=prob))
if __package__ is None or __package__ == "":
from Grammars import URL_GRAMMAR, extend_grammar
else:
from .Grammars import URL_GRAMMAR, extend_grammar
if __name__ == "__main__":
probabilistic_url_grammar = extend_grammar(URL_GRAMMAR)
set_prob(probabilistic_url_grammar, "<scheme>", "ftps", 0.8)
assert is_valid_probabilistic_grammar(probabilistic_url_grammar)
if __name__ == "__main__":
probabilistic_url_grammar["<scheme>"]
if __name__ == "__main__":
prob_url_fuzzer = ProbabilisticGrammarFuzzer(probabilistic_url_grammar)
for i in range(10):
print(prob_url_fuzzer.fuzz())
if __name__ == "__main__":
set_prob(probabilistic_url_grammar, "<scheme>", "ftps", 0.0)
assert is_valid_probabilistic_grammar(probabilistic_url_grammar)
def __init__(self, parser, log=False):
assert isinstance(parser, Parser)
self.grammar = extend_grammar(parser.grammar())
self.parser = parser
self.log = log
self.reset()
urlparse_fuzzer = GrammarFuzzer(URLPARSE_C_GRAMMAR)
print(urlparse_fuzzer.fuzz())
# ## Synthesizing Oracles
if __name__ == "__main__":
print('\n## Synthesizing Oracles')
if __package__ is None or __package__ == "":
from GeneratorGrammarFuzzer import GeneratorGrammarFuzzer, ProbabilisticGeneratorGrammarFuzzer
else:
from .GeneratorGrammarFuzzer import GeneratorGrammarFuzzer, ProbabilisticGeneratorGrammarFuzzer
URLPARSE_ORACLE_GRAMMAR = extend_grammar(
URLPARSE_GRAMMAR, {
"<call>": [("assert urlparse('<url>').geturl() == '<url>'",
opts(post=lambda url_1, url_2: [None, url_1]))]
})
if __name__ == "__main__":
urlparse_oracle_fuzzer = GeneratorGrammarFuzzer(URLPARSE_ORACLE_GRAMMAR)
test = urlparse_oracle_fuzzer.fuzz()
print(test)
if __name__ == "__main__":
exec(test)
URLPARSE_ORACLE_GRAMMAR = extend_grammar(
URLPARSE_GRAMMAR,
{
"<call>": [(
if __name__ == "__main__":
EXPR_GRAMMAR["<factor>"]
# ### Extending Grammars for Context Coverage Manually
if __name__ == "__main__":
print('\n### Extending Grammars for Context Coverage Manually')
if __name__ == "__main__":
dup_expr_grammar = extend_grammar(
EXPR_GRAMMAR, {
"<factor>": [
"+<factor>", "-<factor>", "(<expr>)",
"<integer-1>.<integer-2>", "<integer>"
],
"<integer-1>": ["<digit-1><integer-1>", "<digit-1>"],
"<integer-2>": ["<digit-2><integer-2>", "<digit-2>"],
"<digit-1>": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"],
"<digit-2>": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]
})
if __name__ == "__main__":
assert is_valid_grammar(dup_expr_grammar)
if __name__ == "__main__":
f = GrammarCoverageFuzzer(dup_expr_grammar, start_symbol="<factor>")
for i in range(10):
print(f.fuzz())
# ### Extending Grammars for Context Coverage Programmatically
if __name__ == "__main__":
print(pairs[:20])
def pairwise(option_list):
return [option_1 + option_2
for (option_1, option_2) in combinations(option_list, 2)]
if __name__ == "__main__":
print(pairwise(option_list)[:20])
if __name__ == "__main__":
notedown_grammar = notedown_runner.grammar()
pairwise_notedown_grammar = extend_grammar(notedown_grammar)
pairwise_notedown_grammar["<option>"] = pairwise(notedown_grammar["<option>"])
assert is_valid_grammar(pairwise_notedown_grammar)
if __name__ == "__main__":
notedown_fuzzer = GrammarCoverageFuzzer(
pairwise_notedown_grammar, max_nonterminals=4)
if __name__ == "__main__":
for i in range(10):
print(notedown_fuzzer.fuzz())
if __name__ == "__main__":
# ### Example: More Numeric Ranges
if __name__ == "__main__":
print('\n### Example: More Numeric Ranges')
if __name__ == "__main__":
expr_100_200_grammar = extend_grammar(
EXPR_GRAMMAR,
{
"<factor>": [
"+<factor>",
"-<factor>",
"(<expr>)",
# Generate only the integer part with a function;
# the fractional part comes from
# the grammar
("<integer>.<integer>",
opts(pre=lambda: [random.randint(100, 200), None])),
# Generate the entire integer
# from the function
("<integer>", opts(pre=lambda: random.randint(100, 200))),
],
})
if __name__ == "__main__":
expr_100_200_fuzzer = GeneratorGrammarFuzzer(expr_100_200_grammar)
expr_100_200_fuzzer.fuzz()
# ### Support for Python Generators
if __name__ == "__main__":
print('\n## Directed Fuzzing')
def set_prob(grammar, symbol, expansion, prob):
"""Set the probability of the given expansion of grammar[symbol]"""
set_opts(grammar, symbol, expansion, opts(prob=prob))
if __package__ is None or __package__ == "":
from Grammars import URL_GRAMMAR, extend_grammar
else:
from .Grammars import URL_GRAMMAR, extend_grammar
if __name__ == "__main__":
probabilistic_url_grammar = extend_grammar(URL_GRAMMAR)
set_prob(probabilistic_url_grammar, "<scheme>", "ftps", 0.8)
assert is_valid_probabilistic_grammar(probabilistic_url_grammar)
if __name__ == "__main__":
probabilistic_url_grammar["<scheme>"]
if __name__ == "__main__":
prob_url_fuzzer = ProbabilisticGrammarFuzzer(probabilistic_url_grammar)
for i in range(10):
print(prob_url_fuzzer.fuzz())
if __name__ == "__main__":
set_prob(probabilistic_url_grammar, "<scheme>", "ftps", 0.0)
assert is_valid_probabilistic_grammar(probabilistic_url_grammar)
def int_grammar_with_range(start, end):
int_grammar = extend_grammar(INT_GRAMMAR)
set_opts(int_grammar, "<int>", "<_int>",
opts(pre=lambda: random.randint(start, end)))
return int_grammar
def invert_probs(grammar):
inverted_grammar = extend_grammar(grammar)
for symbol in grammar:
inverted_grammar[symbol] = invert_expansion(grammar[symbol])
return inverted_grammar
def float_grammar_with_range(start, end):
float_grammar = extend_grammar(FLOAT_GRAMMAR)
set_opts(float_grammar, "<float>", "<_float>",
opts(pre=lambda: start + random.random() * (end - start)))
return float_grammar
