def test_file_set_during_grammar_load(self):
"""Ensure that __file__ is set when loading a new grammar file
"""
named_tmp = tempfile.NamedTemporaryFile()
named_tmp.write(gutils.binstr(r"""
from gramfuzz.fields import *
Def("file_test", __file__, cat="default")
"""))
named_tmp.flush()
# should not raise an exception
self.fuzzer.load_grammar(named_tmp.name)
output = self.fuzzer.gen(num=1, cat="default")
self.assertEqual(output[0], gutils.binstr(named_tmp.name))
def test_auto_process(self):
named_tmp = tempfile.NamedTemporaryFile()
named_tmp.write(gutils.binstr(r"""
import gramfuzz
from gramfuzz.fields import *
Def("test", "hello")
# should also get pruned
Def("other_test", Ref("not_reachable", cat="other"))
# should get pruned
Def("not_reachable", Ref("doesnt_exist", cat="other"), cat="other")
"""))
named_tmp.flush()
self.fuzzer.load_grammar(named_tmp.name)
named_tmp.close()
self.assertTrue(self.fuzzer._rules_processed == False)
data = self.fuzzer.gen(cat="default", num=1)[0]
self.assertTrue(self.fuzzer._rules_processed == True)
with self.assertRaises(gramfuzz.errors.GramFuzzError) as ctx:
self.fuzzer.get_ref("other", "not_reachable")
self.assertEqual(
str(ctx.exception),
"referenced definition ('not_reachable') not defined"
)
def test_or_probabilities(self):
"""Make sure that Or does its probabilities correctly
"""
# for two items, their probability of being generated should
# be 50%
hello_count = 0
int_count = 0
data = Or(UInt, "hello")
for x in six.moves.range(LOOP_NUM):
res = data.build()
val = gutils.val(res)
if val == b"hello":
hello_count += 1
elif re.match(gutils.binstr(r'^\d+$'), val) is not None:
int_count += 1
else:
self.assertTrue(False, "was neither an int or hello")
for v in [hello_count, int_count]:
percent = v / float(LOOP_NUM)
diff = abs(percent - 0.50)
self.assertLess(
diff, PERCENT_ERROR,
"{}/{} == {}% error, bad".format(v, LOOP_NUM, diff))
def __init__(self, name, *values, **options):
"""Create a new rule definition. Simply instantiating a new rule definition
will add it to the current ``GramFuzzer`` instance.
:param str name: The name of the rule being defined
:param list values: The list of values that define the value of the rule
(will be concatenated when built)
:param str cat: The category to create the rule in (default=``"default"``).
:param bool no_prune: If this rule should not be pruned *EVEN IF* it is found to be
unreachable (default=``False``)
"""
self.name = name
self.options = options
self.values = list(map(maybe_binstr, values))
self.sep = binstr(self.options.setdefault("sep", self.sep))
self.cat = self.options.setdefault("cat", self.cat)
self.no_prune = self.options.setdefault("no_prune", self.no_prune)
self.fuzzer = GramFuzzer.instance()
frame,mod_path,_,_,_,_ = inspect.stack()[1]
module_name = os.path.basename(mod_path).replace(".pyc", "").replace(".py", "")
if "TOP_CAT" in frame.f_locals:
self.fuzzer.cat_group_defaults[module_name] = frame.f_locals["TOP_CAT"]
self.fuzzer.add_definition(self.cat, self.name, self, no_prune=self.no_prune, gram_file=module_name)
def _repr_escape(self, val):
"""Perform a repr escape on 'val', trimming the ``b`` off of the
resulting data, if it exists.
"""
res = binstr(repr(val))
# this is safe - res after repr will always start with either a
# single quote or a double quote
if res.startswith(b"b"):
res = res[1:]
return res
def __init__(self, *values, **kwargs):
"""Create a new ``And`` field instance.
:param list values: The list of values to be concatenated
"""
self.sep = binstr(kwargs.setdefault("sep", self.sep))
self.values = list(map(maybe_binstr, values))
# to be used internally, is not intended to be set directly by a user
self.rolling = kwargs.setdefault("rolling", False)
self.fuzzer = GramFuzzer.instance()
def __init__(self, value=None, **kwargs):
"""Create a new instance of the ``String`` field.
:param value: The hard-coded value of the String field
:param int min: The minimum size of the String when built
:param int max: The maximum size of the String when built
:param str charset: The character-set to be used when building the string
"""
super(String, self).__init__(value, **kwargs)
self.charset = binstr(kwargs.setdefault("charset", self.charset))
def test_relative_grammar_import(self):
tmpdir = tempfile.mkdtemp()
try:
file1 = os.path.join(tmpdir, "file1.py")
file2 = os.path.join(tmpdir, "file2.py")
with open(file1, "wb") as f:
f.write(gutils.binstr(r"""
import gramfuzz
from gramfuzz.fields import *
import file2
TOP_CAT = "file1"
Def("file1def", Ref("file2def", cat=file2.TOP_CAT), cat="file1")
"""))
with open(file2, "wb") as f:
f.write(gutils.binstr(r"""
import gramfuzz
from gramfuzz.fields import *
TOP_CAT = "file2"
Def("file2def", "hi from file2", cat="file2")
"""))
self.fuzzer.load_grammar(file1)
res = self.fuzzer.gen(cat_group="file1", num=1)[0]
self.assertEqual(res, b"hi from file2")
# always clean up
finally:
shutil.rmtree(tmpdir)
def __init__(self, *values, **kwargs):
"""Create a new instance of the ``Join`` class.
:param list values: The values to join
:param str sep: The string with which to separate each of the values (default=``","``)
:param int max: The maximum number of times (inclusive) to build the first item in ``values``.
This can be useful when a variable number of items in a list is needed. E.g.:
.. code-block:: python
Join(Int, max=5, sep=",")
"""
self.values = list(map(maybe_binstr, values))
self.sep = binstr(kwargs.setdefault("sep", self.sep))
self.max = kwargs.setdefault("max", None)
def test_load_grammar(self):
named_tmp = tempfile.NamedTemporaryFile()
named_tmp.write(gutils.binstr(r"""
import gramfuzz
from gramfuzz.fields import *
Def("test_def", Join(Int, Int, Opt("hello"), sep="|"))
Def("test_def", Join(Float, Float, Opt("hello"), sep="|"))
Def("test_def_ref", "this.", String(min=1), "=", Q(Ref("test_def")))
"""))
named_tmp.flush()
self.fuzzer.load_grammar(named_tmp.name)
named_tmp.close()
res = self.fuzzer.get_ref("default", "test_def_ref").build()
def test_weighted_or_probabilities(self):
"""Make sure that WeightedOr does its probabilities correctly
"""
counts = {
"int": {
"prob": 0.1,
"val": UInt,
"count": 0,
"match": lambda x: re.match(gutils.binstr(r'^\d+$'), x) is not None
},
"hello": {
"prob": 0.6,
"val": b"hello",
"count": 0,
"match": lambda x: x == b"hello"
},
"a": {
"prob": 0.3,
"val": b"a",
"count": 0,
"match": lambda x: x == b"a"
},
}
values = [(v["val"], v["prob"]) for k, v in counts.items()]
data = WeightedOr(*values)
for x in six.moves.range(LOOP_NUM):
res = data.build()
val = gutils.val(res)
matched = False
for val_name, val_info in counts.items():
if val_info["match"](val):
val_info["count"] += 1
matched = True
if matched is False:
raise Exception("Something went wrong, could not match to a value")
for val_name, val_info in counts.items():
percent = val_info["count"] / float(LOOP_NUM)
diff = abs(percent - val_info["prob"])
self.assertLess(diff, PERCENT_ERROR, "{}: {}/{} == {}% error, bad".format(
val_name,
val_info["count"],
LOOP_NUM,
diff
))
def __init__(self, value=None, **kwargs):
"""Create a new instance of the ``String`` field.
:param value: The hard-coded value of the String field
:param int min: The minimum size of the String when built
:param int max: The maximum size of the String when built
:param str charset: The character-set to be used when building the string
"""
if "min" in kwargs or "max" in kwargs:
self.odds = []
self.min = kwargs.setdefault("min", self.min)
self.max = kwargs.setdefault("max", self.max)
self.odds = kwargs.setdefault("odds", self.odds)
self.value = value
self.charset = binstr(kwargs.setdefault("charset", self.charset))
def test_gen_cat_group(self):
named_tmp = tempfile.NamedTemporaryFile()
named_tmp.write(gutils.binstr(r"""
import gramfuzz
from gramfuzz.fields import *
TOP_CAT = "other"
# should get pruned since b isn't defined
Def("a", "hello", cat="other")
"""))
named_tmp.flush()
self.fuzzer.load_grammar(named_tmp.name)
cat_group = os.path.basename(named_tmp.name)
named_tmp.close()
res = self.fuzzer.gen(cat_group=cat_group, num=1)[0]
self.assertEqual(res, b"hello")
def test_no_prune_rules(self):
named_tmp = tempfile.NamedTemporaryFile()
named_tmp.write(gutils.binstr(r"""
import gramfuzz
from gramfuzz.fields import *
# should get pruned since b isn't defined
Def("a", UInt, Ref("b"), no_prune=True)
Def("c", UInt)
"""))
named_tmp.flush()
self.fuzzer.load_grammar(named_tmp.name)
named_tmp.close()
self.fuzzer.preprocess_rules()
self.assertIn("a", self.fuzzer.defs["default"])
self.assertIn("c", self.fuzzer.defs["default"])
def test_join_max(self):
num_items = {}
for x in six.moves.range(LOOP_NUM):
# should generate 0-9 items, not 10
j = Join(UInt, sep=",", max=10)
res = j.build()
self.assertRegexpMatches(res, gutils.binstr(r'^\d+(,\d+)*'))
sep_count = res.count(b",")
num_items.setdefault(sep_count, 0)
num_items[sep_count] += 1
self.assertEqual(len(num_items), 10)
for k, v in six.iteritems(num_items):
percent = v / float(LOOP_NUM)
diff = abs(percent - 0.10)
self.assertLess(
diff, PERCENT_ERROR,
"{}/{} == {}% error, bad".format(v, LOOP_NUM, diff))
def test_default_cat_for_cat_group(self):
named_tmp = tempfile.NamedTemporaryFile()
named_tmp.write(gutils.binstr(r"""
import gramfuzz
from gramfuzz.fields import *
TOP_CAT = "other"
# should get pruned since b isn't defined
Def("a", UInt, Ref("b", cat="other"), cat="other")
Def("c", UInt, cat="other")
"""))
named_tmp.flush()
self.fuzzer.load_grammar(named_tmp.name)
cat_group = os.path.basename(named_tmp.name)
named_tmp.close()
self.assertIn(cat_group, self.fuzzer.cat_group_defaults)
self.assertEqual(self.fuzzer.cat_group_defaults[cat_group], "other")
def test_or_explicit(self):
data = Or(UInt, "hello")
res = data.build()
val = gutils.val(res)
self.assertRegexpMatches(val, gutils.binstr(r'^(\d+|hello)'))
def test_or_operator(self):
data = UInt | "hello"
res = data.build()
val = gutils.val(res)
self.assertRegexpMatches(val, gutils.binstr(r'^(\d+|hello)'))
def test_and_explicit(self):
data = And(UInt, ",", UInt)
res = data.build()
val = gutils.val(res)
self.assertRegexpMatches(val, gutils.binstr(r'^\d+,\d+$'))
def test_and_operator(self):
data = UInt & "," & UInt
res = data.build()
val = gutils.val(res)
self.assertRegexpMatches(val, gutils.binstr(r'^\d+,\d+$'))
def test_join_fields2(self):
j = Join(UInt, "b", sep="X")
res = j.build()
self.assertRegexpMatches(res, gutils.binstr(r'^\d+Xb'))
def test_string_charset(self):
s = String(charset="abc")
res = s.build()
self.assertRegexpMatches(res, gutils.binstr(r'^[abc]*$'))
class String(Field):
"""Defines a string field
"""
min = 0
max = 0x100
odds = [
(0.85, [0, 20]),
(0.10, 1),
(0.025, 0),
(0.025, [20, 100]),
]
"""Unlike numeric ``Field`` types, the odds value for the ``String`` field
defines the *length* of the field, not characters used in the string.
See the :any:`gramfuzz.fields.Field.odds` member for details on the format of the ``odds`` probability
list.
"""
charset_alpha_lower = b"abcdefghijklmnopqrstuvwxyz"
"""A lower-case alphabet character set
"""
charset_alpha_upper = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
"""An upper-case alphabet character set
"""
charset_alpha = charset_alpha_lower + charset_alpha_upper
"""Upper- and lower-case alphabet
"""
charset_spaces = b"\n\r\t "
"""Whitespace character set
"""
charset_num = b"1234567890"
"""Numeric character set
"""
charset_alphanum = charset_alpha + charset_num
"""Alpha-numeric character set (upper- and lower-case alphabet + numbers)
"""
charset_all = b"".join(binstr(chr(x)) for x in six.moves.range(0x100))
"""All possible binary characters (``0x0-0xff``)
"""
charset = charset_alpha
"""The default character set of the ``String`` field class (default=charset_alpha)
"""
def __init__(self, value=None, **kwargs):
"""Create a new instance of the ``String`` field.
:param value: The hard-coded value of the String field
:param int min: The minimum size of the String when built
:param int max: The maximum size of the String when built
:param str charset: The character-set to be used when building the string
"""
if "min" in kwargs or "max" in kwargs:
self.odds = []
self.min = kwargs.setdefault("min", self.min)
self.max = kwargs.setdefault("max", self.max)
self.odds = kwargs.setdefault("odds", self.odds)
self.value = value
self.charset = binstr(kwargs.setdefault("charset", self.charset))
def build(self, pre=None, shortest=False):
"""Build the String instance
:param list pre: The prerequisites list (optional, default=None)
:param bool shortest: Whether or not the shortest reference-chain (most minimal) version of the field should be generated.
"""
if pre is None:
pre = []
if self.value is not None and rand.maybe():
return utils.val(self.value, pre, shortest=shortest)
length = self._odds_val()
res = rand.data(length, self.charset)
return res
def test_uint(self):
i = UInt
res = i().build()
val = gutils.val(res)
self.assertRegexpMatches(val, gutils.binstr(r'^\d+$'))
def test_ufloat(self):
f = UFloat
res = f().build()
val = gutils.val(res)
self.assertRegexpMatches(val, gutils.binstr(r'^\d+(\.\d+)?$'))
