def generator_mode():
if description is not None:
# description can be explicit
yield 'description', 'meta', description
# transform the yielded data, and add type annotations
for data in fn(*args, **kw):
# if not 2 items, then it is assumed to be already formatted with a type:
# e.g. ("bls_setting", "meta", 1)
if len(data) != 2:
yield data
continue
# Try to infer the type, but keep it as-is if it's not a SSZ type or bytes.
(key, value) = data
if value is None:
continue
if isinstance(value, View):
yield key, 'ssz', serialize(value)
elif isinstance(value, bytes):
yield key, 'ssz', value
elif isinstance(value, list) and all([isinstance(el, (View, bytes)) for el in value]):
for i, el in enumerate(value):
if isinstance(el, View):
yield f'{key}_{i}', 'ssz', serialize(el)
elif isinstance(el, bytes):
yield f'{key}_{i}', 'ssz', el
yield f'{key}_count', 'meta', len(value)
else:
# Not a ssz value.
# The data will now just be yielded as any python data,
# something that should be encodable by the generator runner.
yield key, 'data', value
def invalid_cases():
rng = Random(1234)
for (name, (typ, offsets)) in PRESET_CONTAINERS.items():
# using mode_max_count, so that the extra byte cannot be picked up as normal list content
yield f'{name}_extra_byte', \
invalid_test_case(lambda: serialize(
container_case_fn(rng, RandomizationMode.mode_max_count, typ)) + b'\xff')
if len(offsets) != 0:
# Note: there are many more ways to have invalid offsets,
# these are just example to get clients started looking into hardening ssz.
for mode in [
RandomizationMode.mode_random,
RandomizationMode.mode_nil_count,
RandomizationMode.mode_one_count,
RandomizationMode.mode_max_count
]:
if len(offsets) != 0:
for offset_index in offsets:
yield f'{name}_offset_{offset_index}_plus_one', \
invalid_test_case(lambda: mod_offset(
b=serialize(container_case_fn(rng, mode, typ)),
offset_index=offset_index,
change=lambda x: x + 1
))
yield f'{name}_offset_{offset_index}_zeroed', \
invalid_test_case(lambda: mod_offset(
b=serialize(container_case_fn(rng, mode, typ)),
offset_index=offset_index,
change=lambda x: 0
))
def invalid_cases():
# zero length vectors are illegal
for (name, typ) in BASIC_TYPES.items():
yield f'vec_{name}_0', invalid_test_case(lambda: b'')
rng = Random(1234)
for (name, typ) in BASIC_TYPES.items():
random_modes = [
RandomizationMode.mode_zero, RandomizationMode.mode_max
]
if name != 'bool':
random_modes.append(RandomizationMode.mode_random)
for length in [1, 2, 3, 4, 5, 8, 16, 31, 512, 513]:
yield f'vec_{name}_{length}_nil', invalid_test_case(lambda: b'')
for mode in random_modes:
if length == 1:
# empty bytes, no elements. It may seem valid, but empty fixed-size elements are not valid SSZ.
yield f'vec_{name}_{length}_{mode.to_name()}_one_less', \
invalid_test_case(lambda: b"")
else:
yield f'vec_{name}_{length}_{mode.to_name()}_one_less', \
invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length - 1)))
yield f'vec_{name}_{length}_{mode.to_name()}_one_more', \
invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length + 1)))
yield f'vec_{name}_{length}_{mode.to_name()}_one_byte_less', \
invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length))[:-1])
yield f'vec_{name}_{length}_{mode.to_name()}_one_byte_more', \
invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length))
+ serialize(basic_vector_case_fn(rng, mode, uint8, 1)))
def encode(value, include_hash_tree_roots=False):
if isinstance(value, uint):
# Larger uints are boxed and the class declares their byte length
if value.__class__.type_byte_length() > 8:
return str(int(value))
return int(value)
elif isinstance(value, boolean):
return value == 1
elif isinstance(value, (Bitlist, Bitvector)):
return '0x' + serialize(value).hex()
elif isinstance(value, list): # normal python lists
return [encode(element, include_hash_tree_roots) for element in value]
elif isinstance(value, (List, Vector)):
return [encode(element, include_hash_tree_roots) for element in value]
elif isinstance(value, bytes): # bytes, ByteList, ByteVector
return '0x' + value.hex()
elif isinstance(value, Container):
ret = {}
for field_name in value.fields().keys():
field_value = getattr(value, field_name)
ret[field_name] = encode(field_value, include_hash_tree_roots)
if include_hash_tree_roots:
ret[field_name + "_hash_tree_root"] = '0x' + hash_tree_root(
field_value).hex()
if include_hash_tree_roots:
ret["hash_tree_root"] = '0x' + hash_tree_root(value).hex()
return ret
else:
raise Exception(
f"Type not recognized: value={value}, typ={type(value)}")
def case_fn():
value = value_fn()
yield "value", "data", encode(value)
yield "serialized", "ssz", serialize(value)
yield "root", "meta", '0x' + hash_tree_root(value).hex()
if isinstance(value, Container):
yield "signing_root", "meta", '0x' + signing_root(value).hex()
def test_decoder():
rng = Random(123)
# check these types only, Block covers a lot of operation types already.
for typ in [spec.AttestationDataAndCustodyBit, spec.BeaconState, spec.BeaconBlock]:
# create a random pyspec value
original = random_value.get_random_ssz_object(rng, typ, 100, 10,
mode=random_value.RandomizationMode.mode_random,
chaos=True)
# serialize it, using pyspec
pyspec_data = spec_ssz_impl.serialize(original)
# get the py-ssz type for it
block_sedes = translate_typ(typ)
# try decoding using the py-ssz type
raw_value = block_sedes.deserialize(pyspec_data)
# serialize it using py-ssz
pyssz_data = block_sedes.serialize(raw_value)
# now check if the serialized form is equal. If so, we confirmed decoding and encoding to work.
assert pyspec_data == pyssz_data
# now translate the py-ssz value in a pyspec-value
block = translate_value(raw_value, typ)
# and see if the hash-tree-root of the original matches the hash-tree-root of the decoded & translated value.
original_hash_tree_root = spec_ssz_impl.hash_tree_root(original)
assert original_hash_tree_root == spec_ssz_impl.hash_tree_root(block)
assert original_hash_tree_root == block_sedes.get_hash_tree_root(raw_value)
def create_test_case(rng: Random, typ, mode: random_value.RandomizationMode, chaos: bool) -> Iterable[gen_typing.TestCasePart]:
value = random_value.get_random_ssz_object(rng, typ, MAX_BYTES_LENGTH, MAX_LIST_LENGTH, mode, chaos)
yield "value", "data", encode.encode(value)
yield "serialized", "ssz", serialize(value)
roots_data = {
"root": '0x' + hash_tree_root(value).hex()
}
yield "roots", "data", roots_data
def create_test_case(rng: Random, typ, mode: random_value.RandomizationMode,
chaos: bool) -> Iterable[gen_typing.TestCasePart]:
value = random_value.get_random_ssz_object(rng, typ, MAX_BYTES_LENGTH,
MAX_LIST_LENGTH, mode, chaos)
yield "value", "data", encode.encode(value)
yield "serialized", "ssz", serialize(value)
roots_data = {"root": '0x' + hash_tree_root(value).hex()}
if isinstance(value, Container) and hasattr(value, "signature"):
roots_data["signing_root"] = '0x' + signing_root(value).hex()
yield "roots", "data", roots_data
def FuzzerRunOne(input_data: bytes) -> typing.Optional[bytes]:
test_case = translate_value(deposit_sedes.deserialize(input_data),
DepositTestCase)
try:
# modifies state in place
spec.process_deposit(state=test_case.pre, deposit=test_case.deposit)
return serialize(test_case.pre)
except (AssertionError, IndexError):
return None
def main(argv: typing.Optional[typing.Collection[str]] = None) -> int:
op_registry = load_builtin_registry()
args = get_args(argv, op_registry.keys())
if args.verbose:
logging.getLogger().setLevel(logging.DEBUG)
try:
op_details = op_registry[args.operation_name]
except KeyError as e:
raise ValueError(
f"Operation name '{args.operation_name}' not supported."
) from e
op_dest = args.out_dir or pathlib.Path(op_details.name + "_corpora")
if args.force:
# TODO print warning and wait for user confirmation?
shutil.rmtree(op_dest, ignore_errors=True)
op_dest.mkdir(parents=True, exist_ok=True)
args.state_out_dir.mkdir(parents=True, exist_ok=True)
state_mapping, next_id = get_existing_states(args.state_out_dir)
num_states_pre = len(state_mapping)
logging.info("Found %s existing states.", num_states_pre)
state_mapping, next_id = collect_found_states(
args.search_root, args.state_out_dir, state_mapping, next_id
)
logging.info(
"Found and imported %s new states.", len(state_mapping) - num_states_pre
)
test_names: typing.Set[str] = set()
num_ops = 0
for op in get_operations(args.search_root, op_details):
num_ops += 1
# Combine with every possible state
for state_id in state_mapping.values():
test_case = op_details.test_type_factory(state_id, op)
logging.debug("Created test case: %s", test_case)
raw = serialize(test_case)
# libfuzzer also uses sha1 names!
out_path = op_dest / hashlib.sha1(raw).hexdigest()
# this protects against duplicate test cases
logging.debug("Saving to %s", out_path)
out_path.write_bytes(raw)
test_names.add(out_path.name)
logging.info(
"Wrote %s unique test cases from %s unique operations and %s states.",
len(test_names),
num_ops,
len(state_mapping),
)
return 0
def FuzzerRunOne(input_data: bytes) -> typing.Optional[bytes]:
test_case = translate_value(block_header_sedes.deserialize(input_data),
BlockHeaderTestCase)
try:
# modifies state in place
spec.process_block_header(state=test_case.pre, block=test_case.block)
# NOTE - signature verification should do nothing with bls disabled
return serialize(test_case.pre)
except (AssertionError, IndexError):
return None
def invalid_cases():
yield 'bitlist_no_delimiter_empty', invalid_test_case(lambda: b'')
yield 'bitlist_no_delimiter_zero_byte', invalid_test_case(lambda: b'\x00')
yield 'bitlist_no_delimiter_zeroes', invalid_test_case(
lambda: b'\x00\x00\x00')
rng = Random(1234)
for (typ_limit, test_limit) in [(1, 2), (1, 8), (1, 9), (2, 3), (3, 4),
(4, 5), (5, 6), (8, 9), (32, 64), (32, 33),
(512, 513)]:
yield f'bitlist_{typ_limit}_but_{test_limit}', \
invalid_test_case(lambda: serialize(
bitlist_case_fn(rng, RandomizationMode.mode_max_count, test_limit)))
def FuzzerRunOne(FuzzerInput):
state_block = translate_value(state_block_sedes.deserialize(FuzzerInput),
StateBlock)
prestate = copy.deepcopy(prestates[state_block.stateID])
try:
poststate = spec.state_transition(prestate, state_block.block, False)
return serialize(poststate)
except AssertionError as e:
pass
except IndexError:
pass
def FuzzerRunOne(input_data: bytes) -> typing.Optional[bytes]:
test_case = translate_value(
proposer_slashing_sedes.deserialize(input_data),
ProposerSlashingTestCase)
try:
# modifies state in place
spec.process_proposer_slashing(test_case.pre,
test_case.proposer_slashing)
# NOTE - signature verification should do nothing with bls disabled
return serialize(test_case.pre)
except (AssertionError, IndexError):
return None
def FuzzerRunOne(fuzzer_input):
state_block = translate_value(block_sedes.deserialize(fuzzer_input),
BlockTestCase)
try:
# NOTE we don't validate state root here
poststate = spec.state_transition(
state=state_block.pre,
block=state_block.block,
validate_state_root=VALIDATE_STATE_ROOT,
)
return serialize(poststate)
except (AssertionError, IndexError):
return None
def invalid_cases():
# zero length bitvecors are illegal
yield 'bitvec_0', invalid_test_case(lambda: b'')
rng = Random(1234)
# Create a vector with test_size bits, but make the type typ_size instead,
# which is invalid when used with the given type size
# (and a bit set just after typ_size bits if necessary to avoid the valid 0 padding-but-same-last-byte case)
for (typ_size, test_size) in [(1, 2), (2, 3), (3, 4), (4, 5),
(5, 6), (8, 9), (9, 8), (16, 8), (32, 33),
(512, 513)]:
for mode in [
RandomizationMode.mode_random, RandomizationMode.mode_zero,
RandomizationMode.mode_max
]:
yield f'bitvec_{typ_size}_{mode.to_name()}_{test_size}', \
invalid_test_case(lambda: serialize(bitvector_case_fn(rng, mode, test_size,
invalid_making_pos=typ_size)))
def create_test_case_contents(value, typ):
yield "value", encode.encode(value, typ)
yield "serialized", '0x' + serialize(value).hex()
yield "root", '0x' + hash_tree_root(value).hex()
if hasattr(value, "signature"):
yield "signing_root", '0x' + signing_root(value).hex()
def write_post_state(post_state, out_file):
# Encode state as SSZ
post_raw_ssz = spec_ssz_impl.serialize(post_state)
# Write poststate
write_or_stdout(post_raw_ssz, out_file)
def case_fn():
value = value_fn()
yield "value", "data", encode(value)
yield "serialized", "ssz", serialize(value)
yield "root", "meta", '0x' + hash_tree_root(value).hex()
