def generator_mode(): if description is not None: # description can be explicit yield 'description', 'meta', description # transform the yielded data, and add type annotations for data in fn(*args, **kw): # if not 2 items, then it is assumed to be already formatted with a type: # e.g. ("bls_setting", "meta", 1) if len(data) != 2: yield data continue # Try to infer the type, but keep it as-is if it's not a SSZ type or bytes. (key, value) = data if value is None: continue if isinstance(value, View): yield key, 'ssz', serialize(value) elif isinstance(value, bytes): yield key, 'ssz', value elif isinstance(value, list) and all([isinstance(el, (View, bytes)) for el in value]): for i, el in enumerate(value): if isinstance(el, View): yield f'{key}_{i}', 'ssz', serialize(el) elif isinstance(el, bytes): yield f'{key}_{i}', 'ssz', el yield f'{key}_count', 'meta', len(value) else: # Not a ssz value. # The data will now just be yielded as any python data, # something that should be encodable by the generator runner. yield key, 'data', value
def invalid_cases(): rng = Random(1234) for (name, (typ, offsets)) in PRESET_CONTAINERS.items(): # using mode_max_count, so that the extra byte cannot be picked up as normal list content yield f'{name}_extra_byte', \ invalid_test_case(lambda: serialize( container_case_fn(rng, RandomizationMode.mode_max_count, typ)) + b'\xff') if len(offsets) != 0: # Note: there are many more ways to have invalid offsets, # these are just example to get clients started looking into hardening ssz. for mode in [ RandomizationMode.mode_random, RandomizationMode.mode_nil_count, RandomizationMode.mode_one_count, RandomizationMode.mode_max_count ]: if len(offsets) != 0: for offset_index in offsets: yield f'{name}_offset_{offset_index}_plus_one', \ invalid_test_case(lambda: mod_offset( b=serialize(container_case_fn(rng, mode, typ)), offset_index=offset_index, change=lambda x: x + 1 )) yield f'{name}_offset_{offset_index}_zeroed', \ invalid_test_case(lambda: mod_offset( b=serialize(container_case_fn(rng, mode, typ)), offset_index=offset_index, change=lambda x: 0 ))
def invalid_cases(): # zero length vectors are illegal for (name, typ) in BASIC_TYPES.items(): yield f'vec_{name}_0', invalid_test_case(lambda: b'') rng = Random(1234) for (name, typ) in BASIC_TYPES.items(): random_modes = [ RandomizationMode.mode_zero, RandomizationMode.mode_max ] if name != 'bool': random_modes.append(RandomizationMode.mode_random) for length in [1, 2, 3, 4, 5, 8, 16, 31, 512, 513]: yield f'vec_{name}_{length}_nil', invalid_test_case(lambda: b'') for mode in random_modes: if length == 1: # empty bytes, no elements. It may seem valid, but empty fixed-size elements are not valid SSZ. yield f'vec_{name}_{length}_{mode.to_name()}_one_less', \ invalid_test_case(lambda: b"") else: yield f'vec_{name}_{length}_{mode.to_name()}_one_less', \ invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length - 1))) yield f'vec_{name}_{length}_{mode.to_name()}_one_more', \ invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length + 1))) yield f'vec_{name}_{length}_{mode.to_name()}_one_byte_less', \ invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length))[:-1]) yield f'vec_{name}_{length}_{mode.to_name()}_one_byte_more', \ invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length)) + serialize(basic_vector_case_fn(rng, mode, uint8, 1)))
def encode(value, include_hash_tree_roots=False): if isinstance(value, uint): # Larger uints are boxed and the class declares their byte length if value.__class__.type_byte_length() > 8: return str(int(value)) return int(value) elif isinstance(value, boolean): return value == 1 elif isinstance(value, (Bitlist, Bitvector)): return '0x' + serialize(value).hex() elif isinstance(value, list): # normal python lists return [encode(element, include_hash_tree_roots) for element in value] elif isinstance(value, (List, Vector)): return [encode(element, include_hash_tree_roots) for element in value] elif isinstance(value, bytes): # bytes, ByteList, ByteVector return '0x' + value.hex() elif isinstance(value, Container): ret = {} for field_name in value.fields().keys(): field_value = getattr(value, field_name) ret[field_name] = encode(field_value, include_hash_tree_roots) if include_hash_tree_roots: ret[field_name + "_hash_tree_root"] = '0x' + hash_tree_root( field_value).hex() if include_hash_tree_roots: ret["hash_tree_root"] = '0x' + hash_tree_root(value).hex() return ret else: raise Exception( f"Type not recognized: value={value}, typ={type(value)}")
def case_fn(): value = value_fn() yield "value", "data", encode(value) yield "serialized", "ssz", serialize(value) yield "root", "meta", '0x' + hash_tree_root(value).hex() if isinstance(value, Container): yield "signing_root", "meta", '0x' + signing_root(value).hex()
def test_decoder(): rng = Random(123) # check these types only, Block covers a lot of operation types already. for typ in [spec.AttestationDataAndCustodyBit, spec.BeaconState, spec.BeaconBlock]: # create a random pyspec value original = random_value.get_random_ssz_object(rng, typ, 100, 10, mode=random_value.RandomizationMode.mode_random, chaos=True) # serialize it, using pyspec pyspec_data = spec_ssz_impl.serialize(original) # get the py-ssz type for it block_sedes = translate_typ(typ) # try decoding using the py-ssz type raw_value = block_sedes.deserialize(pyspec_data) # serialize it using py-ssz pyssz_data = block_sedes.serialize(raw_value) # now check if the serialized form is equal. If so, we confirmed decoding and encoding to work. assert pyspec_data == pyssz_data # now translate the py-ssz value in a pyspec-value block = translate_value(raw_value, typ) # and see if the hash-tree-root of the original matches the hash-tree-root of the decoded & translated value. original_hash_tree_root = spec_ssz_impl.hash_tree_root(original) assert original_hash_tree_root == spec_ssz_impl.hash_tree_root(block) assert original_hash_tree_root == block_sedes.get_hash_tree_root(raw_value)
def create_test_case(rng: Random, typ, mode: random_value.RandomizationMode, chaos: bool) -> Iterable[gen_typing.TestCasePart]: value = random_value.get_random_ssz_object(rng, typ, MAX_BYTES_LENGTH, MAX_LIST_LENGTH, mode, chaos) yield "value", "data", encode.encode(value) yield "serialized", "ssz", serialize(value) roots_data = { "root": '0x' + hash_tree_root(value).hex() } yield "roots", "data", roots_data
def create_test_case(rng: Random, typ, mode: random_value.RandomizationMode, chaos: bool) -> Iterable[gen_typing.TestCasePart]: value = random_value.get_random_ssz_object(rng, typ, MAX_BYTES_LENGTH, MAX_LIST_LENGTH, mode, chaos) yield "value", "data", encode.encode(value) yield "serialized", "ssz", serialize(value) roots_data = {"root": '0x' + hash_tree_root(value).hex()} if isinstance(value, Container) and hasattr(value, "signature"): roots_data["signing_root"] = '0x' + signing_root(value).hex() yield "roots", "data", roots_data
def FuzzerRunOne(input_data: bytes) -> typing.Optional[bytes]: test_case = translate_value(deposit_sedes.deserialize(input_data), DepositTestCase) try: # modifies state in place spec.process_deposit(state=test_case.pre, deposit=test_case.deposit) return serialize(test_case.pre) except (AssertionError, IndexError): return None
def main(argv: typing.Optional[typing.Collection[str]] = None) -> int: op_registry = load_builtin_registry() args = get_args(argv, op_registry.keys()) if args.verbose: logging.getLogger().setLevel(logging.DEBUG) try: op_details = op_registry[args.operation_name] except KeyError as e: raise ValueError( f"Operation name '{args.operation_name}' not supported." ) from e op_dest = args.out_dir or pathlib.Path(op_details.name + "_corpora") if args.force: # TODO print warning and wait for user confirmation? shutil.rmtree(op_dest, ignore_errors=True) op_dest.mkdir(parents=True, exist_ok=True) args.state_out_dir.mkdir(parents=True, exist_ok=True) state_mapping, next_id = get_existing_states(args.state_out_dir) num_states_pre = len(state_mapping) logging.info("Found %s existing states.", num_states_pre) state_mapping, next_id = collect_found_states( args.search_root, args.state_out_dir, state_mapping, next_id ) logging.info( "Found and imported %s new states.", len(state_mapping) - num_states_pre ) test_names: typing.Set[str] = set() num_ops = 0 for op in get_operations(args.search_root, op_details): num_ops += 1 # Combine with every possible state for state_id in state_mapping.values(): test_case = op_details.test_type_factory(state_id, op) logging.debug("Created test case: %s", test_case) raw = serialize(test_case) # libfuzzer also uses sha1 names! out_path = op_dest / hashlib.sha1(raw).hexdigest() # this protects against duplicate test cases logging.debug("Saving to %s", out_path) out_path.write_bytes(raw) test_names.add(out_path.name) logging.info( "Wrote %s unique test cases from %s unique operations and %s states.", len(test_names), num_ops, len(state_mapping), ) return 0
def FuzzerRunOne(input_data: bytes) -> typing.Optional[bytes]: test_case = translate_value(block_header_sedes.deserialize(input_data), BlockHeaderTestCase) try: # modifies state in place spec.process_block_header(state=test_case.pre, block=test_case.block) # NOTE - signature verification should do nothing with bls disabled return serialize(test_case.pre) except (AssertionError, IndexError): return None
def invalid_cases(): yield 'bitlist_no_delimiter_empty', invalid_test_case(lambda: b'') yield 'bitlist_no_delimiter_zero_byte', invalid_test_case(lambda: b'\x00') yield 'bitlist_no_delimiter_zeroes', invalid_test_case( lambda: b'\x00\x00\x00') rng = Random(1234) for (typ_limit, test_limit) in [(1, 2), (1, 8), (1, 9), (2, 3), (3, 4), (4, 5), (5, 6), (8, 9), (32, 64), (32, 33), (512, 513)]: yield f'bitlist_{typ_limit}_but_{test_limit}', \ invalid_test_case(lambda: serialize( bitlist_case_fn(rng, RandomizationMode.mode_max_count, test_limit)))
def FuzzerRunOne(FuzzerInput): state_block = translate_value(state_block_sedes.deserialize(FuzzerInput), StateBlock) prestate = copy.deepcopy(prestates[state_block.stateID]) try: poststate = spec.state_transition(prestate, state_block.block, False) return serialize(poststate) except AssertionError as e: pass except IndexError: pass
def FuzzerRunOne(input_data: bytes) -> typing.Optional[bytes]: test_case = translate_value( proposer_slashing_sedes.deserialize(input_data), ProposerSlashingTestCase) try: # modifies state in place spec.process_proposer_slashing(test_case.pre, test_case.proposer_slashing) # NOTE - signature verification should do nothing with bls disabled return serialize(test_case.pre) except (AssertionError, IndexError): return None
def FuzzerRunOne(fuzzer_input): state_block = translate_value(block_sedes.deserialize(fuzzer_input), BlockTestCase) try: # NOTE we don't validate state root here poststate = spec.state_transition( state=state_block.pre, block=state_block.block, validate_state_root=VALIDATE_STATE_ROOT, ) return serialize(poststate) except (AssertionError, IndexError): return None
def invalid_cases(): # zero length bitvecors are illegal yield 'bitvec_0', invalid_test_case(lambda: b'') rng = Random(1234) # Create a vector with test_size bits, but make the type typ_size instead, # which is invalid when used with the given type size # (and a bit set just after typ_size bits if necessary to avoid the valid 0 padding-but-same-last-byte case) for (typ_size, test_size) in [(1, 2), (2, 3), (3, 4), (4, 5), (5, 6), (8, 9), (9, 8), (16, 8), (32, 33), (512, 513)]: for mode in [ RandomizationMode.mode_random, RandomizationMode.mode_zero, RandomizationMode.mode_max ]: yield f'bitvec_{typ_size}_{mode.to_name()}_{test_size}', \ invalid_test_case(lambda: serialize(bitvector_case_fn(rng, mode, test_size, invalid_making_pos=typ_size)))
def create_test_case_contents(value, typ): yield "value", encode.encode(value, typ) yield "serialized", '0x' + serialize(value).hex() yield "root", '0x' + hash_tree_root(value).hex() if hasattr(value, "signature"): yield "signing_root", '0x' + signing_root(value).hex()
def write_post_state(post_state, out_file): # Encode state as SSZ post_raw_ssz = spec_ssz_impl.serialize(post_state) # Write poststate write_or_stdout(post_raw_ssz, out_file)
def case_fn(): value = value_fn() yield "value", "data", encode(value) yield "serialized", "ssz", serialize(value) yield "root", "meta", '0x' + hash_tree_root(value).hex()