def run_test(filename, testname, testdata): testdata = fixture_to_bytes(testdata) try: rlpdata = decode_hex(testdata["rlp"][2:]) print rlpdata.encode('hex') o = {} tx = rlp.decode(rlpdata, transactions.Transaction) o["sender"] = tx.sender o["transaction"] = { "data": b'0x' * (len(tx.data) > 0) + encode_hex(tx.data), "gasLimit": str_to_bytes(str(tx.startgas)), "gasPrice": str_to_bytes(str(tx.gasprice)), "nonce": str_to_bytes(str(tx.nonce)), "r": b'0x' + encode_hex(utils.zpad(utils.int_to_big_endian(tx.r), 32)), "s": b'0x' + encode_hex(utils.zpad(utils.int_to_big_endian(tx.s), 32)), "v": str_to_bytes(str(tx.v)), "value": str_to_bytes(str(tx.value)), "to": encode_hex(tx.to), } except: tx = None if 'transaction' not in testdata: # expected to fail assert tx is None else: assert set(o['transaction'].keys()) == set( testdata.get("transaction", dict()).keys()) o.get("transaction", None) == testdata.get("transaction", None) assert encode_hex(o.get("sender", '')) == testdata.get("sender", '')
def test_transaction(filename, testname, testdata): testdata = fixture_to_bytes(testdata) try: rlpdata = decode_hex(testdata["rlp"][2:]) o = {} tx = rlp.decode(rlpdata, transactions.Transaction) blknum = int(testdata["blocknumber"]) if blknum >= config.default_config["HOMESTEAD_FORK_BLKNUM"]: tx.check_low_s() o["sender"] = tx.sender o["transaction"] = { "data": b'0x' * (len(tx.data) > 0) + encode_hex(tx.data), "gasLimit": str_to_bytes(str(tx.startgas)), "gasPrice": str_to_bytes(str(tx.gasprice)), "nonce": str_to_bytes(str(tx.nonce)), "r": b'0x' + encode_hex(utils.zpad(utils.int_to_big_endian(tx.r), 32)), "s": b'0x' + encode_hex(utils.zpad(utils.int_to_big_endian(tx.s), 32)), "v": str_to_bytes(str(tx.v)), "value": str_to_bytes(str(tx.value)), "to": encode_hex(tx.to), } except Exception as e: tx = None sys.stderr.write(str(e)) if 'transaction' not in testdata: # expected to fail assert tx is None else: assert set(o['transaction'].keys()) == set( testdata.get("transaction", dict()).keys()) o.get("transaction", None) == testdata.get("transaction", None) assert encode_hex(o.get("sender", '')) == testdata.get("sender", '')
def test_transaction(filename, testname, testdata): testdata = fixture_to_bytes(testdata) try: rlpdata = decode_hex(testdata["rlp"][2:]) o = {} tx = rlp.decode(rlpdata, transactions.Transaction) blknum = int(testdata["blocknumber"]) if blknum >= config.default_config["HOMESTEAD_FORK_BLKNUM"]: tx.check_low_s() o["sender"] = tx.sender o["transaction"] = { "data": b'0x' * (len(tx.data) > 0) + encode_hex(tx.data), "gasLimit": str_to_bytes(str(tx.startgas)), "gasPrice": str_to_bytes(str(tx.gasprice)), "nonce": str_to_bytes(str(tx.nonce)), "r": b'0x' + encode_hex(utils.zpad(utils.int_to_big_endian(tx.r), 32)), "s": b'0x' + encode_hex(utils.zpad(utils.int_to_big_endian(tx.s), 32)), "v": str_to_bytes(str(tx.v)), "value": str_to_bytes(str(tx.value)), "to": encode_hex(tx.to), } except Exception as e: tx = None sys.stderr.write(str(e)) if 'transaction' not in testdata: # expected to fail assert tx is None else: assert set(o['transaction'].keys()) == set(testdata.get("transaction", dict()).keys()) o.get("transaction", None) == testdata.get("transaction", None) assert encode_hex(o.get("sender", '')) == testdata.get("sender", '')
def run_test(filename, testname, testdata): testdata = fixture_to_bytes(testdata) try: rlpdata = decode_hex(testdata["rlp"][2:]) print rlpdata.encode("hex") o = {} tx = rlp.decode(rlpdata, transactions.Transaction) o["sender"] = tx.sender o["transaction"] = { "data": b"0x" * (len(tx.data) > 0) + encode_hex(tx.data), "gasLimit": str_to_bytes(str(tx.startgas)), "gasPrice": str_to_bytes(str(tx.gasprice)), "nonce": str_to_bytes(str(tx.nonce)), "r": b"0x" + encode_hex(utils.zpad(utils.int_to_big_endian(tx.r), 32)), "s": b"0x" + encode_hex(utils.zpad(utils.int_to_big_endian(tx.s), 32)), "v": str_to_bytes(str(tx.v)), "value": str_to_bytes(str(tx.value)), "to": encode_hex(tx.to), } except: tx = None if "transaction" not in testdata: # expected to fail assert tx is None else: assert set(o["transaction"].keys()) == set(testdata.get("transaction", dict()).keys()) o.get("transaction", None) == testdata.get("transaction", None) assert encode_hex(o.get("sender", "")) == testdata.get("sender", "")
def host_port_pubkey_from_uri(uri): b_node_uri_scheme = str_to_bytes(node_uri_scheme) b_uri = str_to_bytes(uri) assert b_uri.startswith(b_node_uri_scheme) and \ b'@' in b_uri and b':' in b_uri, b_uri pubkey_hex, ip_port = b_uri[len(b_node_uri_scheme):].split(b'@') assert len(pubkey_hex) == 2 * 512 // 8 ip, port = ip_port.split(b':') return ip, port, decode_hex(pubkey_hex)
def _iter_branch(self, node): '''yield (key, value) stored in this and the descendant nodes :param node: node in form of list, or BLANK_NODE .. note:: Here key is in full form, rather than key of the individual node ''' if node == BLANK_NODE: raise StopIteration node_type = self._get_node_type(node) if is_key_value_type(node_type): nibbles = without_terminator(unpack_to_nibbles(node[0])) key = b'+'.join([to_string(x) for x in nibbles]) if node_type == NODE_TYPE_EXTENSION: sub_tree = self._iter_branch(self._decode_to_node(node[1])) else: sub_tree = [(to_string(NIBBLE_TERMINATOR), node[1])] # prepend key of this node to the keys of children for sub_key, sub_value in sub_tree: full_key = (key + b'+' + sub_key).strip(b'+') yield (full_key, sub_value) elif node_type == NODE_TYPE_BRANCH: for i in range(16): sub_tree = self._iter_branch(self._decode_to_node(node[i])) for sub_key, sub_value in sub_tree: full_key = (str_to_bytes(str(i)) + b'+' + sub_key).strip(b'+') yield (full_key, sub_value) if node[16]: yield (to_string(NIBBLE_TERMINATOR), node[-1])
def _mkpingid(self, echoed, node): assert node.pubkey pid = str_to_bytes(echoed) + node.pubkey log.debug('mkpingid', echoed=encode_hex(echoed), node=encode_hex(node.pubkey)) return pid
def to_bytes(value): if isinstance(value, str): return utils.str_to_bytes(value) elif isinstance(value, list): return [to_bytes(v) for v in value] else: return value
def test_normal_frame(self): cipher1, cipher2 = self.setup_ciphers() dummy_msg_type = 1 dummy_payload = helpers.generate_bytearray(123) dummy_protocol = 0 frames = frame_utils.get_frames( dummy_msg_type, memoryview(dummy_payload), dummy_protocol, eth_common_constants.DEFAULT_FRAME_SIZE) input_buffer = InputBuffer() for frame in frames: encrypted_frame = cipher1.encrypt_frame(frame) encrypted_frame_bytes = bytearray( rlp_utils.str_to_bytes(encrypted_frame)) input_buffer.add_bytes(encrypted_frame_bytes) # adding some dummy bytes but less than header size len dummy_bytes_len = int(eth_common_constants.FRAME_HDR_TOTAL_LEN / 2) input_buffer.add_bytes(helpers.generate_bytearray(dummy_bytes_len)) framed_input_buffer = FramedInputBuffer(cipher2) is_full, msg_type = framed_input_buffer.peek_message(input_buffer) self.assertTrue(is_full) self.assertEqual(msg_type, dummy_msg_type) message, full_msg_type = framed_input_buffer.get_full_message() self.assertEqual(message, dummy_payload) self.assertEqual(full_msg_type, dummy_msg_type) self.assertEqual(input_buffer.length, dummy_bytes_len)
def _to_dict(self, node): if node == BLANK_NODE: return {} node_type = self._get_node_type(node) if is_key_value_type(node_type): nibbles = without_terminator(unpack_to_nibbles(node[0])) key = b'+'.join([utils.to_string(x) for x in nibbles]) if node_type == NODE_TYPE_EXTENSION: sub_dict = self._to_dict(self._decode_to_node(node[1])) else: sub_dict = {utils.to_string(NIBBLE_TERMINATOR): node[1]} res = {} for sub_key, sub_value in sub_dict.items(): full_key = (key + b'+' + sub_key).strip(b'+') res[full_key] = sub_value return res elif node_type == NODE_TYPE_BRANCH: res = {} for i in range(16): sub_dict = self._to_dict(self._decode_to_node(node[i])) for sub_key, sub_value in sub_dict.items(): full_key = ( str_to_bytes( str(i)) + b'+' + sub_key).strip(b'+') res[full_key] = sub_value if node[16]: res[utils.to_string(NIBBLE_TERMINATOR)] = node[-1] return res
def _iter_branch(self, node): """yield (key, value) stored in this and the descendant nodes :param node: node in form of list, or BLANK_NODE .. note:: Here key is in full form, rather than key of the individual node """ if node == BLANK_NODE: raise StopIteration node_type = self._get_node_type(node) if is_key_value_type(node_type): nibbles = without_terminator(unpack_to_nibbles(node[0])) key = b'+'.join([to_string(x) for x in nibbles]) if node_type == NODE_TYPE_EXTENSION: sub_tree = self._iter_branch(self._decode_to_node(node[1])) else: sub_tree = [(to_string(NIBBLE_TERMINATOR), node[1])] # prepend key of this node to the keys of children for sub_key, sub_value in sub_tree: full_key = (key + b'+' + sub_key).strip(b'+') yield (full_key, sub_value) elif node_type == NODE_TYPE_BRANCH: for i in range(16): sub_tree = self._iter_branch(self._decode_to_node(node[i])) for sub_key, sub_value in sub_tree: full_key = (str_to_bytes(str(i)) + b'+' + sub_key).strip(b'+') yield (full_key, sub_value) if node[16]: yield (to_string(NIBBLE_TERMINATOR), node[-1])
def _iter_branch(self, node): if node == BLANK_NODE: raise StopIteration node_type = self._get_node_type(node) if is_key_value_type(node_type): nibbles = without_terminator(unpack_to_nibbles(node[0])) key = b'+'.join([utils.to_string(x) for x in nibbles]) if node_type == NODE_TYPE_EXTENSION: sub_tree = self._iter_branch(self._decode_to_node(node[1])) else: sub_tree = [(utils.to_string(NIBBLE_TERMINATOR), node[1])] for sub_key, sub_value in sub_tree: full_key = (key + b'+' + sub_key).strip(b'+') yield (full_key, sub_value) elif node_type == NODE_TYPE_BRANCH: for i in range(16): sub_tree = self._iter_branch(self._decode_to_node(node[i])) for sub_key, sub_value in sub_tree: full_key = ( str_to_bytes( str(i)) + b'+' + sub_key).strip(b'+') yield (full_key, sub_value) if node[16]: yield (utils.to_string(NIBBLE_TERMINATOR), node[-1])
def fixture_to_bytes(value): if isinstance(value, str): return str_to_bytes(value) elif isinstance(value, list): return [fixture_to_bytes(v) for v in value] elif isinstance(value, dict): ret = {} for k, v in list(value.items()): if isinstance(k, str) and (len(k) == 40 or k[:2] == '0x'): key = str_to_bytes(k) else: key = k ret[key] = fixture_to_bytes(v) return ret else: return value
def intrinsic_gas_used(self): num_zero_bytes = str_to_bytes(self.data).count(ascii_chr(0)) num_non_zero_bytes = len(self.data) - num_zero_bytes return (opcodes.GTXCOST # + (0 if self.to else opcodes.CREATE[3]) + opcodes.GTXDATAZERO * num_zero_bytes + opcodes.GTXDATANONZERO * num_non_zero_bytes)
class MockProtocol(devp2p.protocol.BaseProtocol): protocol_id = n max_cmd_id = size name = str_to_bytes('mock%d' % n) version = 1 def __init__(self, *args, **kwargs): super(MockProtocol, self).__init__(*args, **kwargs) self.cmd_by_id = ['mock_cmd%d' % i for i in range(size + 1)]
def intrinsic_gas_used(self): num_zero_bytes = str_to_bytes(self.data).count(ascii_chr(0)) num_non_zero_bytes = len(self.data) - num_zero_bytes return ( opcodes.GTXCOST + (opcodes.CREATE[3] if not self.to else 0) + opcodes.GTXDATAZERO * num_zero_bytes + opcodes.GTXDATANONZERO * num_non_zero_bytes + (opcodes.GTXXSHARDCOST if self.is_cross_shard() else 0) )
def ecies_encrypt(cls, data, raw_pubkey, shared_mac_data=''): """ ECIES Encrypt, where P = recipient public key is: 1) generate r = random value 2) generate shared-secret = kdf( ecdhAgree(r, P) ) 3) generate R = rG [same op as generating a public key] 4) send 0x04 || R || AsymmetricEncrypt(shared-secret, plaintext) || tag currently used by go: ECIES_AES128_SHA256 = &ECIESParams{ Hash: sha256.New, hashAlgo: crypto.SHA256, Cipher: aes.NewCipher, BlockSize: aes.BlockSize, KeyLen: 16, } """ # 1) generate r = random value ephem = cls(None) # 2) generate shared-secret = kdf( ecdhAgree(r, P) ) key_material = ephem.raw_get_ecdh_key(pubkey_x=raw_pubkey[:32], pubkey_y=raw_pubkey[32:]) assert len(key_material) == 32 key = eciesKDF(key_material, 32) assert len(key) == 32 key_enc, key_mac = key[:16], key[16:] key_mac = hashlib.sha256(key_mac).digest() # !!! assert len(key_mac) == 32 # 3) generate R = rG [same op as generating a public key] ephem_pubkey = ephem.raw_pubkey # encrypt iv = pyelliptic.Cipher.gen_IV(cls.ecies_ciphername) assert len(iv) == 16 ctx = pyelliptic.Cipher(key_enc, iv, 1, cls.ecies_ciphername) ciphertext = ctx.ciphering(data) assert len(ciphertext) == len(data) # 4) send 0x04 || R || AsymmetricEncrypt(shared-secret, plaintext) # || tag msg = rlp_utils.ascii_chr(0x04) + ephem_pubkey + iv + ciphertext # the MAC of a message (called the tag) as per SEC 1, 3.5. tag = pyelliptic.hmac_sha256( key_mac, msg[1 + 64:] + rlp_utils.str_to_bytes(shared_mac_data)) assert len(tag) == 32 msg += tag assert len(msg) == 1 + 64 + 16 + 32 + len(data) == 113 + len(data) assert len(msg) - cls.ecies_encrypt_overhead_length == len(data) return msg
def _encode_node(self, node, put_in_db=True): if node == BLANK_NODE: return BLANK_NODE rlpnode = encode_optimized(node) if len(rlpnode) < 32: return node hashkey = utils.sha3(rlpnode) if put_in_db: self.db.put(hashkey, str_to_bytes(rlpnode)) return hashkey
def _encode_node(self, node, put_in_db=True): if node == BLANK_NODE: return BLANK_NODE # assert isinstance(node, list) rlpnode = rlp_encode(node) if len(rlpnode) < 32: return node hashkey = utils.sha3(rlpnode) if put_in_db: self.db.put(hashkey, str_to_bytes(rlpnode)) return hashkey
def test_transaction(filename, testname, testdata): try: rlpdata = decode_hex(testdata["rlp"][2:]) o = {} tx = rlp.decode(rlpdata, transactions.Transaction) blknum = int(testdata["blocknumber"]) # if blknum >= config.default_config["HOMESTEAD_FORK_BLKNUM"]: # tx.check_low_s_homestead() assert config_fork_specific_validation(konfig, blknum, tx) assert tx.startgas >= tx.intrinsic_gas_used if tx.sender == null_address: assert tx.value == 0 and tx.gasprice == 0 and tx.nonce == 0 o["sender"] = tx.sender o["transaction"] = { "data": '0x' * (len(tx.data) > 0) + encode_hex(tx.data), "gasLimit": str(tx.startgas), "gasPrice": str(tx.gasprice), "nonce": str(tx.nonce), "r": '0x' + encode_hex(utils.zpad(utils.int_to_big_endian(tx.r), 32)), "s": '0x' + encode_hex(utils.zpad(utils.int_to_big_endian(tx.s), 32)), "v": str(tx.v), "value": str(tx.value), "to": encode_hex(tx.to), } except Exception as e: tx = None sys.stderr.write(str(e)) if 'transaction' not in testdata: # expected to fail # print(tx.to_dict(), testdata) assert tx is None else: assert set(o['transaction'].keys()) == set( testdata.get("transaction", dict()).keys()) o.get("transaction", None) == testdata.get("transaction", None) assert str_to_bytes(encode_hex(o.get("sender", ''))) == str_to_bytes( testdata.get("sender", ''))
def run_test(filename, testname, testdata): testdata = fixture_to_bytes(testdata) try: rlpdata = decode_hex(testdata["rlp"][2:]) o = {} tx = rlp.decode(rlpdata, transactions.Transaction) o["sender"] = tx.sender o["transaction"] = { "data": b'0x' * (len(tx.data) > 0) + encode_hex(tx.data), "gasLimit": str_to_bytes(str(tx.startgas)), "gasPrice": str_to_bytes(str(tx.gasprice)), "nonce": str_to_bytes(str(tx.nonce)), "r": b'0x' + encode_hex(utils.zpad(utils.int_to_big_endian(tx.r), 32)), "s": b'0x' + encode_hex(utils.zpad(utils.int_to_big_endian(tx.s), 32)), "v": str_to_bytes(str(tx.v)), "value": str_to_bytes(str(tx.value)), "to": encode_hex(tx.to), } except Exception, e: tx = None sys.stderr.write(str(e))
def run_block_test(params): b = blocks.genesis(env, start_alloc=params["pre"]) gbh = params["genesisBlockHeader"] b.bloom = utils.scanners['int256b'](gbh["bloom"]) b.timestamp = utils.scanners['int'](gbh["timestamp"]) b.nonce = utils.scanners['bin'](gbh["nonce"]) b.extra_data = utils.scanners['bin'](gbh["extraData"]) b.gas_limit = utils.scanners['int'](gbh["gasLimit"]) b.gas_used = utils.scanners['int'](gbh["gasUsed"]) b.coinbase = utils.scanners['addr'](decode_hex(gbh["coinbase"])) b.difficulty = utils.parse_int_or_hex(gbh["difficulty"]) b.prevhash = utils.scanners['bin'](gbh["parentHash"]) b.mixhash = utils.scanners['bin'](gbh["mixHash"]) assert b.receipts.root_hash == \ utils.scanners['bin'](gbh["receiptTrie"]) assert b.transactions.root_hash == \ utils.scanners['bin'](gbh["transactionsTrie"]) assert utils.sha3rlp(b.uncles) == \ utils.scanners['bin'](gbh["uncleHash"]) h = encode_hex(b.state.root_hash) if h != str_to_bytes(gbh["stateRoot"]): raise Exception("state root mismatch") if b.hash != utils.scanners['bin'](gbh["hash"]): raise Exception("header hash mismatch") assert b.header.check_pow() blockmap = {b.hash: b} env.db.put(b.hash, rlp.encode(b)) for blk in params["blocks"]: if 'blockHeader' not in blk: try: rlpdata = decode_hex(blk["rlp"][2:]) blkparent = rlp.decode( rlp.encode(rlp.decode(rlpdata)[0]), blocks.BlockHeader).prevhash b2 = rlp.decode(rlpdata, blocks.Block, parent=blockmap[blkparent], env=env) success = b2.validate_uncles() except (ValueError, TypeError, AttributeError, VerificationFailed, DecodingError, DeserializationError, InvalidTransaction, KeyError): success = False assert not success else: rlpdata = decode_hex(blk["rlp"][2:]) blkparent = rlp.decode(rlp.encode(rlp.decode(rlpdata)[0]), blocks.BlockHeader).prevhash b2 = rlp.decode(rlpdata, blocks.Block, parent=blockmap[blkparent], env=env) assert b2.validate_uncles() blockmap[b2.hash] = b2 env.db.put(b2.hash, rlp.encode(b2))
def pack(self, cmd_id, payload): """ UDP packets are structured as follows: hash || signature || packet-type || packet-data packet-type: single byte < 2**7 // valid values are [1,4] packet-data: RLP encoded list. Packet properties are serialized in the order in which they're defined. See packet-data below. Offset | 0 | MDC | Ensures integrity of packet, 65 | signature | Ensures authenticity of sender, `SIGN(sender-privkey, MDC)` 97 | type | Single byte in range [1, 4] that determines the structure of Data 98 | data | RLP encoded, see section Packet Data The packets are signed and authenticated. The sender's Node ID is determined by recovering the public key from the signature. sender-pubkey = ECRECOVER(Signature) The integrity of the packet can then be verified by computing the expected MDC of the packet as: MDC = SHA3(sender-pubkey || type || data) As an optimization, implementations may look up the public key by the UDP sending address and compute MDC before recovering the sender ID. If the MDC values do not match, the packet can be dropped. """ assert cmd_id in self.cmd_id_map.values() assert isinstance(payload, list) cmd_id = str_to_bytes(self.encoders['cmd_id'](cmd_id)) expiration = self.encoders['expiration'](int(time.time() + self.expiration)) encoded_data = rlp.encode(payload + [expiration]) signed_data = crypto.sha3(cmd_id + encoded_data) signature = crypto.sign(signed_data, self.privkey) # assert crypto.verify(self.pubkey, signature, signed_data) # assert self.pubkey == crypto.ecdsa_recover(signed_data, signature) # assert crypto.verify(self.pubkey, signature, signed_data) assert len(signature) == 65 mdc = crypto.sha3(signature + cmd_id + encoded_data) assert len(mdc) == 32 return mdc + signature + cmd_id + encoded_data
def _to_dict(self, node): '''convert (key, value) stored in this and the descendant nodes to dict items. :param node: node in form of list, or BLANK_NODE .. note:: Here key is in full form, rather than key of the individual node ''' if node == BLANK_NODE: return {} node_type = self._get_node_type(node) if is_key_value_type(node_type): nibbles = key_nibbles_from_key_value_node(node) key = b'+'.join([to_string(x) for x in nibbles]) if node_type == NODE_TYPE_EXTENSION: sub_dict = self._to_dict( self._get_inner_node_from_extension(node)) else: sub_dict = {to_string(NIBBLE_TERMINATOR): node[1]} # prepend key of this node to the keys of children res = {} for sub_key, sub_value in sub_dict.items(): full_key = (key + b'+' + sub_key).strip(b'+') res[full_key] = sub_value return res elif node_type == NODE_TYPE_BRANCH: res = {} for i in range(16): sub_dict = self._to_dict(self._decode_to_node(node[i])) for sub_key, sub_value in sub_dict.items(): full_key = (str_to_bytes(str(i)) + b'+' + sub_key).strip(b'+') res[full_key] = sub_value if node[16]: res[to_string(NIBBLE_TERMINATOR)] = node[-1] return res
def __init__(self, ip, udp_port, tcp_port=0, from_binary=False): tcp_port = tcp_port or udp_port if from_binary: self.udp_port = dec_port(udp_port) self.tcp_port = dec_port(tcp_port) else: assert is_integer(udp_port) assert is_integer(tcp_port) self.udp_port = udp_port self.tcp_port = tcp_port try: # `ip` could be in binary or ascii format, independent of # from_binary's truthy. We use ad-hoc regexp to determine format _ip = str_to_bytes(ip) _ip = (bytes_to_str(ip) if PY3 else unicode(ip)) if ip_pattern.match(_ip) else _ip self._ip = ipaddress.ip_address(_ip) except ipaddress.AddressValueError as e: log.debug("failed to parse ip", error=e, ip=ip) raise e
def _to_dict(self, node): '''convert (key, value) stored in this and the descendant nodes to dict items. :param node: node in form of list, or BLANK_NODE .. note:: Here key is in full form, rather than key of the individual node ''' if node == BLANK_NODE: return {} node_type = self._get_node_type(node) if is_key_value_type(node_type): nibbles = key_nibbles_from_key_value_node(node) key = b'+'.join([to_string(x) for x in nibbles]) if node_type == NODE_TYPE_EXTENSION: sub_dict = self._to_dict(self._get_inner_node_from_extension(node)) else: sub_dict = {to_string(NIBBLE_TERMINATOR): node[1]} # prepend key of this node to the keys of children res = {} for sub_key, sub_value in sub_dict.items(): full_key = (key + b'+' + sub_key).strip(b'+') res[full_key] = sub_value return res elif node_type == NODE_TYPE_BRANCH: res = {} for i in range(16): sub_dict = self._to_dict(self._decode_to_node(node[i])) for sub_key, sub_value in sub_dict.items(): full_key = (str_to_bytes(str(i)) + b'+' + sub_key).strip(b'+') res[full_key] = sub_value if node[16]: res[to_string(NIBBLE_TERMINATOR)] = node[-1] return res
def test_chunked_frames(self): cipher1, cipher2 = self.setup_ciphers() dummy_msg_type = 10 expected_frames_count = 10 dummy_payload = helpers.generate_bytearray(self.TEST_FRAME_SIZE * (expected_frames_count - 1)) dummy_protocol = 0 frames = frame_utils.get_frames(dummy_msg_type, memoryview(dummy_payload), dummy_protocol, self.TEST_FRAME_SIZE) self.assertEqual(len(frames), expected_frames_count) frames_bytes = bytearray(0) for frame in frames: encrypted_frame = cipher1.encrypt_frame(frame) encrypted_frame_bytes = bytearray( rlp_utils.str_to_bytes(encrypted_frame)) frames_bytes += encrypted_frame_bytes # adding some dummy bytes but less than header size len dummy_bytes_len = int(eth_common_constants.FRAME_HDR_TOTAL_LEN / 2) frames_bytes += helpers.generate_bytearray(dummy_bytes_len) input_buffer = InputBuffer() framed_input_buffer = FramedInputBuffer(cipher2) read_start = 0 read_size = eth_common_constants.FRAME_HDR_TOTAL_LEN is_full = False msg_type = None while not is_full and read_start < len(frames_bytes): input_buffer.add_bytes(frames_bytes[read_start:read_start + read_size]) is_full, msg_type = framed_input_buffer.peek_message(input_buffer) read_start += read_size self.assertTrue(is_full) self.assertEqual(msg_type, dummy_msg_type)
def do_test_bloom(test_logs): """ The logs sections is a mapping between the blooms and their corresponding logentries. Each logentry has the format: address: The address of the logentry. data: The data of the logentry. topics: The topics of the logentry, given as an array of values. """ for data in test_logs: address = data["address"] # Test via bloom b = bloom.bloom_insert(0, decode_hex(address)) for t in data["topics"]: b = bloom.bloom_insert(b, decode_hex(t)) # Test via Log topics = [decode_int_from_hex(x) for x in data["topics"]] log = pb.Log(decode_hex(address), topics, "") log_bloom = bloom.b64(bloom.bloom_from_list(log.bloomables())) assert encode_hex(log_bloom) == encode_hex_from_int(b) assert str_to_bytes(data["bloom"]) == encode_hex(log_bloom)
def do_test_bloom(test_logs): """ The logs sections is a mapping between the blooms and their corresponding logentries. Each logentry has the format: address: The address of the logentry. data: The data of the logentry. topics: The topics of the logentry, given as an array of values. """ for data in test_logs: address = data['address'] # Test via bloom b = bloom.bloom_insert(0, decode_hex(address)) for t in data['topics']: b = bloom.bloom_insert(b, decode_hex(t)) # Test via Log topics = [decode_int_from_hex(x) for x in data['topics']] log = pb.Log(decode_hex(address), topics, '') log_bloom = bloom.b64(bloom.bloom_from_list(log.bloomables())) assert encode_hex(log_bloom) == encode_hex_from_int(b) assert str_to_bytes(data['bloom']) == encode_hex(log_bloom)
def test_encryption(): initiator, responder = test_session() for i in range(5): msg_frame = sha3(str_to_bytes(str(i)) + b'f') * i + b'notpadded' msg_frame_padded = rzpad16(msg_frame) frame_size = len(msg_frame) msg_header = struct.pack('>I', frame_size)[1:] + sha3(str(i))[:16 - 3] msg_ct = initiator.encrypt(msg_header, msg_frame_padded) r = responder.decrypt(msg_ct) assert r['header'] == msg_header assert r['frame'] == msg_frame for i in range(5): msg_frame = sha3(str(i) + 'f') msg_header = struct.pack('>I', len(msg_frame))[1:] + sha3( str(i))[:16 - 3] msg_ct = responder.encrypt(msg_header, msg_frame) r = initiator.decrypt(msg_ct) assert r['header'] == msg_header assert r['frame'] == msg_frame
return [evaluate(e) for e in ll] else: return ll def to_bytes(value): if isinstance(value, str): return utils.str_to_bytes(value) elif isinstance(value, list): return [to_bytes(v) for v in value] else: return value with open('tests/rlptest.json') as f: test_data = json.loads(f.read()) test_pieces = [(name, {'in': to_bytes(in_out['in']), 'out': utils.str_to_bytes(in_out['out'])}) for name, in_out in test_data.items()] @pytest.mark.parametrize('name, in_out', test_pieces) def test_encode(name, in_out): msg_format = 'Test {} failed (encoded {} to {} instead of {})' data = in_out['in'] result = utils.encode_hex(encode(data)).upper() expected = in_out['out'].upper() if result != expected: pytest.fail(msg_format.format(name, data, result, expected)) @pytest.mark.parametrize('name, in_out', test_pieces) def test_decode(name, in_out):
def run_vm_test(params, mode, profiler=None): pre = params['pre'] exek = params['exec'] env = params['env'] if 'previousHash' not in env: env['previousHash'] = encode_hex(db_env.config['GENESIS_PREVHASH']) assert set(env.keys()) == set(['currentGasLimit', 'currentTimestamp', 'previousHash', 'currentCoinbase', 'currentDifficulty', 'currentNumber']) # setup env header = blocks.BlockHeader( prevhash=decode_hex(env['previousHash']), number=parse_int_or_hex(env['currentNumber']), coinbase=decode_hex(env['currentCoinbase']), difficulty=parse_int_or_hex(env['currentDifficulty']), gas_limit=parse_int_or_hex(env['currentGasLimit']), timestamp=parse_int_or_hex(env['currentTimestamp'])) blk = blocks.Block(header, env=db_env) # setup state for address, h in list(pre.items()): assert len(address) == 40 address = decode_hex(address) assert set(h.keys()) == set(['code', 'nonce', 'balance', 'storage']) blk.set_nonce(address, parse_int_or_hex(h['nonce'])) blk.set_balance(address, parse_int_or_hex(h['balance'])) blk.set_code(address, decode_hex(h['code'][2:])) for k, v in h['storage'].items(): blk.set_storage_data(address, utils.big_endian_to_int(decode_hex(k[2:])), utils.big_endian_to_int(decode_hex(v[2:]))) # execute transactions sender = decode_hex(exek['caller']) # a party that originates a call recvaddr = decode_hex(exek['address']) nonce = blk._get_acct_item(sender, 'nonce') gasprice = parse_int_or_hex(exek['gasPrice']) startgas = parse_int_or_hex(exek['gas']) value = parse_int_or_hex(exek['value']) data = decode_hex(exek['data'][2:]) # bypass gas check in tx initialization by temporarily increasing startgas num_zero_bytes = str_to_bytes(data).count(ascii_chr(0)) num_non_zero_bytes = len(data) - num_zero_bytes intrinsic_gas = (opcodes.GTXCOST + opcodes.GTXDATAZERO * num_zero_bytes + opcodes.GTXDATANONZERO * num_non_zero_bytes) startgas += intrinsic_gas tx = transactions.Transaction(nonce=nonce, gasprice=gasprice, startgas=startgas, to=recvaddr, value=value, data=data) tx.startgas -= intrinsic_gas tx.sender = sender # capture apply_message calls apply_message_calls = [] orig_apply_msg = pb.apply_msg ext = pb.VMExt(blk, tx) def msg_wrapper(msg): hexdata = encode_hex(msg.data.extract_all()) apply_message_calls.append(dict(gasLimit=to_string(msg.gas), value=to_string(msg.value), destination=encode_hex(msg.to), data=b'0x' + hexdata)) return 1, msg.gas, b'' def create_wrapper(msg): sender = decode_hex(msg.sender) if \ len(msg.sender) == 40 else msg.sender nonce = utils.encode_int(ext._block.get_nonce(msg.sender)) addr = utils.sha3(rlp.encode([sender, nonce]))[12:] hexdata = encode_hex(msg.data.extract_all()) apply_message_calls.append(dict(gasLimit=to_string(msg.gas), value=to_string(msg.value), destination=b'', data=b'0x' + hexdata)) return 1, msg.gas, addr ext.msg = msg_wrapper ext.create = create_wrapper def blkhash(n): if n >= ext.block_number or n < ext.block_number - 256: return b'' else: return utils.sha3(to_string(n)) ext.block_hash = blkhash msg = vm.Message(tx.sender, tx.to, tx.value, tx.startgas, vm.CallData([safe_ord(x) for x in tx.data])) code = decode_hex(exek['code'][2:]) time_pre = time.time() if profiler: profiler.enable() success, gas_remained, output = vm.vm_execute(ext, msg, code) if profiler: profiler.disable() pb.apply_msg = orig_apply_msg blk.commit_state() for s in blk.suicides: blk.del_account(s) time_post = time.time() """ generally expected that the test implementer will read env, exec and pre then check their results against gas, logs, out, post and callcreates. If an exception is expected, then latter sections are absent in the test. Since the reverting of the state is not part of the VM tests. """ params2 = copy.deepcopy(params) if success: params2['callcreates'] = apply_message_calls params2['out'] = b'0x' + encode_hex(b''.join(map(ascii_chr, output))) params2['gas'] = to_string(gas_remained) params2['logs'] = [log.to_dict() for log in blk.logs] params2['post'] = blk.to_dict(with_state=True)['state'] if mode == FILL: return params2 elif mode == VERIFY: if not success: assert 'post' not in params, 'failed, but expected to succeed' params1 = copy.deepcopy(params) shouldbe, reallyis = params1.get('post', None), params2.get('post', None) compare_post_states(shouldbe, reallyis) def normalize_value(k, p): if k in p: if k == 'gas': return parse_int_or_hex(p[k]) elif k == 'callcreates': return list(map(callcreate_standard_form, p[k])) else: return utils.to_string(k) return None for k in ['pre', 'exec', 'env', 'callcreates', 'out', 'gas', 'logs']: shouldbe = normalize_value(k, params1) reallyis = normalize_value(k, params2) if shouldbe != reallyis: raise Exception("Mismatch: " + k + ':\n shouldbe %r\n reallyis %r' % (shouldbe, reallyis)) elif mode == TIME: return time_post - time_pre
try: from Crypto.Hash import keccak sha3_256 = lambda x: keccak.new(digest_bits=256, data=x).digest() except: import sha3 as _sha3 sha3_256 = lambda x: _sha3.sha3_256(x).digest() from bitcoin import privtopub import sys import rlp from rlp.sedes import big_endian_int, BigEndianInt, Binary from rlp.utils import decode_hex, encode_hex, ascii_chr, str_to_bytes import random big_endian_to_int = lambda x: big_endian_int.deserialize( str_to_bytes(x).lstrip(b'\x00')) int_to_big_endian = lambda x: big_endian_int.serialize(x) TT256 = 2**256 TT256M1 = 2**256 - 1 TT255 = 2**255 if sys.version_info.major == 2: is_numeric = lambda x: isinstance(x, (int, long)) is_string = lambda x: isinstance(x, (str, unicode)) def to_string(value): return str(value) def int_to_bytes(value): if isinstance(value, str):
def mac(data=b''): data = str_to_bytes(data) self.egress_mac.update(data) return self.egress_mac.digest()
def intrinsic_gas_used(tx): num_zero_bytes = str_to_bytes(tx.data).count(ascii_chr(0)) num_non_zero_bytes = len(tx.data) - num_zero_bytes return (opcodes.GTXCOST + opcodes.GTXDATAZERO * num_zero_bytes + opcodes.GTXDATANONZERO * num_non_zero_bytes)
def run_block_test(params, config_overrides = {}): b = blocks.genesis(env, start_alloc=params["pre"]) gbh = params["genesisBlockHeader"] b.bloom = utils.scanners['int256b'](gbh["bloom"]) b.timestamp = utils.scanners['int'](gbh["timestamp"]) b.nonce = utils.scanners['bin'](gbh["nonce"]) b.extra_data = utils.scanners['bin'](gbh["extraData"]) b.gas_limit = utils.scanners['int'](gbh["gasLimit"]) b.gas_used = utils.scanners['int'](gbh["gasUsed"]) b.coinbase = utils.scanners['addr'](decode_hex(gbh["coinbase"])) b.difficulty = utils.parse_int_or_hex(gbh["difficulty"]) b.prevhash = utils.scanners['bin'](gbh["parentHash"]) b.mixhash = utils.scanners['bin'](gbh["mixHash"]) assert b.receipts.root_hash == \ utils.scanners['bin'](gbh["receiptTrie"]) assert b.transactions.root_hash == \ utils.scanners['bin'](gbh["transactionsTrie"]) assert utils.sha3rlp(b.uncles) == \ utils.scanners['bin'](gbh["uncleHash"]) h = encode_hex(b.state.root_hash) if h != str_to_bytes(gbh["stateRoot"]): raise Exception("state root mismatch") if b.hash != utils.scanners['bin'](gbh["hash"]): raise Exception("header hash mismatch") # assert b.header.check_pow() blockmap = {b.hash: b} env.db.put(b.hash, rlp.encode(b)) old_config = copy.deepcopy(env.config) for k, v in config_overrides.items(): env.config[k] = v b2 = None for blk in params["blocks"]: if 'blockHeader' not in blk: try: rlpdata = decode_hex(blk["rlp"][2:]) blkparent = rlp.decode( rlp.encode(rlp.decode(rlpdata)[0]), blocks.BlockHeader).prevhash b2 = rlp.decode(rlpdata, blocks.Block, parent=blockmap[blkparent], env=env) success = b2.validate_uncles() except (ValueError, TypeError, AttributeError, VerificationFailed, DecodingError, DeserializationError, InvalidTransaction, KeyError): success = False assert not success else: rlpdata = decode_hex(blk["rlp"][2:]) blkparent = rlp.decode(rlp.encode(rlp.decode(rlpdata)[0]), blocks.BlockHeader).prevhash b2 = rlp.decode(rlpdata, blocks.Block, parent=blockmap[blkparent], env=env) assert b2.validate_uncles() blockmap[b2.hash] = b2 env.db.put(b2.hash, rlp.encode(b2)) if b2: print('Block %d with state root %s' % (b2.number, encode_hex(b2.state.root_hash))) # blkdict = b.to_dict(False, True, False, True) # assert blk["blockHeader"] == \ # translate_keys(blkdict["header"], translator_list, lambda y, x: x, []) # assert blk["transactions"] == \ # [translate_keys(t, translator_list, valueconv, ['hash']) # for t in blkdict["transactions"]] # assert blk["uncleHeader"] == \ # [translate_keys(u, translator_list, lambda x: x, []) # for u in blkdict["uncles"]] env.config = old_config
def run_vm_test(params, mode, profiler=None): pre = params['pre'] exek = params['exec'] env = params['env'] if 'previousHash' not in env: env['previousHash'] = encode_hex(db_env.config['GENESIS_PREVHASH']) assert set(env.keys()) == set([ 'currentGasLimit', 'currentTimestamp', 'previousHash', 'currentCoinbase', 'currentDifficulty', 'currentNumber' ]) # setup env header = blocks.BlockHeader( prevhash=decode_hex(env['previousHash']), number=parse_int_or_hex(env['currentNumber']), coinbase=decode_hex(env['currentCoinbase']), difficulty=parse_int_or_hex(env['currentDifficulty']), gas_limit=parse_int_or_hex(env['currentGasLimit']), timestamp=parse_int_or_hex(env['currentTimestamp'])) blk = blocks.Block(header, env=db_env) # setup state for address, h in list(pre.items()): assert len(address) == 40 address = decode_hex(address) assert set(h.keys()) == set(['code', 'nonce', 'balance', 'storage']) blk.set_nonce(address, parse_int_or_hex(h['nonce'])) blk.set_balance(address, parse_int_or_hex(h['balance'])) blk.set_code(address, decode_hex(h['code'][2:])) for k, v in h['storage'].items(): blk.set_storage_data(address, utils.big_endian_to_int(decode_hex(k[2:])), utils.big_endian_to_int(decode_hex(v[2:]))) # execute transactions sender = decode_hex(exek['caller']) # a party that originates a call recvaddr = decode_hex(exek['address']) nonce = blk._get_acct_item(sender, 'nonce') gasprice = parse_int_or_hex(exek['gasPrice']) startgas = parse_int_or_hex(exek['gas']) value = parse_int_or_hex(exek['value']) data = decode_hex(exek['data'][2:]) # bypass gas check in tx initialization by temporarily increasing startgas num_zero_bytes = str_to_bytes(data).count(ascii_chr(0)) num_non_zero_bytes = len(data) - num_zero_bytes intrinsic_gas = (opcodes.GTXCOST + opcodes.GTXDATAZERO * num_zero_bytes + opcodes.GTXDATANONZERO * num_non_zero_bytes) startgas += intrinsic_gas tx = transactions.Transaction(nonce=nonce, gasprice=gasprice, startgas=startgas, to=recvaddr, value=value, data=data) tx.startgas -= intrinsic_gas tx.sender = sender # capture apply_message calls apply_message_calls = [] orig_apply_msg = pb.apply_msg ext = pb.VMExt(blk, tx) def msg_wrapper(msg): hexdata = encode_hex(msg.data.extract_all()) apply_message_calls.append( dict(gasLimit=to_string(msg.gas), value=to_string(msg.value), destination=encode_hex(msg.to), data=b'0x' + hexdata)) return 1, msg.gas, b'' def create_wrapper(msg): sender = decode_hex(msg.sender) if \ len(msg.sender) == 40 else msg.sender nonce = utils.encode_int(ext._block.get_nonce(msg.sender)) addr = utils.sha3(rlp.encode([sender, nonce]))[12:] hexdata = encode_hex(msg.data.extract_all()) apply_message_calls.append( dict(gasLimit=to_string(msg.gas), value=to_string(msg.value), destination=b'', data=b'0x' + hexdata)) return 1, msg.gas, addr ext.msg = msg_wrapper ext.create = create_wrapper def blkhash(n): if n >= ext.block_number or n < ext.block_number - 256: return b'' else: return utils.sha3(to_string(n)) ext.block_hash = blkhash msg = vm.Message(tx.sender, tx.to, tx.value, tx.startgas, vm.CallData([safe_ord(x) for x in tx.data])) code = decode_hex(exek['code'][2:]) time_pre = time.time() if profiler: profiler.enable() success, gas_remained, output = vm.vm_execute(ext, msg, code) if profiler: profiler.disable() pb.apply_msg = orig_apply_msg blk.commit_state() for s in blk.suicides: blk.del_account(s) time_post = time.time() """ generally expected that the test implementer will read env, exec and pre then check their results against gas, logs, out, post and callcreates. If an exception is expected, then latter sections are absent in the test. Since the reverting of the state is not part of the VM tests. """ params2 = copy.deepcopy(params) if success: params2['callcreates'] = apply_message_calls params2['out'] = b'0x' + encode_hex(b''.join(map(ascii_chr, output))) params2['gas'] = to_string(gas_remained) params2['logs'] = [log.to_dict() for log in blk.logs] params2['post'] = blk.to_dict(with_state=True)['state'] if mode == FILL: return params2 elif mode == VERIFY: if not success: assert 'post' not in params, 'failed, but expected to succeed' params1 = copy.deepcopy(params) shouldbe, reallyis = params1.get('post', None), params2.get('post', None) compare_post_states(shouldbe, reallyis) def normalize_value(k, p): if k in p: if k == 'gas': return parse_int_or_hex(p[k]) elif k == 'callcreates': return list(map(callcreate_standard_form, p[k])) else: return utils.to_string(k) return None for k in ['pre', 'exec', 'env', 'callcreates', 'out', 'gas', 'logs']: shouldbe = normalize_value(k, params1) reallyis = normalize_value(k, params2) if shouldbe != reallyis: raise Exception("Mismatch: " + k + ':\n shouldbe %r\n reallyis %r' % (shouldbe, reallyis)) elif mode == TIME: return time_post - time_pre
def __hash__(self): return utils.big_endian_to_int(str_to_bytes(self.__repr__()))
def big_endian_to_int(x): return big_endian_int.deserialize(str_to_bytes(x).lstrip(b'\x00'))
def test_binary(): b1 = Binary() f = { '': b'', 'asdf': b'asdf', ('\x00' * 20): (b'\x00' * 20), 'fdsa': b'fdsa' } for k in f: assert b1.serialize(k) == f[k] for d in ([], 5, str): with pytest.raises(SerializationError): b1.serialize(d) b2 = Binary.fixed_length(5) f = { 'asdfg': b'asdfg', b'\x00\x01\x02\x03\x04': b'\x00\x01\x02\x03\x04', utils.str_to_bytes('ababa'): b'ababa' } for k in f: assert b2.serialize(k) == f[k] for d in ('asdf', 'asdfgh', '', 'bababa'): with pytest.raises(SerializationError): b2.serialize(d) b3 = Binary(2, 4) f = { 'as': b'as', 'dfg': b'dfg', 'hjkl': b'hjkl', b'\x00\x01\x02': b'\x00\x01\x02' } for k in f: assert b3.serialize(k) == f[k] for d in ('', 'a', 'abcde', 'äää'): with pytest.raises(SerializationError): b3.serialize(d) b4 = Binary(min_length=3) f = {'abc': b'abc', 'abcd': b'abcd', ('x' * 132): (b'x' * 132)} for k in f: assert b4.serialize(k) == f[k] for d in ('ab', '', 'a', 'xy'): with pytest.raises(SerializationError): b4.serialize(d) b5 = Binary(max_length=3) f = {'': b'', 'ab': b'ab', 'abc': b'abc'} for k in f: assert b5.serialize(k) == f[k] for d in ('abcd', 'vwxyz', 'a' * 32): with pytest.raises(SerializationError): b5.serialize(d) b6 = Binary(min_length=3, max_length=5, allow_empty=True) f = {'': b'', 'abc': b'abc', 'abcd': b'abcd', 'abcde': b'abcde'} for k in f: assert b6.serialize(k) == f[k] for d in ('a', 'ab', 'abcdef', 'abcdefgh' * 10): with pytest.raises(SerializationError): b6.serialize(d)
from sha3 import sha3_256 from bitcoin import privtopub import struct import os import sys import rlp from rlp.sedes import big_endian_int, BigEndianInt, Binary from rlp.utils import decode_hex, encode_hex, ascii_chr, str_to_bytes import random big_endian_to_int = lambda x: big_endian_int.deserialize(str_to_bytes(x).lstrip(b"\x00")) int_to_big_endian = lambda x: big_endian_int.serialize(x) TT256 = 2 ** 256 TT256M1 = 2 ** 256 - 1 TT255 = 2 ** 255 if sys.version_info.major == 2: is_numeric = lambda x: isinstance(x, (int, long)) is_string = lambda x: isinstance(x, (str, unicode)) def to_string(value): return str(value) def int_to_bytes(value): if isinstance(value, str): return value return int_to_big_endian(value)