def parse(cls, s, testnet=False):
'''Takes a byte stream and parses the transaction at the start
return a Tx object
'''
# s.read(n) will return n bytes
# version is an integer in 4 bytes, little-endian
version = little_endian_to_int(s.read(4))
# num_inputs is a varint, use read_varint(s)
num_inputs = read_varint(s)
# parse num_inputs number of TxIns
inputs = []
for _ in range(num_inputs):
inputs.append(TxIn.parse(s))
# num_outputs is a varint, use read_varint(s)
num_outputs = read_varint(s)
# parse num_outputs number of TxOuts
outputs = []
for _ in range(num_outputs):
outputs.append(TxOut.parse(s))
# locktime is an integer in 4 bytes, little-endian
locktime = little_endian_to_int(s.read(4))
# return an instance of the class (see __init__ for args)
return cls(version, inputs, outputs, locktime, testnet=testnet)
def test_varint(self):
tests = [
(0x01, unhexlify('01')),
(0xfd, unhexlify('fdfd00')),
(0xfe, unhexlify('fdfe00')),
(0xff, unhexlify('fdff00')),
(0x1234, unhexlify('fd3412')),
(0x123456, unhexlify('fe56341200')),
(0x123456789a01, unhexlify('ff019a785634120000')),
]
for num, encoded in tests:
self.assertEqual(encode_varint(num), encoded)
s = BytesIO(encoded)
self.assertEqual(read_varint(s), num)
self.assertRaises(ValueError, encode_varint, 2**65)
def parse_segwit(cls, s, testnet=False):
'''Takes a byte stream and parses a segwit transaction'''
# s.read(n) will return n bytes
# version has 4 bytes, little-endian, interpret as int
version = little_endian_to_int(s.read(4))
# next two bytes need to be 0x00 and 0x01, otherwise raise RuntimeError
marker = s.read(2)
if marker != b'\x00\x01':
raise RuntimeError('Not a segwit transaction {}'.format(marker))
# num_inputs is a varint, use read_varint(s)
num_inputs = read_varint(s)
# each input needs parsing, create inputs array
inputs = []
# parse each input and add to the inputs array
for _ in range(num_inputs):
inputs.append(TxIn.parse(s))
# num_outputs is a varint, use read_varint(s)
num_outputs = read_varint(s)
# each output needs parsing, create outputs array
outputs = []
# parse each output and add to the outputs array
for _ in range(num_outputs):
outputs.append(TxOut.parse(s))
# there is a witness for each input
for tx_in in inputs:
# use Witness.parse to grab the witness
tx_in.witness = Witness.parse(s)
# locktime is 4 bytes, little-endian
locktime = little_endian_to_int(s.read(4))
# return an instance of the class (cls(...))
return cls(version,
inputs,
outputs,
locktime,
testnet=testnet,
segwit=True)
def parse_segwit(cls, s, testnet=False):
'''Takes a byte stream and parses a segwit transaction'''
# s.read(n) will return n bytes
# version has 4 bytes, little-endian, interpret as int
version = little_endian_to_int(s.read(4))
# next two bytes need to be 0x00 and 0x01
marker = s.read(2)
if marker != b'\x00\x01':
raise RuntimeError('Not a segwit transaction {}'.format(marker))
# num_inputs is a varint, use read_varint(s)
num_inputs = read_varint(s)
# each input needs parsing
inputs = []
for _ in range(num_inputs):
inputs.append(TxIn.parse(s))
# num_outputs is a varint, use read_varint(s)
num_outputs = read_varint(s)
# each output needs parsing
outputs = []
for _ in range(num_outputs):
outputs.append(TxOut.parse(s))
# now parse the witness
for tx_in in inputs:
num_items = read_varint(s)
items = []
for _ in range(num_items):
item_len = read_varint(s)
if item_len == 0:
items.append(0)
else:
items.append(s.read(item_len))
tx_in.witness = items
# locktime is 4 bytes, little-endian
locktime = little_endian_to_int(s.read(4))
# return an instance of the class (cls(...))
return cls(version, inputs, outputs, locktime, testnet=testnet, segwit=True)
def parse(cls, s):
'''Takes a byte stream and parses the transaction at the start
return a Tx object
'''
# s.read(n) will return n bytes
# version has 4 bytes, little-endian, interpret as int
version = little_endian_to_int(s.read(4))
prev_block_hash = s.read(32)[::-1]
# num_inputs is a varint, use read_varint(s)
num_inputs = read_varint(s)
# each input needs parsing
inputs = []
for _ in range(num_inputs):
inputs.append(TxIn.parse(s))
# num_outputs is a varint, use read_varint(s)
num_outputs = read_varint(s)
# each output needs parsing
outputs = []
for _ in range(num_outputs):
outputs.append(TxOut.parse(s))
# locktime is 4 bytes, little-endian
locktime = little_endian_to_int(s.read(4))
# return an instance of the class (cls(...))
return cls(version, inputs, outputs, locktime, prev_block_hash=prev_block_hash)
def parse(cls, s, coinbase_mode=False):
# get the length of the entire field
length = read_varint(s)
if coinbase_mode:
return cls([], coinbase=s.read(length))
# initialize the instructions array
instructions = []
# initialize the number of bytes we've read to 0
count = 0
# loop until we've read length bytes
while count < length:
# get the current byte
current = s.read(1)
# increment the bytes we've read
count += 1
# convert the current byte to an integer
current_byte = current[0]
# if the current byte is between 1 and 75 inclusive
if current_byte >= 1 and current_byte <= 75:
# we have an instruction set n to be the current byte
n = current_byte
# add the next n bytes as an instruction
instructions.append(s.read(n))
# increase the count by n
count += n
elif current_byte == 76:
# op_pushdata1
data_length = little_endian_to_int(s.read(1))
instructions.append(s.read(data_length))
count += data_length + 1
elif current_byte == 77:
# op_pushdata2
data_length = little_endian_to_int(s.read(2))
instructions.append(s.read(data_length))
count += data_length + 2
elif current_byte == 78:
# op_pushdata4
data_length = little_endian_to_int(s.read(4))
instructions.append(s.read(data_length))
count += data_length + 4
else:
# we have an op code. set the current byte to op_code
op_code = current_byte
# add the op_code to the list of instructions
instructions.append(op_code)
if count != length:
raise RuntimeError('parsing script failed')
return cls(instructions)
def parse(cls, s):
length = read_varint(s)
items = []
count = 0
while count < length:
current = s.read(1)
count += 1
current_byte = current[0]
if current_byte >= 1 and current_byte <= 75:
n = current_byte
items.append(s.read(n))
count += n
else:
op_code = current_byte
items.append(op_code)
return cls(items)
def parse(cls, s):
# number of headers is in a varint
num_headers = read_varint(s)
# initialize the headers array
headers = []
# loop through number of headers times
for _ in range(num_headers):
# parse a header using Block.parse(s)
header = Block.parse(s)
# add the header to the headers array
headers.append(header)
# check that the length of the tx_hashes to be 0 or raise a RuntimeError
if len(header.tx_hashes) != 0:
raise RuntimeError('number of txs not 0')
# return a class instance
return cls(headers)
def parse(cls, s):
version = little_endian_to_int(s.read(4))
prev_block = s.read(32)[::-1]
merkle_root = s.read(32)[::-1]
timestamp = little_endian_to_int(s.read(4))
bits = s.read(4)
nonce = s.read(4)
num_tx = read_varint(s)
txs = []
for i in range(num_tx):
if i == 0:
txs.append(Tx.parse(s, coinbase_tx=True))
else:
txs.append(Tx.parse(s))
return cls(version, prev_block, merkle_root, timestamp, bits, nonce,
txs)
def parse(cls, s):
'''Takes a byte stream and parses a merkle block. Returns a Merkle Block object'''
# version - 4 bytes, Little-Endian integer
header = Block.parse_header(s)
# total transactions in block - 4 bytes, Little-Endian integer
total = little_endian_to_int(s.read(4))
# number of transaction hashes - varint
num_txs = read_varint(s)
# each transaction is 32 bytes, Little-Endian
hashes = []
for _ in range(num_txs):
hashes.append(s.read(32)[::-1])
# flags field - varstr
flags = read_varstr(s)
# initialize class
return cls(header, total, hashes, flags)
def parse(cls, s):
'''Takes a byte stream and parses the tx_input at the start
return a TxIn object
'''
# s.read(n) will return n bytes
# prev_tx is 32 bytes, little endian
prev_tx = s.read(32)[::-1]
# prev_index is 4 bytes, little endian, interpret as int
prev_index = little_endian_to_int(s.read(4))
# script_sig is a variable field (length followed by the data)
# get the length by using read_varint(s)
script_sig_length = read_varint(s)
script_sig = s.read(script_sig_length)
# sequence is 4 bytes, little-endian, interpret as int
sequence = little_endian_to_int(s.read(4))
# return an instance of the class (cls(...))
return cls(prev_tx, prev_index, script_sig, sequence)
def parse(cls, s):
'''Takes a byte stream and parses the tx_output at the start
return a TxOut object
'''
# amount is an integer in 8 bytes, little endian
amount = little_endian_to_int(s.read(8))
#scriptpublickey
scritp_pubkey_length = read_varint(s)
script_pubkey = s.read(scritp_pubkey_length)
# use Script.parse to get the ScriptPubKey
#try:
#script_pubkey = Script.parse(s)
#except:
#print("BAD SCRIPT PUBLIC KEY.")
#script_pubkey = Script()
# return an instance of the class (see __init__ for args)
return cls(amount, script_pubkey)
def test_exercise_10(self):
block_hex = '0000000000044b01a9440b34f582fe171c7b8642fedd0ebfccf8fdf6a1810900'
block_hash = bytes.fromhex(block_hex)
node = SimpleNode('tbtc.programmingblockchain.com', testnet=True)
node.handshake()
getdata = GetDataMessage()
getdata.add_data(BLOCK_DATA_TYPE, block_hash)
node.send(getdata.command, getdata.serialize())
block_envelope = node.wait_for_commands([b'block'])
stream = block_envelope.stream()
b = Block.parse(stream)
self.assertTrue(b.check_pow())
num_txs = read_varint(stream)
tx_hashes = []
for _ in range(num_txs):
t = Tx.parse(stream)
tx_hashes.append(t.hash())
b.tx_hashes = tx_hashes
self.assertTrue(b.validate_merkle_root())
def parse(cls, s):
# get the length of the entire field
length = read_varint(s)
# initialize the cmds array
cmds = []
# initialize the number of bytes we've read to 0
count = 0
# loop until we've read length bytes
while count < length:
# get the current byte
current = s.read(1)
# increment the bytes we've read
count += 1
# convert the current byte to an integer
current_byte = current[0]
# if the current byte is between 1 and 75 inclusive
if current_byte >= 1 and current_byte <= 75:
# we have an cmd set n to be the current byte
n = current_byte
# add the next n bytes as an cmd
cmds.append(s.read(n))
# increase the count by n
count += n
elif current_byte == 76:
# op_pushdata1
data_length = little_endian_to_int(s.read(1))
cmds.append(s.read(data_length))
count += data_length + 1
elif current_byte == 77:
# op_pushdata2
data_length = little_endian_to_int(s.read(2))
cmds.append(s.read(data_length))
count += data_length + 2
else:
# we have an opcode. set the current byte to op_code
op_code = current_byte
# add the op_code to the list of cmds
cmds.append(op_code)
if count != length:
raise SyntaxError('parsing script failed')
return cls(cmds)
def parse(cls, s):
# script serialization always starts with the length of the script.
length = read_varint(s)
cmds = []
count = 0
# parse until whole script has been parsed.
while count < length:
# this byte's value determines if we have an opcode or an element.
current = s.read(1)
count += 1
# this converts the current byte into an int.
current_byte_as_int = current[0]
print('curr byte', current_byte_as_int)
# for a number between 1 and 75, we know the next n bytes are an element.
if current_byte_as_int >= 1 and current_byte_as_int <= 75:
n = current_byte_as_int
# push the element into the stack.
cmds.append(s.read(n))
# update the count.
count += n
# 76 is OP_PUSHDATA1, so the next byte tells us how many bytes the next element is.
elif current_byte_as_int == 76:
# n is the number of bytes to read
n = little_endian_to_int(s.read(1))
# push the element into the stack.
cmds.append(s.read(n))
# update the count.
count += n + 1
# 77 is OP_PUSHDATA2, so the next 2 bytes tell us how many bytes the next element is.
elif current_byte_as_int == 77:
n = little_endian_to_int(s.read(2))
cmds.append(s.read(n))
count += n + 2
# else we push the opcode onto the stack
else:
op_code = current_byte_as_int
cmds.append(op_code)
# script should have consumed exactly the number of bytes expected. If not we raise an error.
print("count, length", count, length)
if count != length:
raise SyntaxError('Parsing script failed.')
return cls(cmds)
async def process_queue(self):
print("start processing")
start = time.time()
while self.keep_looping:
envelope = await self.q.get()
command = envelope.command.strip(b'\x00').decode('ascii')
if command == 'version':
self.send(b'verack', b'')
elif command == 'sendheaders':
self.send(b'headers', encode_varint(0))
elif command == 'ping':
self.send(b'pong', envelope.payload)
if not self._sent:
# tell them we have a tx
self.send(b'inv', self.inv_payload)
elif not self._accepted:
self.send(b'tx', self.raw_tx)
self.send(b'mempool', b'')
elif command == 'getdata':
if envelope.payload == self.inv_payload:
self.send(b'tx', self.raw_tx)
self._sent = True
elif self._sent:
print('TX rejected')
self.keep_looping = False
elif command == 'feefilter':
minimum = little_endian_to_int(envelope.payload)
print('TX requires fee: {} minimum'.format(minimum))
elif command == 'inv':
stream = BytesIO(envelope.payload)
num_inv = read_varint(stream)
for _ in range(num_inv):
inv_type = little_endian_to_int(stream.read(4))
inv_hash = stream.read(32)
if inv_type == 1 and inv_hash == self.tx_hash:
print('TX successfully sent')
self.keep_looping = False
self.send(b'inv', self.inv_payload)
else:
print(envelope)
if time.time() - start < self.timeout:
self.keep_looping = False
def save(self):
store_height = self.store_height()
if exists(self.filename):
with open(self.filename, 'rb') as f:
file_height = read_varint(f)
if store_height == file_height:
# no need to save as we're synced
return
rest = f.read()
else:
file_height = -1
rest = b''
with open(self.filename, 'wb') as f:
f.write(encode_varint(store_height))
for height in range(store_height, file_height, -1):
h = self.header_by_height(height)
f.write(h.serialize())
f.write(h.cfheader)
f.write(h.cfhash)
f.write(rest)
def parse(cls, s):
'''Takes a byte stream and parses a block. Returns a BlockHeader object'''
# s.read(n) will read n bytes from the stream
# version - 4 bytes, little endian, interpret as int
version = little_endian_to_int(s.read(4))
# prev_block - 32 bytes, little endian (use [::-1] to reverse)
prev_block = s.read(32)[::-1]
# merkle_root - 32 bytes, little endian (use [::-1] to reverse)
merkle_root = s.read(32)[::-1]
# timestamp - 4 bytes, little endian, interpret as int
timestamp = little_endian_to_int(s.read(4))
# bits - 4 bytes
bits = s.read(4)
# nonce - 4 bytes
nonce = s.read(4)
# read the actual transactions
num_txns = read_varint(s)
txns = [Tx.parse(s) for _ in range(num_txns)]
# initialize class
return cls(version, prev_block, merkle_root, timestamp, bits, nonce,
txns)
def __init__(self, node, filename=None, full=False, include=None):
self.node = node
self.testnet = node.testnet
if filename is None:
if self.testnet:
self.filename = 'testnet.blocks'
else:
self.filename = 'mainnet.blocks'
else:
self.filename = filename
self.full = full
self.include = include
self.start_height = 0
if not exists(self.filename):
# new store
# start from the genesis block
if self.testnet:
header = TESTNET_GENESIS_BLOCK_HEADER
else:
header = GENESIS_BLOCK_HEADER
self.headers = [Block.parse_header(BytesIO(header))]
else:
with open(self.filename, 'rb') as f:
headers = []
file_height = read_varint(f)
if self.full:
self.start_height = 0
elif self.include and self.include < file_height:
# start of difficulty adjustment that includes this block
self.start_height = (self.include // 2016) * 2016
else:
# one full difficulty adjustment + remaining
self.start_height = (file_height // 2016 - 1) * 2016
for _ in range(self.start_height, file_height + 1):
header = Block.parse_header(f)
header.cfheader = f.read(32)
header.cfhash = f.read(32)
headers.append(header)
self.headers = headers[::-1]
self.headers_lookup = {h.hash(): h for h in self.headers}
def parse_from_blk(cls, s):
'''Takes a byte stream and parses a block. Returns a Block object'''
# s.read(n) will read n bytes from the stream
magic = little_endian_to_int(s.read(4))
size = little_endian_to_int(s.read(4))
# version - 4 bytes, little endian, interpret as int
version = little_endian_to_int(s.read(4))
# prev_block - 32 bytes, little endian (use [::-1] to reverse)
prev_block = s.read(32)[::-1]
# merkle_root - 32 bytes, little endian (use [::-1] to reverse)
merkle_root = s.read(32)[::-1]
# timestamp - 4 bytes, little endian, interpret as int
timestamp = little_endian_to_int(s.read(4))
# bits - 4 bytes
bits = s.read(4)
# nonce - 4 bytes
nonce = s.read(4)
#THIS IS ADDED AT THE END.
tx_hashes = read_varint(s)
# initialize class
return cls(version, prev_block, merkle_root, timestamp, bits, nonce,
tx_hashes, magic, size)
def parse(cls, s):
'''Takes a byte stream and parses the tx_input at the start
return a TxIn object
'''
# prev_tx is 32 bytes, little endian
prev_tx = s.read(32)[::-1]
# prev_index is an integer in 4 bytes, little endian
prev_index = little_endian_to_int(s.read(4))
#read the script sig and convert it to hex
sig_length = read_varint(s)
script_sig = s.read(sig_length)
#script_sig = script_sig.hex()
# use Script.parse to get the ScriptSig
#try:
#script_sig = Script.parse(s)
#except:
#print(f"BAD SCRIPT SIGNATURE")
#script_sig = Script()
# sequence is an integer in 4 bytes, little-endian
sequence = little_endian_to_int(s.read(4))
# return an instance of the class (see __init__ for args)
return cls(prev_tx, prev_index, script_sig, sequence)
def parse(cls, s: BufferedIOBase) -> Script:
# get the length of the entire field
length = read_varint(s)
# initialize the commands array
commands = []
# initialize the number of bytes we've read to 0
count = 0
# loop until we've read length bytes
while count < length:
# get the current byte
current = s.read(1)
# increment the bytes we've read
count += 1
# convert the current byte to an integer
current_int = current[0]
# if the current byte is between 1 and 75 inclusive
if current_int <= 75:
# add the next n bytes as a command
commands.append(s.read(current_int))
count += current_int
elif current == OP_PUSHDATA1:
# op_pushdata1
data_length = byte_to_int(s.read(1))
commands.append(s.read(data_length))
count += data_length + 1
elif current == OP_PUSHDATA2:
# op_pushdata2
data_length = little_endian_to_int(s.read(2))
commands.append(s.read(data_length))
count += data_length + 2
else:
# add the command to the list of commands
commands.append(current)
if count != length:
raise SyntaxError(f'parsing script failed {commands}')
return cls(commands)
def parse(cls, s):
salt = s.read(32)
data_len = read_varint(s)
encrypted_data = s.read(data_len)
return cls(salt, encrypted_data)
def parse(cls, stream):
count = read_varint(stream)
address_list = []
for _ in range(count):
address_list.append(Address.parse(stream))
return cls(address_list)
def read_varstr(s):
item_length = read_varint(s)
return s.read(item_length)
def parse(cls, s):
# get the length of the entire field
#print(s)
length = read_varint(s)
#print(f"length script: {length}")
# initialize the cmds array
cmds = []
# initialize the number of bytes we've read to 0
count = 0
# loop until we've read length bytes
while count < length:
# get the current byte
#print(f"WHILE: count {count} and length {length}")
current = s.read(1)
# increment the bytes we've read
count += 1
# convert the current byte to an integer
current_byte = current[0]
#print(f"current_byte : {current_byte}")
# if the current byte is between 1 and 75 inclusive
if current_byte >= 1 and current_byte <= 75:
# we have an cmd set n to be the current byte
n = current_byte
# add the next n bytes as an cmd
cmds.append(s.read(n))
# increase the count by n
count += n
elif current_byte == 76:
# op_pushdata1
data_length = little_endian_to_int(s.read(1))
if (data_length + count) > length:
print(
f"BAD SCRIPT: original script data_length {data_length} "
)
data_length = length - count - 1
print(f"Fixed script data_length to {data_length }")
s.read(data_length)
raise SyntaxError('parsing script failed')
cmds.append(s.read(data_length))
count += data_length + 1
elif current_byte == 77:
# op_pushdata2
data_length = little_endian_to_int(s.read(2))
if (data_length + count) > length:
print(
f"BAD SCRIPT: original script data_length {data_length} "
)
data_length = length - count - 2
print(f"Fixed script data_length to {data_length}")
s.read(data_length)
raise SyntaxError('parsing script failed')
cmds.append(s.read(data_length))
count += data_length + 2
else:
# we have an opcode. set the current byte to op_code
op_code = current_byte
#print(f"op_code : {op_code}")
# add the op_code to the list of cmds
cmds.append(op_code)
if count != length:
raise SyntaxError('parsing script failed')
return cls(cmds)