def tx_hash( cls, tx ):
"""
Calculate the hash of a transction
"""
tx_hex = bits.tx_serialize( tx )
tx_hash = pybitcoin.bin_double_sha256(tx_hex.decode('hex'))[::-1].encode('hex')
return tx_hash
def tx_hash( cls, tx ):
"""
Calculate the hash of a transction
"""
tx_hex = virtualchain.tx_to_hex( tx )
tx_hash = pybitcoin.bin_double_sha256(tx_hex.decode('hex'))[::-1].encode('hex')
return tx_hash
def tx_hash(cls, tx):
"""
Calculate the hash of a transction
"""
tx_hex = virtualchain.tx_to_hex(tx)
tx_hash = pybitcoin.bin_double_sha256(
tx_hex.decode('hex'))[::-1].encode('hex')
return tx_hash
def tx_hash(cls, tx):
"""
Calculate the hash of a transction
"""
tx_hex = bits.tx_serialize(tx)
tx_hash = pybitcoin.bin_double_sha256(
tx_hex.decode('hex'))[::-1].encode('hex')
return tx_hash
def tx_get_hash(tx_serialized):
"""
Make a transaction hash from a hex tx
"""
tx_reversed_bin_hash = pybitcoin.bin_double_sha256(
binascii.unhexlify(tx_serialized))
tx_candidate_hash = binascii.hexlify(tx_reversed_bin_hash[::-1])
return tx_candidate_hash
def block_header_verify( block_data, prev_hash, block_hash ):
"""
Verify whether or not bitcoind's block header matches the hash we expect.
"""
serialized_header = block_header_to_hex( block_data, prev_hash )
candidate_hash_bin_reversed = pybitcoin.bin_double_sha256(binascii.unhexlify(serialized_header))
candidate_hash = binascii.hexlify( candidate_hash_bin_reversed[::-1] )
return block_hash == candidate_hash
def block_header_verify( block_data, prev_hash, block_hash ):
"""
Verify whether or not bitcoind's block header matches the hash we expect.
"""
serialized_header = block_header_to_hex( block_data, prev_hash )
candidate_hash_bin_reversed = pybitcoin.bin_double_sha256(binascii.unhexlify(serialized_header))
candidate_hash = binascii.hexlify( candidate_hash_bin_reversed[::-1] )
return block_hash == candidate_hash
def tx_verify( tx, tx_hash ):
"""
Confirm that a bitcoin transaction has the given hash.
"""
tx_serialized = tx_to_hex( tx )
tx_reversed_bin_hash = pybitcoin.bin_double_sha256( binascii.unhexlify(tx_serialized) )
tx_candidate_hash = binascii.hexlify(tx_reversed_bin_hash[::-1])
return tx_hash == tx_candidate_hash
def tx_verify(tx, tx_hash):
"""
Confirm that a bitcoin transaction has the given hash.
"""
tx_serialized = tx_to_hex(tx)
tx_reversed_bin_hash = pybitcoin.bin_double_sha256(
binascii.unhexlify(tx_serialized))
tx_candidate_hash = binascii.hexlify(tx_reversed_bin_hash[::-1])
return tx_hash == tx_candidate_hash
def flush_transactions( self ):
"""
TESTING ONLY
Send the bufferred list of transactions as a block.
Save the resulting transactions to a temporary file.
"""
# next block
txs = self.next_block_txs
self.next_block_txs = []
# add a fake coinbase
txs.append( "01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff53038349040d00456c69676975730052d8f72ffabe6d6dd991088decd13e658bbecc0b2b4c87306f637828917838c02a5d95d0e1bdff9b0400000000000000002f73733331312f00906b570400000000e4050000ffffffff01bf208795000000001976a9145399c3093d31e4b0af4be1215d59b857b861ad5d88ac00000000" )
block_txs = {}
block_txids = []
for tx in txs:
txid = make_txid( tx )
block_txids.append( txid )
block_txs[ txid ] = tx
version = '01000000'
t_hex = "%08X" % self.time
difficulty_hex = "%08X" % self.difficulty
tx_merkle_tree = pybitcoin.MerkleTree( block_txids )
tx_merkle_root = tx_merkle_tree.root()
prev_block_hash = self.block_hashes[ self.end_block - 1 ]
# next block
block = {
'merkleroot': tx_merkle_root,
'nonce': 0, # mock
'previousblockhash': prev_block_hash,
'version': 3,
'tx': block_txids,
'chainwork': '00' * 32, # mock
'height': self.end_block,
'difficulty': Decimal(0.0), # mock
'nextblockhash': None, # to be filled in
'confirmations': None, # to be filled in
'time': self.time, # mock
'bits': "0x00000000", # mock
'size': sum( [len(tx) for tx in txs] ) + 32 # mock
}
block_header = bitcoin.main.encode(block['version'], 256, 4)[::-1] + \
block['previousblockhash'].decode('hex')[::-1] + \
block['merkleroot'].decode('hex')[::-1] + \
bitcoin.main.encode(block['time'], 256, 4)[::-1] + \
bitcoin.main.encode(int(block['bits'], 16), 256, 4)[::-1] + \
bitcoin.main.encode(block['nonce'], 256, 4)[::-1]
block['hash'] = pybitcoin.bin_double_sha256( block_header )[::-1].encode('hex')
block['header'] = block_header
for txid in block['tx']:
# update txid --> blockhash map
self.txid_to_blockhash[ txid ] = block['hash']
# update nextblockhash at least
self.blocks[prev_block_hash]['nextblockhash'] = block['hash']
self.block_hashes[ self.end_block ] = block['hash']
self.blocks[ block['hash'] ] = block
self.txs.update( block_txs )
self.time += 600 # 10 minutes
self.difficulty += 1
self.end_block += 1
if self.save_file is not None:
self.save( self.save_file )
if self.spv_headers_path is not None:
with open(self.spv_headers_path, "a+") as f:
f.write( block_header )
f.write( "00".decode('hex') ) # our SPV client expects varint for tx count to be zero
return [ make_txid( tx ) for tx in txs ]
return None
try:
for resp in resp_json:
assert 'result' in resp, "Missing result"
txhex = resp['result']
assert txhex is not None, "Invalid RPC response '%s' (for %s)" % (
simplejson.dumps(resp), txids[resp['id']])
try:
tx_bin = txhex.decode('hex')
assert tx_bin is not None
tx_hash_bin = pybitcoin.bin_double_sha256(tx_bin)[::-1]
assert tx_hash_bin is not None
tx_hash = tx_hash_bin.encode('hex')
assert tx_hash is not None
except Exception, e:
log.error("Failed to calculate txid of %s" % txhex)
raise
# solicited transaction?
assert tx_hash in txids, "Unsolicited transaction %s" % tx_hash
# unique?
if tx_hash in ret.keys():
continue
def testnet_encode( pk_wif ):
s = pybitcoin.b58check_decode(pk_wif )
s = '\xef' + s
ret = base58.b58encode( s + pybitcoin.bin_double_sha256(s)[0:4] )
return ret
def flush_transactions( self ):
"""
TESTING ONLY
Send the bufferred list of transactions as a block.
Save the resulting transactions to a temporary file.
"""
# next block
txs = []
if self.next_block_txs_path is not None and os.path.exists( self.next_block_txs_path ):
txs = self.restore_next( self.next_block_txs_path )
os.unlink( self.next_block_txs_path )
# add a fake coinbase
txs.append( "01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff53038349040d00456c69676975730052d8f72ffabe6d6dd991088decd13e658bbecc0b2b4c87306f637828917838c02a5d95d0e1bdff9b0400000000000000002f73733331312f00906b570400000000e4050000ffffffff01bf208795000000001976a9145399c3093d31e4b0af4be1215d59b857b861ad5d88ac00000000" )
block_txs = {}
block_txids = []
for tx in txs:
txid = make_txid( str(tx) )
block_txids.append( txid )
block_txs[ txid ] = tx
version = '01000000'
t_hex = "%08X" % self.time
difficulty_hex = "%08X" % self.difficulty
tx_merkle_tree = pybitcoin.MerkleTree( block_txids )
tx_merkle_root = tx_merkle_tree.root()
prev_block_hash = self.block_hashes[ self.end_block - 1 ]
# next block
block = {
'merkleroot': tx_merkle_root,
'nonce': 0, # mock
'previousblockhash': prev_block_hash,
'version': 3,
'tx': block_txids,
'chainwork': '00' * 32, # mock
'height': self.end_block,
'difficulty': Decimal(0.0), # mock
'nextblockhash': None, # to be filled in
'confirmations': None, # to be filled in
'time': self.time, # mock
'bits': "0x00000000", # mock
'size': sum( [len(tx) for tx in txs] ) + 32 # mock
}
block_header = bitcoin.main.encode(block['version'], 256, 4)[::-1] + \
block['previousblockhash'].decode('hex')[::-1] + \
block['merkleroot'].decode('hex')[::-1] + \
bitcoin.main.encode(block['time'], 256, 4)[::-1] + \
bitcoin.main.encode(int(block['bits'], 16), 256, 4)[::-1] + \
bitcoin.main.encode(block['nonce'], 256, 4)[::-1]
block['hash'] = pybitcoin.bin_double_sha256( block_header )[::-1].encode('hex')
block['header'] = block_header
for txid in block['tx']:
# update txid --> blockhash map
self.txid_to_blockhash[ txid ] = block['hash']
# update nextblockhash at least
self.blocks[prev_block_hash]['nextblockhash'] = block['hash']
self.block_hashes[ self.end_block ] = block['hash']
self.blocks[ block['hash'] ] = block
self.txs.update( block_txs )
self.time += 600 # 10 minutes
self.difficulty += 1
self.end_block += 1
if self.save_file is not None:
self.save( self.save_file )
if self.spv_headers_path is not None:
with open(self.spv_headers_path, "a+") as f:
f.write( block_header )
f.write( "00".decode('hex') ) # our SPV client expects varint for tx count to be zero
return [ make_txid( str(tx) ) for tx in txs ]
def get_nulldata_txs_in_blocks(workpool,
bitcoind_opts,
blocks_ids,
first_block_hash=None):
"""
Obtain the set of transactions over a range of blocks that have an OP_RETURN with nulldata.
Each returned transaction record will contain:
* vin (list of inputs from bitcoind)
* vout (list of outputs from bitcoind)
* txid (transaction ID, as a hex string)
* txindex (transaction index in the block)
* senders (a list of {"script_pubkey":, "amount":, and "addresses":} dicts; the "script_pubkey" field is the hex-encoded op script).
* fee (total amount sent)
* nulldata (input data to the transaction's script; encodes virtual chain operations)
Farm out the requisite RPCs to a workpool of processes, each
of which have their own bitcoind RPC client.
Returns [(block_number, [txs])], where each tx contains the above.
"""
nulldata_tx_map = {} # {block_number: {"tx": [tx]}}
block_bandwidth = {
} # {block_number: {"time": time taken to process, "size": number of bytes}}
nulldata_txs = []
# break work up into slices of blocks, so we don't run out of memory
slice_len = multiprocess_batch_size(bitcoind_opts)
slice_count = 0
last_block_hash = first_block_hash
while slice_count * slice_len < len(blocks_ids):
block_hashes = {} # map block ID to block hash
block_datas = {} # map block hashes to block data
block_hash_futures = []
block_data_futures = []
tx_futures = []
nulldata_tx_futures = []
all_nulldata_tx_futures = []
block_times = {} # {block_number: time taken to process}
block_slice = blocks_ids[(slice_count *
slice_len):min((slice_count + 1) *
slice_len, len(blocks_ids))]
if len(block_slice) == 0:
log.debug("Zero-length block slice")
break
start_slice_time = time.time()
# get all block hashes
for block_number in block_slice:
block_times[block_number] = time.time()
block_hash_fut = getblockhash_async(workpool, bitcoind_opts,
block_number)
block_hash_futures.append((block_number, block_hash_fut))
# coalesce all block hashes
block_hash_time_start = time.time()
block_hash_time_end = 0
for i in xrange(0, len(block_hash_futures)):
block_number, block_hash_fut = future_next(block_hash_futures,
lambda f: f[1])
# NOTE: interruptable blocking get(), but should not block since future_next found one that's ready
block_hash = future_get_result(block_hash_fut, 10000000000000000L)
block_hashes[block_number] = block_hash
# start getting each block's data
if block_hash is not None:
block_data_fut = getblock_async(workpool, bitcoind_opts,
block_hash)
block_data_futures.append((block_number, block_data_fut))
else:
raise Exception("BUG: Block %s: no block hash" % block_number)
block_data_time_start = time.time()
block_data_time_end = 0
# coalesce block data
for i in xrange(0, len(block_data_futures)):
block_number, block_data_fut = future_next(block_data_futures,
lambda f: f[1])
block_hash_time_end = time.time()
# NOTE: interruptable blocking get(), but should not block since future_next found one that's ready
block_data = future_get_result(block_data_fut, 1000000000000000L)
if 'tx' not in block_data:
raise Exception("BUG: No tx data in block %s" % block_number)
block_datas[block_hashes[block_number]] = block_data
# verify blockchain headers
for i in xrange(0, len(block_slice)):
block_id = block_slice[i]
block_hash = block_hashes[block_id]
prev_block_hash = None
if i > 0:
prev_block_id = block_slice[i - 1]
prev_block_hash = block_hashes[prev_block_id]
elif last_block_hash is not None:
prev_block_hash = last_block_hash
else:
continue
if not block_header_verify(block_datas[block_hash],
prev_block_hash, block_hash):
serialized_header = block_header_to_hex(
block_datas[block_hash], prev_block_hash)
candidate_hash_reversed = pybitcoin.bin_double_sha256(
binascii.unhexlify(serialized_header))
candidate_hash = binascii.hexlify(
candidate_hash_reversed[::-1])
raise Exception(
"Hash mismatch on block %s: got invalid block hash (expected %s, got %s)"
% (block_id, block_hash, candidate_hash))
last_block_hash = block_hashes[block_slice[-1]]
for block_number in block_slice:
block_hash = block_hashes[block_number]
block_data = block_datas[block_hash]
# verify block data txs
rc = block_verify(block_data)
if not rc:
raise Exception(
"Hash mismatch on block %s: got invalid Merkle root (expected %s)"
% (block_hash, block_data['merkleroot']))
# go get each transaction
tx_hashes = block_data['tx']
log.debug("Get %s transactions from block %s" %
(len(tx_hashes), block_hash))
# can get transactions asynchronously with a workpool (but preserve tx order!)
if len(tx_hashes) > 0:
for j in xrange(0, len(tx_hashes)):
tx_hash = tx_hashes[j]
tx_fut = getrawtransaction_async(workpool, bitcoind_opts,
tx_hash, 1)
tx_futures.append((block_number, j, tx_fut))
else:
raise Exception("BUG: Zero-transaction block %s" %
block_number)
block_tx_time_start = time.time()
block_tx_time_end = 0
# coalesce raw transaction queries...
for i in xrange(0, len(tx_futures)):
block_number, tx_index, tx_fut = future_next(
tx_futures, lambda f: f[2])
block_data_time_end = time.time()
# NOTE: interruptable blocking get(), but should not block since future_next found one that's ready
tx = future_get_result(tx_fut, 1000000000000000L)
#if len(tx['vin']) > 0 and 'coinbase' not in tx['vin'][0].keys():
if not tx_is_coinbase(tx):
# verify non-coinbase transaction
tx_hash = tx['txid']
if not tx_verify(tx, tx_hash):
raise Exception(
"Transaction hash mismatch in %s (index %s) in block %s"
% (tx['txid'], tx_index, block_number))
if tx and has_nulldata(tx):
# go get input transactions for this transaction (since it's the one with nulldata, i.e., a virtual chain operation),
# but tag each future with the hash of the current tx, so we can reassemble the in-flight inputs back into it.
nulldata_tx_futs_and_output_idxs = process_nulldata_tx_async(
workpool, bitcoind_opts, tx)
nulldata_tx_futures.append((block_number, tx_index, tx,
nulldata_tx_futs_and_output_idxs))
else:
# maybe done with this block
# NOTE will be called multiple times; we expect the last write to be the total time taken by this block
total_time = time.time() - block_times[block_number]
block_bandwidth[block_number] = bandwidth_record(
total_time, None)
block_nulldata_tx_time_start = time.time()
block_nulldata_tx_time_end = 0
# coalesce queries on the inputs to each nulldata transaction from this block...
for (block_number, tx_index, tx,
nulldata_tx_futs_and_output_idxs) in nulldata_tx_futures:
if ('vin' not in tx) or ('vout' not in tx) or ('txid' not in tx):
continue
outputs = tx['vout']
total_in = 0 # total input paid
senders = []
ordered_senders = []
# gather this tx's nulldata-bearing transactions
for i in xrange(0, len(nulldata_tx_futs_and_output_idxs)):
input_idx, input_tx_fut, tx_output_index = future_next(
nulldata_tx_futs_and_output_idxs, lambda f: f[1])
# NOTE: interruptable blocking get(), but should not block since future_next found one that's ready
input_tx = future_get_result(input_tx_fut, 1000000000000000L)
input_tx_hash = input_tx['txid']
# verify (but skip coinbase)
if not tx_is_coinbase(input_tx):
try:
if not tx_verify(input_tx, input_tx_hash):
raise Exception(
"Input transaction hash mismatch %s from tx %s (index %s)"
% (input_tx['txid'], tx['txid'],
tx_output_index))
except:
pp = pprint.PrettyPrinter()
pp.pprint(input_tx)
raise
sender, amount_in = get_sender_and_amount_in_from_txn(
input_tx, tx_output_index)
if sender is None or amount_in is None:
continue
total_in += amount_in
# preserve sender order...
ordered_senders.append((input_idx, sender))
# sort on input_idx, so the list of senders matches the given transaction's list of inputs
ordered_senders.sort()
senders = [sender for (_, sender) in ordered_senders]
total_out = get_total_out(outputs)
nulldata = get_nulldata(tx)
# extend tx to explicitly record its nulldata (i.e. the virtual chain op),
# the list of senders (i.e. their script hexs),
# and the total amount paid
tx['nulldata'] = nulldata
tx['senders'] = senders
tx['fee'] = total_in - total_out
# track the order of nulldata-containing transactions in this block
if not nulldata_tx_map.has_key(block_number):
nulldata_tx_map[block_number] = [(tx_index, tx)]
else:
nulldata_tx_map[block_number].append((tx_index, tx))
# maybe done with this block
# NOTE will be called multiple times; we expect the last write to be the total time taken by this block
total_time = time.time() - block_times[block_number]
block_bandwidth[block_number] = bandwidth_record(total_time, None)
# record bandwidth information
for block_number in block_slice:
block_data = None
if nulldata_tx_map.has_key(block_number):
tx_list = nulldata_tx_map[block_number] # [(tx_index, tx)]
tx_list.sort(
) # sorts on tx_index--preserves order in the block
txs = [tx for (_, tx) in tx_list]
block_data = txs
if not block_bandwidth.has_key(block_number):
# done with this block now
total_time = time.time() - block_times[block_number]
block_bandwidth[block_number] = bandwidth_record(
total_time, block_data)
block_tx_time_end = time.time()
block_nulldata_tx_time_end = time.time()
end_slice_time = time.time()
total_processing_time = sum(
map(lambda block_id: block_bandwidth[block_id]["time"],
block_bandwidth.keys()))
total_data = sum(
map(lambda block_id: block_bandwidth[block_id]["size"],
block_bandwidth.keys()))
block_hash_time = block_hash_time_end - block_hash_time_start
block_data_time = block_data_time_end - block_data_time_start
block_tx_time = block_tx_time_end - block_tx_time_start
block_nulldata_tx_time = block_nulldata_tx_time_end - block_nulldata_tx_time_start
# log some stats...
log.debug("blocks %s-%s (%s):" %
(block_slice[0], block_slice[-1], len(block_slice)))
log.debug(" Time total: %s" % total_processing_time)
log.debug(" Data total: %s" % total_data)
log.debug(" Total goodput: %s" % (total_data /
(total_processing_time + 1e-7)))
log.debug(" block hash time: %s" % block_hash_time)
log.debug(" block data time: %s" % block_data_time)
log.debug(" block tx time: %s" % block_tx_time)
log.debug(" block nulldata tx time: %s" % block_nulldata_tx_time)
# next slice
slice_count += 1
# get the blockchain-ordered list of nulldata-containing transactions.
# this is the blockchain-agreed list of all virtual chain operations, as well as the amount paid per transaction and the
# principal(s) who created each transaction.
# convert {block_number: [tx]} to [(block_number, [tx])] where [tx] is ordered by the order in which the transactions occurred in the block
for block_number in blocks_ids:
txs = []
if block_number in nulldata_tx_map.keys():
tx_list = nulldata_tx_map[block_number] # [(tx_index, tx)]
tx_list.sort() # sorts on tx_index--preserves order in the block
# preserve index
for (tx_index, tx) in tx_list:
tx['txindex'] = tx_index
txs = [tx for (_, tx) in tx_list]
nulldata_txs.append((block_number, txs))
return nulldata_txs
def parse_nameop( opcode, payload, fake_pubkey, recipient=None, recipient_address=None, import_update_hash=None ):
opcode_name = OPCODE_NAMES[opcode]
pubk = pybitcoin.BitcoinPublicKey(fake_pubkey)
address = pubk.address()
script_pubkey = pybitcoin.make_pay_to_address_script( address )
senders = [{
"script_pubkey": script_pubkey,
"script_type": "pubkeyhash",
"addresses": [ address ]
}]
# just enough to get the public key
inputs = [{
"scriptSig": {
"asm": "ignored %s" % fake_pubkey,
}
}]
script = "OP_RETURN %s" % payload
try:
scripthex = pybitcoin.make_op_return_script( payload )
except:
if len(payload) == 0:
scripthex = "6a"
else:
raise
outputs = [{
"scriptPubKey": {
"asm": script,
"hex": scripthex,
"addresses": []
}}]
if recipient_address is not None:
script = "OP_DUP OP_HASH160 %s OP_EQUALVERIFY OP_CHECKSIG" % binascii.hexlify( pybitcoin.bin_double_sha256( fake_pubkey ) )
scripthex = pybitcoin.make_pay_to_address_script( recipient_address )
outputs.append( {
"scriptPubKey": {
"asm": script,
"hex": scripthex,
"addresses": [ recipient_address ]
}
})
if import_update_hash is not None:
script = "OP_DUP OP_HASH160 %s OP_EQUALVERIFY OP_CHECKSIG" % import_update_hash
scripthex = pybitcoin.make_pay_to_address_script( pybitcoin.hex_hash160_to_address( import_update_hash ) )
outputs.append( {
"scriptPubKey": {
"asm": script,
"hex": scripthex,
"addresses": [ pybitcoin.hex_hash160_to_address(import_update_hash) ]
}
})
try:
op = op_extract( opcode_name, payload, senders, inputs, outputs, 373601, 0, "00" * 64 )
except AssertionError, ae:
# failed to parse
return None
def __init__(self, data): bindoublehash = bin_double_sha256(data) reversedhash = reverse_hash(bindoublehash) self.dgst = reversedhash
def parse_nameop(opcode,
payload,
fake_pubkey,
recipient=None,
recipient_address=None,
import_update_hash=None):
opcode_name = OPCODE_NAMES[opcode]
pubk = pybitcoin.BitcoinPublicKey(fake_pubkey)
address = pubk.address()
script_pubkey = pybitcoin.make_pay_to_address_script(address)
senders = [{
"script_pubkey": script_pubkey,
"script_type": "pubkeyhash",
"addresses": [address]
}]
# just enough to get the public key
inputs = [{
"scriptSig": {
"asm": "ignored %s" % fake_pubkey,
}
}]
script = "OP_RETURN %s" % payload
try:
scripthex = pybitcoin.make_op_return_script(payload)
except:
if len(payload) == 0:
scripthex = "6a"
else:
raise
outputs = [{
"scriptPubKey": {
"asm": script,
"hex": scripthex,
"addresses": []
}
}]
if recipient_address is not None:
script = "OP_DUP OP_HASH160 %s OP_EQUALVERIFY OP_CHECKSIG" % binascii.hexlify(
pybitcoin.bin_double_sha256(fake_pubkey))
scripthex = pybitcoin.make_pay_to_address_script(recipient_address)
outputs.append({
"scriptPubKey": {
"asm": script,
"hex": scripthex,
"addresses": [recipient_address]
}
})
if import_update_hash is not None:
script = "OP_DUP OP_HASH160 %s OP_EQUALVERIFY OP_CHECKSIG" % import_update_hash
scripthex = pybitcoin.make_pay_to_address_script(
pybitcoin.hex_hash160_to_address(import_update_hash))
outputs.append({
"scriptPubKey": {
"asm": script,
"hex": scripthex,
"addresses":
[pybitcoin.hex_hash160_to_address(import_update_hash)]
}
})
try:
op = op_extract(opcode_name, payload, senders, inputs, outputs, 373601,
0, "00" * 64)
except AssertionError, ae:
# failed to parse
return None
def get_nulldata_txs_in_blocks( workpool, bitcoind_opts, blocks_ids, first_block_hash=None ):
"""
Obtain the set of transactions over a range of blocks that have an OP_RETURN with nulldata.
Each returned transaction record will contain:
* vin (list of inputs from bitcoind)
* vout (list of outputs from bitcoind)
* txid (transaction ID, as a hex string)
* txindex (transaction index in the block)
* senders (a list of {"script_pubkey":, "amount":, and "addresses":} dicts; the "script_pubkey" field is the hex-encoded op script).
* fee (total amount sent)
* nulldata (input data to the transaction's script; encodes virtual chain operations)
Farm out the requisite RPCs to a workpool of processes, each
of which have their own bitcoind RPC client.
Returns [(block_number, [txs])], where each tx contains the above.
"""
nulldata_tx_map = {} # {block_number: {"tx": [tx]}}
block_bandwidth = {} # {block_number: {"time": time taken to process, "size": number of bytes}}
nulldata_txs = []
# break work up into slices of blocks, so we don't run out of memory
slice_len = multiprocess_batch_size( bitcoind_opts )
slice_count = 0
last_block_hash = first_block_hash
while slice_count * slice_len < len(blocks_ids):
block_hashes = {} # map block ID to block hash
block_datas = {} # map block hashes to block data
block_hash_futures = []
block_data_futures = []
tx_futures = []
nulldata_tx_futures = []
all_nulldata_tx_futures = []
block_times = {} # {block_number: time taken to process}
block_slice = blocks_ids[ (slice_count * slice_len) : min((slice_count+1) * slice_len, len(blocks_ids)) ]
if len(block_slice) == 0:
log.debug("Zero-length block slice")
break
start_slice_time = time.time()
# get all block hashes
for block_number in block_slice:
block_times[block_number] = time.time()
block_hash_fut = getblockhash_async( workpool, bitcoind_opts, block_number )
block_hash_futures.append( (block_number, block_hash_fut) )
# coalesce all block hashes
block_hash_time_start = time.time()
block_hash_time_end = 0
for i in xrange(0, len(block_hash_futures)):
block_number, block_hash_fut = future_next( block_hash_futures, lambda f: f[1] )
# NOTE: interruptable blocking get(), but should not block since future_next found one that's ready
block_hash = future_get_result( block_hash_fut, 10000000000000000L )
block_hashes[block_number] = block_hash
# start getting each block's data
if block_hash is not None:
block_data_fut = getblock_async( workpool, bitcoind_opts, block_hash )
block_data_futures.append( (block_number, block_data_fut) )
else:
raise Exception("BUG: Block %s: no block hash" % block_number)
block_data_time_start = time.time()
block_data_time_end = 0
# coalesce block data
for i in xrange(0, len(block_data_futures)):
block_number, block_data_fut = future_next( block_data_futures, lambda f: f[1] )
block_hash_time_end = time.time()
# NOTE: interruptable blocking get(), but should not block since future_next found one that's ready
block_data = future_get_result( block_data_fut, 1000000000000000L )
if 'tx' not in block_data:
raise Exception("BUG: No tx data in block %s" % block_number)
block_datas[ block_hashes[block_number] ] = block_data
# verify blockchain headers
for i in xrange(0, len(block_slice)):
block_id = block_slice[i]
block_hash = block_hashes[block_id]
prev_block_hash = None
if i > 0:
prev_block_id = block_slice[i-1]
prev_block_hash = block_hashes[prev_block_id]
elif last_block_hash is not None:
prev_block_hash = last_block_hash
else:
continue
if not block_header_verify( block_datas[block_hash], prev_block_hash, block_hash ):
serialized_header = block_header_to_hex( block_datas[block_hash], prev_block_hash )
candidate_hash_reversed = pybitcoin.bin_double_sha256(binascii.unhexlify(serialized_header))
candidate_hash = binascii.hexlify(candidate_hash_reversed[::-1])
raise Exception("Hash mismatch on block %s: got invalid block hash (expected %s, got %s)" % (block_id, block_hash, candidate_hash))
last_block_hash = block_hashes[ block_slice[-1] ]
for block_number in block_slice:
block_hash = block_hashes[block_number]
block_data = block_datas[block_hash]
# verify block data txs
rc = block_verify( block_data )
if not rc:
raise Exception("Hash mismatch on block %s: got invalid Merkle root (expected %s)" % (block_hash, block_data['merkleroot']))
# go get each transaction
tx_hashes = block_data['tx']
log.debug("Get %s transactions from block %s" % (len(tx_hashes), block_hash))
# can get transactions asynchronously with a workpool (but preserve tx order!)
if len(tx_hashes) > 0:
for j in xrange(0, len(tx_hashes)):
tx_hash = tx_hashes[j]
tx_fut = getrawtransaction_async( workpool, bitcoind_opts, tx_hash, 1 )
tx_futures.append( (block_number, j, tx_fut) )
else:
raise Exception("BUG: Zero-transaction block %s" % block_number)
block_tx_time_start = time.time()
block_tx_time_end = 0
# coalesce raw transaction queries...
for i in xrange(0, len(tx_futures)):
block_number, tx_index, tx_fut = future_next( tx_futures, lambda f: f[2] )
block_data_time_end = time.time()
# NOTE: interruptable blocking get(), but should not block since future_next found one that's ready
tx = future_get_result( tx_fut, 1000000000000000L )
#if len(tx['vin']) > 0 and 'coinbase' not in tx['vin'][0].keys():
if not tx_is_coinbase( tx ):
# verify non-coinbase transaction
tx_hash = tx['txid']
if not tx_verify( tx, tx_hash ):
raise Exception("Transaction hash mismatch in %s (index %s) in block %s" % (tx['txid'], tx_index, block_number))
if tx and has_nulldata(tx):
# go get input transactions for this transaction (since it's the one with nulldata, i.e., a virtual chain operation),
# but tag each future with the hash of the current tx, so we can reassemble the in-flight inputs back into it.
nulldata_tx_futs_and_output_idxs = process_nulldata_tx_async( workpool, bitcoind_opts, tx )
nulldata_tx_futures.append( (block_number, tx_index, tx, nulldata_tx_futs_and_output_idxs) )
else:
# maybe done with this block
# NOTE will be called multiple times; we expect the last write to be the total time taken by this block
total_time = time.time() - block_times[ block_number ]
block_bandwidth[ block_number ] = bandwidth_record( total_time, None )
block_nulldata_tx_time_start = time.time()
block_nulldata_tx_time_end = 0
# coalesce queries on the inputs to each nulldata transaction from this block...
for (block_number, tx_index, tx, nulldata_tx_futs_and_output_idxs) in nulldata_tx_futures:
if ('vin' not in tx) or ('vout' not in tx) or ('txid' not in tx):
continue
outputs = tx['vout']
total_in = 0 # total input paid
senders = []
ordered_senders = []
# gather this tx's nulldata-bearing transactions
for i in xrange(0, len(nulldata_tx_futs_and_output_idxs)):
input_idx, input_tx_fut, tx_output_index = future_next( nulldata_tx_futs_and_output_idxs, lambda f: f[1] )
# NOTE: interruptable blocking get(), but should not block since future_next found one that's ready
input_tx = future_get_result( input_tx_fut, 1000000000000000L )
input_tx_hash = input_tx['txid']
# verify (but skip coinbase)
if not tx_is_coinbase( input_tx ):
try:
if not tx_verify( input_tx, input_tx_hash ):
raise Exception("Input transaction hash mismatch %s from tx %s (index %s)" % (input_tx['txid'], tx['txid'], tx_output_index))
except:
pp = pprint.PrettyPrinter()
pp.pprint(input_tx)
raise
sender, amount_in = get_sender_and_amount_in_from_txn( input_tx, tx_output_index )
if sender is None or amount_in is None:
continue
total_in += amount_in
# preserve sender order...
ordered_senders.append( (input_idx, sender) )
# sort on input_idx, so the list of senders matches the given transaction's list of inputs
ordered_senders.sort()
senders = [sender for (_, sender) in ordered_senders]
total_out = get_total_out( outputs )
nulldata = get_nulldata( tx )
# extend tx to explicitly record its nulldata (i.e. the virtual chain op),
# the list of senders (i.e. their script hexs),
# and the total amount paid
tx['nulldata'] = nulldata
tx['senders'] = senders
tx['fee'] = total_in - total_out
# track the order of nulldata-containing transactions in this block
if not nulldata_tx_map.has_key( block_number ):
nulldata_tx_map[ block_number ] = [(tx_index, tx)]
else:
nulldata_tx_map[ block_number ].append( (tx_index, tx) )
# maybe done with this block
# NOTE will be called multiple times; we expect the last write to be the total time taken by this block
total_time = time.time() - block_times[ block_number ]
block_bandwidth[ block_number ] = bandwidth_record( total_time, None )
# record bandwidth information
for block_number in block_slice:
block_data = None
if nulldata_tx_map.has_key( block_number ):
tx_list = nulldata_tx_map[ block_number ] # [(tx_index, tx)]
tx_list.sort() # sorts on tx_index--preserves order in the block
txs = [ tx for (_, tx) in tx_list ]
block_data = txs
if not block_bandwidth.has_key( block_number ):
# done with this block now
total_time = time.time() - block_times[ block_number ]
block_bandwidth[ block_number ] = bandwidth_record( total_time, block_data )
block_tx_time_end = time.time()
block_nulldata_tx_time_end = time.time()
end_slice_time = time.time()
total_processing_time = sum( map( lambda block_id: block_bandwidth[block_id]["time"], block_bandwidth.keys() ) )
total_data = sum( map( lambda block_id: block_bandwidth[block_id]["size"], block_bandwidth.keys() ) )
block_hash_time = block_hash_time_end - block_hash_time_start
block_data_time = block_data_time_end - block_data_time_start
block_tx_time = block_tx_time_end - block_tx_time_start
block_nulldata_tx_time = block_nulldata_tx_time_end - block_nulldata_tx_time_start
# log some stats...
log.debug("blocks %s-%s (%s):" % (block_slice[0], block_slice[-1], len(block_slice)) )
log.debug(" Time total: %s" % total_processing_time )
log.debug(" Data total: %s" % total_data )
log.debug(" Total goodput: %s" % (total_data / (total_processing_time + 1e-7)))
log.debug(" block hash time: %s" % block_hash_time)
log.debug(" block data time: %s" % block_data_time)
log.debug(" block tx time: %s" % block_tx_time)
log.debug(" block nulldata tx time: %s" % block_nulldata_tx_time)
# next slice
slice_count += 1
# get the blockchain-ordered list of nulldata-containing transactions.
# this is the blockchain-agreed list of all virtual chain operations, as well as the amount paid per transaction and the
# principal(s) who created each transaction.
# convert {block_number: [tx]} to [(block_number, [tx])] where [tx] is ordered by the order in which the transactions occurred in the block
for block_number in blocks_ids:
txs = []
if block_number in nulldata_tx_map.keys():
tx_list = nulldata_tx_map[ block_number ] # [(tx_index, tx)]
tx_list.sort() # sorts on tx_index--preserves order in the block
# preserve index
for (tx_index, tx) in tx_list:
tx['txindex'] = tx_index
txs = [ tx for (_, tx) in tx_list ]
nulldata_txs.append( (block_number, txs) )
return nulldata_txs
def testnet_encode( pk_wif ):
s = pybitcoin.b58check_decode(pk_wif )
s = '\xef' + s
ret = base58.b58encode( s + pybitcoin.bin_double_sha256(s)[0:4] )
return ret
log.error("Failed to parse transactions")
return None
try:
for resp in resp_json:
assert 'result' in resp, "Missing result"
txhex = resp['result']
assert txhex is not None, "Invalid RPC response '%s' (for %s)" % (simplejson.dumps(resp), txids[resp['id']])
try:
tx_bin = txhex.decode('hex')
assert tx_bin is not None
tx_hash_bin = pybitcoin.bin_double_sha256(tx_bin)[::-1]
assert tx_hash_bin is not None
tx_hash = tx_hash_bin.encode('hex')
assert tx_hash is not None
except Exception, e:
log.error("Failed to calculate txid of %s" % txhex)
raise
# solicited transaction?
assert tx_hash in txids, "Unsolicited transaction %s" % tx_hash
# unique?
if tx_hash in ret.keys():
continue
