def getNextBlock(f):
global blkCounter
f.seek(MAGIC_NUMBER_LENGTH, 1)
blkSize = binary_to_int(f.read(BLOCK_SIZE_LENGTH), LITTLEENDIAN)
result = None
if blkSize > 0:
binunpack = BinaryUnpacker(f.read(blkSize))
blkHdrBinary = binunpack.get(BINARY_CHUNK, HEADER_LENGTH)
blkHdr = parseBlockHeader(blkHdrBinary)
blkCounter += 1
txCount = binunpack.get(VAR_INT)
txBinary = binunpack.get(BINARY_CHUNK, binunpack.getRemainingSize())
txOffsetList = getTxOffsetList(txBinary, txCount)
txList = []
for i in range(len(txOffsetList)):
tx = txBinary[txOffsetList[i]:txOffsetList[i+1] if i < len(txOffsetList) - 1 else len(txBinary)]
txList.append(getTx(tx, blkCounter))
result = Block(blkCounter, blkSize,
blkHdr,
txCount,
txBinary,
txOffsetList,txList)
else:
f.seek(0,2)
return result
def testBinaryUnpacker(self):
ts = '\xff\xff\xff'
bu = BinaryUnpacker(ts)
self.assertEqual(bu.getSize(), len(ts))
bu.advance(1)
self.assertEqual(bu.getRemainingSize(), len(ts)-1)
self.assertEqual(bu.getBinaryString(), ts)
self.assertEqual(bu.getRemainingString(), ts[1:])
bu.rewind(1)
self.assertEqual(bu.getRemainingSize(), len(ts))
bu.resetPosition(2)
self.assertEqual(bu.getRemainingSize(), len(ts) - 2)
self.assertEqual(bu.getPosition(), 2)
bu.resetPosition()
self.assertEqual(bu.getRemainingSize(), len(ts))
self.assertEqual(bu.getPosition(), 0)
bu.append(ts)
self.assertEqual(bu.getBinaryString(), ts + ts)
def testBinaryUnpacker(self):
ts = '\xff\xff\xff'
bu = BinaryUnpacker(ts)
self.assertEqual(bu.getSize(), len(ts))
bu.advance(1)
self.assertEqual(bu.getRemainingSize(), len(ts) - 1)
self.assertEqual(bu.getBinaryString(), ts)
self.assertEqual(bu.getRemainingString(), ts[1:])
bu.rewind(1)
self.assertEqual(bu.getRemainingSize(), len(ts))
bu.resetPosition(2)
self.assertEqual(bu.getRemainingSize(), len(ts) - 2)
self.assertEqual(bu.getPosition(), 2)
bu.resetPosition()
self.assertEqual(bu.getRemainingSize(), len(ts))
self.assertEqual(bu.getPosition(), 0)
bu.append(ts)
self.assertEqual(bu.getBinaryString(), ts + ts)
def processTxOutScr(txOutScr, blkHash, blkPos, txIdx, txOutIdx):
'''Function processing a TxOut script.'''
# Proceed only if there's data to read.
retVal = txOutScr
txOutScrUnpack = BinaryUnpacker(txOutScr)
txOutAddress = BinaryPacker()
txType = TxType.unknownTx
txOutScrSize = txOutScrUnpack.getRemainingSize()
if(txOutScrSize > 0):
# Read the initial byte and determine what TxOut type it is.
initByte = txOutScrUnpack.get(BINARY_CHUNK, 1)
# 0x21/0x41 = Pay2PubKey
if(initByte == '\x21' or initByte == '\x41'):
# Make sure it's a valid pub key before declaring it valid.
pkLen = isPubKey(txOutScrUnpack)
if(pkLen != 0):
# Save the pub key.
txOutKey = txOutScrUnpack.get(BINARY_CHUNK, pkLen)
txOutAddress.put(BINARY_CHUNK, txOutKey)
txType = TxType.p2pKey
# OP_DUP = Pay2PubKeyHash
elif(initByte == OP_DUP):
# HACK ALERT: Some bright bulb has created OP_* TxOuts that have
# nothing but the OP_* code. Check the remaining size upfront.
# (Checking after every read is more robust, really. I'm just lazy
# and don't want to retrofit this chunk of code. :) )
if(txOutScrUnpack.getRemainingSize() > 0):
hashByte = txOutScrUnpack.get(BINARY_CHUNK, 1)
if(hashByte == OP_HASH160):
hashSize = txOutScrUnpack.get(BINARY_CHUNK, 1)
hashRemSize = txOutScrUnpack.getRemainingSize()
if(hashSize == '\x14' and \
hashRemSize >= binary_to_int(hashSize)):
txOutHash = txOutScrUnpack.get(BINARY_CHUNK, \
binary_to_int(hashSize))
# Save the hash.
txOutAddress.put(BINARY_CHUNK, txOutHash)
eqVerByte = txOutScrUnpack.get(BINARY_CHUNK, 1)
if(eqVerByte == OP_EQUALVERIFY):
checkSigByte = txOutScrUnpack.get(BINARY_CHUNK, 1)
if(checkSigByte == OP_CHECKSIG):
txType = TxType.p2pHash
# OP_HASH160 = Pay2ScriptHash
elif(initByte == OP_HASH160):
hashSize = txOutScrUnpack.get(BINARY_CHUNK, 1)
hashRemSize = txOutScrUnpack.getRemainingSize()
if(hashSize == '\x14' and hashRemSize >= binary_to_int(hashSize)):
txOutHash = txOutScrUnpack.get(BINARY_CHUNK, \
binary_to_int(hashSize))
# Save the hash.
txOutAddress.put(BINARY_CHUNK, txOutHash)
eqByte = txOutScrUnpack.get(BINARY_CHUNK, 1)
if(eqByte == OP_EQUAL):
txType = TxType.p2sh
# OP_1/2/3 = MultiSig
elif(initByte == OP_1 or initByte == OP_2 or initByte == OP_3):
validKeys = True
readByte = 0
numKeys = 0
# HACK ALERT 1: Some scripts are weird and initially appear to be
# multi-sig but really aren't. We should compensate. One particular
# way is to require at least 36 bytes (assume 1-of-1 w/ compressed
# key) beyond the initial byte.
#
# HACK ALERT 2: There are some multisig TxOuts that, for unknown
# reasons have things like compressed keys that where the first byte
# is 0x00, not 0x02 or 0x03. For now, we just mark them as unknown
# Tx and move on.
if(txOutScrUnpack.getRemainingSize() >= 36):
readByte = txOutScrUnpack.get(BINARY_CHUNK, 1)
while((readByte == '\x21' or readByte == '\x41') and numKeys < 3
and validKeys == True):
pkLen = isPubKey(txOutScrUnpack)
if(pkLen != 0):
txOutKey = txOutScrUnpack.get(BINARY_CHUNK, pkLen)
# Save the key.
txOutAddress.put(BINARY_CHUNK, txOutKey)
numKeys += 1
readByte = txOutScrUnpack.get(BINARY_CHUNK, 1)
else:
validKeys = False
else:
validKeys = False
if(validKeys == True):
if((readByte == OP_1 or readByte == OP_2 or readByte == OP_3) \
and binary_to_int(initByte) <= binary_to_int(readByte)):
cmsByte = txOutScrUnpack.get(BINARY_CHUNK, 1)
if(cmsByte == OP_CHECKMULTISIG):
txType = TxType.multiSig
# OP_RETURN = Arbitrary data attached to a Tx.
# Official as of BC-Core 0.9. https://bitcoinfoundation.org/blog/?p=290
# and https://github.com/bitcoin/bitcoin/pull/2738 have the details of
# the initial commit, with https://github.com/bitcoin/bitcoin/pull/3737
# having the revision down to 40 bytes.
elif(initByte == OP_RETURN):
# If the 1st byte is OP_RETURN, as of BC-Core v0.9, there can be
# arbitrary data placed afterwards. This makes the TxOut immediately
# prunable, meaning it can never be used as a TxIn. (It can still be
# spent, mind you.) The final BC-Core 0.9 only accepts <=40 bytes,
# but preview versions accepted <=80. In theory, any amount of data
# is valid, but miners won't accept non-standard amounts by default.
#
# Anyway, since it's arbitrary, we don't care what's present and
# just assume it's valid. Save all the data as the TxOut address.
opRetData = txOutScrUnpack.get(BINARY_CHUNK, \
txOutScrUnpack.getRemainingSize())
# Save the data.
txOutAddress.put(BINARY_CHUNK, opRetData)
txType = TxType.opReturn
# Everything else isn't standard. For now, we'll do nothing.
# else:
# print("DEBUG: 1st BYTE (TxOut) IS TOTALLY UNKNOWN!!! BYTE={0}".format(binary_to_hex(initByte)))
# Could it be that there is no TxOut script? Is this legal? Let's take a
# peek and write some debug code.
else:
print("DEBUG: At block {0}, we have an empty TxOut script!".format(blkPos))
# If we have a known TxOut type, we'll return an address instead of the
# entire TxOut script. Note that it's unlikely but possible that an address
# value for, say, OP_RETURN could match a pub key hash. In production code,
# we'd want to add extra data to ensure that no collisions have occurred
# (e.g., 2 different addresses are sharing a common balance), or possibly
# use a different approach altogether.
if(txType != TxType.unknownTx):
retVal = txOutAddress.getBinaryString()
if(txType == TxType.p2pKey):
step1 = hash160(retVal)
step2 = hash160_to_addrStr(step1)
retVal = base58_to_binary(step2)
elif(txType == TxType.p2pHash):
step1 = hash160_to_addrStr(retVal)
retVal = base58_to_binary(step1)
return retVal
def processTxInScr(txInScr, blkHash, blkPos, txIdx, txInIdx, txInFile):
'''Function processing a TxIn script.'''
# Proceed only if there's data to read.
txInScrUnpack = BinaryUnpacker(txInScr)
retVal = TxType.unknownTx
txInScrSize = txInScrUnpack.getRemainingSize()
if(txInScrSize > 0):
# Read the initial byte and determine what TxOut type it is.
initByte = txInScrUnpack.get(BINARY_CHUNK, 1)
# Except for multisig and possibly OP_RETURN, all should start with a
# sig.
if(initByte >= '\x43' and initByte <= '\x4b'):
# Make sure it's a valid pub key before declaring it valid.
# CATCH: We'll rewind because the first byte of the sig isn't
# repeated, meaning the stack uses the first byte of the sig to push
# the rest of the sig onto the stack. The rewind isn't necessary but
# I'd like to keep the sig verification whole.
txInScrUnpack.rewind(1)
sigLen = isSigShell(txInScrUnpack, False)
if(sigLen != 0):
txInScrUnpack.advance(sigLen)
if(txInScrUnpack.getRemainingSize() == 0):
retVal = TxType.p2pKey
else:
readByte = txInScrUnpack.get(BINARY_CHUNK, 1)
if(readByte == '\x21' or readByte == '\x41'):
pkLen = isPubKey(txInScrUnpack)
if(pkLen != 0):
retVal = TxType.p2pHash
# OP_0 = P2SH or MultiSig
elif(initByte == OP_0):
numBytesAdv = isSigShell(txInScrUnpack, True)
# Proceed only if there was at least 1 valid sig.
if(numBytesAdv != 0):
txInScrUnpack.advance(numBytesAdv)
numBytesRem = txInScrUnpack.getRemainingSize()
if(numBytesRem != 0):
# Confirm that the remaining bytes are a standard script
# before marking this as a P2SH script. (There are P2SH
# scripts that aren't standard, so we'll mark the entire
# script as unknown and save it.) In a fully robust system,
# we'd Hash160 and compare against the Hash160 in the
# ref'd TxOut to confirm that this is valid.
# NB: In the real world, it looks like all scripts don't
# match the normal TxOut types! Just mark this as P2SH and
# write it out anyway.
# p2shScript = txInScrUnpack.get(BINARY_CHUNK, numBytesRem)
# if(processTxOutScr(p2shScript, blkHash, blkPos, txIdx, \
# txOutIdx, txOutFile) != TxType.unknownTx):
# retVal = TxType.p2sh
# print("HEY, WE GOT A GOOD SCRIPT! {0}".format(binary_to_hex(p2shScript)))
# else:
# print("OH NO, WE HAVE A BAD SCRIPT! {0}".format(binary_to_hex(p2shScript)))
retVal = TxType.p2sh
else:
# We have multi-sig.
retVal = TxType.multiSig
# We have an unknown script type. We'll report it. There's a chance it
# refers to an OP_RETURN TxOut script but we'll ignore that possibility
# for now in order to keep things simple.
else:
print("DEBUG: Block {0}: 1st BYTE (TxIn) IS TOTALLY UNKNOWN!!! " \
"BYTE={1}".format(blkPos, binary_to_hex(initByte)))
# If a script is unknown or is P2SH, write it out here.
# NB: After running this code several times, it appears that the multisig
# code uses keys with invalid first bytes. I'm not sure what's going on. The
# scripts seem valid otherwise.
# if(retVal == TxType.unknownTx or retVal == TxType.p2sh):
# if(retVal == TxType.p2sh):
# print("P2SH script")
# else:
# print("Unknown TxIn script")
if retVal == TxType.unknownTx:
print("TxIn: {0}".format(binary_to_hex(txInScr)), \
file=txInFile)
print("Block Number: {0}".format(blkPos), file=txInFile)
print("Block Hash: {0}".format(binary_to_hex(blkHash)), \
file=txInFile)
print("Tx Hash: {0}", binary_to_hex(curTxHash, endOut=BIGENDIAN), \
file=txInFile)
print("Tx Index: {0}", txIdx, file=txInFile)
print("TxIn Index: {0}", txInIdx, file=txInFile)
print("---------------------------------------", file=txInFile)
return retVal
def processTxOutScr(txOutScr, blkHash, blkPos, txIdx, txOutIdx, txOutFile):
'''Function processing a TxOut script.'''
# Proceed only if there's data to read.
txOutScrUnpack = BinaryUnpacker(txOutScr)
retVal = TxType.unknownTx
txOutScrSize = txOutScrUnpack.getRemainingSize()
if(txOutScrSize > 0):
# Read the initial byte and determine what TxOut type it is.
initByte = txOutScrUnpack.get(BINARY_CHUNK, 1)
# 0x21/0x41 = Pay2PubKey
if(initByte == '\x21' or initByte == '\x41'):
# Make sure it's a valid pub key before declaring it valid.
pkLen = isPubKey(txOutScrUnpack)
if(pkLen != 0):
retVal = TxType.p2pKey
# OP_DUP = Pay2PubKeyHash
elif(initByte == OP_DUP):
# HACK ALERT: Some bright bulb has created OP_* TxOuts that have
# nothing but the OP_* code. Check the remaining size upfront.
# (Checking after every read is more robust, really. I'm just lazy
# and don't want to retrofit this chunk of code. :) )
if(txOutScrUnpack.getRemainingSize() > 0):
hashByte = txOutScrUnpack.get(BINARY_CHUNK, 1)
if(hashByte == OP_HASH160):
hashSize = txOutScrUnpack.get(BINARY_CHUNK, 1)
hashRemSize = txOutScrUnpack.getRemainingSize()
if(hashSize == '\x14' and hashRemSize >= binary_to_int(hashSize)):
txOutScrUnpack.advance(binary_to_int(hashSize))
eqVerByte = txOutScrUnpack.get(BINARY_CHUNK, 1)
if(eqVerByte == OP_EQUALVERIFY):
checkSigByte = txOutScrUnpack.get(BINARY_CHUNK, 1)
if(checkSigByte == OP_CHECKSIG):
retVal = TxType.p2pHash
# OP_HASH160 = Pay2ScriptHash
elif(initByte == OP_HASH160):
hashSize = txOutScrUnpack.get(BINARY_CHUNK, 1)
hashRemSize = txOutScrUnpack.getRemainingSize()
if(hashSize == '\x14' and hashRemSize >= binary_to_int(hashSize)):
txOutScrUnpack.advance(binary_to_int(hashSize))
eqByte = txOutScrUnpack.get(BINARY_CHUNK, 1)
if(eqByte == OP_EQUAL):
retVal = TxType.p2sh
# OP_1/2/3 = MultiSig
elif(initByte == OP_1 or initByte == OP_2 or initByte == OP_3):
validKeys = True
readByte = 0
numKeys = 0
# HACK ALERT 1: Some scripts are weird and initially appear to be
# multi-sig but really aren't. We should compensate. One particular
# way is to require at least 36 bytes (assume 1-of-1 w/ compressed
# key) beyond the initial byte.
#
# HACK ALERT 2: There are some multisig TxOuts that, for unknown
# reasons have things like compressed keys that where the first byte
# is 0x00, not 0x02 or 0x03. For now, we just mark them as unknown
# Tx and move on.
if(txOutScrUnpack.getRemainingSize() >= 36):
readByte = txOutScrUnpack.get(BINARY_CHUNK, 1)
while((readByte == '\x21' or readByte == '\x41') and numKeys < 3
and validKeys == True):
pkLen = isPubKey(txOutScrUnpack)
if(pkLen != 0):
txOutScrUnpack.advance(pkLen)
numKeys += 1
readByte = txOutScrUnpack.get(BINARY_CHUNK, 1)
else:
validKeys = False
else:
validKeys = False
if(validKeys == True):
if((readByte == OP_1 or readByte == OP_2 or readByte == OP_3) \
and binary_to_int(initByte) <= binary_to_int(readByte)):
cmsByte = txOutScrUnpack.get(BINARY_CHUNK, 1)
if(cmsByte == OP_CHECKMULTISIG):
retVal = TxType.multiSig
# OP_RETURN = Arbitrary data attached to a Tx.
# Official as of BC-Core 0.9. https://bitcoinfoundation.org/blog/?p=290
# and https://github.com/bitcoin/bitcoin/pull/2738 have the details of
# the initial commit, with https://github.com/bitcoin/bitcoin/pull/3737
# having the revision down to 40 bytes.
elif(initByte == OP_RETURN):
# If the 1st byte is OP_RETURN, as of BC-Core v0.9, there can be
# arbitrary data placed afterwards. This makes the TxOut immediately
# prunable, meaning it can never be used as a TxIn. (It can still be
# spent, mind you.) The final BC-Core 0.9 only accepts <=40 bytes,
# but preview versions accepted <=80. In theory, any amount of data
# is valid, but miners won't accept non-standard amounts by default.
#
# Anyway, since it's arbitrary, we don't care what's present and
# just assume it's valid.
retVal = TxType.opReturn
# Everything else is weird and should be reported.
else:
print("DEBUG: Block {0} - Tx Hash {1}: 1st BYTE (TxOut) IS " \
"TOTALLY UNKNOWN!!! BYTE={2}".format(blkPos, \
binary_to_hex(curTxHash, endOut=BIGENDIAN), \
binary_to_hex(initByte)))
# Write the script to the file if necessary.
if(retVal == TxType.unknownTx):
print("TxOut: {0}".format(binary_to_hex(txOutScr)), \
file=txOutFile)
print("Block Number: {0}".format(blkPos), file=txOutFile)
print("Block Hash: {0}".format(binary_to_hex(blkHash)), \
file=txOutFile)
print("Tx Hash: {0}", binary_to_hex(curTxHash, endOut=BIGENDIAN), \
file=txOutFile)
print("Tx Index: {0}", txIdx, file=txOutFile)
print("TxOut Index: {0}", txOutIdx, file=txOutFile)
print("---------------------------------------", file=txOutFile)
return retVal
