def get_txid(self, rtype=hex, endianness="LE"):
""" Computes the transaction id (i.e: transaction hash for non-segwit txs).
:param rtype: Defines the type of return, either hex str or bytes.
:type rtype: str or bin
:param endianness: Whether the id is returned in BE (Big endian) or LE (Little Endian) (default one)
:type endianness: str
:return: The hash of the transaction (i.e: transaction id)
:rtype: hex str or bin, depending on rtype parameter.
"""
if rtype not in [hex, bin]:
raise Exception(
"Invalid return type (rtype). It should be either hex or bin.")
if endianness not in ["BE", "LE"]:
raise Exception(
"Invalid endianness type. It should be either BE or LE.")
if rtype is hex:
tx_id = hexlify(
sha256(sha256(self.serialize(rtype=bin)).digest()).digest())
if endianness == "BE":
tx_id = change_endianness(tx_id)
else:
tx_id = sha256(sha256(self.serialize(rtype=bin)).digest()).digest()
if endianness == "BE":
tx_id = unhexlify(change_endianness(hexlify(tx_id)))
return tx_id
def serialize(self, rtype=hex):
""" Serialize all the transaction fields arranged in the proper order, resulting in a hexadecimal string
ready to be broadcast to the network.
:param self: self
:type self: TX
:param rtype: Whether the serialized transaction is returned as a hex str or a byte array.
:type rtype: hex or bool
:return: Serialized transaction representation (hexadecimal or bin depending on rtype parameter).
:rtype: hex str / bin
"""
if rtype not in [hex, bin]:
raise Exception(
"Invalid return type (rtype). It should be either hex or bin.")
serialized_tx = change_endianness(int2bytes(
self.version, 4)) # 4-byte version number (LE).
# INPUTS
serialized_tx += encode_varint(self.inputs) # Varint number of inputs.
for i in range(self.inputs):
serialized_tx += change_endianness(
self.prev_tx_id[i]
) # 32-byte hash of the previous transaction (LE).
serialized_tx += change_endianness(
int2bytes(self.prev_out_index[i],
4)) # 4-byte output index (LE)
serialized_tx += encode_varint(len(self.scriptSig[i].content) /
2) # Varint input script length.
# ScriptSig
serialized_tx += self.scriptSig[i].content # Input script.
serialized_tx += int2bytes(self.nSequence[i],
4) # 4-byte sequence number.
# OUTPUTS
serialized_tx += encode_varint(
self.outputs) # Varint number of outputs.
if self.outputs != 0:
for i in range(self.outputs):
serialized_tx += change_endianness(int2bytes(
self.value[i], 8)) # 8-byte field Satoshi value (LE)
# ScriptPubKey
serialized_tx += encode_varint(
len(self.scriptPubKey[i].content) /
2) # Varint Output script length.
serialized_tx += self.scriptPubKey[i].content # Output script.
serialized_tx += int2bytes(self.nLockTime, 4) # 4-byte lock time field
# If return type has been set to binary, the serialized transaction is converted.
if rtype is bin:
serialized_tx = unhexlify(serialized_tx)
return serialized_tx
def deserialize(cls, hex_tx):
""" Builds a transaction object from the hexadecimal serialization format of a transaction that
could be obtained, for example, from a blockexplorer.
:param hex_tx: Hexadecimal serialized transaction.
:type hex_tx: hex str
:return: The transaction build using the provided hex serialized transaction.
:rtype: TX
"""
tx = cls()
tx.hex = hex_tx
tx.version = int(change_endianness(parse_element(tx, 4)), 16)
# INPUTS
tx.inputs = int(parse_varint(tx), 16)
for i in range(tx.inputs):
tx.prev_tx_id.append(change_endianness(parse_element(tx, 32)))
tx.prev_out_index.append(
int(change_endianness(parse_element(tx, 4)), 16))
# ScriptSig
tx.scriptSig_len.append(int(parse_varint(tx), 16))
tx.scriptSig.append(
InputScript.from_hex(parse_element(tx, tx.scriptSig_len[i])))
tx.nSequence.append(int(parse_element(tx, 4), 16))
# OUTPUTS
tx.outputs = int(parse_varint(tx), 16)
for i in range(tx.outputs):
tx.value.append(int(change_endianness(parse_element(tx, 8)), 16))
# ScriptPubKey
tx.scriptPubKey_len.append(int(parse_varint(tx), 16))
tx.scriptPubKey.append(
OutputScript.from_hex(parse_element(tx,
tx.scriptPubKey_len[i])))
tx.nLockTime = int(parse_element(tx, 4), 16)
if tx.offset != len(tx.hex):
raise Exception(
"There is some error in the serialized transaction passed as input. Transaction can't"
" be built")
else:
tx.offset = 0
return tx
def get_chainstate_lastblock(fin_name=CFG.chainstate_path):
"""
Gets the block hash of the last block a given chainstate folder is updated to.
:param fin_name: chainstate folder name
:type fin_name: str
:return: The block hash (Big Endian)
:rtype: str
"""
# Open the chainstate
db = plyvel.DB(fin_name, compression=None)
# Load obfuscation key (if it exists)
o_key = db.get((unhexlify("0e00") + "obfuscate_key"))
# Get the key itself (the leading byte indicates only its size)
if o_key is not None:
o_key = hexlify(o_key)[2:]
# Get the obfuscated block hash
o_height = db.get(b'B')
# Deobfuscate the height
height = deobfuscate_value(o_key, hexlify(o_height))
return change_endianness(height)
def transaction_dump(fin_name, fout_name):
# Transaction dump
# Input file
fin = open(CFG.data_path + fin_name, 'r')
# Output file
fout = open(CFG.data_path + fout_name, 'w')
for line in fin:
data = loads(line[:-1])
utxo = decode_utxo(data["value"])
imprt = sum([out["amount"] for out in utxo.get("outs")])
result = {
"tx_id": change_endianness(data["key"][2:]),
"num_utxos": len(utxo.get("outs")),
"total_value": imprt,
"total_len": (len(data["key"]) + len(data["value"])) / 2,
"height": utxo["height"],
"coinbase": utxo["coinbase"],
"version": utxo["version"]
}
fout.write(dumps(result) + '\n')
fout.close()
def display(self):
""" Displays all the information related to the transaction object, properly split and arranged.
Data between parenthesis corresponds to the data encoded following the serialized transaction format.
(replicates the same encoding being done in serialize method)
:param self: self
:type self: TX
:return: None.
:rtype: None
"""
print "version: " + str(self.version) + " (" + change_endianness(
int2bytes(self.version, 4)) + ")"
print "number of inputs: " + str(self.inputs) + " (" + encode_varint(
self.inputs) + ")"
for i in range(self.inputs):
print "input " + str(i)
print "\t previous txid (little endian): " + self.prev_tx_id[i] + \
" (" + change_endianness(self.prev_tx_id[i]) + ")"
print "\t previous tx output (little endian): " + str(self.prev_out_index[i]) + \
" (" + change_endianness(int2bytes(self.prev_out_index[i], 4)) + ")"
print "\t input script (scriptSig) length: " + str(self.scriptSig_len[i]) \
+ " (" + encode_varint((self.scriptSig_len[i])) + ")"
print "\t input script (scriptSig): " + self.scriptSig[i].content
print "\t decoded scriptSig: " + Script.deserialize(
self.scriptSig[i].content)
if self.scriptSig[i].type is "P2SH":
print "\t \t decoded redeemScript: " + InputScript.deserialize(
self.scriptSig[i].get_element(-1)[1:-1])
print "\t nSequence: " + str(self.nSequence[i]) + " (" + int2bytes(
self.nSequence[i], 4) + ")"
print "number of outputs: " + str(self.outputs) + " (" + encode_varint(
self.outputs) + ")"
for i in range(self.outputs):
print "output " + str(i)
print "\t Satoshis to be spent (little endian): " + str(self.value[i]) + \
" (" + change_endianness(int2bytes(self.value[i], 8)) + ")"
print "\t output script (scriptPubKey) length: " + str(self.scriptPubKey_len[i]) \
+ " (" + encode_varint(self.scriptPubKey_len[i]) + ")"
print "\t output script (scriptPubKey): " + self.scriptPubKey[
i].content
print "\t decoded scriptPubKey: " + Script.deserialize(
self.scriptPubKey[i].content)
print "nLockTime: " + str(self.nLockTime) + " (" + int2bytes(
self.nLockTime, 4) + ")"
def utxo_dump(fin_name, fout_name, count_p2sh=False, non_std_only=False):
# UTXO dump
# Input file
fin = open(CFG.data_path + fin_name, 'r')
# Output file
fout = open(CFG.data_path + fout_name, 'w')
# Standard UTXO types
std_types = [0, 1, 2, 3, 4, 5]
for line in fin:
data = loads(line[:-1])
utxo = decode_utxo(data["value"])
for out in utxo.get("outs"):
# Checks whether we are looking for every type of UTXO or just for non-standard ones.
if not non_std_only or (non_std_only
and out["out_type"] not in std_types
and not check_multisig(out['data'])):
# Calculates the dust threshold for every UTXO value and every fee per byte ratio between min and max.
min_size = get_min_input_size(out, utxo["height"], count_p2sh)
# Initialize dust, lm and the fee_per_byte ratio.
dust = 0
lm = 0
fee_per_byte = MIN_FEE_PER_BYTE
# Check whether the utxo is dust/lm for the fee_per_byte range.
while MAX_FEE_PER_BYTE > fee_per_byte and lm == 0:
# Set the dust and loss_making thresholds.
if dust is 0 and min_size * fee_per_byte > out[
"amount"] / 3:
dust = fee_per_byte
if lm is 0 and out["amount"] < min_size * fee_per_byte:
lm = fee_per_byte
# Increase the ratio
fee_per_byte += FEE_STEP
# Builds the output dictionary
result = {
"tx_id": change_endianness(data["key"][2:]),
"tx_height": utxo["height"],
"utxo_data_len": len(out["data"]) / 2,
"dust": dust,
"loss_making": lm
}
# Updates the dictionary with the remaining data from out, and stores it in disk.
result.update(out)
fout.write(dumps(result) + '\n')
fout.close()
def ecdsa_tx_sign(unsigned_tx, sk, hashflag=SIGHASH_ALL, deterministic=True):
""" Performs and ECDSA sign over a given transaction using a given secret key.
:param unsigned_tx: unsigned transaction that will be double-sha256 and signed.
:type unsigned_tx: hex str
:param sk: ECDSA private key that will sign the transaction.
:type sk: SigningKey
:param hashflag: hash type that will be used during the signature process and will identify the signature format.
:type hashflag: int
:param deterministic: Whether the signature is performed using a deterministic k or not. Set by default.
:type deterministic: bool
:return:
"""
# Encode the hash type as a 4-byte hex value.
if hashflag in [SIGHASH_ALL, SIGHASH_SINGLE, SIGHASH_NONE]:
hc = int2bytes(hashflag, 4)
else:
raise Exception("Wrong hash flag.")
# ToDo: Deal with SIGHASH_ANYONECANPAY
# sha-256 the unsigned transaction together with the hash type (little endian).
h = sha256(unhexlify(unsigned_tx + change_endianness(hc))).digest()
# Sign the transaction (using a sha256 digest, that will conclude with the double-sha256)
# If deterministic is set, the signature will be performed deterministically choosing a k from the given transaction
if deterministic:
s = sk.sign_deterministic(h,
hashfunc=sha256,
sigencode=sigencode_der_canonize)
# Otherwise, k will be chosen at random. Notice that this can lead to a private key disclosure if two different
# messages are signed using the same k.
else:
s = sk.sign(h, hashfunc=sha256, sigencode=sigencode_der_canonize)
# Finally, add the hashtype to the end of the signature as a 2-byte big endian hex value.
return hexlify(s) + hc[-2:]
def parse_ldb(fout_name, fin_name='chainstate', version=0.15):
largestBlock = 0
smallestBlock = 100000000 # should not be bigger than this number
"""
Parsed data from the chainstate LevelDB and stores it in a output file.
:param fout_name: Name of the file to output the data.
:type fout_name: str
:param fin_name: Name of the LevelDB folder (chainstate by default)
:type fin_name: str
:param version: Bitcoin Core client version. Determines the prefix of the transaction entries.
:param version: float
:return: None
:rtype: None
"""
if 0.08 <= version < 0.15:
prefix = b'c'
elif version < 0.08:
raise Exception(
"The utxo decoder only works for version 0.08 onwards.")
else:
prefix = b'C'
# Output file
fout = open(CFG.data_path + fout_name, 'wb')
# Open the LevelDB
db = plyvel.DB(CFG.btc_core_path + "/" + fin_name,
compression=None) # Change with path to chainstate
# Load obfuscation key (if it exists)
o_key = db.get((unhexlify("0e00") + "obfuscate_key"))
# If the key exists, the leading byte indicates the length of the key (8 byte by default). If there is no key,
# 8-byte zeros are used (since the key will be XORed with the given values).
if o_key is not None:
o_key = hexlify(o_key)[2:]
# For every UTXO (identified with a leading 'c'), the key (tx_id) and the value (encoded utxo) is displayed.
# UTXOs are obfuscated using the obfuscation key (o_key), in order to get them non-obfuscated, a XOR between the
# value and the key (concatenated until the length of the value is reached) if performed).
std_types = [0, 1, 2, 3, 4, 5]
non_std_only = False
count_p2sh = False
counter = 0
for key, o_value in db.iterator(prefix=prefix):
if o_key is not None:
value = deobfuscate_value(o_key, hexlify(o_value))
else:
value = hexlify(o_value)
hexed_key = hexlify(key)
# fout.write(ujson.dumps({"key": hexlify(key), "value": value}) + "\n")
# utxo = decode_utxo(value, None, version)
utxo = decode_utxo(value, hexlify(key), version)
tx_id = change_endianness(utxo.get('tx_id'))
for out in utxo.get("outs"):
# Checks whether we are looking for every type of UTXO or just for non-standard ones.
if not non_std_only or (non_std_only
and out["out_type"] not in std_types
and not check_multisig(out['data'])):
# Calculates the dust threshold for every UTXO value and every fee per byte ratio between min and max.
min_size = get_min_input_size(out, utxo["height"], count_p2sh)
dust = 0
lm = 0
if min_size > 0:
raw_dust = out["amount"] / float(3 * min_size)
raw_lm = out["amount"] / float(min_size)
dust = roundup_rate(raw_dust, FEE_STEP)
lm = roundup_rate(raw_lm, FEE_STEP)
# Adds multisig type info
if out["out_type"] == 0:
non_std_type = "std"
else:
continue
# non_std_type = check_multisig_type(out["data"]) #don't check multisig for now
# Builds the output dictionary
result = {
"tx_id": tx_id, # transaction id
"tx_height": utxo["height"], # block num
"data": out["data"], # public key hash
"amount": out["amount"] # amount
# ,
# "utxo_data_len": len(out["data"]) / 2,
# "dust": dust,
# "loss_making": lm,
# "non_std_type": non_std_type
}
# Index added at the end when updated the result with the out, since the index is not part of the
# encoded data anymore (coin) but of the entry identifier (outpoint), we add it manually.
if version >= 0.15:
result['index'] = utxo['index']
# result['register_len'] = len(value) / 2 + len(hexed_key) / 2
# Updates the dictionary with the remaining data from out, and stores it in disk.
# result.update(out)
# print result
# print unhexlify(result['tx_id'])
# print result["amount"]
## Raw data
fout.write(unhexlify(result['tx_id']))
fout.write(struct.pack('<Q', result['amount']))
fout.write(struct.pack('<I', result['tx_height']))
fout.write(unhexlify(result['data']))
fout.write(struct.pack('<I', result['index']))
if smallestBlock > result['tx_height']:
smallestBlock = result['tx_height']
if largestBlock < result['tx_height']:
largestBlock = result['tx_height']
## Json
# fout.write(ujson.dumps(result) + '\n')
counter += 1
if counter % 1000000 == 0:
print "[>] Currently at ", counter
db.close()
print "[+] Finished parsing raw utxo"
print "[>] Largest block :", largestBlock
print "[>] Smallest block :", smallestBlock
def decode_utxo_v08_v014(utxo):
""" Disclaimer: The internal structure of the chainstate LevelDB has been changed with Bitcoin Core v 0.15 release.
Therefore, this function works for chainstate created with Bitcoin Core v 0.08-v0.14, for v 0.15 onwards use
decode_utxo.
Decodes a LevelDB serialized UTXO for Bitcoin core v 0.08 - v 0.14. The serialized format is defined in the Bitcoin
Core source as follows:
Serialized format:
- VARINT(nVersion)
- VARINT(nCode)
- unspentness bitvector, for vout[2] and further; least significant byte first
- the non-spent CTxOuts (via CTxOutCompressor)
- VARINT(nHeight)
The nCode value consists of:
- bit 1: IsCoinBase()
- bit 2: vout[0] is not spent
- bit 4: vout[1] is not spent
- The higher bits encode N, the number of non-zero bytes in the following bitvector.
- In case both bit 2 and bit 4 are unset, they encode N-1, as there must be at
least one non-spent output).
VARINT refers to the CVarint used along the Bitcoin Core client, that is base128 encoding. A CTxOut contains the
compressed amount of satoshi that the UTXO holds. That amount is encoded using the equivalent to txout_compress +
b128_encode.
:param utxo: UTXO to be decoded (extracted from the chainstate)
:type utxo: hex str
:return; The decoded UTXO.
:rtype: dict
"""
# Version is extracted from the first varint of the serialized utxo
version, offset = parse_b128(utxo)
version = b128_decode(version)
# The next MSB base 128 varint is parsed to extract both is the utxo is coin base (first bit) and which of the
# outputs are not spent.
code, offset = parse_b128(utxo, offset)
code = b128_decode(code)
coinbase = code & 0x01
# Check if the first two outputs are spent
vout = [(code | 0x01) & 0x02, (code | 0x01) & 0x04]
# The higher bits of the current byte (from the fourth onwards) encode n, the number of non-zero bytes of
# the following bitvector. If both vout[0] and vout[1] are spent (v[0] = v[1] = 0) then the higher bits encodes n-1,
# since there should be at least one non-spent output.
if not vout[0] and not vout[1]:
n = (code >> 3) + 1
vout = []
else:
n = code >> 3
vout = [i for i in xrange(len(vout)) if vout[i] is not 0]
# If n is set, the encoded value contains a bitvector. The following bytes are parsed until n non-zero bytes have
# been extracted. (If a 00 is found, the parsing continues but n is not decreased)
if n > 0:
bitvector = ""
while n:
data = utxo[offset:offset + 2]
if data != "00":
n -= 1
bitvector += data
offset += 2
# Once the value is parsed, the endianness of the value is switched from LE to BE and the binary representation
# of the value is checked to identify the non-spent output indexes.
bin_data = format(int(change_endianness(bitvector), 16),
'0' + str(n * 8) + 'b')[::-1]
# Every position (i) with a 1 encodes the index of a non-spent output as i+2, since the two first outs (v[0] and
# v[1] has been already counted)
# (e.g: 0440 (LE) = 4004 (BE) = 0100 0000 0000 0100. It encodes outs 4 (i+2 = 2+2) and 16 (i+2 = 14+2).
extended_vout = [
i + 2 for i in xrange(len(bin_data)) if bin_data.find('1', i) == i
] # Finds the index of '1's and adds 2.
# Finally, the first two vouts are included to the list (if they are non-spent).
vout += extended_vout
# Once the number of outs and their index is known, they could be parsed.
outs = []
for i in vout:
# The satoshi amount is parsed, decoded and decompressed.
data, offset = parse_b128(utxo, offset)
amount = txout_decompress(b128_decode(data))
# The output type is also parsed.
out_type, offset = parse_b128(utxo, offset)
out_type = b128_decode(out_type)
# Depending on the type, the length of the following data will differ. Types 0 and 1 refers to P2PKH and P2SH
# encoded outputs. They are always followed 20 bytes of data, corresponding to the hash160 of the address (in
# P2PKH outputs) or to the scriptHash (in P2PKH). Notice that the leading and tailing opcodes are not included.
# If 2-5 is found, the following bytes encode a public key. The first byte in this case should be also included,
# since it determines the format of the key.
if out_type in [0, 1]:
data_size = 40 # 20 bytes
elif out_type in [2, 3, 4, 5]:
data_size = 66 # 33 bytes (1 byte for the type + 32 bytes of data)
offset -= 2
# Finally, if another value is found, it represents the length of the following data, which is uncompressed.
else:
data_size = (
out_type - NSPECIALSCRIPTS
) * 2 # If the data is not compacted, the out_type corresponds
# to the data size adding the number os special scripts (nSpecialScripts).
# And finally the address (the hash160 of the public key actually)
data, offset = utxo[offset:offset + data_size], offset + data_size
outs.append({
'index': i,
'amount': amount,
'out_type': out_type,
'data': data
})
# Once all the outs are processed, the block height is parsed
height, offset = parse_b128(utxo, offset)
height = b128_decode(height)
# And the length of the serialized utxo is compared with the offset to ensure that no data remains unchecked.
assert len(utxo) == offset
return {
'version': version,
'coinbase': coinbase,
'outs': outs,
'height': height
}
def display(self):
""" Displays all the information related to the transaction object, properly split and arranged.
Data between parenthesis corresponds to the data encoded following the serialized transaction format.
(replicates the same encoding being done in serialize method)
:param self: self
:type self: TX
:return: None.
:rtype: None
"""
print("version: " + str(self.version) + " (" +
change_endianness(int2bytes(self.version, 4)) + ")")
if self.version == 2:
print(
'BIP68 Relative lock-time using consensus-enforced sequence numbers'
)
if self.isWitness:
print('Witness TX')
else:
print('Non-Witness TX')
print("number of inputs: " + str(self.inputs) + " (" +
encode_varint(self.inputs) + ")")
for i in range(self.inputs):
print("input " + str(i))
print("\t previous txid (little endian): " + self.prev_tx_id[i] +
" (" + change_endianness(self.prev_tx_id[i]) + ")")
print("\t previous tx output (little endian): " +
str(self.prev_out_index[i]) + " (" +
change_endianness(int2bytes(self.prev_out_index[i], 4)) +
")")
print("\t input script (scriptSig) length: " +
str(self.scriptSig_len[i]) + " (" +
encode_varint((self.scriptSig_len[i])) + ")")
print("\t input script (scriptSig): " + self.scriptSig[i].content)
print("\t decoded scriptSig: " +
Script.deserialize(self.scriptSig[i].content))
if self.scriptSig[i].type == "P2SH":
print("\t \t decoded redeemScript: " + InputScript.deserialize(
self.scriptSig[i].get_element(-1)[1:-1]))
print("\t nSequence: " + str(self.nSequence[i]) + " (" +
int2bytes(self.nSequence[i], 4) + ")")
if self.isWitness: # todo! chack if there a TX with more than 1 witnes vin or with witness 0
print("\t txinwitness:")
for _ in self.scriptWitness[i]:
print('\t\t', _)
print("number of outputs: " + str(self.outputs) + " (" +
encode_varint(self.outputs) + ")")
for i in range(self.outputs):
print("output " + str(i))
print("\t Satoshis to be spent (little endian): " +
str(self.value[i]) + " (" +
change_endianness(int2bytes(self.value[i], 8)) + ")")
print("\t output script (scriptPubKey) length: " +
str(self.scriptPubKey_len[i]) + " (" +
encode_varint(self.scriptPubKey_len[i]) + ")")
print("\t output script (scriptPubKey): " +
self.scriptPubKey[i].content)
print("\t decoded scriptPubKey: " +
Script.deserialize(self.scriptPubKey[i].content))
print("nLockTime: " + str(self.nLockTime) + " (" +
int2bytes(self.nLockTime, 4) + ")")
def deserialize(cls, hex_tx):
""" Builds a transaction object from the hexadecimal serialization format of a transaction that
could be obtained, for example, from a blockexplorer.
:param hex_tx: Hexadecimal serialized transaction.
:type hex_tx: hex str
:return: The transaction build using the provided hex serialized transaction.
:rtype: TX
"""
tx = cls()
tx.hex = hex_tx
# first 4 bytes - TX version
tx.version = int(change_endianness(parse_element(tx, 4)), 16)
# INPUTS
tx.inputs = decode_varint(parse_varint(tx))
if tx.inputs > 0: # regular TX
tx.isWitness = False
else: # witness TX
tx.isWitness = True
flag = parse_element(tx, 1) # get flag and shift 2 bytes
# get witness TX inputs count as varint
tx.inputs = decode_varint(parse_varint(tx))
for i in range(tx.inputs):
# outpoint txid
tx.prev_tx_id.append(change_endianness(parse_element(tx, 32)))
#outpoint vout #
tx.prev_out_index.append(
int(change_endianness(parse_element(tx, 4)), 16))
# ScriptSig
tx.scriptSig_len.append(decode_varint(parse_varint(tx)))
# asm/hex
tx.scriptSig.append(
InputScript.from_hex(parse_element(tx, tx.scriptSig_len[i])))
# sequence
tx.nSequence.append(int(parse_element(tx, 4), 16))
# OUTPUTS
tx.outputs = decode_varint(parse_varint(tx))
for i in range(tx.outputs):
tx.value.append(int(change_endianness(parse_element(tx, 8)), 16))
# ScriptPubKey
tx.scriptPubKey_len.append(decode_varint(parse_varint(tx)))
tx.scriptPubKey.append(
OutputScript.from_hex(parse_element(tx,
tx.scriptPubKey_len[i])))
# WITNESS DATA
if tx.isWitness:
for tx_in in range(tx.inputs):
tx.witness_count.append(decode_varint(parse_varint(tx)))
for _ in range(tx.witness_count[tx_in]):
tx.scriptWitness_len[tx_in].append(
decode_varint(parse_varint(tx)))
tx.scriptWitness[tx_in].append(
parse_element(tx, tx.scriptWitness_len[tx_in][_]))
tx.scriptWitness_len.append([]) # todo remove unnecessary list
tx.scriptWitness.append([]) # todo remove unnecessary list
del tx.scriptWitness_len[-1]
del tx.scriptWitness[-1]
# nLockTime
tx.nLockTime = int(change_endianness(parse_element(tx, 4)), 16)
if tx.offset != len(tx.hex): # and not tx.isWitness:
raise Exception(
"There is some error in the serialized transaction passed as input. Transaction can't"
" be built")
else:
tx.offset = 0
return tx
def transaction_dump(fin_name, fout_name, version=0.15):
# ToDo: Profile this function
# Transaction dump
if version < 0.15:
# Input file
fin = open(CFG.data_path + fin_name, 'r')
# Output file
fout = open(CFG.data_path + fout_name, 'w')
for line in fin:
data = ujson.loads(line[:-1])
utxo = decode_utxo(data["value"], None, version)
imprt = sum([out["amount"] for out in utxo.get("outs")])
result = {
"tx_id": change_endianness(data["key"][2:]),
"num_utxos": len(utxo.get("outs")),
"total_value": imprt,
"total_len": (len(data["key"]) + len(data["value"])) / 2,
"height": utxo["height"],
"coinbase": utxo["coinbase"],
"version": utxo["version"]
}
fout.write(ujson.dumps(result) + '\n')
fout.close()
fin.close()
else:
# Input file
fin = open(CFG.data_path + fin_name, 'r')
# Temp file (unsorted & non-aggregated tx data)
fout = open(CFG.data_path + "temp.json", 'w')
# [1] Create temp file
for line in fin:
data = ujson.loads(line[:-1])
utxo = decode_utxo(data["value"], data["key"], version)
result = OrderedDict([
("tx_id", change_endianness(utxo.get('tx_id'))),
("num_utxos", 1),
("total_value", utxo.get('outs')[0].get('amount')),
("total_len", (len(data["key"]) + len(data["value"])) / 2),
("height", utxo["height"]), ("coinbase", utxo["coinbase"]),
("version", None)
])
fout.write(ujson.dumps(result) + '\n')
fout.close()
fin.close()
# [2] Sort file
call([
"sort", CFG.data_path + "temp.json", "-o",
CFG.data_path + "temp.json"
])
# [3] Aggregate tx data
fin = open(CFG.data_path + "temp.json", 'r')
fout = open(CFG.data_path + fout_name, 'w')
line_1 = fin.readline()
line_2 = fin.readline()
line_1 = ujson.loads(line_1) if line_1 else None
line_2 = ujson.loads(line_2) if line_2 else None
while line_1:
total_len = line_1["total_len"]
total_value = line_1["total_value"]
num_utxos = line_1["num_utxos"]
while line_2 and (line_1["tx_id"] == line_2["tx_id"]):
total_len += line_2["total_len"]
total_value += line_2["total_value"]
num_utxos += line_2["num_utxos"]
line_2 = fin.readline()
line_2 = ujson.loads(line_2) if line_2 else None
result = OrderedDict([("tx_id", line_1["tx_id"]),
("num_utxos", num_utxos),
("total_value", total_value),
("total_len", total_len),
("height", line_1["height"]),
("coinbase", line_1["coinbase"]),
("version", line_1["version"])])
fout.write(ujson.dumps(result) + '\n')
line_1 = line_2
line_2 = fin.readline()
line_2 = ujson.loads(line_2) if line_2 else None
fin.close()
fout.close()
remove(CFG.data_path + "temp.json")
def utxo_dump(fin_name,
fout_name,
version=0.15,
count_p2sh=False,
non_std_only=False):
# UTXO dump
# Input file
fin = open(CFG.data_path + fin_name, 'r')
# Output file
fout = open(CFG.data_path + fout_name, 'w')
# Standard UTXO types
std_types = [0, 1, 2, 3, 4, 5]
for line in fin:
data = ujson.loads(line[:-1])
if version < 0.15:
utxo = decode_utxo(data["value"], None, version)
tx_id = change_endianness(data["key"][2:])
else:
utxo = decode_utxo(data["value"], data['key'], version)
tx_id = change_endianness(utxo.get('tx_id'))
for out in utxo.get("outs"):
# Checks whether we are looking for every type of UTXO or just for non-standard ones.
if not non_std_only or (non_std_only
and out["out_type"] not in std_types
and not check_multisig(out['data'])):
# Calculates the dust threshold for every UTXO value and every fee per byte ratio between min and max.
min_size = get_min_input_size(out, utxo["height"], count_p2sh)
dust = 0
lm = 0
if min_size > 0:
raw_dust = out["amount"] / float(3 * min_size)
raw_lm = out["amount"] / float(min_size)
dust = roundup_rate(raw_dust, FEE_STEP)
lm = roundup_rate(raw_lm, FEE_STEP)
# Adds multisig type info
if out["out_type"] in [0, 1, 2, 3, 4, 5]:
non_std_type = "std"
else:
non_std_type = check_multisig_type(out["data"])
# Builds the output dictionary
result = {
"tx_id": tx_id,
"tx_height": utxo["height"],
"utxo_data_len": len(out["data"]) / 2,
"dust": dust,
"loss_making": lm,
"non_std_type": non_std_type
}
# Index added at the end when updated the result with the out, since the index is not part of the
# encoded data anymore (coin) but of the entry identifier (outpoint), we add it manually.
if version >= 0.15:
result['index'] = utxo['index']
result['register_len'] = len(data["value"]) / 2 + len(
data["key"]) / 2
# Updates the dictionary with the remaining data from out, and stores it in disk.
result.update(out)
fout.write(ujson.dumps(result) + '\n')
fout.close()