def get_txid(self, rtype=hex, endianness="LE"): """ Computes the transaction id (i.e: transaction hash for non-segwit txs). :param rtype: Defines the type of return, either hex str or bytes. :type rtype: str or bin :param endianness: Whether the id is returned in BE (Big endian) or LE (Little Endian) (default one) :type endianness: str :return: The hash of the transaction (i.e: transaction id) :rtype: hex str or bin, depending on rtype parameter. """ if rtype not in [hex, bin]: raise Exception( "Invalid return type (rtype). It should be either hex or bin.") if endianness not in ["BE", "LE"]: raise Exception( "Invalid endianness type. It should be either BE or LE.") if rtype is hex: tx_id = hexlify( sha256(sha256(self.serialize(rtype=bin)).digest()).digest()) if endianness == "BE": tx_id = change_endianness(tx_id) else: tx_id = sha256(sha256(self.serialize(rtype=bin)).digest()).digest() if endianness == "BE": tx_id = unhexlify(change_endianness(hexlify(tx_id))) return tx_id
def serialize(self, rtype=hex): """ Serialize all the transaction fields arranged in the proper order, resulting in a hexadecimal string ready to be broadcast to the network. :param self: self :type self: TX :param rtype: Whether the serialized transaction is returned as a hex str or a byte array. :type rtype: hex or bool :return: Serialized transaction representation (hexadecimal or bin depending on rtype parameter). :rtype: hex str / bin """ if rtype not in [hex, bin]: raise Exception( "Invalid return type (rtype). It should be either hex or bin.") serialized_tx = change_endianness(int2bytes( self.version, 4)) # 4-byte version number (LE). # INPUTS serialized_tx += encode_varint(self.inputs) # Varint number of inputs. for i in range(self.inputs): serialized_tx += change_endianness( self.prev_tx_id[i] ) # 32-byte hash of the previous transaction (LE). serialized_tx += change_endianness( int2bytes(self.prev_out_index[i], 4)) # 4-byte output index (LE) serialized_tx += encode_varint(len(self.scriptSig[i].content) / 2) # Varint input script length. # ScriptSig serialized_tx += self.scriptSig[i].content # Input script. serialized_tx += int2bytes(self.nSequence[i], 4) # 4-byte sequence number. # OUTPUTS serialized_tx += encode_varint( self.outputs) # Varint number of outputs. if self.outputs != 0: for i in range(self.outputs): serialized_tx += change_endianness(int2bytes( self.value[i], 8)) # 8-byte field Satoshi value (LE) # ScriptPubKey serialized_tx += encode_varint( len(self.scriptPubKey[i].content) / 2) # Varint Output script length. serialized_tx += self.scriptPubKey[i].content # Output script. serialized_tx += int2bytes(self.nLockTime, 4) # 4-byte lock time field # If return type has been set to binary, the serialized transaction is converted. if rtype is bin: serialized_tx = unhexlify(serialized_tx) return serialized_tx
def deserialize(cls, hex_tx): """ Builds a transaction object from the hexadecimal serialization format of a transaction that could be obtained, for example, from a blockexplorer. :param hex_tx: Hexadecimal serialized transaction. :type hex_tx: hex str :return: The transaction build using the provided hex serialized transaction. :rtype: TX """ tx = cls() tx.hex = hex_tx tx.version = int(change_endianness(parse_element(tx, 4)), 16) # INPUTS tx.inputs = int(parse_varint(tx), 16) for i in range(tx.inputs): tx.prev_tx_id.append(change_endianness(parse_element(tx, 32))) tx.prev_out_index.append( int(change_endianness(parse_element(tx, 4)), 16)) # ScriptSig tx.scriptSig_len.append(int(parse_varint(tx), 16)) tx.scriptSig.append( InputScript.from_hex(parse_element(tx, tx.scriptSig_len[i]))) tx.nSequence.append(int(parse_element(tx, 4), 16)) # OUTPUTS tx.outputs = int(parse_varint(tx), 16) for i in range(tx.outputs): tx.value.append(int(change_endianness(parse_element(tx, 8)), 16)) # ScriptPubKey tx.scriptPubKey_len.append(int(parse_varint(tx), 16)) tx.scriptPubKey.append( OutputScript.from_hex(parse_element(tx, tx.scriptPubKey_len[i]))) tx.nLockTime = int(parse_element(tx, 4), 16) if tx.offset != len(tx.hex): raise Exception( "There is some error in the serialized transaction passed as input. Transaction can't" " be built") else: tx.offset = 0 return tx
def get_chainstate_lastblock(fin_name=CFG.chainstate_path): """ Gets the block hash of the last block a given chainstate folder is updated to. :param fin_name: chainstate folder name :type fin_name: str :return: The block hash (Big Endian) :rtype: str """ # Open the chainstate db = plyvel.DB(fin_name, compression=None) # Load obfuscation key (if it exists) o_key = db.get((unhexlify("0e00") + "obfuscate_key")) # Get the key itself (the leading byte indicates only its size) if o_key is not None: o_key = hexlify(o_key)[2:] # Get the obfuscated block hash o_height = db.get(b'B') # Deobfuscate the height height = deobfuscate_value(o_key, hexlify(o_height)) return change_endianness(height)
def transaction_dump(fin_name, fout_name): # Transaction dump # Input file fin = open(CFG.data_path + fin_name, 'r') # Output file fout = open(CFG.data_path + fout_name, 'w') for line in fin: data = loads(line[:-1]) utxo = decode_utxo(data["value"]) imprt = sum([out["amount"] for out in utxo.get("outs")]) result = { "tx_id": change_endianness(data["key"][2:]), "num_utxos": len(utxo.get("outs")), "total_value": imprt, "total_len": (len(data["key"]) + len(data["value"])) / 2, "height": utxo["height"], "coinbase": utxo["coinbase"], "version": utxo["version"] } fout.write(dumps(result) + '\n') fout.close()
def display(self): """ Displays all the information related to the transaction object, properly split and arranged. Data between parenthesis corresponds to the data encoded following the serialized transaction format. (replicates the same encoding being done in serialize method) :param self: self :type self: TX :return: None. :rtype: None """ print "version: " + str(self.version) + " (" + change_endianness( int2bytes(self.version, 4)) + ")" print "number of inputs: " + str(self.inputs) + " (" + encode_varint( self.inputs) + ")" for i in range(self.inputs): print "input " + str(i) print "\t previous txid (little endian): " + self.prev_tx_id[i] + \ " (" + change_endianness(self.prev_tx_id[i]) + ")" print "\t previous tx output (little endian): " + str(self.prev_out_index[i]) + \ " (" + change_endianness(int2bytes(self.prev_out_index[i], 4)) + ")" print "\t input script (scriptSig) length: " + str(self.scriptSig_len[i]) \ + " (" + encode_varint((self.scriptSig_len[i])) + ")" print "\t input script (scriptSig): " + self.scriptSig[i].content print "\t decoded scriptSig: " + Script.deserialize( self.scriptSig[i].content) if self.scriptSig[i].type is "P2SH": print "\t \t decoded redeemScript: " + InputScript.deserialize( self.scriptSig[i].get_element(-1)[1:-1]) print "\t nSequence: " + str(self.nSequence[i]) + " (" + int2bytes( self.nSequence[i], 4) + ")" print "number of outputs: " + str(self.outputs) + " (" + encode_varint( self.outputs) + ")" for i in range(self.outputs): print "output " + str(i) print "\t Satoshis to be spent (little endian): " + str(self.value[i]) + \ " (" + change_endianness(int2bytes(self.value[i], 8)) + ")" print "\t output script (scriptPubKey) length: " + str(self.scriptPubKey_len[i]) \ + " (" + encode_varint(self.scriptPubKey_len[i]) + ")" print "\t output script (scriptPubKey): " + self.scriptPubKey[ i].content print "\t decoded scriptPubKey: " + Script.deserialize( self.scriptPubKey[i].content) print "nLockTime: " + str(self.nLockTime) + " (" + int2bytes( self.nLockTime, 4) + ")"
def utxo_dump(fin_name, fout_name, count_p2sh=False, non_std_only=False): # UTXO dump # Input file fin = open(CFG.data_path + fin_name, 'r') # Output file fout = open(CFG.data_path + fout_name, 'w') # Standard UTXO types std_types = [0, 1, 2, 3, 4, 5] for line in fin: data = loads(line[:-1]) utxo = decode_utxo(data["value"]) for out in utxo.get("outs"): # Checks whether we are looking for every type of UTXO or just for non-standard ones. if not non_std_only or (non_std_only and out["out_type"] not in std_types and not check_multisig(out['data'])): # Calculates the dust threshold for every UTXO value and every fee per byte ratio between min and max. min_size = get_min_input_size(out, utxo["height"], count_p2sh) # Initialize dust, lm and the fee_per_byte ratio. dust = 0 lm = 0 fee_per_byte = MIN_FEE_PER_BYTE # Check whether the utxo is dust/lm for the fee_per_byte range. while MAX_FEE_PER_BYTE > fee_per_byte and lm == 0: # Set the dust and loss_making thresholds. if dust is 0 and min_size * fee_per_byte > out[ "amount"] / 3: dust = fee_per_byte if lm is 0 and out["amount"] < min_size * fee_per_byte: lm = fee_per_byte # Increase the ratio fee_per_byte += FEE_STEP # Builds the output dictionary result = { "tx_id": change_endianness(data["key"][2:]), "tx_height": utxo["height"], "utxo_data_len": len(out["data"]) / 2, "dust": dust, "loss_making": lm } # Updates the dictionary with the remaining data from out, and stores it in disk. result.update(out) fout.write(dumps(result) + '\n') fout.close()
def ecdsa_tx_sign(unsigned_tx, sk, hashflag=SIGHASH_ALL, deterministic=True): """ Performs and ECDSA sign over a given transaction using a given secret key. :param unsigned_tx: unsigned transaction that will be double-sha256 and signed. :type unsigned_tx: hex str :param sk: ECDSA private key that will sign the transaction. :type sk: SigningKey :param hashflag: hash type that will be used during the signature process and will identify the signature format. :type hashflag: int :param deterministic: Whether the signature is performed using a deterministic k or not. Set by default. :type deterministic: bool :return: """ # Encode the hash type as a 4-byte hex value. if hashflag in [SIGHASH_ALL, SIGHASH_SINGLE, SIGHASH_NONE]: hc = int2bytes(hashflag, 4) else: raise Exception("Wrong hash flag.") # ToDo: Deal with SIGHASH_ANYONECANPAY # sha-256 the unsigned transaction together with the hash type (little endian). h = sha256(unhexlify(unsigned_tx + change_endianness(hc))).digest() # Sign the transaction (using a sha256 digest, that will conclude with the double-sha256) # If deterministic is set, the signature will be performed deterministically choosing a k from the given transaction if deterministic: s = sk.sign_deterministic(h, hashfunc=sha256, sigencode=sigencode_der_canonize) # Otherwise, k will be chosen at random. Notice that this can lead to a private key disclosure if two different # messages are signed using the same k. else: s = sk.sign(h, hashfunc=sha256, sigencode=sigencode_der_canonize) # Finally, add the hashtype to the end of the signature as a 2-byte big endian hex value. return hexlify(s) + hc[-2:]
def parse_ldb(fout_name, fin_name='chainstate', version=0.15): largestBlock = 0 smallestBlock = 100000000 # should not be bigger than this number """ Parsed data from the chainstate LevelDB and stores it in a output file. :param fout_name: Name of the file to output the data. :type fout_name: str :param fin_name: Name of the LevelDB folder (chainstate by default) :type fin_name: str :param version: Bitcoin Core client version. Determines the prefix of the transaction entries. :param version: float :return: None :rtype: None """ if 0.08 <= version < 0.15: prefix = b'c' elif version < 0.08: raise Exception( "The utxo decoder only works for version 0.08 onwards.") else: prefix = b'C' # Output file fout = open(CFG.data_path + fout_name, 'wb') # Open the LevelDB db = plyvel.DB(CFG.btc_core_path + "/" + fin_name, compression=None) # Change with path to chainstate # Load obfuscation key (if it exists) o_key = db.get((unhexlify("0e00") + "obfuscate_key")) # If the key exists, the leading byte indicates the length of the key (8 byte by default). If there is no key, # 8-byte zeros are used (since the key will be XORed with the given values). if o_key is not None: o_key = hexlify(o_key)[2:] # For every UTXO (identified with a leading 'c'), the key (tx_id) and the value (encoded utxo) is displayed. # UTXOs are obfuscated using the obfuscation key (o_key), in order to get them non-obfuscated, a XOR between the # value and the key (concatenated until the length of the value is reached) if performed). std_types = [0, 1, 2, 3, 4, 5] non_std_only = False count_p2sh = False counter = 0 for key, o_value in db.iterator(prefix=prefix): if o_key is not None: value = deobfuscate_value(o_key, hexlify(o_value)) else: value = hexlify(o_value) hexed_key = hexlify(key) # fout.write(ujson.dumps({"key": hexlify(key), "value": value}) + "\n") # utxo = decode_utxo(value, None, version) utxo = decode_utxo(value, hexlify(key), version) tx_id = change_endianness(utxo.get('tx_id')) for out in utxo.get("outs"): # Checks whether we are looking for every type of UTXO or just for non-standard ones. if not non_std_only or (non_std_only and out["out_type"] not in std_types and not check_multisig(out['data'])): # Calculates the dust threshold for every UTXO value and every fee per byte ratio between min and max. min_size = get_min_input_size(out, utxo["height"], count_p2sh) dust = 0 lm = 0 if min_size > 0: raw_dust = out["amount"] / float(3 * min_size) raw_lm = out["amount"] / float(min_size) dust = roundup_rate(raw_dust, FEE_STEP) lm = roundup_rate(raw_lm, FEE_STEP) # Adds multisig type info if out["out_type"] == 0: non_std_type = "std" else: continue # non_std_type = check_multisig_type(out["data"]) #don't check multisig for now # Builds the output dictionary result = { "tx_id": tx_id, # transaction id "tx_height": utxo["height"], # block num "data": out["data"], # public key hash "amount": out["amount"] # amount # , # "utxo_data_len": len(out["data"]) / 2, # "dust": dust, # "loss_making": lm, # "non_std_type": non_std_type } # Index added at the end when updated the result with the out, since the index is not part of the # encoded data anymore (coin) but of the entry identifier (outpoint), we add it manually. if version >= 0.15: result['index'] = utxo['index'] # result['register_len'] = len(value) / 2 + len(hexed_key) / 2 # Updates the dictionary with the remaining data from out, and stores it in disk. # result.update(out) # print result # print unhexlify(result['tx_id']) # print result["amount"] ## Raw data fout.write(unhexlify(result['tx_id'])) fout.write(struct.pack('<Q', result['amount'])) fout.write(struct.pack('<I', result['tx_height'])) fout.write(unhexlify(result['data'])) fout.write(struct.pack('<I', result['index'])) if smallestBlock > result['tx_height']: smallestBlock = result['tx_height'] if largestBlock < result['tx_height']: largestBlock = result['tx_height'] ## Json # fout.write(ujson.dumps(result) + '\n') counter += 1 if counter % 1000000 == 0: print "[>] Currently at ", counter db.close() print "[+] Finished parsing raw utxo" print "[>] Largest block :", largestBlock print "[>] Smallest block :", smallestBlock
def decode_utxo_v08_v014(utxo): """ Disclaimer: The internal structure of the chainstate LevelDB has been changed with Bitcoin Core v 0.15 release. Therefore, this function works for chainstate created with Bitcoin Core v 0.08-v0.14, for v 0.15 onwards use decode_utxo. Decodes a LevelDB serialized UTXO for Bitcoin core v 0.08 - v 0.14. The serialized format is defined in the Bitcoin Core source as follows: Serialized format: - VARINT(nVersion) - VARINT(nCode) - unspentness bitvector, for vout[2] and further; least significant byte first - the non-spent CTxOuts (via CTxOutCompressor) - VARINT(nHeight) The nCode value consists of: - bit 1: IsCoinBase() - bit 2: vout[0] is not spent - bit 4: vout[1] is not spent - The higher bits encode N, the number of non-zero bytes in the following bitvector. - In case both bit 2 and bit 4 are unset, they encode N-1, as there must be at least one non-spent output). VARINT refers to the CVarint used along the Bitcoin Core client, that is base128 encoding. A CTxOut contains the compressed amount of satoshi that the UTXO holds. That amount is encoded using the equivalent to txout_compress + b128_encode. :param utxo: UTXO to be decoded (extracted from the chainstate) :type utxo: hex str :return; The decoded UTXO. :rtype: dict """ # Version is extracted from the first varint of the serialized utxo version, offset = parse_b128(utxo) version = b128_decode(version) # The next MSB base 128 varint is parsed to extract both is the utxo is coin base (first bit) and which of the # outputs are not spent. code, offset = parse_b128(utxo, offset) code = b128_decode(code) coinbase = code & 0x01 # Check if the first two outputs are spent vout = [(code | 0x01) & 0x02, (code | 0x01) & 0x04] # The higher bits of the current byte (from the fourth onwards) encode n, the number of non-zero bytes of # the following bitvector. If both vout[0] and vout[1] are spent (v[0] = v[1] = 0) then the higher bits encodes n-1, # since there should be at least one non-spent output. if not vout[0] and not vout[1]: n = (code >> 3) + 1 vout = [] else: n = code >> 3 vout = [i for i in xrange(len(vout)) if vout[i] is not 0] # If n is set, the encoded value contains a bitvector. The following bytes are parsed until n non-zero bytes have # been extracted. (If a 00 is found, the parsing continues but n is not decreased) if n > 0: bitvector = "" while n: data = utxo[offset:offset + 2] if data != "00": n -= 1 bitvector += data offset += 2 # Once the value is parsed, the endianness of the value is switched from LE to BE and the binary representation # of the value is checked to identify the non-spent output indexes. bin_data = format(int(change_endianness(bitvector), 16), '0' + str(n * 8) + 'b')[::-1] # Every position (i) with a 1 encodes the index of a non-spent output as i+2, since the two first outs (v[0] and # v[1] has been already counted) # (e.g: 0440 (LE) = 4004 (BE) = 0100 0000 0000 0100. It encodes outs 4 (i+2 = 2+2) and 16 (i+2 = 14+2). extended_vout = [ i + 2 for i in xrange(len(bin_data)) if bin_data.find('1', i) == i ] # Finds the index of '1's and adds 2. # Finally, the first two vouts are included to the list (if they are non-spent). vout += extended_vout # Once the number of outs and their index is known, they could be parsed. outs = [] for i in vout: # The satoshi amount is parsed, decoded and decompressed. data, offset = parse_b128(utxo, offset) amount = txout_decompress(b128_decode(data)) # The output type is also parsed. out_type, offset = parse_b128(utxo, offset) out_type = b128_decode(out_type) # Depending on the type, the length of the following data will differ. Types 0 and 1 refers to P2PKH and P2SH # encoded outputs. They are always followed 20 bytes of data, corresponding to the hash160 of the address (in # P2PKH outputs) or to the scriptHash (in P2PKH). Notice that the leading and tailing opcodes are not included. # If 2-5 is found, the following bytes encode a public key. The first byte in this case should be also included, # since it determines the format of the key. if out_type in [0, 1]: data_size = 40 # 20 bytes elif out_type in [2, 3, 4, 5]: data_size = 66 # 33 bytes (1 byte for the type + 32 bytes of data) offset -= 2 # Finally, if another value is found, it represents the length of the following data, which is uncompressed. else: data_size = ( out_type - NSPECIALSCRIPTS ) * 2 # If the data is not compacted, the out_type corresponds # to the data size adding the number os special scripts (nSpecialScripts). # And finally the address (the hash160 of the public key actually) data, offset = utxo[offset:offset + data_size], offset + data_size outs.append({ 'index': i, 'amount': amount, 'out_type': out_type, 'data': data }) # Once all the outs are processed, the block height is parsed height, offset = parse_b128(utxo, offset) height = b128_decode(height) # And the length of the serialized utxo is compared with the offset to ensure that no data remains unchecked. assert len(utxo) == offset return { 'version': version, 'coinbase': coinbase, 'outs': outs, 'height': height }
def display(self): """ Displays all the information related to the transaction object, properly split and arranged. Data between parenthesis corresponds to the data encoded following the serialized transaction format. (replicates the same encoding being done in serialize method) :param self: self :type self: TX :return: None. :rtype: None """ print("version: " + str(self.version) + " (" + change_endianness(int2bytes(self.version, 4)) + ")") if self.version == 2: print( 'BIP68 Relative lock-time using consensus-enforced sequence numbers' ) if self.isWitness: print('Witness TX') else: print('Non-Witness TX') print("number of inputs: " + str(self.inputs) + " (" + encode_varint(self.inputs) + ")") for i in range(self.inputs): print("input " + str(i)) print("\t previous txid (little endian): " + self.prev_tx_id[i] + " (" + change_endianness(self.prev_tx_id[i]) + ")") print("\t previous tx output (little endian): " + str(self.prev_out_index[i]) + " (" + change_endianness(int2bytes(self.prev_out_index[i], 4)) + ")") print("\t input script (scriptSig) length: " + str(self.scriptSig_len[i]) + " (" + encode_varint((self.scriptSig_len[i])) + ")") print("\t input script (scriptSig): " + self.scriptSig[i].content) print("\t decoded scriptSig: " + Script.deserialize(self.scriptSig[i].content)) if self.scriptSig[i].type == "P2SH": print("\t \t decoded redeemScript: " + InputScript.deserialize( self.scriptSig[i].get_element(-1)[1:-1])) print("\t nSequence: " + str(self.nSequence[i]) + " (" + int2bytes(self.nSequence[i], 4) + ")") if self.isWitness: # todo! chack if there a TX with more than 1 witnes vin or with witness 0 print("\t txinwitness:") for _ in self.scriptWitness[i]: print('\t\t', _) print("number of outputs: " + str(self.outputs) + " (" + encode_varint(self.outputs) + ")") for i in range(self.outputs): print("output " + str(i)) print("\t Satoshis to be spent (little endian): " + str(self.value[i]) + " (" + change_endianness(int2bytes(self.value[i], 8)) + ")") print("\t output script (scriptPubKey) length: " + str(self.scriptPubKey_len[i]) + " (" + encode_varint(self.scriptPubKey_len[i]) + ")") print("\t output script (scriptPubKey): " + self.scriptPubKey[i].content) print("\t decoded scriptPubKey: " + Script.deserialize(self.scriptPubKey[i].content)) print("nLockTime: " + str(self.nLockTime) + " (" + int2bytes(self.nLockTime, 4) + ")")
def deserialize(cls, hex_tx): """ Builds a transaction object from the hexadecimal serialization format of a transaction that could be obtained, for example, from a blockexplorer. :param hex_tx: Hexadecimal serialized transaction. :type hex_tx: hex str :return: The transaction build using the provided hex serialized transaction. :rtype: TX """ tx = cls() tx.hex = hex_tx # first 4 bytes - TX version tx.version = int(change_endianness(parse_element(tx, 4)), 16) # INPUTS tx.inputs = decode_varint(parse_varint(tx)) if tx.inputs > 0: # regular TX tx.isWitness = False else: # witness TX tx.isWitness = True flag = parse_element(tx, 1) # get flag and shift 2 bytes # get witness TX inputs count as varint tx.inputs = decode_varint(parse_varint(tx)) for i in range(tx.inputs): # outpoint txid tx.prev_tx_id.append(change_endianness(parse_element(tx, 32))) #outpoint vout # tx.prev_out_index.append( int(change_endianness(parse_element(tx, 4)), 16)) # ScriptSig tx.scriptSig_len.append(decode_varint(parse_varint(tx))) # asm/hex tx.scriptSig.append( InputScript.from_hex(parse_element(tx, tx.scriptSig_len[i]))) # sequence tx.nSequence.append(int(parse_element(tx, 4), 16)) # OUTPUTS tx.outputs = decode_varint(parse_varint(tx)) for i in range(tx.outputs): tx.value.append(int(change_endianness(parse_element(tx, 8)), 16)) # ScriptPubKey tx.scriptPubKey_len.append(decode_varint(parse_varint(tx))) tx.scriptPubKey.append( OutputScript.from_hex(parse_element(tx, tx.scriptPubKey_len[i]))) # WITNESS DATA if tx.isWitness: for tx_in in range(tx.inputs): tx.witness_count.append(decode_varint(parse_varint(tx))) for _ in range(tx.witness_count[tx_in]): tx.scriptWitness_len[tx_in].append( decode_varint(parse_varint(tx))) tx.scriptWitness[tx_in].append( parse_element(tx, tx.scriptWitness_len[tx_in][_])) tx.scriptWitness_len.append([]) # todo remove unnecessary list tx.scriptWitness.append([]) # todo remove unnecessary list del tx.scriptWitness_len[-1] del tx.scriptWitness[-1] # nLockTime tx.nLockTime = int(change_endianness(parse_element(tx, 4)), 16) if tx.offset != len(tx.hex): # and not tx.isWitness: raise Exception( "There is some error in the serialized transaction passed as input. Transaction can't" " be built") else: tx.offset = 0 return tx
def transaction_dump(fin_name, fout_name, version=0.15): # ToDo: Profile this function # Transaction dump if version < 0.15: # Input file fin = open(CFG.data_path + fin_name, 'r') # Output file fout = open(CFG.data_path + fout_name, 'w') for line in fin: data = ujson.loads(line[:-1]) utxo = decode_utxo(data["value"], None, version) imprt = sum([out["amount"] for out in utxo.get("outs")]) result = { "tx_id": change_endianness(data["key"][2:]), "num_utxos": len(utxo.get("outs")), "total_value": imprt, "total_len": (len(data["key"]) + len(data["value"])) / 2, "height": utxo["height"], "coinbase": utxo["coinbase"], "version": utxo["version"] } fout.write(ujson.dumps(result) + '\n') fout.close() fin.close() else: # Input file fin = open(CFG.data_path + fin_name, 'r') # Temp file (unsorted & non-aggregated tx data) fout = open(CFG.data_path + "temp.json", 'w') # [1] Create temp file for line in fin: data = ujson.loads(line[:-1]) utxo = decode_utxo(data["value"], data["key"], version) result = OrderedDict([ ("tx_id", change_endianness(utxo.get('tx_id'))), ("num_utxos", 1), ("total_value", utxo.get('outs')[0].get('amount')), ("total_len", (len(data["key"]) + len(data["value"])) / 2), ("height", utxo["height"]), ("coinbase", utxo["coinbase"]), ("version", None) ]) fout.write(ujson.dumps(result) + '\n') fout.close() fin.close() # [2] Sort file call([ "sort", CFG.data_path + "temp.json", "-o", CFG.data_path + "temp.json" ]) # [3] Aggregate tx data fin = open(CFG.data_path + "temp.json", 'r') fout = open(CFG.data_path + fout_name, 'w') line_1 = fin.readline() line_2 = fin.readline() line_1 = ujson.loads(line_1) if line_1 else None line_2 = ujson.loads(line_2) if line_2 else None while line_1: total_len = line_1["total_len"] total_value = line_1["total_value"] num_utxos = line_1["num_utxos"] while line_2 and (line_1["tx_id"] == line_2["tx_id"]): total_len += line_2["total_len"] total_value += line_2["total_value"] num_utxos += line_2["num_utxos"] line_2 = fin.readline() line_2 = ujson.loads(line_2) if line_2 else None result = OrderedDict([("tx_id", line_1["tx_id"]), ("num_utxos", num_utxos), ("total_value", total_value), ("total_len", total_len), ("height", line_1["height"]), ("coinbase", line_1["coinbase"]), ("version", line_1["version"])]) fout.write(ujson.dumps(result) + '\n') line_1 = line_2 line_2 = fin.readline() line_2 = ujson.loads(line_2) if line_2 else None fin.close() fout.close() remove(CFG.data_path + "temp.json")
def utxo_dump(fin_name, fout_name, version=0.15, count_p2sh=False, non_std_only=False): # UTXO dump # Input file fin = open(CFG.data_path + fin_name, 'r') # Output file fout = open(CFG.data_path + fout_name, 'w') # Standard UTXO types std_types = [0, 1, 2, 3, 4, 5] for line in fin: data = ujson.loads(line[:-1]) if version < 0.15: utxo = decode_utxo(data["value"], None, version) tx_id = change_endianness(data["key"][2:]) else: utxo = decode_utxo(data["value"], data['key'], version) tx_id = change_endianness(utxo.get('tx_id')) for out in utxo.get("outs"): # Checks whether we are looking for every type of UTXO or just for non-standard ones. if not non_std_only or (non_std_only and out["out_type"] not in std_types and not check_multisig(out['data'])): # Calculates the dust threshold for every UTXO value and every fee per byte ratio between min and max. min_size = get_min_input_size(out, utxo["height"], count_p2sh) dust = 0 lm = 0 if min_size > 0: raw_dust = out["amount"] / float(3 * min_size) raw_lm = out["amount"] / float(min_size) dust = roundup_rate(raw_dust, FEE_STEP) lm = roundup_rate(raw_lm, FEE_STEP) # Adds multisig type info if out["out_type"] in [0, 1, 2, 3, 4, 5]: non_std_type = "std" else: non_std_type = check_multisig_type(out["data"]) # Builds the output dictionary result = { "tx_id": tx_id, "tx_height": utxo["height"], "utxo_data_len": len(out["data"]) / 2, "dust": dust, "loss_making": lm, "non_std_type": non_std_type } # Index added at the end when updated the result with the out, since the index is not part of the # encoded data anymore (coin) but of the entry identifier (outpoint), we add it manually. if version >= 0.15: result['index'] = utxo['index'] result['register_len'] = len(data["value"]) / 2 + len( data["key"]) / 2 # Updates the dictionary with the remaining data from out, and stores it in disk. result.update(out) fout.write(ujson.dumps(result) + '\n') fout.close()