def serialize(self):
return b''.join((
self.prev_hash,
pack_le_uint32(self.prev_idx),
pack_varbytes(self.script),
pack_le_uint32(self.sequence),
))
def add_unflushed(
self,
*,
hashXs_by_tx: Sequence[Sequence[bytes]],
first_tx_num: int,
txhash_to_txnum_map: Dict[bytes, int],
txo_to_spender_map: Dict[Tuple[bytes, int], bytes], # (tx_hash, txout_idx) -> tx_hash
):
unflushed_hashxs = self._unflushed_hashxs
count = 0
tx_num = None
for tx_num, hashXs in enumerate(hashXs_by_tx, start=first_tx_num):
tx_numb = pack_txnum(tx_num)
hashXs = set(hashXs)
for hashX in hashXs:
unflushed_hashxs[hashX] += tx_numb
count += len(hashXs)
self._unflushed_hashxs_count += count
if tx_num is not None:
assert self.hist_db_tx_count_next + len(hashXs_by_tx) == tx_num + 1
self.hist_db_tx_count_next = tx_num + 1
self._unflushed_txhash_to_txnum_map.update(txhash_to_txnum_map)
unflushed_spenders = self._unflushed_txo_to_spender
get_txnum_for_txhash = self.get_txnum_for_txhash
for (prev_hash, prev_idx), spender_hash in txo_to_spender_map.items():
prev_txnum = get_txnum_for_txhash(prev_hash)
assert prev_txnum is not None
spender_txnum = get_txnum_for_txhash(spender_hash)
assert spender_txnum is not None
prev_idx_packed = pack_le_uint32(prev_idx)[:TXOUTIDX_LEN]
prev_txnumb = pack_txnum(prev_txnum)
unflushed_spenders[prev_txnumb+prev_idx_packed] = spender_txnum
def serialize(self):
assert len(self.merkleRootMNList) == 32
return (
pack_le_uint16(self.version) + # version
pack_le_uint32(self.height) + # height
self.merkleRootMNList # merkleRootMNList
)
def serialize(self):
return b''.join(
(pack_le_int32(self.version), pack_varint(len(self.inputs)),
b''.join(tx_in.serialize() for tx_in in self.inputs),
pack_varint(len(self.outputs)),
b''.join(tx_out.serialize() for tx_out in self.outputs),
pack_le_uint32(self.locktime)))
def serialize(self):
return b''.join((
self.keyimage,
self.ringsize,
pack_varbytes(self.script),
pack_le_uint32(self.sequence),
))
def _read_tx_parts(self, produce_hash=True):
start = self.cursor
version = self._read_le_int32()
inputs = self._read_inputs()
outputs = self._read_outputs()
locktime = self._read_le_uint32()
expiry = self._read_le_uint32()
end_prefix = self.cursor
witness = self._read_witness(len(inputs))
# Drop the coinbase-like input from a vote tx as it creates problems
# with UTXOs lookups and mempool management
if inputs[0].is_generation and len(inputs) > 1:
inputs = inputs[1:]
if produce_hash:
# TxSerializeNoWitness << 16 == 0x10000
no_witness_header = pack_le_uint32(0x10000 | (version & 0xffff))
prefix_tx = no_witness_header + self.binary[start+4:end_prefix]
tx_hash = self.blake256(prefix_tx)
else:
tx_hash = None
return TxDcr(
version,
inputs,
outputs,
locktime,
expiry,
witness
), tx_hash, self.cursor - start
def _read_tx_parts(self, produce_hash=True):
start = self.cursor
version = self._read_le_int32()
inputs = self._read_inputs()
outputs = self._read_outputs()
locktime = self._read_le_uint32()
expiry = self._read_le_uint32()
end_prefix = self.cursor
witness = self._read_witness(len(inputs))
if produce_hash:
# TxSerializeNoWitness << 16 == 0x10000
no_witness_header = pack_le_uint32(0x10000 | (version & 0xffff))
prefix_tx = no_witness_header + self.binary[start+4:end_prefix]
tx_hash = self.blake256(prefix_tx)
else:
tx_hash = None
return TxDcr(
version,
inputs,
outputs,
locktime,
expiry,
witness
), tx_hash, self.cursor - start
def serialize(self):
_output = b''.join((
hex_str_to_hash(self.asset_id),
pack_le_int64(self.value),
pack_le_uint32(self.output_lock),
bytes.fromhex(self.pk_script), # uint168
))
return _output
def serialize(self, payload_version):
_count = len(self.signs)
return b''.join(
(self.illegalType, pack_le_uint32(self.height),
pack_varbytes(self.illegalSigner), self.evidence.serialize(),
self.compareEvidence.serialize(),
pack_varbytes(self.genesisBlockAddress), pack_varint(_count),
b''.join((pack_varbytes(self.signs[i] for i in range(_count))))))
def advance_txs(self, txs, is_unspendable, height, blocktime):
result = super().advance_txs(txs, is_unspendable)
height_numb = pack_le_uint32(height)
tx_num = self.tx_count - len(txs)
split_name_script = self.coin.split_name_script
script_name_hashX = self.coin.name_hashX_from_script
script_key_hashX = self.coin.key_hashX_from_script
script_name_key_hashX = self.coin.name_key_hashX_from_script
script_key_value_hashX = self.coin.key_value_hashX_from_script
update_touched = self.touched.update
hashXs_by_tx = []
append_hashXs = hashXs_by_tx.append
keva_scripts = []
for tx, _tx_hash in txs:
hashXs = []
append_hashX = hashXs.append
# Add the new UTXOs and associate them with the name script
for txout in tx.outputs:
# Get the hashX of the name script. Ignore non-name scripts.
hashX = script_name_hashX(txout.pk_script)
if hashX:
append_hashX(hashX)
# Store key-value script by tx id.
_, address_script = split_name_script(txout.pk_script)
keva_script_len = len(
txout.pk_script) - len(address_script)
# The first 4 bytes are height.
keva_scripts.append(
(_tx_hash,
height_numb + txout.pk_script[0:keva_script_len]))
hashKeyX = script_key_hashX(txout.pk_script)
if hashKeyX:
append_hashX(hashKeyX)
hashKeyNameX = script_name_key_hashX(txout.pk_script)
if hashKeyNameX:
append_hashX(hashKeyNameX)
hashKeyValueX = script_key_value_hashX(txout.pk_script)
for h in hashKeyValueX or []:
append_hashX(h)
append_hashXs(hashXs)
update_touched(hashXs)
tx_num += 1
# Write transaction info to db.
self.db.tx_db.add_tx_info(self.coin, txs, height, blocktime)
self.db.keva.put_keva_script_batch(keva_scripts)
self.db.history.add_unflushed(hashXs_by_tx, self.tx_count - len(txs))
return result
def serialize_unsigned(self):
return b''.join(
(int_to_byte(self.type), int_to_byte(self.payload_version),
self.payload.serialize() if self.payload else b'',
pack_varint(len(self.attributes)),
b''.join(attr.serialize() for attr in self.attributes),
pack_varint(len(self.inputs)), b''.join(tx_in.serialize()
for tx_in in self.inputs),
pack_varint(len(self.outputs)),
b''.join(tx_out.serialize() for tx_out in self.outputs),
pack_le_uint32(self.locktime)))
def serialize(self):
assert len(self.merkleRootMNList) == 32
res = (
pack_le_uint16(self.version) + # version
pack_le_uint32(self.height) + # height
self.merkleRootMNList # merkleRootMNList
)
if self.version > 1:
assert len(self.merkleRootQuorums) == 32
res += self.merkleRootQuorums # merkleRootQuorums
return res
def push_data(cls, data):
'''Returns the opcodes to push the data on the stack.'''
assert isinstance(data, (bytes, bytearray))
n = len(data)
if n < OpCodes.OP_PUSHDATA1:
return bytes([n]) + data
if n < 256:
return bytes([OpCodes.OP_PUSHDATA1, n]) + data
if n < 65536:
return bytes([OpCodes.OP_PUSHDATA2]) + pack_le_uint16(n) + data
return bytes([OpCodes.OP_PUSHDATA4]) + pack_le_uint32(n) + data
def backup_txs(
self,
txs: Sequence[Tuple[Tx, bytes]],
is_unspendable: Callable[[bytes], bool],
):
# Prevout values, in order down the block (coinbase first if present)
# undo_info is in reverse block order
undo_info = self.db.read_undo_info(self.height)
if undo_info is None:
raise ChainError(f'no undo information found for height '
f'{self.height:,d}')
n = len(undo_info)
# Use local vars for speed in the loops
put_utxo = self.utxo_cache.__setitem__
spend_utxo = self.spend_utxo
add_touched_hashx = self.touched_hashxs.add
add_touched_outpoint = self.touched_outpoints.add
undo_hist_spend = self.undo_historical_spends.append
undo_entry_len = HASHX_LEN + TXNUM_LEN + 8
for tx, tx_hash in reversed(txs):
for idx, txout in enumerate(tx.outputs):
# Spend the TX outputs. Be careful with unspendable
# outputs - we didn't save those in the first place.
if is_unspendable(txout.pk_script):
continue
# Get the hashX
cache_value = spend_utxo(tx_hash, idx)
hashX = cache_value[:HASHX_LEN]
add_touched_hashx(hashX)
add_touched_outpoint((tx_hash, idx))
# Restore the inputs
for txin in reversed(tx.inputs):
if txin.is_generation():
continue
n -= undo_entry_len
undo_item = undo_info[n:n + undo_entry_len]
prevout = txin.prev_hash + pack_le_uint32(
txin.prev_idx)[:TXOUTIDX_LEN]
put_utxo(prevout, undo_item)
hashX = undo_item[:HASHX_LEN]
add_touched_hashx(hashX)
add_touched_outpoint((txin.prev_hash, txin.prev_idx))
undo_hist_spend(prevout)
self.undo_tx_hashes.append(b''.join(tx_hash for tx, tx_hash in txs))
assert n == 0
self.tx_count -= len(txs)
def get_spender_txnum_for_txo(self, prev_txnum: int, txout_idx: int) -> Optional[int]:
'''For an outpoint, returns the tx_num that spent it.
If the outpoint is unspent, or even if it never existed (!), returns None.
'''
prev_idx_packed = pack_le_uint32(txout_idx)[:TXOUTIDX_LEN]
prev_txnumb = pack_txnum(prev_txnum)
prevout = prev_txnumb + prev_idx_packed
spender_txnum = self._unflushed_txhash_to_txnum_map.get(prevout)
if spender_txnum is None:
db_key = b's' + prevout
spender_txnumb = self.db.get(db_key)
if spender_txnumb:
spender_txnum = unpack_txnum(spender_txnumb)
return spender_txnum
def spend_utxo(self, tx_hash, tx_idx, spend_tx_hash):
'''Spend a UTXO and return the 33-byte value.
If the UTXO is not in the cache it must be on disk. We store
all UTXOs so not finding one indicates a logic error or DB
corruption.
'''
# Fast track is it being in the cache
idx_packed = pack_le_uint32(tx_idx)
cache_value = self.utxo_cache.pop(tx_hash + idx_packed, None)
if cache_value:
return cache_value
# Spend it from the DB.
# Key: b'h' + compressed_tx_hash + tx_idx + tx_num
# Value: hashX
prefix = b'h' + tx_hash[:4] + idx_packed
candidates = {
db_key: hashX
for db_key, hashX in self.db.utxo_db.iterator(prefix=prefix)
}
for hdb_key, hashX in candidates.items():
tx_num_packed = hdb_key[-4:]
if len(candidates) > 1:
tx_num, = unpack_le_uint32(tx_num_packed)
hash, _height = self.db.fs_tx_hash(tx_num)
if hash != tx_hash:
assert hash is not None # Should always be found
continue
# Key: b'u' + address_hashX + tx_idx + tx_num
# Value: the UTXO value as a 64-bit unsigned integer
udb_key = b'u' + hashX + hdb_key[-8:]
utxo_value_packed = self.db.utxo_db.get(udb_key)
if utxo_value_packed:
# Remove both entries for this UTXO
self.db_deletes.append(hdb_key)
self.db_deletes.append(udb_key)
return hashX + tx_num_packed + utxo_value_packed
if spend_tx_hash in self.dup_tx_hashes:
return None
raise ChainError(
'UTXO {}:{:,d} not found in "h" table at height {}, spend tx {}'.
format(hash_to_hex_str(tx_hash), tx_idx, self.height,
hash_to_hex_str(spend_tx_hash)))
def serialize(self):
nLocktime = pack_le_uint32(self.locktime)
txins = (pack_varint(len(self.inputs)) +
b''.join(tx_in.serialize() for tx_in in self.inputs))
txouts = (pack_varint(len(self.outputs)) +
b''.join(tx_out.serialize() for tx_out in self.outputs))
if self.tx_type:
uVersion = pack_le_uint16(self.version)
uTxType = pack_le_uint16(self.tx_type)
vExtra = self._serialize_extra_payload()
return uVersion + uTxType + txins + txouts + nLocktime + vExtra
else:
nVersion = pack_le_int32(self.version)
return nVersion + txins + txouts + nLocktime
def lookup_hashX(tx_hash, tx_idx):
idx_packed = pack_le_uint32(tx_idx)
# Key: b'h' + compressed_tx_hash + tx_idx + tx_num
# Value: hashX
prefix = b'h' + tx_hash[:4] + idx_packed
# Find which entry, if any, the TX_HASH matches.
for db_key, hashX in self.utxo_db.iterator(prefix=prefix):
tx_num_packed = db_key[-5:]
tx_num, = unpack_le_uint64(tx_num_packed + bytes(3))
fs_hash, _height = self.fs_tx_hash(tx_num)
if fs_hash == tx_hash:
return hashX, idx_packed + tx_num_packed
return None, None
def serialize(self, tx_version):
_output = b''.join((
self.asset_id,
pack_le_int64(self.value),
pack_le_uint32(self.output_lock),
self.pk_script, # uint168
))
if tx_version >= 0x09:
_output = b''.join((
_output,
int_to_byte(self.output_type)
if self.output_type is not None else b'',
self.output_payload.serialize()
if self.output_payload else b'',
))
return _output
def backup_txs(self, txs, is_unspendable):
undo_info = self.db.read_undo_info(self.height)
if undo_info is None:
raise ChainError(
f'no undo information found for height {self.height:,d}')
# Use local vars for speed in the loops
put_utxo = self.utxo_cache.__setitem__
spend_utxo = self.spend_utxo
add_touched_hashx = self.touched_hashxs.add
add_touched_outpoint = self.touched_outpoints.add
undo_entry_len = HASHX_LEN + TXNUM_LEN + 8
# Restore coins that had been spent
# (may include coins made then spent in this block)
n = 0
for tx, tx_hash in txs:
for txin in tx.inputs:
if txin.is_generation():
continue
undo_item = undo_info[n:n + undo_entry_len]
prevout = txin.prev_hash + pack_le_uint32(
txin.prev_idx)[:TXOUTIDX_LEN]
put_utxo(prevout, undo_item)
add_touched_hashx(undo_item[:HASHX_LEN])
add_touched_outpoint((txin.prev_hash, txin.prev_idx))
n += undo_entry_len
assert n == len(undo_info)
# Remove tx outputs made in this block, by spending them.
for tx, tx_hash in txs:
for idx, txout in enumerate(tx.outputs):
# Spend the TX outputs. Be careful with unspendable
# outputs - we didn't save those in the first place.
if is_unspendable(txout.pk_script):
continue
# Get the hashX
cache_value = spend_utxo(tx_hash, idx)
hashX = cache_value[:HASHX_LEN]
add_touched_hashx(hashX)
add_touched_outpoint((tx_hash, idx))
self.undo_tx_hashes.append(b''.join(tx_hash for tx, tx_hash in txs))
self.tx_count -= len(txs)
def spend_utxo(self, tx_hash: bytes, tx_idx: int, spend_tx_hash) -> bytes:
'''Spend a UTXO and return (hashX + tx_num + value_sats).
If the UTXO is not in the cache it must be on disk. We store
all UTXOs so not finding one indicates a logic error or DB
corruption.
'''
# Fast track is it being in the cache
idx_packed = pack_le_uint32(tx_idx)
cache_value = self.utxo_cache.pop(tx_hash + idx_packed, None)
if cache_value:
return cache_value
# Spend it from the DB.
txnum_padding = bytes(8-TXNUM_LEN)
# Key: b'h' + compressed_tx_hash + tx_idx + tx_num
# Value: hashX
prefix = b'h' + tx_hash[:COMP_TXID_LEN] + idx_packed
candidates = {db_key: hashX for db_key, hashX
in self.db.utxo_db.iterator(prefix=prefix)}
for hdb_key, hashX in candidates.items():
tx_num_packed = hdb_key[-TXNUM_LEN:]
if len(candidates) > 1:
tx_num, = unpack_le_uint64(tx_num_packed + txnum_padding)
hash, _height = self.db.fs_tx_hash(tx_num)
if hash != tx_hash:
assert hash is not None # Should always be found
continue
# Key: b'u' + address_hashX + tx_idx + tx_num
# Value: the UTXO value as a 64-bit unsigned integer
udb_key = b'u' + hashX + hdb_key[-4-TXNUM_LEN:]
utxo_value_packed = self.db.utxo_db.get(udb_key)
if utxo_value_packed:
# Remove both entries for this UTXO
self.db_deletes.append(hdb_key)
self.db_deletes.append(udb_key)
return hashX + tx_num_packed + utxo_value_packed
if spend_tx_hash in self.dup_tx_hashes:
return None
raise ChainError(f'UTXO {hash_to_hex_str(tx_hash)} / {tx_idx:,d} not '
f'found in "h" table')
def _get_hashX_for_utxo(
self,
tx_hash: bytes,
txout_idx: int,
) -> Tuple[Optional[bytes], Optional[bytes]]:
idx_packed = pack_le_uint32(txout_idx)[:TXOUTIDX_LEN]
tx_num = self.fs_txnum_for_txhash(tx_hash)
if tx_num is None:
return None, None
tx_numb = pack_txnum(tx_num)
# Key: b'h' + tx_num + txout_idx
# Value: hashX
db_key = b'h' + tx_numb + idx_packed
hashX = self.utxo_db.get(db_key)
if hashX is None:
return None, None
return hashX, tx_numb + idx_packed
def backup_txs(self, txs, is_unspendable):
undo_info = self.db.read_undo_info(self.height)
if undo_info is None:
raise ChainError(
'no undo information found for height {:,d}'.format(
self.height))
# Use local vars for speed in the loops
put_utxo = self.utxo_cache.__setitem__
spend_utxo = self.spend_utxo
script_hashX = self.coin.hashX_from_script
add_touched = self.touched.add
undo_entry_len = 12 + HASHX_LEN
# Restore coins that had been spent
# (may include coins made then spent in this block)
n = 0
for tx, tx_hash in txs:
for txin in tx.inputs:
if txin.is_generation():
continue
undo_item = undo_info[n:n + undo_entry_len]
put_utxo(txin.prev_hash + pack_le_uint32(txin.prev_idx),
undo_item)
add_touched(undo_item[:-12])
n += undo_entry_len
assert n == len(undo_info)
# Remove tx outputs made in this block, by spending them.
for tx, tx_hash in txs:
for idx, txout in enumerate(tx.outputs):
# Spend the TX outputs. Be careful with unspendable
# outputs - we didn't save those in the first place.
if is_unspendable(txout.pk_script):
continue
# Get the hashX
hashX = script_hashX(txout.script)
cache_value = spend_utxo(tx_hash, idx, tx_hash)
if cache_value is not None:
add_touched(cache_value[:-12])
self.tx_count -= len(txs)
def backup_txs(self, txs, is_unspendable):
# Prevout values, in order down the block (coinbase first if present)
# undo_info is in reverse block order
undo_info = self.db.read_undo_info(self.height)
if undo_info is None:
raise ChainError(
'no undo information found for height {:,d}'.format(
self.height))
n = len(undo_info)
# Use local vars for speed in the loops
put_utxo = self.utxo_cache.__setitem__
spend_utxo = self.spend_utxo
script_hashX = self.coin.hashX_from_script
touched = self.touched
undo_entry_len = 13 + HASHX_LEN
for tx, tx_hash in reversed(txs):
for idx, txout in enumerate(tx.outputs):
# Spend the TX outputs. Be careful with unspendable
# outputs - we didn't save those in the first place.
if is_unspendable(txout.pk_script):
continue
# Get the hashX
hashX = script_hashX(txout.pk_script)
cache_value = spend_utxo(tx_hash, idx)
touched.add(cache_value[:-13])
# Restore the inputs
for txin in reversed(tx.inputs):
if txin.is_generation():
continue
n -= undo_entry_len
undo_item = undo_info[n:n + undo_entry_len]
put_utxo(txin.prev_hash + pack_le_uint32(txin.prev_idx),
undo_item)
touched.add(undo_item[:-13])
assert n == 0
self.tx_count -= len(txs)
def spend_utxo(self, tx_hash: bytes, txout_idx: int) -> bytes:
'''Spend a UTXO and return (hashX + tx_num + value_sats).
If the UTXO is not in the cache it must be on disk. We store
all UTXOs so not finding one indicates a logic error or DB
corruption.
'''
# Fast track is it being in the cache
idx_packed = pack_le_uint32(txout_idx)[:TXOUTIDX_LEN]
cache_value = self.utxo_cache.pop(tx_hash + idx_packed, None)
if cache_value:
return cache_value
# Spend it from the DB.
tx_num = self.db.fs_txnum_for_txhash(tx_hash)
if tx_num is None:
raise ChainError(
f'UTXO {hash_to_hex_str(tx_hash)} / {txout_idx:,d} has '
f'no corresponding tx_num in DB')
tx_numb = pack_txnum(tx_num)
# Key: b'h' + tx_num + txout_idx
# Value: hashX
hdb_key = b'h' + tx_numb + idx_packed
hashX = self.db.utxo_db.get(hdb_key)
if hashX is None:
raise ChainError(
f'UTXO {hash_to_hex_str(tx_hash)} / {txout_idx:,d} not '
f'found in "h" table')
# Key: b'u' + address_hashX + tx_num + txout_idx
# Value: the UTXO value as a 64-bit unsigned integer
udb_key = b'u' + hashX + tx_numb + idx_packed
utxo_value_packed = self.db.utxo_db.get(udb_key)
if utxo_value_packed is None:
raise ChainError(
f'UTXO {hash_to_hex_str(tx_hash)} / {txout_idx:,d} not '
f'found in "u" table')
# Remove both entries for this UTXO
self.db_deletes.append(hdb_key)
self.db_deletes.append(udb_key)
return hashX + tx_numb + utxo_value_packed
def serialize(self):
return b''.join((
self.prev_hash,
pack_le_uint16(self.prev_idx),
pack_le_uint32(self.sequence),
))
def serialize(self, payload_version):
return b''.join(
(pack_le_uint32(self.startHeight), pack_le_uint32(self.endHeight)))
def serialize(self, payload_version):
_count = len(self.arbitrators)
return b''.join(
(pack_varbytes(self.sponsor), pack_le_uint32(self.blockHeight),
pack_varint(_count), b''.join(
(pack_varbytes(self.arbitrators[i]) for i in range(_count)))))
def serialize_unsigned(self):
return b''.join(
(pack_le_uint32(self.coinType), pack_le_uint32(self.blockHeight),
self.evidence.serialize_unsigned(),
self.compareEvidence.serialize_unsigned()))
def serialize(self):
return b''.join((self.proposal.serialize(), pack_varbytes(self.header),
pack_le_uint32(self.height)))
