def test_block(block_details):
coin, block_info = block_details
raw_block = unhexlify(block_info['block'])
block = coin.block(raw_block, block_info['height'])
assert coin.header_hash(block.header) == hex_str_to_hash(
block_info['hash'])
assert (coin.header_prevhash(block.header) == hex_str_to_hash(
block_info['previousblockhash']))
for n, (tx, txid) in enumerate(block.transactions):
assert txid == hex_str_to_hash(block_info['tx'][n])
async def make_raw_header(self, b):
pbh = b.get('previousblockhash')
if pbh is None:
pbh = '0' * 64
return b''.join([
pack('<L', b.get('version')),
hex_str_to_hash(pbh),
hex_str_to_hash(b.get('merkleroot')),
pack('<L', self.timestamp_safe(b['time'])),
pack('<L', int(b.get('bits'), 16)),
pack('<L', int(b.get('nonce')))
])
def test_block(block_infos):
# From electrumx test_block 342930
coin = LBC
block_info = block_infos['342930']
raw_block = unhexlify(block_info['block'])
block = coin.block(raw_block, block_info['height'])
assert coin.header_hash(block.header) == hex_str_to_hash(
block_info['hash'])
assert (coin.header_prevhash(block.header) == hex_str_to_hash(
block_info['previousblockhash']))
for n, (tx, txid) in enumerate(block.transactions):
assert txid == hex_str_to_hash(block_info['tx'][n])
def __init__(self, env, db, daemon, notifications):
self.env = env
self.db = db
self.daemon = daemon
self.notifications = notifications
self.coin = env.coin
self.blocks_event = asyncio.Event()
self.prefetcher = Prefetcher(daemon, env.coin, self.blocks_event)
self.logger = class_logger(__name__, self.__class__.__name__)
# Meta
self.next_cache_check = 0
self.touched = set()
self.reorg_count = 0
self.height = -1
self.tip = None
self.tx_count = 0
self._caught_up_event = None
# Caches of unflushed items.
self.headers = []
self.tx_hashes = []
self.undo_infos = []
# UTXO cache
self.utxo_cache = {}
self.db_deletes = []
# If the lock is successfully acquired, in-memory chain state
# is consistent with self.height
self.state_lock = asyncio.Lock()
self.dup_tx_hashes = {
hex_str_to_hash(
'7702eaa0e042846d39d01eeb4c87f774913022e9958cfd714c5c2942af380569'
),
hex_str_to_hash(
'a5210b0bdfe0edaff3f1fb7ac24a379f55bbc51dcc224dc5efc04c1de8b30b2f'
),
hex_str_to_hash(
'1bf147bdaaad84364f6ff49661c66a0d7d4545c0eab2cd997d2ea0f3490393ec'
),
hex_str_to_hash(
'95e55038b16a4f6f81bbdcf3a44b0a76ffc76e395c57c0967229f26088d05fa7'
),
hex_str_to_hash(
'83890738940d7afd1f94a67db072f8fc4fdeea60c1f32e46f082f86ff4be3a48'
),
}
async def transaction_get_merkle(self, tx_hash, height):
'''Return the markle tree to a confirmed transaction given its hash
and height.
tx_hash: the transaction hash as a hexadecimal string
height: the height of the block it is in
'''
self.assert_tx_hash(tx_hash)
height = non_negative_integer(height)
hex_hashes = await self.daemon_request('block_hex_hashes', height, 1)
block_hash = hex_hashes[0]
block = await self.daemon_request('deserialised_block', block_hash)
tx_hashes = block['tx']
try:
pos = tx_hashes.index(tx_hash)
except ValueError:
raise RPCError(
BAD_REQUEST, f'tx hash {tx_hash} not in '
f'block {block_hash} at height {height:,d}')
hashes = [hex_str_to_hash(hash) for hash in tx_hashes]
branch, root = self.bp.merkle.branch_and_root(hashes, pos)
branch = [hash_to_hex_str(hash) for hash in branch]
return {"block_height": height, "merkle": branch, "pos": pos}
async def _refresh_hashes(self, synchronized_event):
'''Refresh our view of the daemon's mempool.'''
# touched_* accumulates between calls to on_mempool and each
# call transfers ownership
touched_hashxs = set()
touched_outpoints = set()
while True:
height = self.api.cached_height()
hex_hashes = await self.api.mempool_hashes()
if height != await self.api.height():
continue
hashes = {hex_str_to_hash(hh) for hh in hex_hashes}
try:
async with self.lock:
await self._process_mempool(
all_hashes=hashes,
touched_hashxs=touched_hashxs,
touched_outpoints=touched_outpoints,
mempool_height=height,
)
except DBSyncError:
# The UTXO DB is not at the same height as the
# mempool; wait and try again
self.logger.debug('waiting for DB to sync')
else:
synchronized_event.set()
synchronized_event.clear()
await self.api.on_mempool(
touched_hashxs=touched_hashxs,
touched_outpoints=touched_outpoints,
height=height,
)
touched_hashxs = set()
touched_outpoints = set()
await sleep(self.refresh_secs)
async def tx_merkle(self, tx_hash, height):
'''tx_hash is a hex string.'''
hex_hashes = await self.daemon_request('block_hex_hashes', height, 1)
block = await self.daemon_request('deserialised_block', hex_hashes[0])
tx_hashes = block['tx']
try:
pos = tx_hashes.index(tx_hash)
except ValueError:
raise RPCError(
BAD_REQUEST, f'tx hash {tx_hash} not in '
f'block {hex_hashes[0]} at height {height:,d}')
idx = pos
hashes = [hex_str_to_hash(txh) for txh in tx_hashes]
merkle_branch = []
while len(hashes) > 1:
if len(hashes) & 1:
hashes.append(hashes[-1])
idx = idx - 1 if (idx & 1) else idx + 1
merkle_branch.append(hash_to_str(hashes[idx]))
idx //= 2
hashes = [
double_sha256(hashes[n] + hashes[n + 1])
for n in range(0, len(hashes), 2)
]
return {"block_height": height, "merkle": merkle_branch, "pos": pos}
def test_block(block_details):
coin, block_info = block_details
raw_block = unhexlify(block_info['block'])
block = coin.block(raw_block, block_info['height'])
try:
assert coin.header_hash(
block.header) == hex_str_to_hash(block_info['hash'])
except ImportError as e:
pytest.skip(str(e))
assert (coin.header_prevhash(block.header)
== hex_str_to_hash(block_info['previousblockhash']))
assert len(block_info['tx']) == len(block.transactions)
for n, (tx, txid) in enumerate(block.transactions):
assert txid == hex_str_to_hash(block_info['tx'][n])
def serialize(self):
_output = b''.join((
hex_str_to_hash(self.asset_id),
pack_le_int64(self.value),
pack_le_uint32(self.output_lock),
bytes.fromhex(self.pk_script), # uint168
))
return _output
async def raw_transactions(self, hex_hashes):
'''Query bitcoind for the serialized raw transactions with the given
hashes. Missing transactions are returned as None.
hex_hashes is an iterable of hexadecimal hash strings.'''
await sleep(0)
hashes = [hex_str_to_hash(hex_hash) for hex_hash in hex_hashes]
return [self.raw_txs.get(hash) for hash in hashes]
def scripthash_to_hashX(scripthash):
try:
bin_hash = hex_str_to_hash(scripthash)
if len(bin_hash) == 32:
return bin_hash[:HASHX_LEN]
except Exception:
pass
raise RPCError(BAD_REQUEST, f'{scripthash} is not a valid script hash')
async def mock_tx_hashes_at_blockheight(height):
COST = 0.2502
hashes = ["aec10c10352cd6a519f5dc9ceda52aa8ef17570f6730d7d2347dfc1f5c963196",
"b2378093f853cbc635153950d8f3bcec1a785e3a62deec652533c7d8e8613866",
"d28df756c9cc2beadce3ef692a9e6419c4ef73a12cbdd56b23acd7452d320022",
"5c52e28bc0961fb5cc552023d4ab7a68320e4dae567c1d7d58a185bd84e12a3d",
"ed5a81e439e1cd9139ddb81da80bfa7cfc31e323aea544ca92a9ee1d84b9fb2f"]
hashes = [hex_str_to_hash(x) for x in hashes]
return hashes, COST
def test_block(block_details):
coin, block_info = block_details
raw_block = unhexlify(block_info['block'])
block = coin.block(raw_block, block_info['height'])
h = coin.electrum_header(block.header, block_info['height'])
assert block_info['merkleroot'] == h['merkle_root']
assert block_info['time'] == h['timestamp']
assert block_info['previousblockhash'] == h['prev_block_hash']
assert block_info['height'] == h['block_height']
assert block_info['nonce'] == h['nonce']
assert block_info['bits'] == pack_be_uint32(h['bits']).hex()
assert coin.header_hash(block.header) == hex_str_to_hash(
block_info['hash'])
assert (coin.header_prevhash(block.header) == hex_str_to_hash(
block_info['previousblockhash']))
for n, (tx, txid) in enumerate(block.transactions):
assert txid == hex_str_to_hash(block_info['tx'][n])
def _get_merkle_branch(self, tx_hashes, tx_pos):
'''Return a merkle branch to a transaction.
tx_hashes: ordered list of hex strings of tx hashes in a block
tx_pos: index of transaction in tx_hashes to create branch for
'''
hashes = [hex_str_to_hash(hash) for hash in tx_hashes]
branch, root = self.db.merkle.branch_and_root(hashes, tx_pos)
branch = [hash_to_hex_str(hash) for hash in branch]
return branch
async def _refresh_hashes(self, synchronized_event):
'''Refresh our view of the daemon's mempool.'''
while True:
height = self.api.cached_height()
hex_hashes = await self.api.mempool_hashes()
if height != await self.api.height():
continue
hashes = set(hex_str_to_hash(hh) for hh in hex_hashes)
async with self.lock:
touched = await self._process_mempool(hashes)
synchronized_event.set()
synchronized_event.clear()
await self.api.on_mempool(touched, height)
await sleep(self.refresh_secs)
async def _refresh_hashes(self, once):
'''Refresh our view of the daemon's mempool.'''
for loop_count in itertools.count():
height = self.daemon.cached_height()
hex_hashes = await self.daemon.mempool_hashes()
if height != await self.daemon.height():
continue
hashes = set(hex_str_to_hash(hh) for hh in hex_hashes)
touched = await self._process_mempool(hashes)
await self.notifications.on_mempool(touched, height)
# Refresh the cached histogram periodically. Thread it as it
# can be expensive.
if loop_count % 100 == 0:
await self.tasks.run_in_thread(self._update_histogram)
if once:
return
await asyncio.sleep(5)
async def _refresh_hashes(self, synchronized_event):
'''Refresh our view of the daemon's mempool.'''
sleep = 5
histogram_refresh = self.coin.MEMPOOL_HISTOGRAM_REFRESH_SECS // sleep
for loop_count in itertools.count():
height = self.daemon.cached_height()
hex_hashes = await self.daemon.mempool_hashes()
if height != await self.daemon.height():
continue
hashes = set(hex_str_to_hash(hh) for hh in hex_hashes)
touched = await self._process_mempool(hashes)
synchronized_event.set()
await self.notifications.on_mempool(touched, height)
# Thread mempool histogram refreshes - they can be expensive
if loop_count % histogram_refresh == 0:
await run_in_thread(self._update_histogram)
await asyncio.sleep(sleep)
def get_utxos(self, hashX):
'''Return an unordered list of UTXO named tuples from mempool
transactions that pay to hashX.
This does not consider if any other mempool transactions spend
the outputs.
'''
utxos = []
# hashXs is a defaultdict, so use get() to query
for hex_hash in self.hashXs.get(hashX, []):
item = self.txs.get(hex_hash)
if not item:
continue
txout_pairs = item[1]
for pos, (hX, value) in enumerate(txout_pairs):
if hX == hashX:
# Unfortunately UTXO holds a binary hash
utxos.append(
UTXO(-1, pos, hex_str_to_hash(hex_hash), 0, value))
return utxos
async def transaction_merkle(self, tx_hash, height):
'''Return the markle tree to a confirmed transaction given its hash
and height.
tx_hash: the transaction hash as a hexadecimal string
height: the height of the block it is in
'''
assert_tx_hash(tx_hash)
block_hash, tx_hashes = await self.block_hash_and_tx_hashes(height)
try:
pos = tx_hashes.index(tx_hash)
except ValueError:
raise RPCError(
BAD_REQUEST, f'tx hash {tx_hash} not in '
f'block {block_hash} at height {height:,d}')
hashes = [hex_str_to_hash(hash) for hash in tx_hashes]
branch, root = self.bp.merkle.branch_and_root(hashes, pos)
branch = [hash_to_hex_str(hash) for hash in branch]
return {"block_height": height, "merkle": branch, "pos": pos}
def backup_blocks(self, raw_blocks, hex_hashes):
'''Backup the raw blocks and flush.
The blocks should be in order of decreasing height, starting at.
self.height. A flush is performed once the blocks are backed up.
'''
self.db.assert_flushed(self.flush_data())
assert self.height >= len(raw_blocks)
coin = self.coin
for raw_block, hex_hash in zip(raw_blocks, hex_hashes):
# Check and update self.tip
block = coin.block(raw_block, self.height)
header_hash = hex_str_to_hash(hex_hash)
if header_hash != self.tip:
raise ChainError('backup block {} not tip {} at height {:,d}'
.format(hash_to_hex_str(header_hash),
hash_to_hex_str(self.tip),
self.height))
self.tip = coin.header_prevhash(block.header)
self.backup_txs(block.transactions)
self.height -= 1
self.db.tx_counts.pop()
def is_one(flag):
return flag == hex_str_to_hash('01') or flag == hex_str_to_hash(
'51')
async def _process_mempool(self, all_hashes, touched, mempool_height):
# Re-sync with the new set of hashes
txs = self.txs
hashXs = self.hashXs
tx_to_create = self.tx_to_asset_create
tx_to_reissue = self.tx_to_asset_reissue
creates = self.asset_creates
reissues = self.asset_reissues
if mempool_height != self.api.db_height():
raise DBSyncError
# First handle txs that have disappeared
for tx_hash in set(txs).difference(all_hashes):
tx = txs.pop(tx_hash)
reissued_asset = tx_to_reissue.pop(tx_hash, None)
if reissued_asset:
del reissues[reissued_asset]
created_asset = tx_to_create.pop(tx_hash, None)
if created_asset:
del creates[created_asset]
tx_hashXs = set(hashX for hashX, value, _, _ in tx.in_pairs)
tx_hashXs.update(hashX for hashX, value, _, _ in tx.out_pairs)
for hashX in tx_hashXs:
hashXs[hashX].remove(tx_hash)
if not hashXs[hashX]:
del hashXs[hashX]
touched.update(tx_hashXs)
# Process new transactions
new_hashes = list(all_hashes.difference(txs))
if new_hashes:
group = TaskGroup()
for hashes in chunks(new_hashes, 200):
coro = self._fetch_and_accept(hashes, all_hashes, touched)
await group.spawn(coro)
tx_map = {}
utxo_map = {}
async for task in group:
(deferred,
unspent), internal_creates, internal_reissues = task.result()
# Store asset changes
for asset, stats in internal_creates.items():
tx_to_create[hex_str_to_hash(
stats['source']['tx_hash'])] = asset
creates[asset] = stats
for asset, stats in internal_reissues.items():
tx_to_reissue[hex_str_to_hash(
stats['source']['tx_hash'])] = asset
reissues[asset] = stats
tx_map.update(deferred)
utxo_map.update(unspent)
prior_count = 0
# FIXME: this is not particularly efficient
while tx_map and len(tx_map) != prior_count:
prior_count = len(tx_map)
tx_map, utxo_map = self._accept_transactions(
tx_map, utxo_map, touched)
if tx_map:
self.logger.error(f'{len(tx_map)} txs dropped')
return touched
def create_raw_transaction(self, did, json_payload):
LOG.info("Creating raw transaction...")
spec = json_payload["header"]["specification"]
operation = json_payload["header"]["operation"]
verification = json_payload["proof"]["verificationMethod"]
signature = json_payload["proof"]["signature"]
payload = json_payload["payload"]
try:
did_sidechain_rpc = DidSidechainRpc()
addresses = [self.wallets[self.current_wallet_index]["address"]]
utxo_txid, asset_id, value, prev_idx = did_sidechain_rpc.get_utxos(
addresses)
wallet_exhausted = 0
while float(value) < 0.000001:
if wallet_exhausted == config.NUM_WALLETS:
LOG.info(
"None of the wallets have enough UTXOs to send a transaction"
)
return None
self.current_wallet_index += 1
if self.current_wallet_index > config.NUM_WALLETS - 1:
self.current_wallet_index = 0
utxo_txid, asset_id, value, prev_idx = did_sidechain_rpc.get_utxos(
addresses)
wallet_exhausted += 1
change = int((10**8) * (float(value) - self.did_sidechain_fee))
previous_txid = ""
if operation == "update":
previous_txid = json_payload["header"]["previousTxid"]
tx_header = tx_ela.DIDHeaderInfo(
specification=str.encode(spec),
operation=str.encode(operation),
previoustxid=str.encode(previous_txid))
tx_proof = tx_ela.DIDProofInfo(
type=b"ECDSAsecp256r1",
verification_method=str.encode(verification),
signature=str.encode(signature))
tx_payload = tx_ela.TxPayloadDIDOperation(
header=tx_header, payload=str.encode(payload),
proof=tx_proof).serialize()
sender_hashed_public_key = self.address_to_programhash(
self.wallets[self.current_wallet_index]["address"], False)
did_hashed = self.address_to_programhash(did, False)
# Variables needed for raw_tx
tx_type = b'\x0a' # DID transaction
payload_version = struct.pack("<B", 0) # one byte
output_count = struct.pack("<B", 2) # one byte
lock_time = struct.pack("<L", 0) # 4 bytes
program_count = struct.pack("<B", 1) # one byte
tx_attributes = tx_ela.TxAttribute(usage=129,
data=b'1234567890').serialize()
tx_input = tx_ela.TxInputELA(prev_hash=hex_str_to_hash(utxo_txid),
prev_idx=prev_idx,
sequence=0).serialize()
# DID requires 2 outputs. The first one is DID string with amount 0 and the second one is change address
# and amount. Fee is about 100 sela (.000001 ELA)
output1 = tx_ela.TxOutputELA(asset_id=hex_str_to_hash(asset_id),
value=0,
output_lock=0,
pk_script=did_hashed,
output_type=None,
output_payload=None).serialize(
tx_ela.TransferAsset)
output2 = tx_ela.TxOutputELA(asset_id=hex_str_to_hash(asset_id),
value=change,
output_lock=0,
pk_script=sender_hashed_public_key,
output_type=None,
output_payload=None).serialize(
tx_ela.TransferAsset)
raw_tx_string = (tx_type + payload_version + tx_payload +
program_count + tx_attributes + program_count +
tx_input + output_count + output1 + output2 +
lock_time)
code = self.get_code_from_pb()
signature = self.ecdsa_sign(raw_tx_string)
parameter = (struct.pack("B", len(signature)) + signature).hex()
parameter_bytes = bytes.fromhex(parameter)
code_bytes = bytes.fromhex(code)
script = (pack_varint(len(parameter_bytes)) + parameter_bytes +
pack_varint(len(code_bytes)) + code_bytes)
real_tx = (tx_type + payload_version + tx_payload + program_count +
tx_attributes + program_count + tx_input +
output_count + output1 + output2 + lock_time +
program_count + script)
return real_tx
except Exception as err:
message = "Error: " + str(err) + "\n"
exc_type, exc_obj, exc_tb = sys.exc_info()
message += "Unexpected error: " + str(exc_type) + "\n"
message += ' File "' + exc_tb.tb_frame.f_code.co_filename + '", line ' + str(
exc_tb.tb_lineno) + "\n"
LOG.info(f"Error while creating a transaction: {message}")
return None
def is_zero(flag):
return flag == hex_str_to_hash('') or flag == hex_str_to_hash('00')
def test_hex_str_to_hash():
assert lib_hash.hex_str_to_hash('7274735f6f745f68736168') == b'hash_to_str'
def process_raw_txs(self, raw_tx_map, pending):
'''Process the dictionary of raw transactions and return a dictionary
of updates to apply to self.txs.
This runs in the executor so should not update any member
variables it doesn't own. Atomic reads of self.txs that do
not depend on the result remaining the same are fine.
'''
script_hashX = self.coin.hashX_from_script
deserializer = self.coin.DESERIALIZER
db_utxo_lookup = self.db.db_utxo_lookup
txs = self.txs
# Deserialize each tx and put it in a pending list
for tx_hash, raw_tx in raw_tx_map.items():
if tx_hash not in txs:
continue
tx, tx_size = deserializer(raw_tx).read_tx_and_vsize()
# Convert the tx outputs into (hashX, value) pairs
txout_pairs = [(script_hashX(txout.pk_script), txout.value)
for txout in tx.outputs]
# Convert the tx inputs to ([prev_hex_hash, prev_idx) pairs
txin_pairs = [(hash_to_hex_str(txin.prev_hash), txin.prev_idx)
for txin in tx.inputs]
pending.append((tx_hash, txin_pairs, txout_pairs, tx_size))
# Now process what we can
result = {}
deferred = []
for item in pending:
if self.stop:
break
tx_hash, old_txin_pairs, txout_pairs, tx_size = item
if tx_hash not in txs:
continue
mempool_missing = False
txin_pairs = []
try:
for prev_hex_hash, prev_idx in old_txin_pairs:
tx_info = txs.get(prev_hex_hash, 0)
if tx_info is None:
tx_info = result.get(prev_hex_hash)
if not tx_info:
mempool_missing = True
continue
if tx_info:
txin_pairs.append(tx_info[1][prev_idx])
elif not mempool_missing:
prev_hash = hex_str_to_hash(prev_hex_hash)
txin_pairs.append(db_utxo_lookup(prev_hash, prev_idx))
except (self.db.MissingUTXOError, self.db.DBError):
# DBError can happen when flushing a newly processed
# block. MissingUTXOError typically happens just
# after the daemon has accepted a new block and the
# new mempool has deps on new txs in that block.
continue
if mempool_missing:
deferred.append(item)
else:
# Compute fee
tx_fee = (sum(v for hashX, v in txin_pairs) -
sum(v for hashX, v in txout_pairs))
result[tx_hash] = (txin_pairs, txout_pairs, tx_fee, tx_size)
return result, deferred
async def raw_transactions(self, hex_hashes):
await sleep(0)
hashes = [hex_str_to_hash(hex_hash) for hex_hash in hex_hashes]
return [self.raw_txs.get(hash) for hash in hashes]
