def assert_address_history_tx(address_history, spk, height, txid, subscribed):
history_element = address_history[hashes.script_to_scripthash(spk)]
assert history_element["history"][0]["height"] == height
assert history_element["history"][0]["tx_hash"] == txid
#fee always zero, its easier to test because otherwise you have
# to use Decimal to stop float weirdness
if height == 0:
assert history_element["history"][0]["fee"] == 0
assert history_element["subscribed"] == subscribed
def build_address_history(self, monitored_scriptpubkeys):
self.log("Building history with " + str(len(monitored_scriptpubkeys)) +
" addresses . . .")
st = time.time()
address_history = {}
for spk in monitored_scriptpubkeys:
address_history[hashes.script_to_scripthash(spk)] = {
'history': [],
'subscribed': False
}
wallet_addr_scripthashes = set(address_history.keys())
self.reorganizable_txes = []
#populate history
#which is a blockheight-ordered list of ("txhash", height)
#unconfirmed transactions go at the end as ("txhash", 0, fee)
# 0=unconfirmed -1=unconfirmed with unconfirmed parents
BATCH_SIZE = 1000
ret = list(range(BATCH_SIZE))
t = 0
count = 0
obtained_txids = set()
last_tx = None
while len(ret) == BATCH_SIZE:
ret = self.rpc.call("listtransactions", ["*", BATCH_SIZE, t, True])
self.debug("listtransactions skip=" + str(t) + " len(ret)=" +
str(len(ret)))
if t == 0 and len(ret) > 0:
last_tx = ret[-1]
t += len(ret)
for tx in ret:
if "txid" not in tx or "category" not in tx:
continue
if tx["category"] not in ("receive", "send"):
continue
if tx["confirmations"] < 0:
continue #conflicted
if tx["txid"] in obtained_txids:
continue
self.debug("adding obtained tx=" + str(tx["txid"]))
obtained_txids.add(tx["txid"])
#obtain all the addresses this transaction is involved with
output_scriptpubkeys, input_scriptpubkeys, txd = \
self.get_input_and_output_scriptpubkeys(tx["txid"])
output_scripthashes = [
hashes.script_to_scripthash(sc)
for sc in output_scriptpubkeys
]
sh_to_add = wallet_addr_scripthashes.intersection(
set(output_scripthashes))
input_scripthashes = [
hashes.script_to_scripthash(sc)
for sc in input_scriptpubkeys
]
sh_to_add |= wallet_addr_scripthashes.intersection(
set(input_scripthashes))
if len(sh_to_add) == 0:
continue
for wal in self.deterministic_wallets:
overrun_depths = wal.have_scriptpubkeys_overrun_gaplimit(
output_scriptpubkeys)
if overrun_depths != None:
self.log("ERROR: Not enough addresses imported.")
self.log("Delete wallet.dat and increase the value " +
"of `initial_import_count` in the file " +
"`config.cfg` then reimport and rescan")
#TODO make it so users dont have to delete wallet.dat
# check whether all initial_import_count addresses are
# imported rather than just the first one
return False
new_history_element = self.generate_new_history_element(
tx, txd)
for scripthash in sh_to_add:
address_history[scripthash]["history"].append(
new_history_element)
if tx["confirmations"] > 0 and (tx["confirmations"] <
CONFIRMATIONS_SAFE_FROM_REORG):
self.reorganizable_txes.append(
(tx["txid"], tx["blockhash"],
new_history_element["height"], sh_to_add))
count += 1
unconfirmed_txes = defaultdict(list)
for scrhash, his in address_history.items():
uctx = self.sort_address_history_list(his)
for u in uctx:
unconfirmed_txes[u["tx_hash"]].append(scrhash)
self.debug("unconfirmed_txes = " + str(unconfirmed_txes))
self.debug("reorganizable_txes = " + str(self.reorganizable_txes))
if len(ret) > 0:
#txid doesnt uniquely identify transactions from listtransactions
#but the tuple (txid, address) does
self.last_known_wallet_txid = (last_tx["txid"],
last_tx.get("address", None))
else:
self.last_known_wallet_txid = None
self.debug("last_known_wallet_txid = " +
str(self.last_known_wallet_txid))
et = time.time()
self.debug("address_history =\n" + pprint.pformat(address_history))
self.log("Found " + str(count) + " txes. History built in " +
str(et - st) + "sec")
self.address_history = address_history
self.unconfirmed_txes = unconfirmed_txes
return True
def check_for_new_txes(self):
MAX_TX_REQUEST_COUNT = 256
tx_request_count = 2
max_attempts = int(math.log(MAX_TX_REQUEST_COUNT, 2))
for i in range(max_attempts):
self.debug("listtransactions tx_request_count=" +
str(tx_request_count))
##how listtransactions works
##skip and count parameters take most-recent txes first
## so skip=0 count=1 will return the most recent tx
##and skip=0 count=3 will return the 3 most recent txes
##but the actual list returned has the REVERSED order
##skip=0 count=3 will return a list with the most recent tx LAST
ret = self.rpc.call("listtransactions",
["*", tx_request_count, 0, True])
ret = ret[::-1]
if self.last_known_wallet_txid == None:
recent_tx_index = len(ret) #=0 means no new txes
break
else:
txid_list = [(tx["txid"], tx.get("address", None))
for tx in ret]
recent_tx_index = next(
(i for i, (txid, addr) in enumerate(txid_list)
if txid == self.last_known_wallet_txid[0]
and addr == self.last_known_wallet_txid[1]), -1)
if recent_tx_index != -1:
break
tx_request_count *= 2
#TODO low priority: handle a user getting more than 255 new
# transactions in 15 seconds
self.debug("recent tx index = " + str(recent_tx_index) + " ret = " +
str([(t["txid"], t.get("address", None)) for t in ret]))
if len(ret) > 0:
self.last_known_wallet_txid = (ret[0]["txid"],
ret[0].get("address", None))
self.debug("last_known_wallet_txid = " +
str(self.last_known_wallet_txid))
assert (recent_tx_index != -1)
if recent_tx_index == 0:
return set()
new_txes = ret[:recent_tx_index][::-1]
self.debug("new txes = " + str(new_txes))
obtained_txids = set()
updated_scripthashes = []
for tx in new_txes:
if "txid" not in tx or "category" not in tx:
continue
if tx["category"] not in ("receive", "send"):
continue
if tx["confirmations"] < 0:
continue #conflicted
if tx["txid"] in obtained_txids:
continue
obtained_txids.add(tx["txid"])
output_scriptpubkeys, input_scriptpubkeys, txd = \
self.get_input_and_output_scriptpubkeys(tx["txid"])
matching_scripthashes = []
for spk in (output_scriptpubkeys + input_scriptpubkeys):
scripthash = hashes.script_to_scripthash(spk)
if scripthash in self.address_history:
matching_scripthashes.append(scripthash)
if len(matching_scripthashes) == 0:
continue
for wal in self.deterministic_wallets:
overrun_depths = wal.have_scriptpubkeys_overrun_gaplimit(
output_scriptpubkeys)
if overrun_depths != None:
for change, import_count in overrun_depths.items():
spks = wal.get_new_scriptpubkeys(change, import_count)
for spk in spks:
self.address_history[hashes.script_to_scripthash(
spk)] = {
'history': [],
'subscribed': False
}
new_addrs = [
hashes.script_to_address(s, self.rpc) for s in spks
]
self.debug("importing " + str(len(spks)) +
" into change=" + str(change))
import_addresses(self.rpc, new_addrs, self.debug,
self.log)
updated_scripthashes.extend(matching_scripthashes)
new_history_element = self.generate_new_history_element(tx, txd)
self.log("Found new tx: " + str(new_history_element))
for scrhash in matching_scripthashes:
self.address_history[scrhash]["history"].append(
new_history_element)
if new_history_element["height"] == 0:
self.unconfirmed_txes[tx["txid"]].append(scrhash)
if tx["confirmations"] > 0:
self.reorganizable_txes.append(
(tx["txid"], tx["blockhash"],
new_history_element["height"], matching_scripthashes))
return set(updated_scripthashes)
def test():
debugf = lambda x: x
logf = lambda x: x
#empty deterministic wallets
deterministic_wallets = [TestDeterministicWallet()]
test_spk1 = "deadbeefdeadbeefdeadbeefdeadbeef"
test_containing_block1 = "blockhash-placeholder1"
test_paying_in_tx1 = {
"txid": "placeholder-test-txid1",
"vin": [{"txid": "placeholder-unknown-input-txid", "vout": 0}],
"vout": [{"value": 1, "scriptPubKey": {"hex": test_spk1}}],
"address": test_spk_to_address(test_spk1),
"category": "receive",
"confirmations": 1,
"blockhash": test_containing_block1,
"hex": "placeholder-test-txhex1"
}
test_spk2 = "deadbeefdeadbeefdeadbeef"
test_containing_block2 = "blockhash-placeholder2"
test_paying_in_tx2 = {
"txid": "placeholder-test-txid2",
"vin": [{"txid": "placeholder-unknown-input-txid", "vout": 0}],
"vout": [{"value": 1, "scriptPubKey": {"hex": test_spk2}}],
"address": test_spk_to_address(test_spk2),
"category": "receive",
"confirmations": 1,
"blockhash": test_containing_block2,
"hex": "placeholder-test-txhex2"
}
###single confirmed tx in wallet belonging to us, address history built
rpc = TestJsonRpc([test_paying_in_tx1], [],
{test_containing_block1: 420000})
txmonitor1 = TransactionMonitor(rpc, deterministic_wallets, debugf, logf)
assert txmonitor1.build_address_history([test_spk1])
assert len(txmonitor1.address_history) == 1
assert_address_history_tx(txmonitor1.address_history, spk=test_spk1,
height=420000, txid=test_paying_in_tx1["txid"], subscribed=False)
###two confirmed txes in wallet belonging to us, addr history built
rpc = TestJsonRpc([test_paying_in_tx1, test_paying_in_tx2], [],
{test_containing_block1: 1, test_containing_block2: 2})
deterministic_wallets = [TestDeterministicWallet()]
txmonitor2 = TransactionMonitor(rpc, deterministic_wallets, debugf, logf)
assert txmonitor2.build_address_history([test_spk1, test_spk2])
assert len(txmonitor2.address_history) == 2
assert_address_history_tx(txmonitor2.address_history, spk=test_spk1,
height=1, txid=test_paying_in_tx1["txid"], subscribed=False)
assert_address_history_tx(txmonitor2.address_history, spk=test_spk2,
height=2, txid=test_paying_in_tx2["txid"], subscribed=False)
###one unconfirmed tx in wallet belonging to us, with confirmed inputs,
### addr history built, then tx confirms, not subscribed to address
test_spk3 = "deadbeefdeadbeef"
test_containing_block3 = "blockhash-placeholder3"
input_utxo3 = {"txid": "placeholder-unknown-input-txid", "vout": 0,
"value": 1, "confirmations": 1}
test_paying_in_tx3 = {
"txid": "placeholder-test-txid3",
"vin": [input_utxo3],
"vout": [{"value": 1, "scriptPubKey": {"hex": test_spk3}}],
"address": test_spk_to_address(test_spk3),
"category": "receive",
"confirmations": 0,
"blockhash": test_containing_block3,
"hex": "placeholder-test-txhex3"
}
rpc = TestJsonRpc([test_paying_in_tx3], [input_utxo3],
{test_containing_block3: 10})
txmonitor3 = TransactionMonitor(rpc, deterministic_wallets, debugf, logf)
assert txmonitor3.build_address_history([test_spk3])
assert len(txmonitor3.address_history) == 1
assert_address_history_tx(txmonitor3.address_history, spk=test_spk3,
height=0, txid=test_paying_in_tx3["txid"], subscribed=False)
assert len(list(txmonitor3.check_for_updated_txes())) == 0
test_paying_in_tx3["confirmations"] = 1 #tx confirms
#not subscribed so still only returns an empty list
assert len(list(txmonitor3.check_for_updated_txes())) == 0
assert_address_history_tx(txmonitor3.address_history, spk=test_spk3,
height=10, txid=test_paying_in_tx3["txid"], subscribed=False)
###build empty address history, subscribe one address
### an unconfirmed tx appears, then confirms
test_spk4 = "deadbeefdeadbeefaa"
test_containing_block4 = "blockhash-placeholder4"
input_utxo4 = {"txid": "placeholder-unknown-input-txid", "vout": 0,
"value": 1, "confirmations": 1}
test_paying_in_tx4 = {
"txid": "placeholder-test-txid4",
"vin": [input_utxo4],
"vout": [{"value": 1, "scriptPubKey": {"hex": test_spk4}}],
"address": test_spk_to_address(test_spk4),
"category": "receive",
"confirmations": 0,
"blockhash": test_containing_block4,
"hex": "placeholder-test-txhex4"
}
rpc = TestJsonRpc([], [input_utxo4], {test_containing_block4: 10})
txmonitor4 = TransactionMonitor(rpc, deterministic_wallets, debugf, logf)
assert txmonitor4.build_address_history([test_spk4])
assert len(txmonitor4.address_history) == 1
sh4 = hashes.script_to_scripthash(test_spk4)
assert len(txmonitor4.get_electrum_history(sh4)) == 0
txmonitor4.subscribe_address(sh4)
# unconfirm transaction appears
assert len(list(txmonitor4.check_for_updated_txes())) == 0
rpc.add_transaction(test_paying_in_tx4)
assert len(list(txmonitor4.check_for_updated_txes())) == 1
assert_address_history_tx(txmonitor4.address_history, spk=test_spk4,
height=0, txid=test_paying_in_tx4["txid"], subscribed=True)
# transaction confirms
test_paying_in_tx4["confirmations"] = 1
assert len(list(txmonitor4.check_for_updated_txes())) == 1
assert_address_history_tx(txmonitor4.address_history, spk=test_spk4,
height=10, txid=test_paying_in_tx4["txid"], subscribed=True)
###transaction that has nothing to do with our wallet
test_spk5 = "deadbeefdeadbeefbb"
test_containing_block5 = "blockhash-placeholder5"
test_paying_in_tx5 = {
"txid": "placeholder-test-txid5",
"vin": [{"txid": "placeholder-unknown-input-txid", "vout": 0}],
"vout": [{"value": 1, "scriptPubKey": {"hex": test_spk5}}],
"address": test_spk_to_address(test_spk5),
"category": "receive",
"confirmations": 0,
"blockhash": test_containing_block5,
"hex": "placeholder-test-txhex5"
}
test_spk5_1 = "deadbeefdeadbeefcc"
rpc = TestJsonRpc([test_paying_in_tx5], [], {test_containing_block4: 10})
txmonitor5 = TransactionMonitor(rpc, deterministic_wallets, debugf, logf)
assert txmonitor5.build_address_history([test_spk5_1])
assert len(txmonitor5.address_history) == 1
assert len(txmonitor5.get_electrum_history(hashes.script_to_scripthash(
test_spk5_1))) == 0
###transaction which arrives to an address which already has a tx on it
test_spk6 = "deadbeefdeadbeefdd"
test_containing_block6 = "blockhash-placeholder6"
test_paying_in_tx6 = {
"txid": "placeholder-test-txid6",
"vin": [{"txid": "placeholder-unknown-input-txid", "vout": 0}],
"vout": [{"value": 1, "scriptPubKey": {"hex": test_spk6}}],
"address": test_spk_to_address(test_spk6),
"category": "receive",
"confirmations": 1,
"blockhash": test_containing_block6,
"hex": "placeholder-test-txhex6"
}
test_paying_in_tx6_1 = {
"txid": "placeholder-test-txid6_1",
"vin": [{"txid": "placeholder-unknown-input-txid", "vout": 0}],
"vout": [{"value": 1, "scriptPubKey": {"hex": test_spk6}}],
"address": test_spk_to_address(test_spk6),
"category": "receive",
"confirmations": 1,
"blockhash": test_containing_block6,
"hex": "placeholder-test-txhex6"
}
rpc = TestJsonRpc([test_paying_in_tx6], [], {test_containing_block6: 10})
txmonitor6 = TransactionMonitor(rpc, deterministic_wallets, debugf, logf)
assert txmonitor6.build_address_history([test_spk6])
sh = hashes.script_to_scripthash(test_spk6)
assert len(txmonitor6.get_electrum_history(sh)) == 1
rpc.add_transaction(test_paying_in_tx6_1)
assert len(txmonitor6.get_electrum_history(sh)) == 1
txmonitor6.check_for_updated_txes()
assert len(txmonitor6.get_electrum_history(sh)) == 2
###transaction spending FROM one of our addresses
test_spk7 = "deadbeefdeadbeefee"
test_input_containing_block7 = "blockhash-input-placeholder7"
test_input_tx7 = {
"txid": "placeholder-input-test-txid7",
"vin": [{"txid": "placeholder-unknown-input-txid", "vout": 0}],
"vout": [{"value": 1, "scriptPubKey": {"hex": test_spk7}}],
"address": test_spk_to_address(test_spk7),
"category": "send",
"confirmations": 2,
"blockhash": test_input_containing_block7,
"hex": "placeholder-input-test-txhex7"
}
test_containing_block7 = "blockhash-placeholder7"
test_paying_from_tx7 = {
"txid": "placeholder-test-txid7",
"vin": [{"txid": test_input_tx7["txid"], "vout": 0}],
"vout": [{"value": 1, "scriptPubKey": {"hex": "deadbeef"}}],
"address": test_spk_to_address(test_spk7),
"category": "receive",
"confirmations": 1,
"blockhash": test_containing_block7,
"hex": "placeholder-test-txhex7"
}
rpc = TestJsonRpc([test_input_tx7, test_paying_from_tx7], [],
{test_containing_block7: 9, test_input_containing_block7: 8})
txmonitor7 = TransactionMonitor(rpc, deterministic_wallets, debugf, logf)
assert txmonitor7.build_address_history([test_spk7])
sh = hashes.script_to_scripthash(test_spk7)
assert len(txmonitor7.get_electrum_history(sh)) == 2
###transaction from one address to the other, both addresses in wallet
test_spk8 = "deadbeefdeadbeefee"
test_spk8_1 = "deadbeefdeadbeefff"
test_input_containing_block8 = "blockhash-input-placeholder8"
test_input_tx8 = {
"txid": "placeholder-input-test-txid8",
"vin": [{"txid": "placeholder-unknown-input-txid", "vout": 0}],
"vout": [{"value": 1, "scriptPubKey": {"hex": test_spk8}}],
"address": test_spk_to_address(test_spk8),
"category": "send",
"confirmations": 2,
"blockhash": test_input_containing_block8,
"hex": "placeholder-input-test-txhex8"
}
test_containing_block8 = "blockhash-placeholder8"
test_paying_from_tx8 = {
"txid": "placeholder-test-txid8",
"vin": [{"txid": test_input_tx8["txid"], "vout": 0}],
"vout": [{"value": 1, "scriptPubKey": {"hex": test_spk8_1}}],
"address": test_spk_to_address(test_spk8),
"category": "receive",
"confirmations": 1,
"blockhash": test_containing_block8,
"hex": "placeholder-test-txhex8"
}
rpc = TestJsonRpc([test_input_tx8, test_paying_from_tx8], [],
{test_containing_block8: 9, test_input_containing_block8: 8})
txmonitor8 = TransactionMonitor(rpc, deterministic_wallets, debugf, logf)
assert txmonitor8.build_address_history([test_spk8, test_spk8_1])
assert len(txmonitor8.get_electrum_history(hashes.script_to_scripthash(
test_spk8))) == 2
assert len(txmonitor8.get_electrum_history(hashes.script_to_scripthash(
test_spk8_1))) == 1
###overrun gap limit so import address is needed
test_spk9 = "deadbeefdeadbeef00"
test_containing_block9 = "blockhash-placeholder9"
test_paying_in_tx9 = {
"txid": "placeholder-test-txid9",
"vin": [{"txid": "placeholder-unknown-input-txid", "vout": 0}],
"vout": [{"value": 1, "scriptPubKey": {"hex": test_spk9}}],
"address": test_spk_to_address(test_spk9),
"category": "receive",
"confirmations": 1,
"blockhash": test_containing_block9,
"hex": "placeholder-test-txhex9"
}
test_spk9_imported = "deadbeefdeadbeef11"
class TestImportDeterministicWallet(DeterministicWallet):
def __init__(self):
pass
def have_scriptpubkeys_overrun_gaplimit(self, scriptpubkeys):
return {0: 1} #overrun by one
def get_new_scriptpubkeys(self, change, count):
return [test_spk9_imported]
rpc = TestJsonRpc([], [], {test_containing_block9: 10})
txmonitor9 = TransactionMonitor(rpc, [TestImportDeterministicWallet()],
debugf, logf)
assert txmonitor9.build_address_history([test_spk9])
assert len(txmonitor9.address_history) == 1
assert len(list(txmonitor9.check_for_updated_txes())) == 0
assert len(txmonitor9.get_electrum_history(hashes.script_to_scripthash(
test_spk9))) == 0
rpc.add_transaction(test_paying_in_tx9)
assert len(list(txmonitor9.check_for_updated_txes())) == 0
assert len(txmonitor9.get_electrum_history(hashes.script_to_scripthash(
test_spk9))) == 1
assert len(txmonitor9.get_electrum_history(hashes.script_to_scripthash(
test_spk9_imported))) == 0
assert len(rpc.get_imported_addresses()) == 1
assert rpc.get_imported_addresses()[0] == test_spk_to_address(
test_spk9_imported)
###conflicted transaction
test_spk10 = "deadbeefdeadbeefcccc"
test_paying_in_tx10 = {
"txid": "placeholder-test-txid10",
"vin": [{"txid": "placeholder-unknown-input-txid", "vout": 0}],
"vout": [{"value": 1, "scriptPubKey": {"hex": test_spk10}}],
"address": test_spk_to_address(test_spk10),
"category": "receive",
"confirmations": -1,
"hex": "placeholder-test-txhex10"
}
rpc = TestJsonRpc([test_paying_in_tx10], [], {})
txmonitor10 = TransactionMonitor(rpc, deterministic_wallets, debugf, logf)
assert txmonitor10.build_address_history([test_spk10])
assert len(txmonitor10.address_history) == 1
assert len(txmonitor10.get_electrum_history(hashes.script_to_scripthash(
test_spk10))) == 0 #shouldnt show up after build history
rpc.add_transaction(test_paying_in_tx10)
assert len(list(txmonitor10.check_for_updated_txes())) == 0
assert len(txmonitor10.get_electrum_history(hashes.script_to_scripthash(
test_spk10))) == 0 #shouldnt show up after tx is added
#other possible stuff to test:
#finding confirmed and unconfirmed tx, in that order, then both confirm
#finding unconfirmed and confirmed tx, in that order, then both confirm
print("\nAll tests passed")