def _verify_ioauth_data(self, ioauth_data):
verified_data = []
# Need to authorize against the btc pubkey first.
for nick, nickdata in ioauth_data.items():
utxo_list, auth_pub, cj_addr, change_addr, btc_sig, maker_pk = nickdata
if not self.auth_counterparty(btc_sig, auth_pub, maker_pk):
jlog.debug(
"Counterparty encryption verification failed, aborting: " + nick)
# This counterparty must be rejected
continue
if not validate_address(cj_addr)[0]\
or not validate_address(change_addr)[0]:
jlog.warn("Counterparty provided invalid address: {}".format(
(cj_addr, change_addr)))
# Interpreted as malicious
self.add_ignored_makers([nick])
continue
try:
maker_inputs_data = self._verify_ioauth_inputs(
nick, utxo_list, auth_pub)
except IoauthInputVerificationError as e:
for msg in e.messages:
jlog.warning(msg)
continue
verified_data.append(maker_inputs_data._replace(
utxo_list=utxo_list, cj_addr=cj_addr, change_addr=change_addr))
return verified_data
def test_b58_invalid_addresses(setup_addresses):
#none of these are valid as any kind of key or address
with open(os.path.join(testdir,"base58_keys_invalid.json"), "r") as f:
json_data = f.read()
invalid_key_list = json.loads(json_data)
for k in invalid_key_list:
bad_key = k[0]
res, message = validate_address(bad_key)
assert res == False, "Incorrectly validated address: " + bad_key + " with message: " + message
def test_invalid_bech32_addresses():
invalids = [
"tc1qw508d6qejxtdg4y5r3zarvary0c5xw7kg3g4ty",
"bc1qw508d6qejxtdg4y5r3zarvary0c5xw7kv8f3t5",
"BC13W508D6QEJXTDG4Y5R3ZARVARY0C5XW7KN40WF2",
"bc1rw5uspcuh",
"bc10w508d6qejxtdg4y5r3zarvary0c5xw7kw508d6qejxtdg4y5r3zarvary0c5xw7kw5rljs90",
"BC1QR508D6QEJXTDG4Y5R3ZARVARYV98GJ9P",
"tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3q0sL5k7",
"bc1zw508d6qejxtdg4y5r3zarvaryvqyzf3du",
"tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3pjxtptv",
"bc1gmk9yu"]
for iva in invalids:
print("Testing this address: ", iva)
res, message = validate_address(iva)
assert res == False, "Incorrectly validated address: " + iva
def test_valid_bech32_addresses():
valids = ["BC1QW508D6QEJXTDG4Y5R3ZARVARY0C5XW7KV8F3T4",
"tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3q0sl5k7",
"bc1pw508d6qejxtdg4y5r3zarvary0c5xw7kw508d6qejxtdg4y5r3zarvary0c5xw7k7grplx",
"BC1SW50QA3JX3S",
"bc1zw508d6qejxtdg4y5r3zarvaryvg6kdaj",
"tb1qqqqqp399et2xygdj5xreqhjjvcmzhxw4aywxecjdzew6hylgvsesrxh6hy"]
for va in valids:
print("Testing this address: ", va)
if va.lower()[:2] == "bc":
jm_single().config.set("BLOCKCHAIN", "network", "mainnet")
else:
jm_single().config.set("BLOCKCHAIN", "network", "testnet")
res, message = validate_address(va)
assert res == True, "Incorrect failed to validate address: " + va + " with message: " + message
jm_single().config.set("BLOCKCHAIN", "network", "testnet")
def test_b58_valid_addresses():
with open(os.path.join(testdir,"base58_keys_valid.json"), "r") as f:
json_data = f.read()
valid_keys_list = json.loads(json_data)
for a in valid_keys_list:
addr, pubkey, prop_dict = a
if not prop_dict["isPrivkey"]:
if prop_dict["isTestnet"]:
jm_single().config.set("BLOCKCHAIN", "network", "testnet")
else:
jm_single().config.set("BLOCKCHAIN", "network", "mainnet")
#if using pytest -s ; sanity check to see what's actually being tested
print('testing this address: ', addr)
res, message = validate_address(addr)
assert res == True, "Incorrectly failed to validate address: " + addr + " with message: " + message
jm_single().config.set("BLOCKCHAIN", "network", "testnet")
def test_valid_bip341_scriptpubkeys_addresses():
with ChainParams("bitcoin"):
with open(os.path.join(testdir, "bip341_wallet_test_vectors.json"),
"r") as f:
json_data = json.loads(f.read())
for x in json_data["scriptPubKey"]:
sPK = hextobin(x["expected"]["scriptPubKey"])
addr = x["expected"]["bip350Address"]
res, message = validate_address(addr)
assert res, message
print("address {} was valid bech32m".format(addr))
# test this specific conversion because this is how
# our human readable outputs work:
assert str(CCoinAddress.from_scriptPubKey(
btc.CScript(sPK))) == addr
print("and it converts correctly from scriptPubKey: {}".format(
btc.CScript(sPK)))
def test_valid_bech32_addresses():
valids = ["BC1QW508D6QEJXTDG4Y5R3ZARVARY0C5XW7KV8F3T4",
"tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3q0sl5k7",
# TODO these are valid bech32 addresses but rejected by bitcointx
# because they are not witness version 0; add others.
#"bc1pw508d6qejxtdg4y5r3zarvary0c5xw7kw508d6qejxtdg4y5r3zarvary0c5xw7k7grplx",
#"BC1SW50QA3JX3S",
#"bc1zw508d6qejxtdg4y5r3zarvaryvg6kdaj",
"tb1qqqqqp399et2xygdj5xreqhjjvcmzhxw4aywxecjdzew6hylgvsesrxh6hy"]
for va in valids:
if va.lower()[:2] == "bc":
jm_single().config.set("BLOCKCHAIN", "network", "mainnet")
btc.select_chain_params("bitcoin")
else:
jm_single().config.set("BLOCKCHAIN", "network", "testnet")
btc.select_chain_params("bitcoin/testnet")
res, message = validate_address(va)
assert res == True, "Incorrect failed to validate address: " + va + " with message: " + message
jm_single().config.set("BLOCKCHAIN", "network", "testnet")
btc.select_chain_params("bitcoin/regtest")
def receive_utxos(self, ioauth_data):
"""Triggered when the daemon returns utxo data from
makers who responded; this is the completion of phase 1
of the protocol
"""
if self.aborted:
return (False, "User aborted")
#Temporary list used to aggregate all ioauth data that must be removed
rejected_counterparties = []
#Need to authorize against the btc pubkey first.
for nick, nickdata in ioauth_data.items():
utxo_list, auth_pub, cj_addr, change_addr, btc_sig, maker_pk = nickdata
if not self.auth_counterparty(btc_sig, auth_pub, maker_pk):
jlog.debug(
"Counterparty encryption verification failed, aborting: " +
nick)
#This counterparty must be rejected
rejected_counterparties.append(nick)
if not validate_address(cj_addr)[0] or not validate_address(
change_addr)[0]:
jlog.warn("Counterparty provided invalid address: {}".format(
(cj_addr, change_addr)))
# Interpreted as malicious
self.add_ignored_makers([nick])
rejected_counterparties.append(nick)
for rc in rejected_counterparties:
del ioauth_data[rc]
self.maker_utxo_data = {}
for nick, nickdata in ioauth_data.items():
utxo_list, auth_pub, cj_addr, change_addr, _, _ = nickdata
utxo_data = jm_single().bc_interface.query_utxo_set(utxo_list)
self.utxos[nick] = utxo_list
if None in utxo_data:
jlog.warn(('ERROR outputs unconfirmed or already spent. '
'utxo_data={}').format(pprint.pformat(utxo_data)))
jlog.warn('Disregarding this counterparty.')
del self.utxos[nick]
continue
#Complete maker authorization:
#Extract the address fields from the utxos
#Construct the Bitcoin address for the auth_pub field
#Ensure that at least one address from utxos corresponds.
for inp in utxo_data:
try:
if self.wallet_service.pubkey_has_script(
auth_pub, inp['script']):
break
except EngineError as e:
pass
else:
jlog.warn("ERROR maker's (" + nick + ")"
" authorising pubkey is not included "
"in the transaction!")
#this will not be added to the transaction, so we will have
#to recheck if we have enough
continue
total_input = sum([d['value'] for d in utxo_data])
real_cjfee = calc_cj_fee(self.orderbook[nick]['ordertype'],
self.orderbook[nick]['cjfee'],
self.cjamount)
change_amount = (total_input - self.cjamount -
self.orderbook[nick]['txfee'] + real_cjfee)
# certain malicious and/or incompetent liquidity providers send
# inputs totalling less than the coinjoin amount! this leads to
# a change output of zero satoshis; this counterparty must be removed.
if change_amount < jm_single().DUST_THRESHOLD:
fmt = ('ERROR counterparty requires sub-dust change. nick={}'
'totalin={:d} cjamount={:d} change={:d}').format
jlog.warn(fmt(nick, total_input, self.cjamount, change_amount))
jlog.warn("Invalid change, too small, nick= " + nick)
continue
self.outputs.append({
'address': change_addr,
'value': change_amount
})
fmt = ('fee breakdown for {} totalin={:d} '
'cjamount={:d} txfee={:d} realcjfee={:d}').format
jlog.info(
fmt(nick, total_input, self.cjamount,
self.orderbook[nick]['txfee'], real_cjfee))
self.outputs.append({'address': cj_addr, 'value': self.cjamount})
self.cjfee_total += real_cjfee
self.maker_txfee_contributions += self.orderbook[nick]['txfee']
self.maker_utxo_data[nick] = utxo_data
#We have succesfully processed the data from this nick:
try:
self.nonrespondants.remove(nick)
except Exception as e:
jlog.warn("Failure to remove counterparty from nonrespondants list: " + str(nick) + \
", error message: " + repr(e))
#Apply business logic of how many counterparties are enough; note that
#this must occur after the above ioauth data processing, since we only now
#know for sure that the data meets all business-logic requirements.
if len(self.maker_utxo_data) < jm_single().config.getint(
"POLICY", "minimum_makers"):
self.taker_info_callback("INFO",
"Not enough counterparties, aborting.")
return (False,
"Not enough counterparties responded to fill, giving up")
self.taker_info_callback("INFO", "Got all parts, enough to build a tx")
#The list self.nonrespondants is now reset and
#used to track return of signatures for phase 2
self.nonrespondants = list(self.maker_utxo_data.keys())
my_total_in = sum([va['value'] for u, va in self.input_utxos.items()])
if self.my_change_addr:
#Estimate fee per choice of next/3/6 blocks targetting.
estimated_fee = estimate_tx_fee(
len(sum(self.utxos.values(), [])),
len(self.outputs) + 2,
txtype=self.wallet_service.get_txtype())
jlog.info("Based on initial guess: " +
btc.amount_to_str(self.total_txfee) +
", we estimated a miner fee of: " +
btc.amount_to_str(estimated_fee))
#reset total
self.total_txfee = estimated_fee
my_txfee = max(self.total_txfee - self.maker_txfee_contributions, 0)
my_change_value = (my_total_in - self.cjamount - self.cjfee_total -
my_txfee)
#Since we could not predict the maker's inputs, we may end up needing
#too much such that the change value is negative or small. Note that
#we have tried to avoid this based on over-estimating the needed amount
#in SendPayment.create_tx(), but it is still a possibility if one maker
#uses a *lot* of inputs.
if self.my_change_addr:
if my_change_value < -1:
raise ValueError(
"Calculated transaction fee of: " +
btc.amount_to_str(self.total_txfee) +
" is too large for our inputs; Please try again.")
if my_change_value <= jm_single().BITCOIN_DUST_THRESHOLD:
jlog.info("Dynamically calculated change lower than dust: " +
btc.amount_to_str(my_change_value) + "; dropping.")
self.my_change_addr = None
my_change_value = 0
jlog.info(
'fee breakdown for me totalin=%d my_txfee=%d makers_txfee=%d cjfee_total=%d => changevalue=%d'
% (my_total_in, my_txfee, self.maker_txfee_contributions,
self.cjfee_total, my_change_value))
if self.my_change_addr is None:
if my_change_value != 0 and abs(my_change_value) != 1:
# seems you wont always get exactly zero because of integer
# rounding so 1 satoshi extra or fewer being spent as miner
# fees is acceptable
jlog.info(
('WARNING CHANGE NOT BEING USED\nCHANGEVALUE = {}').format(
btc.amount_to_str(my_change_value)))
# we need to check whether the *achieved* txfee-rate is outside
# the range allowed by the user in config; if not, abort the tx.
# this is done with using the same estimate fee function and comparing
# the totals; this ratio will correspond to the ratio of the feerates.
num_ins = len([u for u in sum(self.utxos.values(), [])])
num_outs = len(self.outputs) + 2
new_total_fee = estimate_tx_fee(
num_ins, num_outs, txtype=self.wallet_service.get_txtype())
feeratio = self.total_txfee / new_total_fee
jlog.debug(
"Ratio of actual to estimated sweep fee: {}".format(feeratio))
sweep_delta = float(jm_single().config.get("POLICY",
"max_sweep_fee_change"))
if feeratio < 1 - sweep_delta or feeratio > 1 + sweep_delta:
jlog.warn(
"Transaction fee for sweep: {} too far from expected:"
" {}; check the setting 'max_sweep_fee_change'"
" in joinmarket.cfg. Aborting this attempt.".format(
self.total_txfee, new_total_fee))
return (False, "Unacceptable feerate for sweep, giving up.")
else:
self.outputs.append({
'address': self.my_change_addr,
'value': my_change_value
})
self.utxo_tx = [u for u in sum(self.utxos.values(), [])]
self.outputs.append({
'address': self.coinjoin_address(),
'value': self.cjamount
})
# pre-Nov-2020/v0.8.0: transactions used ver 1 and nlocktime 0
# so only the new "pit" (using native segwit) will use the updated
# version 2 and nlocktime ~ current block as per normal payments.
# TODO makers do not check this; while there is no security risk,
# it might be better for them to sanity check.
if self.wallet_service.get_txtype() == "p2wpkh":
n_version = 2
locktime = compute_tx_locktime()
else:
n_version = 1
locktime = 0
self.latest_tx = btc.make_shuffled_tx(self.utxo_tx,
self.outputs,
version=n_version,
locktime=locktime)
jlog.info('obtained tx\n' +
btc.human_readable_transaction(self.latest_tx))
self.taker_info_callback("INFO",
"Built tx, sending to counterparties.")
return (True, list(self.maker_utxo_data.keys()),
bintohex(self.latest_tx.serialize()))
def test_non_addresses(setup_addresses):
#could flesh this out with other examples
res, msg = validate_address(2)
assert res == False, "Incorrectly accepted number"
def receive_utxos(self, ioauth_data):
"""Triggered when the daemon returns utxo data from
makers who responded; this is the completion of phase 1
of the protocol
"""
if self.aborted:
return (False, "User aborted")
#Temporary list used to aggregate all ioauth data that must be removed
rejected_counterparties = []
#Need to authorize against the btc pubkey first.
for nick, nickdata in ioauth_data.iteritems():
utxo_list, auth_pub, cj_addr, change_addr, btc_sig, maker_pk = nickdata
if not self.auth_counterparty(btc_sig, auth_pub, maker_pk):
jlog.debug(
"Counterparty encryption verification failed, aborting: " +
nick)
#This counterparty must be rejected
rejected_counterparties.append(nick)
if not validate_address(cj_addr)[0] or not validate_address(
change_addr)[0]:
jlog.warn("Counterparty provided invalid address: {}".format(
(cj_addr, change_addr)))
# Interpreted as malicious
self.add_ignored_makers([nick])
rejected_counterparties.append(nick)
for rc in rejected_counterparties:
del ioauth_data[rc]
self.maker_utxo_data = {}
for nick, nickdata in ioauth_data.iteritems():
utxo_list, auth_pub, cj_addr, change_addr, btc_sig, maker_pk = nickdata
self.utxos[nick] = utxo_list
utxo_data = jm_single().bc_interface.query_utxo_set(
self.utxos[nick])
if None in utxo_data:
jlog.warn(('ERROR outputs unconfirmed or already spent. '
'utxo_data={}').format(pprint.pformat(utxo_data)))
jlog.warn('Disregarding this counterparty.')
del self.utxos[nick]
continue
#Complete maker authorization:
#Extract the address fields from the utxos
#Construct the Bitcoin address for the auth_pub field
#Ensure that at least one address from utxos corresponds.
input_addresses = [d['address'] for d in utxo_data]
# FIXME: This only works if taker's commitment address is of same type
# as our wallet.
auth_address = self.wallet.pubkey_to_addr(unhexlify(auth_pub))
if not auth_address in input_addresses:
jlog.warn("ERROR maker's (" + nick + ")"
" authorising pubkey is not included "
"in the transaction: " + str(auth_address))
#this will not be added to the transaction, so we will have
#to recheck if we have enough
continue
total_input = sum([d['value'] for d in utxo_data])
real_cjfee = calc_cj_fee(self.orderbook[nick]['ordertype'],
self.orderbook[nick]['cjfee'],
self.cjamount)
change_amount = (total_input - self.cjamount -
self.orderbook[nick]['txfee'] + real_cjfee)
# certain malicious and/or incompetent liquidity providers send
# inputs totalling less than the coinjoin amount! this leads to
# a change output of zero satoshis; this counterparty must be removed.
if change_amount < jm_single().DUST_THRESHOLD:
fmt = ('ERROR counterparty requires sub-dust change. nick={}'
'totalin={:d} cjamount={:d} change={:d}').format
jlog.warn(fmt(nick, total_input, self.cjamount, change_amount))
jlog.warn("Invalid change, too small, nick= " + nick)
continue
self.outputs.append({
'address': change_addr,
'value': change_amount
})
fmt = ('fee breakdown for {} totalin={:d} '
'cjamount={:d} txfee={:d} realcjfee={:d}').format
jlog.info(
fmt(nick, total_input, self.cjamount,
self.orderbook[nick]['txfee'], real_cjfee))
self.outputs.append({'address': cj_addr, 'value': self.cjamount})
self.cjfee_total += real_cjfee
self.maker_txfee_contributions += self.orderbook[nick]['txfee']
self.maker_utxo_data[nick] = utxo_data
#We have succesfully processed the data from this nick:
try:
self.nonrespondants.remove(nick)
except Exception as e:
jlog.warn("Failure to remove counterparty from nonrespondants list: " + str(nick) + \
", error message: " + repr(e))
#Apply business logic of how many counterparties are enough; note that
#this must occur after the above ioauth data processing, since we only now
#know for sure that the data meets all business-logic requirements.
if len(self.maker_utxo_data.keys()) < jm_single().config.getint(
"POLICY", "minimum_makers"):
self.taker_info_callback("INFO",
"Not enough counterparties, aborting.")
return (False,
"Not enough counterparties responded to fill, giving up")
self.taker_info_callback("INFO", "Got all parts, enough to build a tx")
#The list self.nonrespondants is now reset and
#used to track return of signatures for phase 2
self.nonrespondants = list(self.maker_utxo_data.keys())
my_total_in = sum(
[va['value'] for u, va in self.input_utxos.iteritems()])
if self.my_change_addr:
#Estimate fee per choice of next/3/6 blocks targetting.
estimated_fee = estimate_tx_fee(len(sum(self.utxos.values(), [])),
len(self.outputs) + 2,
txtype=self.wallet.get_txtype())
jlog.info("Based on initial guess: " + str(self.total_txfee) +
", we estimated a miner fee of: " + str(estimated_fee))
#reset total
self.total_txfee = estimated_fee
my_txfee = max(self.total_txfee - self.maker_txfee_contributions, 0)
my_change_value = (my_total_in - self.cjamount - self.cjfee_total -
my_txfee)
#Since we could not predict the maker's inputs, we may end up needing
#too much such that the change value is negative or small. Note that
#we have tried to avoid this based on over-estimating the needed amount
#in SendPayment.create_tx(), but it is still a possibility if one maker
#uses a *lot* of inputs.
if self.my_change_addr and my_change_value <= 0:
raise ValueError("Calculated transaction fee of: " +
str(self.total_txfee) +
" is too large for our inputs;Please try again.")
elif self.my_change_addr and my_change_value <= jm_single(
).BITCOIN_DUST_THRESHOLD:
jlog.info("Dynamically calculated change lower than dust: " +
str(my_change_value) + "; dropping.")
self.my_change_addr = None
my_change_value = 0
jlog.info(
'fee breakdown for me totalin=%d my_txfee=%d makers_txfee=%d cjfee_total=%d => changevalue=%d'
% (my_total_in, my_txfee, self.maker_txfee_contributions,
self.cjfee_total, my_change_value))
if self.my_change_addr is None:
if my_change_value != 0 and abs(my_change_value) != 1:
# seems you wont always get exactly zero because of integer
# rounding so 1 satoshi extra or fewer being spent as miner
# fees is acceptable
jlog.info(('WARNING CHANGE NOT BEING '
'USED\nCHANGEVALUE = {}').format(my_change_value))
else:
self.outputs.append({
'address': self.my_change_addr,
'value': my_change_value
})
self.utxo_tx = [
dict([('output', u)]) for u in sum(self.utxos.values(), [])
]
self.outputs.append({
'address': self.coinjoin_address(),
'value': self.cjamount
})
random.shuffle(self.utxo_tx)
random.shuffle(self.outputs)
tx = btc.mktx(self.utxo_tx, self.outputs)
jlog.info('obtained tx\n' + pprint.pformat(btc.deserialize(tx)))
self.latest_tx = btc.deserialize(tx)
for index, ins in enumerate(self.latest_tx['ins']):
utxo = ins['outpoint']['hash'] + ':' + str(
ins['outpoint']['index'])
if utxo not in self.input_utxos.keys():
continue
# placeholders required
ins['script'] = 'deadbeef'
self.taker_info_callback("INFO",
"Built tx, sending to counterparties.")
return (True, self.maker_utxo_data.keys(), tx)
def address_valid_somewhere(addr):
for x in ["bitcoin", "bitcoin/testnet", "bitcoin/regtest"]:
btc.select_chain_params(x)
if validate_address(addr)[0]:
return True
return False
