def add_tx_notify(self,
txd,
unconfirmfun,
confirmfun,
spentfun,
notifyaddr,
timeoutfun=None):
if not self.notifythread:
self.notifythread = BitcoinCoreNotifyThread(self)
self.notifythread.start()
one_addr_imported = False
for outs in txd['outs']:
addr = btc.script_to_address(outs['script'], get_p2pk_vbyte())
if self.rpc('getaccount', [addr]) != '':
one_addr_imported = True
break
if not one_addr_imported:
self.rpc('importaddress', [notifyaddr, 'joinmarket-notify', False])
tx_output_set = set([(sv['script'], sv['value'])
for sv in txd['outs']])
self.txnotify_fun.append(
(btc.txhash(btc.serialize(txd)), tx_output_set, unconfirmfun,
confirmfun, spentfun, timeoutfun, False))
#create unconfirm timeout here, create confirm timeout in the other thread
if timeoutfun:
threading.Timer(cs_single().config.getint('TIMEOUT',
'unconfirm_timeout_sec'),
bitcoincore_timeout_callback,
args=(False, tx_output_set, self.txnotify_fun,
timeoutfun)).start()
def add_new_utxos(self, tx, txid):
added_utxos = {}
for index, outs in enumerate(tx['outs']):
addr = btc.script_to_address(outs['script'], get_p2pk_vbyte())
if addr not in self.addr_cache:
continue
addrdict = {'address': addr, 'value': outs['value']}
utxo = txid + ':' + str(index)
added_utxos[utxo] = addrdict
self.unspent[utxo] = addrdict
log.debug('added utxos, wallet now is \n' +
pprint.pformat(self.get_utxos_by_mixdepth()))
return added_utxos
def add_tx_notify(self,
txd,
unconfirmfun,
confirmfun,
notifyaddr,
wallet_name=None,
timeoutfun=None,
spentfun=None,
txid_flag=True,
n=0,
c=1,
vb=None):
"""Given a deserialized transaction txd,
callback functions for broadcast and confirmation of the transaction,
an address to import, and a callback function for timeout, set up
a polling loop to check for events on the transaction. Also optionally set
to trigger "confirmed" callback on number of confirmations c. Also checks
for spending (if spentfun is not None) of the outpoint n.
If txid_flag is True, we create a watcher loop on the txid (hence only
really usable in a segwit context, and only on fully formed transactions),
else we create a watcher loop on the output set of the transaction (taken
from the outs field of the txd).
"""
if not vb:
vb = get_p2pk_vbyte()
if isinstance(self, BitcoinCoreInterface) or isinstance(
self, RegtestBitcoinCoreInterface):
#This code ensures that a walletnotify is triggered, by
#ensuring that at least one of the output addresses is
#imported into the wallet (note the sweep special case, where
#none of the output addresses belong to me).
one_addr_imported = False
for outs in txd['outs']:
addr = btc.script_to_address(outs['script'], vb)
if self.rpc('getaccount', [addr]) != '':
one_addr_imported = True
break
if not one_addr_imported:
self.rpc('importaddress',
[notifyaddr, 'joinmarket-notify', False])
#Warning! In case of txid_flag false, this is *not* a valid txid,
#but only a hash of an incomplete transaction serialization.
txid = btc.txhash(btc.serialize(txd))
if not txid_flag:
tx_output_set = set([(sv['script'], sv['value'])
for sv in txd['outs']])
loop = task.LoopingCall(self.outputs_watcher, wallet_name,
notifyaddr, tx_output_set, unconfirmfun,
confirmfun, timeoutfun)
log.debug("Created watcher loop for address: " + notifyaddr)
loopkey = notifyaddr
else:
loop = task.LoopingCall(self.tx_watcher, txd, unconfirmfun,
confirmfun, spentfun, c, n)
log.debug("Created watcher loop for txid: " + txid)
loopkey = txid
self.tx_watcher_loops[loopkey] = [loop, False, False, False]
#Hardcoded polling interval, but in any case it can be very short.
loop.start(5.0)
#Give up on un-broadcast transactions and broadcast but not confirmed
#transactions as per settings in the config.
reactor.callLater(
float(jm_single().config.get("TIMEOUT", "unconfirm_timeout_sec")),
self.tx_network_timeout, loopkey)
confirm_timeout_sec = int(jm_single().config.get(
"TIMEOUT", "confirm_timeout_hours")) * 3600
reactor.callLater(confirm_timeout_sec, self.tx_timeout, txd, loopkey,
timeoutfun)
def sync_addresses(self, wallet, restart_cb=None):
from jmclient.wallet import BitcoinCoreWallet
if isinstance(wallet, BitcoinCoreWallet):
return
log.debug('requesting detailed wallet history')
wallet_name = self.get_wallet_name(wallet)
#TODO It is worth considering making this user configurable:
addr_req_count = 20
wallet_addr_list = []
for mix_depth in range(wallet.max_mix_depth):
for forchange in [0, 1]:
#If we have an index-cache available, we can use it
#to decide how much to import (note that this list
#*always* starts from index 0 on each branch).
#In cases where the Bitcoin Core instance is fresh,
#this will allow the entire import+rescan to occur
#in 2 steps only.
if wallet.index_cache != [[0, 0]] * wallet.max_mix_depth:
#Need to request N*addr_req_count where N is least s.t.
#N*addr_req_count > index_cache val. This is so that the batching
#process in the main loop *always* has already imported enough
#addresses to complete.
req_count = int(wallet.index_cache[mix_depth][forchange] /
addr_req_count) + 1
req_count *= addr_req_count
else:
#If we have *nothing* - no index_cache, and no info
#in Core wallet (imports), we revert to a batching mode
#with a default size.
#In this scenario it could require several restarts *and*
#rescans; perhaps user should set addr_req_count high
#(see above TODO)
req_count = addr_req_count
wallet_addr_list += [
wallet.get_new_addr(mix_depth, forchange)
for _ in range(req_count)
]
#Indices are reset here so that the next algorithm step starts
#from the beginning of each branch
wallet.index[mix_depth][forchange] = 0
# makes more sense to add these in an account called "joinmarket-imported" but its much
# simpler to add to the same account here
for privkey_list in wallet.imported_privkeys.values():
for privkey in privkey_list:
imported_addr = btc.privtoaddr(privkey,
magicbyte=get_p2pk_vbyte())
wallet_addr_list.append(imported_addr)
imported_addr_list = self.rpc('getaddressesbyaccount', [wallet_name])
if not set(wallet_addr_list).issubset(set(imported_addr_list)):
self.add_watchonly_addresses(wallet_addr_list, wallet_name,
restart_cb)
return
buf = self.rpc('listtransactions', [wallet_name, 1000, 0, True])
txs = buf
# If the buffer's full, check for more, until it ain't
while len(buf) == 1000:
buf = self.rpc(
'listtransactions',
[wallet_name, 1000, len(txs), True])
txs += buf
# TODO check whether used_addr_list can be a set, may be faster (if
# its a hashset) and allows using issubset() here and setdiff() for
# finding which addresses need importing
# TODO also check the fastest way to build up python lists, i suspect
# using += is slow
used_addr_list = [
tx['address'] for tx in txs if tx['category'] == 'receive'
]
too_few_addr_mix_change = []
for mix_depth in range(wallet.max_mix_depth):
for forchange in [0, 1]:
unused_addr_count = 0
last_used_addr = ''
breakloop = False
while not breakloop:
if unused_addr_count >= wallet.gaplimit and \
is_index_ahead_of_cache(wallet, mix_depth,
forchange):
break
mix_change_addrs = [
wallet.get_new_addr(mix_depth, forchange)
for _ in range(addr_req_count)
]
for mc_addr in mix_change_addrs:
if mc_addr not in imported_addr_list:
too_few_addr_mix_change.append(
(mix_depth, forchange))
breakloop = True
break
if mc_addr in used_addr_list:
last_used_addr = mc_addr
unused_addr_count = 0
else:
unused_addr_count += 1
#index setting here depends on whether we broke out of the loop
#early; if we did, it means we need to prepare the index
#at the level of the last used address or zero so as to not
#miss any imports in add_watchonly_addresses.
#If we didn't, we need to respect the index_cache to avoid
#potential address reuse.
if breakloop:
if last_used_addr == '':
wallet.index[mix_depth][forchange] = 0
else:
wallet.index[mix_depth][forchange] = \
wallet.addr_cache[last_used_addr][2] + 1
else:
if last_used_addr == '':
next_avail_idx = max(
[wallet.index_cache[mix_depth][forchange], 0])
else:
next_avail_idx = max([
wallet.addr_cache[last_used_addr][2] + 1,
wallet.index_cache[mix_depth][forchange]
])
wallet.index[mix_depth][forchange] = next_avail_idx
wallet_addr_list = []
if len(too_few_addr_mix_change) > 0:
indices = [
wallet.index[mc[0]][mc[1]] for mc in too_few_addr_mix_change
]
log.debug('too few addresses in ' + str(too_few_addr_mix_change) +
' at ' + str(indices))
for mix_depth, forchange in too_few_addr_mix_change:
wallet_addr_list += [
wallet.get_new_addr(mix_depth, forchange)
for _ in range(addr_req_count * 3)
]
self.add_watchonly_addresses(wallet_addr_list, wallet_name,
restart_cb)
return
self.wallet_synced = True
def test_net_byte():
load_test_config()
assert struct.unpack(b'B', get_p2pk_vbyte())[0] == 0x6f
assert struct.unpack(b'B', get_p2sh_vbyte())[0] == 196
def receive_utxos(self, ioauth_data):
"""Triggered when the daemon returns utxo data from
makers who responded; this is the completion of phase 1
of the protocol
"""
if self.aborted:
return (False, "User aborted")
rejected_counterparties = []
#Enough data, but need to authorize against the btc pubkey first.
for nick, nickdata in ioauth_data.iteritems():
utxo_list, auth_pub, cj_addr, change_addr, btc_sig, maker_pk = nickdata
if not self.auth_counterparty(btc_sig, auth_pub, maker_pk):
jlog.debug(
"Counterparty encryption verification failed, aborting")
#This counterparty must be rejected
rejected_counterparties.append(nick)
for rc in rejected_counterparties:
del ioauth_data[rc]
self.maker_utxo_data = {}
for nick, nickdata in ioauth_data.iteritems():
utxo_list, auth_pub, cj_addr, change_addr, btc_sig, maker_pk = nickdata
self.utxos[nick] = utxo_list
utxo_data = jm_single().bc_interface.query_utxo_set(
self.utxos[nick])
if None in utxo_data:
jlog.debug(('ERROR outputs unconfirmed or already spent. '
'utxo_data={}').format(pprint.pformat(utxo_data)))
# when internal reviewing of makers is created, add it here to
# immediately quit; currently, the timeout thread suffices.
continue
#Complete maker authorization:
#Extract the address fields from the utxos
#Construct the Bitcoin address for the auth_pub field
#Ensure that at least one address from utxos corresponds.
input_addresses = [d['address'] for d in utxo_data]
auth_address = btc.pubkey_to_address(auth_pub, get_p2pk_vbyte())
if not auth_address in input_addresses:
jlog.warn("ERROR maker's (" + nick + ")"
" authorising pubkey is not included "
"in the transaction: " + str(auth_address))
#this will not be added to the transaction, so we will have
#to recheck if we have enough
continue
total_input = sum([d['value'] for d in utxo_data])
real_cjfee = calc_cj_fee(self.orderbook[nick]['ordertype'],
self.orderbook[nick]['cjfee'],
self.cjamount)
change_amount = (total_input - self.cjamount -
self.orderbook[nick]['txfee'] + real_cjfee)
# certain malicious and/or incompetent liquidity providers send
# inputs totalling less than the coinjoin amount! this leads to
# a change output of zero satoshis; this counterparty must be removed.
if change_amount < jm_single().DUST_THRESHOLD:
fmt = ('ERROR counterparty requires sub-dust change. nick={}'
'totalin={:d} cjamount={:d} change={:d}').format
jlog.debug(fmt(nick, total_input, self.cjamount,
change_amount))
jlog.warn("Invalid change, too small, nick= " + nick)
continue
self.outputs.append({
'address': change_addr,
'value': change_amount
})
fmt = ('fee breakdown for {} totalin={:d} '
'cjamount={:d} txfee={:d} realcjfee={:d}').format
jlog.debug(
fmt(nick, total_input, self.cjamount,
self.orderbook[nick]['txfee'], real_cjfee))
self.outputs.append({'address': cj_addr, 'value': self.cjamount})
self.cjfee_total += real_cjfee
self.maker_txfee_contributions += self.orderbook[nick]['txfee']
self.maker_utxo_data[nick] = utxo_data
#Apply business logic of how many counterparties are enough:
if len(self.maker_utxo_data.keys()) < jm_single().config.getint(
"POLICY", "minimum_makers"):
self.taker_info_callback("INFO",
"Not enough counterparties, aborting.")
return (False,
"Not enough counterparties responded to fill, giving up")
self.taker_info_callback("INFO", "Got all parts, enough to build a tx")
self.nonrespondants = list(self.maker_utxo_data.keys())
my_total_in = sum(
[va['value'] for u, va in self.input_utxos.iteritems()])
if self.my_change_addr:
#Estimate fee per choice of next/3/6 blocks targetting.
estimated_fee = estimate_tx_fee(len(sum(self.utxos.values(), [])),
len(self.outputs) + 2)
jlog.info("Based on initial guess: " + str(self.total_txfee) +
", we estimated a miner fee of: " + str(estimated_fee))
#reset total
self.total_txfee = estimated_fee
my_txfee = max(self.total_txfee - self.maker_txfee_contributions, 0)
my_change_value = (my_total_in - self.cjamount - self.cjfee_total -
my_txfee)
#Since we could not predict the maker's inputs, we may end up needing
#too much such that the change value is negative or small. Note that
#we have tried to avoid this based on over-estimating the needed amount
#in SendPayment.create_tx(), but it is still a possibility if one maker
#uses a *lot* of inputs.
if self.my_change_addr and my_change_value <= 0:
raise ValueError("Calculated transaction fee of: " +
str(self.total_txfee) +
" is too large for our inputs;Please try again.")
elif self.my_change_addr and my_change_value <= jm_single(
).BITCOIN_DUST_THRESHOLD:
jlog.info("Dynamically calculated change lower than dust: " +
str(my_change_value) + "; dropping.")
self.my_change_addr = None
my_change_value = 0
jlog.info(
'fee breakdown for me totalin=%d my_txfee=%d makers_txfee=%d cjfee_total=%d => changevalue=%d'
% (my_total_in, my_txfee, self.maker_txfee_contributions,
self.cjfee_total, my_change_value))
if self.my_change_addr is None:
if my_change_value != 0 and abs(my_change_value) != 1:
# seems you wont always get exactly zero because of integer
# rounding so 1 satoshi extra or fewer being spent as miner
# fees is acceptable
jlog.debug(('WARNING CHANGE NOT BEING '
'USED\nCHANGEVALUE = {}').format(my_change_value))
else:
self.outputs.append({
'address': self.my_change_addr,
'value': my_change_value
})
self.utxo_tx = [
dict([('output', u)]) for u in sum(self.utxos.values(), [])
]
self.outputs.append({
'address': self.coinjoin_address(),
'value': self.cjamount
})
random.shuffle(self.utxo_tx)
random.shuffle(self.outputs)
tx = btc.mktx(self.utxo_tx, self.outputs)
jlog.debug('obtained tx\n' + pprint.pformat(btc.deserialize(tx)))
self.latest_tx = btc.deserialize(tx)
for index, ins in enumerate(self.latest_tx['ins']):
utxo = ins['outpoint']['hash'] + ':' + str(
ins['outpoint']['index'])
if utxo not in self.input_utxos.keys():
continue
# placeholders required
ins['script'] = 'deadbeef'
self.taker_info_callback("INFO",
"Built tx, sending to counterparties.")
return (True, self.maker_utxo_data.keys(), tx)
def get_key_from_addr(self, addr):
self.ensure_wallet_unlocked()
wifkey = jm_single().bc_interface.rpc('dumpprivkey', [addr])
return btc.from_wif_privkey(wifkey, vbyte=get_p2pk_vbyte())
def get_vbyte(self):
return get_p2pk_vbyte()
def get_addr(self, mixing_depth, forchange, i):
return btc.privtoaddr(self.get_key(mixing_depth, forchange, i),
magicbyte=get_p2pk_vbyte())
def read_wallet_file_data(self, filename, pwd=None, wallet_dir=None):
self.path = None
wallet_dir = wallet_dir if wallet_dir else 'wallets'
self.index_cache = [[0, 0]] * self.max_mix_depth
path = os.path.join(wallet_dir, filename)
if not os.path.isfile(path):
if get_network() == 'testnet':
log.debug('filename interpreted as seed, only available in '
'testnet because this probably has lower entropy')
return "FAKESEED" + filename
else:
raise IOError('wallet file not found')
if not pwd:
log.info("Password required for non-testnet seed wallet")
return None
self.path = path
fd = open(path, 'r')
walletfile = fd.read()
fd.close()
walletdata = json.loads(walletfile)
if walletdata['network'] != get_network():
raise ValueError('wallet network(%s) does not match '
'joinmarket configured network(%s)' %
(walletdata['network'], get_network()))
if 'index_cache' in walletdata:
self.index_cache = walletdata['index_cache']
if self.max_mix_depth > len(self.index_cache):
#This can happen e.g. in tumbler when we need more mixdepths
#than currently exist. Since we have no info for those extra
#depths, we must default to (0,0) (but sync should find used
#adddresses).
self.index_cache += [[0, 0]] * (self.max_mix_depth -
len(self.index_cache))
password_key = btc.bin_dbl_sha256(pwd)
if 'encrypted_seed' in walletdata: #accept old field name
encrypted_entropy = walletdata['encrypted_seed']
elif 'encrypted_entropy' in walletdata:
encrypted_entropy = walletdata['encrypted_entropy']
try:
decrypted_entropy = decryptData(
password_key, encrypted_entropy.decode('hex')).encode('hex')
# there is a small probability of getting a valid PKCS7
# padding by chance from a wrong password; sanity check the
# seed length
if len(decrypted_entropy) != 32:
raise ValueError
except ValueError:
log.info('Incorrect password')
return None
if 'encrypted_mnemonic_extension' in walletdata:
try:
cleartext = decryptData(
password_key,
walletdata['encrypted_mnemonic_extension'].decode('hex'))
#theres a small chance of not getting a ValueError from the wrong
# password so also check the sum
if cleartext[-9] != '\xff':
raise ValueError
chunks = cleartext.split('\xff')
if len(chunks) < 3 or cleartext[-8:] != btc.dbl_sha256(
chunks[1])[:8]:
raise ValueError
mnemonic_extension = chunks[1]
except ValueError:
log.info('incorrect password')
return None
else:
mnemonic_extension = None
if self.storepassword:
self.password_key = password_key
self.walletdata = walletdata
if 'imported_keys' in walletdata:
for epk_m in walletdata['imported_keys']:
privkey = decryptData(
password_key,
epk_m['encrypted_privkey'].decode('hex')).encode('hex')
#Imported keys are stored as 32 byte strings only, so the
#second version below is sufficient, really.
if len(privkey) != 64:
raise Exception(
"Unexpected privkey format; already compressed?:" +
privkey)
privkey += "01"
if epk_m['mixdepth'] not in self.imported_privkeys:
self.imported_privkeys[epk_m['mixdepth']] = []
self.addr_cache[btc.privtoaddr(
privkey, magicbyte=get_p2pk_vbyte())] = (
epk_m['mixdepth'], -1,
len(self.imported_privkeys[epk_m['mixdepth']]))
self.imported_privkeys[epk_m['mixdepth']].append(privkey)
if mnemonic_extension:
return (decrypted_entropy, mnemonic_extension)
else:
return decrypted_entropy
def script_to_address(self, script):
"""Return the address for a given output script,
which will be p2pkh for the default Wallet object,
and reading the correct network byte from the config.
"""
return btc.script_to_address(script, get_p2pk_vbyte())
def test_net_byte():
load_program_config()
assert get_p2pk_vbyte() == 0x6f
assert get_p2sh_vbyte() == 196
def get_vbyte():
return get_p2pk_vbyte()
