def on_tx_confirmed(self, cjorder, confirmations, txid): to_announce = [] for i, out in enumerate(cjorder.tx['outs']): addr = btc.script_to_address(out['script'], get_p2pk_vbyte()) if addr == cjorder.change_addr: neworder = { 'oid': self.get_next_oid(), 'ordertype': 'absorder', 'minsize': 12000, 'maxsize': out['value'], 'txfee': 10000, 'cjfee': 100000, 'utxo': txid + ':' + str(i) } to_announce.append(neworder) if addr == cjorder.cj_addr: neworder = { 'oid': self.get_next_oid(), 'ordertype': 'absorder', 'minsize': 12000, 'maxsize': out['value'], 'txfee': 10000, 'cjfee': 100000, 'utxo': txid + ':' + str(i) } to_announce.append(neworder) return [], to_announce
def add_tx_notify(self, txd, unconfirmfun, confirmfun, notifyaddr, timeoutfun=None): if not self.notifythread: self.notifythread = BitcoinCoreNotifyThread(self) self.notifythread.start() one_addr_imported = False for outs in txd['outs']: addr = btc.script_to_address(outs['script'], get_p2pk_vbyte()) if self.rpc('getaccount', [addr]) != '': one_addr_imported = True break if not one_addr_imported: self.rpc('importaddress', [notifyaddr, 'joinmarket-notify', False]) tx_output_set = set([(sv['script'], sv['value']) for sv in txd['outs']]) self.txnotify_fun.append( (tx_output_set, unconfirmfun, confirmfun, timeoutfun, False)) #create unconfirm timeout here, create confirm timeout in the other thread if timeoutfun: threading.Timer(jm_single().config.getint('TIMEOUT', 'unconfirm_timeout_sec'), bitcoincore_timeout_callback, args=(False, tx_output_set, self.txnotify_fun, timeoutfun)).start()
def add_tx_notify(self, txd, unconfirmfun, confirmfun, notifyaddr, timeoutfun=None): if not self.notifythread: self.notifythread = BitcoinCoreNotifyThread(self) self.notifythread.start() one_addr_imported = False for outs in txd['outs']: addr = btc.script_to_address(outs['script'], get_p2pk_vbyte()) if self.rpc('getaccount', [addr]) != '': one_addr_imported = True break if not one_addr_imported: self.rpc('importaddress', [notifyaddr, 'joinmarket-notify', False]) tx_output_set = set([(sv['script'], sv['value']) for sv in txd['outs']]) self.txnotify_fun.append((tx_output_set, unconfirmfun, confirmfun, timeoutfun, False)) #create unconfirm timeout here, create confirm timeout in the other thread if timeoutfun: threading.Timer(jm_single().config.getint('TIMEOUT', 'unconfirm_timeout_sec'), bitcoincore_timeout_callback, args=(False, tx_output_set, self.txnotify_fun, timeoutfun)).start()
def verify_unsigned_tx(self, txd): tx_utxo_set = set(ins['outpoint']['hash'] + ':' + str( ins['outpoint']['index']) for ins in txd['ins']) # complete authentication: check the tx input uses the authing pubkey input_utxo_data = jm_single().bc_interface.query_utxo_set( list(tx_utxo_set)) if None in input_utxo_data: return False, 'some utxos already spent or not confirmed yet' input_addresses = [u['address'] for u in input_utxo_data] if btc.pubtoaddr( self.i_utxo_pubkey, get_p2pk_vbyte()) not in input_addresses: return False, "authenticating bitcoin address is not contained" my_utxo_set = set(self.utxos.keys()) if not tx_utxo_set.issuperset(my_utxo_set): return False, 'my utxos are not contained' my_total_in = sum([va['value'] for va in self.utxos.values()]) self.real_cjfee = calc_cj_fee( self.ordertype, self.cjfee, self.cj_amount) expected_change_value = ( my_total_in - self.cj_amount - self.txfee + self.real_cjfee) log.debug('potentially earned = {}'.format( self.real_cjfee - self.txfee)) log.debug('mycjaddr, mychange = {}, {}'.format( self.cj_addr, self.change_addr)) times_seen_cj_addr = 0 times_seen_change_addr = 0 for outs in txd['outs']: addr = btc.script_to_address(outs['script'], get_p2pk_vbyte()) if addr == self.cj_addr: times_seen_cj_addr += 1 if outs['value'] != self.cj_amount: return False, 'Wrong cj_amount. I expect ' + str( self.cj_amount) if addr == self.change_addr: times_seen_change_addr += 1 if outs['value'] != expected_change_value: return False, 'wrong change, i expect ' + str( expected_change_value) if times_seen_cj_addr != 1 or times_seen_change_addr != 1: fmt = ('cj or change addr not in tx ' 'outputs once, #cjaddr={}, #chaddr={}').format return False, (fmt(times_seen_cj_addr, times_seen_change_addr)) return True, None
def verify_unsigned_tx(self, txd): tx_utxo_set = set(ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) for ins in txd['ins']) # complete authentication: check the tx input uses the authing pubkey input_utxo_data = jm_single().bc_interface.query_utxo_set( list(tx_utxo_set)) if None in input_utxo_data: return False, 'some utxos already spent or not confirmed yet' input_addresses = [u['address'] for u in input_utxo_data] if btc.pubtoaddr(self.i_utxo_pubkey, get_p2pk_vbyte()) not in input_addresses: return False, "authenticating bitcoin address is not contained" my_utxo_set = set(self.utxos.keys()) if not tx_utxo_set.issuperset(my_utxo_set): return False, 'my utxos are not contained' my_total_in = sum([va['value'] for va in self.utxos.values()]) self.real_cjfee = calc_cj_fee(self.ordertype, self.cjfee, self.cj_amount) expected_change_value = (my_total_in - self.cj_amount - self.txfee + self.real_cjfee) log.debug('potentially earned = {}'.format(self.real_cjfee - self.txfee)) log.debug('mycjaddr, mychange = {}, {}'.format(self.cj_addr, self.change_addr)) times_seen_cj_addr = 0 times_seen_change_addr = 0 for outs in txd['outs']: addr = btc.script_to_address(outs['script'], get_p2pk_vbyte()) if addr == self.cj_addr: times_seen_cj_addr += 1 if outs['value'] != self.cj_amount: return False, 'Wrong cj_amount. I expect ' + str( self.cj_amount) if addr == self.change_addr: times_seen_change_addr += 1 if outs['value'] != expected_change_value: return False, 'wrong change, i expect ' + str( expected_change_value) if times_seen_cj_addr != 1 or times_seen_change_addr != 1: fmt = ('cj or change addr not in tx ' 'outputs once, #cjaddr={}, #chaddr={}').format return False, (fmt(times_seen_cj_addr, times_seen_change_addr)) return True, None
def read_wallet_file_data(self, filename): self.path = None self.index_cache = [[0, 0]] * self.max_mix_depth path = os.path.join('wallets', filename) if not os.path.isfile(path): if get_network() == 'testnet': log.debug('filename interpreted as seed, only available in ' 'testnet because this probably has lower entropy') return filename else: raise IOError('wallet file not found') self.path = path fd = open(path, 'r') walletfile = fd.read() fd.close() walletdata = json.loads(walletfile) if walletdata['network'] != get_network(): print ('wallet network(%s) does not match ' 'joinmarket configured network(%s)' % ( walletdata['network'], get_network())) sys.exit(0) if 'index_cache' in walletdata: self.index_cache = walletdata['index_cache'] decrypted = False while not decrypted: password = getpass('Enter wallet decryption passphrase: ') password_key = btc.bin_dbl_sha256(password) encrypted_seed = walletdata['encrypted_seed'] try: decrypted_seed = decryptData( password_key, encrypted_seed.decode('hex')).encode('hex') # there is a small probability of getting a valid PKCS7 # padding by chance from a wrong password; sanity check the # seed length if len(decrypted_seed) == 32: decrypted = True else: raise ValueError except ValueError: print('Incorrect password') decrypted = False if self.storepassword: self.password_key = password_key self.walletdata = walletdata if 'imported_keys' in walletdata: for epk_m in walletdata['imported_keys']: privkey = decryptData( password_key, epk_m['encrypted_privkey'].decode( 'hex')).encode('hex') privkey = btc.encode_privkey(privkey, 'hex_compressed') if epk_m['mixdepth'] not in self.imported_privkeys: self.imported_privkeys[epk_m['mixdepth']] = [] self.addr_cache[btc.privtoaddr( privkey, get_p2pk_vbyte())] = (epk_m['mixdepth'], -1, len(self.imported_privkeys[epk_m['mixdepth']])) self.imported_privkeys[epk_m['mixdepth']].append(privkey) return decrypted_seed
def auth_counterparty(self, nick, cr): #deserialize the commitment revelation cr_dict = btc.PoDLE.deserialize_revelation(cr) #check the validity of the proof of discrete log equivalence tries = jm_single().config.getint("POLICY", "taker_utxo_retries") def reject(msg): log.info("Counterparty commitment not accepted, reason: " + msg) return False if not btc.verify_podle(cr_dict['P'], cr_dict['P2'], cr_dict['sig'], cr_dict['e'], self.maker.commit, index_range=range(tries)): reason = "verify_podle failed" return reject(reason) #finally, check that the proffered utxo is real, old enough, large enough, #and corresponds to the pubkey res = jm_single().bc_interface.query_utxo_set([cr_dict['utxo']], includeconf=True) if len(res) != 1 or not res[0]: reason = "authorizing utxo is not valid" return reject(reason) age = jm_single().config.getint("POLICY", "taker_utxo_age") if res[0]['confirms'] < age: reason = "commitment utxo not old enough: " + str(res[0]['confirms']) return reject(reason) reqd_amt = int(self.cj_amount * jm_single().config.getint( "POLICY", "taker_utxo_amtpercent") / 100.0) if res[0]['value'] < reqd_amt: reason = "commitment utxo too small: " + str(res[0]['value']) return reject(reason) if res[0]['address'] != btc.pubkey_to_address(cr_dict['P'], get_p2pk_vbyte()): reason = "Invalid podle pubkey: " + str(cr_dict['P']) return reject(reason) # authorisation of taker passed # Send auth request to taker # Need to choose an input utxo pubkey to sign with # (no longer using the coinjoin pubkey from 0.2.0) # Just choose the first utxo in self.utxos and retrieve key from wallet. auth_address = self.utxos[self.utxos.keys()[0]]['address'] auth_key = self.maker.wallet.get_key_from_addr(auth_address) auth_pub = btc.privtopub(auth_key) btc_sig = btc.ecdsa_sign(self.kp.hex_pk(), auth_key) self.maker.msgchan.send_ioauth(nick, self.utxos.keys(), auth_pub, self.cj_addr, self.change_addr, btc_sig) #In case of *blacklisted (ie already used) commitments, we already #broadcasted them on receipt; in case of valid, and now used commitments, #we broadcast them here, and not early - to avoid accidentally #blacklisting commitments that are broadcast between makers in real time #for the same transaction. self.maker.transfer_commitment(self.maker.commit) #now persist the fact that the commitment is actually used. check_utxo_blacklist(self.maker.commit, persist=True) return True
def __init__(self, blockr_domain, txd, unconfirmfun, confirmfun): threading.Thread.__init__(self) self.daemon = True self.blockr_domain = blockr_domain self.unconfirmfun = unconfirmfun self.confirmfun = confirmfun self.tx_output_set = set([(sv['script'], sv['value']) for sv in txd['outs']]) self.output_addresses = [ btc.script_to_address(scrval[0], get_p2pk_vbyte()) for scrval in self.tx_output_set] log.debug('txoutset=' + pprint.pformat(self.tx_output_set)) log.debug('outaddrs=' + ','.join(self.output_addresses))
def add_new_utxos(self, tx, txid): added_utxos = {} for index, outs in enumerate(tx['outs']): addr = btc.script_to_address(outs['script'], get_p2pk_vbyte()) if addr not in self.addr_cache: continue addrdict = {'address': addr, 'value': outs['value']} utxo = txid + ':' + str(index) added_utxos[utxo] = addrdict self.unspent[utxo] = addrdict log.debug('added utxos, wallet now is \n' + pprint.pformat( self.get_utxos_by_mixdepth())) return added_utxos
def add_tx_notify(self, txd, unconfirmfun, confirmfun, notifyaddr): if not self.notifythread: self.notifythread = BitcoinCoreNotifyThread(self) self.notifythread.start() one_addr_imported = False for outs in txd['outs']: addr = btc.script_to_address(outs['script'], get_p2pk_vbyte()) if self.rpc('getaccount', [addr]) != '': one_addr_imported = True break if not one_addr_imported: self.rpc('importaddress', [notifyaddr, 'joinmarket-notify', False]) tx_output_set = set([(sv['script'], sv['value']) for sv in txd['outs']]) self.txnotify_fun.append((tx_output_set, unconfirmfun, confirmfun))
def add_tx_notify(self, txd, unconfirmfun, confirmfun, notifyaddr): if not self.notifythread: self.notifythread = BitcoinCoreNotifyThread(self) self.notifythread.start() self.notifythread.running.wait() one_addr_imported = False for outs in txd['outs']: addr = btc.script_to_address(outs['script'], get_p2pk_vbyte()) if self.rpc('getaccount', [addr]) != '': one_addr_imported = True break if not one_addr_imported: self.rpc('importaddress', [notifyaddr, 'joinmarket-notify', False]) tx_output_set = set([(sv['script'], sv['value']) for sv in txd['outs']]) self.txnotify_fun.append((tx_output_set, unconfirmfun, confirmfun))
def donation_address(cjtx): from bitcoin.main import multiply, G, deterministic_generate_k, add_pubkeys reusable_donation_pubkey = ('02be838257fbfddabaea03afbb9f16e852' '9dfe2de921260a5c46036d97b5eacf2a') donation_utxo_data = cjtx.input_utxos.iteritems().next() global donation_utxo donation_utxo = donation_utxo_data[0] privkey = cjtx.wallet.get_key_from_addr(donation_utxo_data[1]['address']) # tx without our inputs and outputs tx = btc.mktx(cjtx.utxo_tx, cjtx.outputs) msghash = btc.bin_txhash(tx, btc.SIGHASH_ALL) # generate unpredictable k global sign_k sign_k = deterministic_generate_k(msghash, privkey) c = btc.sha256(multiply(reusable_donation_pubkey, sign_k)) sender_pubkey = add_pubkeys(reusable_donation_pubkey, multiply(G, c)) sender_address = btc.pubtoaddr(sender_pubkey, get_p2pk_vbyte()) log.debug('sending coins to ' + sender_address) return sender_address
def donation_address(cjtx): from bitcoin.main import multiply, G, deterministic_generate_k, add_pubkeys reusable_donation_pubkey = ('02be838257fbfddabaea03afbb9f16e852' '9dfe2de921260a5c46036d97b5eacf2a') donation_utxo_data = cjtx.input_utxos.iteritems().next() global donation_utxo donation_utxo = donation_utxo_data[0] privkey = cjtx.wallet.get_key_from_addr(donation_utxo_data[1]['address']) # tx without our inputs and outputs tx = btc.mktx(cjtx.utxo_tx, cjtx.outputs) msghash = btc.bin_txhash(tx, btc.SIGHASH_ALL) # generate unpredictable k global sign_k sign_k = deterministic_generate_k(msghash, privkey) c = btc.sha256(multiply(reusable_donation_pubkey, sign_k)) sender_pubkey = add_pubkeys( reusable_donation_pubkey, multiply( G, c)) sender_address = btc.pubtoaddr(sender_pubkey, get_p2pk_vbyte()) log.debug('sending coins to ' + sender_address) return sender_address
def on_tx_confirmed(self, cjorder, confirmations, txid): to_announce = [] for i, out in enumerate(cjorder.tx['outs']): addr = btc.script_to_address(out['script'], get_p2pk_vbyte()) if addr == cjorder.change_addr: neworder = {'oid': self.get_next_oid(), 'ordertype': 'absoffer', 'minsize': 12000, 'maxsize': out['value'], 'txfee': 10000, 'cjfee': 100000, 'utxo': txid + ':' + str(i)} to_announce.append(neworder) if addr == cjorder.cj_addr: neworder = {'oid': self.get_next_oid(), 'ordertype': 'absoffer', 'minsize': 12000, 'maxsize': out['value'], 'txfee': 10000, 'cjfee': 100000, 'utxo': txid + ':' + str(i)} to_announce.append(neworder) return [], to_announce
def verify_unsigned_tx(self, txd): tx_utxo_set = set(ins['outpoint']['hash'] + ':' + str( ins['outpoint']['index']) for ins in txd['ins']) my_utxo_set = set(self.utxos.keys()) if not tx_utxo_set.issuperset(my_utxo_set): return False, 'my utxos are not contained' my_total_in = sum([va['value'] for va in self.utxos.values()]) self.real_cjfee = calc_cj_fee( self.ordertype, self.cjfee, self.cj_amount) expected_change_value = ( my_total_in - self.cj_amount - self.txfee + self.real_cjfee) log.info('potentially earned = {}'.format( self.real_cjfee - self.txfee)) log.debug('mycjaddr, mychange = {}, {}'.format( self.cj_addr, self.change_addr)) times_seen_cj_addr = 0 times_seen_change_addr = 0 for outs in txd['outs']: addr = btc.script_to_address(outs['script'], get_p2pk_vbyte()) if addr == self.cj_addr: times_seen_cj_addr += 1 if outs['value'] != self.cj_amount: return False, 'Wrong cj_amount. I expect ' + str( self.cj_amount) if addr == self.change_addr: times_seen_change_addr += 1 if outs['value'] != expected_change_value: return False, 'wrong change, i expect ' + str( expected_change_value) if times_seen_cj_addr != 1 or times_seen_change_addr != 1: fmt = ('cj or change addr not in tx ' 'outputs once, #cjaddr={}, #chaddr={}').format return False, (fmt(times_seen_cj_addr, times_seen_change_addr)) return True, None
def verify_unsigned_tx(self, txd): tx_utxo_set = set(ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) for ins in txd['ins']) my_utxo_set = set(self.utxos.keys()) if not tx_utxo_set.issuperset(my_utxo_set): return False, 'my utxos are not contained' my_total_in = sum([va['value'] for va in self.utxos.values()]) self.real_cjfee = calc_cj_fee(self.ordertype, self.cjfee, self.cj_amount) expected_change_value = (my_total_in - self.cj_amount - self.txfee + self.real_cjfee) log.debug('potentially earned = {}'.format(self.real_cjfee - self.txfee)) log.debug('mycjaddr, mychange = {}, {}'.format(self.cj_addr, self.change_addr)) times_seen_cj_addr = 0 times_seen_change_addr = 0 for outs in txd['outs']: addr = btc.script_to_address(outs['script'], get_p2pk_vbyte()) if addr == self.cj_addr: times_seen_cj_addr += 1 if outs['value'] != self.cj_amount: return False, 'Wrong cj_amount. I expect ' + str( self.cj_amount) if addr == self.change_addr: times_seen_change_addr += 1 if outs['value'] != expected_change_value: return False, 'wrong change, i expect ' + str( expected_change_value) if times_seen_cj_addr != 1 or times_seen_change_addr != 1: fmt = ('cj or change addr not in tx ' 'outputs once, #cjaddr={}, #chaddr={}').format return False, (fmt(times_seen_cj_addr, times_seen_change_addr)) return True, None
def get_addr(self, mixing_depth, forchange, i): return btc.privtoaddr( self.get_key(mixing_depth, forchange, i), magicbyte=get_p2pk_vbyte())
def get_key_from_addr(self, addr): self.ensure_wallet_unlocked() wifkey = jm_single().bc_interface.rpc('dumpprivkey', [addr]) return btc.from_wif_privkey(wifkey, vbyte=get_p2pk_vbyte())
def recv_txio(self, nick, utxo_list, auth_pub, cj_addr, change_addr): if nick not in self.nonrespondants: log.debug(('recv_txio => nick={} not in ' 'nonrespondants {}').format(nick, self.nonrespondants)) return self.utxos[nick] = utxo_list utxo_data = jm_single().bc_interface.query_utxo_set(self.utxos[nick]) if None in utxo_data: log.debug(('ERROR outputs unconfirmed or already spent. ' 'utxo_data={}').format(pprint.pformat(utxo_data))) # when internal reviewing of makers is created, add it here to # immediately quit; currently, the timeout thread suffices. return #Complete maker authorization: #Extract the address fields from the utxos #Construct the Bitcoin address for the auth_pub field #Ensure that at least one address from utxos corresponds. input_addresses = [d['address'] for d in utxo_data] auth_address = btc.pubkey_to_address(auth_pub, get_p2pk_vbyte()) if not auth_address in input_addresses: log.debug("ERROR maker's authorising pubkey is not included " "in the transaction: " + str(auth_address)) return total_input = sum([d['value'] for d in utxo_data]) real_cjfee = calc_cj_fee(self.active_orders[nick]['ordertype'], self.active_orders[nick]['cjfee'], self.cj_amount) change_amount = (total_input - self.cj_amount - self.active_orders[nick]['txfee'] + real_cjfee) # certain malicious and/or incompetent liquidity providers send # inputs totalling less than the coinjoin amount! this leads to # a change output of zero satoshis, so the invalid transaction # fails harmlessly; let's fail earlier, with a clear message. if change_amount < jm_single().DUST_THRESHOLD: fmt = ('ERROR counterparty requires sub-dust change. nick={}' 'totalin={:d} cjamount={:d} change={:d}').format log.debug(fmt(nick, total_input, self.cj_amount, change_amount)) return # timeout marks this maker as nonresponsive self.outputs.append({'address': change_addr, 'value': change_amount}) fmt = ('fee breakdown for {} totalin={:d} ' 'cjamount={:d} txfee={:d} realcjfee={:d}').format log.debug( fmt(nick, total_input, self.cj_amount, self.active_orders[nick]['txfee'], real_cjfee)) self.outputs.append({'address': cj_addr, 'value': self.cj_amount}) self.cjfee_total += real_cjfee self.maker_txfee_contributions += self.active_orders[nick]['txfee'] self.nonrespondants.remove(nick) if len(self.nonrespondants) > 0: log.debug('nonrespondants = ' + str(self.nonrespondants)) return log.debug('got all parts, enough to build a tx') self.nonrespondants = list(self.active_orders.keys()) my_total_in = sum( [va['value'] for u, va in self.input_utxos.iteritems()]) if self.my_change_addr: #Estimate fee per choice of next/3/6 blocks targetting. estimated_fee = estimate_tx_fee(len(sum(self.utxos.values(), [])), len(self.outputs) + 2) log.debug("Based on initial guess: " + str(self.total_txfee) + ", we estimated a fee of: " + str(estimated_fee)) #reset total self.total_txfee = estimated_fee my_txfee = max(self.total_txfee - self.maker_txfee_contributions, 0) my_change_value = (my_total_in - self.cj_amount - self.cjfee_total - my_txfee) #Since we could not predict the maker's inputs, we may end up needing #too much such that the change value is negative or small. Note that #we have tried to avoid this based on over-estimating the needed amount #in SendPayment.create_tx(), but it is still a possibility if one maker #uses a *lot* of inputs. if self.my_change_addr and my_change_value <= 0: raise ValueError("Calculated transaction fee of: " + str(self.total_txfee) + " is too large for our inputs;Please try again.") elif self.my_change_addr and my_change_value <= jm_single( ).DUST_THRESHOLD: log.debug("Dynamically calculated change lower than dust: " + str(my_change_value) + "; dropping.") self.my_change_addr = None my_change_value = 0 log.debug( 'fee breakdown for me totalin=%d my_txfee=%d makers_txfee=%d cjfee_total=%d => changevalue=%d' % (my_total_in, my_txfee, self.maker_txfee_contributions, self.cjfee_total, my_change_value)) if self.my_change_addr is None: if my_change_value != 0 and abs(my_change_value) != 1: # seems you wont always get exactly zero because of integer # rounding so 1 satoshi extra or fewer being spent as miner # fees is acceptable log.debug(('WARNING CHANGE NOT BEING ' 'USED\nCHANGEVALUE = {}').format(my_change_value)) else: self.outputs.append({ 'address': self.my_change_addr, 'value': my_change_value }) self.utxo_tx = [ dict([('output', u)]) for u in sum(self.utxos.values(), []) ] self.outputs.append({ 'address': self.coinjoin_address(), 'value': self.cj_amount }) random.shuffle(self.utxo_tx) random.shuffle(self.outputs) tx = btc.mktx(self.utxo_tx, self.outputs) log.debug('obtained tx\n' + pprint.pformat(btc.deserialize(tx))) #Re-calculate a sensible timeout wait based on the throttling #settings and the tx size. #Calculation: Let tx size be S; tx undergoes two b64 expansions, 1.8*S #So we're sending N*1.8*S over the wire, and the #maximum bytes/sec = B, means we need (1.8*N*S/B) seconds, #and need to add some leeway for network delays, we just add the #contents of jm_single().maker_timeout_sec (the user configured value) self.maker_timeout_sec = (len(tx) * 1.8 * len(self.active_orders.keys( ))) / (B_PER_SEC) + jm_single().maker_timeout_sec log.debug("Based on transaction size: " + str(len(tx)) + ", calculated time to wait for replies: " + str(self.maker_timeout_sec)) self.all_responded = True with self.timeout_lock: self.timeout_lock.notify() self.msgchan.send_tx(self.active_orders.keys(), tx) self.latest_tx = btc.deserialize(tx) for index, ins in enumerate(self.latest_tx['ins']): utxo = ins['outpoint']['hash'] + ':' + str( ins['outpoint']['index']) if utxo not in self.input_utxos.keys(): continue # placeholders required ins['script'] = 'deadbeef'
def read_wallet_file_data(self, filename, pwd=None): self.path = None self.index_cache = [[0, 0]] * self.max_mix_depth path = os.path.join('wallets', filename) if not os.path.isfile(path): if get_network() == 'testnet': log.debug('filename interpreted as seed, only available in ' 'testnet because this probably has lower entropy') return filename else: raise IOError('wallet file not found') self.path = path fd = open(path, 'r') walletfile = fd.read() fd.close() walletdata = json.loads(walletfile) if walletdata['network'] != get_network(): print('wallet network(%s) does not match ' 'joinmarket configured network(%s)' % (walletdata['network'], get_network())) sys.exit(0) if 'index_cache' in walletdata: self.index_cache = walletdata['index_cache'] if self.max_mix_depth > len(self.index_cache): #This can happen e.g. in tumbler when we need more mixdepths #than currently exist. Since we have no info for those extra #depths, we must default to (0,0) (but sync should find used #adddresses). self.index_cache += [[0, 0]] * (self.max_mix_depth - len(self.index_cache)) decrypted = False while not decrypted: if pwd: password = pwd else: password = getpass('Enter wallet decryption passphrase: ') password_key = btc.bin_dbl_sha256(password) encrypted_seed = walletdata['encrypted_seed'] try: decrypted_seed = decryptData( password_key, encrypted_seed.decode('hex')).encode('hex') # there is a small probability of getting a valid PKCS7 # padding by chance from a wrong password; sanity check the # seed length if len(decrypted_seed) == 32: decrypted = True else: raise ValueError except ValueError: print('Incorrect password') if pwd: raise decrypted = False if self.storepassword: self.password_key = password_key self.walletdata = walletdata if 'imported_keys' in walletdata: for epk_m in walletdata['imported_keys']: privkey = decryptData( password_key, epk_m['encrypted_privkey'].decode('hex')).encode('hex') #Imported keys are stored as 32 byte strings only, so the #second version below is sufficient, really. if len(privkey) != 64: raise Exception( "Unexpected privkey format; already compressed?:" + privkey) privkey += "01" if epk_m['mixdepth'] not in self.imported_privkeys: self.imported_privkeys[epk_m['mixdepth']] = [] self.addr_cache[btc.privtoaddr( privkey, magicbyte=get_p2pk_vbyte())] = ( epk_m['mixdepth'], -1, len(self.imported_privkeys[epk_m['mixdepth']])) self.imported_privkeys[epk_m['mixdepth']].append(privkey) return decrypted_seed
def auth_counterparty(self, nick, cr): #deserialize the commitment revelation cr_dict = btc.PoDLE.deserialize_revelation(cr) #check the validity of the proof of discrete log equivalence tries = jm_single().config.getint("POLICY", "taker_utxo_retries") def reject(msg): log.debug("Counterparty commitment not accepted, reason: " + msg) return False if not btc.verify_podle(cr_dict['P'], cr_dict['P2'], cr_dict['sig'], cr_dict['e'], self.maker.commit, index_range=range(tries)): reason = "verify_podle failed" return reject(reason) #finally, check that the proffered utxo is real, old enough, large enough, #and corresponds to the pubkey res = jm_single().bc_interface.query_utxo_set([cr_dict['utxo']], includeconf=True) if len(res) != 1 or not res[0]: reason = "authorizing utxo is not valid" return reject(reason) age = jm_single().config.getint("POLICY", "taker_utxo_age") if res[0]['confirms'] < age: reason = "commitment utxo not old enough: " + str( res[0]['confirms']) return reject(reason) reqd_amt = int( self.cj_amount * jm_single().config.getint("POLICY", "taker_utxo_amtpercent") / 100.0) if res[0]['value'] < reqd_amt: reason = "commitment utxo too small: " + str(res[0]['value']) return reject(reason) if res[0]['address'] != btc.pubkey_to_address(cr_dict['P'], get_p2pk_vbyte()): reason = "Invalid podle pubkey: " + str(cr_dict['P']) return reject(reason) # authorisation of taker passed # Send auth request to taker # Need to choose an input utxo pubkey to sign with # (no longer using the coinjoin pubkey from 0.2.0) # Just choose the first utxo in self.utxos and retrieve key from wallet. auth_address = self.utxos[self.utxos.keys()[0]]['address'] auth_key = self.maker.wallet.get_key_from_addr(auth_address) auth_pub = btc.privtopub(auth_key) btc_sig = btc.ecdsa_sign(self.kp.hex_pk(), auth_key) self.maker.msgchan.send_ioauth(nick, self.utxos.keys(), auth_pub, self.cj_addr, self.change_addr, btc_sig) #In case of *blacklisted (ie already used) commitments, we already #broadcasted them on receipt; in case of valid, and now used commitments, #we broadcast them here, and not early - to avoid accidentally #blacklisting commitments that are broadcast between makers in real time #for the same transaction. self.maker.transfer_commitment(self.maker.commit) #now persist the fact that the commitment is actually used. check_utxo_blacklist(self.maker.commit, persist=True) return True
def recv_txio(self, nick, utxo_list, cj_pub, change_addr): if nick not in self.nonrespondants: log.debug(('recv_txio => nick={} not in ' 'nonrespondants {}').format(nick, self.nonrespondants)) return self.utxos[nick] = utxo_list utxo_data = jm_single().bc_interface.query_utxo_set(self.utxos[nick]) if None in utxo_data: log.debug(('ERROR outputs unconfirmed or already spent. ' 'utxo_data={}').format(pprint.pformat(utxo_data))) # when internal reviewing of makers is created, add it here to # immediately quit return # ignore this message, eventually the timeout thread will recover total_input = sum([d['value'] for d in utxo_data]) real_cjfee = calc_cj_fee(self.active_orders[nick]['ordertype'], self.active_orders[nick]['cjfee'], self.cj_amount) self.outputs.append({'address': change_addr, 'value': total_input - self.cj_amount - self.active_orders[nick]['txfee'] + real_cjfee}) fmt = ('fee breakdown for {} totalin={:d} ' 'cjamount={:d} txfee={:d} realcjfee={:d}').format log.debug(fmt(nick, total_input, self.cj_amount, self.active_orders[nick]['txfee'], real_cjfee)) cj_addr = btc.pubtoaddr(cj_pub, get_p2pk_vbyte()) self.outputs.append({'address': cj_addr, 'value': self.cj_amount}) self.cjfee_total += real_cjfee self.maker_txfee_contributions += self.active_orders[nick]['txfee'] self.nonrespondants.remove(nick) if len(self.nonrespondants) > 0: log.debug('nonrespondants = ' + str(self.nonrespondants)) return self.all_responded = True with self.timeout_lock: self.timeout_lock.notify() log.debug('got all parts, enough to build a tx') self.nonrespondants = list(self.active_orders.keys()) my_total_in = sum([va['value'] for u, va in self.input_utxos.iteritems()]) if self.my_change_addr: #Estimate fee per choice of next/3/6 blocks targetting. estimated_fee = estimate_tx_fee(len(sum( self.utxos.values(),[])), len(self.outputs)+2) log.debug("Based on initial guess: "+str( self.total_txfee)+", we estimated a fee of: "+str(estimated_fee)) #reset total self.total_txfee = estimated_fee my_txfee = max(self.total_txfee - self.maker_txfee_contributions, 0) my_change_value = ( my_total_in - self.cj_amount - self.cjfee_total - my_txfee) #Since we could not predict the maker's inputs, we may end up needing #too much such that the change value is negative or small. Note that #we have tried to avoid this based on over-estimating the needed amount #in SendPayment.create_tx(), but it is still a possibility if one maker #uses a *lot* of inputs. if self.my_change_addr and my_change_value <= 0: raise ValueError("Calculated transaction fee of: "+str( self.total_txfee)+" is too large for our inputs;Please try again.") elif self.my_change_addr and my_change_value <= jm_single().DUST_THRESHOLD: log.debug("Dynamically calculated change lower than dust: "+str( my_change_value)+"; dropping.") self.my_change_addr = None my_change_value = 0 log.debug('fee breakdown for me totalin=%d my_txfee=%d makers_txfee=%d cjfee_total=%d => changevalue=%d' % (my_total_in, my_txfee, self.maker_txfee_contributions, self.cjfee_total, my_change_value)) if self.my_change_addr is None: if my_change_value != 0 and abs(my_change_value) != 1: # seems you wont always get exactly zero because of integer # rounding so 1 satoshi extra or fewer being spent as miner # fees is acceptable log.debug(('WARNING CHANGE NOT BEING ' 'USED\nCHANGEVALUE = {}').format(my_change_value)) else: self.outputs.append({'address': self.my_change_addr, 'value': my_change_value}) self.utxo_tx = [dict([('output', u)]) for u in sum(self.utxos.values(), [])] self.outputs.append({'address': self.coinjoin_address(), 'value': self.cj_amount}) random.shuffle(self.utxo_tx) random.shuffle(self.outputs) tx = btc.mktx(self.utxo_tx, self.outputs) log.debug('obtained tx\n' + pprint.pformat(btc.deserialize(tx))) self.msgchan.send_tx(self.active_orders.keys(), tx) self.latest_tx = btc.deserialize(tx) for index, ins in enumerate(self.latest_tx['ins']): utxo = ins['outpoint']['hash'] + ':' + str( ins['outpoint']['index']) if utxo not in self.input_utxos.keys(): continue # placeholders required ins['script'] = 'deadbeef'
def get_addr(self, mixing_depth, forchange, i): return btc.privtoaddr( self.get_key(mixing_depth, forchange, i), get_p2pk_vbyte())
def recv_txio(self, nick, utxo_list, cj_pub, change_addr): if nick not in self.nonrespondants: log.debug(('recv_txio => nick={} not in ' 'nonrespondants {}').format(nick, self.nonrespondants)) return self.utxos[nick] = utxo_list utxo_data = jm_single().bc_interface.query_utxo_set(self.utxos[nick]) if None in utxo_data: log.debug(('ERROR outputs unconfirmed or already spent. ' 'utxo_data={}').format(pprint.pformat(utxo_data))) # when internal reviewing of makers is created, add it here to # immediately quit; currently, the timeout thread suffices. return total_input = sum([d['value'] for d in utxo_data]) real_cjfee = calc_cj_fee(self.active_orders[nick]['ordertype'], self.active_orders[nick]['cjfee'], self.cj_amount) change_amount = (total_input - self.cj_amount - self.active_orders[nick]['txfee'] + real_cjfee) # certain malicious and/or incompetent liquidity providers send # inputs totalling less than the coinjoin amount! this leads to # a change output of zero satoshis, so the invalid transaction # fails harmlessly; let's fail earlier, with a clear message. if change_amount < jm_single().DUST_THRESHOLD: fmt = ('ERROR counterparty requires sub-dust change. nick={}' 'totalin={:d} cjamount={:d} change={:d}').format log.debug(fmt(nick, total_input, self.cj_amount, change_amount)) return # timeout marks this maker as nonresponsive self.outputs.append({'address': change_addr, 'value': change_amount}) fmt = ('fee breakdown for {} totalin={:d} ' 'cjamount={:d} txfee={:d} realcjfee={:d}').format log.debug(fmt(nick, total_input, self.cj_amount, self.active_orders[nick]['txfee'], real_cjfee)) cj_addr = btc.pubtoaddr(cj_pub, get_p2pk_vbyte()) self.outputs.append({'address': cj_addr, 'value': self.cj_amount}) self.cjfee_total += real_cjfee self.maker_txfee_contributions += self.active_orders[nick]['txfee'] self.nonrespondants.remove(nick) if len(self.nonrespondants) > 0: log.debug('nonrespondants = ' + str(self.nonrespondants)) return log.debug('got all parts, enough to build a tx') self.nonrespondants = list(self.active_orders.keys()) my_total_in = sum([va['value'] for u, va in self.input_utxos.iteritems()]) if self.my_change_addr: #Estimate fee per choice of next/3/6 blocks targetting. estimated_fee = estimate_tx_fee(len(sum( self.utxos.values(),[])), len(self.outputs)+2) log.debug("Based on initial guess: "+str( self.total_txfee)+", we estimated a fee of: "+str(estimated_fee)) #reset total self.total_txfee = estimated_fee my_txfee = max(self.total_txfee - self.maker_txfee_contributions, 0) my_change_value = ( my_total_in - self.cj_amount - self.cjfee_total - my_txfee) #Since we could not predict the maker's inputs, we may end up needing #too much such that the change value is negative or small. Note that #we have tried to avoid this based on over-estimating the needed amount #in SendPayment.create_tx(), but it is still a possibility if one maker #uses a *lot* of inputs. if self.my_change_addr and my_change_value <= 0: raise ValueError("Calculated transaction fee of: "+str( self.total_txfee)+" is too large for our inputs;Please try again.") elif self.my_change_addr and my_change_value <= jm_single().DUST_THRESHOLD: log.debug("Dynamically calculated change lower than dust: "+str( my_change_value)+"; dropping.") self.my_change_addr = None my_change_value = 0 log.debug('fee breakdown for me totalin=%d my_txfee=%d makers_txfee=%d cjfee_total=%d => changevalue=%d' % (my_total_in, my_txfee, self.maker_txfee_contributions, self.cjfee_total, my_change_value)) if self.my_change_addr is None: if my_change_value != 0 and abs(my_change_value) != 1: # seems you wont always get exactly zero because of integer # rounding so 1 satoshi extra or fewer being spent as miner # fees is acceptable log.debug(('WARNING CHANGE NOT BEING ' 'USED\nCHANGEVALUE = {}').format(my_change_value)) else: self.outputs.append({'address': self.my_change_addr, 'value': my_change_value}) self.utxo_tx = [dict([('output', u)]) for u in sum(self.utxos.values(), [])] self.outputs.append({'address': self.coinjoin_address(), 'value': self.cj_amount}) random.shuffle(self.utxo_tx) random.shuffle(self.outputs) tx = btc.mktx(self.utxo_tx, self.outputs) log.debug('obtained tx\n' + pprint.pformat(btc.deserialize(tx))) #Re-calculate a sensible timeout wait based on the throttling #settings and the tx size. #Calculation: Let tx size be S; tx undergoes two b64 expansions, 1.8*S #So we're sending N*1.8*S over the wire, and the #maximum bytes/sec = B, means we need (1.8*N*S/B) seconds, #and need to add some leeway for network delays, we just add the #contents of jm_single().maker_timeout_sec (the user configured value) self.maker_timeout_sec = (len(tx) * 1.8 * len( self.active_orders.keys()))/(B_PER_SEC) + jm_single().maker_timeout_sec log.debug("Based on transaction size: " + str( len(tx)) + ", calculated time to wait for replies: " + str( self.maker_timeout_sec)) self.all_responded = True with self.timeout_lock: self.timeout_lock.notify() self.msgchan.send_tx(self.active_orders.keys(), tx) self.latest_tx = btc.deserialize(tx) for index, ins in enumerate(self.latest_tx['ins']): utxo = ins['outpoint']['hash'] + ':' + str( ins['outpoint']['index']) if utxo not in self.input_utxos.keys(): continue # placeholders required ins['script'] = 'deadbeef'
def sync_addresses(self, wallet): from joinmarket.wallet import BitcoinCoreWallet if isinstance(wallet, BitcoinCoreWallet): return log.debug('requesting wallet history') wallet_name = self.get_wallet_name(wallet) #TODO It is worth considering making this user configurable: addr_req_count = 20 wallet_addr_list = [] for mix_depth in range(wallet.max_mix_depth): for forchange in [0, 1]: #If we have an index-cache available, we can use it #to decide how much to import (note that this list #*always* starts from index 0 on each branch). #In cases where the Bitcoin Core instance is fresh, #this will allow the entire import+rescan to occur #in 2 steps only. if wallet.index_cache != [[0, 0]] * wallet.max_mix_depth: #Need to request N*addr_req_count where N is least s.t. #N*addr_req_count > index_cache val. This is so that the batching #process in the main loop *always* has already imported enough #addresses to complete. req_count = int(wallet.index_cache[mix_depth][forchange] / addr_req_count) + 1 req_count *= addr_req_count else: #If we have *nothing* - no index_cache, and no info #in Core wallet (imports), we revert to a batching mode #with a default size. #In this scenario it could require several restarts *and* #rescans; perhaps user should set addr_req_count high #(see above TODO) req_count = addr_req_count wallet_addr_list += [wallet.get_new_addr(mix_depth, forchange) for _ in range(req_count)] #Indices are reset here so that the next algorithm step starts #from the beginning of each branch wallet.index[mix_depth][forchange] = 0 # makes more sense to add these in an account called "joinmarket-imported" but its much # simpler to add to the same account here for privkey_list in wallet.imported_privkeys.values(): for privkey in privkey_list: imported_addr = btc.privtoaddr(privkey, magicbyte=get_p2pk_vbyte()) wallet_addr_list.append(imported_addr) imported_addr_list = self.rpc('getaddressesbyaccount', [wallet_name]) if not set(wallet_addr_list).issubset(set(imported_addr_list)): self.add_watchonly_addresses(wallet_addr_list, wallet_name) return buf = self.rpc('listtransactions', [wallet_name, 1000, 0, True]) txs = buf # If the buffer's full, check for more, until it ain't while len(buf) == 1000: buf = self.rpc('listtransactions', [wallet_name, 1000, len(txs), True]) txs += buf # TODO check whether used_addr_list can be a set, may be faster (if # its a hashset) and allows using issubset() here and setdiff() for # finding which addresses need importing # TODO also check the fastest way to build up python lists, i suspect # using += is slow used_addr_list = [tx['address'] for tx in txs if tx['category'] == 'receive'] too_few_addr_mix_change = [] for mix_depth in range(wallet.max_mix_depth): for forchange in [0, 1]: unused_addr_count = 0 last_used_addr = '' breakloop = False while not breakloop: if unused_addr_count >= wallet.gaplimit and \ is_index_ahead_of_cache(wallet, mix_depth, forchange): break mix_change_addrs = [ wallet.get_new_addr(mix_depth, forchange) for _ in range(addr_req_count) ] for mc_addr in mix_change_addrs: if mc_addr not in imported_addr_list: too_few_addr_mix_change.append((mix_depth, forchange )) breakloop = True break if mc_addr in used_addr_list: last_used_addr = mc_addr unused_addr_count = 0 else: unused_addr_count += 1 #index setting here depends on whether we broke out of the loop #early; if we did, it means we need to prepare the index #at the level of the last used address or zero so as to not #miss any imports in add_watchonly_addresses. #If we didn't, we need to respect the index_cache to avoid #potential address reuse. if breakloop: if last_used_addr == '': wallet.index[mix_depth][forchange] = 0 else: wallet.index[mix_depth][forchange] = \ wallet.addr_cache[last_used_addr][2] + 1 else: if last_used_addr == '': next_avail_idx = max([wallet.index_cache[mix_depth][ forchange], 0]) else: next_avail_idx = max([wallet.addr_cache[last_used_addr][ 2] + 1, wallet.index_cache[mix_depth][forchange]]) wallet.index[mix_depth][forchange] = next_avail_idx wallet_addr_list = [] if len(too_few_addr_mix_change) > 0: indices = [wallet.index[mc[0]][mc[1]] for mc in too_few_addr_mix_change] log.debug('too few addresses in ' + str(too_few_addr_mix_change) + ' at ' + str(indices)) for mix_depth, forchange in too_few_addr_mix_change: wallet_addr_list += [ wallet.get_new_addr(mix_depth, forchange) for _ in range(addr_req_count * 3) ] self.add_watchonly_addresses(wallet_addr_list, wallet_name) return self.wallet_synced = True
def read_wallet_file_data(self, filename, pwd=None): self.path = None self.index_cache = [[0, 0]] * self.max_mix_depth path = os.path.join('wallets', filename) if not os.path.isfile(path): if get_network() == 'testnet': log.debug('filename interpreted as seed, only available in ' 'testnet because this probably has lower entropy') return filename else: raise IOError('wallet file not found') self.path = path fd = open(path, 'r') walletfile = fd.read() fd.close() walletdata = json.loads(walletfile) if walletdata['network'] != get_network(): print ('wallet network(%s) does not match ' 'joinmarket configured network(%s)' % ( walletdata['network'], get_network())) sys.exit(0) if 'index_cache' in walletdata: self.index_cache = walletdata['index_cache'] if self.max_mix_depth > len(self.index_cache): #This can happen e.g. in tumbler when we need more mixdepths #than currently exist. Since we have no info for those extra #depths, we must default to (0,0) (but sync should find used #adddresses). self.index_cache += [[0,0]] * ( self.max_mix_depth - len(self.index_cache)) decrypted = False trieddefault = False while not decrypted: if pwd: password = pwd else: if not trieddefault: password = '' trieddefault = True else: password = getpass('Enter wallet decryption passphrase: ') password_key = btc.bin_dbl_sha256(password) encrypted_seed = walletdata['encrypted_seed'] try: decrypted_seed = decryptData( password_key, encrypted_seed.decode('hex')).encode('hex') # there is a small probability of getting a valid PKCS7 # padding by chance from a wrong password; sanity check the # seed length if len(decrypted_seed) == 32: decrypted = True else: raise ValueError except ValueError: if not trieddefault: print('Incorrect password') if pwd: raise decrypted = False if self.storepassword: self.password_key = password_key self.walletdata = walletdata if 'imported_keys' in walletdata: for epk_m in walletdata['imported_keys']: privkey = decryptData( password_key, epk_m['encrypted_privkey'].decode( 'hex')).encode('hex') #Imported keys are stored as 32 byte strings only, so the #second version below is sufficient, really. if len(privkey) != 64: raise Exception( "Unexpected privkey format; already compressed?:" + privkey) privkey += "01" if epk_m['mixdepth'] not in self.imported_privkeys: self.imported_privkeys[epk_m['mixdepth']] = [] self.addr_cache[btc.privtoaddr( privkey, magicbyte=get_p2pk_vbyte())] = (epk_m['mixdepth'], -1, len(self.imported_privkeys[epk_m['mixdepth']])) self.imported_privkeys[epk_m['mixdepth']].append(privkey) return decrypted_seed
def recv_txio(self, nick, utxo_list, cj_pub, change_addr): if nick not in self.nonrespondants: log.debug(('recv_txio => nick={} not in ' 'nonrespondants {}').format(nick, self.nonrespondants)) return self.utxos[nick] = utxo_list order = self.db.execute( 'SELECT ordertype, txfee, cjfee FROM ' 'orderbook WHERE oid=? AND counterparty=?', (self.active_orders[nick], nick)).fetchone() utxo_data = jm_single().bc_interface.query_utxo_set(self.utxos[nick]) if None in utxo_data: log.debug(('ERROR outputs unconfirmed or already spent. ' 'utxo_data={}').format(pprint.pformat(utxo_data))) # when internal reviewing of makers is created, add it here to # immediately quit return # ignore this message, eventually the timeout thread will recover total_input = sum([d['value'] for d in utxo_data]) real_cjfee = calc_cj_fee(order['ordertype'], order['cjfee'], self.cj_amount) self.outputs.append({ 'address': change_addr, 'value': total_input - self.cj_amount - order['txfee'] + real_cjfee }) fmt = ('fee breakdown for {} totalin={:d} ' 'cjamount={:d} txfee={:d} realcjfee={:d}').format log.debug( fmt(nick, total_input, self.cj_amount, order['txfee'], real_cjfee)) cj_addr = btc.pubtoaddr(cj_pub, get_p2pk_vbyte()) self.outputs.append({'address': cj_addr, 'value': self.cj_amount}) self.cjfee_total += real_cjfee self.maker_txfee_contributions += order['txfee'] self.nonrespondants.remove(nick) if len(self.nonrespondants) > 0: log.debug('nonrespondants = ' + str(self.nonrespondants)) return self.all_responded = True with self.timeout_lock: self.timeout_lock.notify() log.debug('got all parts, enough to build a tx') self.nonrespondants = list(self.active_orders.keys()) my_total_in = sum( [va['value'] for u, va in self.input_utxos.iteritems()]) if self.my_change_addr: #Estimate fee per choice of next/3/6 blocks targetting. estimated_fee = estimate_tx_fee(len(sum(self.utxos.values(), [])), len(self.outputs) + 2) log.debug("Based on initial guess: " + str(self.total_txfee) + ", we estimated a fee of: " + str(estimated_fee)) #reset total self.total_txfee = estimated_fee my_txfee = max(self.total_txfee - self.maker_txfee_contributions, 0) my_change_value = (my_total_in - self.cj_amount - self.cjfee_total - my_txfee) #Since we could not predict the maker's inputs, we may end up needing #too much such that the change value is negative or small. Note that #we have tried to avoid this based on over-estimating the needed amount #in SendPayment.create_tx(), but it is still a possibility if one maker #uses a *lot* of inputs. if self.my_change_addr and my_change_value <= 0: raise ValueError("Calculated transaction fee of: " + str(self.total_txfee) + " is too large for our inputs;Please try again.") elif self.my_change_addr and my_change_value <= jm_single( ).DUST_THRESHOLD: log.debug("Dynamically calculated change lower than dust: " + str(my_change_value) + "; dropping.") self.my_change_addr = None my_change_value = 0 log.debug( 'fee breakdown for me totalin=%d my_txfee=%d makers_txfee=%d cjfee_total=%d => changevalue=%d' % (my_total_in, my_txfee, self.maker_txfee_contributions, self.cjfee_total, my_change_value)) if self.my_change_addr is None: if my_change_value != 0 and abs(my_change_value) != 1: # seems you wont always get exactly zero because of integer # rounding so 1 satoshi extra or fewer being spent as miner # fees is acceptable log.debug(('WARNING CHANGE NOT BEING ' 'USED\nCHANGEVALUE = {}').format(my_change_value)) else: self.outputs.append({ 'address': self.my_change_addr, 'value': my_change_value }) self.utxo_tx = [ dict([('output', u)]) for u in sum(self.utxos.values(), []) ] self.outputs.append({ 'address': self.coinjoin_address(), 'value': self.cj_amount }) random.shuffle(self.utxo_tx) random.shuffle(self.outputs) tx = btc.mktx(self.utxo_tx, self.outputs) log.debug('obtained tx\n' + pprint.pformat(btc.deserialize(tx))) self.msgchan.send_tx(self.active_orders.keys(), tx) self.latest_tx = btc.deserialize(tx) for index, ins in enumerate(self.latest_tx['ins']): utxo = ins['outpoint']['hash'] + ':' + str( ins['outpoint']['index']) if utxo not in self.input_utxos.keys(): continue # placeholders required ins['script'] = 'deadbeef'
def sync_addresses(self, wallet): from joinmarket.wallet import BitcoinCoreWallet if isinstance(wallet, BitcoinCoreWallet): return log.debug('requesting wallet history') wallet_name = self.get_wallet_name(wallet) #TODO It is worth considering making this user configurable: addr_req_count = 20 wallet_addr_list = [] for mix_depth in range(wallet.max_mix_depth): for forchange in [0, 1]: #If we have an index-cache available, we can use it #to decide how much to import (note that this list #*always* starts from index 0 on each branch). #In cases where the Bitcoin Core instance is fresh, #this will allow the entire import+rescan to occur #in 2 steps only. if wallet.index_cache != [[0, 0]] * wallet.max_mix_depth: #Need to request N*addr_req_count where N is least s.t. #N*addr_req_count > index_cache val. This is so that the batching #process in the main loop *always* has already imported enough #addresses to complete. req_count = int(wallet.index_cache[mix_depth][forchange] / addr_req_count) + 1 req_count *= addr_req_count else: #If we have *nothing* - no index_cache, and no info #in Core wallet (imports), we revert to a batching mode #with a default size. #In this scenario it could require several restarts *and* #rescans; perhaps user should set addr_req_count high #(see above TODO) req_count = addr_req_count wallet_addr_list += [ wallet.get_new_addr(mix_depth, forchange) for _ in range(req_count) ] #Indices are reset here so that the next algorithm step starts #from the beginning of each branch wallet.index[mix_depth][forchange] = 0 # makes more sense to add these in an account called "joinmarket-imported" but its much # simpler to add to the same account here for privkey_list in wallet.imported_privkeys.values(): for privkey in privkey_list: imported_addr = btc.privtoaddr(privkey, magicbyte=get_p2pk_vbyte()) wallet_addr_list.append(imported_addr) imported_addr_list = self.rpc('getaddressesbyaccount', [wallet_name]) if not set(wallet_addr_list).issubset(set(imported_addr_list)): self.add_watchonly_addresses(wallet_addr_list, wallet_name) return buf = self.rpc('listtransactions', [wallet_name, 1000, 0, True]) txs = buf # If the buffer's full, check for more, until it ain't while len(buf) == 1000: buf = self.rpc( 'listtransactions', [wallet_name, 1000, len(txs), True]) txs += buf # TODO check whether used_addr_list can be a set, may be faster (if # its a hashset) and allows using issubset() here and setdiff() for # finding which addresses need importing # TODO also check the fastest way to build up python lists, i suspect # using += is slow used_addr_list = [ tx['address'] for tx in txs if tx['category'] == 'receive' ] too_few_addr_mix_change = [] for mix_depth in range(wallet.max_mix_depth): for forchange in [0, 1]: unused_addr_count = 0 last_used_addr = '' breakloop = False while not breakloop: if unused_addr_count >= wallet.gaplimit and \ is_index_ahead_of_cache(wallet, mix_depth, forchange): break mix_change_addrs = [ wallet.get_new_addr(mix_depth, forchange) for _ in range(addr_req_count) ] for mc_addr in mix_change_addrs: if mc_addr not in imported_addr_list: too_few_addr_mix_change.append( (mix_depth, forchange)) breakloop = True break if mc_addr in used_addr_list: last_used_addr = mc_addr unused_addr_count = 0 else: unused_addr_count += 1 #index setting here depends on whether we broke out of the loop #early; if we did, it means we need to prepare the index #at the level of the last used address or zero so as to not #miss any imports in add_watchonly_addresses. #If we didn't, we need to respect the index_cache to avoid #potential address reuse. if breakloop: if last_used_addr == '': wallet.index[mix_depth][forchange] = 0 else: wallet.index[mix_depth][forchange] = \ wallet.addr_cache[last_used_addr][2] + 1 else: if last_used_addr == '': next_avail_idx = max( [wallet.index_cache[mix_depth][forchange], 0]) else: next_avail_idx = max([ wallet.addr_cache[last_used_addr][2] + 1, wallet.index_cache[mix_depth][forchange] ]) wallet.index[mix_depth][forchange] = next_avail_idx wallet_addr_list = [] if len(too_few_addr_mix_change) > 0: indices = [ wallet.index[mc[0]][mc[1]] for mc in too_few_addr_mix_change ] log.debug('too few addresses in ' + str(too_few_addr_mix_change) + ' at ' + str(indices)) for mix_depth, forchange in too_few_addr_mix_change: wallet_addr_list += [ wallet.get_new_addr(mix_depth, forchange) for _ in range(addr_req_count * 3) ] self.add_watchonly_addresses(wallet_addr_list, wallet_name) return self.wallet_synced = True
def sync_addresses(self, wallet): from joinmarket.wallet import BitcoinCoreWallet if isinstance(wallet, BitcoinCoreWallet): return log.debug('requesting wallet history') wallet_name = self.get_wallet_name(wallet) addr_req_count = 20 wallet_addr_list = [] for mix_depth in range(wallet.max_mix_depth): for forchange in [0, 1]: wallet_addr_list += [ wallet.get_new_addr(mix_depth, forchange) for _ in range(addr_req_count) ] wallet.index[mix_depth][forchange] = 0 # makes more sense to add these in an account called "joinmarket-imported" but its much # simpler to add to the same account here for privkey_list in wallet.imported_privkeys.values(): for privkey in privkey_list: imported_addr = btc.privtoaddr(privkey, magicbyte=get_p2pk_vbyte()) wallet_addr_list.append(imported_addr) imported_addr_list = self.rpc('getaddressesbyaccount', [wallet_name]) if not set(wallet_addr_list).issubset(set(imported_addr_list)): self.add_watchonly_addresses(wallet_addr_list, wallet_name) return buf = self.rpc('listtransactions', [wallet_name, 1000, 0, True]) txs = buf # If the buffer's full, check for more, until it ain't while len(buf) == 1000: buf = self.rpc( 'listtransactions', [wallet_name, 1000, len(txs), True]) txs += buf # TODO check whether used_addr_list can be a set, may be faster (if # its a hashset) and allows using issubset() here and setdiff() for # finding which addresses need importing # TODO also check the fastest way to build up python lists, i suspect # using += is slow used_addr_list = [ tx['address'] for tx in txs if tx['category'] == 'receive' ] too_few_addr_mix_change = [] for mix_depth in range(wallet.max_mix_depth): for forchange in [0, 1]: unused_addr_count = 0 last_used_addr = '' breakloop = False while not breakloop: if unused_addr_count >= wallet.gaplimit and \ is_index_ahead_of_cache(wallet, mix_depth, forchange): break mix_change_addrs = [ wallet.get_new_addr(mix_depth, forchange) for _ in range(addr_req_count) ] for mc_addr in mix_change_addrs: if mc_addr not in imported_addr_list: too_few_addr_mix_change.append( (mix_depth, forchange)) breakloop = True break if mc_addr in used_addr_list: last_used_addr = mc_addr unused_addr_count = 0 else: unused_addr_count += 1 if last_used_addr == '': wallet.index[mix_depth][forchange] = 0 else: wallet.index[mix_depth][forchange] = \ wallet.addr_cache[last_used_addr][2] + 1 wallet_addr_list = [] if len(too_few_addr_mix_change) > 0: log.debug('too few addresses in ' + str(too_few_addr_mix_change)) for mix_depth, forchange in too_few_addr_mix_change: wallet_addr_list += [ wallet.get_new_addr(mix_depth, forchange) for _ in range(addr_req_count * 3) ] self.add_watchonly_addresses(wallet_addr_list, wallet_name) return self.wallet_synced = True
def sync_addresses(self, wallet): from joinmarket.wallet import BitcoinCoreWallet if isinstance(wallet, BitcoinCoreWallet): return log.debug('requesting wallet history') wallet_name = self.get_wallet_name(wallet) addr_req_count = 20 wallet_addr_list = [] for mix_depth in range(wallet.max_mix_depth): for forchange in [0, 1]: wallet_addr_list += [wallet.get_new_addr(mix_depth, forchange) for _ in range(addr_req_count)] wallet.index[mix_depth][forchange] = 0 # makes more sense to add these in an account called "joinmarket-imported" but its much # simpler to add to the same account here for privkey_list in wallet.imported_privkeys.values(): for privkey in privkey_list: imported_addr = btc.privtoaddr(privkey, get_p2pk_vbyte()) wallet_addr_list.append(imported_addr) imported_addr_list = self.rpc('getaddressesbyaccount', [wallet_name]) if not set(wallet_addr_list).issubset(set(imported_addr_list)): self.add_watchonly_addresses(wallet_addr_list, wallet_name) return buf = self.rpc('listtransactions', [wallet_name, 1000, 0, True]) txs = buf # If the buffer's full, check for more, until it ain't while len(buf) == 1000: buf = self.rpc('listtransactions', [wallet_name, 1000, len(txs), True]) txs += buf # TODO check whether used_addr_list can be a set, may be faster (if # its a hashset) and allows using issubset() here and setdiff() for # finding which addresses need importing # TODO also check the fastest way to build up python lists, i suspect # using += is slow used_addr_list = [tx['address'] for tx in txs if tx['category'] == 'receive'] too_few_addr_mix_change = [] for mix_depth in range(wallet.max_mix_depth): for forchange in [0, 1]: unused_addr_count = 0 last_used_addr = '' breakloop = False while not breakloop: if unused_addr_count >= wallet.gaplimit and \ is_index_ahead_of_cache(wallet, mix_depth, forchange): break mix_change_addrs = [ wallet.get_new_addr(mix_depth, forchange) for _ in range(addr_req_count)] for mc_addr in mix_change_addrs: if mc_addr not in imported_addr_list: too_few_addr_mix_change.append( (mix_depth, forchange)) breakloop = True break if mc_addr in used_addr_list: last_used_addr = mc_addr unused_addr_count = 0 else: unused_addr_count += 1 if last_used_addr == '': wallet.index[mix_depth][forchange] = 0 else: wallet.index[mix_depth][forchange] = \ wallet.addr_cache[last_used_addr][2] + 1 wallet_addr_list = [] if len(too_few_addr_mix_change) > 0: log.debug('too few addresses in ' + str(too_few_addr_mix_change)) for mix_depth, forchange in too_few_addr_mix_change: wallet_addr_list += [ wallet.get_new_addr(mix_depth, forchange) for _ in range(addr_req_count * 3)] self.add_watchonly_addresses(wallet_addr_list, wallet_name) return self.wallet_synced = True