def add_tx_notify(self, txd, unconfirmfun, confirmfun, spentfun, notifyaddr, timeoutfun=None): if not self.notifythread: self.notifythread = BitcoinCoreNotifyThread(self) self.notifythread.start() one_addr_imported = False for outs in txd['outs']: addr = btc.script_to_address(outs['script'], get_p2pk_vbyte()) if self.rpc('getaccount', [addr]) != '': one_addr_imported = True break if not one_addr_imported: self.rpc('importaddress', [notifyaddr, 'joinmarket-notify', False]) tx_output_set = set([(sv['script'], sv['value']) for sv in txd['outs']]) self.txnotify_fun.append( (btc.txhash(btc.serialize(txd)), tx_output_set, unconfirmfun, confirmfun, spentfun, timeoutfun, False)) #create unconfirm timeout here, create confirm timeout in the other thread if timeoutfun: threading.Timer(cs_single().config.getint('TIMEOUT', 'unconfirm_timeout_sec'), bitcoincore_timeout_callback, args=(False, tx_output_set, self.txnotify_fun, timeoutfun)).start()
def add_new_utxos(self, tx, txid): added_utxos = {} for index, outs in enumerate(tx['outs']): addr = btc.script_to_address(outs['script'], get_p2pk_vbyte()) if addr not in self.addr_cache: continue addrdict = {'address': addr, 'value': outs['value']} utxo = txid + ':' + str(index) added_utxos[utxo] = addrdict self.unspent[utxo] = addrdict log.debug('added utxos, wallet now is \n' + pprint.pformat(self.get_utxos_by_mixdepth())) return added_utxos
def add_tx_notify(self, txd, unconfirmfun, confirmfun, notifyaddr, wallet_name=None, timeoutfun=None, spentfun=None, txid_flag=True, n=0, c=1, vb=None): """Given a deserialized transaction txd, callback functions for broadcast and confirmation of the transaction, an address to import, and a callback function for timeout, set up a polling loop to check for events on the transaction. Also optionally set to trigger "confirmed" callback on number of confirmations c. Also checks for spending (if spentfun is not None) of the outpoint n. If txid_flag is True, we create a watcher loop on the txid (hence only really usable in a segwit context, and only on fully formed transactions), else we create a watcher loop on the output set of the transaction (taken from the outs field of the txd). """ if not vb: vb = get_p2pk_vbyte() if isinstance(self, BitcoinCoreInterface) or isinstance( self, RegtestBitcoinCoreInterface): #This code ensures that a walletnotify is triggered, by #ensuring that at least one of the output addresses is #imported into the wallet (note the sweep special case, where #none of the output addresses belong to me). one_addr_imported = False for outs in txd['outs']: addr = btc.script_to_address(outs['script'], vb) if self.rpc('getaccount', [addr]) != '': one_addr_imported = True break if not one_addr_imported: self.rpc('importaddress', [notifyaddr, 'joinmarket-notify', False]) #Warning! In case of txid_flag false, this is *not* a valid txid, #but only a hash of an incomplete transaction serialization. txid = btc.txhash(btc.serialize(txd)) if not txid_flag: tx_output_set = set([(sv['script'], sv['value']) for sv in txd['outs']]) loop = task.LoopingCall(self.outputs_watcher, wallet_name, notifyaddr, tx_output_set, unconfirmfun, confirmfun, timeoutfun) log.debug("Created watcher loop for address: " + notifyaddr) loopkey = notifyaddr else: loop = task.LoopingCall(self.tx_watcher, txd, unconfirmfun, confirmfun, spentfun, c, n) log.debug("Created watcher loop for txid: " + txid) loopkey = txid self.tx_watcher_loops[loopkey] = [loop, False, False, False] #Hardcoded polling interval, but in any case it can be very short. loop.start(5.0) #Give up on un-broadcast transactions and broadcast but not confirmed #transactions as per settings in the config. reactor.callLater( float(jm_single().config.get("TIMEOUT", "unconfirm_timeout_sec")), self.tx_network_timeout, loopkey) confirm_timeout_sec = int(jm_single().config.get( "TIMEOUT", "confirm_timeout_hours")) * 3600 reactor.callLater(confirm_timeout_sec, self.tx_timeout, txd, loopkey, timeoutfun)
def sync_addresses(self, wallet, restart_cb=None): from jmclient.wallet import BitcoinCoreWallet if isinstance(wallet, BitcoinCoreWallet): return log.debug('requesting detailed wallet history') wallet_name = self.get_wallet_name(wallet) #TODO It is worth considering making this user configurable: addr_req_count = 20 wallet_addr_list = [] for mix_depth in range(wallet.max_mix_depth): for forchange in [0, 1]: #If we have an index-cache available, we can use it #to decide how much to import (note that this list #*always* starts from index 0 on each branch). #In cases where the Bitcoin Core instance is fresh, #this will allow the entire import+rescan to occur #in 2 steps only. if wallet.index_cache != [[0, 0]] * wallet.max_mix_depth: #Need to request N*addr_req_count where N is least s.t. #N*addr_req_count > index_cache val. This is so that the batching #process in the main loop *always* has already imported enough #addresses to complete. req_count = int(wallet.index_cache[mix_depth][forchange] / addr_req_count) + 1 req_count *= addr_req_count else: #If we have *nothing* - no index_cache, and no info #in Core wallet (imports), we revert to a batching mode #with a default size. #In this scenario it could require several restarts *and* #rescans; perhaps user should set addr_req_count high #(see above TODO) req_count = addr_req_count wallet_addr_list += [ wallet.get_new_addr(mix_depth, forchange) for _ in range(req_count) ] #Indices are reset here so that the next algorithm step starts #from the beginning of each branch wallet.index[mix_depth][forchange] = 0 # makes more sense to add these in an account called "joinmarket-imported" but its much # simpler to add to the same account here for privkey_list in wallet.imported_privkeys.values(): for privkey in privkey_list: imported_addr = btc.privtoaddr(privkey, magicbyte=get_p2pk_vbyte()) wallet_addr_list.append(imported_addr) imported_addr_list = self.rpc('getaddressesbyaccount', [wallet_name]) if not set(wallet_addr_list).issubset(set(imported_addr_list)): self.add_watchonly_addresses(wallet_addr_list, wallet_name, restart_cb) return buf = self.rpc('listtransactions', [wallet_name, 1000, 0, True]) txs = buf # If the buffer's full, check for more, until it ain't while len(buf) == 1000: buf = self.rpc( 'listtransactions', [wallet_name, 1000, len(txs), True]) txs += buf # TODO check whether used_addr_list can be a set, may be faster (if # its a hashset) and allows using issubset() here and setdiff() for # finding which addresses need importing # TODO also check the fastest way to build up python lists, i suspect # using += is slow used_addr_list = [ tx['address'] for tx in txs if tx['category'] == 'receive' ] too_few_addr_mix_change = [] for mix_depth in range(wallet.max_mix_depth): for forchange in [0, 1]: unused_addr_count = 0 last_used_addr = '' breakloop = False while not breakloop: if unused_addr_count >= wallet.gaplimit and \ is_index_ahead_of_cache(wallet, mix_depth, forchange): break mix_change_addrs = [ wallet.get_new_addr(mix_depth, forchange) for _ in range(addr_req_count) ] for mc_addr in mix_change_addrs: if mc_addr not in imported_addr_list: too_few_addr_mix_change.append( (mix_depth, forchange)) breakloop = True break if mc_addr in used_addr_list: last_used_addr = mc_addr unused_addr_count = 0 else: unused_addr_count += 1 #index setting here depends on whether we broke out of the loop #early; if we did, it means we need to prepare the index #at the level of the last used address or zero so as to not #miss any imports in add_watchonly_addresses. #If we didn't, we need to respect the index_cache to avoid #potential address reuse. if breakloop: if last_used_addr == '': wallet.index[mix_depth][forchange] = 0 else: wallet.index[mix_depth][forchange] = \ wallet.addr_cache[last_used_addr][2] + 1 else: if last_used_addr == '': next_avail_idx = max( [wallet.index_cache[mix_depth][forchange], 0]) else: next_avail_idx = max([ wallet.addr_cache[last_used_addr][2] + 1, wallet.index_cache[mix_depth][forchange] ]) wallet.index[mix_depth][forchange] = next_avail_idx wallet_addr_list = [] if len(too_few_addr_mix_change) > 0: indices = [ wallet.index[mc[0]][mc[1]] for mc in too_few_addr_mix_change ] log.debug('too few addresses in ' + str(too_few_addr_mix_change) + ' at ' + str(indices)) for mix_depth, forchange in too_few_addr_mix_change: wallet_addr_list += [ wallet.get_new_addr(mix_depth, forchange) for _ in range(addr_req_count * 3) ] self.add_watchonly_addresses(wallet_addr_list, wallet_name, restart_cb) return self.wallet_synced = True
def test_net_byte(): load_test_config() assert struct.unpack(b'B', get_p2pk_vbyte())[0] == 0x6f assert struct.unpack(b'B', get_p2sh_vbyte())[0] == 196
def receive_utxos(self, ioauth_data): """Triggered when the daemon returns utxo data from makers who responded; this is the completion of phase 1 of the protocol """ if self.aborted: return (False, "User aborted") rejected_counterparties = [] #Enough data, but need to authorize against the btc pubkey first. for nick, nickdata in ioauth_data.iteritems(): utxo_list, auth_pub, cj_addr, change_addr, btc_sig, maker_pk = nickdata if not self.auth_counterparty(btc_sig, auth_pub, maker_pk): jlog.debug( "Counterparty encryption verification failed, aborting") #This counterparty must be rejected rejected_counterparties.append(nick) for rc in rejected_counterparties: del ioauth_data[rc] self.maker_utxo_data = {} for nick, nickdata in ioauth_data.iteritems(): utxo_list, auth_pub, cj_addr, change_addr, btc_sig, maker_pk = nickdata self.utxos[nick] = utxo_list utxo_data = jm_single().bc_interface.query_utxo_set( self.utxos[nick]) if None in utxo_data: jlog.debug(('ERROR outputs unconfirmed or already spent. ' 'utxo_data={}').format(pprint.pformat(utxo_data))) # when internal reviewing of makers is created, add it here to # immediately quit; currently, the timeout thread suffices. continue #Complete maker authorization: #Extract the address fields from the utxos #Construct the Bitcoin address for the auth_pub field #Ensure that at least one address from utxos corresponds. input_addresses = [d['address'] for d in utxo_data] auth_address = btc.pubkey_to_address(auth_pub, get_p2pk_vbyte()) if not auth_address in input_addresses: jlog.warn("ERROR maker's (" + nick + ")" " authorising pubkey is not included " "in the transaction: " + str(auth_address)) #this will not be added to the transaction, so we will have #to recheck if we have enough continue total_input = sum([d['value'] for d in utxo_data]) real_cjfee = calc_cj_fee(self.orderbook[nick]['ordertype'], self.orderbook[nick]['cjfee'], self.cjamount) change_amount = (total_input - self.cjamount - self.orderbook[nick]['txfee'] + real_cjfee) # certain malicious and/or incompetent liquidity providers send # inputs totalling less than the coinjoin amount! this leads to # a change output of zero satoshis; this counterparty must be removed. if change_amount < jm_single().DUST_THRESHOLD: fmt = ('ERROR counterparty requires sub-dust change. nick={}' 'totalin={:d} cjamount={:d} change={:d}').format jlog.debug(fmt(nick, total_input, self.cjamount, change_amount)) jlog.warn("Invalid change, too small, nick= " + nick) continue self.outputs.append({ 'address': change_addr, 'value': change_amount }) fmt = ('fee breakdown for {} totalin={:d} ' 'cjamount={:d} txfee={:d} realcjfee={:d}').format jlog.debug( fmt(nick, total_input, self.cjamount, self.orderbook[nick]['txfee'], real_cjfee)) self.outputs.append({'address': cj_addr, 'value': self.cjamount}) self.cjfee_total += real_cjfee self.maker_txfee_contributions += self.orderbook[nick]['txfee'] self.maker_utxo_data[nick] = utxo_data #Apply business logic of how many counterparties are enough: if len(self.maker_utxo_data.keys()) < jm_single().config.getint( "POLICY", "minimum_makers"): self.taker_info_callback("INFO", "Not enough counterparties, aborting.") return (False, "Not enough counterparties responded to fill, giving up") self.taker_info_callback("INFO", "Got all parts, enough to build a tx") self.nonrespondants = list(self.maker_utxo_data.keys()) my_total_in = sum( [va['value'] for u, va in self.input_utxos.iteritems()]) if self.my_change_addr: #Estimate fee per choice of next/3/6 blocks targetting. estimated_fee = estimate_tx_fee(len(sum(self.utxos.values(), [])), len(self.outputs) + 2) jlog.info("Based on initial guess: " + str(self.total_txfee) + ", we estimated a miner fee of: " + str(estimated_fee)) #reset total self.total_txfee = estimated_fee my_txfee = max(self.total_txfee - self.maker_txfee_contributions, 0) my_change_value = (my_total_in - self.cjamount - self.cjfee_total - my_txfee) #Since we could not predict the maker's inputs, we may end up needing #too much such that the change value is negative or small. Note that #we have tried to avoid this based on over-estimating the needed amount #in SendPayment.create_tx(), but it is still a possibility if one maker #uses a *lot* of inputs. if self.my_change_addr and my_change_value <= 0: raise ValueError("Calculated transaction fee of: " + str(self.total_txfee) + " is too large for our inputs;Please try again.") elif self.my_change_addr and my_change_value <= jm_single( ).BITCOIN_DUST_THRESHOLD: jlog.info("Dynamically calculated change lower than dust: " + str(my_change_value) + "; dropping.") self.my_change_addr = None my_change_value = 0 jlog.info( 'fee breakdown for me totalin=%d my_txfee=%d makers_txfee=%d cjfee_total=%d => changevalue=%d' % (my_total_in, my_txfee, self.maker_txfee_contributions, self.cjfee_total, my_change_value)) if self.my_change_addr is None: if my_change_value != 0 and abs(my_change_value) != 1: # seems you wont always get exactly zero because of integer # rounding so 1 satoshi extra or fewer being spent as miner # fees is acceptable jlog.debug(('WARNING CHANGE NOT BEING ' 'USED\nCHANGEVALUE = {}').format(my_change_value)) else: self.outputs.append({ 'address': self.my_change_addr, 'value': my_change_value }) self.utxo_tx = [ dict([('output', u)]) for u in sum(self.utxos.values(), []) ] self.outputs.append({ 'address': self.coinjoin_address(), 'value': self.cjamount }) random.shuffle(self.utxo_tx) random.shuffle(self.outputs) tx = btc.mktx(self.utxo_tx, self.outputs) jlog.debug('obtained tx\n' + pprint.pformat(btc.deserialize(tx))) self.latest_tx = btc.deserialize(tx) for index, ins in enumerate(self.latest_tx['ins']): utxo = ins['outpoint']['hash'] + ':' + str( ins['outpoint']['index']) if utxo not in self.input_utxos.keys(): continue # placeholders required ins['script'] = 'deadbeef' self.taker_info_callback("INFO", "Built tx, sending to counterparties.") return (True, self.maker_utxo_data.keys(), tx)
def get_key_from_addr(self, addr): self.ensure_wallet_unlocked() wifkey = jm_single().bc_interface.rpc('dumpprivkey', [addr]) return btc.from_wif_privkey(wifkey, vbyte=get_p2pk_vbyte())
def get_vbyte(self): return get_p2pk_vbyte()
def get_addr(self, mixing_depth, forchange, i): return btc.privtoaddr(self.get_key(mixing_depth, forchange, i), magicbyte=get_p2pk_vbyte())
def read_wallet_file_data(self, filename, pwd=None, wallet_dir=None): self.path = None wallet_dir = wallet_dir if wallet_dir else 'wallets' self.index_cache = [[0, 0]] * self.max_mix_depth path = os.path.join(wallet_dir, filename) if not os.path.isfile(path): if get_network() == 'testnet': log.debug('filename interpreted as seed, only available in ' 'testnet because this probably has lower entropy') return "FAKESEED" + filename else: raise IOError('wallet file not found') if not pwd: log.info("Password required for non-testnet seed wallet") return None self.path = path fd = open(path, 'r') walletfile = fd.read() fd.close() walletdata = json.loads(walletfile) if walletdata['network'] != get_network(): raise ValueError('wallet network(%s) does not match ' 'joinmarket configured network(%s)' % (walletdata['network'], get_network())) if 'index_cache' in walletdata: self.index_cache = walletdata['index_cache'] if self.max_mix_depth > len(self.index_cache): #This can happen e.g. in tumbler when we need more mixdepths #than currently exist. Since we have no info for those extra #depths, we must default to (0,0) (but sync should find used #adddresses). self.index_cache += [[0, 0]] * (self.max_mix_depth - len(self.index_cache)) password_key = btc.bin_dbl_sha256(pwd) if 'encrypted_seed' in walletdata: #accept old field name encrypted_entropy = walletdata['encrypted_seed'] elif 'encrypted_entropy' in walletdata: encrypted_entropy = walletdata['encrypted_entropy'] try: decrypted_entropy = decryptData( password_key, encrypted_entropy.decode('hex')).encode('hex') # there is a small probability of getting a valid PKCS7 # padding by chance from a wrong password; sanity check the # seed length if len(decrypted_entropy) != 32: raise ValueError except ValueError: log.info('Incorrect password') return None if 'encrypted_mnemonic_extension' in walletdata: try: cleartext = decryptData( password_key, walletdata['encrypted_mnemonic_extension'].decode('hex')) #theres a small chance of not getting a ValueError from the wrong # password so also check the sum if cleartext[-9] != '\xff': raise ValueError chunks = cleartext.split('\xff') if len(chunks) < 3 or cleartext[-8:] != btc.dbl_sha256( chunks[1])[:8]: raise ValueError mnemonic_extension = chunks[1] except ValueError: log.info('incorrect password') return None else: mnemonic_extension = None if self.storepassword: self.password_key = password_key self.walletdata = walletdata if 'imported_keys' in walletdata: for epk_m in walletdata['imported_keys']: privkey = decryptData( password_key, epk_m['encrypted_privkey'].decode('hex')).encode('hex') #Imported keys are stored as 32 byte strings only, so the #second version below is sufficient, really. if len(privkey) != 64: raise Exception( "Unexpected privkey format; already compressed?:" + privkey) privkey += "01" if epk_m['mixdepth'] not in self.imported_privkeys: self.imported_privkeys[epk_m['mixdepth']] = [] self.addr_cache[btc.privtoaddr( privkey, magicbyte=get_p2pk_vbyte())] = ( epk_m['mixdepth'], -1, len(self.imported_privkeys[epk_m['mixdepth']])) self.imported_privkeys[epk_m['mixdepth']].append(privkey) if mnemonic_extension: return (decrypted_entropy, mnemonic_extension) else: return decrypted_entropy
def script_to_address(self, script): """Return the address for a given output script, which will be p2pkh for the default Wallet object, and reading the correct network byte from the config. """ return btc.script_to_address(script, get_p2pk_vbyte())
def test_net_byte(): load_program_config() assert get_p2pk_vbyte() == 0x6f assert get_p2sh_vbyte() == 196
def get_vbyte(): return get_p2pk_vbyte()