def get_commitment(self, utxos, amount):
"""Create commitment to fulfil anti-DOS requirement of makers,
storing the corresponding reveal/proof data for next step.
"""
while True:
self.commitment, self.reveal_commitment = self.commitment_creator(
self.wallet, utxos, amount)
if (self.commitment) or (jm_single().wait_for_commitments == 0):
break
log.info("Failed to source commitments, waiting 3 minutes")
time.sleep(3 * 60)
if not self.commitment:
log.error("Cannot construct transaction, failed to generate "
"commitment, shutting down. Please read commitments_debug.txt "
"for some information on why this is, and what can be "
"done to remedy it.")
#TODO: would like to raw_input here to show the user, but
#interactivity is undesirable here.
#Test only:
if jm_single().config.get(
"BLOCKCHAIN", "blockchain_source") == 'regtest':
raise btc.PoDLEError("For testing raising podle exception")
#The timeout/recovery code is designed to handle non-responsive
#counterparties, but this condition means that the current bot
#is not able to create transactions following its *own* rules,
#so shutting down is appropriate no matter what style
#of bot this is.
import thread
thread.interrupt_main()
return False
return True
def sync_unspent(self, wallet):
from joinmarket.wallet import BitcoinCoreWallet
if isinstance(wallet, BitcoinCoreWallet):
return
st = time.time()
wallet_name = self.get_wallet_name(wallet)
wallet.unspent = {}
listunspent_args = []
if 'listunspent_args' in jm_single().config.options('POLICY'):
listunspent_args = ast.literal_eval(
jm_single().config.get('POLICY', 'listunspent_args'))
unspent_list = self.rpc('listunspent', listunspent_args)
for u in unspent_list:
if 'account' not in u:
continue
if u['account'] != wallet_name:
continue
if u['address'] not in wallet.addr_cache:
continue
wallet.unspent[u['txid'] + ':' + str(u['vout'])] = {
'address': u['address'],
'value': int(Decimal(str(u['amount'])) * Decimal('1e8'))}
et = time.time()
log.debug('bitcoind sync_unspent took ' + str((et - st)) + 'sec')
def recv_tx(self, nick, txhex):
try:
self.tx = btc.deserialize(txhex)
except IndexError as e:
self.maker.msgchan.send_error(nick, 'malformed txhex. ' + repr(e))
log.info('obtained tx\n' + pprint.pformat(self.tx))
goodtx, errmsg = self.verify_unsigned_tx(self.tx)
if not goodtx:
log.info('not a good tx, reason=' + errmsg)
self.maker.msgchan.send_error(nick, errmsg)
# TODO: the above 3 errors should be encrypted, but it's a bit messy.
log.info('goodtx')
sigs = []
for index, ins in enumerate(self.tx['ins']):
utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index'])
if utxo not in self.utxos:
continue
addr = self.utxos[utxo]['address']
txs = btc.sign(txhex, index,
self.maker.wallet.get_key_from_addr(addr))
sigs.append(base64.b64encode(btc.deserialize(txs)['ins'][index][
'script'].decode('hex')))
# len(sigs) > 0 guarenteed since i did verify_unsigned_tx()
jm_single().bc_interface.add_tx_notify(
self.tx, self.unconfirm_callback,
self.confirm_callback, self.cj_addr)
self.maker.msgchan.send_sigs(nick, sigs)
self.maker.active_orders[nick] = None
def push(self):
tx = btc.serialize(self.latest_tx)
log.debug('\n' + tx)
self.txid = btc.txhash(tx)
log.debug('txid = ' + self.txid)
tx_broadcast = jm_single().config.get('POLICY', 'tx_broadcast')
if tx_broadcast == 'self':
pushed = jm_single().bc_interface.pushtx(tx)
elif tx_broadcast in ['random-peer', 'not-self']:
n = len(self.active_orders)
if tx_broadcast == 'random-peer':
i = random.randrange(n + 1)
else:
i = random.randrange(n)
if i == n:
pushed = jm_single().bc_interface.pushtx(tx)
else:
self.msgchan.push_tx(self.active_orders.keys()[i], tx)
pushed = True
elif tx_broadcast == 'random-maker':
crow = self.db.execute(
'SELECT DISTINCT counterparty FROM orderbook ORDER BY ' +
'RANDOM() LIMIT 1;'
).fetchone()
counterparty = crow['counterparty']
log.debug('pushing tx to ' + counterparty)
self.msgchan.push_tx(counterparty, tx)
pushed = True
if not pushed:
log.debug('unable to pushtx')
return pushed
def do_HEAD(self):
pages = ('/walletnotify?', '/alertnotify?')
if self.path.startswith('/walletnotify?'):
txid = self.path[len(pages[0]):]
if not re.match('^[0-9a-fA-F]*$', txid):
log.debug('not a txid')
return
tx = self.btcinterface.rpc('getrawtransaction', [txid])
if not re.match('^[0-9a-fA-F]*$', tx):
log.debug('not a txhex')
return
txd = btc.deserialize(tx)
tx_output_set = set([(sv['script'], sv['value']) for sv in txd[
'outs']])
unconfirmfun, confirmfun = None, None
for tx_out, ucfun, cfun in self.btcinterface.txnotify_fun:
if tx_out == tx_output_set:
unconfirmfun = ucfun
confirmfun = cfun
break
if unconfirmfun is None:
log.debug('txid=' + txid + ' not being listened for')
else:
# on rare occasions people spend their output without waiting
# for a confirm
txdata = None
for n in range(len(txd['outs'])):
txdata = self.btcinterface.rpc('gettxout', [txid, n, True])
if txdata is not None:
break
assert txdata is not None
if txdata['confirmations'] == 0:
unconfirmfun(txd, txid)
# TODO pass the total transfered amount value here somehow
# wallet_name = self.get_wallet_name()
# amount =
# bitcoin-cli move wallet_name "" amount
log.debug('ran unconfirmfun')
else:
confirmfun(txd, txid, txdata['confirmations'])
self.btcinterface.txnotify_fun.remove(
(tx_out, unconfirmfun, confirmfun))
log.debug('ran confirmfun')
elif self.path.startswith('/alertnotify?'):
jm_single().core_alert[0] = urllib.unquote(self.path[len(pages[1]):])
log.debug('Got an alert!\nMessage=' + jm_single().core_alert[0])
else:
log.debug('ERROR: This is not a handled URL path. You may want to check your notify URL for typos.')
os.system('curl -sI --connect-timeout 1 http://localhost:' + str(
self.base_server.server_address[1] + 1) + self.path)
self.send_response(200)
# self.send_header('Connection', 'close')
self.end_headers()
def on_order_seen(self, counterparty, oid, ordertype, minsize, maxsize,
txfee, cjfee):
try:
self.dblock.acquire(True)
if int(oid) < 0 or int(oid) > sys.maxint:
log.debug(
"Got invalid order ID: " + oid + " from " + counterparty)
return
# delete orders eagerly, so in case a buggy maker sends an
# invalid offer, we won't accidentally !fill based on the ghost
# of its previous message.
self.db.execute(("DELETE FROM orderbook WHERE counterparty=? "
"AND oid=?;"), (counterparty, oid))
# now validate the remaining fields
if int(minsize) < 0 or int(minsize) > 21 * 10 ** 14:
log.debug("Got invalid minsize: {} from {}".format(
minsize, counterparty))
return
if int(minsize) < jm_single().DUST_THRESHOLD:
minsize = jm_single().DUST_THRESHOLD
log.debug("{} has dusty minsize, capping at {}".format(
counterparty, minsize))
# do not pass return, go not drop this otherwise fine offer
if int(maxsize) < 0 or int(maxsize) > 21 * 10 ** 14:
log.debug("Got invalid maxsize: " + maxsize + " from " +
counterparty)
return
if int(txfee) < 0:
log.debug("Got invalid txfee: {} from {}".format(
txfee, counterparty))
return
if int(minsize) > int(maxsize):
fmt = ("Got minsize bigger than maxsize: {} - {} "
"from {}").format
log.debug(fmt(minsize, maxsize, counterparty))
return
if ordertype == 'absoffer' and not isinstance(cjfee, int):
try:
cjfee = int(cjfee)
except ValueError:
log.debug("Got non integer coinjoin fee: " + str(cjfee) +
" for an absoffer from " + counterparty)
return
self.db.execute(
'INSERT INTO orderbook VALUES(?, ?, ?, ?, ?, ?, ?);',
(counterparty, oid, ordertype, minsize, maxsize, txfee,
str(Decimal(
cjfee)))) # any parseable Decimal is a valid cjfee
except InvalidOperation:
log.debug("Got invalid cjfee: " + cjfee + " from " + counterparty)
except Exception as e:
log.debug("Error parsing order " + oid + " from " + counterparty)
log.debug("Exception was: " + repr(e))
finally:
self.dblock.release()
def auth_counterparty(self, nick, cr):
#deserialize the commitment revelation
cr_dict = btc.PoDLE.deserialize_revelation(cr)
#check the validity of the proof of discrete log equivalence
tries = jm_single().config.getint("POLICY", "taker_utxo_retries")
def reject(msg):
log.info("Counterparty commitment not accepted, reason: " + msg)
return False
if not btc.verify_podle(cr_dict['P'], cr_dict['P2'], cr_dict['sig'],
cr_dict['e'], self.maker.commit,
index_range=range(tries)):
reason = "verify_podle failed"
return reject(reason)
#finally, check that the proffered utxo is real, old enough, large enough,
#and corresponds to the pubkey
res = jm_single().bc_interface.query_utxo_set([cr_dict['utxo']],
includeconf=True)
if len(res) != 1 or not res[0]:
reason = "authorizing utxo is not valid"
return reject(reason)
age = jm_single().config.getint("POLICY", "taker_utxo_age")
if res[0]['confirms'] < age:
reason = "commitment utxo not old enough: " + str(res[0]['confirms'])
return reject(reason)
reqd_amt = int(self.cj_amount * jm_single().config.getint(
"POLICY", "taker_utxo_amtpercent") / 100.0)
if res[0]['value'] < reqd_amt:
reason = "commitment utxo too small: " + str(res[0]['value'])
return reject(reason)
if res[0]['address'] != btc.pubkey_to_address(cr_dict['P'],
get_p2pk_vbyte()):
reason = "Invalid podle pubkey: " + str(cr_dict['P'])
return reject(reason)
# authorisation of taker passed
# Send auth request to taker
# Need to choose an input utxo pubkey to sign with
# (no longer using the coinjoin pubkey from 0.2.0)
# Just choose the first utxo in self.utxos and retrieve key from wallet.
auth_address = self.utxos[self.utxos.keys()[0]]['address']
auth_key = self.maker.wallet.get_key_from_addr(auth_address)
auth_pub = btc.privtopub(auth_key)
btc_sig = btc.ecdsa_sign(self.kp.hex_pk(), auth_key)
self.maker.msgchan.send_ioauth(nick, self.utxos.keys(), auth_pub,
self.cj_addr, self.change_addr, btc_sig)
#In case of *blacklisted (ie already used) commitments, we already
#broadcasted them on receipt; in case of valid, and now used commitments,
#we broadcast them here, and not early - to avoid accidentally
#blacklisting commitments that are broadcast between makers in real time
#for the same transaction.
self.maker.transfer_commitment(self.maker.commit)
#now persist the fact that the commitment is actually used.
check_utxo_blacklist(self.maker.commit, persist=True)
return True
def estimate_tx_fee(ins, outs, txtype='p2pkh'):
'''Returns an estimate of the number of satoshis required
for a transaction with the given number of inputs and outputs,
based on information from the blockchain interface.
'''
tx_estimated_bytes = btc.estimate_tx_size(ins, outs, txtype)
log.debug("Estimated transaction size: "+str(tx_estimated_bytes))
fee_per_kb = jm_single().bc_interface.estimate_fee_per_kb(
jm_single().config.getint("POLICY","tx_fees"))
log.debug("got estimated tx bytes: "+str(tx_estimated_bytes))
return int((tx_estimated_bytes * fee_per_kb)/Decimal(1000.0))
def __init__(self, maker, nick, oid, amount, taker_pk):
self.tx = None
self.i_utxo_pubkey = None
self.maker = maker
self.oid = oid
self.cj_amount = amount
if self.cj_amount <= jm_single().DUST_THRESHOLD:
self.maker.msgchan.send_error(nick, 'amount below dust threshold')
# the btc pubkey of the utxo that the taker plans to use as input
self.taker_pk = taker_pk
# create DH keypair on the fly for this Order object
self.kp = init_keypair()
# the encryption channel crypto box for this Order object
self.crypto_box = as_init_encryption(self.kp,
init_pubkey(taker_pk))
order_s = [o for o in maker.orderlist if o['oid'] == oid]
if len(order_s) == 0:
self.maker.msgchan.send_error(nick, 'oid not found')
order = order_s[0]
if amount < order['minsize'] or amount > order['maxsize']:
self.maker.msgchan.send_error(nick, 'amount out of range')
self.ordertype = order['ordertype']
self.txfee = order['txfee']
self.cjfee = order['cjfee']
log.debug('new cjorder nick=%s oid=%d amount=%d' % (nick, oid, amount))
self.utxos, self.cj_addr, self.change_addr = maker.oid_to_order(
self, oid, amount)
self.maker.wallet.update_cache_index()
if not self.utxos:
self.maker.msgchan.send_error(
nick, 'unable to fill order constrained by dust avoidance')
# TODO make up orders offers in a way that this error cant appear
# check nothing has messed up with the wallet code, remove this
# code after a while
import pprint
log.debug('maker utxos = ' + pprint.pformat(self.utxos))
utxo_list = self.utxos.keys()
utxo_data = jm_single().bc_interface.query_utxo_set(utxo_list)
if None in utxo_data:
log.debug('wrongly using an already spent utxo. utxo_data = ' +
pprint.pformat(utxo_data))
sys.exit(0)
for utxo, data in zip(utxo_list, utxo_data):
if self.utxos[utxo]['value'] != data['value']:
fmt = 'wrongly labeled utxo, expected value: {} got {}'.format
log.debug(fmt(self.utxos[utxo]['value'], data['value']))
sys.exit(0)
# always a new address even if the order ends up never being
# furfilled, you dont want someone pretending to fill all your
# orders to find out which addresses you use
self.maker.msgchan.send_pubkey(nick, self.kp.hex_pk())
def confirm_callback(self, txd, txid, confirmations):
self.maker.wallet_unspent_lock.acquire()
try:
jm_single().bc_interface.sync_unspent(self.maker.wallet)
finally:
self.maker.wallet_unspent_lock.release()
log.info('tx in a block')
log.info('earned = ' + str(self.real_cjfee - self.txfee))
to_cancel, to_announce = self.maker.on_tx_confirmed(self, confirmations,
txid)
self.maker.modify_orders(to_cancel, to_announce)
def estimate_tx_fee(ins, outs, txtype='p2pkh'):
'''Returns an estimate of the number of satoshis required
for a transaction with the given number of inputs and outputs,
based on information from the blockchain interface.
'''
tx_estimated_bytes = btc.estimate_tx_size(ins, outs, txtype)
log.debug("Estimated transaction size: "+str(tx_estimated_bytes))
fee_per_kb = jm_single().bc_interface.estimate_fee_per_kb(
jm_single().config.getint("POLICY", "tx_fees"))
absurd_fee = jm_single().config.getint("POLICY", "absurd_fee_per_kb")
if fee_per_kb > absurd_fee:
#This error is considered critical; for safety reasons, shut down.
raise ValueError("Estimated fee per kB greater than absurd value: " + \
str(absurd_fee) + ", quitting.")
log.debug("got estimated tx bytes: "+str(tx_estimated_bytes))
return int((tx_estimated_bytes * fee_per_kb)/Decimal(1000.0))
def _pubmsg(self, message):
line = "PRIVMSG " + self.channel + " :" + message
assert len(line) <= MAX_PRIVMSG_LEN
ob = False
if any([x in line for x in jm_single().ordername_list]):
ob = True
self.send_raw(line, ob)
def __init__(self,
msgchan,
wallet,
db,
cj_amount,
orders,
input_utxos,
my_cj_addr,
my_change_addr,
total_txfee,
finishcallback,
choose_orders_recover,
commitment_creator
):
"""
if my_change is None then there wont be a change address
thats used if you want to entirely coinjoin one utxo with no change left over
orders is the orders you want to fill {'counterpartynick': order1, 'cp2': order2}
each order object is a dict of properties {'oid': 0, 'maxsize': 2000000, 'minsize':
5000, 'cjfee': 10000, 'txfee': 5000}
"""
log.info(
'starting cj to ' + str(my_cj_addr) + ' with change at ' + str(
my_change_addr))
# parameters
self.msgchan = msgchan
self.wallet = wallet
self.db = db
self.cj_amount = cj_amount
self.active_orders = dict(orders)
self.input_utxos = input_utxos
self.finishcallback = finishcallback
self.total_txfee = total_txfee
self.my_cj_addr = my_cj_addr
self.my_change_addr = my_change_addr
self.choose_orders_recover = choose_orders_recover
self.commitment_creator = commitment_creator
self.timeout_lock = threading.Condition() # used to wait() and notify()
# used to restrict access to certain variables across threads
self.timeout_thread_lock = threading.Condition()
self.end_timeout_thread = False
self.maker_timeout_sec = jm_single().maker_timeout_sec
CoinJoinTX.TimeoutThread(self).start()
# state variables
self.txid = None
self.cjfee_total = 0
self.maker_txfee_contributions = 0
self.nonrespondants = list(self.active_orders.keys())
self.all_responded = False
self.latest_tx = None
# None means they belong to me
self.utxos = {None: self.input_utxos.keys()}
self.outputs = []
# create DH keypair on the fly for this Tx object
self.kp = init_keypair()
self.crypto_boxes = {}
if not self.get_commitment(input_utxos, self.cj_amount):
return
self.msgchan.fill_orders(self.active_orders, self.cj_amount,
self.kp.hex_pk(), self.commitment)
def estimate_fee_per_kb(self, N):
if not self.absurd_fees:
return super(RegtestBitcoinCoreInterface,
self).estimate_fee_per_kb(N)
else:
return jm_single().config.getint("POLICY",
"absurd_fee_per_kb") + 100
def add_tx_notify(self,
txd,
unconfirmfun,
confirmfun,
notifyaddr,
timeoutfun=None):
if not self.notifythread:
self.notifythread = BitcoinCoreNotifyThread(self)
self.notifythread.start()
one_addr_imported = False
for outs in txd['outs']:
addr = btc.script_to_address(outs['script'], get_p2pk_vbyte())
if self.rpc('getaccount', [addr]) != '':
one_addr_imported = True
break
if not one_addr_imported:
self.rpc('importaddress', [notifyaddr, 'joinmarket-notify', False])
tx_output_set = set([(sv['script'], sv['value']) for sv in txd['outs']])
self.txnotify_fun.append((tx_output_set, unconfirmfun, confirmfun,
timeoutfun, False))
#create unconfirm timeout here, create confirm timeout in the other thread
if timeoutfun:
threading.Timer(jm_single().config.getint('TIMEOUT',
'unconfirm_timeout_sec'),
bitcoincore_timeout_callback,
args=(False, tx_output_set, self.txnotify_fun,
timeoutfun)).start()
def on_set_topic(newtopic):
chunks = newtopic.split('|')
for msg in chunks[1:]:
try:
msg = msg.strip()
params = msg.split(' ')
min_version = int(params[0])
max_version = int(params[1])
alert = msg[msg.index(params[1]) + len(params[1]):].strip()
except ValueError, IndexError:
continue
if min_version < jm_single().JM_VERSION < max_version:
print('=' * 60)
print('JOINMARKET ALERT')
print(alert)
print('=' * 60)
jm_single().joinmarket_alert[0] = alert
def on_push_tx(self, nick, txhex):
log.debug('received txhex from ' + nick + ' to push\n' + txhex)
pushed = jm_single().bc_interface.pushtx(txhex)
if pushed:
log.debug('pushed tx ' + btc.txhash(txhex))
else:
log.debug('failed to push tx sent by taker')
self.msgchan.send_error(nick, 'Unable to push tx')
def __init__(self, fromaccount):
super(BitcoinCoreWallet, self).__init__()
if not isinstance(jm_single().bc_interface,
BitcoinCoreInterface):
raise RuntimeError('Bitcoin Core wallet can only be used when '
'blockchain interface is BitcoinCoreInterface')
self.fromaccount = fromaccount
self.max_mix_depth = 1
def ensure_wallet_unlocked():
wallet_info = jm_single().bc_interface.rpc('getwalletinfo', [])
if 'unlocked_until' in wallet_info and wallet_info[
'unlocked_until'] <= 0:
while True:
password = getpass(
'Enter passphrase to unlock wallet: ')
if password == '':
raise RuntimeError('Aborting wallet unlock')
try:
# TODO cleanly unlock wallet after use, not with arbitrary timeout
jm_single().bc_interface.rpc(
'walletpassphrase', [password, 10])
break
except jm_single().JsonRpcError as exc:
if exc.code != -14:
raise exc
def on_push_tx(self, nick, txhex):
log.debug('received txhex from ' + nick + ' to push\n' + txhex)
pushed = jm_single().bc_interface.pushtx(txhex)
if pushed[0]:
log.debug('pushed tx ' + str(pushed[1]))
else:
log.debug('failed to push tx, reason: '+str(pushed[1]))
self.msgchan.send_error(nick, 'Unable to push tx')
def on_commitment_seen(self, nick, commitment):
"""Triggered when we see a commitment for blacklisting
appear in the public pit channel. If the policy is set,
we blacklist this commitment.
"""
if jm_single().config.has_option("POLICY", "accept_commitment_broadcasts"):
blacklist_add = jm_single().config.getint("POLICY",
"accept_commitment_broadcasts")
else:
blacklist_add = 0
if blacklist_add > 0:
#just add if necessary, ignore return value.
check_utxo_blacklist(commitment, persist=True)
log.debug("Received commitment broadcast by other maker: " + str(
commitment) + ", now blacklisted.")
else:
log.debug("Received commitment broadcast by other maker: " + str(
commitment) + ", ignored.")
def recover_from_nonrespondants(self):
def restart():
self.end_timeout_thread = True
if self.finishcallback is not None:
self.finishcallback(self)
# finishcallback will check if self.all_responded is True
# and will know it came from here
log.info('nonresponding makers = ' + str(self.nonrespondants))
# if there is no choose_orders_recover then end and call finishcallback
# so the caller can handle it in their own way, notable for sweeping
# where simply replacing the makers wont work
if not self.choose_orders_recover:
restart()
return
if self.latest_tx is None:
# nonresponding to !fill-!auth, proceed with transaction anyway as long
# as number of makers is at least POLICY.minimum_makers (and not zero,
# i.e. disallow this kind of continuation).
log.debug('nonresponse to !fill')
for nr in self.nonrespondants:
del self.active_orders[nr]
minmakers = jm_single().config.getint("POLICY", "minimum_makers")
if len(self.active_orders.keys()) >= minmakers and minmakers != 0:
log.info("Completing the transaction with: " + str(
len(self.active_orders.keys())) + " makers.")
self.recv_txio(None, None, None, None, None)
elif minmakers == 0:
#Revert to the old algorithm: re-source number of orders
#still needed, but ignoring non-respondants and currently active
new_orders, new_makers_fee = self.choose_orders_recover(
self.cj_amount, len(self.nonrespondants),
self.nonrespondants,
self.active_orders.keys())
for nick, order in new_orders.iteritems():
self.active_orders[nick] = order
self.nonrespondants = list(new_orders.keys())
log.debug(('new active_orders = {} \nnew nonrespondants = '
'{}').format(
pprint.pformat(self.active_orders),
pprint.pformat(self.nonrespondants)))
#Re-source commitment; previous attempt will have been blacklisted
self.get_commitment(self.input_utxos, self.cj_amount)
self.msgchan.fill_orders(new_orders, self.cj_amount,
self.kp.hex_pk(), self.commitment)
else:
log.info("Too few makers responded to complete, trying again.")
restart()
else:
log.debug('nonresponse to !tx')
# have to restart tx from the beginning
restart()
def add_watchonly_addresses(self, addr_list, wallet_name):
log.debug('importing ' + str(len(addr_list)) +
' addresses into account ' + wallet_name)
for addr in addr_list:
self.rpc('importaddress', [addr, wallet_name, False])
if jm_single().config.get(
"BLOCKCHAIN", "blockchain_source") != 'regtest':
print('restart Bitcoin Core with -rescan if you\'re '
'recovering an existing wallet from backup seed')
print(' otherwise just restart this joinmarket script')
sys.exit(0)
def push(self):
tx = btc.serialize(self.latest_tx)
log.debug('\n' + tx)
self.txid = btc.txhash(tx)
log.info('txid = ' + self.txid)
tx_broadcast = jm_single().config.get('POLICY', 'tx_broadcast')
if tx_broadcast == 'self':
pushed = jm_single().bc_interface.pushtx(tx)
elif tx_broadcast in ['random-peer', 'not-self']:
n = len(self.active_orders)
if tx_broadcast == 'random-peer':
i = random.randrange(n + 1)
else:
i = random.randrange(n)
if i == n:
pushed = jm_single().bc_interface.pushtx(tx)
else:
self.msgchan.push_tx(self.active_orders.keys()[i], tx)
pushed = True
elif tx_broadcast == 'random-maker':
crow = self.db.execute(
'SELECT DISTINCT counterparty FROM orderbook ORDER BY ' +
'RANDOM() LIMIT 1;'
).fetchone()
counterparty = crow['counterparty']
log.info('pushing tx to ' + counterparty)
self.msgchan.push_tx(counterparty, tx)
pushed = True
elif tx_broadcast == 'tor':
socks5_host = jm_single().config.get("MESSAGING",
"socks5_host").split(",")[0]
socks5_port = int(jm_single().config.get("MESSAGING",
"socks5_port").split(",")[0])
pushed = tor_broadcast_tx(tx, (socks5_host, socks5_port),
testnet=(get_network() == "testnet"))
if not pushed:
log.error('unable to pushtx')
return pushed
def push(self, txd):
tx = btc.serialize(txd)
log.debug('\n' + tx)
log.debug('txid = ' + btc.txhash(tx))
# TODO send to a random maker or push myself
# TODO need to check whether the other party sent it
# self.msgchan.push_tx(self.active_orders.keys()[0], txhex)
pushed = jm_single().bc_interface.pushtx(tx)
if pushed[0]:
self.txid = btc.txhash(tx)
else:
log.debug('unable to pushtx, reason: '+str(pushed[1]))
return pushed[0]
def __init__(self, mchannels):
self.mchannels = mchannels
#To keep track of chosen channels
#for private messaging counterparties.
self.active_channels = {}
#To keep track of message channel status;
#0: not started 1: started 2: failed/broken/inactive
self.mc_status = dict([(x, 0) for x in self.mchannels])
#To keep track of counterparties having at least once
#made their presence known on a channel
self.nicks_seen = {}
for mc in self.mchannels:
self.nicks_seen[mc] = set()
#callback to mark nicks as seen when they privmsg
mc.on_privmsg_trigger = self.on_privmsg
#keep track of whether we want to deliberately
#shut down the connections
self.give_up = False
#only allow on_welcome() to fire once.
self.welcomed = False
#control access
self.mc_lock = threading.Lock()
#Create an ephemeral keypair for the duration
#of this run, same across all message channels,
#and set the nickname for all message channels using it.
self.nick_priv = hashlib.sha256(os.urandom(16)).hexdigest() + '01'
self.nick_pubkey = btc.privtopub(self.nick_priv)
self.nick_pkh_raw = hashlib.sha256(self.nick_pubkey).digest()[
:NICK_HASH_LENGTH]
self.nick_pkh = btc.changebase(self.nick_pkh_raw, 256, 58)
#right pad to maximum possible; b58 is not fixed length.
#Use 'O' as one of the 4 not included chars in base58.
self.nick_pkh += 'O' * (NICK_MAX_ENCODED - len(self.nick_pkh))
#The constructed length will be 1 + 1 + NICK_MAX_ENCODED
self.nick = JOINMARKET_NICK_HEADER + str(
jm_single().JM_VERSION) + self.nick_pkh
jm_single().nickname = self.nick
for mc in self.mchannels:
mc.set_nick(self.nick, self.nick_priv, self.nick_pubkey)
def populate_utxo_data():
self.utxos, self.cj_addr, self.change_addr = maker.oid_to_order(
self, oid, amount)
self.maker.wallet.update_cache_index()
if not self.utxos:
self.maker.msgchan.send_error(
nick, 'unable to fill order constrained by dust avoidance')
# TODO make up orders offers in a way that this error cant appear
# check nothing has messed up with the wallet code, remove this
# code after a while
log.debug('maker utxos = ' + pprint.pformat(self.utxos))
utxos = self.utxos.keys()
return (utxos, jm_single().bc_interface.query_utxo_set(utxos))
def get_utxos_by_mixdepth(self):
unspent_list = jm_single().bc_interface.rpc('listunspent', [])
result = {0: {}}
for u in unspent_list:
if not u['spendable']:
continue
if self.fromaccount and (
('account' not in u) or u['account'] !=
self.fromaccount):
continue
result[0][u['txid'] + ':' + str(u['vout'])] = {
'address': u['address'],
'value': int(Decimal(str(u['amount'])) * Decimal('1e8'))}
return result
def __init__(self):
self.max_mix_depth = 0
self.utxo_selector = btc.select # default fallback: upstream
try:
config = jm_single().config
if config.get("POLICY", "merge_algorithm") == "gradual":
self.utxo_selector = select_gradual
elif config.get("POLICY", "merge_algorithm") == "greedy":
self.utxo_selector = select_greedy
elif config.get("POLICY", "merge_algorithm") == "greediest":
self.utxo_selector = select_greediest
elif config.get("POLICY", "merge_algorithm") != "default":
raise Exception("Unknown merge algorithm")
except NoSectionError:
pass
def run(self):
log.debug(('started timeout thread for coinjoin of amount {} to '
'addr {}').format(self.cjtx.cj_amount,
self.cjtx.my_cj_addr))
# how the threading to check for nonresponding makers works like this
# there is a Condition object
# in a loop, call cond.wait(timeout)
# after it returns, check a boolean
# to see if if the messages have arrived
while not self.cjtx.end_timeout_thread:
log.debug('waiting for all replies.. timeout=' + str(
jm_single().maker_timeout_sec))
with self.cjtx.timeout_lock:
self.cjtx.timeout_lock.wait(jm_single().maker_timeout_sec)
if self.cjtx.all_responded:
log.debug(('timeout thread woken by notify(), '
'makers responded in time'))
self.cjtx.all_responded = False
else:
log.debug('timeout thread woken by timeout, '
'makers didnt respond')
with self.cjtx.timeout_thread_lock:
self.cjtx.recover_from_nonrespondants()
def do_HEAD(self):
pages = ('/walletnotify?', '/alertnotify?')
if self.path.startswith('/walletnotify?'):
txid = self.path[len(pages[0]):]
if not re.match('^[0-9a-fA-F]*$', txid):
log.debug('not a txid')
return
try:
tx = self.btcinterface.rpc('getrawtransaction', [txid])
except (JsonRpcError, JsonRpcConnectionError) as e:
log.debug('transaction not found, probably a conflict')
return
if not re.match('^[0-9a-fA-F]*$', tx):
log.debug('not a txhex')
return
txd = btc.deserialize(tx)
tx_output_set = set([(sv['script'], sv['value'])
for sv in txd['outs']])
txnotify_tuple = None
unconfirmfun, confirmfun, timeoutfun, uc_called = (None, None,
None, None)
for tnf in self.btcinterface.txnotify_fun:
tx_out = tnf[0]
if tx_out == tx_output_set:
txnotify_tuple = tnf
tx_out, unconfirmfun, confirmfun, timeoutfun, uc_called = tnf
break
if unconfirmfun is None:
log.debug('txid=' + txid + ' not being listened for')
else:
# on rare occasions people spend their output without waiting
# for a confirm
txdata = None
for n in range(len(txd['outs'])):
txdata = self.btcinterface.rpc('gettxout', [txid, n, True])
if txdata is not None:
break
assert txdata is not None
if txdata['confirmations'] == 0:
unconfirmfun(txd, txid)
# TODO pass the total transfered amount value here somehow
# wallet_name = self.get_wallet_name()
# amount =
# bitcoin-cli move wallet_name "" amount
self.btcinterface.txnotify_fun.remove(txnotify_tuple)
self.btcinterface.txnotify_fun.append(txnotify_tuple[:-1] +
(True, ))
log.debug('ran unconfirmfun')
if timeoutfun:
threading.Timer(jm_single().config.getfloat(
'TIMEOUT', 'confirm_timeout_hours') * 60 * 60,
bitcoincore_timeout_callback,
args=(True, tx_output_set,
self.btcinterface.txnotify_fun,
timeoutfun)).start()
else:
if not uc_called:
unconfirmfun(txd, txid)
log.debug('saw confirmed tx before unconfirmed, ' +
'running unconfirmfun first')
confirmfun(txd, txid, txdata['confirmations'])
self.btcinterface.txnotify_fun.remove(txnotify_tuple)
log.debug('ran confirmfun')
elif self.path.startswith('/alertnotify?'):
jm_single().core_alert[0] = urllib.unquote(
self.path[len(pages[1]):])
log.debug('Got an alert!\nMessage=' + jm_single().core_alert[0])
else:
log.debug(
'ERROR: This is not a handled URL path. You may want to check your notify URL for typos.'
)
request = urllib2.Request('http://localhost:' +
str(self.base_server.server_address[1] + 1) +
self.path)
request.get_method = lambda: 'HEAD'
try:
urllib2.urlopen(request)
except urllib2.URLError:
pass
self.send_response(200)
# self.send_header('Connection', 'close')
self.end_headers()
def recv_txio(self, nick, utxo_list, auth_pub, cj_addr, change_addr):
if nick not in self.nonrespondants:
log.debug(('recv_txio => nick={} not in '
'nonrespondants {}').format(nick, self.nonrespondants))
return
self.utxos[nick] = utxo_list
utxo_data = jm_single().bc_interface.query_utxo_set(self.utxos[nick])
if None in utxo_data:
log.debug(('ERROR outputs unconfirmed or already spent. '
'utxo_data={}').format(pprint.pformat(utxo_data)))
# when internal reviewing of makers is created, add it here to
# immediately quit; currently, the timeout thread suffices.
return
#Complete maker authorization:
#Extract the address fields from the utxos
#Construct the Bitcoin address for the auth_pub field
#Ensure that at least one address from utxos corresponds.
input_addresses = [d['address'] for d in utxo_data]
auth_address = btc.pubkey_to_address(auth_pub, get_p2pk_vbyte())
if not auth_address in input_addresses:
log.debug("ERROR maker's authorising pubkey is not included "
"in the transaction: " + str(auth_address))
return
total_input = sum([d['value'] for d in utxo_data])
real_cjfee = calc_cj_fee(self.active_orders[nick]['ordertype'],
self.active_orders[nick]['cjfee'],
self.cj_amount)
change_amount = (total_input - self.cj_amount -
self.active_orders[nick]['txfee'] + real_cjfee)
# certain malicious and/or incompetent liquidity providers send
# inputs totalling less than the coinjoin amount! this leads to
# a change output of zero satoshis, so the invalid transaction
# fails harmlessly; let's fail earlier, with a clear message.
if change_amount < jm_single().DUST_THRESHOLD:
fmt = ('ERROR counterparty requires sub-dust change. nick={}'
'totalin={:d} cjamount={:d} change={:d}').format
log.debug(fmt(nick, total_input, self.cj_amount, change_amount))
return # timeout marks this maker as nonresponsive
self.outputs.append({'address': change_addr, 'value': change_amount})
fmt = ('fee breakdown for {} totalin={:d} '
'cjamount={:d} txfee={:d} realcjfee={:d}').format
log.debug(
fmt(nick, total_input, self.cj_amount,
self.active_orders[nick]['txfee'], real_cjfee))
self.outputs.append({'address': cj_addr, 'value': self.cj_amount})
self.cjfee_total += real_cjfee
self.maker_txfee_contributions += self.active_orders[nick]['txfee']
self.nonrespondants.remove(nick)
if len(self.nonrespondants) > 0:
log.debug('nonrespondants = ' + str(self.nonrespondants))
return
log.debug('got all parts, enough to build a tx')
self.nonrespondants = list(self.active_orders.keys())
my_total_in = sum(
[va['value'] for u, va in self.input_utxos.iteritems()])
if self.my_change_addr:
#Estimate fee per choice of next/3/6 blocks targetting.
estimated_fee = estimate_tx_fee(len(sum(self.utxos.values(), [])),
len(self.outputs) + 2)
log.debug("Based on initial guess: " + str(self.total_txfee) +
", we estimated a fee of: " + str(estimated_fee))
#reset total
self.total_txfee = estimated_fee
my_txfee = max(self.total_txfee - self.maker_txfee_contributions, 0)
my_change_value = (my_total_in - self.cj_amount - self.cjfee_total -
my_txfee)
#Since we could not predict the maker's inputs, we may end up needing
#too much such that the change value is negative or small. Note that
#we have tried to avoid this based on over-estimating the needed amount
#in SendPayment.create_tx(), but it is still a possibility if one maker
#uses a *lot* of inputs.
if self.my_change_addr and my_change_value <= 0:
raise ValueError("Calculated transaction fee of: " +
str(self.total_txfee) +
" is too large for our inputs;Please try again.")
elif self.my_change_addr and my_change_value <= jm_single(
).DUST_THRESHOLD:
log.debug("Dynamically calculated change lower than dust: " +
str(my_change_value) + "; dropping.")
self.my_change_addr = None
my_change_value = 0
log.debug(
'fee breakdown for me totalin=%d my_txfee=%d makers_txfee=%d cjfee_total=%d => changevalue=%d'
% (my_total_in, my_txfee, self.maker_txfee_contributions,
self.cjfee_total, my_change_value))
if self.my_change_addr is None:
if my_change_value != 0 and abs(my_change_value) != 1:
# seems you wont always get exactly zero because of integer
# rounding so 1 satoshi extra or fewer being spent as miner
# fees is acceptable
log.debug(('WARNING CHANGE NOT BEING '
'USED\nCHANGEVALUE = {}').format(my_change_value))
else:
self.outputs.append({
'address': self.my_change_addr,
'value': my_change_value
})
self.utxo_tx = [
dict([('output', u)]) for u in sum(self.utxos.values(), [])
]
self.outputs.append({
'address': self.coinjoin_address(),
'value': self.cj_amount
})
random.shuffle(self.utxo_tx)
random.shuffle(self.outputs)
tx = btc.mktx(self.utxo_tx, self.outputs)
log.debug('obtained tx\n' + pprint.pformat(btc.deserialize(tx)))
#Re-calculate a sensible timeout wait based on the throttling
#settings and the tx size.
#Calculation: Let tx size be S; tx undergoes two b64 expansions, 1.8*S
#So we're sending N*1.8*S over the wire, and the
#maximum bytes/sec = B, means we need (1.8*N*S/B) seconds,
#and need to add some leeway for network delays, we just add the
#contents of jm_single().maker_timeout_sec (the user configured value)
self.maker_timeout_sec = (len(tx) * 1.8 * len(self.active_orders.keys(
))) / (B_PER_SEC) + jm_single().maker_timeout_sec
log.debug("Based on transaction size: " + str(len(tx)) +
", calculated time to wait for replies: " +
str(self.maker_timeout_sec))
self.all_responded = True
with self.timeout_lock:
self.timeout_lock.notify()
self.msgchan.send_tx(self.active_orders.keys(), tx)
self.latest_tx = btc.deserialize(tx)
for index, ins in enumerate(self.latest_tx['ins']):
utxo = ins['outpoint']['hash'] + ':' + str(
ins['outpoint']['index'])
if utxo not in self.input_utxos.keys():
continue
# placeholders required
ins['script'] = 'deadbeef'
def make_commitment(self, wallet, input_utxos, cjamount):
"""The Taker default commitment function, which uses PoDLE.
Alternative commitment types should use a different commit type byte.
This will allow future upgrades to provide different style commitments
by subclassing Taker and changing the commit_type_byte; existing makers
will simply not accept this new type of commitment.
In case of success, return the commitment and its opening.
In case of failure returns (None, None) and constructs a detailed
log for the user to read and discern the reason.
"""
def filter_by_coin_age_amt(utxos, age, amt):
results = jm_single().bc_interface.query_utxo_set(utxos,
includeconf=True)
newresults = []
too_old = []
too_small = []
for i, r in enumerate(results):
#results return "None" if txo is spent; drop this
if not r:
continue
valid_age = r['confirms'] >= age
valid_amt = r['value'] >= amt
if not valid_age:
too_old.append(utxos[i])
if not valid_amt:
too_small.append(utxos[i])
if valid_age and valid_amt:
newresults.append(utxos[i])
return newresults, too_old, too_small
def priv_utxo_pairs_from_utxos(utxos, age, amt):
#returns pairs list of (priv, utxo) for each valid utxo;
#also returns lists "too_old" and "too_small" for any
#utxos that did not satisfy the criteria for debugging.
priv_utxo_pairs = []
new_utxos, too_old, too_small = filter_by_coin_age_amt(
utxos.keys(), age, amt)
new_utxos_dict = {k: v for k, v in utxos.items() if k in new_utxos}
for k, v in new_utxos_dict.iteritems():
addr = v['address']
priv = wallet.get_key_from_addr(addr)
if priv: #can be null from create-unsigned
priv_utxo_pairs.append((priv, k))
return priv_utxo_pairs, too_old, too_small
commit_type_byte = "P"
podle_data = None
tries = jm_single().config.getint("POLICY", "taker_utxo_retries")
age = jm_single().config.getint("POLICY", "taker_utxo_age")
#Minor rounding errors don't matter here
amt = int(
cjamount *
jm_single().config.getint("POLICY", "taker_utxo_amtpercent") /
100.0)
priv_utxo_pairs, to, ts = priv_utxo_pairs_from_utxos(
input_utxos, age, amt)
#Note that we ignore the "too old" and "too small" lists in the first
#pass through, because the same utxos appear in the whole-wallet check.
#For podle data format see: btc.podle.PoDLE.reveal()
#In first round try, don't use external commitments
podle_data = btc.generate_podle(priv_utxo_pairs, tries)
if not podle_data:
#We defer to a second round to try *all* utxos in wallet;
#this is because it's much cleaner to use the utxos involved
#in the transaction, about to be consumed, rather than use
#random utxos that will persist after. At this step we also
#allow use of external utxos in the json file.
if wallet.unspent:
priv_utxo_pairs, to, ts = priv_utxo_pairs_from_utxos(
wallet.unspent, age, amt)
#Pre-filter the set of external commitments that work for this
#transaction according to its size and age.
dummy, extdict = btc.get_podle_commitments()
if len(extdict.keys()) > 0:
ext_valid, ext_to, ext_ts = filter_by_coin_age_amt(
extdict.keys(), age, amt)
else:
ext_valid = None
podle_data = btc.generate_podle(priv_utxo_pairs, tries, ext_valid)
if podle_data:
log.debug("Generated PoDLE: " + pprint.pformat(podle_data))
revelation = btc.PoDLE(u=podle_data['utxo'],
P=podle_data['P'],
P2=podle_data['P2'],
s=podle_data['sig'],
e=podle_data['e']).serialize_revelation()
return (commit_type_byte + podle_data["commit"], revelation)
else:
#we know that priv_utxo_pairs all passed age and size tests, so
#they must have failed the retries test. Summarize this info
#and publish to commitments_debug.txt
with open("commitments_debug.txt", "wb") as f:
f.write("THIS IS A TEMPORARY FILE FOR DEBUGGING; "
"IT CAN BE SAFELY DELETED ANY TIME.\n")
f.write("***\n")
f.write("1: Utxos that passed age and size limits, but have "
"been used too many times (see taker_utxo_retries "
"in the config):\n")
if len(priv_utxo_pairs) == 0:
f.write("None\n")
else:
for p, u in priv_utxo_pairs:
f.write(str(u) + "\n")
f.write("2: Utxos that have less than " +
jm_single().config.get("POLICY", "taker_utxo_age") +
" confirmations:\n")
if len(to) == 0:
f.write("None\n")
else:
for t in to:
f.write(str(t) + "\n")
f.write("3: Utxos that were not at least " + \
jm_single().config.get(
"POLICY", "taker_utxo_amtpercent") + "% of the "
"size of the coinjoin amount " + str(
self.proposed_cj_amount) + "\n")
if len(ts) == 0:
f.write("None\n")
else:
for t in ts:
f.write(str(t) + "\n")
f.write('***\n')
f.write(
"Utxos that appeared in item 1 cannot be used again.\n")
f.write(
"Utxos only in item 2 can be used by waiting for more "
"confirmations, (set by the value of taker_utxo_age).\n")
f.write("Utxos only in item 3 are not big enough for this "
"coinjoin transaction, set by the value "
"of taker_utxo_amtpercent.\n")
f.write(
"If you cannot source a utxo from your wallet according "
"to these rules, use the tool add-utxo.py to source a "
"utxo external to your joinmarket wallet. Read the help "
"with 'python add-utxo.py --help'\n\n")
f.write("You can also reset the rules in the joinmarket.cfg "
"file, but this is generally inadvisable.\n")
f.write(
"***\nFor reference, here are the utxos in your wallet:\n")
f.write("\n" + str(self.proposed_wallet.unspent))
return (None, None)
def get_key_from_addr(self, addr):
self.ensure_wallet_unlocked()
wifkey = jm_single().bc_interface.rpc('dumpprivkey', [addr])
return btc.from_wif_privkey(wifkey, vbyte=get_p2pk_vbyte())
def __init__(self, maker, nick, oid, amount, taker_pk):
self.tx = None
self.i_utxo_pubkey = None
self.maker = maker
self.oid = oid
self.cj_amount = amount
if self.cj_amount <= jm_single().DUST_THRESHOLD:
self.maker.msgchan.send_error(nick, 'amount below dust threshold')
# the btc pubkey of the utxo that the taker plans to use as input
self.taker_pk = taker_pk
# create DH keypair on the fly for this Order object
self.kp = init_keypair()
# the encryption channel crypto box for this Order object.
# Invalid pubkeys must be handled by giving up gracefully (otherwise DOS)
try:
self.crypto_box = as_init_encryption(self.kp,
init_pubkey(taker_pk))
except NaclError as e:
log.debug("Unable to setup crypto box with counterparty: " +
repr(e))
self.maker.msgchan.send_error(nick,
"invalid nacl pubkey: " + taker_pk)
return
order_s = [o for o in maker.orderlist if o['oid'] == oid]
if len(order_s) == 0:
self.maker.msgchan.send_error(nick, 'oid not found')
order = order_s[0]
if amount < order['minsize'] or amount > order['maxsize']:
self.maker.msgchan.send_error(nick, 'amount out of range')
self.ordertype = order['ordertype']
self.txfee = order['txfee']
self.cjfee = order['cjfee']
log.debug('new cjorder nick=%s oid=%d amount=%d' % (nick, oid, amount))
def populate_utxo_data():
self.utxos, self.cj_addr, self.change_addr = maker.oid_to_order(
self, oid, amount)
self.maker.wallet.update_cache_index()
if not self.utxos:
self.maker.msgchan.send_error(
nick, 'unable to fill order constrained by dust avoidance')
# TODO make up orders offers in a way that this error cant appear
# check nothing has messed up with the wallet code, remove this
# code after a while
log.debug('maker utxos = ' + pprint.pformat(self.utxos))
utxos = self.utxos.keys()
return (utxos, jm_single().bc_interface.query_utxo_set(utxos))
utxo_list, utxo_data = populate_utxo_data()
while None in utxo_data:
log.debug('wrongly selected stale utxos! utxo_data = ' +
pprint.pformat(utxo_data))
self.maker.wallet_unspent_lock.acquire()
try:
jm_single().bc_interface.sync_unspent(self.maker.wallet)
finally:
self.maker.wallet_unspent_lock.release()
utxo_list, utxo_data = populate_utxo_data()
for utxo, data in zip(utxo_list, utxo_data):
if self.utxos[utxo]['value'] != data['value']:
fmt = 'wrongly labeled utxo, expected value: {} got {}'.format
log.debug(fmt(self.utxos[utxo]['value'], data['value']))
sys.exit(0)
# always a new address even if the order ends up never being
# furfilled, you dont want someone pretending to fill all your
# orders to find out which addresses you use
self.maker.msgchan.send_pubkey(nick, self.kp.hex_pk())
def add_tx_notify(self,
txd,
unconfirmfun,
confirmfun,
notifyaddr,
timeoutfun=None):
unconfirm_timeout = jm_single().config.getint('TIMEOUT',
'unconfirm_timeout_sec')
unconfirm_poll_period = 5
confirm_timeout = jm_single().config.getfloat(
'TIMEOUT', 'confirm_timeout_hours') * 60 * 60
confirm_poll_period = 5 * 60
class NotifyThread(threading.Thread):
def __init__(self, blockr_domain, txd, unconfirmfun, confirmfun,
timeoutfun):
threading.Thread.__init__(self, name='BlockrNotifyThread')
self.daemon = True
self.blockr_domain = blockr_domain
self.unconfirmfun = unconfirmfun
self.confirmfun = confirmfun
self.timeoutfun = timeoutfun
self.tx_output_set = set([(sv['script'], sv['value'])
for sv in txd['outs']])
self.output_addresses = [
btc.script_to_address(scrval[0], get_p2pk_vbyte())
for scrval in self.tx_output_set
]
log.debug('txoutset=' + pprint.pformat(self.tx_output_set))
log.debug('outaddrs=' + ','.join(self.output_addresses))
def run(self):
st = int(time.time())
unconfirmed_txid = None
unconfirmed_txhex = None
while not unconfirmed_txid:
time.sleep(unconfirm_poll_period)
if int(time.time()) - st > unconfirm_timeout:
log.debug('checking for unconfirmed tx timed out')
if self.timeoutfun:
self.timeoutfun(False)
return
blockr_url = 'https://' + self.blockr_domain
blockr_url += '.blockr.io/api/v1/address/unspent/'
random.shuffle(self.output_addresses
) # seriously weird bug with blockr.io
data = json.loads(
btc.make_request(blockr_url +
','.join(self.output_addresses) +
'?unconfirmed=1'))['data']
shared_txid = None
for unspent_list in data:
txs = set([
str(txdata['tx'])
for txdata in unspent_list['unspent']
])
if not shared_txid:
shared_txid = txs
else:
shared_txid = shared_txid.intersection(txs)
log.debug('sharedtxid = ' + str(shared_txid))
if len(shared_txid) == 0:
continue
time.sleep(
2
) # here for some race condition bullshit with blockr.io
blockr_url = 'https://' + self.blockr_domain
blockr_url += '.blockr.io/api/v1/tx/raw/'
data = json.loads(
btc.make_request(blockr_url +
','.join(shared_txid)))['data']
if not isinstance(data, list):
data = [data]
for txinfo in data:
txhex = str(txinfo['tx']['hex'])
outs = set([(sv['script'], sv['value'])
for sv in btc.deserialize(txhex)['outs']])
log.debug('unconfirm query outs = ' + str(outs))
if outs == self.tx_output_set:
unconfirmed_txid = txinfo['tx']['txid']
unconfirmed_txhex = str(txinfo['tx']['hex'])
break
self.unconfirmfun(btc.deserialize(unconfirmed_txhex),
unconfirmed_txid)
st = int(time.time())
confirmed_txid = None
confirmed_txhex = None
while not confirmed_txid:
time.sleep(confirm_poll_period)
if int(time.time()) - st > confirm_timeout:
log.debug('checking for confirmed tx timed out')
if self.timeoutfun:
self.timeoutfun(True)
return
blockr_url = 'https://' + self.blockr_domain
blockr_url += '.blockr.io/api/v1/address/txs/'
data = json.loads(
btc.make_request(
blockr_url +
','.join(self.output_addresses)))['data']
shared_txid = None
for addrtxs in data:
txs = set(
str(txdata['tx']) for txdata in addrtxs['txs'])
if not shared_txid:
shared_txid = txs
else:
shared_txid = shared_txid.intersection(txs)
log.debug('sharedtxid = ' + str(shared_txid))
if len(shared_txid) == 0:
continue
blockr_url = 'https://' + self.blockr_domain
blockr_url += '.blockr.io/api/v1/tx/raw/'
data = json.loads(
btc.make_request(blockr_url +
','.join(shared_txid)))['data']
if not isinstance(data, list):
data = [data]
for txinfo in data:
txhex = str(txinfo['tx']['hex'])
outs = set([(sv['script'], sv['value'])
for sv in btc.deserialize(txhex)['outs']])
log.debug('confirm query outs = ' + str(outs))
if outs == self.tx_output_set:
confirmed_txid = txinfo['tx']['txid']
confirmed_txhex = str(txinfo['tx']['hex'])
break
self.confirmfun(btc.deserialize(confirmed_txhex),
confirmed_txid, 1)
NotifyThread(self.blockr_domain, txd, unconfirmfun, confirmfun,
timeoutfun).start()
def add_signature(self, nick, sigb64):
if nick not in self.nonrespondants:
log.debug(
('add_signature => nick={} '
'not in nonrespondants {}').format(nick, self.nonrespondants))
return
sig = base64.b64decode(sigb64).encode('hex')
inserted_sig = False
txhex = btc.serialize(self.latest_tx)
# batch retrieval of utxo data
utxo = {}
ctr = 0
for index, ins in enumerate(self.latest_tx['ins']):
utxo_for_checking = ins['outpoint']['hash'] + ':' + str(
ins['outpoint']['index'])
if (ins['script'] != ''
or utxo_for_checking in self.input_utxos.keys()):
continue
utxo[ctr] = [index, utxo_for_checking]
ctr += 1
utxo_data = jm_single().bc_interface.query_utxo_set(
[x[1] for x in utxo.values()])
# insert signatures
for i, u in utxo.iteritems():
if utxo_data[i] is None:
continue
sig_good = btc.verify_tx_input(txhex, u[0], utxo_data[i]['script'],
*btc.deserialize_script(sig))
if sig_good:
log.debug('found good sig at index=%d' % (u[0]))
self.latest_tx['ins'][u[0]]['script'] = sig
inserted_sig = True
# check if maker has sent everything possible
self.utxos[nick].remove(u[1])
if len(self.utxos[nick]) == 0:
log.debug(('nick = {} sent all sigs, removing from '
'nonrespondant list').format(nick))
self.nonrespondants.remove(nick)
break
if not inserted_sig:
log.debug('signature did not match anything in the tx')
# TODO what if the signature doesnt match anything
# nothing really to do except drop it, carry on and wonder why the
# other guy sent a failed signature
tx_signed = True
for ins in self.latest_tx['ins']:
if ins['script'] == '':
tx_signed = False
if not tx_signed:
return
self.end_timeout_thread = True
self.all_responded = True
with self.timeout_lock:
self.timeout_lock.notify()
log.debug('all makers have sent their signatures')
for index, ins in enumerate(self.latest_tx['ins']):
# remove placeholders
if ins['script'] == 'deadbeef':
ins['script'] = ''
if self.finishcallback is not None:
self.finishcallback(self)
def recv_txio(self, nick, utxo_list, cj_pub, change_addr):
if nick not in self.nonrespondants:
log.debug(('recv_txio => nick={} not in '
'nonrespondants {}').format(nick, self.nonrespondants))
return
self.utxos[nick] = utxo_list
order = self.db.execute(
'SELECT ordertype, txfee, cjfee FROM '
'orderbook WHERE oid=? AND counterparty=?',
(self.active_orders[nick], nick)).fetchone()
utxo_data = jm_single().bc_interface.query_utxo_set(self.utxos[nick])
if None in utxo_data:
log.debug(('ERROR outputs unconfirmed or already spent. '
'utxo_data={}').format(pprint.pformat(utxo_data)))
# when internal reviewing of makers is created, add it here to
# immediately quit
return
# ignore this message, eventually the timeout thread will recover
total_input = sum([d['value'] for d in utxo_data])
real_cjfee = calc_cj_fee(order['ordertype'], order['cjfee'],
self.cj_amount)
self.outputs.append({
'address':
change_addr,
'value':
total_input - self.cj_amount - order['txfee'] + real_cjfee
})
fmt = ('fee breakdown for {} totalin={:d} '
'cjamount={:d} txfee={:d} realcjfee={:d}').format
log.debug(
fmt(nick, total_input, self.cj_amount, order['txfee'], real_cjfee))
cj_addr = btc.pubtoaddr(cj_pub, get_p2pk_vbyte())
self.outputs.append({'address': cj_addr, 'value': self.cj_amount})
self.cjfee_total += real_cjfee
self.maker_txfee_contributions += order['txfee']
self.nonrespondants.remove(nick)
if len(self.nonrespondants) > 0:
log.debug('nonrespondants = ' + str(self.nonrespondants))
return
self.all_responded = True
with self.timeout_lock:
self.timeout_lock.notify()
log.debug('got all parts, enough to build a tx')
self.nonrespondants = list(self.active_orders.keys())
my_total_in = sum(
[va['value'] for u, va in self.input_utxos.iteritems()])
if self.my_change_addr:
#Estimate fee per choice of next/3/6 blocks targetting.
estimated_fee = estimate_tx_fee(len(sum(self.utxos.values(), [])),
len(self.outputs) + 2)
log.debug("Based on initial guess: " + str(self.total_txfee) +
", we estimated a fee of: " + str(estimated_fee))
#reset total
self.total_txfee = estimated_fee
my_txfee = max(self.total_txfee - self.maker_txfee_contributions, 0)
my_change_value = (my_total_in - self.cj_amount - self.cjfee_total -
my_txfee)
#Since we could not predict the maker's inputs, we may end up needing
#too much such that the change value is negative or small. Note that
#we have tried to avoid this based on over-estimating the needed amount
#in SendPayment.create_tx(), but it is still a possibility if one maker
#uses a *lot* of inputs.
if self.my_change_addr and my_change_value <= 0:
raise ValueError("Calculated transaction fee of: " +
str(self.total_txfee) +
" is too large for our inputs;Please try again.")
elif self.my_change_addr and my_change_value <= jm_single(
).DUST_THRESHOLD:
log.debug("Dynamically calculated change lower than dust: " +
str(my_change_value) + "; dropping.")
self.my_change_addr = None
my_change_value = 0
log.debug(
'fee breakdown for me totalin=%d my_txfee=%d makers_txfee=%d cjfee_total=%d => changevalue=%d'
% (my_total_in, my_txfee, self.maker_txfee_contributions,
self.cjfee_total, my_change_value))
if self.my_change_addr is None:
if my_change_value != 0 and abs(my_change_value) != 1:
# seems you wont always get exactly zero because of integer
# rounding so 1 satoshi extra or fewer being spent as miner
# fees is acceptable
log.debug(('WARNING CHANGE NOT BEING '
'USED\nCHANGEVALUE = {}').format(my_change_value))
else:
self.outputs.append({
'address': self.my_change_addr,
'value': my_change_value
})
self.utxo_tx = [
dict([('output', u)]) for u in sum(self.utxos.values(), [])
]
self.outputs.append({
'address': self.coinjoin_address(),
'value': self.cj_amount
})
random.shuffle(self.utxo_tx)
random.shuffle(self.outputs)
tx = btc.mktx(self.utxo_tx, self.outputs)
log.debug('obtained tx\n' + pprint.pformat(btc.deserialize(tx)))
self.msgchan.send_tx(self.active_orders.keys(), tx)
self.latest_tx = btc.deserialize(tx)
for index, ins in enumerate(self.latest_tx['ins']):
utxo = ins['outpoint']['hash'] + ':' + str(
ins['outpoint']['index'])
if utxo not in self.input_utxos.keys():
continue
# placeholders required
ins['script'] = 'deadbeef'
def auth_counterparty(self, nick, cr):
#deserialize the commitment revelation
cr_dict = btc.PoDLE.deserialize_revelation(cr)
#check the validity of the proof of discrete log equivalence
tries = jm_single().config.getint("POLICY", "taker_utxo_retries")
def reject(msg):
log.debug("Counterparty commitment not accepted, reason: " + msg)
return False
if not btc.verify_podle(cr_dict['P'],
cr_dict['P2'],
cr_dict['sig'],
cr_dict['e'],
self.maker.commit,
index_range=range(tries)):
reason = "verify_podle failed"
return reject(reason)
#finally, check that the proffered utxo is real, old enough, large enough,
#and corresponds to the pubkey
res = jm_single().bc_interface.query_utxo_set([cr_dict['utxo']],
includeconf=True)
if len(res) != 1 or not res[0]:
reason = "authorizing utxo is not valid"
return reject(reason)
age = jm_single().config.getint("POLICY", "taker_utxo_age")
if res[0]['confirms'] < age:
reason = "commitment utxo not old enough: " + str(
res[0]['confirms'])
return reject(reason)
reqd_amt = int(
self.cj_amount *
jm_single().config.getint("POLICY", "taker_utxo_amtpercent") /
100.0)
if res[0]['value'] < reqd_amt:
reason = "commitment utxo too small: " + str(res[0]['value'])
return reject(reason)
if res[0]['address'] != btc.pubkey_to_address(cr_dict['P'],
get_p2pk_vbyte()):
reason = "Invalid podle pubkey: " + str(cr_dict['P'])
return reject(reason)
# authorisation of taker passed
# Send auth request to taker
# Need to choose an input utxo pubkey to sign with
# (no longer using the coinjoin pubkey from 0.2.0)
# Just choose the first utxo in self.utxos and retrieve key from wallet.
auth_address = self.utxos[self.utxos.keys()[0]]['address']
auth_key = self.maker.wallet.get_key_from_addr(auth_address)
auth_pub = btc.privtopub(auth_key)
btc_sig = btc.ecdsa_sign(self.kp.hex_pk(), auth_key)
self.maker.msgchan.send_ioauth(nick, self.utxos.keys(), auth_pub,
self.cj_addr, self.change_addr, btc_sig)
#In case of *blacklisted (ie already used) commitments, we already
#broadcasted them on receipt; in case of valid, and now used commitments,
#we broadcast them here, and not early - to avoid accidentally
#blacklisting commitments that are broadcast between makers in real time
#for the same transaction.
self.maker.transfer_commitment(self.maker.commit)
#now persist the fact that the commitment is actually used.
check_utxo_blacklist(self.maker.commit, persist=True)
return True
def get_internal_addr(self, mixing_depth):
return jm_single().bc_interface.rpc('getrawchangeaddress', [])
def get_key_from_addr(self, addr):
self.ensure_wallet_unlocked()
return jm_single().bc_interface.rpc('dumpprivkey', [addr])
import base64, abc, threading, time, hashlib, os, binascii
from joinmarket.enc_wrapper import encrypt_encode, decode_decrypt
from joinmarket.support import get_log, chunks
from joinmarket.configure import jm_single
import bitcoin as btc
from functools import wraps
COMMAND_PREFIX = '!'
JOINMARKET_NICK_HEADER = 'J'
NICK_HASH_LENGTH = 10
NICK_MAX_ENCODED = 14 #comes from base58 expansion; recalculate if above changes
encrypted_commands = ["auth", "ioauth", "tx", "sig"]
plaintext_commands = ["fill", "error", "pubkey", "orderbook", "push"]
plaintext_commands += jm_single().ordername_list
plaintext_commands += jm_single().commitment_broadcast_list
log = get_log()
class CJPeerError(StandardError):
pass
class MChannelThread(threading.Thread):
def __init__(self, mc):
threading.Thread.__init__(self, name='MCThread')
self.daemon = True
self.mc = mc
def run(self):
self.mc.run()
def run(self):
self.waiting = {}
self.built_privmsg = {}
self.give_up = False
self.ping_reply = True
self.lockcond = threading.Condition()
self.lockthrottle = threading.Condition()
PingThread(self).start()
ThrottleThread(self).start()
while not self.give_up:
try:
config = jm_single().config
log.debug('connecting')
if self.newnyms:
log.debug("Grabbing new Tor identity")
self.newnym()
self.nick = random_nick()
if config.get("MESSAGING", "socks5").lower() == 'true':
log.debug("Using socks5 proxy %s:%d" %
(self.socks5_host, self.socks5_port))
setdefaultproxy(PROXY_TYPE_SOCKS5, self.socks5_host,
self.socks5_port, True)
self.sock = socksocket()
else:
self.sock = socket.socket(socket.AF_INET,
socket.SOCK_STREAM)
self.sock.connect(self.serverport)
if config.get("MESSAGING", "usessl").lower() == 'true':
self.sock = ssl.wrap_socket(self.sock)
self.fd = self.sock.makefile()
self.password = None
if self.given_password:
self.password = self.given_password
self.send_raw('CAP REQ :sasl')
self.send_raw('USER %s b c :%s' % self.userrealname)
self.nick = self.given_nick
self.send_raw('NICK ' + self.nick)
while 1:
try:
line = self.fd.readline()
except AttributeError as e:
raise IOError(repr(e))
if line is None:
log.debug('line returned null')
break
if len(line) == 0:
log.debug('line was zero length')
break
self.__handle_line(line)
except IOError as e:
log.debug(repr(e))
finally:
try:
self.fd.close()
self.sock.close()
except Exception as e:
print(repr(e))
if self.on_disconnect:
self.on_disconnect()
log.debug('disconnected irc')
if not self.give_up:
time.sleep(self.reconnect_delay)
if self.newnyms:
time.sleep(random.randint(0, self.newnym_delay))
log.debug('ending irc')
self.give_up = True
def estimate_fee_per_kb(self, N):
if not self.absurd_fees:
return super(RegtestBitcoinCoreInterface, self).estimate_fee_per_kb(N)
else:
return jm_single().config.getint("POLICY", "absurd_fee_per_kb") + 100
def do_HEAD(self):
pages = ('/walletnotify?', '/alertnotify?')
if self.path.startswith('/walletnotify?'):
txid = self.path[len(pages[0]):]
if not re.match('^[0-9a-fA-F]*$', txid):
log.debug('not a txid')
return
tx = self.btcinterface.rpc('getrawtransaction', [txid])
if not re.match('^[0-9a-fA-F]*$', tx):
log.debug('not a txhex')
return
txd = btc.deserialize(tx)
tx_output_set = set([(sv['script'], sv['value'])
for sv in txd['outs']])
unconfirmfun, confirmfun = None, None
for tx_out, ucfun, cfun in self.btcinterface.txnotify_fun:
if tx_out == tx_output_set:
unconfirmfun = ucfun
confirmfun = cfun
break
if unconfirmfun is None:
log.debug('txid=' + txid + ' not being listened for')
else:
# on rare occasions people spend their output without waiting
# for a confirm
txdata = None
for n in range(len(txd['outs'])):
txdata = self.btcinterface.rpc('gettxout', [txid, n, True])
if txdata is not None:
break
assert txdata is not None
if txdata['confirmations'] == 0:
unconfirmfun(txd, txid)
# TODO pass the total transfered amount value here somehow
# wallet_name = self.get_wallet_name()
# amount =
# bitcoin-cli move wallet_name "" amount
log.debug('ran unconfirmfun')
else:
confirmfun(txd, txid, txdata['confirmations'])
self.btcinterface.txnotify_fun.remove(
(tx_out, unconfirmfun, confirmfun))
log.debug('ran confirmfun')
elif self.path.startswith('/alertnotify?'):
jm_single().core_alert[0] = urllib.unquote(
self.path[len(pages[1]):])
log.debug('Got an alert!\nMessage=' + jm_single().core_alert[0])
else:
log.debug(
'ERROR: This is not a handled URL path. You may want to check your notify URL for typos.'
)
os.system('curl -sI --connect-timeout 1 http://localhost:' +
str(self.base_server.server_address[1] + 1) + self.path)
self.send_response(200)
# self.send_header('Connection', 'close')
self.end_headers()
