def test_external_commitments(setup_podle):
"""Add this generated commitment to the external list
{txid:N:{'P':pubkey, 'reveal':{1:{'P2':P2,'s':s,'e':e}, 2:{..},..}}}
Note we do this *after* the sendpayment test so that the external
commitments will not erroneously used (they are fake).
"""
ecs = {}
tries = jm_single().config.getint("POLICY","taker_utxo_retries")
for i in range(10):
priv = os.urandom(32)
dummy_utxo = btc.sha256(priv)+":2"
ecs[dummy_utxo] = {}
ecs[dummy_utxo]['reveal']={}
for j in range(tries):
P, P2, s, e, commit = generate_single_podle_sig(priv, j)
if 'P' not in ecs[dummy_utxo]:
ecs[dummy_utxo]['P']=P
ecs[dummy_utxo]['reveal'][j] = {'P2':P2, 's':s, 'e':e}
btc.add_external_commitments(ecs)
used, external = btc.get_podle_commitments()
for u in external:
assert external[u]['P'] == ecs[u]['P']
for i in range(tries):
for x in ['P2', 's', 'e']:
assert external[u]['reveal'][str(i)][x] == ecs[u]['reveal'][i][x]
def test_external_commitments(setup_podle):
"""Add this generated commitment to the external list
{txid:N:{'P':pubkey, 'reveal':{1:{'P2':P2,'s':s,'e':e}, 2:{..},..}}}
Note we do this *after* the sendpayment test so that the external
commitments will not erroneously used (they are fake).
"""
ecs = {}
tries = jm_single().config.getint("POLICY", "taker_utxo_retries")
for i in range(10):
priv = os.urandom(32)
dummy_utxo = btc.sha256(priv) + ":2"
ecs[dummy_utxo] = {}
ecs[dummy_utxo]['reveal'] = {}
for j in range(tries):
P, P2, s, e, commit = generate_single_podle_sig(priv, j)
if 'P' not in ecs[dummy_utxo]:
ecs[dummy_utxo]['P'] = P
ecs[dummy_utxo]['reveal'][j] = {'P2': P2, 's': s, 'e': e}
btc.add_external_commitments(ecs)
used, external = btc.get_podle_commitments()
for u in external:
assert external[u]['P'] == ecs[u]['P']
for i in range(tries):
for x in ['P2', 's', 'e']:
assert external[u]['reveal'][str(
i)][x] == ecs[u]['reveal'][i][x]
def main():
parser = OptionParser(
usage=
'usage: %prog [options] [txid:n]',
description="Adds one or more utxos to the list that can be used to make "
"commitments for anti-snooping. Note that this utxo, and its "
"PUBkey, will be revealed to makers, so consider the privacy "
"implication. "
"It may be useful to those who are having trouble making "
"coinjoins due to several unsuccessful attempts (especially "
"if your joinmarket wallet is new). "
"'Utxo' means unspent transaction output, it must not "
"already be spent. "
"The options -w, -r and -R offer ways to load these utxos "
"from a file or wallet. "
"If you enter a single utxo without these options, you will be "
"prompted to enter the private key here - it must be in "
"WIF compressed format. "
"BE CAREFUL about handling private keys! "
"Don't do this in insecure environments. "
"Also note this ONLY works for standard (p2pkh) utxos."
)
parser.add_option(
'-r',
'--read-from-file',
action='store',
type='str',
dest='in_file',
help='name of plain text csv file containing utxos, one per line, format: '
'txid:N, WIF-compressed-privkey'
)
parser.add_option(
'-R',
'--read-from-json',
action='store',
type='str',
dest='in_json',
help='name of json formatted file containing utxos with private keys, as '
'output from "python wallet-tool.py -u -p walletname showutxos"'
)
parser.add_option(
'-w',
'--load-wallet',
action='store',
type='str',
dest='loadwallet',
help='name of wallet from which to load utxos and use as commitments.'
)
parser.add_option(
'-g',
'--gap-limit',
action='store',
type='int',
dest='gaplimit',
default = 6,
help='Only to be used with -w; gap limit for Joinmarket wallet, default 6.'
)
parser.add_option(
'-M',
'--max-mixdepth',
action='store',
type='int',
dest='maxmixdepth',
default=5,
help='Only to be used with -w; number of mixdepths for wallet, default 5.'
)
parser.add_option(
'-d',
'--delete-external',
action='store_true',
dest='delete_ext',
help='deletes the current list of external commitment utxos',
default=False
)
parser.add_option(
'-v',
'--validate-utxos',
action='store_true',
dest='validate',
help='validate the utxos and pubkeys provided against the blockchain',
default=False
)
parser.add_option(
'-o',
'--validate-only',
action='store_true',
dest='vonly',
help='only validate the provided utxos (file or command line), not add',
default=False
)
(options, args) = parser.parse_args()
load_program_config()
utxo_data = []
if options.delete_ext:
other = options.in_file or options.in_json or options.loadwallet
if len(args) > 0 or other:
if raw_input("You have chosen to delete commitments, other arguments "
"will be ignored; continue? (y/n)") != 'y':
print "Quitting"
sys.exit(0)
c, e = btc.get_podle_commitments()
print pformat(e)
if raw_input(
"You will remove the above commitments; are you sure? (y/n): ") != 'y':
print "Quitting"
sys.exit(0)
btc.update_commitments(external_to_remove=e)
print "Commitments deleted."
sys.exit(0)
#Three options (-w, -r, -R) for loading utxo and privkey pairs from a wallet,
#csv file or json file.
if options.loadwallet:
os.chdir('..') #yuck (see earlier comment about package)
wallet = Wallet(options.loadwallet,
options.maxmixdepth,
options.gaplimit)
os.chdir(os.path.join(os.getcwd(), 'cmttools'))
jm_single().bc_interface.sync_wallet(wallet)
unsp = {}
for u, av in wallet.unspent.iteritems():
addr = av['address']
key = wallet.get_key_from_addr(addr)
wifkey = btc.wif_compressed_privkey(key, vbyte=get_p2pk_vbyte())
unsp[u] = {'address': av['address'],
'value': av['value'], 'privkey': wifkey}
for u, pva in unsp.iteritems():
utxo_data.append((u, pva['privkey']))
elif options.in_file:
with open(options.in_file, "rb") as f:
utxo_info = f.readlines()
for ul in utxo_info:
ul = ul.rstrip()
if ul:
u, priv = get_utxo_info(ul)
if not u:
quit(parser, "Failed to parse utxo info: " + str(ul))
utxo_data.append((u, priv))
elif options.in_json:
if not os.path.isfile(options.in_json):
print "File: " + options.in_json + " not found."
sys.exit(0)
with open(options.in_json, "rb") as f:
try:
utxo_json = json.loads(f.read())
except:
print "Failed to read json from " + options.in_json
sys.exit(0)
for u, pva in utxo_json.iteritems():
utxo_data.append((u, pva['privkey']))
elif len(args) == 1:
u = args[0]
priv = raw_input(
'input private key for ' + u + ', in WIF compressed format : ')
u, priv = get_utxo_info(','.join([u, priv]))
if not u:
quit(parser, "Failed to parse utxo info: " + u)
utxo_data.append((u, priv))
else:
quit(parser, 'Invalid syntax')
if options.validate or options.vonly:
if not validate_utxo_data(utxo_data):
quit(parser, "Utxos did not validate, quitting")
if options.vonly:
sys.exit(0)
#We are adding utxos to the external list
assert len(utxo_data)
add_external_commitments(utxo_data)
def test_commitments_empty(setup_podle):
"""Ensure that empty commitments file
results in {}
"""
assert btc.get_podle_commitments() == ([], {})
def test_commitments_empty(setup_podle):
"""Ensure that empty commitments file
results in {}
"""
assert btc.get_podle_commitments() == ([], {})
def make_commitment(self, wallet, input_utxos, cjamount):
"""The Taker default commitment function, which uses PoDLE.
Alternative commitment types should use a different commit type byte.
This will allow future upgrades to provide different style commitments
by subclassing Taker and changing the commit_type_byte; existing makers
will simply not accept this new type of commitment.
In case of success, return the commitment and its opening.
In case of failure returns (None, None) and constructs a detailed
log for the user to read and discern the reason.
"""
def filter_by_coin_age_amt(utxos, age, amt):
results = jm_single().bc_interface.query_utxo_set(utxos,
includeconf=True)
newresults = []
too_old = []
too_small = []
for i, r in enumerate(results):
#results return "None" if txo is spent; drop this
if not r:
continue
valid_age = r['confirms'] >= age
valid_amt = r['value'] >= amt
if not valid_age:
too_old.append(utxos[i])
if not valid_amt:
too_small.append(utxos[i])
if valid_age and valid_amt:
newresults.append(utxos[i])
return newresults, too_old, too_small
def priv_utxo_pairs_from_utxos(utxos, age, amt):
#returns pairs list of (priv, utxo) for each valid utxo;
#also returns lists "too_old" and "too_small" for any
#utxos that did not satisfy the criteria for debugging.
priv_utxo_pairs = []
new_utxos, too_old, too_small = filter_by_coin_age_amt(
utxos.keys(), age, amt)
new_utxos_dict = {k: v for k, v in utxos.items() if k in new_utxos}
for k, v in new_utxos_dict.iteritems():
addr = v['address']
priv = wallet.get_key_from_addr(addr)
if priv: #can be null from create-unsigned
priv_utxo_pairs.append((priv, k))
return priv_utxo_pairs, too_old, too_small
commit_type_byte = "P"
podle_data = None
tries = jm_single().config.getint("POLICY", "taker_utxo_retries")
age = jm_single().config.getint("POLICY", "taker_utxo_age")
#Minor rounding errors don't matter here
amt = int(
cjamount *
jm_single().config.getint("POLICY", "taker_utxo_amtpercent") /
100.0)
priv_utxo_pairs, to, ts = priv_utxo_pairs_from_utxos(
input_utxos, age, amt)
#Note that we ignore the "too old" and "too small" lists in the first
#pass through, because the same utxos appear in the whole-wallet check.
#For podle data format see: btc.podle.PoDLE.reveal()
#In first round try, don't use external commitments
podle_data = btc.generate_podle(priv_utxo_pairs, tries)
if not podle_data:
#We defer to a second round to try *all* utxos in wallet;
#this is because it's much cleaner to use the utxos involved
#in the transaction, about to be consumed, rather than use
#random utxos that will persist after. At this step we also
#allow use of external utxos in the json file.
if wallet.unspent:
priv_utxo_pairs, to, ts = priv_utxo_pairs_from_utxos(
wallet.unspent, age, amt)
#Pre-filter the set of external commitments that work for this
#transaction according to its size and age.
dummy, extdict = btc.get_podle_commitments()
if len(extdict.keys()) > 0:
ext_valid, ext_to, ext_ts = filter_by_coin_age_amt(
extdict.keys(), age, amt)
else:
ext_valid = None
podle_data = btc.generate_podle(priv_utxo_pairs, tries, ext_valid)
if podle_data:
log.debug("Generated PoDLE: " + pprint.pformat(podle_data))
revelation = btc.PoDLE(u=podle_data['utxo'],
P=podle_data['P'],
P2=podle_data['P2'],
s=podle_data['sig'],
e=podle_data['e']).serialize_revelation()
return (commit_type_byte + podle_data["commit"], revelation)
else:
#we know that priv_utxo_pairs all passed age and size tests, so
#they must have failed the retries test. Summarize this info
#and publish to commitments_debug.txt
with open("commitments_debug.txt", "wb") as f:
f.write("THIS IS A TEMPORARY FILE FOR DEBUGGING; "
"IT CAN BE SAFELY DELETED ANY TIME.\n")
f.write("***\n")
f.write("1: Utxos that passed age and size limits, but have "
"been used too many times (see taker_utxo_retries "
"in the config):\n")
if len(priv_utxo_pairs) == 0:
f.write("None\n")
else:
for p, u in priv_utxo_pairs:
f.write(str(u) + "\n")
f.write("2: Utxos that have less than " +
jm_single().config.get("POLICY", "taker_utxo_age") +
" confirmations:\n")
if len(to) == 0:
f.write("None\n")
else:
for t in to:
f.write(str(t) + "\n")
f.write("3: Utxos that were not at least " + \
jm_single().config.get(
"POLICY", "taker_utxo_amtpercent") + "% of the "
"size of the coinjoin amount " + str(
self.proposed_cj_amount) + "\n")
if len(ts) == 0:
f.write("None\n")
else:
for t in ts:
f.write(str(t) + "\n")
f.write('***\n')
f.write(
"Utxos that appeared in item 1 cannot be used again.\n")
f.write(
"Utxos only in item 2 can be used by waiting for more "
"confirmations, (set by the value of taker_utxo_age).\n")
f.write("Utxos only in item 3 are not big enough for this "
"coinjoin transaction, set by the value "
"of taker_utxo_amtpercent.\n")
f.write(
"If you cannot source a utxo from your wallet according "
"to these rules, use the tool add-utxo.py to source a "
"utxo external to your joinmarket wallet. Read the help "
"with 'python add-utxo.py --help'\n\n")
f.write("You can also reset the rules in the joinmarket.cfg "
"file, but this is generally inadvisable.\n")
f.write(
"***\nFor reference, here are the utxos in your wallet:\n")
f.write("\n" + str(self.proposed_wallet.unspent))
return (None, None)
def make_commitment(self, wallet, input_utxos, cjamount):
"""The Taker default commitment function, which uses PoDLE.
Alternative commitment types should use a different commit type byte.
This will allow future upgrades to provide different style commitments
by subclassing Taker and changing the commit_type_byte; existing makers
will simply not accept this new type of commitment.
In case of success, return the commitment and its opening.
In case of failure returns (None, None) and constructs a detailed
log for the user to read and discern the reason.
"""
def filter_by_coin_age_amt(utxos, age, amt):
results = jm_single().bc_interface.query_utxo_set(utxos,
includeconf=True)
newresults = []
too_old = []
too_small = []
for i, r in enumerate(results):
#results return "None" if txo is spent; drop this
if not r:
continue
valid_age = r['confirms'] >= age
valid_amt = r['value'] >= amt
if not valid_age:
too_old.append(utxos[i])
if not valid_amt:
too_small.append(utxos[i])
if valid_age and valid_amt:
newresults.append(utxos[i])
return newresults, too_old, too_small
def priv_utxo_pairs_from_utxos(utxos, age, amt):
#returns pairs list of (priv, utxo) for each valid utxo;
#also returns lists "too_old" and "too_small" for any
#utxos that did not satisfy the criteria for debugging.
priv_utxo_pairs = []
new_utxos, too_old, too_small = filter_by_coin_age_amt(
utxos.keys(), age, amt)
new_utxos_dict = {k: v for k, v in utxos.items() if k in new_utxos}
for k, v in new_utxos_dict.iteritems():
addr = v['address']
priv = wallet.get_key_from_addr(addr)
if priv: #can be null from create-unsigned
priv_utxo_pairs.append((priv, k))
return priv_utxo_pairs, too_old, too_small
commit_type_byte = "P"
podle_data = None
tries = jm_single().config.getint("POLICY", "taker_utxo_retries")
age = jm_single().config.getint("POLICY", "taker_utxo_age")
#Minor rounding errors don't matter here
amt = int(cjamount * jm_single().config.getint(
"POLICY", "taker_utxo_amtpercent") / 100.0)
priv_utxo_pairs, to, ts = priv_utxo_pairs_from_utxos(input_utxos, age, amt)
#Note that we ignore the "too old" and "too small" lists in the first
#pass through, because the same utxos appear in the whole-wallet check.
#For podle data format see: btc.podle.PoDLE.reveal()
#In first round try, don't use external commitments
podle_data = btc.generate_podle(priv_utxo_pairs, tries)
if not podle_data:
#We defer to a second round to try *all* utxos in wallet;
#this is because it's much cleaner to use the utxos involved
#in the transaction, about to be consumed, rather than use
#random utxos that will persist after. At this step we also
#allow use of external utxos in the json file.
if wallet.unspent:
priv_utxo_pairs, to, ts = priv_utxo_pairs_from_utxos(
wallet.unspent, age, amt)
#Pre-filter the set of external commitments that work for this
#transaction according to its size and age.
dummy, extdict = btc.get_podle_commitments()
if len(extdict.keys()) > 0:
ext_valid, ext_to, ext_ts = filter_by_coin_age_amt(extdict.keys(),
age, amt)
else:
ext_valid = None
podle_data = btc.generate_podle(priv_utxo_pairs, tries, ext_valid)
if podle_data:
log.debug("Generated PoDLE: " + pprint.pformat(podle_data))
revelation = btc.PoDLE(u=podle_data['utxo'],P=podle_data['P'],
P2=podle_data['P2'],s=podle_data['sig'],
e=podle_data['e']).serialize_revelation()
return (commit_type_byte + podle_data["commit"], revelation)
else:
#we know that priv_utxo_pairs all passed age and size tests, so
#they must have failed the retries test. Summarize this info
#and publish to commitments_debug.txt
with open("commitments_debug.txt", "wb") as f:
f.write("THIS IS A TEMPORARY FILE FOR DEBUGGING; "
"IT CAN BE SAFELY DELETED ANY TIME.\n")
f.write("***\n")
f.write("1: Utxos that passed age and size limits, but have "
"been used too many times (see taker_utxo_retries "
"in the config):\n")
if len(priv_utxo_pairs) == 0:
f.write("None\n")
else:
for p, u in priv_utxo_pairs:
f.write(str(u) + "\n")
f.write("2: Utxos that have less than " + jm_single().config.get(
"POLICY", "taker_utxo_age") + " confirmations:\n")
if len(to) == 0:
f.write("None\n")
else:
for t in to:
f.write(str(t) + "\n")
f.write("3: Utxos that were not at least " + \
jm_single().config.get(
"POLICY", "taker_utxo_amtpercent") + "% of the "
"size of the coinjoin amount " + str(
self.proposed_cj_amount) + "\n")
if len(ts) == 0:
f.write("None\n")
else:
for t in ts:
f.write(str(t) + "\n")
f.write('***\n')
f.write("Utxos that appeared in item 1 cannot be used again.\n")
f.write("Utxos only in item 2 can be used by waiting for more "
"confirmations, (set by the value of taker_utxo_age).\n")
f.write("Utxos only in item 3 are not big enough for this "
"coinjoin transaction, set by the value "
"of taker_utxo_amtpercent.\n")
f.write("If you cannot source a utxo from your wallet according "
"to these rules, use the tool add-utxo.py to source a "
"utxo external to your joinmarket wallet. Read the help "
"with 'python add-utxo.py --help'\n\n")
f.write("You can also reset the rules in the joinmarket.cfg "
"file, but this is generally inadvisable.\n")
f.write("***\nFor reference, here are the utxos in your wallet:\n")
f.write("\n" + str(self.proposed_wallet.unspent))
return (None, None)