def execute(payload: dict): my_id = plugin.rpc.getinfo().get('id') peer_id = peer_from_scid(plugin, payload, payload['scid'], my_id) get_channel(plugin, payload, peer_id, payload['scid']) # ensures or raises error test_or_set_chunks(plugin, payload, my_id) plugin.log("%s %s %d%% %d chunks" % (payload['command'], payload['scid'], payload['percentage'], payload['chunks'])) # iterate of chunks, default just one for chunk in range(payload['chunks']): # we discover remaining capacities for each chunk, # as fees from previous chunks affect reserves spendable, receivable = spendable_from_scid(plugin, payload) total = spendable + receivable amount = total * 0.01 * payload['percentage'] / payload['chunks'] # if capacity exceeds, limit amount to full or empty channel if payload['command'] == "drain" and amount > spendable: amount = spendable if payload['command'] == "fill" and amount > receivable: amount = receivable result = False try: # we need to try with different HTLC_FEE values # until we dont get capacity error on first hop htlc_fee = HTLC_FEE_NUL htlc_stp = HTLC_FEE_STP while htlc_fee < HTLC_FEE_MAX and result is False: # When getting close to 100% we need to account for HTLC commitment fee if payload[ 'command'] == 'drain' and spendable - amount <= htlc_fee: if amount < htlc_fee: raise RpcError( payload['command'], payload, {'message': 'channel too low to cover fees'}) amount -= htlc_fee plugin.log( "Trying... chunk:%s/%s spendable:%s receivable:%s htlc_fee:%s => amount:%s" % (chunk + 1, payload['chunks'], spendable, receivable, htlc_fee, amount)) try: result = try_for_htlc_fee(plugin, payload, my_id, peer_id, amount, chunk, spendable) except Exception as err: if "htlc_fee unknown" in str(err): if htlc_fee == HTLC_FEE_NUL: htlc_fee = HTLC_FEE_MIN - HTLC_FEE_STP htlc_fee += htlc_stp htlc_stp *= 1.1 # exponential increase steps plugin.log( "Retrying with additional HTLC onchain fees: %s" % htlc_fee) continue if "htlc_fee is" in str(err): htlc_fee = Millisatoshi(str(err)[12:]) plugin.log( "Retrying with exact HTLC onchain fees: %s" % htlc_fee) continue raise err # If result is still false, we tried allowed htlc_fee range unsuccessfully if result is False: raise RpcError(payload['command'], payload, { 'message': 'Cannot determine required htlc commitment fees.' }) except Exception as e: return cleanup(plugin, payload, e) return cleanup(plugin, payload)
def rebalance(plugin, outgoing_scid, incoming_scid, msatoshi: Millisatoshi = None, maxfeepercent: float = 0.5, retry_for: int = 60, exemptfee: Millisatoshi = Millisatoshi(5000)): """Rebalancing channel liquidity with circular payments. This tool helps to move some msatoshis between your channels. """ if msatoshi: msatoshi = Millisatoshi(msatoshi) maxfeepercent = float(maxfeepercent) retry_for = int(retry_for) exemptfee = Millisatoshi(exemptfee) payload = { "outgoing_scid": outgoing_scid, "incoming_scid": incoming_scid, "msatoshi": msatoshi, "maxfeepercent": maxfeepercent, "retry_for": retry_for, "exemptfee": exemptfee } my_node_id = plugin.rpc.getinfo().get('id') outgoing_node_id = peer_from_scid(plugin, outgoing_scid, my_node_id, payload) incoming_node_id = peer_from_scid(plugin, incoming_scid, my_node_id, payload) get_channel(plugin, payload, outgoing_node_id, outgoing_scid, True) get_channel(plugin, payload, incoming_node_id, incoming_scid, True) out_ours, out_total = amounts_from_scid(plugin, outgoing_scid) in_ours, in_total = amounts_from_scid(plugin, incoming_scid) plugin.log("Outgoing node: %s, channel: %s" % (outgoing_node_id, outgoing_scid)) plugin.log("Incoming node: %s, channel: %s" % (incoming_node_id, incoming_scid)) # If amount was not given, calculate a suitable 50/50 rebalance amount if msatoshi is None: msatoshi = calc_optimal_amount(out_ours, out_total, in_ours, in_total, payload) plugin.log("Estimating optimal amount %s" % msatoshi) # Check requested amounts are selected channels if msatoshi > out_ours or msatoshi > in_total - in_ours: raise RpcError("rebalance", payload, {'message': 'Channel capacities too low'}) route_out = { 'id': outgoing_node_id, 'channel': outgoing_scid, 'direction': int(not my_node_id < outgoing_node_id) } route_in = { 'id': my_node_id, 'channel': incoming_scid, 'direction': int(not incoming_node_id < my_node_id) } start_ts = int(time.time()) label = "Rebalance-" + str(uuid.uuid4()) description = "%s to %s" % (outgoing_scid, incoming_scid) invoice = plugin.rpc.invoice(msatoshi, label, description, retry_for + 60) payment_hash = invoice['payment_hash'] plugin.log("Invoice payment_hash: %s" % payment_hash) success_msg = "" try: excludes = [] # excude all own channels to prevent unwanted shortcuts [out,mid,in] mychannels = plugin.rpc.listchannels(source=my_node_id)['channels'] for channel in mychannels: excludes += [ channel['short_channel_id'] + '/0', channel['short_channel_id'] + '/1' ] while int(time.time()) - start_ts < retry_for: r = plugin.rpc.getroute(incoming_node_id, msatoshi, riskfactor=1, cltv=9, fromid=outgoing_node_id, exclude=excludes) route_mid = r['route'] route = [route_out] + route_mid + [route_in] setup_routing_fees(plugin, route, msatoshi) fees = route[0]['amount_msat'] - msatoshi # check fee and exclude worst channel the next time # NOTE: the int(msat) casts are just a workaround for outdated pylightning versions if fees > exemptfee and int( fees) > int(msatoshi) * maxfeepercent / 100: worst_channel_id = find_worst_channel(route) if worst_channel_id is None: raise RpcError("rebalance", payload, {'message': 'Insufficient fee'}) excludes += [worst_channel_id + '/0', worst_channel_id + '/1'] continue success_msg = "%d msat sent over %d hops to rebalance %d msat" % ( msatoshi + fees, len(route), msatoshi) plugin.log( "Sending %s over %d hops to rebalance %s" % (msatoshi + fees, len(route), msatoshi), 'debug') for r in route: plugin.log( " - %s %14s %s" % (r['id'], r['channel'], r['amount_msat']), 'debug') try: plugin.rpc.sendpay(route, payment_hash) plugin.rpc.waitsendpay(payment_hash, retry_for + start_ts - int(time.time())) return success_msg except RpcError as e: plugin.log("RpcError: " + str(e)) erring_channel = e.error.get('data', {}).get('erring_channel') if erring_channel == incoming_scid: raise RpcError("rebalance", payload, {'message': 'Error with incoming channel'}) if erring_channel == outgoing_scid: raise RpcError("rebalance", payload, {'message': 'Error with outgoing channel'}) erring_direction = e.error.get('data', {}).get('erring_direction') if erring_channel is not None and erring_direction is not None: excludes.append(erring_channel + '/' + str(erring_direction)) except Exception as e: plugin.log("Exception: " + str(e)) return cleanup(plugin, label, payload, success_msg, e) return cleanup(plugin, label, payload, success_msg)
def try_for_htlc_fee(plugin, payload, my_id, peer_id, amount, chunk, spendable_before): start_ts = int(time.time()) label = payload['command'] + "-" + str(uuid.uuid4()) payload['labels'] += [label] description = "%s %s %s%s [%d/%d]" % (payload['command'], payload['scid'], payload['percentage'], '%', chunk + 1, payload['chunks']) invoice = plugin.rpc.invoice("any", label, description, payload['retry_for'] + 60) payment_hash = invoice['payment_hash'] plugin.log("Invoice payment_hash: %s" % payment_hash) # exclude selected channel to prevent unwanted shortcuts excludes = [payload['scid'] + '/0', payload['scid'] + '/1'] mychannels = plugin.rpc.listchannels(source=my_id).get('channels') # exclude local channels known to have too little capacity. # getroute currently does not do this. for channel in mychannels: if channel['short_channel_id'] == payload['scid']: continue # already added few lines above spend, recv = spendable_from_scid(plugin, payload, channel['short_channel_id']) if payload['command'] == 'drain' and recv < amount: excludes += [ channel['short_channel_id'] + '/0', channel['short_channel_id'] + '/1' ] if payload['command'] == 'fill' and spend < amount: excludes += [ channel['short_channel_id'] + '/0', channel['short_channel_id'] + '/1' ] while int(time.time()) - start_ts < payload['retry_for']: if payload['command'] == 'drain': r = plugin.rpc.getroute(my_id, amount, riskfactor=0, cltv=9, fromid=peer_id, fuzzpercent=0, exclude=excludes) route_out = { 'id': peer_id, 'channel': payload['scid'], 'direction': int(my_id >= peer_id) } route = [route_out] + r['route'] setup_routing_fees(plugin, payload, route, amount, True) if payload['command'] == 'fill': r = plugin.rpc.getroute(peer_id, amount, riskfactor=0, cltv=9, fromid=my_id, fuzzpercent=0, exclude=excludes) route_in = { 'id': my_id, 'channel': payload['scid'], 'direction': int(peer_id >= my_id) } route = r['route'] + [route_in] setup_routing_fees(plugin, payload, route, amount, False) fees = route[0]['amount_msat'] - route[-1]['amount_msat'] # check fee and exclude worst channel the next time # NOTE: the int(msat) casts are just a workaround for outdated pylightning versions if fees > payload['exemptfee'] and int( fees) > int(amount) * payload['maxfeepercent'] / 100: worst_channel_id = find_worst_channel(route) if worst_channel_id is None: raise RpcError(payload['command'], payload, {'message': 'Insufficient fee'}) excludes += [worst_channel_id + '/0', worst_channel_id + '/1'] continue plugin.log("[%d/%d] Sending over %d hops to %s %s using %s fees" % (chunk + 1, payload['chunks'], len(route), payload['command'], amount, fees)) for r in route: plugin.log(" - %s %14s %s" % (r['id'], r['channel'], r['amount_msat'])) try: plugin.rpc.sendpay(route, payment_hash, label) result = plugin.rpc.waitsendpay( payment_hash, payload['retry_for'] + start_ts - int(time.time())) if result.get('status') == 'complete': payload['success_msg'] += [ "%dmsat sent over %d hops to %s %dmsat [%d/%d]" % (amount + fees, len(route), payload['command'], amount, chunk + 1, payload['chunks']) ] # we need to wait for gossipd to update to new state, # so remaining amounts will be calculated correctly for the next chunk spendable, _ = spendable_from_scid(plugin, payload) while spendable == spendable_before: time.sleep(0.5) spendable, _ = spendable_from_scid(plugin, payload) return True return False except RpcError as e: erring_message = e.error.get('message', '') erring_channel = e.error.get('data', {}).get('erring_channel') erring_index = e.error.get('data', {}).get('erring_index') erring_direction = e.error.get('data', {}).get('erring_direction') # detect exceeding of HTLC commitment fee if 'Capacity exceeded' in erring_message and erring_index == 0: match = HTLC_FEE_PAT.search(erring_message) if match: # new servers tell htlc_fee via exception (#2691) raise ValueError("htlc_fee is %s" % match.group(1)) raise ValueError("htlc_fee unknown") if erring_channel == payload['scid']: raise RpcError(payload['command'], payload, { 'message': 'Error with selected channel: %s' % erring_message }) plugin.log("RpcError: " + str(e)) if erring_channel is not None and erring_direction is not None: excludes.append(erring_channel + '/' + str(erring_direction))
def test_or_set_chunks(plugin, payload, my_id): scid = payload['scid'] cmd = payload['command'] spendable, receivable = spendable_from_scid(plugin, payload) total = spendable + receivable amount = total * 0.01 * payload['percentage'] # if capacity exceeds, limit amount to full or empty channel if cmd == "drain" and amount > spendable: amount = spendable if cmd == "fill" and amount > receivable: amount = receivable # get all spendable/receivables for our channels channels = {} for channel in plugin.rpc.listchannels(source=my_id).get('channels'): if channel['short_channel_id'] == scid: continue spend, recv = spendable_from_scid(plugin, payload, channel['short_channel_id']) channels[channel['short_channel_id']] = { 'spendable': spend, 'receivable': recv, } # test if selected chunks fit into other channel capacities if payload['chunks'] >= 1: chunks = payload['chunks'] chunksize = amount / chunks fit = 0 for i in channels: channel = channels[i] if cmd == "drain": fit += int(channel['receivable']) // int(chunksize) if cmd == "fill": fit += int(channel['spendable']) // int(chunksize) if fit >= chunks: return if cmd == "drain": raise RpcError( payload['command'], payload, { 'message': 'Selected chunks (%d) will not fit incoming channel capacities.' % chunks }) if cmd == "fill": raise RpcError( payload['command'], payload, { 'message': 'Selected chunks (%d) will not fit outgoing channel capacities.' % chunks }) # if chunks is 0 -> auto detect from 1 to 16 (max) chunks until amounts fit else: chunks = 0 while chunks < 16: chunks += 1 chunksize = amount / chunks fit = 0 for i in channels: channel = channels[i] if cmd == "drain" and int(channel['receivable']) > 0: fit += int(channel['receivable']) // int(chunksize) if cmd == "fill" and int(channel['spendable']) > 0: fit += int(channel['spendable']) // int(chunksize) if fit >= chunks: payload['chunks'] = chunks return if cmd == "drain": raise RpcError( payload['command'], payload, { 'message': 'Cannot detect required chunks to perform operation. Incoming capacity problem.' }) if cmd == "fill": raise RpcError( payload['command'], payload, { 'message': 'Cannot detect required chunks to perform operation. Outgoing capacity problem.' })
def test_or_set_chunks(plugin, payload): scid = payload['scid'] cmd = payload['command'] spendable, receivable = spendable_from_scid(plugin, payload) total = spendable + receivable amount = Millisatoshi(int(int(total) * (0.01 * payload['percentage']))) # if capacity exceeds, limit amount to full or empty channel if cmd == "drain" and amount > spendable: amount = spendable if cmd == "fill" and amount > receivable: amount = receivable if amount == Millisatoshi(0): raise RpcError( payload['command'], payload, { 'message': 'Cannot detect required chunks to perform operation. Amount would be 0msat.' }) # get all spendable/receivables for our channels channels = {} for channel in plugin.rpc.listchannels( source=payload['my_id']).get('channels'): if channel['short_channel_id'] == scid: continue try: spend, recv = spendable_from_scid(plugin, payload, channel['short_channel_id'], True) except RpcError: continue channels[channel['short_channel_id']] = { 'spendable': spend, 'receivable': recv, } if len(channels) == 0: raise RpcError(payload['command'], payload, { 'message': 'Not enough usable channels to perform cyclic routing.' }) # test if selected chunks fit into other channel capacities chunks = payload['chunks'] if chunks > 0: chunksize = amount / chunks fit = 0 for i in channels: channel = channels[i] if cmd == "drain": fit += int(channel['receivable']) // int(chunksize) if cmd == "fill": fit += int(channel['spendable']) // int(chunksize) if fit >= chunks: return if cmd == "drain": raise RpcError( payload['command'], payload, { 'message': 'Selected chunks (%d) will not fit incoming channel capacities.' % chunks }) if cmd == "fill": raise RpcError( payload['command'], payload, { 'message': 'Selected chunks (%d) will not fit outgoing channel capacities.' % chunks }) # if chunks is 0 -> auto detect from 1 to 16 (max) chunks until amounts fit else: chunks = 0 while chunks < 16: chunks += 1 chunksize = amount / chunks fit = 0 for i in channels: channel = channels[i] if cmd == "drain" and int(channel['receivable']) > 0: fit += int(channel['receivable']) // int(chunksize) if cmd == "fill" and int(channel['spendable']) > 0: fit += int(channel['spendable']) // int(chunksize) if fit >= chunks: payload['chunks'] = chunks return if cmd == "drain": raise RpcError( payload['command'], payload, { 'message': 'Cannot detect required chunks to perform operation. Incoming capacity problem.' }) if cmd == "fill": raise RpcError( payload['command'], payload, { 'message': 'Cannot detect required chunks to perform operation. Outgoing capacity problem.' })
def rebalance(plugin, outgoing_scid, incoming_scid, msatoshi: Millisatoshi = None, retry_for: int = 60, maxfeepercent: float = 0.5, exemptfee: Millisatoshi = Millisatoshi(5000), getroute_method=None): """Rebalancing channel liquidity with circular payments. This tool helps to move some msatoshis between your channels. """ if msatoshi: msatoshi = Millisatoshi(msatoshi) retry_for = int(retry_for) maxfeepercent = float(maxfeepercent) if getroute_method is None: getroute = plugin.getroute else: getroute = getroute_switch(getroute_method) exemptfee = Millisatoshi(exemptfee) payload = { "outgoing_scid": outgoing_scid, "incoming_scid": incoming_scid, "msatoshi": msatoshi, "retry_for": retry_for, "maxfeepercent": maxfeepercent, "exemptfee": exemptfee } my_node_id = plugin.rpc.getinfo().get('id') outgoing_node_id = peer_from_scid(plugin, outgoing_scid, my_node_id, payload) incoming_node_id = peer_from_scid(plugin, incoming_scid, my_node_id, payload) get_channel(plugin, payload, outgoing_node_id, outgoing_scid, True) get_channel(plugin, payload, incoming_node_id, incoming_scid, True) out_ours, out_total = amounts_from_scid(plugin, outgoing_scid) in_ours, in_total = amounts_from_scid(plugin, incoming_scid) # If amount was not given, calculate a suitable 50/50 rebalance amount if msatoshi is None: msatoshi = calc_optimal_amount(out_ours, out_total, in_ours, in_total, payload) plugin.log("Estimating optimal amount %s" % msatoshi) # Check requested amounts are selected channels if msatoshi > out_ours or msatoshi > in_total - in_ours: raise RpcError("rebalance", payload, {'message': 'Channel capacities too low'}) plugin.log( f"starting rebalance out_scid:{outgoing_scid} in_scid:{incoming_scid} amount:{msatoshi}", 'debug') route_out = { 'id': outgoing_node_id, 'channel': outgoing_scid, 'direction': int(not my_node_id < outgoing_node_id) } route_in = { 'id': my_node_id, 'channel': incoming_scid, 'direction': int(not incoming_node_id < my_node_id) } start_ts = int(time.time()) label = "Rebalance-" + str(uuid.uuid4()) description = "%s to %s" % (outgoing_scid, incoming_scid) invoice = plugin.rpc.invoice(msatoshi, label, description, retry_for + 60) payment_hash = invoice['payment_hash'] rpc_result = None excludes = [my_node_id] # excude all own channels to prevent shortcuts nodes = {} # here we store erring node counts plugin.maxhopidx = 1 # start with short routes and increase plugin.msatfactoridx = plugin.msatfactor # start with high msatoshi factor to reduce # WIRE_TEMPORARY failures because of imbalances # 'disable' maxhops filter if set to <= 0 # I know this is ugly, but we don't ruin the rest of the code this way if plugin.maxhops <= 0: plugin.maxhopidx = 20 # trace stats count = 0 count_sendpay = 0 time_getroute = 0 time_sendpay = 0 try: while int(time.time() ) - start_ts < retry_for and not plugin.rebalance_stop: count += 1 try: time_start = time.time() r = getroute(plugin, targetid=incoming_node_id, fromid=outgoing_node_id, excludes=excludes, msatoshi=msatoshi) time_getroute += time.time() - time_start except NoRouteException: # no more chance for a successful getroute rpc_result = { 'status': 'error', 'message': 'No suitable routes found' } return cleanup(plugin, label, payload, rpc_result) except RpcError as e: # getroute can be successful next time with different parameters if e.method == "getroute" and e.error.get('code') == 205: continue else: raise e route_mid = r['route'] route = [route_out] + route_mid + [route_in] setup_routing_fees(plugin, route, msatoshi) fees = route[0]['amount_msat'] - msatoshi # check fee and exclude worst channel the next time # NOTE: the int(msat) casts are just a workaround for outdated pylightning versions if fees > exemptfee and int( fees) > int(msatoshi) * maxfeepercent / 100: worst_channel = find_worst_channel(route) if worst_channel is None: raise RpcError("rebalance", payload, {'message': 'Insufficient fee'}) excludes.append(worst_channel['channel'] + '/' + str(worst_channel['direction'])) continue rpc_result = { "sent": msatoshi + fees, "received": msatoshi, "fee": fees, "hops": len(route), "outgoing_scid": outgoing_scid, "incoming_scid": incoming_scid, "status": "complete", "message": f"{msatoshi + fees} sent over {len(route)} hops to rebalance {msatoshi}" } plugin.log( "Sending %s over %d hops to rebalance %s" % (msatoshi + fees, len(route), msatoshi), 'debug') for r in route: plugin.log( " - %s %14s %s" % (r['id'], r['channel'], r['amount_msat']), 'debug') time_start = time.time() count_sendpay += 1 try: plugin.rpc.sendpay(route, payment_hash) running_for = int(time.time()) - start_ts result = plugin.rpc.waitsendpay( payment_hash, max(retry_for - running_for, 0)) time_sendpay += time.time() - time_start if result.get('status') == "complete": rpc_result[ "stats"] = f"running_for:{int(time.time()) - start_ts} count_getroute:{count} time_getroute:{time_getroute} time_getroute_avg:{time_getroute / count} count_sendpay:{count_sendpay} time_sendpay:{time_sendpay} time_sendpay_avg:{time_sendpay / count_sendpay}" return cleanup(plugin, label, payload, rpc_result) except RpcError as e: time_sendpay += time.time() - time_start plugin.log( f"maxhops:{plugin.maxhopidx} msatfactor:{plugin.msatfactoridx} running_for:{int(time.time()) - start_ts} count_getroute:{count} time_getroute:{time_getroute} time_getroute_avg:{time_getroute / count} count_sendpay:{count_sendpay} time_sendpay:{time_sendpay} time_sendpay_avg:{time_sendpay / count_sendpay}", 'debug') #plugin.log(f"RpcError: {str(e)}", 'debug') # check if we ran into the `rpc.waitsendpay` timeout if e.method == "waitsendpay" and e.error.get('code') == 200: raise RpcError("rebalance", payload, {'message': 'Timeout reached'}) # check if we have problems with our own channels erring_node = e.error.get('data', {}).get('erring_node') erring_channel = e.error.get('data', {}).get('erring_channel') erring_direction = e.error.get('data', {}).get('erring_direction') if erring_channel == incoming_scid: raise RpcError("rebalance", payload, {'message': 'Error with incoming channel'}) if erring_channel == outgoing_scid: raise RpcError("rebalance", payload, {'message': 'Error with outgoing channel'}) # exclude other erroring channels if erring_channel is not None and erring_direction is not None: excludes.append(erring_channel + '/' + str(erring_direction)) # count and exclude nodes that produce a lot of errors if erring_node and plugin.erringnodes > 0: if nodes.get(erring_node) is None: nodes[erring_node] = 0 nodes[erring_node] += 1 if nodes[erring_node] >= plugin.erringnodes: excludes.append(erring_node) except Exception as e: return cleanup(plugin, label, payload, rpc_result, e) rpc_result = {'status': 'error', 'message': 'Timeout reached'} return cleanup(plugin, label, payload, rpc_result)
def peer_from_scid(plugin, short_channel_id, my_node_id, payload): channels = plugin.rpc.listchannels(short_channel_id).get('channels') for ch in channels: if ch['source'] == my_node_id: return ch['destination'] raise RpcError("rebalance", payload, {'message': 'Cannot find peer for channel: ' + short_channel_id})