def rebalanceall(plugin: Plugin, min_amount: Millisatoshi = Millisatoshi("50000sat"), feeratio: float = 0.5):
"""Rebalance all unbalanced channels if possible for a very low fee.
Default minimum rebalancable amount is 50000sat. Default feeratio = 0.5, half of our node's default fee.
To be economical, it tries to fix the liquidity cheaper than it can be ruined by transaction forwards.
It may run for a long time (hours) in the background, but can be stopped with the rebalancestop method.
"""
# some early checks before we start the async thread
if plugin.mutex.locked():
return {"message": "Rebalance is already running, this may take a while. To stop it use the cli method 'rebalancestop'."}
channels = get_open_channels(plugin)
if len(channels) <= 1:
return {"message": "Error: Not enough open channels to rebalance anything"}
our = sum(ch["to_us_msat"] for ch in channels)
total = sum(ch["total_msat"] for ch in channels)
min_amount = Millisatoshi(min_amount)
if total - our < min_amount or our < min_amount:
return {"message": "Error: Not enough liquidity to rebalance anything"}
# param parsing ensure correct type
plugin.feeratio = float(feeratio)
plugin.min_amount = min_amount
# run the job
t = Thread(target=rebalanceall_thread, args=(plugin, ))
t.start()
return {"message": f"Rebalance started with min rebalancable amount: {plugin.min_amount}, feeratio: {plugin.feeratio}"}
def feeadjust(plugin: Plugin, scid: str = None):
"""Adjust fees for all channels (default) or just a given `scid`.
This method is automatically called in plugin init, or can be called manually after a successful payment.
Otherwise, the plugin keeps the fees up-to-date.
"""
plugin.mutex.acquire(blocking=True)
plugin.peers = plugin.rpc.listpeers()["peers"]
if plugin.fee_strategy == get_fees_median and not plugin.listchannels_by_dst:
plugin.channels = plugin.rpc.listchannels()['channels']
channels_adjusted = 0
for peer in plugin.peers:
for chan in peer["channels"]:
if chan["state"] == "CHANNELD_NORMAL":
_scid = chan["short_channel_id"]
if scid != None and scid != _scid:
continue
plugin.adj_balances[_scid] = {
"our": int(chan["to_us_msat"]),
"total": int(chan["total_msat"])
}
channels_adjusted += maybe_adjust_fees(plugin, [_scid])
msg = f"{channels_adjusted} channel(s) adjusted"
plugin.log(msg)
plugin.mutex.release()
return msg
def feeadjust_would_be_nice(plugin: Plugin):
commands = [c for c in plugin.rpc.help().get("help") if c["command"].split()[0] == "feeadjust"]
if len(commands) == 1:
msg = plugin.rpc.feeadjust()
plugin.log(f"Feeadjust succeeded: {msg}")
else:
plugin.log("The feeadjuster plugin would be useful here")
def on_init(options: Mapping[str, str], plugin: Plugin, **kwargs):
# Reach into the DB and
configs = plugin.rpc.listconfigs()
plugin.db_path = configs['wallet']
destination = options['backup-destination']
# Ensure that we don't inadventently switch the destination
if not os.path.exists("backup.lock"):
print("Files in the current directory {}".format(", ".join(
os.listdir("."))))
return abort(
"Could not find backup.lock in the lightning-dir, have you initialized using the backup-cli utility?"
)
d = json.load(open("backup.lock", 'r'))
if destination is None or destination == 'null':
destination = d['backend_url']
elif destination != d['backend_url']:
abort("The destination specified as option does not match the one "
"specified in backup.lock. Please check your settings")
if not plugin.db_path.startswith('sqlite3'):
abort("The backup plugin only works with the sqlite3 database.")
plugin.backend = get_backend(destination, require_init=True)
for c in plugin.early_writes:
apply_write(plugin, c)
def shutdown(plugin: Plugin):
plugin.stop_incoming_htlcs = True
num_inflight_htlcs = 1
while num_inflight_htlcs > 0:
num_inflight_htlcs = count_inflight_htlcs()
if num_inflight_htlcs > 0:
plugin.log(f"Waiting for {num_inflight_htlcs} HTLCs to resolve...")
time.sleep(3)
plugin.rpc.stop()
def rebalancestop(plugin: Plugin):
"""It stops the ongoing rebalanceall.
"""
if not plugin.mutex.locked():
return {"message": "No rebalance is running, nothing to stop"}
plugin.rebalance_stop = True
plugin.mutex.acquire(blocking=True)
plugin.rebalance_stop = False
plugin.mutex.release()
return {"message": "Rebalance stopped"}
def setchannelfee(plugin: Plugin, scid: str, base: int, ppm: int):
fees = get_chan_fees(plugin, scid)
if fees is None or base == fees['base'] and ppm == fees['ppm']:
return False
try:
plugin.rpc.setchannelfee(scid, base, ppm)
return True
except RpcError as e:
plugin.log(f"Could not adjust fees for channel {scid}: '{e}'", level="error")
return False
def test_argument_coercion():
p = Plugin(autopatch=False)
def test1(msat: Millisatoshi):
assert isinstance(msat, Millisatoshi)
ba = p._bind_kwargs(test1, {"msat": "100msat"}, None)
test1(*ba.args)
ba = p._bind_pos(test1, ["100msat"], None)
test1(*ba.args, **ba.kwargs)
def maybe_rebalance_pairs(plugin: Plugin, ch1, ch2, failed_pairs: list):
scid1 = ch1["short_channel_id"]
scid2 = ch2["short_channel_id"]
result = {"success": False, "fee_spent": Millisatoshi(0)}
if scid1 + ":" + scid2 in failed_pairs:
return result
i = 0
while not plugin.rebalance_stop:
liquidity1 = liquidity_info(ch1, plugin.enough_liquidity,
plugin.ideal_ratio)
liquidity2 = liquidity_info(ch2, plugin.enough_liquidity,
plugin.ideal_ratio)
amount1 = min(must_send(liquidity1), could_receive(liquidity2))
amount2 = min(should_send(liquidity1), should_receive(liquidity2))
amount3 = min(could_send(liquidity1), must_receive(liquidity2))
amount = max(amount1, amount2, amount3)
if amount < plugin.min_amount:
return result
amount = min(amount, get_max_amount(i, plugin))
maxfee = get_max_fee(plugin, amount)
plugin.log(
f"Try to rebalance: {scid1} -> {scid2}; amount={amount}; maxfee={maxfee}"
)
start_ts = time.time()
try:
res = rebalance(plugin,
outgoing_scid=scid1,
incoming_scid=scid2,
msatoshi=amount,
maxfeepercent=0,
retry_for=1200,
exemptfee=maxfee)
except Exception:
failed_pairs.append(scid1 + ":" + scid2)
# rebalance failed, let's try with a smaller amount
while (get_max_amount(i, plugin) >= amount and
get_max_amount(i, plugin) != get_max_amount(i + 1, plugin)):
i += 1
if amount > get_max_amount(i, plugin):
continue
return result
elapsed_time = timedelta(seconds=time.time() - start_ts)
res["elapsed_time"] = str(elapsed_time)[:-3]
plugin.log(f"Rebalance succeeded: {res}")
result["success"] = True
result["fee_spent"] += res["fee"]
# refresh channels
time.sleep(10)
ch1 = get_chan(plugin, scid1)
assert ch1 is not None
ch2 = get_chan(plugin, scid2)
assert ch2 is not None
return result
def feeadjuster_toggle(plugin: Plugin, value: bool = None):
"""Activates/Deactivates automatic fee updates for forward events.
The status will be set to value.
"""
msg = {"forward_event_subscription": {"previous": plugin.forward_event_subscription}}
if value is None:
plugin.forward_event_subscription = not plugin.forward_event_subscription
else:
plugin.forward_event_subscription = bool(value)
msg["forward_event_subscription"]["current"] = plugin.forward_event_subscription
return msg
def maybe_setchannelfee(plugin: Plugin, scid: str, base: int, ppm: int):
fees = get_chan_fees(plugin, scid)
if fees is None or base == fees["base_fee_millisatoshi"] and ppm == fees[
"fee_per_millionth"]:
return False
try:
plugin.rpc.setchannelfee(scid, base, ppm)
return True
except RpcError as e:
plugin.log(f"Could not adjust fees for channel {scid}: '{e}'",
level="error")
return False
def init(plugin: Plugin, options: Mapping[str, str], **_kwargs):
plugin.boltz_api = options["boltz-api"]
plugin.boltz_node = options["boltz-node"]
plugin.data_location = path.join(plugin.rpc.listconfigs()["lightning-dir"], "channel-creation.json")
print("Channel Creation data location: {}".format(plugin.data_location))
read_channel_creation(plugin)
print("Started channel-creation plugin: {}".format({
"boltz_api": plugin.boltz_api,
"boltz_node": plugin.boltz_node,
}))
def rebalanceall(plugin: Plugin, min_amount: Millisatoshi = Millisatoshi("50000sat"), feeratio: float = 0.5):
"""Rebalance all unbalanced channels if possible for a very low fee.
Default minimum rebalancable amount is 50000sat. Default feeratio = 0.5, half of our node's default fee.
To be economical, it tries to fix the liquidity cheaper than it can be ruined by transaction forwards.
It may run for a long time (hours) in the background, but can be stopped with the rebalancestop method.
"""
if plugin.mutex.locked():
return {"message": "Rebalance is already running, this may take a while. To stop it use the cli method 'rebalancestop'."}
plugin.feeratio = float(feeratio)
plugin.min_amount = Millisatoshi(min_amount)
t = Thread(target=rebalanceall_thread, args=(plugin, ))
t.start()
return {"message": "Rebalance started"}
def maybe_adjust_fees(plugin: Plugin, scids: list):
channels_adjusted = 0
for scid in scids:
our = plugin.adj_balances[scid]["our"]
total = plugin.adj_balances[scid]["total"]
percentage = our / total
base = plugin.adj_basefee
ppm = plugin.adj_ppmfee
# select ideal values per channel
fees = plugin.fee_strategy(plugin, scid)
if fees is not None:
base = fees['base']
ppm = fees['ppm']
# reset to normal fees if imbalance is not high enough
if (percentage > plugin.imbalance and percentage < 1 - plugin.imbalance):
if setchannelfee(plugin, scid, base, ppm):
plugin.log(f"Set default fees as imbalance is too low: {scid}")
plugin.adj_balances[scid]["last_liquidity"] = our
channels_adjusted += 1
continue
if not significant_update(plugin, scid):
continue
percentage = get_adjusted_percentage(plugin, scid)
assert 0 <= percentage and percentage <= 1
ratio = plugin.get_ratio(percentage)
if setchannelfee(plugin, scid, int(base), int(ppm * ratio)):
plugin.log(f"Adjusted fees of {scid} with a ratio of {ratio}")
plugin.adj_balances[scid]["last_liquidity"] = our
channels_adjusted += 1
return channels_adjusted
def rebalancestop(plugin: Plugin):
"""It stops the ongoing rebalanceall.
"""
if not plugin.mutex.locked():
if plugin.rebalanceall_msg is None:
return {"message": "No rebalance is running, nothing to stop."}
return {
"message":
f"No rebalance is running, nothing to stop. "
f"Last 'rebalanceall' gave: {plugin.rebalanceall_msg}"
}
plugin.rebalance_stop = True
plugin.mutex.acquire(blocking=True)
plugin.rebalance_stop = False
plugin.mutex.release()
return {"message": plugin.rebalanceall_msg}
def getroute_iterative(plugin: Plugin, targetid, fromid, excludes,
msatoshi: Millisatoshi):
""" This searches for 'shorter and bigger pipes' first in order
to increase likelyhood of success on short timeout.
Can be useful for manual `rebalance`.
"""
try:
return plugin.rpc.getroute(targetid,
fromid=fromid,
exclude=excludes,
msatoshi=msatoshi * plugin.msatfactoridx,
maxhops=plugin.maxhopidx,
riskfactor=10,
cltv=9)
except RpcError as e:
# could not find route -> change params and restart loop
if e.method == "getroute" and e.error.get('code') == 205:
# reduce _msatfactor to look for smaller channels now
plugin.msatfactoridx -= 1
if plugin.msatfactoridx < 1:
# when we reached neutral msat factor:
# increase _maxhops and restart with msatfactor
plugin.maxhopidx += 1
plugin.msatfactoridx = plugin.msatfactor
# abort if we reached maxhop limit
if plugin.maxhops > 0 and plugin.maxhopidx > plugin.maxhops:
raise NoRouteException
raise e
def maybe_adjust_fees(plugin: Plugin, scids: list):
channels_adjusted = 0
for scid in scids:
our = plugin.adj_balances[scid]["our"]
total = plugin.adj_balances[scid]["total"]
percentage = our / total
# reset to normal fees if imbalance is not high enough
if (percentage > plugin.imbalance
and percentage < 1 - plugin.imbalance):
if maybe_setchannelfee(plugin, scid, plugin.adj_basefee,
plugin.adj_ppmfee):
plugin.log(f"Set default fees as imbalance is too low: {scid}")
plugin.adj_balances[scid]["last_liquidity"] = our
channels_adjusted += 1
continue
if not significant_update(plugin, scid):
continue
percentage = get_adjusted_percentage(plugin, scid)
assert 0 <= percentage and percentage <= 1
ratio = plugin.get_ratio(percentage)
if maybe_setchannelfee(plugin, scid, int(plugin.adj_basefee * ratio),
int(plugin.adj_ppmfee * ratio)):
plugin.log(f"Adjusted fees of {scid} with a ratio of {ratio}")
plugin.adj_balances[scid]["last_liquidity"] = our
channels_adjusted += 1
return channels_adjusted
def rebalanceall_thread(plugin: Plugin):
if not plugin.mutex.acquire(blocking=False):
return
try:
start_ts = time.time()
feeadjuster_state = feeadjuster_toggle(plugin, False)
channels = get_open_channels(plugin)
plugin.enough_liquidity = get_enough_liquidity_threshold(channels)
plugin.ideal_ratio = get_ideal_ratio(channels, plugin.enough_liquidity)
plugin.log(
f"Automatic rebalance is running with enough liquidity threshold: {plugin.enough_liquidity}, "
f"ideal liquidity ratio: {plugin.ideal_ratio * 100:.2f}%, "
f"min rebalancable amount: {plugin.min_amount}, "
f"feeratio: {plugin.feeratio}")
failed_channels = []
success = 0
fee_spent = Millisatoshi(0)
while not plugin.rebalance_stop:
result = maybe_rebalance_once(plugin, failed_channels)
if not result["success"]:
break
success += 1
fee_spent += result["fee_spent"]
feeadjust_would_be_nice(plugin)
feeadjuster_toggle(plugin, feeadjuster_state)
elapsed_time = timedelta(seconds=time.time() - start_ts)
plugin.rebalanceall_msg = f"Automatic rebalance finished: {success} successful rebalance, {fee_spent} fee spent, it took {str(elapsed_time)[:-3]}"
plugin.log(plugin.rebalanceall_msg)
finally:
plugin.mutex.release()
def maybe_add_new_balances(plugin: Plugin, scids: list):
for scid in scids:
if scid not in plugin.adj_balances:
chan = get_chan(plugin, scid)
assert chan is not None
plugin.adj_balances[scid] = {
"our": int(chan["to_us_msat"]),
"total": int(chan["total_msat"])
}
def get_stub():
plugin = Plugin()
plugin.avail_interval = 60
plugin.avail_window = 3600
plugin.persist = {}
plugin.persist['peerstate'] = {}
plugin.persist['availcount'] = 0
return plugin
def forward_event(plugin: Plugin, forward_event: dict, **kwargs):
if not plugin.forward_event_subscription:
return
plugin.mutex.acquire(blocking=True)
if forward_event["status"] == "settled":
in_scid = forward_event["in_channel"]
out_scid = forward_event["out_channel"]
maybe_add_new_balances(plugin, [in_scid, out_scid])
plugin.adj_balances[in_scid]["our"] += forward_event["in_msatoshi"]
plugin.adj_balances[out_scid]["our"] -= forward_event["out_msatoshi"]
try:
# Pseudo-randomly add some hysterisis to the update
if not plugin.deactivate_fuzz and random.randint(0, 9) == 9:
time.sleep(random.randint(0, 5))
maybe_adjust_fees(plugin, [in_scid, out_scid])
except Exception as e:
plugin.log("Adjusting fees: " + str(e), level="error")
plugin.mutex.release()
def test_method_exceptions():
"""A bunch of tests that should fail calling the methods."""
p = Plugin(autopatch=False)
def fake_write_result(resultdict):
global result_dict
result_dict = resultdict
@p.method("test_raise")
def test_raise():
raise RpcException("testing RpcException", code=-1000)
req = Request(p, 1, "test_raise", {})
req._write_result = fake_write_result
p._dispatch_request(req)
assert result_dict['jsonrpc'] == '2.0'
assert result_dict['id'] == 1
assert result_dict['error']['code'] == -1000
assert result_dict['error']['message'] == "testing RpcException"
@p.method("test_raise2")
def test_raise2():
raise Exception("normal exception")
req = Request(p, 1, "test_raise2", {})
req._write_result = fake_write_result
p._dispatch_request(req)
assert result_dict['jsonrpc'] == '2.0'
assert result_dict['id'] == 1
assert result_dict['error']['code'] == -32600
assert result_dict['error'][
'message'] == "Error while processing test_raise2: normal exception"
def feeadjust(plugin: Plugin):
"""Adjust fees for all existing channels.
This method is automatically called in plugin init, or can be called manually after a successful payment.
Otherwise, the plugin keeps the fees up-to-date.
"""
plugin.mutex.acquire(blocking=True)
peers = plugin.rpc.listpeers()["peers"]
channels_adjusted = 0
for peer in peers:
for chan in peer["channels"]:
if chan["state"] == "CHANNELD_NORMAL":
scid = chan["short_channel_id"]
plugin.adj_balances[scid] = {
"our": int(chan["to_us_msat"]),
"total": int(chan["total_msat"])
}
channels_adjusted += maybe_adjust_fees(plugin, [scid])
msg = f"{channels_adjusted} channels adjusted"
plugin.log(msg)
plugin.mutex.release()
return msg
def maybe_add_new_balances(plugin: Plugin, scids: list):
for scid in scids:
if scid not in plugin.adj_balances:
# At this point we could call listchannels and pass the scid as
# argument, and deduce the peer id (!our_id). But -as unlikely as
# it is- we may not find it in our gossip (eg some corruption of
# the gossip_store occured just before the forwarding event).
# However, it MUST be present in a listpeers() entry.
chan = get_chan(plugin, scid)
assert chan is not None
plugin.adj_balances[scid] = {
"our": int(chan["to_us_msat"]),
"total": int(chan["total_msat"])
}
def get_fees_median(plugin: Plugin, scid: str):
""" Median fees from peers or peer.
The assumption is that our node competes in fees to other peers of a peer.
"""
peer_id = get_peer_id_for_scid(plugin, scid)
assert peer_id is not None
if plugin.listchannels_by_dst:
plugin.channels = plugin.rpc.call("listchannels",
{"destination": peer_id})['channels']
channels_to_peer = [ch for ch in plugin.channels
if ch['destination'] == peer_id
and ch['source'] != plugin.our_node_id]
if len(channels_to_peer) == 0:
return None
fees_ppm = [ch['fee_per_millionth'] for ch in channels_to_peer]
return {"base": plugin.adj_basefee, "ppm": statistics.median(fees_ppm)}
def read_channel_creation(plugin: Plugin):
try:
with open(plugin.data_location, 'r') as file:
raw_data = json.load(file)
plugin.channel_creation = ChannelCreation(
status=Status[raw_data["status"]],
id=raw_data["id"],
private_key=raw_data["private_key"],
redeem_script=raw_data["redeem_script"],
private=raw_data["private"],
invoice_amount=raw_data["invoice_amount"],
inbound_percentage=raw_data["inbound_percentage"],
invoice_label=raw_data["invoice_label"],
preimage_hash=raw_data["preimage_hash"],
address=raw_data["address"],
expected_amount=raw_data["expected_amount"],
bip21=raw_data["bip21"],
)
print("Read exiting channel creation state: {}".format(format_channel_creation(plugin.channel_creation)))
except FileNotFoundError:
print("Did not file existing channel creation state")
def test_duplicate_result():
p = Plugin(autopatch=False)
def test1(request):
request.set_result(1) # MARKER1
request.set_result(1)
req = Request(p, req_id=1, method="test1", params=[])
ba = p._bind_kwargs(test1, {}, req)
with pytest.raises(
ValueError,
match=r'current state is RequestState\.FINISHED(.*\n.*)*MARKER1'):
test1(*ba.args)
def test2(request):
request.set_exception(1) # MARKER2
request.set_exception(1)
req = Request(p, req_id=2, method="test2", params=[])
ba = p._bind_kwargs(test2, {}, req)
with pytest.raises(
ValueError,
match=r'current state is RequestState\.FAILED(.*\n*.*)*MARKER2'):
test2(*ba.args)
def test3(request):
request.set_exception(1) # MARKER3
request.set_result(1)
req = Request(p, req_id=3, method="test3", params=[])
ba = p._bind_kwargs(test3, {}, req)
with pytest.raises(
ValueError,
match=r'current state is RequestState\.FAILED(.*\n*.*)*MARKER3'):
test3(*ba.args)
def test4(request):
request.set_result(1) # MARKER4
request.set_exception(1)
req = Request(p, req_id=4, method="test4", params=[])
ba = p._bind_kwargs(test4, {}, req)
with pytest.raises(
ValueError,
match=r'current state is RequestState\.FINISHED(.*\n*.*)*MARKER4'):
test4(*ba.args)
def rebalanceall_thread(plugin: Plugin):
if not plugin.mutex.acquire(blocking=False):
return
try:
channels = get_our_channels(plugin)
plugin.enough_liquidity = get_enough_liquidity_threshold(channels)
plugin.ideal_ratio = get_ideal_ratio(channels, plugin.enough_liquidity)
plugin.log(f"Automatic rebalance is running with enough liquidity threshold: {plugin.enough_liquidity}, "
f"ideal liquidity ratio: {plugin.ideal_ratio * 100:.2f}%, "
f"min rebalancable amount: {plugin.min_amount}, "
f"feeratio: {plugin.feeratio}")
failed_pairs = []
success = 0
fee_spent = Millisatoshi(0)
while not plugin.rebalance_stop:
result = maybe_rebalance_once(plugin, failed_pairs)
if not result["success"]:
break
success += 1
fee_spent += result["fee_spent"]
feeadjust_would_be_nice(plugin)
plugin.log(f"Automatic rebalance finished: {success} successful rebalance, {fee_spent} fee spent")
finally:
plugin.mutex.release()
from tqdm import tqdm
from typing import Mapping, Type, Iterator
from urllib.parse import urlparse
import json
import logging
import os
import re
import shutil
import struct
import sys
import sqlite3
import tempfile
import time
import psutil
plugin = Plugin()
root = logging.getLogger()
root.setLevel(logging.INFO)
handler = logging.StreamHandler(sys.stdout)
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(message)s')
handler.setFormatter(formatter)
root.addHandler(handler)
# A change that was proposed by c-lightning that needs saving to the
# backup. `version` is the database version before the transaction was
# applied. The optional snapshot reqpresents a complete copy of the database,
# as it was before applying the `transaction`. This is used by the plugin from
# time to time to allow the backend to compress the changelog and forms a new
- -1 indicates that we were unable to find a route to the destination. This
usually indicates that this is a leaf node that is currently offline.
- 16399 is the code for unknown payment details and indicates a successful
probe. The destination received the incoming payment but could not find a
matching `payment_key`, which is expected since we generated the
`payment_hash` at random :-)
"""
from datetime import datetime
from pyln.client import Plugin, RpcError
import random
import string
plugin = Plugin()
@plugin.method("probe")
def bin_search(plugin, node_id, capacity, epsilon, **kwargs):
def loop(lower, upper, i):
i += 1
middle = (lower + upper) // 2
error_code = failcode(node_id, middle)
if (upper - lower) < epsilon:
return middle, i
elif error_code == 16399:
return loop(middle, upper, i)
