def setUp(self):
if self.network_definition is None:
self.skipTest("This class doesn't represent a real test case.")
raise NotImplementedError(
"A network definition needs to be given.")
self.testnet = RegtestNetwork(
binary_folder=bin_dir,
network_definition_location=self.network_definition,
nodedata_folder=test_data_dir,
node_limit='H',
from_scratch=True)
self.testnet.run_nocleanup()
master_node_data_dir = self.testnet.master_node.lnd_data_dir
master_node_port = self.testnet.master_node.grpc_port
self.master_node_networkinfo = self.testnet.master_node.getnetworkinfo(
)
self.lndnode = LndNode(lnd_home=master_node_data_dir,
lnd_host='localhost:' + str(master_node_port),
regtest=True)
self.rebalancer = Rebalancer(self.lndnode,
max_effective_fee_rate=50,
budget_sat=20)
self.graph_test()
def main():
parser = Parser()
# config.ini is expected to be in home/.lndmanage directory
config_file = os.path.join(settings.home_dir, 'config.ini')
# if lndmanage is run with arguments, run once
if len(sys.argv) > 1:
# take arguments from sys.argv
args = parser.parse_arguments()
node = LndNode(config_file=config_file)
parser.run_commands(node, args)
# otherwise enter an interactive mode
else:
history_file = os.path.join(settings.home_dir, "command_history")
try:
readline.read_history_file(history_file)
except FileNotFoundError:
# history will be written later
pass
logger.info("Running in interactive mode. "
"You can type 'help' or 'exit'.")
node = LndNode(config_file=config_file)
while True:
try:
user_input = input("$ lndmanage ")
except (EOFError, KeyboardInterrupt):
readline.write_history_file(history_file)
logger.info("exit")
return 0
if user_input == 'help':
parser.parser.print_help()
continue
elif user_input == 'exit':
readline.write_history_file(history_file)
return 0
args_list = user_input.split(" ")
try:
# need to run with parse_known_args to get an exception
args = parser.parser.parse_args(args_list)
parser.run_commands(node, args)
except:
logger.exception("Exception encountered.")
continue
def setUp(self):
if self.network_definition is None:
self.skipTest("This class doesn't represent a real test case.")
raise NotImplementedError("A network definition path needs to be "
"given.")
self.testnet = Network(
binary_folder=bin_dir,
network_definition_location=self.network_definition,
nodedata_folder=test_data_dir,
node_limit='H',
from_scratch=True
)
self.testnet.run_nocleanup()
# to run the lightning network in the background and do some testing
# here, run:
# $ lnregtest --nodedata_folder /path/to/lndmanage/test/test_data/
# self.testnet.run_from_background()
# logger.info("Generated network information:")
# logger.info(format_dict(self.testnet.node_mapping))
# logger.info(format_dict(self.testnet.channel_mapping))
# logger.info(format_dict(self.testnet.assemble_graph()))
master_node_data_dir = self.testnet.master_node.data_dir
master_node_port = self.testnet.master_node._grpc_port
self.master_node_networkinfo = self.testnet.master_node.getnetworkinfo()
self.lndnode = LndNode(
lnd_home=master_node_data_dir,
lnd_host='localhost:' + str(master_node_port),
regtest=True
)
self.graph_test()
c['fees_total'] = 0
c['fees_total_per_week'] = 0
c['flow_direction'] = float('nan')
c['median_forwarding_out'] = float('nan')
c['median_forwarding_in'] = float('nan')
c['mean_forwarding_out'] = float('nan')
c['mean_forwarding_in'] = float('nan')
c['number_forwardings'] = 0
c['largest_forwarding_amount_in'] = float('nan')
c['largest_forwarding_amount_out'] = float('nan')
c['total_forwarding_in'] = float('nan')
c['total_forwarding_out'] = float('nan')
if abs(c['unbalancedness']) > settings.UNBALANCED_CHANNEL:
c['action_required'] = True
else:
c['action_required'] = False
if c['bandwidth_demand'] > 0.5:
c['action_required'] = True
return channels
if __name__ == '__main__':
import time
import logging.config
logging.config.dictConfig(settings.logger_config)
logger = logging.getLogger()
nd = LndNode()
fa = ForwardingAnalyzer(nd)
fa.initialize_forwarding_data(time_start=0, time_end=time.time())
print(fa.simple_flow_analysis())
channels_in[f['chan_id_in']] += f['amt_in']
total_amount += f['amt_in']
total_fees += f['fee_msat']
transactions += 1
fee_rate += f['effective_fee']
fee_rate /= transactions
print("-------- Forwarding statistics --------")
print("Number of forwardings: {}".format(transactions))
print("Total forwardings [sat]: {}".format(total_amount))
print("Total fees earned [sat]: {:.3f}".format(total_fees / 1000.))
print("Average fee rate: {:.6f}".format(fee_rate))
print("-------- Popular channels --------")
print("Popular channels out:")
for w in sorted(channels_out, key=channels_out.get, reverse=True)[:10]:
print(w, channels_out[w])
print("Popular channels in:")
for w in sorted(channels_in, key=channels_in.get, reverse=True)[:10]:
print(w, channels_in[w])
if __name__ == '__main__':
node = LndNode()
forwardings = node.get_forwarding_events()
# plot_forwardings(forwardings)
# plot_fees(forwardings)
statistics_forwardings(forwardings)
def append_to_history(self, stats: List[dict]):
"""append_history adds the fee setting statistics to a pickle file.
:param stats: fee statistics"""
logger.debug("Saving fee setting stats to fee history.")
with open(self.history_path, "a") as f:
json.dump(stats, f)
f.write("\n")
def read_history(self) -> List[dict]:
"""read_history is a function for unpickling the fee setting history.
:return: list of fee setting statistics
:rtype: list[dict]"""
with open(self.history_path, "r") as f:
history = []
for i, line in enumerate(f):
history.append(json.loads(line))
return history
if __name__ == "__main__":
from lndmanage.lib.node import LndNode
import logging.config
logging.config.dictConfig(settings.logger_config)
nd = LndNode("/home/user/.lndmanage/config.ini")
fee_setter = FeeSetter(nd)
fee_setter.set_fees()
"""
Takes a list of nodes and finds the neighbors with most connections to the nodes.
:param nodes: list
:param blacklist_nodes: list of node_pub_keys to be excluded from counting
:param nnodes: int, limit for the number of nodes returned
:return: list of tuples, (str pub_key, int number of neighbors)
"""
nodes = set(nodes)
# eliminate blacklisted nodes
nodes = nodes.difference(blacklist_nodes)
neighboring_nodes = []
for general_node in self.graph.nodes:
neighbors_general_node = set(self.neighbors(general_node))
intersection_with_nodes = nodes.intersection(neighbors_general_node)
number_of_connection_with_nodes = len(intersection_with_nodes)
neighboring_nodes.append((general_node, number_of_connection_with_nodes))
sorted_neighboring_nodes = sorted(neighboring_nodes, key=lambda x: x[1], reverse=True)
return sorted_neighboring_nodes[:nnodes]
if __name__ == '__main__':
import logging.config
logging.config.dictConfig(settings.logger_config)
from lndmanage.lib.node import LndNode
nd = LndNode('')
print(f"Graph size: {nd.network.graph.size()}")
print(f"Number of channels: {len(nd.network.edges.keys())}")
plt.show()
def plot_cltv(time_locks):
exponent_min = 0
exponent_max = 3
bin_factor = 10
bins_log = 10**np.linspace(
exponent_min, exponent_max, (exponent_max - exponent_min) * bin_factor + 1)
fig, ax = plt.subplots(figsize=standard_figsize, dpi=300)
ax.hist(time_locks, bins=bins_log)
plt.loglog()
ax.set_xlabel("CLTV bins [blocks]")
ax.set_ylabel("Number of channels")
plt.tight_layout()
plt.show()
if __name__ == "__main__":
nd = LndNode('/home/user/.lndmanage/config.ini')
base_fees, fee_rates, time_locks = extract_fee_settings(nd)
plot_fee_rates(fee_rates)
plot_base_fees(base_fees)
plot_cltv(time_locks)
def main():
parser = Parser()
# config.ini is expected to be in home/.lndmanage directory
config_file = os.path.join(settings.home_dir, 'config.ini')
# if lndmanage is run with arguments, run once
if len(sys.argv) > 1:
# take arguments from sys.argv
args = parser.parse_arguments()
node = LndNode(config_file=config_file)
parser.run_commands(node, args)
# otherwise enter an interactive mode
else:
history_file = os.path.join(settings.home_dir, "command_history")
try:
readline.read_history_file(history_file)
except FileNotFoundError:
# history will be written later
pass
logger.info("Running in interactive mode. "
"You can type 'help' or 'exit'.")
node = LndNode(config_file=config_file)
if parser.lncli_path:
logger.info("Enabled lncli: using " + parser.lncli_path)
while True:
try:
user_input = input("$ lndmanage ")
except KeyboardInterrupt:
logger.info("")
continue
except EOFError:
readline.write_history_file(history_file)
logger.info("exit")
return 0
if not user_input or user_input in ['help', '-h', '--help']:
parser.parser.print_help()
continue
elif user_input == 'exit':
readline.write_history_file(history_file)
return 0
args_list = user_input.split(" ")
# lncli execution
if args_list[0] == 'lncli':
if parser.lncli_path:
lncli = Lncli(parser.lncli_path, config_file)
lncli.lncli(args_list[1:])
continue
else:
logger.info(
"lncli not enabled, put lncli in PATH or in ~/.lndmanage"
)
continue
try:
# need to run with parse_known_args to get an exception
args = parser.parser.parse_args(args_list)
parser.run_commands(node, args)
except SystemExit:
# argparse may raise SystemExit on incorrect user input,
# which is a graceful exit. The user gets the standard output
# from argparse of what went wrong.
continue
def get_channel_properties(node: LndNode, time_interval_start: float,
time_interval_end: float) -> Dict:
"""Joins data from listchannels and fwdinghistory to have extended
information about channels.
:return: dict of channel information with channel_id as keys
"""
forwarding_analyzer = ForwardingAnalyzer(node)
forwarding_analyzer.initialize_forwarding_stats(time_interval_start,
time_interval_end)
# dict with channel_id keys
statistics = forwarding_analyzer.get_forwarding_statistics_channels()
logger.debug(f"Time interval (between first and last forwarding) is "
f"{forwarding_analyzer.max_time_interval_days:6.2f} days.")
# join the two data sets:
channels = node.get_unbalanced_channels(unbalancedness_greater_than=0.0)
for k, c in channels.items():
# we may not have forwarding data for every channel
chan_stats = statistics.get(c["chan_id"], {})
c["forwardings_per_channel_age"] = (
chan_stats.get("number_forwardings", 0.01) / c["age"])
c["bandwidth_demand"] = (max(
nan_to_zero(chan_stats.get("mean_forwarding_in", 0)),
nan_to_zero(chan_stats.get("mean_forwarding_out", 0)),
) / c["capacity"])
c["fees_out"] = chan_stats.get("fees_out", 0)
c["fees_in"] = chan_stats.get("fees_in", 0)
c["fees_in_out"] = chan_stats.get("fees_in_out", 0)
try:
c["fees_out_per_week"] = chan_stats.get("fees_out", 0) / (
forwarding_analyzer.max_time_interval_days / 7)
except ZeroDivisionError:
c["fees_out_per_week"] = float("nan")
try:
c["fees_in_per_week"] = chan_stats.get("fees_in", 0) / (
forwarding_analyzer.max_time_interval_days / 7)
except ZeroDivisionError:
c["fees_in_per_week"] = float("nan")
c["flow_direction"] = chan_stats.get("flow_direction", float("nan"))
c["median_forwarding_in"] = chan_stats.get("median_forwarding_in",
float("nan"))
c["median_forwarding_out"] = chan_stats.get("median_forwarding_out",
float("nan"))
c["mean_forwarding_in"] = chan_stats.get("mean_forwarding_in",
float("nan"))
c["mean_forwarding_out"] = chan_stats.get("mean_forwarding_out",
float("nan"))
c["number_forwardings"] = chan_stats.get("number_forwardings", 0)
c["largest_forwarding_amount_in"] = chan_stats.get(
"largest_forwarding_amount_in", float("nan"))
c["largest_forwarding_amount_out"] = chan_stats.get(
"largest_forwarding_amount_out", float("nan"))
c["total_forwarding_in"] = chan_stats.get("total_forwarding_in", 0)
c["total_forwarding_out"] = chan_stats.get("total_forwarding_out", 0)
# action required if flow same direction as unbalancedness
# or bandwidth demand too high
# TODO: refine 'action_required' by better metric
if (c["unbalancedness"] * c["flow_direction"] > 0
and abs(c["unbalancedness"]) > settings.UNBALANCED_CHANNEL):
c["action_required"] = True
else:
c["action_required"] = False
if c["bandwidth_demand"] > 0.5:
c["action_required"] = True
return channels
def get_node_properites(node: LndNode, time_interval_start: float,
time_interval_end: float) -> Dict:
"""Joins data from channels and fwdinghistory to have extended
information about a node.
:return: dict of node information with channel_id as keys
"""
forwarding_analyzer = ForwardingAnalyzer(node)
forwarding_analyzer.initialize_forwarding_stats(time_interval_start,
time_interval_end)
node_forwarding_statistics = forwarding_analyzer.get_forwarding_statistics_nodes(
)
logger.debug(f"Time interval (between first and last forwarding) is "
f"{forwarding_analyzer.max_time_interval_days:6.2f} days.")
channel_id_to_node_id = node.channel_id_to_node_id(open_only=True)
node_ids_with_open_channels = {
nid
for nid in channel_id_to_node_id.values()
}
open_channels = node.get_open_channels()
nodes_properties = {} # type: Dict[str, NodeProperties]
# for each channel, accumulate properties in node properties
for k, c in open_channels.items():
remote_pubkey = c["remote_pubkey"]
try:
properties = nodes_properties[remote_pubkey]
except KeyError:
nodes_properties[remote_pubkey] = NodeProperties(
age=c["age"],
local_fee_rates=[c["local_fee_rate"]],
local_base_fees=[c["local_base_fee"]],
local_balances=[c["local_balance"]],
number_active_channels=1 if c["active"] else 0,
number_channels=1,
number_private_channels=1 if c["private"] else 0,
remote_fee_rates=[c["peer_fee_rate"]],
remote_base_fees=[c["peer_base_fee"]],
remote_balances=[c["remote_balance"]],
sent_received_per_week=c["sent_received_per_week"],
public_capacities=[c["capacity"]] if not c["private"] else [],
private_capacites=[c["capacity"]] if c["private"] else [],
)
else:
properties.age = max(c["age"],
nodes_properties[c["remote_pubkey"]].age)
properties.local_fee_rates.append(c["local_fee_rate"])
properties.local_base_fees.append(c["local_base_fee"])
properties.local_balances.append(c["local_balance"])
properties.number_active_channels += 1 if c["active"] else 0
properties.number_channels += 1
properties.number_private_channels += 1 if c["private"] else 0
properties.remote_fee_rates.append(c["peer_fee_rate"])
properties.remote_base_fees.append(c["peer_base_fee"])
properties.remote_balances.append(c["remote_balance"])
properties.sent_received_per_week += c["sent_received_per_week"]
if not c["private"]:
properties.public_capacities.append(c["capacity"])
else:
properties.private_capacites.append(c["capacity"])
# unify node properties with forwarding data
node_properties_forwardings = {}
# we start with looping over node properties, as this info is complete
for node_id, properties in nodes_properties.items():
local_balance = sum(properties.local_balances)
remote_balance = sum(properties.remote_balances)
capacity = sum(properties.private_capacites) + sum(
properties.public_capacities)
# there can be old forwarding data, which we neglect
if node_id not in node_ids_with_open_channels:
continue
# initial data:
node_properties_forwardings[node_id] = {
"age":
properties.age,
"alias":
node.network.node_alias(node_id),
"local_base_fee":
np.median(properties.local_base_fees),
"local_fee_rate":
np.median(properties.local_fee_rates),
"local_balance":
local_balance,
"max_local_balance":
max(properties.local_balances),
"max_remote_balance":
max(properties.remote_balances),
"number_channels":
properties.number_channels,
"number_active_channels":
properties.number_active_channels,
"number_private_channels":
properties.number_private_channels,
"node_id":
node_id,
"remote_base_fee":
np.median(properties.remote_base_fees),
"remote_fee_rate":
np.median(properties.remote_fee_rates),
"remote_balance":
remote_balance,
"sent_reveived_per_week":
properties.sent_received_per_week,
"total_capacity":
capacity,
"max_public_capacity":
max(properties.public_capacities)
if properties.public_capacities else 0,
"unbalancedness":
local_balance_to_unbalancedness(local_balance, capacity, 0,
False)[0],
}
# add forwarding data if available:
try:
statistics = node_forwarding_statistics[node_id]
except KeyError: # we don't have forwarding data, populate with defaults
node_properties_forwardings[node_id].update({
"fees_out":
0,
"fees_in":
0,
"fees_in_out":
0,
"flow_direction":
float("nan"),
"largest_forwarding_amount_in":
float("nan"),
"largest_forwarding_amount_out":
float("nan"),
"median_forwarding_out":
float("nan"),
"median_forwarding_in":
float("nan"),
"mean_forwarding_out":
float("nan"),
"mean_forwarding_in":
float("nan"),
"number_forwardings":
0,
"total_forwarding_in":
0,
"total_forwarding_out":
0,
"total_forwarding":
0,
})
else:
node_properties_forwardings[node_id].update(**statistics)
try:
node_properties_forwardings[node_id][
"fees_out_per_week"] = node_properties_forwardings[node_id][
"fees_out"] / (forwarding_analyzer.max_time_interval_days /
7)
except ZeroDivisionError:
node_properties_forwardings[node_id]["fees_out_per_week"] = float(
"nan")
try:
node_properties_forwardings[node_id][
"fees_in_per_week"] = node_properties_forwardings[node_id][
"fees_in"] / (forwarding_analyzer.max_time_interval_days /
7)
except ZeroDivisionError:
node_properties_forwardings[node_id]["fees_in_per_week"] = float(
"nan")
# TODO: unify with information from liquidity hints
return node_properties_forwardings
from lndmanage.lib.listchannels import ListChannels
from lndmanage.lib.node import LndNode
from lndmanage import settings
import logging.config
logging.config.dictConfig(settings.logger_config)
if __name__ == '__main__':
node = LndNode()
listchannels = ListChannels(node)
node.print_status()
listchannels.print_channels_unbalanced(
unbalancedness=settings.UNBALANCED_CHANNEL, sort_string='alias')
class CircleTest(TestCase):
network_definition = None
def setUp(self):
if self.network_definition is None:
self.skipTest("This class doesn't represent a real test case.")
raise NotImplementedError(
"A network definition needs to be given.")
self.testnet = RegtestNetwork(
binary_folder=bin_dir,
network_definition_location=self.network_definition,
nodedata_folder=test_data_dir,
node_limit='H',
from_scratch=True)
self.testnet.run_nocleanup()
master_node_data_dir = self.testnet.master_node.lnd_data_dir
master_node_port = self.testnet.master_node.grpc_port
self.master_node_networkinfo = self.testnet.master_node.getnetworkinfo(
)
self.lndnode = LndNode(lnd_home=master_node_data_dir,
lnd_host='localhost:' + str(master_node_port),
regtest=True)
self.rebalancer = Rebalancer(self.lndnode,
max_effective_fee_rate=50,
budget_sat=20)
self.graph_test()
def tearDown(self):
self.testnet.cleanup()
def graph_test(self):
raise NotImplementedError
def circle_and_check(self,
rebalancer,
channel_number_send,
channel_number_receive,
amount_sat,
expected_fees_msat,
dry=False):
"""
Helper function for testing a circular payment.
:param rebalancer:
:type rebalancer: lndmanage.lib.rebalance.Rebalancer
:param channel_number_send: channel whose local balance is decreased
:type channel_number_send: int
:param channel_number_receive: channel whose local balance is increased
:type channel_number_receive: int
:param amount_sat: amount in satoshi to rebalance
:type amount_sat: int
:param expected_fees_msat: expected fees in millisatoshi for
the rebalance
:type expected_fees_msat: int
:param dry: if it should be a dry run
:type dry: bool
"""
# setup
graph_before = self.testnet.assemble_graph()
channel_id_send = self.testnet.channel_mapping[channel_number_send][
'channel_id']
channel_id_receive = self.testnet.channel_mapping[
channel_number_receive]['channel_id']
invoice_r_hash = self.lndnode.get_invoice(amount_sat, '')
# exercise
try:
fees_msat = rebalancer.rebalance_two_channels(
channel_id_send,
channel_id_receive,
amount_sat,
invoice_r_hash,
rebalancer.budget_sat,
dry=dry)
time.sleep(SLEEP_SEC_AFTER_REBALANCING)
except Exception as e:
raise e
# check
graph_after = self.testnet.assemble_graph()
channel_data_send_before = graph_before['A'][channel_number_send]
channel_data_receive_before = graph_before['A'][channel_number_receive]
channel_data_send_after = graph_after['A'][channel_number_send]
channel_data_receive_after = graph_after['A'][channel_number_receive]
listchannels = ListChannels(self.lndnode)
listchannels.print_all_channels('rev_alias')
# test that the fees are correct
self.assertEqual(fees_msat, expected_fees_msat)
# test if sending channel's remote balance has increased correctly
self.assertEqual(
amount_sat, channel_data_send_after['remote_balance'] -
channel_data_send_before['remote_balance'] -
int(expected_fees_msat // 1000), "Sending local balance is wrong")
# test if receiving channel's local balance has increased correctly
self.assertEqual(
amount_sat, channel_data_receive_after['local_balance'] -
channel_data_receive_before['local_balance'],
"Receiving local balance is wrong")
from lndmanage.lib.network_info import NetworkAnalysis
from lndmanage.lib.node import LndNode
from lndmanage import settings
import logging.config
logging.config.dictConfig(settings.logger_config)
logger = logging.getLogger(__name__)
if __name__ == '__main__':
node = LndNode()
network_analysis = NetworkAnalysis(node)
network_analysis.print_node_overview(node.pub_key)
logger.info('-------- Nodes with highest capacity: --------')
for n in network_analysis.get_sorted_nodes_by_property():
logger.info(n)
logger.info('-------- Nodes with highest degree: --------')
for n in network_analysis.get_sorted_nodes_by_property(key='degree'):
logger.info(n)
logger.info('-------- Nodes with highest capacity/channel: --------')
for n in network_analysis.get_sorted_nodes_by_property(
key='capacity_per_channel', min_degree=10):
logger.info(n)
logger.info('-------- Nodes with lowest capacity/channel: --------')
for n in network_analysis.get_sorted_nodes_by_property(
key='capacity_per_channel', min_degree=20, decrementing=False):
logger.info(n)
logger.info('-------- Nodes with most user nodes: --------')
for n in network_analysis.get_sorted_nodes_by_property(key='user_nodes',
min_degree=20):