async def run(port: int, destination: str, localhost: bool) -> None:
if localhost:
ip = "127.0.0.1"
else:
ip = urllib.request.urlopen("https://v4.ident.me/").read().decode("utf8")
transport_opt = f"/ip4/{ip}/tcp/{port}"
host = await new_node(transport_opt=[transport_opt])
await host.get_network().listen(multiaddr.Multiaddr(transport_opt))
if not destination: # its the server
async def stream_handler(stream: INetStream) -> None:
asyncio.ensure_future(read_data(stream))
asyncio.ensure_future(write_data(stream))
host.set_stream_handler(PROTOCOL_ID, stream_handler)
localhost_opt = " --localhost" if localhost else ""
print(
f"Run 'python ./examples/chat/chat.py"
+ localhost_opt
+ f" -p {int(port) + 1} -d /ip4/{ip}/tcp/{port}/p2p/{host.get_id().pretty()}'"
+ " on another console."
)
print("Waiting for incoming connection...")
else: # its the client
maddr = multiaddr.Multiaddr(destination)
info = info_from_p2p_addr(maddr)
# Associate the peer with local ip address
await host.connect(info)
# Start a stream with the destination.
# Multiaddress of the destination peer is fetched from the peerstore using 'peerId'.
stream = await host.new_stream(info.peer_id, [PROTOCOL_ID])
asyncio.ensure_future(read_data(stream))
asyncio.ensure_future(write_data(stream))
print("Connected to peer %s" % info.addrs[0])
def get_addrs(self):
"""
:return: all the multiaddr addresses this host is listening too
"""
p2p_part = multiaddr.Multiaddr('/p2p/{}'.format(
self.get_id().pretty()))
addrs = []
for transport in self.network.listeners.values():
for addr in transport.get_addrs():
addrs.append(addr.encapsulate(p2p_part))
return addrs
def get_addrs(self) -> List[multiaddr.Multiaddr]:
"""
:return: all the multiaddr addresses this host is listening to
"""
# TODO: We don't need "/p2p/{peer_id}" postfix actually.
p2p_part = multiaddr.Multiaddr(f"/p2p/{self.get_id()!s}")
addrs: List[multiaddr.Multiaddr] = []
for transport in self._network.listeners.values():
for addr in transport.get_addrs():
addrs.append(addr.encapsulate(p2p_part))
return addrs
async def test_simple_two_nodes():
node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"])
node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"])
await node_a.get_network().listen(
multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0"))
await node_b.get_network().listen(
multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0"))
supported_protocols = ["/floodsub/1.0.0"]
floodsub_a = FloodSub(supported_protocols)
pubsub_a = Pubsub(node_a, floodsub_a, "a")
floodsub_b = FloodSub(supported_protocols)
pubsub_b = Pubsub(node_b, floodsub_b, "b")
await connect(node_a, node_b)
await asyncio.sleep(0.25)
qb = await pubsub_b.subscribe("my_topic")
await asyncio.sleep(0.25)
node_a_id = str(node_a.get_id())
next_msg_id_func = message_id_generator(0)
msg = generate_RPC_packet(node_a_id, ["my_topic"], "some data",
next_msg_id_func())
await floodsub_a.publish(node_a_id, msg.SerializeToString())
await asyncio.sleep(0.25)
res_b = await qb.get()
# Check that the msg received by node_b is the same
# as the message sent by node_a
assert res_b.SerializeToString() == msg.publish[0].SerializeToString()
# Success, terminate pending tasks.
await cleanup()
async def perform_simple_test(assertion_func, \
transports_for_initiator, transports_for_noninitiator):
# Create libp2p nodes and connect them, then secure the connection, then check
# the proper security was chosen
# TODO: implement -- note we need to introduce the notion of communicating over a raw connection
# for testing, we do NOT want to communicate over a stream so we can't just create two nodes
# and use their conn because our mplex will internally relay messages to a stream
sec_opt1 = transports_for_initiator
sec_opt2 = transports_for_noninitiator
node1 = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"],
sec_opt=sec_opt1)
node2 = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"],
sec_opt=sec_opt2)
await node1.get_network().listen(
multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0"))
await node2.get_network().listen(
multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0"))
await connect(node1, node2)
# Wait a very short period to allow conns to be stored (since the functions
# storing the conns are async, they may happen at slightly different times
# on each node)
await asyncio.sleep(0.1)
# Get conns
node1_conn = node1.get_network().connections[peer_id_for_node(node2)]
node2_conn = node2.get_network().connections[peer_id_for_node(node1)]
# Perform assertion
assertion_func(node1_conn.secured_conn.get_security_details())
assertion_func(node2_conn.secured_conn.get_security_details())
# Success, terminate pending tasks.
await cleanup()
async def test_init():
node = await new_node(transport_opt=["/ip4/127.1/tcp/0"])
await node.get_network().listen(multiaddr.Multiaddr("/ip4/127.1/tcp/0"))
supported_protocols = ["/gossipsub/1.0.0"]
gossipsub = GossipSub(supported_protocols, 3, 2, 4, 30)
pubsub = Pubsub(node, gossipsub, "a")
# Did it work?
assert gossipsub and pubsub
await cleanup()
async def run(port, destination):
host = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/%s" % port])
m = multiaddr.Multiaddr(destination)
info = info_from_p2p_addr(m)
# Associate the peer with local ip address
await host.connect(info)
# Start a stream with the destination.
# Multiaddress of the destination peer is fetched from the peerstore using 'peerId'.
stream = await host.new_stream(info.peer_id, [PROTOCOL_ID])
asyncio.ensure_future(read_data(stream))
asyncio.ensure_future(write_data(stream))
print("Connected to peer %s" % info.addrs[0])
async def run(port, destination):
external_ip = urllib.request.urlopen('https://v4.ident.me/').read().decode(
'utf8')
transport_opt = "/ip4/%s/tcp/%s" % (external_ip, port)
host = await new_node(transport_opt=[transport_opt])
await host.get_network().listen(multiaddr.Multiaddr(transport_opt))
if not destination: # its the server
async def stream_handler(stream):
asyncio.ensure_future(read_data(stream))
asyncio.ensure_future(write_data(stream))
host.set_stream_handler(PROTOCOL_ID, stream_handler)
if not port:
raise RuntimeError("was not able to find the actual local port")
print(
"Run './examples/chat/chat.py -p %s -d /ip4/%s/tcp/%s/p2p/%s' on another console.\n"
% (int(port) + 1, external_ip, port, host.get_id().pretty()))
print("\nWaiting for incoming connection\n\n")
else: # its the client
m = multiaddr.Multiaddr(destination)
info = info_from_p2p_addr(m)
# Associate the peer with local ip address
await host.connect(info)
# Start a stream with the destination.
# Multiaddress of the destination peer is fetched from the peerstore using 'peerId'.
stream = await host.new_stream(info.peer_id, [PROTOCOL_ID])
asyncio.ensure_future(read_data(stream))
asyncio.ensure_future(write_data(stream))
print("Connected to peer %s" % info.addrs[0])
async def initialize_host(key, host='0.0.0.0', port=4025, listen=True, protocol_active=True):
from .peers import publish_host, monitor_hosts
from .protocol import AlephProtocol
from .jobs import reconnect_p2p_job, tidy_http_peers_job
assert key, "Host cannot be initialized without a key"
tasks: List[Coroutine]
priv = import_key(key)
private_key = RSAPrivateKey(priv)
public_key = private_key.get_public_key()
keypair = KeyPair(private_key, public_key)
transport_opt = f"/ip4/{host}/tcp/{port}"
host = await new_node(transport_opt=[transport_opt],
key_pair=keypair)
protocol = None
# gossip = gossipsub.GossipSub([GOSSIPSUB_PROTOCOL_ID], 10, 9, 11, 30)
# psub = Pubsub(host, gossip, host.get_id())
flood = floodsub.FloodSub([FLOODSUB_PROTOCOL_ID, GOSSIPSUB_PROTOCOL_ID])
psub = Pubsub(host, flood, host.get_id())
if protocol_active:
protocol = AlephProtocol(host)
tasks = [
reconnect_p2p_job(),
tidy_http_peers_job(),
]
if listen:
from aleph.web import app
await host.get_network().listen(multiaddr.Multiaddr(transport_opt))
LOGGER.info("Listening on " + f'{transport_opt}/p2p/{host.get_id()}')
ip = await get_IP()
public_address = f'/ip4/{ip}/tcp/{port}/p2p/{host.get_id()}'
http_port = app['config'].p2p.http_port.value
public_http_address = f'http://{ip}:{http_port}'
LOGGER.info("Probable public on " + public_address)
# TODO: set correct interests and args here
tasks += [
publish_host(public_address, psub, peer_type="P2P"),
publish_host(public_http_address, psub, peer_type="HTTP"),
monitor_hosts(psub),
]
# Enable message exchange using libp2p
# host.set_stream_handler(PROTOCOL_ID, stream_handler)
return (host, psub, protocol, tasks)
async def initialize_host(host='0.0.0.0',
port=4025,
key=None,
listen=True,
protocol_active=True):
from .peers import publish_host, monitor_hosts
from .protocol import PROTOCOL_ID, AlephProtocol
from .jobs import reconnect_p2p_job, tidy_http_peers_job
if key is None:
keypair = generate_keypair(print_info=listen)
else:
priv = import_key(key)
private_key = RSAPrivateKey(priv)
public_key = private_key.get_public_key()
keypair = KeyPair(private_key, public_key)
transport_opt = f"/ip4/{host}/tcp/{port}"
host = await new_node(transport_opt=[transport_opt], key_pair=keypair)
protocol = None
# gossip = gossipsub.GossipSub([GOSSIPSUB_PROTOCOL_ID], 10, 9, 11, 30)
# psub = Pubsub(host, gossip, host.get_id())
flood = floodsub.FloodSub([FLOODSUB_PROTOCOL_ID, GOSSIPSUB_PROTOCOL_ID])
psub = Pubsub(host, flood, host.get_id())
if protocol_active:
protocol = AlephProtocol(host)
asyncio.create_task(reconnect_p2p_job())
asyncio.create_task(tidy_http_peers_job())
if listen:
from aleph.web import app
await host.get_network().listen(multiaddr.Multiaddr(transport_opt))
LOGGER.info("Listening on " + f'{transport_opt}/p2p/{host.get_id()}')
ip = await get_IP()
public_address = f'/ip4/{ip}/tcp/{port}/p2p/{host.get_id()}'
http_port = app['config'].p2p.http_port.value
public_http_address = f'http://{ip}:{http_port}'
LOGGER.info("Probable public on " + public_address)
# TODO: set correct interests and args here
asyncio.create_task(publish_host(public_address, psub,
peer_type="P2P"))
asyncio.create_task(
publish_host(public_http_address, psub, peer_type="HTTP"))
asyncio.create_task(monitor_hosts(psub))
# host.set_stream_handler(PROTOCOL_ID, stream_handler)
return (host, psub, protocol)
async def connect_peer(config, peer):
info = info_from_p2p_addr(multiaddr.Multiaddr(peer))
if str(info.peer_id) == str(singleton.host.get_id()):
# LOGGER.debug("Can't connect to myself.")
return
if 'streamer' in config.p2p.clients.value:
if not await singleton.streamer.has_active_streams(info.peer_id):
# network = singleton.host.get_network()
# if info.peer_id in network.connections:
# await network.close_peer(info.peer_id)
# del network[info.peer_id]
await singleton.host.connect(info)
await singleton.streamer.create_connections(info.peer_id)
else:
await singleton.host.connect(info)
async def set_up_routed_hosts() -> Tuple[RoutedHost, RoutedHost]:
router_a, router_b = DummyRouter(), DummyRouter()
transport = "/ip4/127.0.0.1/tcp/0"
host_a = await new_node(transport_opt=[transport], disc_opt=router_a)
host_b = await new_node(transport_opt=[transport], disc_opt=router_b)
address = multiaddr.Multiaddr(transport)
await host_a.get_network().listen(address)
await host_b.get_network().listen(address)
mock_routing_table = {
host_a.get_id(): PeerInfo(host_a.get_id(), host_a.get_addrs()),
host_b.get_id(): PeerInfo(host_b.get_id(), host_b.get_addrs()),
}
router_a._routing_table = router_b._routing_table = mock_routing_table
return cast(RoutedHost, host_a), cast(RoutedHost, host_b)
async def create(cls):
"""
Create a new DummyAccountNode and attach a libp2p node, a floodsub, and a pubsub
instance to this new node
We use create as this serves as a factory function and allows us
to use async await, unlike the init function
"""
self = DummyAccountNode()
libp2p_node = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"])
await libp2p_node.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0"))
self.libp2p_node = libp2p_node
self.floodsub = FloodSub(SUPPORTED_PUBSUB_PROTOCOLS)
self.pubsub = Pubsub(self.libp2p_node, self.floodsub, "a")
return self
def convertAll(_node):
ret = {
"enode": "",
"enr": "",
"multiaddr": "",
"enrdata": {},
"enritems": {}
}
if (_node.startswith("/")):
ret["multiaddr"] = multiaddr.Multiaddr(_node)
elif (_node.startswith("enode://")):
ret["enode"] = _node
ret["multiaddr"] = Handler.enodeToMultiAddress(_node)
elif (_node.startswith("enr:")):
ret["enr"] = _node
ret.update(Handler.enrToMultiAddress(_node))
print(ret)
return ret
async def test_host_connect():
transport_opt_list = [["/ip4/127.0.0.1/tcp/0"], ["/ip4/127.0.0.1/tcp/0"]]
(node_a, node_b) = await set_up_nodes_by_transport_opt(transport_opt_list)
assert not node_a.get_peerstore().peer_ids()
addr = node_b.get_addrs()[0]
info = info_from_p2p_addr(addr)
await node_a.connect(info)
assert len(node_a.get_peerstore().peer_ids()) == 1
await node_a.connect(info)
# make sure we don't do double connection
assert len(node_a.get_peerstore().peer_ids()) == 1
assert node_b.get_id() in node_a.get_peerstore().peer_ids()
ma_node_b = multiaddr.Multiaddr("/p2p/%s" % node_b.get_id().pretty())
for addr in node_a.get_peerstore().addrs(node_b.get_id()):
assert addr.encapsulate(ma_node_b) in node_b.get_addrs()
async def create_libp2p_hosts(num_hosts):
"""
Create libp2p hosts
:param num_hosts: number of hosts to create
"""
hosts = []
tasks_create = []
for i in range(0, num_hosts):
# Create node
tasks_create.append(
asyncio.ensure_future(
new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"])))
hosts = await asyncio.gather(*tasks_create)
tasks_listen = []
for node in hosts:
# Start listener
tasks_listen.append(
asyncio.ensure_future(node.get_network().listen(
multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0"))))
await asyncio.gather(*tasks_listen)
return hosts
async def run(port, destination):
host = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/%s" % port])
if not destination: # its the server
async def stream_handler(stream):
asyncio.ensure_future(read_data(stream))
asyncio.ensure_future(write_data(stream))
host.set_stream_handler(PROTOCOL_ID, stream_handler)
port = None
for listener in host.network.listeners.values():
for addr in listener.get_addrs():
port = int(addr.value_for_protocol('tcp'))
if not port:
raise RuntimeError("was not able to find the actual local port")
print(
"Run './examples/chat/chat.py -p %s -d /ip4/127.0.0.1/tcp/%s/p2p/%s' on another console.\n"
% (int(port) + 1, port, host.get_id().pretty()))
print("You can replace 127.0.0.1 with public IP as well.")
print("\nWaiting for incoming connection\n\n")
else: # its the client
m = multiaddr.Multiaddr(destination)
info = info_from_p2p_addr(m)
# Associate the peer with local ip address
await host.connect(info)
# Start a stream with the destination.
# Multiaddress of the destination peer is fetched from the peerstore using 'peerId'.
stream = await host.new_stream(info.peer_id, [PROTOCOL_ID])
asyncio.ensure_future(read_data(stream))
asyncio.ensure_future(write_data(stream))
print("Connected to peer %s" % info.addrs[0])
async def test_host_connect():
node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"])
node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"])
assert not node_a.get_peerstore().peers()
addr = node_b.get_addrs()[0]
info = info_from_p2p_addr(addr)
await node_a.connect(info)
assert len(node_a.get_peerstore().peers()) == 1
await node_a.connect(info)
# make sure we don't do double connection
assert len(node_a.get_peerstore().peers()) == 1
assert node_b.get_id() in node_a.get_peerstore().peers()
ma_node_b = multiaddr.Multiaddr('/p2p/%s' % node_b.get_id().pretty())
for addr in node_a.get_peerstore().addrs(node_b.get_id()):
assert addr.encapsulate(ma_node_b) in node_b.get_addrs()
# Success, terminate pending tasks.
await cleanup()
import multiaddr
FLOODSUB_PROTOCOL_ID = "/floodsub/1.0.0"
GOSSIPSUB_PROTOCOL_ID = "/gossipsub/1.0.0"
LISTEN_MADDR = multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")
def test_init_():
random_addrs = [random.randint(0, 255) for r in range(4)]
random_id_string = ""
for _ in range(10):
random_id_string += random.SystemRandom().choice(ALPHABETS)
peer_id = ID(random_id_string.encode())
peer_info = PeerInfo(peer_id, random_addrs)
assert peer_info.peer_id == peer_id
assert peer_info.addrs == random_addrs
@pytest.mark.parametrize(
"addr",
(
pytest.param(multiaddr.Multiaddr("/"), id="empty multiaddr"),
pytest.param(
multiaddr.Multiaddr("/ip4/127.0.0.1"),
id="multiaddr without peer_id(p2p protocol)",
),
),
)
def test_info_from_p2p_addr_invalid(addr):
with pytest.raises(InvalidAddrError):
info_from_p2p_addr(addr)
def test_info_from_p2p_addr_valid():
m_addr = multiaddr.Multiaddr(VALID_MULTI_ADDR_STR)
info = info_from_p2p_addr(m_addr)
assert (
def multiaddr_to_url_data(
addr: addr_t,
base: str # type: ignore[no-any-unimported]
) -> ty.Tuple[str, ty.Optional[str], socket.AddressFamily, bool]:
try:
addr = multiaddr.Multiaddr(addr)
except multiaddr.exceptions.ParseError as error:
raise exceptions.AddressError(addr) from error
addr_iter = iter(addr.items())
# Parse the `host`, `family`, `port` & `secure` values from the given
# multiaddr, raising on unsupported `addr` values
try:
# Read host value
proto, host = next(addr_iter)
family = socket.AF_UNSPEC
host_numeric = proto.code in (P_IP4, P_IP6)
uds_path = None # type: ty.Optional[str]
if proto.code in (P_IP4, P_DNS4):
family = socket.AF_INET
elif proto.code in (P_IP6, P_DNS6):
family = socket.AF_INET6
elif proto.code == P_UNIX and AF_UNIX is not NotImplemented:
family = AF_UNIX
uds_path = host
elif proto.code != P_DNS:
raise exceptions.AddressError(addr)
if family == AF_UNIX:
assert uds_path is not None
netloc = urllib.parse.quote(uds_path, safe="")
else:
# Read port value for IP-based transports
proto, port = next(addr_iter)
if proto.code != P_TCP:
raise exceptions.AddressError(addr)
# Pre-format network location URL part based on host+port
if ":" in host and not host.startswith("["):
netloc = "[{0}]:{1}".format(host, port)
else:
netloc = "{0}:{1}".format(host, port)
# Read application-level protocol name
secure = False
try:
proto, value = next(addr_iter)
except StopIteration:
pass
else:
if proto.code == P_HTTPS:
secure = True
elif proto.code != P_HTTP:
raise exceptions.AddressError(addr)
# No further values may follow; this also exhausts the iterator
was_final = all(False for _ in addr_iter)
if not was_final:
raise exceptions.AddressError(addr)
except StopIteration:
raise exceptions.AddressError(addr) from None
if not base.endswith("/"):
base += "/"
# Convert the parsed `addr` values to a URL base and parameters for the
# HTTP library
base_url = urllib.parse.SplitResult(
scheme="http" if not secure else "https",
netloc=netloc,
path=base,
query="",
fragment="").geturl()
return base_url, uds_path, family, host_numeric
def peer_info_from_str(string: str) -> PeerInfo:
peer_id, raw_addrs = json.loads(string)
return PeerInfo(ID.from_base58(peer_id),
list(map(lambda a: multiaddr.Multiaddr(a), raw_addrs)))
async def test_one_notifier_on_two_nodes_with_listen():
events_b = []
messages = ["hello", "hello"]
node_a_key_pair = create_new_key_pair()
node_a_transport_opt = ["/ip4/127.0.0.1/tcp/0"]
node_a = await new_node(node_a_key_pair,
transport_opt=node_a_transport_opt)
await node_a.get_network().listen(
multiaddr.Multiaddr(node_a_transport_opt[0]))
# Set up node_b swarm to pass into host
node_b_key_pair = create_new_key_pair()
node_b_transport_opt = ["/ip4/127.0.0.1/tcp/0"]
node_b_multiaddr = multiaddr.Multiaddr(node_b_transport_opt[0])
node_b_swarm = initialize_default_swarm(node_b_key_pair,
transport_opt=node_b_transport_opt)
node_b = BasicHost(node_b_swarm)
async def my_stream_handler(stream):
# Ensure the listened, connected and opened_stream events were hit in Notifee obj
# and that the stream passed into opened_stream matches the stream created on
# node_b
assert events_b == [
["listenedb", node_b_multiaddr],
["connectedb", stream.mplex_conn],
["opened_streamb", stream],
]
for message in messages:
read_string = (await stream.read(len(message))).decode()
resp = ACK + read_string
await stream.write(resp.encode())
# Add notifee for node_a
events_a = []
assert node_a.get_network().notify(MyNotifee(events_a, "a"))
# Add notifee for node_b
assert node_b.get_network().notify(MyNotifee(events_b, "b"))
# start listen on node_b_swarm
await node_b.get_network().listen(node_b_multiaddr)
node_b.set_stream_handler("/echo/1.0.0", my_stream_handler)
# Associate the peer with local ip address (see default parameters of Libp2p())
node_a.get_peerstore().add_addrs(node_b.get_id(), node_b.get_addrs(), 10)
stream = await node_a.new_stream(node_b.get_id(), ["/echo/1.0.0"])
# Ensure the connected and opened_stream events were hit in MyNotifee obj
# and that stream passed into opened_stream matches the stream created on
# node_a
assert events_a == [["connecteda", stream.mplex_conn],
["opened_streama", stream]]
for message in messages:
expected_resp = ACK + message
await stream.write(message.encode())
response = (await stream.read(len(expected_resp))).decode()
assert response == expected_resp
# Success, terminate pending tasks.
await cleanup()
# -*- coding: utf-8 -*-
"""IPFS API Bindings for Python.
Classes:
* Client – a TCP client for interacting with an IPFS daemon
"""
from __future__ import absolute_import
import os
import warnings
import multiaddr
DEFAULT_ADDR = multiaddr.Multiaddr(
os.environ.get("PY_IPFS_HTTP_CLIENT_DEFAULT_ADDR",
'/dns/localhost/tcp/5001/http'))
DEFAULT_BASE = str(os.environ.get("PY_IPFS_HTTP_CLIENT_DEFAULT_BASE",
'api/v0'))
VERSION_MINIMUM = "0.4.3"
VERSION_BLACKLIST = ["0.4.20"]
VERSION_MAXIMUM = "0.5.0"
from . import bitswap
from . import block
from . import bootstrap
from . import config
#TODO: `from . import dag`
from . import dht
from . import files
def multiaddr_to_url_data(addr: addr_t, base: str) \
-> ty.Tuple[str, socket.AddressFamily, bool]:
try:
addr = multiaddr.Multiaddr(addr)
except multiaddr.exceptions.ParseError as error:
raise exceptions.AddressError(addr) from error
addr_iter = iter(addr.items())
# Parse the `host`, `family`, `port` & `secure` values from the given
# multiaddr, raising on unsupported `addr` values
try:
# Read host value
proto, host = next(addr_iter)
family = socket.AF_UNSPEC
host_numeric = proto.code in (P_IP4, P_IP6)
if proto.code in (P_IP4, P_DNS4):
family = socket.AF_INET
elif proto.code in (P_IP6, P_DNS6):
family = socket.AF_INET6
elif proto.code != P_DNS:
raise exceptions.AddressError(addr)
# Read port value
proto, port = next(addr_iter)
if proto.code != P_TCP:
raise exceptions.AddressError(addr)
# Read application-level protocol name
secure = False
try:
proto, value = next(addr_iter)
except StopIteration:
pass
else:
if proto.code == P_HTTPS:
secure = True
elif proto.code != P_HTTP:
raise exceptions.AddressError(addr)
# No further values may follow; this also exhausts the iterator
was_final = all(False for _ in addr_iter)
if not was_final:
raise exceptions.AddressError(addr)
except StopIteration:
raise exceptions.AddressError(addr) from None
if not base.endswith("/"):
base += "/"
# Convert the parsed `addr` values to a URL base and parameters for the
# HTTP library
if ":" in host and not host.startswith("["):
host = "[{0}]".format(host)
base_url = urllib.parse.SplitResult(
scheme="http" if not secure else "https",
netloc="{0}:{1}".format(host, port),
path=base,
query="",
fragment="").geturl()
return base_url, family, host_numeric
peer_info = PeerInfo(peer_id, peer_data)
assert peer_info.peer_id == peer_id
assert peer_info.addrs == random_addrs
def test_init_no_value():
with pytest.raises(Exception) as _:
# pylint: disable=no-value-for-parameter
PeerInfo()
@pytest.mark.parametrize('addr', (
pytest.param(None),
pytest.param(random.randint(0, 255), id='random integer'),
pytest.param(multiaddr.Multiaddr('/'), id='empty multiaddr'),
pytest.param(multiaddr.Multiaddr('/ip4/127.0.0.1'),
id='multiaddr without peer_id(p2p protocol)'),
))
def test_info_from_p2p_addr_invalid(addr):
with pytest.raises(InvalidAddrError):
info_from_p2p_addr(addr)
def test_info_from_p2p_addr_valid():
# pylint: disable=line-too-long
m_addr = multiaddr.Multiaddr(
'/ip4/127.0.0.1/tcp/8000/p2p/3YgLAeMKSAPcGqZkAt8mREqhQXmJT8SN8VCMN4T6ih4GNX9wvK8mWJnWZ1qA2mLdCQ'
)
info = info_from_p2p_addr(m_addr)
assert info.peer_id.pretty(
def _multiaddr_from_socket(socket):
return multiaddr.Multiaddr("/ip4/%s/tcp/%s" % socket.getsockname())
async def test_lru_cache_two_nodes():
# two nodes with cache_size of 4
# node_a send the following messages to node_b
# [1, 1, 2, 1, 3, 1, 4, 1, 5, 1]
# node_b should only receive the following
# [1, 2, 3, 4, 5, 1]
node_a = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"])
node_b = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"])
await node_a.get_network().listen(
multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0"))
await node_b.get_network().listen(
multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0"))
supported_protocols = ["/floodsub/1.0.0"]
# initialize PubSub with a cache_size of 4
floodsub_a = FloodSub(supported_protocols)
pubsub_a = Pubsub(node_a, floodsub_a, "a", 4)
floodsub_b = FloodSub(supported_protocols)
pubsub_b = Pubsub(node_b, floodsub_b, "b", 4)
await connect(node_a, node_b)
await asyncio.sleep(0.25)
qb = await pubsub_b.subscribe("my_topic")
await asyncio.sleep(0.25)
node_a_id = str(node_a.get_id())
# initialize message_id_generator
# store first message
next_msg_id_func = message_id_generator(0)
first_message = generate_RPC_packet(node_a_id, ["my_topic"], "some data 1",
next_msg_id_func())
await floodsub_a.publish(node_a_id, first_message.SerializeToString())
await asyncio.sleep(0.25)
print(first_message)
messages = [first_message]
# for the next 5 messages
for i in range(2, 6):
# write first message
await floodsub_a.publish(node_a_id, first_message.SerializeToString())
await asyncio.sleep(0.25)
# generate and write next message
msg = generate_RPC_packet(node_a_id, ["my_topic"],
"some data " + str(i), next_msg_id_func())
messages.append(msg)
await floodsub_a.publish(node_a_id, msg.SerializeToString())
await asyncio.sleep(0.25)
# write first message again
await floodsub_a.publish(node_a_id, first_message.SerializeToString())
await asyncio.sleep(0.25)
# check the first five messages in queue
# should only see 1 first_message
for i in range(5):
# Check that the msg received by node_b is the same
# as the message sent by node_a
res_b = await qb.get()
assert res_b.SerializeToString(
) == messages[i].publish[0].SerializeToString()
# the 6th message should be first_message
res_b = await qb.get()
assert res_b.SerializeToString(
) == first_message.publish[0].SerializeToString()
assert qb.empty()
# Success, terminate pending tasks.
await cleanup()
async def initialize_host(key,
host="0.0.0.0",
port=4025,
listen=True,
protocol_active=True
) -> Tuple[BasicHost, Pubsub, Any, List]:
from .protocol import AlephProtocol
from .jobs import reconnect_p2p_job, tidy_http_peers_job
assert key, "Host cannot be initialized without a key"
tasks: List[Coroutine]
priv = import_key(key)
private_key = RSAPrivateKey(priv)
public_key = private_key.get_public_key()
keypair = KeyPair(private_key, public_key)
transport_opt = f"/ip4/{host}/tcp/{port}"
host: BasicHost = await new_node(transport_opt=[transport_opt],
key_pair=keypair)
protocol = None
# gossip = gossipsub.GossipSub([GOSSIPSUB_PROTOCOL_ID], 10, 9, 11, 30)
# psub = Pubsub(host, gossip, host.get_id())
flood = floodsub.FloodSub([FLOODSUB_PROTOCOL_ID, GOSSIPSUB_PROTOCOL_ID])
psub = Pubsub(host, flood, host.get_id())
if protocol_active:
protocol = AlephProtocol(host)
tasks = [
reconnect_p2p_job(),
tidy_http_peers_job(),
]
if listen:
from aleph.web import app
await host.get_network().listen(multiaddr.Multiaddr(transport_opt))
LOGGER.info("Listening on " + f"{transport_opt}/p2p/{host.get_id()}")
ip = await get_IP()
public_address = f"/ip4/{ip}/tcp/{port}/p2p/{host.get_id()}"
http_port = app["config"].p2p.http_port.value
public_adresses.append(public_address)
public_http_address = f"http://{ip}:{http_port}"
LOGGER.info("Probable public on " + public_address)
# TODO: set correct interests and args here
tasks += [
publish_host(
public_address,
psub,
peer_type="P2P",
use_ipfs=app["config"].ipfs.enabled.value,
),
publish_host(
public_http_address,
psub,
peer_type="HTTP",
use_ipfs=app["config"].ipfs.enabled.value,
),
monitor_hosts_p2p(psub),
]
if app["config"].ipfs.enabled.value:
tasks.append(monitor_hosts_ipfs(app["config"]))
try:
public_ipfs_address = await get_public_address()
tasks.append(
publish_host(public_ipfs_address,
psub,
peer_type="IPFS",
use_ipfs=True))
except Exception:
LOGGER.exception("Can't publish public IPFS address")
# Enable message exchange using libp2p
# host.set_stream_handler(PROTOCOL_ID, stream_handler)
return (host, psub, protocol, tasks)
async def perform_test_from_obj(obj):
"""
Perform a floodsub test from a test obj.
test obj are composed as follows:
{
"supported_protocols": ["supported/protocol/1.0.0",...],
"adj_list": {
"node1": ["neighbor1_of_node1", "neighbor2_of_node1", ...],
"node2": ["neighbor1_of_node2", "neighbor2_of_node2", ...],
...
},
"topic_map": {
"topic1": ["node1_subscribed_to_topic1", "node2_subscribed_to_topic1", ...]
},
"messages": [
{
"topics": ["topic1_for_message", "topic2_for_message", ...],
"data": "some contents of the message (newlines are not supported)",
"node_id": "message sender node id"
},
...
]
}
NOTE: In adj_list, for any neighbors A and B, only list B as a neighbor of A
or B as a neighbor of A once. Do NOT list both A: ["B"] and B:["A"] as the behavior
is undefined (even if it may work)
"""
# Step 1) Create graph
adj_list = obj["adj_list"]
node_map = {}
floodsub_map = {}
pubsub_map = {}
supported_protocols = obj["supported_protocols"]
tasks_connect = []
for start_node_id in adj_list:
# Create node if node does not yet exist
if start_node_id not in node_map:
node = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"])
await node.get_network().listen(
multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0"))
node_map[start_node_id] = node
floodsub = FloodSub(supported_protocols)
floodsub_map[start_node_id] = floodsub
pubsub = Pubsub(node, floodsub, start_node_id)
pubsub_map[start_node_id] = pubsub
# For each neighbor of start_node, create if does not yet exist,
# then connect start_node to neighbor
for neighbor_id in adj_list[start_node_id]:
# Create neighbor if neighbor does not yet exist
if neighbor_id not in node_map:
neighbor_node = await new_node(
transport_opt=["/ip4/127.0.0.1/tcp/0"])
await neighbor_node.get_network().listen(
multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0"))
node_map[neighbor_id] = neighbor_node
floodsub = FloodSub(supported_protocols)
floodsub_map[neighbor_id] = floodsub
pubsub = Pubsub(neighbor_node, floodsub, neighbor_id)
pubsub_map[neighbor_id] = pubsub
# Connect node and neighbor
# await connect(node_map[start_node_id], node_map[neighbor_id])
tasks_connect.append(
asyncio.ensure_future(
connect(node_map[start_node_id], node_map[neighbor_id])))
tasks_connect.append(asyncio.sleep(2))
await asyncio.gather(*tasks_connect)
# Allow time for graph creation before continuing
# await asyncio.sleep(0.25)
# Step 2) Subscribe to topics
queues_map = {}
topic_map = obj["topic_map"]
tasks_topic = []
tasks_topic_data = []
for topic in topic_map:
for node_id in topic_map[topic]:
"""
# Subscribe node to topic
q = await pubsub_map[node_id].subscribe(topic)
# Create topic-queue map for node_id if one does not yet exist
if node_id not in queues_map:
queues_map[node_id] = {}
# Store queue in topic-queue map for node
queues_map[node_id][topic] = q
"""
tasks_topic.append(
asyncio.ensure_future(pubsub_map[node_id].subscribe(topic)))
tasks_topic_data.append((node_id, topic))
tasks_topic.append(asyncio.sleep(2))
# Gather is like Promise.all
responses = await asyncio.gather(*tasks_topic, return_exceptions=True)
for i in range(len(responses) - 1):
q = responses[i]
node_id, topic = tasks_topic_data[i]
if node_id not in queues_map:
queues_map[node_id] = {}
# Store queue in topic-queue map for node
queues_map[node_id][topic] = q
# Allow time for subscribing before continuing
# await asyncio.sleep(0.01)
# Step 3) Publish messages
topics_in_msgs_ordered = []
messages = obj["messages"]
tasks_publish = []
next_msg_id_func = message_id_generator(0)
for msg in messages:
topics = msg["topics"]
data = msg["data"]
node_id = msg["node_id"]
# Get actual id for sender node (not the id from the test obj)
actual_node_id = str(node_map[node_id].get_id())
# Create correctly formatted message
msg_talk = generate_RPC_packet(actual_node_id, topics, data,
next_msg_id_func())
# Publish message
# await floodsub_map[node_id].publish(actual_node_id, msg_talk.to_str())
tasks_publish.append(asyncio.ensure_future(floodsub_map[node_id].publish(\
actual_node_id, msg_talk.SerializeToString())))
# For each topic in topics, add topic, msg_talk tuple to ordered test list
# TODO: Update message sender to be correct message sender before
# adding msg_talk to this list
for topic in topics:
topics_in_msgs_ordered.append((topic, msg_talk))
# Allow time for publishing before continuing
# await asyncio.sleep(0.4)
tasks_publish.append(asyncio.sleep(2))
await asyncio.gather(*tasks_publish)
# Step 4) Check that all messages were received correctly.
# TODO: Check message sender too
for i in range(len(topics_in_msgs_ordered)):
topic, actual_msg = topics_in_msgs_ordered[i]
# Look at each node in each topic
for node_id in topic_map[topic]:
# Get message from subscription queue
msg_on_node_str = await queues_map[node_id][topic].get()
assert actual_msg.publish[0].SerializeToString(
) == msg_on_node_str.SerializeToString()
# Success, terminate pending tasks.
await cleanup()
