def __init__(
self,
port: int,
api: Any,
local_type: NodeType,
ping_interval: int,
network_id: str,
root_path: Path,
config: Dict,
name: str = None,
):
# Keeps track of all connections to and from this node.
self.global_connections: PeerConnections = PeerConnections([])
self._port = port # TCP port to identify our node
self._local_type = local_type # NodeType (farmer, full node, timelord, pool, harvester, wallet)
self._ping_interval = ping_interval
# (StreamReader, StreamWriter, NodeType) aiter, gets things from server and clients and
# sends them through the pipeline
self._srwt_aiter: push_aiter = push_aiter()
# Aiter used to broadcase messages
self._outbound_aiter: push_aiter = push_aiter()
# Taks list to keep references to tasks, so they don'y get GCd
self._tasks: List[asyncio.Task] = []
if local_type != NodeType.INTRODUCER:
# Introducers should not keep connections alive, they should close them
self._tasks.append(self._initialize_ping_task())
if name:
self.log = logging.getLogger(name)
else:
self.log = logging.getLogger(__name__)
# Our unique random node id that we will send to other peers, regenerated on launch
node_id = create_node_id()
if hasattr(api, "_set_global_connections"):
api._set_global_connections(self.global_connections)
# Tasks for entire server pipeline
self._pipeline_task: asyncio.Future = asyncio.ensure_future(
initialize_pipeline(
self._srwt_aiter,
api,
self._port,
self._outbound_aiter,
self.global_connections,
self._local_type,
node_id,
network_id,
self.log,
)
)
self.root_path = root_path
self.config = config
def __init__(self, port: int, api: Any, local_type: NodeType):
self._port = port # TCP port to identify our node
self._api = api # API module that will be called from the requests
self._local_type = local_type # NodeType (farmer, full node, timelord, pool, harvester, wallet)
self._srwt_aiter = push_aiter()
self._outbound_aiter = push_aiter()
self._pipeline_task = self.initialize_pipeline(self._srwt_aiter,
self._api, self._port)
self._ping_task = self._initialize_ping_task()
self._node_id = create_node_id()
def __init__(
self,
port: int,
api: Any,
local_type: NodeType,
ping_interval: int,
network_id: str,
root_path: Path,
config: Dict,
name: str = None,
):
# Keeps track of all connections to and from this node.
self.global_connections: PeerConnections = PeerConnections([])
# Optional listening server. You can also use this class without starting one.
self._server: Optional[asyncio.AbstractServer] = None
# Called for inbound connections after successful handshake
self._on_inbound_connect: OnConnectFunc = None
self._port = port # TCP port to identify our node
self._api = api # API module that will be called from the requests
self._local_type = local_type # NodeType (farmer, full node, timelord, pool, harvester, wallet)
self._ping_interval = ping_interval
self._network_id = network_id
# (StreamReader, StreamWriter, NodeType) aiter, gets things from server and clients and
# sends them through the pipeline
self._srwt_aiter: push_aiter = push_aiter()
# Aiter used to broadcase messages
self._outbound_aiter: push_aiter = push_aiter()
# Tasks for entire server pipeline
self._pipeline_task: asyncio.Task = self.initialize_pipeline(
self._srwt_aiter, self._api, self._port)
# Our unique random node id that we will other peers, regenerated on launch
self._node_id = create_node_id()
# Taks list to keep references to tasks, so they don'y get GCd
self._tasks: List[asyncio.Task] = [self._initialize_ping_task()]
if name:
self.log = logging.getLogger(name)
else:
self.log = logging.getLogger(__name__)
self.root_path = root_path
self.config = config
def test_make_delayed_pipeline(self):
def make_wait_index(idx):
async def wait(item):
await asyncio.sleep(item[idx] / 10.)
return item
return wait
TEST_CASE = [
(0, 0, 0, 7),
(5, 0, 0, 0),
(0, 0, 1, 0),
(1, 1, 1, 1),
(2, 0, 0, 1),
(3, 1, 2, 0),
]
q = push_aiter()
aiter = map_aiter(
make_wait_index(0),
map_aiter(
make_wait_index(1),
map_aiter(make_wait_index(2),
map_aiter(make_wait_index(3), q, 10), 10), 10), 10)
q.push(*TEST_CASE)
q.stop()
r = run(get_n(aiter))
r1 = sorted(r, key=lambda x: sum(x))
self.assertEqual(r, r1)
async def start_server_aiter(port):
aiter = push_aiter()
server = await asyncio.start_server(
client_connected_cb=lambda r, w: aiter.push((r, w)), port=port)
aiter.task = asyncio.ensure_future(
server.wait_closed()).add_done_callback(lambda f: aiter.stop())
return server, aiter
def test_filter_pipeline(self):
async def filter(item_list_of_lists):
r = []
for l1 in item_list_of_lists:
for item in l1:
if item != 0:
r.append(item)
return r
TEST_CASE = [
(0, 0, 0, 7),
(5, 0, 0, 0),
(0, 0, 1, 0),
(1, 1, 1, 1),
(2, 0, 0, 1),
(3, 1, 2, 0),
]
q = push_aiter()
aiter = flatten_aiter(map_aiter(filter, q))
q.push(TEST_CASE)
q.stop()
r = run(get_n(aiter, 12))
r1 = [7, 5, 1, 1, 1, 1, 1, 2, 1, 3, 1, 2]
self.assertEqual(r, r1)
def test_join_aiters_1(self):
# make sure nothing's dropped
# even if lots of events come in at once
main_aiter = push_aiter()
child_aiters = []
aiter = join_aiters(main_aiter)
child_aiters.append(push_aiter())
child_aiters[0].push(100)
main_aiter.push(child_aiters[0])
t = run(get_n(aiter, 1))
self.assertEqual(t, [100])
child_aiters.append(push_aiter())
child_aiters[0].push(101)
child_aiters[1].push(200)
child_aiters[1].push(201)
main_aiter.push(child_aiters[1])
t = run(get_n(aiter, 3))
self.assertEqual(set(t), set([101, 200, 201]))
for _ in range(3):
child_aiters.append(push_aiter())
main_aiter.push(child_aiters[-1])
for _, ca in enumerate(child_aiters):
ca.push((_ + 1) * 100)
ca.push((_ + 1) * 100 + 1)
t = run(get_n(aiter, len(child_aiters) * 2))
self.assertEqual(
set(t), set([100, 101, 200, 201, 300, 301, 400, 401, 500, 501]))
child_aiters[-1].push(5000)
main_aiter.stop()
t = run(get_n(aiter, 1))
self.assertEqual(t, [5000])
for ca in child_aiters:
ca.push(99)
ca.stop()
t = run(get_n(aiter))
self.assertEqual(t, [99] * len(child_aiters))
def test_flatten_aiter(self):
q = push_aiter()
fi = flatten_aiter(q)
r = []
q.push([0, 1, 2, 3])
r.extend(run(get_n(fi, 3)))
q.push([4, 5, 6, 7])
r.extend(run(get_n(fi, 5)))
q.stop()
r.extend(run(get_n(fi)))
self.assertEqual(r, list(range(8)))
def test_push_aiter(self):
q = push_aiter()
self.assertEqual(len(q), 0)
q.push(5, 4)
self.assertEqual(len(q), 2)
q.push(3)
self.assertEqual(len(q), 3)
q.stop()
self.assertRaises(ValueError, lambda: q.push(2))
results = list(q.available_iter())
self.assertEqual(results, [5, 4, 3])
results = run(get_n(q))
self.assertEqual(results, [5, 4, 3])
def test_aiter_forker(self):
q = push_aiter()
forker = aiter_forker(q)
q.push(1, 2, 3, 4, 5)
r0 = run(get_n(forker, 3))
f2 = forker.fork()
q.push(*range(7, 14))
q.stop()
r1 = run(get_n(forker))
r2 = run(get_n(f2))
self.assertEqual(r0, [1, 2, 3])
self.assertEqual(r1, [4, 5, 7, 8, 9, 10, 11, 12, 13])
self.assertEqual(r2, [4, 5, 7, 8, 9, 10, 11, 12, 13])
def test_make_pipe(self):
async def map_f(x):
await asyncio.sleep(x / 100.0)
return x * x
q = push_aiter()
aiter = map_aiter(map_f, q)
for _ in range(4):
q.push(_)
for _ in range(3, 9):
q.push(_)
r = run(get_n(aiter, 10))
q.stop()
r.extend(run(get_n(aiter)))
r1 = sorted([_*_ for _ in range(4)] + [_ * _ for _ in range(3, 9)])
self.assertEqual(r, r1)
def test_make_simple_pipeline(self):
q = push_aiter()
aiter = flatten_aiter(flatten_aiter(q))
q.push([
(0, 0, 1, 0),
(1, 1, 1, 1),
(2, 0, 0, 1),
(3, 1, 2, 0),
(0, 0, 0, 7),
])
r = run(get_n(aiter, 11))
self.assertEqual(r, [0, 0, 1, 0, 1, 1, 1, 1, 2, 0, 0])
r.extend(run(get_n(aiter, 8)))
q.stop()
r.extend(run(get_n(aiter)))
self.assertEqual(
r, [0, 0, 1, 0, 1, 1, 1, 1, 2, 0, 0, 1, 3, 1, 2, 0, 0, 0, 0, 7])
def test_syncmap(self):
def make_sync_transformation_f(results):
def sync_transformation_f(item):
results.append(item)
return item + 1
return sync_transformation_f
results = []
q = push_aiter()
q.push(5, 4, 3)
q.stop()
r = list(q.available_iter())
self.assertEqual(r, [5, 4, 3])
aiter = map_aiter(make_sync_transformation_f(results), q)
r = run(get_n(aiter))
self.assertEqual(r, [6, 5, 4])
self.assertEqual(results, [5, 4, 3])
def test_preload_aiter(self):
q = push_aiter()
q.push(*list(range(1000)))
q.stop()
self.assertEqual(len(q), 1000)
aiter = preload_aiter(50, q)
self.assertEqual(len(q), 1000)
r = run(get_n(aiter, 1))
self.assertEqual(len(q), 949)
self.assertEqual(r, [0])
r = run(get_n(aiter, 10))
self.assertEqual(r, list(range(1, 11)))
self.assertEqual(len(q), 939)
r = run(get_n(aiter))
self.assertEqual(r, list(range(11, 1000)))
self.assertEqual(len(q), 0)
def test_aiter_forker_multiple_active(self):
"""
Multiple forks of an aiter_forker both asking for empty q information
at the same time. Make sure the second one doesn't block.
"""
q = push_aiter()
forker = aiter_forker(q)
fork_1 = forker.fork(is_active=True)
fork_2 = forker.fork(is_active=True)
f1 = asyncio.ensure_future(get_n(fork_1, 1))
f2 = asyncio.ensure_future(get_n(fork_2, 1))
run(asyncio.wait([f1, f2], timeout=0.1))
self.assertFalse(f1.done())
self.assertFalse(f2.done())
q.push(1)
run(asyncio.wait([f1, f2], timeout=0.1))
self.assertTrue(f1.done())
self.assertTrue(f2.done())
r1 = run(f1)
r2 = run(f2)
self.assertEqual(r1, [1])
self.assertEqual(r2, [1])
