def __init__( self, port: int, api: Any, local_type: NodeType, ping_interval: int, network_id: str, root_path: Path, config: Dict, name: str = None, ): # Keeps track of all connections to and from this node. self.global_connections: PeerConnections = PeerConnections([]) self._port = port # TCP port to identify our node self._local_type = local_type # NodeType (farmer, full node, timelord, pool, harvester, wallet) self._ping_interval = ping_interval # (StreamReader, StreamWriter, NodeType) aiter, gets things from server and clients and # sends them through the pipeline self._srwt_aiter: push_aiter = push_aiter() # Aiter used to broadcase messages self._outbound_aiter: push_aiter = push_aiter() # Taks list to keep references to tasks, so they don'y get GCd self._tasks: List[asyncio.Task] = [] if local_type != NodeType.INTRODUCER: # Introducers should not keep connections alive, they should close them self._tasks.append(self._initialize_ping_task()) if name: self.log = logging.getLogger(name) else: self.log = logging.getLogger(__name__) # Our unique random node id that we will send to other peers, regenerated on launch node_id = create_node_id() if hasattr(api, "_set_global_connections"): api._set_global_connections(self.global_connections) # Tasks for entire server pipeline self._pipeline_task: asyncio.Future = asyncio.ensure_future( initialize_pipeline( self._srwt_aiter, api, self._port, self._outbound_aiter, self.global_connections, self._local_type, node_id, network_id, self.log, ) ) self.root_path = root_path self.config = config
def __init__(self, port: int, api: Any, local_type: NodeType): self._port = port # TCP port to identify our node self._api = api # API module that will be called from the requests self._local_type = local_type # NodeType (farmer, full node, timelord, pool, harvester, wallet) self._srwt_aiter = push_aiter() self._outbound_aiter = push_aiter() self._pipeline_task = self.initialize_pipeline(self._srwt_aiter, self._api, self._port) self._ping_task = self._initialize_ping_task() self._node_id = create_node_id()
def __init__( self, port: int, api: Any, local_type: NodeType, ping_interval: int, network_id: str, root_path: Path, config: Dict, name: str = None, ): # Keeps track of all connections to and from this node. self.global_connections: PeerConnections = PeerConnections([]) # Optional listening server. You can also use this class without starting one. self._server: Optional[asyncio.AbstractServer] = None # Called for inbound connections after successful handshake self._on_inbound_connect: OnConnectFunc = None self._port = port # TCP port to identify our node self._api = api # API module that will be called from the requests self._local_type = local_type # NodeType (farmer, full node, timelord, pool, harvester, wallet) self._ping_interval = ping_interval self._network_id = network_id # (StreamReader, StreamWriter, NodeType) aiter, gets things from server and clients and # sends them through the pipeline self._srwt_aiter: push_aiter = push_aiter() # Aiter used to broadcase messages self._outbound_aiter: push_aiter = push_aiter() # Tasks for entire server pipeline self._pipeline_task: asyncio.Task = self.initialize_pipeline( self._srwt_aiter, self._api, self._port) # Our unique random node id that we will other peers, regenerated on launch self._node_id = create_node_id() # Taks list to keep references to tasks, so they don'y get GCd self._tasks: List[asyncio.Task] = [self._initialize_ping_task()] if name: self.log = logging.getLogger(name) else: self.log = logging.getLogger(__name__) self.root_path = root_path self.config = config
def test_make_delayed_pipeline(self): def make_wait_index(idx): async def wait(item): await asyncio.sleep(item[idx] / 10.) return item return wait TEST_CASE = [ (0, 0, 0, 7), (5, 0, 0, 0), (0, 0, 1, 0), (1, 1, 1, 1), (2, 0, 0, 1), (3, 1, 2, 0), ] q = push_aiter() aiter = map_aiter( make_wait_index(0), map_aiter( make_wait_index(1), map_aiter(make_wait_index(2), map_aiter(make_wait_index(3), q, 10), 10), 10), 10) q.push(*TEST_CASE) q.stop() r = run(get_n(aiter)) r1 = sorted(r, key=lambda x: sum(x)) self.assertEqual(r, r1)
async def start_server_aiter(port): aiter = push_aiter() server = await asyncio.start_server( client_connected_cb=lambda r, w: aiter.push((r, w)), port=port) aiter.task = asyncio.ensure_future( server.wait_closed()).add_done_callback(lambda f: aiter.stop()) return server, aiter
def test_filter_pipeline(self): async def filter(item_list_of_lists): r = [] for l1 in item_list_of_lists: for item in l1: if item != 0: r.append(item) return r TEST_CASE = [ (0, 0, 0, 7), (5, 0, 0, 0), (0, 0, 1, 0), (1, 1, 1, 1), (2, 0, 0, 1), (3, 1, 2, 0), ] q = push_aiter() aiter = flatten_aiter(map_aiter(filter, q)) q.push(TEST_CASE) q.stop() r = run(get_n(aiter, 12)) r1 = [7, 5, 1, 1, 1, 1, 1, 2, 1, 3, 1, 2] self.assertEqual(r, r1)
def test_join_aiters_1(self): # make sure nothing's dropped # even if lots of events come in at once main_aiter = push_aiter() child_aiters = [] aiter = join_aiters(main_aiter) child_aiters.append(push_aiter()) child_aiters[0].push(100) main_aiter.push(child_aiters[0]) t = run(get_n(aiter, 1)) self.assertEqual(t, [100]) child_aiters.append(push_aiter()) child_aiters[0].push(101) child_aiters[1].push(200) child_aiters[1].push(201) main_aiter.push(child_aiters[1]) t = run(get_n(aiter, 3)) self.assertEqual(set(t), set([101, 200, 201])) for _ in range(3): child_aiters.append(push_aiter()) main_aiter.push(child_aiters[-1]) for _, ca in enumerate(child_aiters): ca.push((_ + 1) * 100) ca.push((_ + 1) * 100 + 1) t = run(get_n(aiter, len(child_aiters) * 2)) self.assertEqual( set(t), set([100, 101, 200, 201, 300, 301, 400, 401, 500, 501])) child_aiters[-1].push(5000) main_aiter.stop() t = run(get_n(aiter, 1)) self.assertEqual(t, [5000]) for ca in child_aiters: ca.push(99) ca.stop() t = run(get_n(aiter)) self.assertEqual(t, [99] * len(child_aiters))
def test_flatten_aiter(self): q = push_aiter() fi = flatten_aiter(q) r = [] q.push([0, 1, 2, 3]) r.extend(run(get_n(fi, 3))) q.push([4, 5, 6, 7]) r.extend(run(get_n(fi, 5))) q.stop() r.extend(run(get_n(fi))) self.assertEqual(r, list(range(8)))
def test_push_aiter(self): q = push_aiter() self.assertEqual(len(q), 0) q.push(5, 4) self.assertEqual(len(q), 2) q.push(3) self.assertEqual(len(q), 3) q.stop() self.assertRaises(ValueError, lambda: q.push(2)) results = list(q.available_iter()) self.assertEqual(results, [5, 4, 3]) results = run(get_n(q)) self.assertEqual(results, [5, 4, 3])
def test_aiter_forker(self): q = push_aiter() forker = aiter_forker(q) q.push(1, 2, 3, 4, 5) r0 = run(get_n(forker, 3)) f2 = forker.fork() q.push(*range(7, 14)) q.stop() r1 = run(get_n(forker)) r2 = run(get_n(f2)) self.assertEqual(r0, [1, 2, 3]) self.assertEqual(r1, [4, 5, 7, 8, 9, 10, 11, 12, 13]) self.assertEqual(r2, [4, 5, 7, 8, 9, 10, 11, 12, 13])
def test_make_pipe(self): async def map_f(x): await asyncio.sleep(x / 100.0) return x * x q = push_aiter() aiter = map_aiter(map_f, q) for _ in range(4): q.push(_) for _ in range(3, 9): q.push(_) r = run(get_n(aiter, 10)) q.stop() r.extend(run(get_n(aiter))) r1 = sorted([_*_ for _ in range(4)] + [_ * _ for _ in range(3, 9)]) self.assertEqual(r, r1)
def test_make_simple_pipeline(self): q = push_aiter() aiter = flatten_aiter(flatten_aiter(q)) q.push([ (0, 0, 1, 0), (1, 1, 1, 1), (2, 0, 0, 1), (3, 1, 2, 0), (0, 0, 0, 7), ]) r = run(get_n(aiter, 11)) self.assertEqual(r, [0, 0, 1, 0, 1, 1, 1, 1, 2, 0, 0]) r.extend(run(get_n(aiter, 8))) q.stop() r.extend(run(get_n(aiter))) self.assertEqual( r, [0, 0, 1, 0, 1, 1, 1, 1, 2, 0, 0, 1, 3, 1, 2, 0, 0, 0, 0, 7])
def test_syncmap(self): def make_sync_transformation_f(results): def sync_transformation_f(item): results.append(item) return item + 1 return sync_transformation_f results = [] q = push_aiter() q.push(5, 4, 3) q.stop() r = list(q.available_iter()) self.assertEqual(r, [5, 4, 3]) aiter = map_aiter(make_sync_transformation_f(results), q) r = run(get_n(aiter)) self.assertEqual(r, [6, 5, 4]) self.assertEqual(results, [5, 4, 3])
def test_preload_aiter(self): q = push_aiter() q.push(*list(range(1000))) q.stop() self.assertEqual(len(q), 1000) aiter = preload_aiter(50, q) self.assertEqual(len(q), 1000) r = run(get_n(aiter, 1)) self.assertEqual(len(q), 949) self.assertEqual(r, [0]) r = run(get_n(aiter, 10)) self.assertEqual(r, list(range(1, 11))) self.assertEqual(len(q), 939) r = run(get_n(aiter)) self.assertEqual(r, list(range(11, 1000))) self.assertEqual(len(q), 0)
def test_aiter_forker_multiple_active(self): """ Multiple forks of an aiter_forker both asking for empty q information at the same time. Make sure the second one doesn't block. """ q = push_aiter() forker = aiter_forker(q) fork_1 = forker.fork(is_active=True) fork_2 = forker.fork(is_active=True) f1 = asyncio.ensure_future(get_n(fork_1, 1)) f2 = asyncio.ensure_future(get_n(fork_2, 1)) run(asyncio.wait([f1, f2], timeout=0.1)) self.assertFalse(f1.done()) self.assertFalse(f2.done()) q.push(1) run(asyncio.wait([f1, f2], timeout=0.1)) self.assertTrue(f1.done()) self.assertTrue(f2.done()) r1 = run(f1) r2 = run(f2) self.assertEqual(r1, [1]) self.assertEqual(r2, [1])