async def test_random_pruning(ignore_duplicates, recomplete_idx, batch_size, task_series, prune_depth): ti = OrderedTaskPreparation( NoPrerequisites, identity, lambda x: x - 1, accept_dangling_tasks=True, max_depth=prune_depth, ) ti.set_finished_dependency(task_series[0]) for idx, task_batch in enumerate(partition_all(batch_size, task_series)): if ignore_duplicates: registerable_tasks = task_batch else: registerable_tasks = set(task_batch) if idx == recomplete_idx: task_to_mark_finished = task_batch[0] - 1 if task_to_mark_finished not in ti._tasks: ti.set_finished_dependency(task_to_mark_finished) try: ti.register_tasks(registerable_tasks, ignore_duplicates=ignore_duplicates) except DuplicateTasks: if ignore_duplicates: raise else: continue if ti.has_ready_tasks(): await wait(ti.ready_tasks())
async def _process_transactions(self) -> None: while self.is_operational: buffer: List[SignedTransactionAPI] = [] # wait for there to be items available on the queue. buffer.extend(await self._internal_queue.get()) # continue to pull items from the queue synchronously until the # queue is either empty or we hit a sufficient size to justify # sending to our peers. while not self._internal_queue.empty(): if len(buffer) > BATCH_LOW_WATER: break buffer.extend(self._internal_queue.get_nowait()) # Now that the queue is either empty or we have an adequate number # to send to our peers, broadcast them to the appropriate peers. for batch in partition_all(BATCH_HIGH_WATER, buffer): for receiving_peer in await self._peer_pool.get_peers(): filtered_tx = self._filter_tx_for_peer( receiving_peer, batch) if len(filtered_tx) == 0: continue self.logger.debug2( 'Relaying %d transactions to %s', len(filtered_tx), receiving_peer, ) receiving_peer.sub_proto.send_transactions(filtered_tx) self._add_txs_to_bloom(receiving_peer.session, filtered_tx) # release to the event loop since this loop processes a # lot of data queue up a lot of outbound messages. await asyncio.sleep(0)
async def send_locations(self, node: Node, *, request_id: int, locations: Collection[Node]) -> int: if node.node_id == self.local_node_id: raise ValueError("Cannot send to self") batches = tuple(partition_all(NODES_PER_PAYLOAD, locations)) self.logger.debug("Sending Locations with %d nodes to %s", len(locations), node) if batches: total_batches = len(batches) for batch in batches: payload = tuple( node.to_payload() for node in batch ) response = Message( Locations(request_id, total_batches, payload), node, ) await self.message_dispatcher.send_message(response) await self.events.sent_locations.trigger(response) return total_batches else: response = Message( Locations(request_id, 1, ()), node, ) await self.message_dispatcher.send_message(response) await self.events.sent_locations.trigger(response) return 1
async def get_linked_validators(event_loop, event_bus) -> Tuple[Validator, Validator]: all_indices = tuple(index for index in range(len(keymap))) global_peer_count = 2 alice_indices, bob_indices = partition_all( len(all_indices) // global_peer_count, all_indices) alice = await get_validator(event_loop, event_bus, alice_indices) bob = await get_validator(event_loop, event_bus, bob_indices) return alice, bob
async def _explore( self, node_id: NodeID, max_distance: int, ) -> None: """ Explore the neighborhood around the given `node_id` out to the specified `max_distance` """ async with trio.open_nursery() as nursery: for distances in partition_all(2, range(max_distance, 0, -1)): try: found_enrs = await self._network.find_nodes( node_id, *distances) except trio.TooSlowError: self.unresponsive.add(node_id) return except MissingEndpointFields: self.unreachable.add(node_id) return except ValidationError: self.invalid.add(node_id) return else: # once we encounter a pair of buckets that elicits an empty # response we assume that all subsequent buckets will also # be empty. if not found_enrs: self.logger.debug( "explore-finish: node_id=%s covered=%d-%d", node_id.hex(), max_distance, distances[0], ) break for enr in found_enrs: try: self._network.enr_db.set_enr(enr) except OldSequenceNumber: pass # check if we have found any new records. If so, queue them and # wake up the new workers. This is guarded by the `condition` # object to ensure we maintain a consistent view of the `seen` # nodes. async with self._condition: new_enrs = tuple(enr for enr in reduce_enrs(found_enrs) if enr.node_id not in self.seen) if new_enrs: self.seen.update(enr.node_id for enr in new_enrs) self._condition.notify_all() # use the `NetworkProtocol.bond` to perform a liveliness check for enr in new_enrs: nursery.start_soon(self._bond_then_send, enr)
def decode_from_bin(input_bin): """ 0100000101010111010000110100100101001001 -> ASCII """ for chunk in partition_all(8, input_bin): yield sum( 2**exp * bit for exp, bit in enumerate(reversed(chunk)) )
async def get_linked_validators(event_loop, event_bus, monkeypatch) -> Tuple[Validator, Validator]: keymap = mk_keymap_of_size(NUM_VALIDATORS) all_indices = tuple(index for index in range(len(keymap))) global_peer_count = 2 alice_indices, bob_indices = partition_all( len(all_indices) // global_peer_count, all_indices) alice = await get_validator(event_loop, event_bus, monkeypatch, alice_indices) bob = await get_validator(event_loop, event_bus, monkeypatch, bob_indices) return alice, bob
async def announce(self, key: bytes, who: Node) -> None: self.logger.debug("Starting announce for: %s", encode_hex(key)) content_id = content_key_to_node_id(key) found_nodes = await self.iterative_lookup(content_id) async def do_advertise(node: Node) -> None: with trio.move_on_after(ADVERTISE_TIMEOUT): await self.client.advertise(node, key=key, who=who) for batch in partition_all(KADEMLIA_ANNOUNCE_CONCURRENCY, found_nodes): async with trio.open_nursery() as nursery: for node in batch: nursery.start_soon(do_advertise, node) self.logger.debug( "Finished announce to %d peers for: %s", len(found_nodes), encode_hex(key), )