async def manage_async_to_thread_tasks(func: Any, items: List[Dict[str, Any]], concurrency: int) -> Dict[str, Any]: """Manages a grouping of async.to_thread tasks, keeping number of active tasks at the concurrency level by starting new tasks whenver one completes. Only use with python3.9+. Args: func (Any): Function to run in threads items (List[Dict[str, Any]]): List of dict with kwargs for each task concurrency (int): max concurrency Returns: Dict[str, Any]: Dict with generi task name as the key, task result as the value """ results = {} arr = [create_task(to_thread(func, **i)) for i in items[:concurrency]] count = len(arr) num_of_items = len(items) while len(results) < num_of_items: await sleep(0.001) for index, task in enumerate(arr): if task.done(): results[task.get_name()] = await task if count < num_of_items: arr[index] = create_task(to_thread(func, **items[count])) count += 1 return results
async def main(): """ This is of course a bad idea. The functions do run, synchronously, because they are evaluated before `gather` is called. That function, however, expects awaitables - instead, it receives strings. """ try: await asyncio.gather( non_awaitable_io_bound_function(1, 2), non_awaitable_io_bound_function(2, 4), ) except TypeError as e: logging.error("Caught a TypeError: %s", e) """ Luckily, we can cheat. Asyncio can run run a function in a thread, and await not the function, but the thread as a whole. In Python, only one thread can run at a time. So, this mechanism cannot be used to speed up calculations. But to complete an IO bound task, a thread does not need to run all the time: it only needs to check if the task (sleeping, in this case) is done every now and then. """ logging.info("\n\nEXAMPLE 2 - TO_THREAD\n") await asyncio.gather( asyncio.to_thread(non_awaitable_io_bound_function, 3, 1), asyncio.to_thread(non_awaitable_io_bound_function, 4, 2), asyncio.to_thread(non_awaitable_io_bound_function, 5, 3), asyncio.to_thread(non_awaitable_io_bound_function, 6, 4), )
def check_if_obj_close_to_side(self): """called by update_ball to check whether the ball is close to the left/right of the screen. """ b = self.ball print(b.x + b.close_to_side) print(self.WIDTH - bx) if b.x + b.close_to_side < 200: await asyncio.gather(asyncio.to_thread(almost_hit_left())) elif self.WIDTH - b.x < 200: await asyncio.gather(asyncio.to_thread(almost_hit_right()))
async def main(): event = asyncio.Event() asyncio.create_task(asyncio.to_thread(blocking_func, event)) await asyncio.sleep(5) # now lets stop event.set()
async def run_cancelable_thread(thread_main: Callable[[], T], on_cancel: Callable[[], None]) -> Awaitable[T]: """ Runs code in a separate thread, while in an async context, like `asyncio.to_thread`, but supporting task cancellation. Args: thread_main: The code to execute in a different thread on_cancel: A callback that will be called when the task is cancelled. BEWARE! This will be called in a different thread to that of `thread_main`, so it should only work with thread-safe objects so as to signal the thread to stop, e.g. by putting some element in a thread-safe queue. Returns: The result of the code in `thread_main` """ thread_task = None try: thread_task = asyncio.create_task(asyncio.to_thread(thread_main)) return await asyncio.shield(thread_task) except asyncio.CancelledError: on_cancel() if thread_task is not None: await thread_task raise
async def check_thread(): print(f"started main at {time.strftime('%X')}") await asyncio.gather(asyncio.to_thread(blocking_io), asyncio.sleep(1), hello_world3()) print(f"finished main at {time.strftime('%X')}")
async def wrap(*args, **kwargs): # return coroutine to not block current actor if asyncio.iscoroutinefunction(func): return func(*args, **kwargs) else: # for sync call, running in thread return asyncio.to_thread(func, *args, **kwargs)
async def spracuj(self, ctx, *, message): #TODO: rename trigger command reading = False result = [] for line in message.split('\n'): if reading: if line.startswith('```'): reading = False break else: result.append(line) else: if line.startswith('```'): reading = True code = '\n'.join(result) try: codeResult = await asyncio.wait_for(asyncio.gather( asyncio.to_thread(executeRun, code)), timeout=1.0) codeResult = ''.join(codeResult) if (not codeResult): await ctx.send("Program prebehol úspešne.") elif (len(codeResult.split('\n')) > 10): await ctx.send( "Výsledok je moc dlhý. Bol by to spam keyže to tu postnem." ) elif (len(codeResult) > 500): await ctx.send( "Output je obmedzený na 500 znakov.\n Toto je už moc aj na mňa." ) else: await ctx.send((codeResult)) except asyncio.TimeoutError: await ctx.send('Čas na spracovanie vypršal.')
async def start(self): shards_per_cluster = int(config["Clustering"]["shards_per_cluster"]) shard_count = int(config["Clustering"].get("shard_count") or await self.fetch_num_shards()) full_clusters, last_cluster_shards = divmod(shard_count, shards_per_cluster) cluster_count = full_clusters + int(bool(last_cluster_shards)) logger.info( f"Launching {cluster_count} clusters to handle {shard_count} shards with {shards_per_cluster} per cluster." ) all_shards = as_chunks(range(shard_count), shards_per_cluster) for cluster_id, shards in enumerate(all_shards): cluster_watcher_func = partial(self.cluster_watcher, (cluster_id, shard_count, shards)) self.monitors[cluster_id] = asyncio.create_task( asyncio.to_thread(cluster_watcher_func)) async def keep_alive(): async with websockets.serve(self.websocket_handler, self.websocket_host, self.websocket_port): await asyncio.gather(*self.monitors.values()) self.keep_clusters_alive.start() self.keep_alive = self.loop.create_task(keep_alive())
async def test_thread(obj, nsubscribers, pre_nitems, post_nitems): """test if the issue is resovled https://github.com/simonsobs/nextline/issues/2 """ nitems = pre_nitems + post_nitems if nsubscribers >= 50 and nitems >= 100: pytest.skip("nsubscribers >= 50 and nitems >= 100") async def subscribe(obj, event): await event.wait() items = [] async for i in obj.subscribe(): items.append(i) return items def send(obj, pre_items, event, post_items, event_end): for i in pre_items: obj.put(i) event.set() for i in post_items: obj.put(i) event_end.set() async def close(obj, event_end): await event_end.wait() await obj.close() items = list(range(nitems)) pre_items = items[:pre_nitems] post_items = items[pre_nitems:] event = ThreadSafeAsyncioEvent() event_end = ThreadSafeAsyncioEvent() coro = asyncio.to_thread(send, obj, pre_items, event, post_items, event_end) task_send = asyncio.create_task(coro) tasks_subscribe = [] for i in range(nsubscribers): coro = subscribe(obj, event) task = asyncio.create_task(coro) tasks_subscribe.append(task) coro = close(obj, event_end) task_close = asyncio.create_task(coro) results = await asyncio.gather(*tasks_subscribe, task_send, task_close) for actual in results[:nsubscribers]: if not actual: # can be empty continue # print(actual[0]) expected = items[items.index(actual[0]):] # no missing or duplicate # items from the first # received item assert actual == expected
async def main(): print(f"started main at {time.strftime('%X')}") await asyncio.gather( asyncio.to_thread(blocking_io), asyncio.sleep(1)) print(f"finished main at {time.strftime('%X')}")
def dispatch_astm_message(message): """Dispatch astm message """ logger.debug('Dispatching ASTM Message') if output: path = os.path.abspath(output) loop.create_task(asyncio.to_thread(write_message, message, path)) if url: session = lims.Session(url) session_args = { 'delay': args.delay, 'retries': args.retries, 'consumer': args.consumer, } loop.create_task( asyncio.to_thread(post_to_senaite, message, session, **session_args))
async def async_do(args): global debug debug = args.debug loop = asyncio.get_running_loop() loop.add_signal_handler(signal.SIGWINCH, sigwinchange_handler) await asyncio.gather(asyncio.to_thread(watcher, args, loop), timer(args))
async def main() -> None: print(1) futures = [asyncio.to_thread(sleeper, count) for count in range(1,60)] print(2) results = await asyncio.gather(*futures) print(3) flatResults = [item for subitem in results for item in subitem] print(flatResults)
async def now_average(currency: CurrencyName): ''' Return an average price from the exchanges configured for the given currency. ''' result = None logger.debug(f'currency: {currency}') def get_candle(exchange: ccxt.Exchange, currency: CurrencyName = currency) -> tuple[ccxt.Exchange, Candle | None]: assert exchange assert currency result = (exchange, None) exchange.load_markets() if currency.value in exchange.currencies: try: candle = None candle = request_single(exchange, currency) if candle: result = exchange, candle except Exception as e: logger.error(f'error requesting data from exchange: {e}') return result tasks = [asyncio.to_thread(get_candle, exchange) for exchange in supported_exchanges.values()] task_results = await asyncio.gather(*tasks) logger.debug(f'task results: {task_results}') candles = [] failed_exchanges = [] for exchange, candle in task_results: if candle: candles.append(candle) else: failed_exchanges.append(exchange.name) logger.debug(f'candles: {candles}') average_price_candle = None if len(candles): average_price_candle = calculate_average_price(candles) else: raise HTTPException( status_code = HTTPStatus.INTERNAL_SERVER_ERROR, detail = 'Spotbit could not get any candle data from the configured exchanges.') exchanges_used = [exchange.name for exchange in supported_exchanges.values() if exchange.name not in failed_exchanges] result = PriceResponse( candle = average_price_candle, exchanges_used = exchanges_used, failed_exchanges = failed_exchanges, ) return result
async def main1(): t0 = time.time() task = asyncio.to_thread( urllib.request.urlopen, "https://api.binance.com/api/v3/depth?symbol=BNBBTC&limit=1000 ", ) t1 = time.time() print(t1 - t0) stream = await task t2 = time.time() print(t2 - t1) content_coro = asyncio.to_thread(stream.read) t3 = time.time() print(t3 - t2) contents = await content_coro t4 = time.time() print(t4 - t3) print(contents)
async def get_candles_at_dates( currency: CurrencyName, exchange: ExchangeName, dates: list[datetime]) -> list[Candle]: ''' Dates should be provided in the body of the request as a json array of dates formatted as ISO8601 "YYYY-MM-DDTHH:mm:SS". ''' result: list[Candle] = [] if exchange.value not in supported_exchanges: raise HTTPException( detail = ServerErrors.EXCHANGE_NOT_SUPPORTED, status_code = HTTPStatus.INTERNAL_SERVER_ERROR) ccxt_exchange = supported_exchanges[exchange.value] ccxt_exchange.load_markets() pair = get_supported_pair_for(currency, ccxt_exchange) # Different exchanges have different ticker formates if not pair: raise HTTPException( detail = f'Spotbit does not support the BTC/{currency.value} pair on {exchange.value}', status_code = HTTPStatus.INTERNAL_SERVER_ERROR) # FIXME(nochiel) Different exchanges return candle data at different resolutions. # I need to get candle data in the lowest possible resolution then filter out the dates needed. limit = 100 timeframe = '1h' if ccxt_exchange.timeframes: if '1h' in ccxt_exchange.timeframes: timeframe = '1h' elif '30m' in ccxt_exchange.timeframes: timeframe = '30m' candles_found: tuple[list[Candle] | None] args = [dict(exchange = ccxt_exchange, limit = limit, timeframe = timeframe, pair = pair, since = date) for date in dates] tasks = [asyncio.to_thread(get_history, **arg) for arg in args] candles_found = await asyncio.gather(*tasks) result = [candles_at[0] for candles_at in candles_found if candles_at] if not result: raise HTTPException( detail = f'Spotbit did not receive any candle history for the requested dates.', status_code = HTTPStatus.INTERNAL_SERVER_ERROR) return result
async def test_to_thread_concurrent(self): func = mock.Mock() futs = [] for _ in range(10): fut = asyncio.to_thread(func) futs.append(fut) await asyncio.gather(*futs) self.assertEqual(func.call_count, 10)
async def main(puzzle_number: int): driver = webdriver.Firefox(executable_path=GeckoDriverManager().install()) try: driver.get(f'http://www.dailykillersudoku.com/puzzle/{puzzle_number}') img_data = driver.execute_script( 'return document.getElementsByClassName("puzzle-canvas")[0].toDataURL()' ) sudoku, _ = await asyncio.gather( asyncio.to_thread(solve_killer_sudoku_from_image_data, img_data), asyncio.to_thread(prepare_browser, driver)) enter_solution(driver, sudoku) finally: driver.close()
async def rebuild(self): """ Downloads the files for vix term structures, and generates a map from trade days to each future settlment date. Most users will prefer to to use the command line program to perform this. :return: nothing """ download_quandl_coro = asyncio.to_thread(v.download_quandle_data, quandl_api_key, self.data_path) ch = asyncio.create_task(cash.get_vix_index_histories(self.data_path)) wide_vix_calendar_coro = asyncio.to_thread( v.vix_futures_trade_dates_and_settlement_dates) (cash_vix, _, wide_vix_calendar) = await asyncio.gather(ch, download_quandl_coro, wide_vix_calendar_coro) wide_vix_calendar.to_pickle(self.data_path / "wide_vix_calendar.pkl") cash_vix.to_pickle(self.data_path / _vix_cash_file)
async def main(): devices = list(range(20)) coroutines = [] for dev in devices: if dev % 2: coro = asyncio.to_thread(netmiko_connect, dev) else: coro = scrapli_connect(dev) coroutines.append(coro) results = await asyncio.gather(*coroutines) return results
async def send_command_to_devices(devices, command): tasks = [] for device in devices: if device["host"] in netmiko_only: task = asyncio.to_thread(connect_ssh_netmiko, device, command) tasks.append(task) else: task = asyncio.create_task(connect_ssh(device, command)) tasks.append(task) results = await asyncio.gather(*tasks) return results
async def main(): nets = map(ip_network, subnets) # get all the ip addresses (using itertools.chain to flatten the ip.hosts() lists) hosts = map(str, itertools.chain(*map(lambda ip: ip.hosts(), nets))) #create tasks tasks = map(asyncio.create_task, map(lambda host: asyncio.to_thread(get_info, host), hosts)) # and wait for completion done, pending = await asyncio.wait(tasks)
async def _get_next_chunk_graph(self, chunk_graph_iter: Iterator[ChunkGraph]) \ -> Optional[ChunkGraph]: def next_chunk_graph(): try: return next(chunk_graph_iter) except StopIteration: return fut = asyncio.to_thread(next_chunk_graph) chunk_graph = await fut return chunk_graph
async def match(self, img): hash_ = await self.getHash(img) sub = lambda n, h1, h2: (n, h1 - h2) tasks = [ asyncio.create_task(asyncio.to_thread(sub, n, h, hash_)) for n, h in self.hash_list ] result = [ r for coro in asyncio.as_completed(tasks) if (r := await coro)[1] <= 10 ] return self.polist[min(result, key=lambda r: r[1])[0]] if any(result) else None
async def acallback(cls, text, context, callback_name=None): try: function = cls._CALLBACKS.get(callback_name, cls.default_callback) if asyncio.iscoroutinefunction(function): coro = function(text, context) else: coro = to_thread(function, text, context) return await coro except cls.CATCH_EXCEPTIONS: if cls.DEFAULT_RETURN is NotSet: return cls.default_callback(text, context) else: return cls.DEFAULT_RETURN
async def test_list(close_register, thread, nsubscribers, nitems): async def subscribe(registry, register_key): ret = [] async for y in registry.subscribe(register_key): ret.append(y) return ret def register(registry, register_key, items): registry.open_register_list(register_key) assert registry.get(register_key) == [] for i, item in enumerate(items): registry.register_list_item(register_key, item) assert registry.get(register_key) == items[: i + 1] for i, item in enumerate(items): registry.deregister_list_item(register_key, item) assert registry.get(register_key) == items[i + 1 :] if close_register: registry.close_register(register_key) async def aregister(registry, register_key, items): return register(registry, register_key, items) registry = Registry() register_key = "item" items = [f"{register_key}-{i+1}" for i in range(nitems)] # subscribe tasks_subscribe = [] for i in range(nsubscribers): task = asyncio.create_task(subscribe(registry, register_key)) tasks_subscribe.append(task) # register if thread: coro = asyncio.to_thread(register, registry, register_key, items) else: coro = aregister(registry, register_key, items) task_register = asyncio.create_task(coro) await asyncio.gather(task_register) task_close = asyncio.create_task(registry.close()) results = await asyncio.gather(*tasks_subscribe, task_close) expected = [items[: i + 1] for i in range(len(items))] expected.extend([items[i + 1 :] for i in range(len(items))]) actuals = results[:nsubscribers] for actual in actuals: assert actual == expected
async def main(): print( f"started main at {datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]}" ) other_task = asyncio.sleep(1) await asyncio.gather( asyncio.to_thread(blokcing_io), other_task, ) print( f"finished main at {datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]}" )
async def main(port, log_storage="/home/tbjc1magic/log"): server = aio.server() task_manager = TaskManager() task_manager_thread = asyncio.to_thread(task_manager.run) data_collector_service_pb2_grpc.add_DataCollectorServicer_to_server( DataCollectorServicer(log_storage, task_manager), server) service_names = ( data_collector_service_pb2.DESCRIPTOR. services_by_name["DataCollector"].full_name, reflection.SERVICE_NAME, ) reflection.enable_server_reflection(service_names, server) server.add_insecure_port(f"[::]:{port}") await server.start() await asyncio.gather(server.wait_for_termination(), task_manager_thread)
async def main(): """ The Incrementer instance has an internal number that starts at zero. We increment it six times, and we do so from different threads "to speed things up". The expected end value of the number six. """ incrementer = Incrementer() logging.info("Number is now: %d", incrementer.number) await asyncio.gather( *[asyncio.to_thread(incrementer.increment, task_id) for task_id in range(6)] ) logging.info("Number is now: %d", incrementer.number) """