async def manage_async_to_thread_tasks(func: Any, items: List[Dict[str, Any]],
concurrency: int) -> Dict[str, Any]:
"""Manages a grouping of async.to_thread tasks, keeping number of active
tasks at the concurrency level by starting new tasks whenver one completes.
Only use with python3.9+.
Args:
func (Any): Function to run in threads
items (List[Dict[str, Any]]): List of dict with kwargs for each task
concurrency (int): max concurrency
Returns:
Dict[str, Any]: Dict with generi task name as the key, task result as the value
"""
results = {}
arr = [create_task(to_thread(func, **i)) for i in items[:concurrency]]
count = len(arr)
num_of_items = len(items)
while len(results) < num_of_items:
await sleep(0.001)
for index, task in enumerate(arr):
if task.done():
results[task.get_name()] = await task
if count < num_of_items:
arr[index] = create_task(to_thread(func, **items[count]))
count += 1
return results
async def main():
"""
This is of course a bad idea. The functions do run, synchronously, because
they are evaluated before `gather` is called. That function, however, expects
awaitables - instead, it receives strings.
"""
try:
await asyncio.gather(
non_awaitable_io_bound_function(1, 2),
non_awaitable_io_bound_function(2, 4),
)
except TypeError as e:
logging.error("Caught a TypeError: %s", e)
"""
Luckily, we can cheat. Asyncio can run run a function in a thread, and await not
the function, but the thread as a whole. In Python, only one thread can run at a
time. So, this mechanism cannot be used to speed up calculations. But to complete an
IO bound task, a thread does not need to run all the time: it only needs to check
if the task (sleeping, in this case) is done every now and then.
"""
logging.info("\n\nEXAMPLE 2 - TO_THREAD\n")
await asyncio.gather(
asyncio.to_thread(non_awaitable_io_bound_function, 3, 1),
asyncio.to_thread(non_awaitable_io_bound_function, 4, 2),
asyncio.to_thread(non_awaitable_io_bound_function, 5, 3),
asyncio.to_thread(non_awaitable_io_bound_function, 6, 4),
)
def check_if_obj_close_to_side(self):
"""called by update_ball to check whether the ball is close to the left/right of the screen. """
b = self.ball
print(b.x + b.close_to_side)
print(self.WIDTH - bx)
if b.x + b.close_to_side < 200:
await asyncio.gather(asyncio.to_thread(almost_hit_left()))
elif self.WIDTH - b.x < 200:
await asyncio.gather(asyncio.to_thread(almost_hit_right()))
async def main():
event = asyncio.Event()
asyncio.create_task(asyncio.to_thread(blocking_func, event))
await asyncio.sleep(5)
# now lets stop
event.set()
async def run_cancelable_thread(thread_main: Callable[[], T],
on_cancel: Callable[[], None]) -> Awaitable[T]:
"""
Runs code in a separate thread, while in an async context, like `asyncio.to_thread`, but supporting task
cancellation.
Args:
thread_main: The code to execute in a different thread
on_cancel: A callback that will be called when the task is cancelled. BEWARE! This will be called in a
different thread to that of `thread_main`, so it should only work with thread-safe objects so as to signal
the thread to stop, e.g. by putting some element in a thread-safe queue.
Returns:
The result of the code in `thread_main`
"""
thread_task = None
try:
thread_task = asyncio.create_task(asyncio.to_thread(thread_main))
return await asyncio.shield(thread_task)
except asyncio.CancelledError:
on_cancel()
if thread_task is not None:
await thread_task
raise
async def check_thread():
print(f"started main at {time.strftime('%X')}")
await asyncio.gather(asyncio.to_thread(blocking_io), asyncio.sleep(1),
hello_world3())
print(f"finished main at {time.strftime('%X')}")
async def wrap(*args, **kwargs):
# return coroutine to not block current actor
if asyncio.iscoroutinefunction(func):
return func(*args, **kwargs)
else:
# for sync call, running in thread
return asyncio.to_thread(func, *args, **kwargs)
async def spracuj(self, ctx, *, message): #TODO: rename trigger command
reading = False
result = []
for line in message.split('\n'):
if reading:
if line.startswith('```'):
reading = False
break
else:
result.append(line)
else:
if line.startswith('```'):
reading = True
code = '\n'.join(result)
try:
codeResult = await asyncio.wait_for(asyncio.gather(
asyncio.to_thread(executeRun, code)),
timeout=1.0)
codeResult = ''.join(codeResult)
if (not codeResult):
await ctx.send("Program prebehol úspešne.")
elif (len(codeResult.split('\n')) > 10):
await ctx.send(
"Výsledok je moc dlhý. Bol by to spam keyže to tu postnem."
)
elif (len(codeResult) > 500):
await ctx.send(
"Output je obmedzený na 500 znakov.\n Toto je už moc aj na mňa."
)
else:
await ctx.send((codeResult))
except asyncio.TimeoutError:
await ctx.send('Čas na spracovanie vypršal.')
async def start(self):
shards_per_cluster = int(config["Clustering"]["shards_per_cluster"])
shard_count = int(config["Clustering"].get("shard_count")
or await self.fetch_num_shards())
full_clusters, last_cluster_shards = divmod(shard_count,
shards_per_cluster)
cluster_count = full_clusters + int(bool(last_cluster_shards))
logger.info(
f"Launching {cluster_count} clusters to handle {shard_count} shards with {shards_per_cluster} per cluster."
)
all_shards = as_chunks(range(shard_count), shards_per_cluster)
for cluster_id, shards in enumerate(all_shards):
cluster_watcher_func = partial(self.cluster_watcher,
(cluster_id, shard_count, shards))
self.monitors[cluster_id] = asyncio.create_task(
asyncio.to_thread(cluster_watcher_func))
async def keep_alive():
async with websockets.serve(self.websocket_handler,
self.websocket_host,
self.websocket_port):
await asyncio.gather(*self.monitors.values())
self.keep_clusters_alive.start()
self.keep_alive = self.loop.create_task(keep_alive())
async def test_thread(obj, nsubscribers, pre_nitems, post_nitems):
"""test if the issue is resovled
https://github.com/simonsobs/nextline/issues/2
"""
nitems = pre_nitems + post_nitems
if nsubscribers >= 50 and nitems >= 100:
pytest.skip("nsubscribers >= 50 and nitems >= 100")
async def subscribe(obj, event):
await event.wait()
items = []
async for i in obj.subscribe():
items.append(i)
return items
def send(obj, pre_items, event, post_items, event_end):
for i in pre_items:
obj.put(i)
event.set()
for i in post_items:
obj.put(i)
event_end.set()
async def close(obj, event_end):
await event_end.wait()
await obj.close()
items = list(range(nitems))
pre_items = items[:pre_nitems]
post_items = items[pre_nitems:]
event = ThreadSafeAsyncioEvent()
event_end = ThreadSafeAsyncioEvent()
coro = asyncio.to_thread(send, obj, pre_items, event, post_items,
event_end)
task_send = asyncio.create_task(coro)
tasks_subscribe = []
for i in range(nsubscribers):
coro = subscribe(obj, event)
task = asyncio.create_task(coro)
tasks_subscribe.append(task)
coro = close(obj, event_end)
task_close = asyncio.create_task(coro)
results = await asyncio.gather(*tasks_subscribe, task_send, task_close)
for actual in results[:nsubscribers]:
if not actual: # can be empty
continue
# print(actual[0])
expected = items[items.index(actual[0]):] # no missing or duplicate
# items from the first
# received item
assert actual == expected
async def main():
print(f"started main at {time.strftime('%X')}")
await asyncio.gather(
asyncio.to_thread(blocking_io),
asyncio.sleep(1))
print(f"finished main at {time.strftime('%X')}")
def dispatch_astm_message(message):
"""Dispatch astm message
"""
logger.debug('Dispatching ASTM Message')
if output:
path = os.path.abspath(output)
loop.create_task(asyncio.to_thread(write_message, message, path))
if url:
session = lims.Session(url)
session_args = {
'delay': args.delay,
'retries': args.retries,
'consumer': args.consumer,
}
loop.create_task(
asyncio.to_thread(post_to_senaite, message, session,
**session_args))
async def async_do(args):
global debug
debug = args.debug
loop = asyncio.get_running_loop()
loop.add_signal_handler(signal.SIGWINCH, sigwinchange_handler)
await asyncio.gather(asyncio.to_thread(watcher, args, loop), timer(args))
async def main() -> None:
print(1)
futures = [asyncio.to_thread(sleeper, count) for count in range(1,60)]
print(2)
results = await asyncio.gather(*futures)
print(3)
flatResults = [item for subitem in results for item in subitem]
print(flatResults)
async def now_average(currency: CurrencyName):
'''
Return an average price from the exchanges configured for the given currency.
'''
result = None
logger.debug(f'currency: {currency}')
def get_candle(exchange: ccxt.Exchange, currency: CurrencyName = currency) -> tuple[ccxt.Exchange, Candle | None]:
assert exchange
assert currency
result = (exchange, None)
exchange.load_markets()
if currency.value in exchange.currencies:
try:
candle = None
candle = request_single(exchange, currency)
if candle:
result = exchange, candle
except Exception as e:
logger.error(f'error requesting data from exchange: {e}')
return result
tasks = [asyncio.to_thread(get_candle, exchange)
for exchange in supported_exchanges.values()]
task_results = await asyncio.gather(*tasks)
logger.debug(f'task results: {task_results}')
candles = []
failed_exchanges = []
for exchange, candle in task_results:
if candle:
candles.append(candle)
else:
failed_exchanges.append(exchange.name)
logger.debug(f'candles: {candles}')
average_price_candle = None
if len(candles):
average_price_candle = calculate_average_price(candles)
else:
raise HTTPException(
status_code = HTTPStatus.INTERNAL_SERVER_ERROR,
detail = 'Spotbit could not get any candle data from the configured exchanges.')
exchanges_used = [exchange.name for exchange in supported_exchanges.values()
if exchange.name not in failed_exchanges]
result = PriceResponse(
candle = average_price_candle,
exchanges_used = exchanges_used,
failed_exchanges = failed_exchanges,
)
return result
async def main1():
t0 = time.time()
task = asyncio.to_thread(
urllib.request.urlopen,
"https://api.binance.com/api/v3/depth?symbol=BNBBTC&limit=1000 ",
)
t1 = time.time()
print(t1 - t0)
stream = await task
t2 = time.time()
print(t2 - t1)
content_coro = asyncio.to_thread(stream.read)
t3 = time.time()
print(t3 - t2)
contents = await content_coro
t4 = time.time()
print(t4 - t3)
print(contents)
async def get_candles_at_dates(
currency: CurrencyName,
exchange: ExchangeName,
dates: list[datetime]) -> list[Candle]:
'''
Dates should be provided in the body of the request as a json array of dates formatted as ISO8601 "YYYY-MM-DDTHH:mm:SS".
'''
result: list[Candle] = []
if exchange.value not in supported_exchanges:
raise HTTPException(
detail = ServerErrors.EXCHANGE_NOT_SUPPORTED,
status_code = HTTPStatus.INTERNAL_SERVER_ERROR)
ccxt_exchange = supported_exchanges[exchange.value]
ccxt_exchange.load_markets()
pair = get_supported_pair_for(currency, ccxt_exchange)
# Different exchanges have different ticker formates
if not pair:
raise HTTPException(
detail = f'Spotbit does not support the BTC/{currency.value} pair on {exchange.value}',
status_code = HTTPStatus.INTERNAL_SERVER_ERROR)
# FIXME(nochiel) Different exchanges return candle data at different resolutions.
# I need to get candle data in the lowest possible resolution then filter out the dates needed.
limit = 100
timeframe = '1h'
if ccxt_exchange.timeframes:
if '1h' in ccxt_exchange.timeframes:
timeframe = '1h'
elif '30m' in ccxt_exchange.timeframes:
timeframe = '30m'
candles_found: tuple[list[Candle] | None]
args = [dict(exchange = ccxt_exchange,
limit = limit,
timeframe = timeframe,
pair = pair,
since = date)
for date in dates]
tasks = [asyncio.to_thread(get_history, **arg)
for arg in args]
candles_found = await asyncio.gather(*tasks)
result = [candles_at[0] for candles_at in candles_found if candles_at]
if not result:
raise HTTPException(
detail = f'Spotbit did not receive any candle history for the requested dates.',
status_code = HTTPStatus.INTERNAL_SERVER_ERROR)
return result
async def test_to_thread_concurrent(self):
func = mock.Mock()
futs = []
for _ in range(10):
fut = asyncio.to_thread(func)
futs.append(fut)
await asyncio.gather(*futs)
self.assertEqual(func.call_count, 10)
async def main(puzzle_number: int):
driver = webdriver.Firefox(executable_path=GeckoDriverManager().install())
try:
driver.get(f'http://www.dailykillersudoku.com/puzzle/{puzzle_number}')
img_data = driver.execute_script(
'return document.getElementsByClassName("puzzle-canvas")[0].toDataURL()'
)
sudoku, _ = await asyncio.gather(
asyncio.to_thread(solve_killer_sudoku_from_image_data, img_data),
asyncio.to_thread(prepare_browser, driver))
enter_solution(driver, sudoku)
finally:
driver.close()
async def rebuild(self):
"""
Downloads the files for vix term structures, and generates a map from trade days to each future settlment date.
Most users will prefer to to use the command line program to perform this.
:return: nothing
"""
download_quandl_coro = asyncio.to_thread(v.download_quandle_data,
quandl_api_key,
self.data_path)
ch = asyncio.create_task(cash.get_vix_index_histories(self.data_path))
wide_vix_calendar_coro = asyncio.to_thread(
v.vix_futures_trade_dates_and_settlement_dates)
(cash_vix, _,
wide_vix_calendar) = await asyncio.gather(ch, download_quandl_coro,
wide_vix_calendar_coro)
wide_vix_calendar.to_pickle(self.data_path / "wide_vix_calendar.pkl")
cash_vix.to_pickle(self.data_path / _vix_cash_file)
async def main():
devices = list(range(20))
coroutines = []
for dev in devices:
if dev % 2:
coro = asyncio.to_thread(netmiko_connect, dev)
else:
coro = scrapli_connect(dev)
coroutines.append(coro)
results = await asyncio.gather(*coroutines)
return results
async def send_command_to_devices(devices, command):
tasks = []
for device in devices:
if device["host"] in netmiko_only:
task = asyncio.to_thread(connect_ssh_netmiko, device, command)
tasks.append(task)
else:
task = asyncio.create_task(connect_ssh(device, command))
tasks.append(task)
results = await asyncio.gather(*tasks)
return results
async def main():
nets = map(ip_network, subnets)
# get all the ip addresses (using itertools.chain to flatten the ip.hosts() lists)
hosts = map(str, itertools.chain(*map(lambda ip: ip.hosts(), nets)))
#create tasks
tasks = map(asyncio.create_task, map(lambda host: asyncio.to_thread(get_info, host), hosts))
# and wait for completion
done, pending = await asyncio.wait(tasks)
async def _get_next_chunk_graph(self,
chunk_graph_iter: Iterator[ChunkGraph]) \
-> Optional[ChunkGraph]:
def next_chunk_graph():
try:
return next(chunk_graph_iter)
except StopIteration:
return
fut = asyncio.to_thread(next_chunk_graph)
chunk_graph = await fut
return chunk_graph
async def match(self, img):
hash_ = await self.getHash(img)
sub = lambda n, h1, h2: (n, h1 - h2)
tasks = [
asyncio.create_task(asyncio.to_thread(sub, n, h, hash_))
for n, h in self.hash_list
]
result = [
r for coro in asyncio.as_completed(tasks)
if (r := await coro)[1] <= 10
]
return self.polist[min(result,
key=lambda r: r[1])[0]] if any(result) else None
async def acallback(cls, text, context, callback_name=None):
try:
function = cls._CALLBACKS.get(callback_name, cls.default_callback)
if asyncio.iscoroutinefunction(function):
coro = function(text, context)
else:
coro = to_thread(function, text, context)
return await coro
except cls.CATCH_EXCEPTIONS:
if cls.DEFAULT_RETURN is NotSet:
return cls.default_callback(text, context)
else:
return cls.DEFAULT_RETURN
async def test_list(close_register, thread, nsubscribers, nitems):
async def subscribe(registry, register_key):
ret = []
async for y in registry.subscribe(register_key):
ret.append(y)
return ret
def register(registry, register_key, items):
registry.open_register_list(register_key)
assert registry.get(register_key) == []
for i, item in enumerate(items):
registry.register_list_item(register_key, item)
assert registry.get(register_key) == items[: i + 1]
for i, item in enumerate(items):
registry.deregister_list_item(register_key, item)
assert registry.get(register_key) == items[i + 1 :]
if close_register:
registry.close_register(register_key)
async def aregister(registry, register_key, items):
return register(registry, register_key, items)
registry = Registry()
register_key = "item"
items = [f"{register_key}-{i+1}" for i in range(nitems)]
# subscribe
tasks_subscribe = []
for i in range(nsubscribers):
task = asyncio.create_task(subscribe(registry, register_key))
tasks_subscribe.append(task)
# register
if thread:
coro = asyncio.to_thread(register, registry, register_key, items)
else:
coro = aregister(registry, register_key, items)
task_register = asyncio.create_task(coro)
await asyncio.gather(task_register)
task_close = asyncio.create_task(registry.close())
results = await asyncio.gather(*tasks_subscribe, task_close)
expected = [items[: i + 1] for i in range(len(items))]
expected.extend([items[i + 1 :] for i in range(len(items))])
actuals = results[:nsubscribers]
for actual in actuals:
assert actual == expected
async def main():
print(
f"started main at {datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]}"
)
other_task = asyncio.sleep(1)
await asyncio.gather(
asyncio.to_thread(blokcing_io),
other_task,
)
print(
f"finished main at {datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]}"
)
async def main(port, log_storage="/home/tbjc1magic/log"):
server = aio.server()
task_manager = TaskManager()
task_manager_thread = asyncio.to_thread(task_manager.run)
data_collector_service_pb2_grpc.add_DataCollectorServicer_to_server(
DataCollectorServicer(log_storage, task_manager), server)
service_names = (
data_collector_service_pb2.DESCRIPTOR.
services_by_name["DataCollector"].full_name,
reflection.SERVICE_NAME,
)
reflection.enable_server_reflection(service_names, server)
server.add_insecure_port(f"[::]:{port}")
await server.start()
await asyncio.gather(server.wait_for_termination(), task_manager_thread)
async def main():
"""
The Incrementer instance has an internal number that starts at zero. We
increment it six times, and we do so from different threads "to speed things up".
The expected end value of the number six.
"""
incrementer = Incrementer()
logging.info("Number is now: %d", incrementer.number)
await asyncio.gather(
*[asyncio.to_thread(incrementer.increment, task_id) for task_id in range(6)]
)
logging.info("Number is now: %d", incrementer.number)
"""
