def input_reader_loop(callback_loop: asyncio.AbstractEventLoop, executor: Executor): task_number = 0 while True: task_number = task_number + 1 line: str = sys.stdin.readline() splits = line.split(' ') coro = None if len(splits) < 2: print('...') continue n = int(splits[1]) if splits[0] == 's': print('Send sleep task...', task_number) asyncio.run_coroutine_threadsafe(do_async_sleep(task_number, n), loop=callback_loop) print('Ok') if splits[0] == 'x': print('Send compute task...', task_number) callback_loop.run_in_executor(executor, execution_job, task_number, n) print('Ok')
async def get_update_skips( merged: pandas.DataFrame, curr_cols: numpy.ndarray, prev_cols: numpy.ndarray, ignored: typing.Optional[typing.List[str]] = None, *, loop: asyncio.AbstractEventLoop, ) -> typing.Tuple[str, typing.AsyncIterator[typing.Tuple[str, typing.List[dict]]]]: """Retrieve the updates & skips for shared rows.""" checks = await loop.run_in_executor( # type: ignore None, functools.partial( get_checks, merged, curr_cols, prev_cols, )) iterable: typing.Tuple[asyncio.Future, ...] = ( loop.run_in_executor( # type: ignore None, functools.partial( split_checks, checks, curr_cols, 'current', )), loop.run_in_executor( # type: ignore None, functools.partial( split_checks, checks, prev_cols, 'previous', ))) splits = {} async for key, value in corelib.task_map(corelib.AsyncIterator, iterable): splits[key] = value iterable = ( loop.run_in_executor( # type: ignore None, functools.partial( get_updates, splits, ignored=ignored, )), loop.run_in_executor( # type: ignore None, functools.partial( get_skips, splits, ignored=ignored, ))) return 'checks', corelib.task_map(corelib.AsyncIterator, iterable)
async def import_from_gpm(*, loop: asyncio.AbstractEventLoop) -> None: batch_iter = corelib.AsyncIterator(await loop.run_in_executor( None, functools.partial(api.get_all_songs, incremental=True))) print('Clearing SQL table.') await loop.run_in_executor(None, sqllib.erase_new_tracks) tasks = [ asyncio.ensure_future(load_batch(batch)) async for batch in batch_iter ] for task in asyncio.as_completed(tasks): print(await task) curr_coro = loop.run_in_executor(None, sqllib.get_current_tracks) prev_coro = loop.run_in_executor(None, sqllib.get_previous_tracks) results = await pdlib.get_ins_upd_del('Tracks', curr_coro, prev_coro, 'id') pprint.pprint(results)
def _begin_validation( session: UpdateSession, config: config.Config, loop: asyncio.AbstractEventLoop, downloaded_update_path: str)\ -> asyncio.futures.Future: """ Start the validation process. """ session.set_stage(Stages.VALIDATING) cert_path = config.update_cert_path\ if config.signature_required else None validation_future \ = asyncio.ensure_future(loop.run_in_executor( None, file_actions.validate_update, downloaded_update_path, session.set_progress, cert_path)) def validation_done(fut): exc = fut.exception() if exc: session.set_error(getattr(exc, 'short', str(type(exc))), str(exc)) else: rootfs_file = fut.result() loop.call_soon_threadsafe(_begin_write, session, loop, rootfs_file) validation_future.add_done_callback(validation_done) return validation_future
def run_command_on_loop( loop: asyncio.AbstractEventLoop, semaphore: asyncio.Semaphore, command: str, env: Mapping[str, str] = None, ) -> bool: """ Run test for one particular feature, check its result and return report. :param loop: Loop to use. :param command: Command to run. :return: Result of the command. """ with (yield from semaphore): runner = partial(run_command, cmd=command, env=env) proc = yield from loop.run_in_executor(None, runner) filename = proc.args.split(" ")[-1] is_fetch = "fetch" in proc.args.split(" ") if proc.returncode == 0: message = f"{['Uploaded', 'Fetched'][is_fetch]}: {filename}" else: error = proc.stderr.decode() message = f"ERROR:\n{error}" if "FileExistsError" in error or "already exists" in error: message = ( f"WARNING: Did not overwrite <{filename}>, " f"please consider --{'osf-' * ~is_fetch}overwrite" ) print(message) return proc.returncode
def extract( query: str, loop: asyncio.AbstractEventLoop = None, ): if not loop: loop = asyncio.get_event_loop() return loop.run_in_executor(None, _extract, query)
async def knock(address: IPv4Address, ports: List[int], loop: asyncio.AbstractEventLoop): """Knock on a set of ports for a given host.""" _LOGGER.debug("Knocking at ports %s on %s", ports, address) await asyncio.wait([ loop.run_in_executor(None, _synch_knock, address, port) for port in ports ])
def run_command_on_loop(loop: asyncio.AbstractEventLoop, fn, command, semaphore, global_state): """ Run command "fn" in loop """ with (yield from semaphore): runner = partial(fn, command, semaphore, global_state) output = yield from loop.run_in_executor(None, runner) #yield from asyncio.sleep(0.01) # Slowing a bit for demonstration purposes return output
def extract( query: str, address: Union[ipaddress.IPv4Address, ipaddress.IPv6Address] = None, video: bool = False, loop: asyncio.AbstractEventLoop = None, ) -> Coroutine: if not loop: loop = asyncio.get_event_loop() return loop.run_in_executor(None, _extract, query, address, video)
async def _call_blocking(loop: asyncio.AbstractEventLoop, executor: concurrent.futures.Executor, func, *args): futures = [loop.run_in_executor(executor, func, *args)] while futures: done, futures = await asyncio.wait(futures, loop=loop, return_when=asyncio.ALL_COMPLETED) for f in done: await f return f.result()
async def session_source( event_loop: asyncio.AbstractEventLoop, session_data: typing.Dict[str, typing.Any] ) -> typing.Generator[NamedTemporaryFile, None, None]: """Create a predefined session data source.""" async with NamedTemporaryFile("wb") as session_file: await asyncio.wait_for( event_loop.run_in_executor( None, partial(pickle.dump, obj=session_data, file=session_file.raw)), timeout=None, ) yield session_file
def run_command_on_loop(loop: asyncio.AbstractEventLoop, command: str) -> bool: """ Run test for one particular feature, check its result and return report. :param loop: Loop to use. :param command: Command to run. :return: Result of the command. """ with (yield from semaphore): runner = partial(run_command, command) output = yield from loop.run_in_executor(None, runner) yield from asyncio.sleep(2) # Slowing a bit for demonstration purposes return output
async def runTest(loop: asyncio.AbstractEventLoop) -> None: pool = ProcessPoolExecutor() try: task = loop.run_in_executor(pool, retried, "runTest") await task except asyncio.CancelledError: print(traceback.format_exc()) print("cancel") except Exception as e: print("error!") print(e) finally: pool.shutdown()
def __init__(self, loop: AbstractEventLoop, topic, bootstrap_servers, frequency_ms=1000 / 60): self.loop = loop self.producer = aiokafka.AIOKafkaProducer( loop=self.loop, bootstrap_servers=bootstrap_servers) self.topic = topic self.running = True # executor = ProcessPoolExecutor() self.task = loop.run_in_executor(executor=None, func=self._flush_measurements) self.frequency = datetime.timedelta(milliseconds=frequency_ms) self.buffer = b''
def _begin_write(session: UpdateSession, loop: asyncio.AbstractEventLoop, rootfs_file_path: str): """ Start the write process. """ session.set_progress(0) session.set_stage(Stages.WRITING) write_future = asyncio.ensure_future( loop.run_in_executor(None, file_actions.write_update, rootfs_file_path, session.set_progress)) def write_done(fut): exc = fut.exception() if exc: session.set_error(getattr(exc, 'short', str(type(exc))), str(exc)) else: session.set_stage(Stages.DONE) write_future.add_done_callback(write_done)
def StartConsumers(self, consumerCount: int = None, listenerQueue: str = None, listenerQueueSettings: dict = None, topicExchange: str = None, topicExchangeSettings: dict = None, directExchange: str = None, directExchangeSettings: dict = None, subscriptions: Union[List[str], str] = None, confirmDelivery: bool = None, prefetchSize: int = None, prefetchCount: int = None, loop: asyncio.AbstractEventLoop = None, executor: ThreadPoolExecutor = None): if executor is None: executor = self._defaultExecutor listenerQueue, listenerQueueSettings = self._AssertListenerQueueIsSet( listenerQueue, listenerQueueSettings) if consumerCount is None: consumerCount = self._defaultConsumerCount if loop is None: loop = asyncio.get_event_loop() tasks = [] for i in range(consumerCount): func = functools.partial( self._StartConsumerWithRetryHandler, listenerQueue=listenerQueue, listenerQueueSettings=listenerQueueSettings, topicExchange=topicExchange, topicExchangeSettings=topicExchangeSettings, directExchange=directExchange, directExchangeSettings=directExchangeSettings, subscriptions=subscriptions, confirmDelivery=confirmDelivery, prefetchSize=prefetchSize, prefetchCount=prefetchCount, loop=loop, executor=executor) task = loop.run_in_executor(executor, func) futureTask = asyncio.ensure_future(task, loop=loop) tasks.append(futureTask) self._allConsumingTasks += tasks return tasks
async def request( poolmanager: urllib3.PoolManager, executor, *, method="GET", url, fields=None, headers=None, loop: asyncio.AbstractEventLoop = None, ): if not loop: loop = asyncio.get_running_loop() request = functools.partial(poolmanager.request, method, url, fields=fields, headers=headers) return loop.run_in_executor(executor, request)
async def invoke(self, rpc: 'venom.stub.RPC', request: 'venom.message.Message', *, context: 'venom.rpc.RequestContext' = None, loop: asyncio.AbstractEventLoop = None, timeout: int = None): if loop is None: loop = asyncio.get_event_loop() future = loop.run_in_executor( None, partial(self._grpc_stub.blocking_unary_unary, self._group, rpc.name, request, timeout=timeout)) return await future
def _begin_validation(session: UpdateSession, loop: asyncio.AbstractEventLoop, downloaded_update_path: str, robot_name: str) -> asyncio.futures.Future: """ Start the validation process. """ session.set_stage(Stages.VALIDATING) validation_future \ = asyncio.ensure_future(loop.run_in_executor( None, validate_update, downloaded_update_path, session.set_progress)) def validation_done(fut): exc = fut.exception() if exc: session.set_error(getattr(exc, 'short', str(type(exc))), str(exc)) else: rootfs_file, bootfs_file = fut.result() loop.call_soon_threadsafe(_begin_write, session, loop, rootfs_file, robot_name) validation_future.add_done_callback(validation_done) return validation_future
def clear_cache(loop: asyncio.AbstractEventLoop = None) -> Coroutine: if not loop: loop = asyncio.get_event_loop() return loop.run_in_executor(None, _clear_cache)
def run(loop: asyncio.AbstractEventLoop): def call(): os.system('node index.js') with concurrent.futures.ThreadPoolExecutor() as pool: loop.run_in_executor(None, call)
def Start(self, listenerQueue: str = None, listenerQueueSettings: dict = None, topicExchange: str = None, topicExchangeSettings: dict = None, directExchange: str = None, directExchangeSettings: dict = None, subscriptions: Union[List[str], str] = None, confirmDelivery: bool = None, prefetchSize: int = None, prefetchCount: int = None, loop: asyncio.AbstractEventLoop = None, executor: ThreadPoolExecutor = None): if executor is None: executor = self._defaultExecutor if loop is None: loop = asyncio.get_event_loop() if prefetchSize is None: prefetchSize = self._defaultPrefetchSize if prefetchCount is None: prefetchCount = self._defaultPrefetchCount listenerQueue, listenerQueueSettings = self._AssertListenerQueueIsSet( listenerQueue, listenerQueueSettings) with pika.BlockingConnection(self._connParams) as connection: channelId = str(uuid.uuid1()) channel: pika.adapters.blocking_connection.BlockingChannel = connection.channel( ) onMessageCallback = functools.partial( self._OnMessageCallBack, connection=connection, channelId=channelId, listenerQueue=listenerQueue, topicExchange=topicExchange, directExchange=directExchange) self._CreateDefaultRabbitMqSetup( channel, listenerQueue, listenerQueueSettings, topicExchange, topicExchangeSettings, directExchange, directExchangeSettings, subscriptions, confirmDelivery) channel.basic_qos(prefetch_size=prefetchSize, prefetch_count=prefetchCount) channel.basic_consume(listenerQueue, onMessageCallback) self._openChannels[channelId] = channel self._openConnections[channelId] = connection self._logger.info( f'Starting new consumer channel with id {channelId} ' f'and {len(self.channels)} ongoing channels.') if not self._connectionHeartbeatIsRunning: heartbeatFunc = functools.partial(self._ConnectionHeartbeat) connectionHeartbeatTask = loop.run_in_executor( executor, heartbeatFunc) futureConnectionHeartbeatTask = asyncio.ensure_future( connectionHeartbeatTask, loop=loop) self._allConsumingTasks += [futureConnectionHeartbeatTask] try: channel.start_consuming() except Exception as exception: if channelId not in self._forceCloseChannelIds: self._logger.debug( f'Consumer with channel id {channelId} ' f'failed due to unknown exception - ' f'{str(type(exception))}: {str(exception)}') finally: self._openChannels.pop(channelId) self._openConnections.pop(channelId) if channelId in self._forceCloseChannelIds: self._forceCloseChannelIds.pop(channelId) else: raise Exception( f'Channel {channelId} stopped unexpectedly.') self._logger.info(f'Closing consumer channel with id {channelId}.')
async def schedule_formatting( sources: Set[Path], fast: bool, write_back: WriteBack, mode: Mode, report: "Report", loop: asyncio.AbstractEventLoop, executor: Executor, ) -> None: """Run formatting of `sources` in parallel using the provided `executor`. (Use ProcessPoolExecutors for actual parallelism.) `write_back`, `fast`, and `mode` options are passed to :func:`format_file_in_place`. """ cache: Cache = {} if write_back not in (WriteBack.DIFF, WriteBack.COLOR_DIFF): cache = read_cache(mode) sources, cached = filter_cached(cache, sources) for src in sorted(cached): report.done(src, Changed.CACHED) if not sources: return cancelled = [] sources_to_cache = [] lock = None if write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF): # For diff output, we need locks to ensure we don't interleave output # from different processes. manager = Manager() lock = manager.Lock() tasks = { asyncio.ensure_future( loop.run_in_executor(executor, format_file_in_place, src, fast, mode, write_back, lock)): src for src in sorted(sources) } pending = tasks.keys() try: loop.add_signal_handler(signal.SIGINT, cancel, pending) loop.add_signal_handler(signal.SIGTERM, cancel, pending) except NotImplementedError: # There are no good alternatives for these on Windows. pass while pending: done, _ = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED) for task in done: src = tasks.pop(task) if task.cancelled(): cancelled.append(task) elif task.exception(): report.failed(src, str(task.exception())) else: changed = Changed.YES if task.result() else Changed.NO # If the file was written back or was successfully checked as # well-formatted, store this information in the cache. if write_back is WriteBack.YES or ( write_back is WriteBack.CHECK and changed is Changed.NO): sources_to_cache.append(src) report.done(src, changed) if cancelled: if sys.version_info >= (3, 7): await asyncio.gather(*cancelled, return_exceptions=True) else: await asyncio.gather(*cancelled, loop=loop, return_exceptions=True) if sources_to_cache: write_cache(cache, sources_to_cache, mode)
async def get_ins_upd_del( data_name: str, curr_coro: typing.Coroutine[typing.List[dict], None, None], prev_coro: typing.Coroutine[typing.List[dict], None, None], index: typing.Union[str, typing.List[str]], ignored: typing.Optional[typing.List[str]] = None, *, loop: asyncio.AbstractEventLoop, ) -> typing.Dict[str, typing.Union[typing.List[dict], typing.List[str], str]]: """ Universal tool for determining inserts/updates/deletes/skips. Results are determined from the current and previous inputs, which must be lists of dicts that have the same keys in all dict entries in both inputs. From those, it figures out what changes are necessary and returns a dict containing the separate inserts/updates/deletes/skips groups. * **inserts** are rows that exist in the *current* but not *previous* data. * **deletes** are rows that exist in the *previous* but not *current* data. * **updates** are rows that exist in both *current* and *previous* data, and have changed. * **skips** are rows that exist in both *current* and *previous* data, and have not changed. Args: current (List[dict]): The current data records to match against. previous (List[dict]): The previous data records to match against. index (str): The name of the field in the data to use as an index. Note: `current` and `previous` are both *lists of dicts*, where the dicts must contain exactly the same keys for this code to work correctly. This is because the columns are matched up against each other for comparison. Returns: Dict[str, Union[List[dict]], List[str], str]: A dictionary containing an entry for *`'inserts'`*, *`'updates'`*, *`'deletes'`*, *`'skips'`*, and *`'index'`*. Each entry contains lists of dictionary records that belong to that particular category of data. """ iterable: typing.Tuple[asyncio.Future, ...] = (get_coro_data('current', curr_coro, index=index), get_coro_data('previous', prev_coro, index=index)) collected = {} async for key, value in corelib.task_map(corelib.AsyncIterator, iterable): collected[key] = value print(' '.join(( f'{data_name} Collected:', f'curr ({collected["current"].size});', f'prev ({collected["previous"].size})', ))) merged, cols = await loop.run_in_executor( # type: ignore None, functools.partial(merge_entries, **collected)) current_cols = [col + '_x' for col in cols] previous_cols = [col + '_y' for col in cols] iterable = ( loop.run_in_executor( # type: ignore None, functools.partial( get_inserts, merged, current_cols, previous_cols, )), loop.run_in_executor( # type: ignore None, functools.partial( get_deletes, merged, current_cols, previous_cols, )), get_update_skips( merged, current_cols, previous_cols, ignored=ignored, )) results = {} async for key, value in corelib.task_map(corelib.AsyncIterator, iterable): if key == 'checks': async for inner_key, inner_value in value: results[inner_key] = inner_value else: results[key] = value print(' '.join(( f'{data_name} Processed:', f'ins ({results["inserts"].__len__()});', f'upd ({results["updates"].__len__()});', f'del ({results["deletes"].__len__()});', f'skp ({results["skips"].__len__()})', ))) return results
async def runTest(loop: asyncio.AbstractEventLoop) -> None: loop.run_in_executor(th, taskWorker2) loop.run_in_executor(th, taskWorker2) loop.run_in_executor(th, taskWorker2) loop.run_in_executor(th, taskWorker2)
def wrap_to_thread(loop: asyncio.AbstractEventLoop, func: Callable[..., T], executor: Executor, *args: Any, **kwargs: Any) -> Awaitable[T]: callee = partial(func, *args, **kwargs) # noinspection PyTypeChecker return loop.run_in_executor(executor, callee)
def render(self, loop: asyncio.AbstractEventLoop=None): """:coro: Returns a rendered version of the table.""" loop = loop or asyncio.get_event_loop() func = functools.partial(self._render) return loop.run_in_executor(None, func)