def input_reader_loop(callback_loop: asyncio.AbstractEventLoop,
executor: Executor):
task_number = 0
while True:
task_number = task_number + 1
line: str = sys.stdin.readline()
splits = line.split(' ')
coro = None
if len(splits) < 2:
print('...')
continue
n = int(splits[1])
if splits[0] == 's':
print('Send sleep task...', task_number)
asyncio.run_coroutine_threadsafe(do_async_sleep(task_number, n),
loop=callback_loop)
print('Ok')
if splits[0] == 'x':
print('Send compute task...', task_number)
callback_loop.run_in_executor(executor, execution_job, task_number,
n)
print('Ok')
async def get_update_skips(
merged: pandas.DataFrame,
curr_cols: numpy.ndarray,
prev_cols: numpy.ndarray,
ignored: typing.Optional[typing.List[str]] = None,
*,
loop: asyncio.AbstractEventLoop,
) -> typing.Tuple[str, typing.AsyncIterator[typing.Tuple[str,
typing.List[dict]]]]:
"""Retrieve the updates & skips for shared rows."""
checks = await loop.run_in_executor( # type: ignore
None, functools.partial(
get_checks,
merged,
curr_cols,
prev_cols,
))
iterable: typing.Tuple[asyncio.Future, ...] = (
loop.run_in_executor( # type: ignore
None,
functools.partial(
split_checks,
checks,
curr_cols,
'current',
)),
loop.run_in_executor( # type: ignore
None,
functools.partial(
split_checks,
checks,
prev_cols,
'previous',
)))
splits = {}
async for key, value in corelib.task_map(corelib.AsyncIterator, iterable):
splits[key] = value
iterable = (
loop.run_in_executor( # type: ignore
None, functools.partial(
get_updates,
splits,
ignored=ignored,
)),
loop.run_in_executor( # type: ignore
None, functools.partial(
get_skips,
splits,
ignored=ignored,
)))
return 'checks', corelib.task_map(corelib.AsyncIterator, iterable)
async def import_from_gpm(*, loop: asyncio.AbstractEventLoop) -> None:
batch_iter = corelib.AsyncIterator(await loop.run_in_executor(
None, functools.partial(api.get_all_songs, incremental=True)))
print('Clearing SQL table.')
await loop.run_in_executor(None, sqllib.erase_new_tracks)
tasks = [
asyncio.ensure_future(load_batch(batch)) async for batch in batch_iter
]
for task in asyncio.as_completed(tasks):
print(await task)
curr_coro = loop.run_in_executor(None, sqllib.get_current_tracks)
prev_coro = loop.run_in_executor(None, sqllib.get_previous_tracks)
results = await pdlib.get_ins_upd_del('Tracks', curr_coro, prev_coro, 'id')
pprint.pprint(results)
def _begin_validation(
session: UpdateSession,
config: config.Config,
loop: asyncio.AbstractEventLoop,
downloaded_update_path: str)\
-> asyncio.futures.Future:
""" Start the validation process. """
session.set_stage(Stages.VALIDATING)
cert_path = config.update_cert_path\
if config.signature_required else None
validation_future \
= asyncio.ensure_future(loop.run_in_executor(
None, file_actions.validate_update,
downloaded_update_path, session.set_progress, cert_path))
def validation_done(fut):
exc = fut.exception()
if exc:
session.set_error(getattr(exc, 'short', str(type(exc))), str(exc))
else:
rootfs_file = fut.result()
loop.call_soon_threadsafe(_begin_write, session, loop, rootfs_file)
validation_future.add_done_callback(validation_done)
return validation_future
def run_command_on_loop(
loop: asyncio.AbstractEventLoop,
semaphore: asyncio.Semaphore,
command: str,
env: Mapping[str, str] = None,
) -> bool:
"""
Run test for one particular feature, check its result and return report.
:param loop: Loop to use.
:param command: Command to run.
:return: Result of the command.
"""
with (yield from semaphore):
runner = partial(run_command, cmd=command, env=env)
proc = yield from loop.run_in_executor(None, runner)
filename = proc.args.split(" ")[-1]
is_fetch = "fetch" in proc.args.split(" ")
if proc.returncode == 0:
message = f"{['Uploaded', 'Fetched'][is_fetch]}: {filename}"
else:
error = proc.stderr.decode()
message = f"ERROR:\n{error}"
if "FileExistsError" in error or "already exists" in error:
message = (
f"WARNING: Did not overwrite <{filename}>, "
f"please consider --{'osf-' * ~is_fetch}overwrite"
)
print(message)
return proc.returncode
def extract(
query: str,
loop: asyncio.AbstractEventLoop = None,
):
if not loop:
loop = asyncio.get_event_loop()
return loop.run_in_executor(None, _extract, query)
async def knock(address: IPv4Address, ports: List[int],
loop: asyncio.AbstractEventLoop):
"""Knock on a set of ports for a given host."""
_LOGGER.debug("Knocking at ports %s on %s", ports, address)
await asyncio.wait([
loop.run_in_executor(None, _synch_knock, address, port)
for port in ports
])
def run_command_on_loop(loop: asyncio.AbstractEventLoop, fn, command,
semaphore, global_state):
"""
Run command "fn" in loop
"""
with (yield from semaphore):
runner = partial(fn, command, semaphore, global_state)
output = yield from loop.run_in_executor(None, runner)
#yield from asyncio.sleep(0.01) # Slowing a bit for demonstration purposes
return output
def extract(
query: str,
address: Union[ipaddress.IPv4Address, ipaddress.IPv6Address] = None,
video: bool = False,
loop: asyncio.AbstractEventLoop = None,
) -> Coroutine:
if not loop:
loop = asyncio.get_event_loop()
return loop.run_in_executor(None, _extract, query, address, video)
async def _call_blocking(loop: asyncio.AbstractEventLoop,
executor: concurrent.futures.Executor, func, *args):
futures = [loop.run_in_executor(executor, func, *args)]
while futures:
done, futures = await asyncio.wait(futures,
loop=loop,
return_when=asyncio.ALL_COMPLETED)
for f in done:
await f
return f.result()
async def session_source(
event_loop: asyncio.AbstractEventLoop,
session_data: typing.Dict[str, typing.Any]
) -> typing.Generator[NamedTemporaryFile, None, None]:
"""Create a predefined session data source."""
async with NamedTemporaryFile("wb") as session_file:
await asyncio.wait_for(
event_loop.run_in_executor(
None,
partial(pickle.dump, obj=session_data, file=session_file.raw)),
timeout=None,
)
yield session_file
def run_command_on_loop(loop: asyncio.AbstractEventLoop, command: str) -> bool:
"""
Run test for one particular feature, check its result and return report.
:param loop: Loop to use.
:param command: Command to run.
:return: Result of the command.
"""
with (yield from semaphore):
runner = partial(run_command, command)
output = yield from loop.run_in_executor(None, runner)
yield from asyncio.sleep(2) # Slowing a bit for demonstration purposes
return output
async def runTest(loop: asyncio.AbstractEventLoop) -> None:
pool = ProcessPoolExecutor()
try:
task = loop.run_in_executor(pool, retried, "runTest")
await task
except asyncio.CancelledError:
print(traceback.format_exc())
print("cancel")
except Exception as e:
print("error!")
print(e)
finally:
pool.shutdown()
def __init__(self,
loop: AbstractEventLoop,
topic,
bootstrap_servers,
frequency_ms=1000 / 60):
self.loop = loop
self.producer = aiokafka.AIOKafkaProducer(
loop=self.loop, bootstrap_servers=bootstrap_servers)
self.topic = topic
self.running = True
# executor = ProcessPoolExecutor()
self.task = loop.run_in_executor(executor=None,
func=self._flush_measurements)
self.frequency = datetime.timedelta(milliseconds=frequency_ms)
self.buffer = b''
def _begin_write(session: UpdateSession, loop: asyncio.AbstractEventLoop,
rootfs_file_path: str):
""" Start the write process. """
session.set_progress(0)
session.set_stage(Stages.WRITING)
write_future = asyncio.ensure_future(
loop.run_in_executor(None, file_actions.write_update, rootfs_file_path,
session.set_progress))
def write_done(fut):
exc = fut.exception()
if exc:
session.set_error(getattr(exc, 'short', str(type(exc))), str(exc))
else:
session.set_stage(Stages.DONE)
write_future.add_done_callback(write_done)
def StartConsumers(self,
consumerCount: int = None,
listenerQueue: str = None,
listenerQueueSettings: dict = None,
topicExchange: str = None,
topicExchangeSettings: dict = None,
directExchange: str = None,
directExchangeSettings: dict = None,
subscriptions: Union[List[str], str] = None,
confirmDelivery: bool = None,
prefetchSize: int = None,
prefetchCount: int = None,
loop: asyncio.AbstractEventLoop = None,
executor: ThreadPoolExecutor = None):
if executor is None:
executor = self._defaultExecutor
listenerQueue, listenerQueueSettings = self._AssertListenerQueueIsSet(
listenerQueue, listenerQueueSettings)
if consumerCount is None:
consumerCount = self._defaultConsumerCount
if loop is None:
loop = asyncio.get_event_loop()
tasks = []
for i in range(consumerCount):
func = functools.partial(
self._StartConsumerWithRetryHandler,
listenerQueue=listenerQueue,
listenerQueueSettings=listenerQueueSettings,
topicExchange=topicExchange,
topicExchangeSettings=topicExchangeSettings,
directExchange=directExchange,
directExchangeSettings=directExchangeSettings,
subscriptions=subscriptions,
confirmDelivery=confirmDelivery,
prefetchSize=prefetchSize,
prefetchCount=prefetchCount,
loop=loop,
executor=executor)
task = loop.run_in_executor(executor, func)
futureTask = asyncio.ensure_future(task, loop=loop)
tasks.append(futureTask)
self._allConsumingTasks += tasks
return tasks
async def request(
poolmanager: urllib3.PoolManager,
executor,
*,
method="GET",
url,
fields=None,
headers=None,
loop: asyncio.AbstractEventLoop = None,
):
if not loop:
loop = asyncio.get_running_loop()
request = functools.partial(poolmanager.request,
method,
url,
fields=fields,
headers=headers)
return loop.run_in_executor(executor, request)
async def invoke(self,
rpc: 'venom.stub.RPC',
request: 'venom.message.Message',
*,
context: 'venom.rpc.RequestContext' = None,
loop: asyncio.AbstractEventLoop = None,
timeout: int = None):
if loop is None:
loop = asyncio.get_event_loop()
future = loop.run_in_executor(
None,
partial(self._grpc_stub.blocking_unary_unary,
self._group,
rpc.name,
request,
timeout=timeout))
return await future
def _begin_validation(session: UpdateSession, loop: asyncio.AbstractEventLoop,
downloaded_update_path: str,
robot_name: str) -> asyncio.futures.Future:
""" Start the validation process. """
session.set_stage(Stages.VALIDATING)
validation_future \
= asyncio.ensure_future(loop.run_in_executor(
None, validate_update,
downloaded_update_path, session.set_progress))
def validation_done(fut):
exc = fut.exception()
if exc:
session.set_error(getattr(exc, 'short', str(type(exc))), str(exc))
else:
rootfs_file, bootfs_file = fut.result()
loop.call_soon_threadsafe(_begin_write, session, loop, rootfs_file,
robot_name)
validation_future.add_done_callback(validation_done)
return validation_future
def clear_cache(loop: asyncio.AbstractEventLoop = None) -> Coroutine:
if not loop:
loop = asyncio.get_event_loop()
return loop.run_in_executor(None, _clear_cache)
def run(loop: asyncio.AbstractEventLoop):
def call():
os.system('node index.js')
with concurrent.futures.ThreadPoolExecutor() as pool:
loop.run_in_executor(None, call)
def Start(self,
listenerQueue: str = None,
listenerQueueSettings: dict = None,
topicExchange: str = None,
topicExchangeSettings: dict = None,
directExchange: str = None,
directExchangeSettings: dict = None,
subscriptions: Union[List[str], str] = None,
confirmDelivery: bool = None,
prefetchSize: int = None,
prefetchCount: int = None,
loop: asyncio.AbstractEventLoop = None,
executor: ThreadPoolExecutor = None):
if executor is None:
executor = self._defaultExecutor
if loop is None:
loop = asyncio.get_event_loop()
if prefetchSize is None:
prefetchSize = self._defaultPrefetchSize
if prefetchCount is None:
prefetchCount = self._defaultPrefetchCount
listenerQueue, listenerQueueSettings = self._AssertListenerQueueIsSet(
listenerQueue, listenerQueueSettings)
with pika.BlockingConnection(self._connParams) as connection:
channelId = str(uuid.uuid1())
channel: pika.adapters.blocking_connection.BlockingChannel = connection.channel(
)
onMessageCallback = functools.partial(
self._OnMessageCallBack,
connection=connection,
channelId=channelId,
listenerQueue=listenerQueue,
topicExchange=topicExchange,
directExchange=directExchange)
self._CreateDefaultRabbitMqSetup(
channel, listenerQueue, listenerQueueSettings, topicExchange,
topicExchangeSettings, directExchange, directExchangeSettings,
subscriptions, confirmDelivery)
channel.basic_qos(prefetch_size=prefetchSize,
prefetch_count=prefetchCount)
channel.basic_consume(listenerQueue, onMessageCallback)
self._openChannels[channelId] = channel
self._openConnections[channelId] = connection
self._logger.info(
f'Starting new consumer channel with id {channelId} '
f'and {len(self.channels)} ongoing channels.')
if not self._connectionHeartbeatIsRunning:
heartbeatFunc = functools.partial(self._ConnectionHeartbeat)
connectionHeartbeatTask = loop.run_in_executor(
executor, heartbeatFunc)
futureConnectionHeartbeatTask = asyncio.ensure_future(
connectionHeartbeatTask, loop=loop)
self._allConsumingTasks += [futureConnectionHeartbeatTask]
try:
channel.start_consuming()
except Exception as exception:
if channelId not in self._forceCloseChannelIds:
self._logger.debug(
f'Consumer with channel id {channelId} '
f'failed due to unknown exception - '
f'{str(type(exception))}: {str(exception)}')
finally:
self._openChannels.pop(channelId)
self._openConnections.pop(channelId)
if channelId in self._forceCloseChannelIds:
self._forceCloseChannelIds.pop(channelId)
else:
raise Exception(
f'Channel {channelId} stopped unexpectedly.')
self._logger.info(f'Closing consumer channel with id {channelId}.')
async def schedule_formatting(
sources: Set[Path],
fast: bool,
write_back: WriteBack,
mode: Mode,
report: "Report",
loop: asyncio.AbstractEventLoop,
executor: Executor,
) -> None:
"""Run formatting of `sources` in parallel using the provided `executor`.
(Use ProcessPoolExecutors for actual parallelism.)
`write_back`, `fast`, and `mode` options are passed to
:func:`format_file_in_place`.
"""
cache: Cache = {}
if write_back not in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
cache = read_cache(mode)
sources, cached = filter_cached(cache, sources)
for src in sorted(cached):
report.done(src, Changed.CACHED)
if not sources:
return
cancelled = []
sources_to_cache = []
lock = None
if write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
# For diff output, we need locks to ensure we don't interleave output
# from different processes.
manager = Manager()
lock = manager.Lock()
tasks = {
asyncio.ensure_future(
loop.run_in_executor(executor, format_file_in_place, src, fast,
mode, write_back, lock)): src
for src in sorted(sources)
}
pending = tasks.keys()
try:
loop.add_signal_handler(signal.SIGINT, cancel, pending)
loop.add_signal_handler(signal.SIGTERM, cancel, pending)
except NotImplementedError:
# There are no good alternatives for these on Windows.
pass
while pending:
done, _ = await asyncio.wait(pending,
return_when=asyncio.FIRST_COMPLETED)
for task in done:
src = tasks.pop(task)
if task.cancelled():
cancelled.append(task)
elif task.exception():
report.failed(src, str(task.exception()))
else:
changed = Changed.YES if task.result() else Changed.NO
# If the file was written back or was successfully checked as
# well-formatted, store this information in the cache.
if write_back is WriteBack.YES or (
write_back is WriteBack.CHECK
and changed is Changed.NO):
sources_to_cache.append(src)
report.done(src, changed)
if cancelled:
if sys.version_info >= (3, 7):
await asyncio.gather(*cancelled, return_exceptions=True)
else:
await asyncio.gather(*cancelled, loop=loop, return_exceptions=True)
if sources_to_cache:
write_cache(cache, sources_to_cache, mode)
async def get_ins_upd_del(
data_name: str,
curr_coro: typing.Coroutine[typing.List[dict], None, None],
prev_coro: typing.Coroutine[typing.List[dict], None, None],
index: typing.Union[str, typing.List[str]],
ignored: typing.Optional[typing.List[str]] = None,
*,
loop: asyncio.AbstractEventLoop,
) -> typing.Dict[str, typing.Union[typing.List[dict], typing.List[str], str]]:
"""
Universal tool for determining inserts/updates/deletes/skips.
Results are determined from the current and previous inputs, which must
be lists of dicts that have the same keys in all dict entries in both
inputs. From those, it figures out what changes are necessary and returns
a dict containing the separate inserts/updates/deletes/skips groups.
* **inserts** are rows that exist in the *current* but not *previous* data.
* **deletes** are rows that exist in the *previous* but not *current* data.
* **updates** are rows that exist in both *current* and *previous* data,
and have changed.
* **skips** are rows that exist in both *current* and *previous* data, and
have not changed.
Args:
current (List[dict]): The current data records to match against.
previous (List[dict]): The previous data records to match against.
index (str): The name of the field in the data to use as an index.
Note:
`current` and `previous` are both *lists of dicts*, where the dicts
must contain exactly the same keys for this code to work correctly.
This is because the columns are matched up against each other for
comparison.
Returns:
Dict[str, Union[List[dict]], List[str], str]: A dictionary containing
an entry for *`'inserts'`*, *`'updates'`*, *`'deletes'`*, *`'skips'`*,
and *`'index'`*. Each entry contains lists of dictionary records that
belong to that particular category of data.
"""
iterable: typing.Tuple[asyncio.Future, ...] = (get_coro_data('current',
curr_coro,
index=index),
get_coro_data('previous',
prev_coro,
index=index))
collected = {}
async for key, value in corelib.task_map(corelib.AsyncIterator, iterable):
collected[key] = value
print(' '.join((
f'{data_name} Collected:',
f'curr ({collected["current"].size});',
f'prev ({collected["previous"].size})',
)))
merged, cols = await loop.run_in_executor( # type: ignore
None, functools.partial(merge_entries, **collected))
current_cols = [col + '_x' for col in cols]
previous_cols = [col + '_y' for col in cols]
iterable = (
loop.run_in_executor( # type: ignore
None,
functools.partial(
get_inserts,
merged,
current_cols,
previous_cols,
)),
loop.run_in_executor( # type: ignore
None,
functools.partial(
get_deletes,
merged,
current_cols,
previous_cols,
)),
get_update_skips(
merged,
current_cols,
previous_cols,
ignored=ignored,
))
results = {}
async for key, value in corelib.task_map(corelib.AsyncIterator, iterable):
if key == 'checks':
async for inner_key, inner_value in value:
results[inner_key] = inner_value
else:
results[key] = value
print(' '.join((
f'{data_name} Processed:',
f'ins ({results["inserts"].__len__()});',
f'upd ({results["updates"].__len__()});',
f'del ({results["deletes"].__len__()});',
f'skp ({results["skips"].__len__()})',
)))
return results
async def runTest(loop: asyncio.AbstractEventLoop) -> None:
loop.run_in_executor(th, taskWorker2)
loop.run_in_executor(th, taskWorker2)
loop.run_in_executor(th, taskWorker2)
loop.run_in_executor(th, taskWorker2)
def wrap_to_thread(loop: asyncio.AbstractEventLoop, func: Callable[..., T],
executor: Executor, *args: Any,
**kwargs: Any) -> Awaitable[T]:
callee = partial(func, *args, **kwargs)
# noinspection PyTypeChecker
return loop.run_in_executor(executor, callee)
def render(self, loop: asyncio.AbstractEventLoop=None):
""":coro: Returns a rendered version of the table."""
loop = loop or asyncio.get_event_loop()
func = functools.partial(self._render)
return loop.run_in_executor(None, func)
