async def test():
broadcaster = ca.curio.client.SharedBroadcaster(log_level=log_level)
await broadcaster.register()
ctx = ca.curio.client.Context(broadcaster, log_level=log_level)
pvs = {}
async with curio.TaskGroup() as connect_task:
async with curio.TaskGroup() as search_task:
for pvname in pv_names:
await search_task.spawn(ctx.search, pvname)
while True:
res = await search_task.next_done()
if res is None:
break
pvname = res.result
await connect_task.spawn(ctx.create_channel, pvname)
while True:
res = await connect_task.next_done()
if res is None:
break
curio_channel = res.result
pvname = curio_channel.channel.name
pvs[pvname] = curio_channel
assert len(pvs) == len(pv_names)
# TODO: can't successfully test as this hammers file creation; this
# will be important to resolve...
await curio.sleep(1)
async def run(self, *, log_pv_names=False):
'Start the server'
self.log.info('Curio server starting up...')
try:
for address in ca.get_beacon_address_list():
sock = ca.bcast_socket(socket)
await sock.connect(address)
interface, _ = sock.getsockname()
self.beacon_socks[address] = (interface, sock)
async def make_socket(interface, port):
return curio.network.tcp_server_socket(interface, port)
self.port, self.tcp_sockets = await self._bind_tcp_sockets_with_consistent_port_number(
make_socket)
async with curio.TaskGroup() as self._task_group:
g = self._task_group
for interface, sock in self.tcp_sockets.items():
# Use run_server instead of tcp_server so we can hand in a
# socket that is already bound, avoiding a race between the
# moment we check for port availability and the moment the
# TCP server binds.
self.log.info("Listening on %s:%d", interface, self.port)
await g.spawn(curio.network.run_server,
sock, self.tcp_handler)
await g.spawn(self._await_stop)
await g.spawn(self.broadcaster_udp_server_loop)
await g.spawn(self.broadcaster_queue_loop)
await g.spawn(self.subscription_queue_loop)
await g.spawn(self.broadcast_beacon_loop)
async_lib = CurioAsyncLayer()
for name, method in self.startup_methods.items():
self.log.debug('Calling startup method %r', name)
await g.spawn(method, async_lib)
self.log.info('Server startup complete.')
if log_pv_names:
self.log.info('PVs available:\n%s', '\n'.join(self.pvdb))
self._log_task_group_exceptions(self._task_group)
except curio.TaskCancelled as ex:
self.log.info('Server task cancelled. Must shut down.')
raise ServerExit() from ex
finally:
self.log.info('Server exiting....')
async_lib = CurioAsyncLayer()
async with curio.TaskGroup() as task_group:
for name, method in self.shutdown_methods.items():
self.log.debug('Calling shutdown method %r', name)
await task_group.spawn(method, async_lib)
self._log_task_group_exceptions(task_group)
for sock in self.tcp_sockets.values():
await sock.close()
for sock in self.udp_socks.values():
await sock.close()
for _interface, sock in self.beacon_socks.values():
await sock.close()
self._task_group = None
async def main():
async with curio.TaskGroup(wait=all) as g:
await g.spawn(
node,
"node0",
"ipc:///tmp/node0.ipc",
["ipc:///tmp/node1.ipc", "ipc:///tmp/node2.ipc"],
daemon=True,
)
await g.spawn(
node,
"node1",
"ipc:///tmp/node1.ipc",
["ipc:///tmp/node2.ipc", "ipc:///tmp/node3.ipc"],
daemon=True,
)
await g.spawn(node,
"node2",
"ipc:///tmp/node2.ipc", ["ipc:///tmp/node3.ipc"],
daemon=True)
await g.spawn(node,
"node3",
"ipc:///tmp/node3.ipc", ["ipc:///tmp/node0.ipc"],
daemon=True)
await curio.sleep(5) # wait % seconde before stop all
await g.join()
async def connect(self, nick, host, port=6667):
"""
Connects to an IRC server specified by host and port with a given nick.
"""
self.nick = nick
self._server = (host, port)
await self._sock.connect(self._server)
await self._send("NICK", self.nick)
await self._send("USER", self.nick, "0", "*", ":" + self.nick)
while True:
line = await self._recv_line()
if line.startswith("PING"):
await self._send(line.replace("PING", "PONG", 1))
continue
msg = self._split_line(line)
if msg.command == "001": # RPL_WELCOME
break
elif msg.command == "433": # ERR_NICKNAMEINUSE
raise ValueError(f"The nickname {self.nick!r} is already used")
async with curio.TaskGroup() as g:
for callback in self._connection_callbacks:
await g.spawn(callback(self))
await g.join()
async def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument(
'mode',
help='Whether the socket should "listen" or "dial"',
choices=['listen', 'dial'],
)
p.add_argument(
'addr',
help='Address to listen or dial; e.g. tcp://127.0.0.1:13134',
)
args = p.parse_args()
with pynng.Pair1(polyamorous=True) as sock:
async with curio.TaskGroup(wait=any) as g:
if args.mode == 'listen':
# the listening socket can get dialled by any number of dialers.
# add a couple callbacks to see when the socket is receiving
# connections.
def pre_connect_cb(pipe):
addr = str(pipe.remote_address)
print('~~~~got connection from {}'.format(addr))
def post_remove_cb(pipe):
addr = str(pipe.remote_address)
print('~~~~goodbye for now from {}'.format(addr))
sock.add_pre_pipe_connect_cb(pre_connect_cb)
sock.add_post_pipe_remove_cb(post_remove_cb)
sock.listen(args.addr)
else:
sock.dial(args.addr)
await g.spawn(recv_eternally, sock)
await g.spawn(send_eternally, sock)
async def run(strategy: strat, exchange: exch, datasource,
strategy_params: str, datasource_path: str):
"""TODO: Add description."""
logger.info("Entering backtest routine.")
# Get curio queues
transaction_queue = curio.Queue()
ticker_queue = curio.Queue()
# Set up objects
data_source_object = datasource(datasource_path, ticker_queue)
exchange_object = exchange(transaction_queue)
strategy_object = strategy(transaction_queue, ticker_queue)
await strategy_object.configure(strategy_params)
# Run the tasks
async with curio.TaskGroup() as g:
await g.spawn(exchange_object.run)
await g.spawn(strategy_object.run)
datasrce_task = await g.spawn(data_source_object.run)
await datasrce_task.join()
await g.cancel_remaining()
async for task in g:
logging.info(str(task) + 'completed.' + str(task.result))
# Clean exit
logger.info("Backtest complete, exiting cleanly.")
async def connect(self, nick, host, port=6667):
"""
Connects to an IRC server specified by host and port with a given nick.
"""
self.nick = nick
self._server = (host, port)
await self._sock.connect(self._server)
await self._send("NICK", self.nick)
await self._send("USER", self.nick, "0", "*", ":" + self.nick)
while True:
line = await self._recv_line()
if line.startswith("PING"):
await self._send(line.replace("PING", "PONG", 1))
continue
msg = self._split_line(line)
if msg.command == "001":
break
async with curio.TaskGroup() as g:
for callback in self._connection_callbacks:
await g.spawn(callback(self))
await g.join()
async def amain(hash_func, a, b=None, num_workers=None):
work_queue = curio.Queue()
output_lock = curio.Lock()
output = {'a': {}}
if b:
output['b'] = {}
hashers = curio.TaskGroup(name='hashers')
if not num_workers:
try:
num_workers = multiprocessing.cpu_count() + 1
except NotImplementedError:
num_workers = 2
for _ in range(num_workers):
await hashers.spawn(hash_file_worker, work_queue, output, output_lock,
hash_func)
for fullpath in walk_all_files(a):
await work_queue.put(('a', fullpath))
if b:
for fullpath in walk_all_files(b):
await work_queue.put(('b', fullpath))
await work_queue.join()
await hashers.cancel_remaining()
return output
async def spawn_drakvuf(profile, domid):
global proc
fmt = 'json'
#fmt = 'kv'
deactivated = ' '.join('-x {}'.format(p) for p in deactivated_plugins)
cmd = ['drakvuf', '-r', profile, '-d', domid, '-o', fmt, deactivated]
#cmd.append('-v')
logging.info("Running %s", ' '.join(cmd))
proc = curio.subprocess.Popen(cmd,
stdout=curio.subprocess.PIPE,
stderr=curio.subprocess.PIPE)
# kick off the stdout, stderr handlers and allow them to run forever
children = list()
async with curio.TaskGroup() as f:
children.append(await f.spawn(read_nop, proc.stderr))
children.append(await f.spawn(handle_output, proc.stdout))
await curio.sleep(100)
if proc.poll():
logging.info("Detected that process has terminated")
[x.cancel() for x in children]
rc = proc.wait()
if rc:
logging.warn("Returned %d, stderr: %s", rc, proc.stderr.read())
proc.terminate()
async def scan(cur, dirs):
'Synchronize the fingerprint database with the file system'
# Fetch already processed paths
cur.execute(SELECT_SQL)
PATHS.update(row[0] for row in cur.fetchall())
# Remove vanished paths from DB
for path in PATHS:
if not os.path.isfile(os.path.join(BASEDIR, path)):
print('-', path)
cur.execute(DELETE_SQL, (path, ))
# Add fingerprints for new files at full throttle
cpus = os.cpu_count()
running = 0
async with curio.TaskGroup() as tasks:
for path in walk(dirs):
trimmed = PREFIX.sub('', path)
assert len(trimmed) < 255
if trimmed not in PATHS:
if running == cpus:
# Limit number of subprocesses to available CPUs
t = await tasks.next_done()
running -= 1
cur.execute(INSERT_SQL, t.result)
running += 1
await tasks.spawn(fingerprint, path)
# Collect remaining results
async for t in tasks:
cur.execute(INSERT_SQL, t.result)
async def kid():
"""
The kid has his/her own task.
Add the curio monitoring framework.
The kid captures the cancel request, complains, then accedes.
6. The kid invites a few friends over to help.
Note - also using the curio sleep timer instead of standard.
:return:
"""
print('Building the Millennium Falcon in Minecraft')
async with curio.TaskGroup() as f:
await f.spawn(friend, 'Max')
await f.spawn(friend, 'Lillian')
await f.spawn(friend, 'Thomas')
try:
await curio.sleep(1000)
except curio.CancelledError as xcp:
print('Fine. Saving my work.')
raise
async def main(): protocol = ProtocolPreGame(rules, gen) server = Server(protocol) async with curio.TaskGroup() as g: await g.spawn(tcp_server(server, '0.0.0.0', 1337)) await g.spawn(ws_server(server, '0.0.0.0', 8080)) await g.spawn(server.run())
async def run(self):
async with TermboxAsync() as tb:
self.tb = tb
async with curio.TaskGroup() as g:
await g.spawn(self._run_net())
await g.spawn(self._run_input())
await g.spawn(self._run_output())
async def main(bind_addr, *server_coros):
async with curio.TaskGroup() as g:
for server_coro in server_coros:
await g.spawn(server_coro)
task = await g.spawn(make_request, bind_addr)
await task.join()
await g.cancel_remaining()
async def go(self):
'Crawler main entry'
start = time.time()
print('Fetching %s, max. depth is %d' % (self.base, self.args.depth))
self.robots_txt.read()
startpage = Page(self.base, 0)
if not self.can_fetch(startpage):
print('URL "%s" is blocked by robots.txt' % startpage.url)
return
# Start crawling
self.queue.append(startpage)
try:
async with curio.TaskGroup() as tasks:
self.tasks = tasks
await tasks.spawn(self.do_page())
except KeyboardInterrupt:
print('Aborted')
print("Finished after %.1f seconds" % (time.time() - start))
async def main(coro, *server_coros):
async with curio.TaskGroup() as g:
for server_coro in server_coros:
await g.spawn(server_coro)
task = await g.spawn(coro)
await task.join()
await g.cancel_remaining()
async def watcher(self):
''' -- monitor a directory
watch a directory for files matching pattern
spawn follower tasks when new files are found
'''
log.print(term.pink("begin watching for new files..."))
async with curio.TaskGroup() as followergroup:
followers = dict()
prev_dirstate = set()
while True:
### find new files todo: if pattern is a tuple, its elements are patterns
dirstate = set(self.target.glob(self.pattern)) #todo: async
new_files = dirstate - prev_dirstate
prev_dirstate = dirstate
### create new followers
if new_files:
for file in sorted(new_files):
follower = await followergroup.spawn(
self.follower, file)
followers[file] = follower
### status report
log.print(
term.pink('scanned count:'), f' \n',
pformat({
file.name: count
for file, count in self._scannedcount.items()
}))
await curio.sleep(self.dir_interval)
async def relay(self, via_client):
try:
async with curio.TaskGroup() as g:
await g.spawn(self._relay(via_client))
await g.spawn(self._reverse_relay(via_client))
await g.next_done(cancel_remaining=True)
except curio.TaskGroupError as e:
gvars.logger.debug(f"group error: {e}")
def __init__(self, id, unit_type, map, player):
self.id = id
self.unit_type = unit_type
self.map = map
self.player = player
self._task_group = curio.TaskGroup()
self._semaphore = curio.Semaphore() # tasks will acquire the semaphore in the same order as they call acquire().
self._action_tasks = {}
async def parent():
async with curio.TaskGroup(wait=any) as g:
await g.spawn(kid, 37, 42)
await g.spawn(countdown, 10)
if g.result is None:
print("Why didn't you finish?")
else:
print("Result:", g.result)
async def main():
with pynng.Pub0() as pub:
n0 = await server(pub)
async with curio.TaskGroup(wait=all) as g:
await g.spawn(client, "client0", 2)
await g.spawn(client, "client1", 3)
await g.spawn(client, "client2", 4)
await n0.cancel()
async def main_coro():
# await curio.spawn(print_memory_growth_statistics(), daemon=True)
async with purerpc.insecure_channel("localhost", 50055) as channel:
for i in range(100):
start = time.time()
async with curio.TaskGroup() as task_group:
for i in range(100):
await task_group.spawn(worker(channel))
print("RPS: {}".format(10000 / (time.time() - start)))
async def test_join_controller_propagates_cancellation(self):
c = MagicMock(**{'join.side_effect': curio.Event().wait})
self.mgr.stop_controller = MagicMock(side_effect=dummy_coro)
async with curio.TaskGroup() as g:
task = await g.spawn(self.mgr._join_controller, c)
while not c.join.called:
await curio.sleep(0)
await task.cancel()
self.mgr.stop_controller.assert_called_once_with(c)
self.mgr.log.exception.assert_not_called()
async def test_watch_signals_signame_missing(self, sq, s):
s.side_effect = ValueError
sq.return_value = FakeSignalQueue()
async with curio.TaskGroup() as g:
await g.spawn(self.mgr._watch_signals)
await sq.return_value.put(15)
async for t in g:
t.result # propagate exceptions
sq.assert_called_once_with(signal.SIGTERM, signal.SIGINT)
self.mgr.log.info.assert_called_once_with('caught signal 15')
async def main():
with pynng.Surveyor0() as surveyor:
n0 = await server(surveyor)
async with curio.TaskGroup(wait=all) as g:
await g.spawn(client, "client0", 3)
await g.spawn(client, "client1", 3)
await g.spawn(client, "client2", 4))
await n0.join()
def __init__(self, circuit, client, context):
super().__init__(circuit, client, context)
self.QueueFull = QueueFull
self.command_queue = QueueWithFullError(ca.MAX_COMMAND_BACKLOG)
self.new_command_condition = curio.Condition()
self.pending_tasks = curio.TaskGroup()
self.events_on = curio.Event()
self.subscription_queue = QueueWithFullError(
ca.MAX_TOTAL_SUBSCRIPTION_BACKLOG)
self.write_event = Event()
async def test_session_stateful(self):
from asks.sessions import Session
s = Session(self.httpbin.url, persist_cookies=True)
async with curio.TaskGroup() as g:
await g.spawn(base_tests.hsession_t_stateful(s))
domain = f'{self.httpbin.host}:{self.httpbin.port}'
cookies = s._cookie_tracker_obj.domain_dict[domain]
assert len(cookies) == 1
assert cookies[0].name == 'cow'
assert cookies[0].value == 'moo'
async def mainloop(self):
"""
Handles keeping the connection alive and event handlers.
"""
while self.running:
line = await self._recv_line()
if not line:
continue
if line.startswith("PING"):
await self._send(line.replace("PING", "PONG", 1))
continue
msg = self._split_line(line)
# The following block handles self.channel_users
if msg.command == "353": # RPL_NAMREPLY
channel = msg.args[2]
nicks = [nick.lstrip("@+")
for nick in msg.args[3].split()]
self.channel_users.setdefault(channel, set()).update(nicks)
elif msg.command == "JOIN":
channel = msg.args[0]
nick = msg.sender.nick
self.channel_users.setdefault(channel, set()).add(nick)
elif msg.command == "PART":
channel = msg.args[0]
nick = msg.sender.nick
self.channel_users.setdefault(channel, set()).discard(nick)
callbacks = self._message_callbacks.get(msg.command, ())
async with curio.TaskGroup() as g:
spawn_callbacks = True
if msg.command == "PRIVMSG":
command, *args = msg.args[1].strip().split(" ")
cmd_callbacks = self._command_callbacks.get(command, ())
for callback, arg_amount in cmd_callbacks:
if arg_amount == NO_SPLITTING:
spawn_callbacks = False
coro = callback(self, msg.sender, msg.args[0],
" ".join(args))
await g.spawn(coro)
elif arg_amount == ANY_ARGUMENTS or \
len(args) == arg_amount:
spawn_callbacks = False
coro = callback(self, msg.sender, msg.args[0],
*args)
await g.spawn(coro)
if ALWAYS_CALLBACK_PRIVMSG or spawn_callbacks:
# Sometimes we don't want to spawn the PRIVMSG callbacks if
# this is a command.
for callback in callbacks:
await g.spawn(callback(self, msg.sender, *msg.args))
await g.join()
async def create_many_outer():
async with curio.TaskGroup() as task:
for name in names:
await task.spawn(connect_one, name)
while True:
res = await task.next_done()
if res is None:
break
name, chan = res.result
channels[name] = chan
def __init__(self, circuit, client, context):
self.connected = True
self.circuit = circuit # a caproto.VirtualCircuit
self.client = client
self.context = context
self.client_hostname = None
self.client_username = None
self.command_queue = curio.Queue()
self.new_command_condition = curio.Condition()
self.pending_tasks = curio.TaskGroup()
self.subscriptions = defaultdict(deque)