async def test_worker():
import logging
logger = logging.getLogger('test_connection')
log_handler = logging.StreamHandler()
log_format = logging.Formatter('%(levelname)s %(asctime)s %(message)s')
log_handler.setFormatter(log_format)
logger.addHandler(log_handler)
logger.setLevel(logging.DEBUG)
getter_queue = curio.Queue()
putter_queue = curio.Queue()
wp = MagneWorkerPool(
worker_nums=2,
worker_timeout=15,
getter_queue=getter_queue,
putter_queue=putter_queue,
task_module_path='magne.process_worker.demo_task',
logger=logger,
)
stask = await curio.spawn(wp.start)
await stask.join()
await curio.sleep(5)
body = {
'channel_number': 1,
'delivery_tag': 1,
'consumer_tag': 1,
'exchange': 1,
'routing_key': 1,
'data': json.dumps({
'func': 'sleep',
'args': [10]
}),
}
await wp.getter_queue.put(json.dumps(body))
await curio.sleep(5)
print(wp.idle_workers, wp.workers, wp.busy_workers)
await curio.sleep(6)
print(wp.putter_queue)
print(wp.idle_workers, wp.workers, wp.busy_workers)
os.kill(wp.idle_workers[0], signal.SIGKILL)
async with curio.SignalQueue(signal.SIGCHLD) as p:
await p.get()
wp.reap_workers()
print('reap done')
print(wp.idle_workers, wp.workers, wp.busy_workers)
print('range send msg...')
for i in range(3):
new_body = {
'delivery_tag': i + 2,
'data': json.dumps({
'func': 'sleep',
'args': [15 * (i + 1) + 1]
})
}
await wp.getter_queue.put(json.dumps(new_body))
await curio.sleep(3)
print(wp.idle_workers, wp.workers, wp.busy_workers, wp.getter_queue)
print('closing...')
await wp.close(warm=False)
print(wp.putter_queue)
return
async def start(self):
self.logger.info('Queue pid: %s' % os.getpid())
ack_queue = curio.Queue()
amqp_queue = curio.Queue()
self.con = SpellsConnection(ack_queue,
amqp_queue,
self.queues,
self.amqp_url,
self.qos,
log_level=self.log_level)
self.spawning_pool = LarvaePool(self.timeout,
self.task_modue,
ack_queue,
amqp_queue,
log_level=self.log_level)
con_run_task = await curio.spawn(self.con.run)
await con_run_task.join()
pool_task = await curio.spawn(self.spawning_pool.run)
await pool_task.join()
signal_task = await curio.spawn(self.watch_signal)
await signal_task.join()
return
def __init__(self, socket: "GRPCSocket", stream_id: int, client_side: bool,
incoming_buffer_size=10, outgoing_buffer_size=10):
self._socket = socket
self._stream_id = stream_id
self._client_side = client_side
self._incoming_events = curio.Queue(incoming_buffer_size)
self._outgoing_messages = curio.Queue(outgoing_buffer_size)
async def run(strategy: strat, exchange: exch, datasource,
strategy_params: str, datasource_path: str):
"""TODO: Add description."""
logger.info("Entering backtest routine.")
# Get curio queues
transaction_queue = curio.Queue()
ticker_queue = curio.Queue()
# Set up objects
data_source_object = datasource(datasource_path, ticker_queue)
exchange_object = exchange(transaction_queue)
strategy_object = strategy(transaction_queue, ticker_queue)
await strategy_object.configure(strategy_params)
# Run the tasks
async with curio.TaskGroup() as g:
await g.spawn(exchange_object.run)
await g.spawn(strategy_object.run)
datasrce_task = await g.spawn(data_source_object.run)
await datasrce_task.join()
await g.cancel_remaining()
async for task in g:
logging.info(str(task) + 'completed.' + str(task.result))
# Clean exit
logger.info("Backtest complete, exiting cleanly.")
def __init__(self, address):
self.address = address
self.numservers = len(config.SERVERS)
self._outgoing = {n: curio.Queue() for n in config.SERVERS}
self._socks = {n: None for n in config.SERVERS} # The other servers
self.server_sock = None
self._msgqueue = curio.Queue() # Incoming messages
self.log = logging.getLogger(f'net.{address}')
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
max_queue_size = 10
self._write_queue = curio.Queue(
maxsize=max_queue_size) # only nmea-datagrams
self._read_queue = curio.Queue(
maxsize=max_queue_size) # only nmea-datagrams
self._write_task_handle = None
self._read_task_handle = None
async def broadcast_client(self, client: curio.io.Socket,
addr: Tuple[str, int]) -> None:
client_name = hash(
client) # i guess getpeername() doesn't work with AF_UNIX
print(f'Unix socket connection: {client_name}', file=sys.stderr)
stream = client.as_stream()
bcast_q = curio.Queue()
self.register_subscriber(bcast_q, f'broadcast_client:{client_name}')
n_readings = 0
last_report = now()
try:
while True:
block = await bcast_q.get()
n_readings += len(block)
delta = block[-1][0] - last_report
if (delta >= 5.0):
print(
f'Broadcasting {n_readings / delta} readings/second.',
file=sys.stderr)
last_report = now()
n_readings = 0
string = json.dumps(block) + '\n'
await curio.timeout_after(60, stream.write,
string.encode('ascii'))
except curio.CancelledError:
await stream.write(
json.dumps([(0, 'END_STREAM', -1)]).encode('ascii'))
raise
except (BrokenPipeError, curio.TaskTimeout):
print(f'Unix socket closed: {client_name}', file=sys.stderr)
finally:
self.unsubscribe(bcast_q)
async def amain(hash_func, a, b=None, num_workers=None):
work_queue = curio.Queue()
output_lock = curio.Lock()
output = {'a': {}}
if b:
output['b'] = {}
hashers = curio.TaskGroup(name='hashers')
if not num_workers:
try:
num_workers = multiprocessing.cpu_count() + 1
except NotImplementedError:
num_workers = 2
for _ in range(num_workers):
await hashers.spawn(hash_file_worker, work_queue, output, output_lock,
hash_func)
for fullpath in walk_all_files(a):
await work_queue.put(('a', fullpath))
if b:
for fullpath in walk_all_files(b):
await work_queue.put(('b', fullpath))
await work_queue.join()
await hashers.cancel_remaining()
return output
async def task(self, error_handler=None):
try:
msg_q = curio.Queue()
self.subscription.subscribe(msg_q, self.name)
while True:
msg = await msg_q.get()
if msg is None:
await msg_q.task_done()
break
for callback, args, kwargs in self.handlers[type(msg)]:
if inspect.iscoroutinefunction(callback):
await callback(msg, *args, **kwargs)
else:
callback(msg, *args, **kwargs)
for callback, args, kwargs in self.handlers[AllMessages]:
if inspect.iscoroutinefunction(callback):
await callback(msg, *args, **kwargs)
else:
callback(msg, *args, **kwargs)
await msg_q.task_done()
except curio.CancelledError:
raise
except Exception as e:
if error_handler is not None:
error_handler(e)
else:
raise
else:
self.subscription.unsubscribe(msg_q)
async def start(self):
print('master %s start' % os.getpid())
# 下面都是建立amqp连接
self.master_queue = curio.Queue()
self.con = Connection(self.amqp_url, self.worker_nums,
self.master_queue)
await self.con.connect()
channel = await self.con.open_channel()
exchange = await self.con.declare_exchange(channel.channel_number,
'curio_amqp_exchange')
queue = await self.con.declare_queue(channel.channel_number,
'curio_amqp_queue')
await self.con.bind(channel.channel_number,
exchange.name,
queue.name,
routing_key='curio_amqp')
await self.con.update_qos(channel.channel_number)
# 构建worker pool
self.pool = WorkerPool(self.worker_nums, self.worker_timeout)
# spawn接收amqp消息的任务和分发msg到worker的任务
consume_task = await curio.spawn(
self.con.start_consume(channel.channel_number, queue.name))
fetch_task = await curio.spawn(self.fetch_amqp_msg())
# wait for signal
await self.wait_signal(consume_task, fetch_task)
return
def __init__(self, host='localhost', port='6379'):
self.host = host
self.port = port
self.should_spawn = True
self._evs = curio.Queue()
self._ev_seq = 0
self._res = {}
return
def __init__(self, pvdb, interfaces=None):
super().__init__(pvdb, interfaces)
self._task_group = None
# _stop_queue is used like a threading.Event, allowing a thread to stop
# the Context in :meth:`.stop`.
self._stop_queue = curio.UniversalQueue()
self.command_bundle_queue = curio.Queue()
self.subscription_queue = curio.UniversalQueue()
async def start(self):
self.logger.info('starting in pid: %s' % (os.getpid()))
amqp_task_queue = curio.Queue()
ack_queue = curio.Queue()
self.con = Connection(ack_queue, amqp_task_queue, list(self._tasks.keys()), self.amqp_url, self.qos, self.log_level)
con_run_task = await curio.spawn(self.con.run)
await con_run_task.join()
self.worker_pool = ThreadWorkerPool(self.thread_nums, self.worker_timeout, self.task_module,
ack_queue, amqp_task_queue, self.log_level
)
pool_task = await curio.spawn(self.worker_pool.run)
await pool_task.join()
signal_task = await curio.spawn(self.watch_signal)
await signal_task.join()
return
def __init__(self, logger, host, port, rules, protocol):
self.logger = logger
self.host = host
self.port = port
self.protocol = protocol
self.queue = curio.Queue()
self.rules = rules
self.tb = None
self.map = None
self.ui = Ui(self)
def __init__(self, players, width, height): self.width = width self.height = height self.cells = [Cell(None) for i in range(width * height)] self.players = players # already a util.IdList(Player) self.units = util.IdList(Unit) self.actions = util.IdList(Action) self.events = curio.Queue()
def __init__(self, app=None):
super().__init__(app)
if app:
self.app = app
self.queue = curio.Queue()
if sys.platform == 'win32':
self.ppe = ProcessPoolExecutor()
def __init__(self, circuit, client, context):
self.connected = True
self.circuit = circuit # a caproto.VirtualCircuit
self.client = client
self.context = context
self.client_hostname = None
self.client_username = None
self.command_queue = curio.Queue()
self.new_command_condition = curio.Condition()
self.pending_tasks = curio.TaskGroup()
self.subscriptions = defaultdict(deque)
def __init__(self, circuit):
self.circuit = circuit # a caproto.VirtualCircuit
self.channels = {} # map cid to Channel
self.ioids = {} # map ioid to Channel
self.ioid_data = {} # map ioid to server response
self.subscriptionids = {} # map subscriptionid to Channel
self.connected = True
self.socket = None
self.command_queue = curio.Queue()
self.new_command_condition = curio.Condition()
self._socket_lock = curio.Lock()
def __init__(self):
self.broadcaster = ca.Broadcaster(our_role=ca.CLIENT)
self.log = self.broadcaster.log
self.command_bundle_queue = curio.Queue()
self.broadcaster_command_condition = curio.Condition()
# UDP socket broadcasting to CA servers
self.udp_sock = ca.bcast_socket(socket)
self.registered = False # refers to RepeaterRegisterRequest
self.loop_ready_event = curio.Event()
self.unanswered_searches = {} # map search id (cid) to name
self.search_results = {} # map name to address
async def test_connection():
# for test
import logging
import os
logger = logging.getLogger('test_connection')
log_handler = logging.StreamHandler()
log_format = logging.Formatter(
'%(levelname)s %(asctime)s %(pathname)s %(lineno)d %(message)s')
log_handler.setFormatter(log_format)
logger.addHandler(log_handler)
logger.setLevel(logging.DEBUG)
getter_queue = curio.Queue()
putter_queue = curio.Queue()
c = MagneConnection(['tc1', 'tc2'], logger, getter_queue, putter_queue)
logger.info('connection pid: %s' % os.getpid())
await c.connect()
await c.run()
await curio.sleep(10)
await c.pre_close()
await c.close()
return
async def database_writer(self) -> None:
try:
write_q = curio.Queue()
self.register_subscriber(write_q, 'database_writer')
while True:
block = await write_q.get()
self.sensor_db.insert_readings(block)
await write_q.task_done()
except curio.CancelledError:
raise
finally:
self.unsubscribe(write_q)
def __init__(self, age, task_module, timeout, task_queue, ack_queue, log_level):
self.alive = True
# 发送task
self.task_queue = task_queue
# 发送task的结果
self.task_module = task_module
self.result_queue = curio.Queue()
self.ack_queue = ack_queue
self.timeout = timeout
self.done_ev = curio.Event()
self.age = age
self.name = 'Magne-ThreadWorker_%s' % age
self.logger = get_component_log(self.name, log_level)
return
async def main():
config = ["Alice", "Bob"]
q = curio.Queue()
alice = RaftNode("Alice", config, q)
alice.status = "Leader"
bob = RaftNode("Bob", config, q)
client = RaftClient(config, q)
tasks = []
nodes = [client, bob, alice]
for n in nodes:
t = await curio.spawn(n)
tasks.append(t)
for t in tasks:
await t.join()
async def subscribe(self, *args, **kwargs):
"Start a new subscription and spawn an async task to receive readings."
command = self.channel.subscribe(*args, **kwargs)
# Stash the subscriptionid to match the response to the request.
queue = curio.Queue()
self.circuit.subscriptionids[command.subscriptionid] = queue
await self.circuit.send(command)
async def _queue_loop():
command = await queue.get()
self.process_subscription(command)
task = await curio.spawn(_queue_loop, daemon=True)
self.monitoring_tasks[command.subscriptionid] = task
return command.subscriptionid
def __init__(self):
self.broadcaster = ca.Broadcaster(our_role=ca.CLIENT)
self.log = self.broadcaster.log
self.command_bundle_queue = curio.Queue()
self.broadcaster_command_condition = curio.Condition()
# UDP socket broadcasting to CA servers
self.udp_sock = ca.bcast_socket(socket)
self.broadcaster.our_address = safe_getsockname(self.udp_sock)
self.registered = False # refers to RepeaterRegisterRequest
self.loop_ready_event = curio.Event()
self.unanswered_searches = {} # map search id (cid) to name
self.search_results = {} # map name to address
self.environ = ca.get_environment_variables()
self.ca_server_port = self.environ['EPICS_CA_SERVER_PORT']
def __init__(self, host, port, pvdb, *, log_level='ERROR'):
self.host = host
self.port = port
self.pvdb = pvdb
self.circuits = set()
self.log_level = log_level
self.broadcaster = ca.Broadcaster(our_role=ca.SERVER)
self.broadcaster.log.setLevel(self.log_level)
self.command_bundle_queue = curio.Queue()
self.subscriptions = defaultdict(deque)
self.subscription_queue = curio.UniversalQueue()
self.beacon_count = 0
self.environ = get_environment_variables()
ignore_addresses = self.environ['EPICS_CAS_IGNORE_ADDR_LIST']
self.ignore_addresses = ignore_addresses.split(' ')
async def amain():
'''An asynchronous version of main function.'''
# Parse arguments
parser = argparse.ArgumentParser(
description="AWA - Adaptive Weighted Aggregation")
parser.add_argument('--config',
'-c',
default='config.json',
type=str,
help='Configuration file in json format')
args = parser.parse_args()
# Load configuration file
with open(args.config) as f:
conf = json.load(f)
# Setup logger
level = conf.get('verbose', logging.INFO)
logger = logging.getLogger(__name__)
logger.setLevel(level)
handler = logging.StreamHandler()
handler.setLevel(level)
logger.addHandler(handler)
# Setup parameters to be optimized
params = conf['parameters']
obj = conf.get('objectives', 1)
use_param_names = conf.get('use_parameter_names', False)
# Setup cache file
cache_path = conf.get('cache', 'solutions.csv')
queue = curio.Queue()
problem = Problem(params, obj, cache_path, use_param_names, queue, logger)
# Setup workers
await start_evaluators(conf['workers'], queue, logger)
# Run GA loop
conf_opt = conf.get('optimizer', {})
import awa.awa as opt
await opt.fmin(problem, conf_opt, logger)
async def follower(self, file: Path):
''' -- monitor a file
spawn new instances of the registered event handlers
tail the file, whenever new lines come in,
use queues to dispatch them to the prompter tasks for each handler.
'''
log.print(term.cyan('new file:'), f' {file.name}')
async with curio.TaskGroup() as handlergroup:
handlers = dict()
queues = dict()
### create handler tasks
for trigger, handler in self._events.items():
log.print(term.dcyan(f'spawn line handler for {file.name}:'),
f' {trigger}')
queue = curio.Queue()
queues[trigger] = queue
prompter = self.Prompter(queue, trigger, file)
### supported parameters for event handlers:
kwargs = dict()
kwargs['prompter'] = prompter
handlers[trigger] = await handlergroup.spawn(handler, kwargs)
### process the file
async with curio.aopen(file, 'r') as fstream:
### fast-forward through already-scanned lines todo
self._scannedcount.setdefault(file, 0)
### follow the file and push new lines to prompter queues
while True:
line = await fstream.readline()
if line:
log.print(term.dpink(file.name), ' put ',
term.dyellow(line.strip()))
self._scannedcount[file] += 1
for trigger, queue in queues.items():
await queue.put((line, self._scannedcount[file]))
async def call(self, method, *args, **kwargs):
self.__waiting_calls.append(method)
id = self.__call_id
self.__call_id += 1
q = curio.Queue()
self.__wait_for_response[id] = q
await self.__send({
"method": method,
"params": args or kwargs,
"id": id
})
# await for answer in the answer queue
res = await q.get()
del self.__wait_for_response[id]
self.__waiting_calls.remove(method)
error = res.get("error")
if error:
raise Exception(error)
return res.get("result")
def __init__(self,
sensor_db: SensorDB,
broadcast_socket: str = DEFAULT_SOCKET,
database_write_interval: float = 0.5,
write_results: bool = False,
report_status: bool = False):
"""
Args:
sensor_db (SensorDB): Database to write to.
polling_interval (float): Global sensor min polling interval.
database_write_interval (float): Data write op min interval.
websocket_host (str): Host address for websocket broadcast; if None, don't broadcast.
websocket_port (int): Host port for websocket broadcast.
Returns:
"""
try:
os.unlink(broadcast_socket)
except OSError:
if os.path.exists(broadcast_socket):
raise
self.readings = curio.Queue()
self.sensor_db = sensor_db
self.write_results = write_results
self.database_write_interval = database_write_interval
self.broadcast_socket = broadcast_socket
self.subscribers = set()
self.subscriber_names = {}
self.reporters = set()
self.report_status = report_status
