def run(self):
with ThreadPool(thread_name_prefix="HandlePool-%d" %
self._counter()) as handle_pool:
with multiprocessing_connection.Listener(
self.address, authkey=self.authkey) as listener:
c_recv, c_send = multiprocessing_connection.Pipe(False)
def after_delete_handle(
t: Tuple[multiprocessing_connection.Connection, bool]):
k, v = t
self.logger.debug("close timeout conn %s" % k)
c_send.send_bytes(b'ok')
k.close()
conn_lru_dict = LRUCache(
size=1024,
timeout=CONN_LRU_TIMEOUT,
after_delete_handle=after_delete_handle)
handle_pool.spawn(self._process, conn_lru_dict, handle_pool,
c_recv, c_send)
while True:
try:
conn = listener.accept()
self.logger.debug("get a new conn %s" % conn)
conn_lru_dict[conn] = True
c_send.send_bytes(b'ok')
except:
logging.exception("error")
def start(self):
managers.SyncManager.start(self)
l = connection.Listener(family='AF_INET', authkey=self._authkey)
for i, host in enumerate(self._hostlist):
host._start_manager(i, self._authkey, l.address, self._files)
for host in self._hostlist:
if host.hostname != 'localhost':
conn = l.accept()
i, address, cpus = conn.recv()
conn.close()
other_host = self._hostlist[i]
other_host.manager = HostManager.from_address(
address, self._authkey)
other_host.slots = other_host.slots or cpus
other_host.Process = other_host.manager.Process
else:
host.slots = host.slots or slot_count
host.Process = LocalProcess
self._slotlist = [
Slot(host) for host in self._hostlist for i in range(host.slots)
]
self._slot_iterator = itertools.cycle(self._slotlist)
self._base_shutdown = self.shutdown
del self.shutdown
def serve_forever(self):
sock = connection.Listener(address=self.address, authkey=self.authkey)
while True:
client = sock.accept()
t = threading.Thread(target=self.handler.handle_connection, args=(client,))
t.daemon = True
t.start()
def __ensure_connection(self):
if self.__failed_com_count >= 5:
raise CommunicationError(
"Failed to connect to AvoPlot daemon process."
" Final attempt to reconnect failed with "
"error: \'%s\'" % (str(self.__failed_com_exception)))
if self.__connection is None:
#find a free port to open the connection on (starting at 25000 and
#working upwards)
port = 25000
while True:
try:
self.listener = connection.Listener(
('localhost', port), authkey=self.__auth_key)
break
except:
port += 1
#start a new AvoPlot instance in a separate process
subprocess.Popen([
sys.executable, '-c',
'from avoplot import pyplot;'
' pyplot._AvoPlotRemoteInstance(%s, \"%s\")' %
(port, self.__auth_key)
],
close_fds=True)
#connect to the avoplot process
self.__connection = self.listener.accept()
def serve(self):
signal.signal(signal.SIGINT, self._signal_handler)
signal.signal(signal.SIGTERM, self._signal_handler)
self.start_time = time.time()
mpl.log("node server started at " + self.addr[0] + ":" + str(self.addr[1]), self.log_addr)
with mpc.Listener(self.addr, "AF_INET", self.conn_buffer_size, None) as lst:
while True:
conn = lst.accept()
threading.Thread(target=self._handle_connection, args=(conn, lst.last_accepted), daemon=True).start()
def connect(self):
"""Method that connects with the external entity.
Typically called by :class:`~WASimulationManager`, but okay to call outside. Will only connect to one client at
the hostname and port provided to the constructor.
If this bridge is a server, a ``multiprocessing.Connection.Listener`` object is used. Otherwise, a ``multiprocessing.Connection.Client`` object is used.
See the `Listener <https://docs.python.org/3/library/multiprocessing.html#multiprocessing.connection.Listener>`_
or `Client <https://docs.python.org/3/library/multiprocessing.html#multiprocessing.connection.Client>`_ docs to see possible errors that may be raised.
"""
if self._server:
self._listener = mp.Listener(self._address)
self._connection = self._listener.accept()
if self._connection is None:
raise RuntimeError("Bridge failed to connect.")
# As the server, it is responsible for _sending_ the system information to the client
# The client should then use these values
msg = {
"type": "init",
"data": {
"step_size": self._system.step_size,
"render_step_size": self._system.render_step_size,
"end_time": self._system.end_time,
}
}
self._connection.send(msg)
# Receive an acknowledgement that we got the message
if not self._connection.poll(self._timeout):
raise RuntimeError(
"Failed to receive acknowledgement from client.")
ack = self._connection.recv()
if ack != 1:
raise RuntimeError("Acknowledgement is corrupted.")
else:
self._connection = mp.Client(self._address)
# As the client, it is responsible for _receiving_ the system information from the server
msg = self._connection.recv()
# Ack
self._connection.send(1)
# Message validation
assert msg["type"] == "init"
assert "data" in msg
assert "step_size" in msg["data"]
assert "render_step_size" in msg["data"]
assert "end_time" in msg["data"]
# Update the system object
self._system.step_size = msg["data"]["step_size"]
self._system.render_step_size = msg["data"]["render_step_size"]
self._system.end_time = msg["data"]["end_time"]
def open_window_listener(root):
listener = connection.Listener(address, authkey=b"limits auth")
while True:
conn = listener.accept()
message = conn.recv()
if message == b"open_window":
root.deiconify()
conn.close()
def sendFrom(q):
listener = connection.Listener(address, authkey="hello")
while True:
conn = listener.accept()
data = conn.recv()
if name in data[1]:
print("message from {}:{}".format(data[0], data[2]))
q.put(data)
if data[2] == "quit":
conn.close()
def run(self):
print('Running sevice at {}'.format(self.address))
with connection.Listener(self.address, authkey=self.authkey) as listener:
while True:
conn = listener.accept()
addr = listener.last_accepted
print('Connection accepted from {}'.format(addr))
if self.multithreaded:
self.start_thread(conn, addr)
else:
self._handle(conn, self.logger)
def open(self):
'''
Opens the listener to start accepting connections from clients.
'''
if self._listener is not None:
print(
'Warning: the ADR interrupt listener cannot be opened twice.')
return
self._listener = mpc.Listener(
self._address,
'AF_INET',
authkey=(self._authkey if len(self._authkey) > 0 else None))
def main_loop():
address = ("127.0.0.1", FLAGS.port)
with connection.Listener(address) as listener:
while True:
with listener.accept() as conn:
while True:
# Try to keep these flushed for exit logging.
sys.stdout.flush()
sys.stderr.flush()
try:
logging.info("Waiting for message from supervisor.")
msg = conn.recv()
logging.info("Message received.")
except EOFError:
logging.warning("[NOT FATAL] EOFError on conn.recv()")
break
msg = serialization.deserialize(msg)
logging.info(f"Incoming msg: {msg}")
if msg.type == messages.MessageType.PROCESS_ITEM:
exe_item = msg.content.execution_item
if isinstance(exe_item, str):
exe_item = serialization.deserialize(exe_item)
logging.info(
f"Processing execution item: {serialization.serialize(exe_item, indent=2)}"
)
entrypoint.worker_run(**exe_item.worker_run_kwargs)
response = messages.Message(
type=messages.MessageType.PROCESS_ITEM,
content=messages.ItemProcessed(
status=messages.ResponseStatus.SUCCESS),
)
logging.info("Successfully processed execution item")
ser_res = serialization.serialize(response)
logging.info("Sending response to supervisor.")
conn.send(ser_res)
logging.info("Clearing keras session.")
tf.keras.backend.clear_session()
# NOTE: I don't I support this, so commenting out.
# elif msg.type == messages.MessageType.KILL:
# return
else:
raise ValueError(
f"Message received with unknown type {msg.type}.")
def __enter__(self):
assert self._listen is None
try:
self._listen = connection.Listener(self._address, self._family,
authkey=self._authkey)
except Exception as e:
if PLATFORM == 'win':
if not isinstance(e, PermissionError):
raise
else:
if not isinstance(e, OSError) or e.errno != errno.EADDRINUSE:
raise
return self
def run_server():
listener = mpc.Listener(('', 6000), authkey='cs')
remote_conn = listener.accept()
print('Connection accepted from:' + listener.last_accepted[0] + ':%d' %
(listener.last_accepted[1]))
cycles_to_do = 1000
latency_total_us = 0
for n in range(cycles_to_do):
latency_total_us += ping(remote_conn)
print['Average latency ',
float(latency_total_us) / cycles_to_do * 1e6, ' us']
remote_conn.send('quit')
def listen_events(self):
self.logger.info("started thread listener %s", self.name)
listener = connection.Listener(self.address, authkey=AUTH_KEY)
while not self.stop_executing:
conn = listener.accept()
if conn:
data = conn.recv()
self.logger.debug("received new event: %s",
"E" + str(data.get('id') or 0))
if data:
self.process_incoming(data)
conn.close()
self.logger.info("stopped listening to events")
def _setup_listener(self):
self._auth = binascii.b2a_hex(os.urandom(32))
while not self._abort:
try:
self._listener = mpc.Listener(('localhost', self._port),
authkey=self._auth)
return True
except socket.error, e:
if e.errno in (errno.EADDRINUSE, errno.EACCES):
self._port += 1
continue
print '\nException has occurred while executing %s. Please, try again.' % self._executable
self._cleanup(e)
return False
def listener() -> None:
nonlocal config
socket = connection.Listener((self.host, self.port + 1))
conn = socket.accept()
while True:
if conn.poll():
message: Dict[str, Any] = conn.recv()
if message["content"] == "close":
conn.close()
break
else:
config = message["content"]
socketio.emit("config", config, broadcast=True)
socketio.sleep(0.0001)
socket.close()
def wait_for_client(self):
"""Wait for a client to connect and return us the connection. Putting
'localhost' prevents connections from outside machines."""
print('Waiting for client')
self.listener == None
while self.listener == None:
try:
self.listener = mpc.Listener((self.address, self.port),
authkey=self.authkey)
self.remote_conn = self.listener.accept()
print('Connection accepted from:' +
self.listener.last_accepted[0] + ':%d' %
(self.listener.last_accepted[1]))
except mp.AuthenticationError:
print('Client had wrong key')
def conStart(self, addr, pw, mode):
lstnr = connection.Listener(addr,
family='AF_PIPE',
authkey=pw.encode())
while 1:
try:
self.client = lstnr.accept()
if mode == 1:
thrd = threading.Thread(target=self.clientLoop,
args=(self.client, ))
if mode != 2:
thrd.daemon = True
thrd.start()
except:
import traceback
print(traceback.format_exc())
def main(_):
address = ("127.0.0.1", FLAGS.port)
with connection.Listener(address) as listener:
while True:
# TODO: Enable multi-threading for connections.
with listener.accept() as conn:
while True:
try:
msg = conn.recv()
except EOFError:
logging.warning("[NOT FATAL] EOFError on conn.recv()")
break
cmd = msg
# logging.info(cmd)
output = subprocess.check_output(cmd)
conn.send(output)
def _listenerProcess(self):
try:
with multi_con.Listener(address=self._address,
authkey=self._authKey) as listener:
self._listenerRegistered = True
self._comManagerInitEvent.set()
traceLog(LogLevel.DEBUG, "ComManager: Listener registered")
self._externalConnectorsLinkedEvent.wait()
traceLog(LogLevel.DEBUG, "ComManager: Listener started")
# TODO: If filename is received at start of program, also start text view
if self._filePath:
self._root.after(10, self._openFile, self._filePath)
while self._listenerFlag:
conn = listener.accept(
) # Will block until connection is accepted
try:
msg = conn.recv(
) # blocking until something is received
self._root.after(10, self._openFile, msg)
except EOFError:
pass
conn.close()
time.sleep(0.05) # No need to loop crazy fast
except OSError:
traceLog(
LogLevel.INFO,
"ComManager: Socket address already used. Com listener likely already running"
)
self._clientSend(self._filePath)
except multi_con.AuthenticationError:
traceLog(LogLevel.WARNING,
"ComManager: Listener authentication error")
except Exception as e:
traceLog(
LogLevel.ERROR, "ComManager: Listener exception [%s]: %s" %
(str(type(e).__name__), str(e)))
self._comManagerInitEvent.set()
def _serve(addr, conn_buffer_size, filename):
signal.signal(signal.SIGINT, lambda signum, frame: exit(0))
signal.signal(signal.SIGTERM, lambda signum, frame: exit(0))
root_logger = logging.getLogger()
root_logger.setLevel(logging.DEBUG)
file_logger = logging.FileHandler(filename, "a", "utf-8")
root_logger.addHandler(file_logger)
log_handler = logging.StreamHandler(sys.stdout)
root_logger.addHandler(log_handler)
with mpc.Listener(addr, "AF_INET", conn_buffer_size, None) as lst:
while True:
conn = lst.accept()
caddr = lst.last_accepted
threading.Thread(target=_handle_connection,
args=(conn, caddr)).start()
def test_conn_accept(self):
(parent_conn, child_conn) = multiprocessing.Pipe()
self.assertEqual(self.plugin._acceptConns(parent_conn), parent_conn)
listener = connection.Listener(('127.0.0.1', 0))
with self.assertRaises(RuntimeError):
self.plugin._acceptConns(listener)
def fake_client(address):
client = connection.Client(address)
time.sleep(10)
client.close()
t = threading.Thread(target=fake_client, args=(listener.address, ))
t.start()
conn = self.plugin._acceptConns(listener)
self.assertTrue(hasattr(conn, "send"))
self.assertTrue(hasattr(conn, "recv"))
def session():
"""Context manager that listens for send().
Use this as a context manager:
# the queue will contain objects from send()
with session() as message_queue:
# start something that processes items in the queue and run
# the application
"""
message_queue = queue.Queue()
with connection.Listener() as listener:
with open(_ADDRESS_FILE, 'w') as file:
print(listener.address, file=file)
thread = threading.Thread(target=_listener2queue,
args=[listener, message_queue], daemon=True)
thread.start()
yield message_queue
def _prepConns(self):
"""
If the ``bind_host`` is not ``None``, return:
(multiprocessing.connection.Listener, (address, port, authkey))
else:
(parent_connection, child_connection)
For the former case: ``accept`` must be called on the listener. In order
to get a ``Connection`` object for the socket.
"""
if self.bind_host is not None:
#prevent "accidental" wire crossing
authkey = os.urandom(20)
address = (self.bind_host, self.bind_port)
listener = connection.Listener(address, authkey=authkey)
return (listener, listener.address + (authkey, ))
else:
return multiprocessing.Pipe()
def __init__(self, address):
"""Opens a socket with the given address and key.
@param address: The socket address.
@raises socket.error: If the address is already in use or is not a valid
path.
@raises TypeError: If the address is not a valid unix domain socket
address.
"""
self._socket = connection.Listener(address, family='AF_UNIX')
# This is done mostly for local testing/dev purposes - the easiest/most
# reliable way to run the container pool locally is as root, but then
# only other processes owned by root can connect to the container.
# Setting open permissions on the socket makes it so that other users
# can connect, which enables developers to then run tests without sudo.
os.chmod(address, 0777)
self._address = address
self._queue = Queue.Queue()
self._thread = None
self._running = False
def server():
global glob_hosts
global glob_vals
serversocket = None
for host in glob_hosts:
try:
serversocket = connection.Listener(
host,
authkey=PROJECT_AUTHKEY,
)
except:
continue
if not serversocket:
print("Error: Network: couldn't connect")
os._exit(1)
print("Connected")
while True:
clientsocket = serversocket.accept()
request = clientsocket.recv()
if isinstance(request, REQUEST_GET_VALUE):
if request.varname in glob_vals:
clientsocket.send(
RESPONSE_GET_VALUE(request.varname,
glob_vals[request.varname]))
else:
clientsocket.send(I_DONT_KNOW())
elif isinstance(request, REQUEST_SET_VALUE):
if request.varname in glob_vals:
glob_vals[request.varname] = request.value
clientsocket.send(ACK())
else:
clientsocket.send(I_DONT_KNOW())
elif isinstance(request, REQUEST_LIST):
clientsocket.send(RESPONSE_LIST(glob_vals.keys()))
clientsocket.close()
serversocket.close()
def start(self):
""" Start this manager and all remote managers. """
super(Cluster, self).start()
hostname = socket.getfqdn()
listener = connection.Listener(address=(hostname, 0),
authkey=self._authkey,
backlog=5) # Default is 1.
# TODO: support multiple addresses if multiple networks are attached.
# Start managers in separate thread to avoid losing connections.
starter = threading.Thread(target=self._start_hosts,
args=(listener.address, get_credentials()))
starter.daemon = True
starter.start()
# Accept callback connections from started managers.
waiting = ['']
retry = 0
while waiting:
host_processed = False
for host in self._hostlist:
host.poll()
if host.state == 'started':
# Accept conection from *any* host.
_LOGGER.debug('waiting for a connection, host %s',
host.hostname)
# This will hang if server doesn't receive our address.
conn = listener.accept()
i, address, pubkey_text = conn.recv()
conn.close()
other_host = self._hostlist[i]
if address is None:
_LOGGER.error('Host %s died: %s', other_host.hostname,
pubkey_text) # Exception text.
continue
other_host.manager = HostManager.from_address(
address, self._authkey)
other_host.state = 'up'
if pubkey_text:
other_host.manager._pubkey = \
decode_public_key(pubkey_text)
host_processed = True
_LOGGER.debug('Host %s is now up', other_host.hostname)
self._up.append(other_host)
# See if there are still hosts to wait for.
waiting = []
for host in self._hostlist:
host.poll()
if host.state == 'init' or host.state == 'started':
waiting.append(host)
if waiting:
if not host_processed:
retry += 1
if retry < 600: # ~60 seconds.
time.sleep(0.1)
else:
_LOGGER.warning('Cluster startup timeout,'
' hosts not started:')
for host in waiting:
_LOGGER.warning(' %s (%s) in dir %s',
host.hostname, host.state,
host.tempdir)
break
else:
break
self._up = sorted(self._up, key=lambda host: host.hostname)
self._base_shutdown = self.shutdown
del self.shutdown
# from multiprocessing.connection import Listener
import multiprocessing.connection as con
import time
import os
import sys
address = ('localhost', 6000) # family is deduced to be 'AF_INET'
try:
listener = con.Listener(address, authkey=b"secret password")
print("Listener created")
conn = listener.accept() # blocking
print("connection accepted from", listener.last_accepted)
time.sleep(5)
while True:
if conn.poll(timeout=0.1):
msg = conn.recv() # blocking
# do something with msg
if msg == "close":
conn.close()
break
time.sleep(2)
print("Loooop")
print("Message received")
listener.close()
except OSError:
print("Socket address already used")
except con.AuthenticationError:
print("AuthenticationError")
except Exception as e:
print("Not able to start listener")
print(e.args)
def add_connection(name):
address = ('localhost', 0)
listener = connection.Listener(address)
address = listener.address
listeners[name] = listener
malt_to_bridge[name] = address
def sendFrom(q):
listener = connection.Listener(("localhost", 10000), authkey="hello")
while True:
conn = listener.accept()
data = conn.recv()
q.put(data)