def run(self):
workers, connections = self._start_workers()
writers = []
for outfile in [self._outfiles.out, self._outfiles.out2]:
if outfile is None:
continue
writers.append(OrderedChunkWriter(outfile))
stats = None
while connections:
ready_connections = multiprocessing.connection.wait(connections)
for connection in ready_connections:
chunk_index = connection.recv()
if chunk_index == -1:
# the worker is done
cur_stats = connection.recv()
if stats == -2:
# An exception has occurred in the worker (see below,
# this happens only when there is an exception sending
# the statistics)
e, tb_str = connection.recv()
# TODO traceback should only be printed in development
logger.debug('%s', tb_str)
raise e
if stats is None:
stats = cur_stats
else:
stats += cur_stats
connections.remove(connection)
continue
elif chunk_index == -2:
# An exception has occurred in the worker
e, tb_str = connection.recv()
# TODO traceback should only be printed in development
# We should use the worker's actual traceback object
# here, but traceback objects are not picklable.
logger.debug('%s', tb_str)
raise e
for writer in writers:
data = connection.recv_bytes()
writer.write(data, chunk_index)
for writer in writers:
assert writer.wrote_everything()
for w in workers:
w.join()
self._reader_process.join()
return stats
def run(self):
while True:
connection = self._listener.accept()
class Closer:
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
connection.close()
with Closer():
while True:
try:
message = connection.recv()
except EOFError:
break
try:
result = self._handle_message(message)
except self._AppException:
result = {
'status':
'failure',
'traceback':
traceback.format_exception(*sys.exc_info()),
}
print(result)
connection.send(result)
async def recv_wait_async(self, name, timeout=None, interval=0.0):
connection = self.recv_connections[name]
start_time = time.perf_counter()
while not connection.poll():
await asyncio.sleep(interval)
if timeout is not None and (time.perf_counter() -
start_time) >= timeout:
warnings.warn()
return None
return connection.recv()
def handle_connection(self, connection):
try:
while True:
func_name, args, kwargs = pickle.loads(connection.recv().encode())
try:
result = self._functions[func_name](*args, **kwargs)
connection.send(pickle.dumps(result))
except Exception as e:
connection.send(pickle.dumps(e))
except EOFError:
pass
def run(self):
workers, connections = self._start_workers()
writers = []
for outfile in [self._outfiles.out, self._outfiles.out2]:
if outfile is None:
continue
writers.append(OrderedChunkWriter(outfile))
stats = None
n = 0 # A running total of the number of processed reads (for progress indicator)
while connections:
ready_connections = multiprocessing.connection.wait(connections)
for connection in ready_connections:
chunk_index = connection.recv()
if chunk_index == -1:
# the worker is done
cur_stats = connection.recv()
if stats == -2:
# An exception has occurred in the worker (see below,
# this happens only when there is an exception sending
# the statistics)
e, tb_str = connection.recv()
# TODO traceback should only be printed in development
logger.debug('%s', tb_str)
raise e
if stats is None:
stats = cur_stats
else:
stats += cur_stats
connections.remove(connection)
continue
elif chunk_index == -2:
# An exception has occurred in the worker
e, tb_str = connection.recv()
# TODO traceback should only be printed in development
# We should use the worker's actual traceback object
# here, but traceback objects are not picklable.
logger.debug('%s', tb_str)
raise e
# No. of reads processed in this chunk
chunk_n = connection.recv()
if chunk_n == -2:
e, tb_str = connection.recv()
logger.debug('%s', tb_str)
raise e
n += chunk_n
self._progress.update(n)
for writer in writers:
data = connection.recv_bytes()
writer.write(data, chunk_index)
for writer in writers:
assert writer.wrote_everything()
for w in workers:
w.join()
self._reader_process.join()
self._progress.stop(n)
return stats
def run(self):
workers, connections = self._start_workers()
writers = []
for outfile in self._outfiles:
if outfile is None:
continue
writers.append(OrderedChunkWriter(outfile))
stats = None
n = 0 # A running total of the number of processed reads (for progress indicator)
while connections:
ready_connections = multiprocessing.connection.wait(connections)
for connection in ready_connections:
chunk_index = connection.recv()
if chunk_index == -1:
# the worker is done
cur_stats = connection.recv()
if stats == -2:
# An exception has occurred in the worker (see below,
# this happens only when there is an exception sending
# the statistics)
e, tb_str = connection.recv()
logger.error('%s', tb_str)
raise e
if stats is None:
stats = cur_stats
else:
stats += cur_stats
connections.remove(connection)
continue
elif chunk_index == -2:
# An exception has occurred in the worker
e, tb_str = connection.recv()
# We should use the worker's actual traceback object
# here, but traceback objects are not picklable.
logger.error('%s', tb_str)
raise e
# No. of reads processed in this chunk
chunk_n = connection.recv()
if chunk_n == -2:
e, tb_str = connection.recv()
logger.error('%s', tb_str)
raise e
n += chunk_n
self._progress.update(n)
for writer in writers:
data = connection.recv_bytes()
writer.write(data, chunk_index)
for writer in writers:
assert writer.wrote_everything()
for w in workers:
w.join()
self._reader_process.join()
self._progress.stop(n)
return stats
async def wait_for_broadcast_async(self, name, timeout=None, interval=0.0):
connection = self.broadcasters[name]
interrupt = self.interrupts.add(name)
start_time = time.perf_counter()
while not interrupt:
if connection.poll():
return connection.recv()
else:
if timeout is not None and (time.perf_counter() -
start_time) >= timeout:
warnings.warn()
return None
await asyncio.sleep(interval)
interrupt.reset()
return None
def clear_recv(self, name):
connection = self.recv_connections[name]
while connection.poll:
connection.recv()
def recv_bytes(self, name, poll=True, timeout=0.0, **kwargs):
connection = self.recv_connections[name]
if not poll or connection.poll(timeout=timeout):
return connection.recv(**kwargs)
else:
return None
def recv(self, name, poll=True, timeout=0.0):
connection = self.recv_connections[name]
if not poll or connection.poll(timeout=timeout):
return connection.recv()
else:
return None
import multiprocessing
import multiprocessing.connection
rx = multiprocessing.connection.Client(('localhost', 12345)).recv()
# ruleid: multiprocessing-recv
connection = multiprocessing.connection.Client(
('localhost', 12345),
)
output = {}
connection.send(output)
# toodoruleid:multiprocessing.recv
rx = connection.recv()
def run(self):
"""
Run the thread, accepting connections and then listening on them until
they are closed. Each message is a call into the function table.
"""
logger.debug("server listening on '%s'", self.pipe)
while self._is_closed is False:
logger.debug("server thread is about to create a server")
ready = False
# test to see if there is a connection waiting on the pipe
if sys.platform == "win32":
try:
mpc_win32.WaitNamedPipe(self.server.address,
self.LISTEN_TIMEOUT * 1000)
ready = True
except WindowsError as e:
logger.debug("Error during WaitNamedPipe:", exc_info=True)
if e.args[0] not in (
mpc_win32.ERROR_SEM_TIMEOUT,
mpc_win32.ERROR_PIPE_BUSY,
):
raise
ready = False
else:
# can use select on osx and linux
(rd, _, _) = select.select([self.server._listener._socket], [],
[], self.LISTEN_TIMEOUT)
ready = len(rd) > 0
if not ready:
logger.debug("server thread could not create server")
continue
connection = None
try:
# connection waiting to be read, accept it
logger.debug("server about to accept connection")
connection = SafePickleConnection(self.server.accept())
logger.debug("server accepted connection")
while self._is_closed is False:
# test to see if there is data waiting on the connection
has_data = connection.poll(self.LISTEN_TIMEOUT)
# see if we need to stop the server
if self._is_closed:
return
# If we timed out waiting for data, go back to sleep.
if not has_data:
continue
# data coming over the connection is a tuple of (name, args, kwargs)
(respond, func_name, args,
kwargs) = pickle.loads(connection.recv())
logger.debug("server calling '%s(%s, %s)'" %
(func_name, args, kwargs))
try:
if func_name not in self._functions:
logger.error(
"unknown function call: '%s', expecting one of '%s'"
% (func_name, self.list_functions()))
raise ValueError("unknown function call: '%s'" %
func_name)
# grab the function from the function table
func = self._functions[func_name]
# execute the function on the main thread. It may do GUI work.
result = self.engine.execute_in_main_thread(
func, *args, **kwargs)
# If the RPC server was stopped, don't bother trying to reply, the connection
# will have been broken on the client side and this will avoid an error
# on the server side when calling send.
if self._SERVER_WAS_STOPPED != result:
# if the client expects the results, send them along
logger.debug("server got result '%s'" % result)
if respond:
connection.send(pickle.dumps(result))
except Exception as e:
# if any of the above fails send the exception back to the client
logger.error("got exception '%s'" % e)
logger.debug(" traceback:\n%s" %
traceback.format_exc())
if respond:
connection.send(pickle.dumps(e))
except (EOFError, IOError):
# let these errors go
# just keep serving new connections
pass
finally:
# It's possible we failed accepting, so the variable may not be defined.
if connection is not None:
logger.debug("server closing")
connection.close()
logger.debug("server closed")
logger.debug("server thread shutting down")
# Kyle Johnson
# ETEC3702 – Concurrency 5:00 PM
# Paul Yost
# 3-25-21
# Lab 7 – Inter-Process Communication with Listeners and Clients
#Client Example
import multiprocessing.connection
print("You are now connected please wait...")
address = ('127.0.0.1', 0x1234)
connection = multiprocessing.connection.Client(address, authkey=b'secret')
sym = connection.recv()
valid = 1
while True:
while valid != 0:
newboard = connection.recv()
if newboard == "P1 Wins!" or newboard == "P2 Wins!" or newboard == 9:
break
servermess = connection.recv()
print(newboard)
print(servermess)
mymove = int(input("Enter number to place move: "))
playerData = {'move': mymove, 'sym': sym}
connection.send(playerData)
waitmess = connection.recv()
updateboard = connection.recv()
winstatus = connection.recv()
if waitmess == "Invalid Entry, try again":
print(updateboard)
print(waitmess)
def run(self):
"""
Run the thread, accepting connections and then listening on them until
they are closed. Each message is a call into the function table.
"""
self._logger.debug("server listening on '%s'", self.pipe)
while True:
# test to see if there is a connection waiting on the pipe
if sys.platform == "win32":
# need to use win32 api for windows
mpc_win32 = multiprocessing.connection.win32
try:
mpc_win32.WaitNamedPipe(self.server.address, self.LISTEN_TIMEOUT*1000)
ready = True
except WindowsError, e:
if e.args[0] not in (mpc_win32.ERROR_SEM_TIMEOUT, mpc_win32.ERROR_PIPE_BUSY):
raise
ready = False
else:
# can use select on osx and linux
(rd, _, _) = select.select([self.server._listener._socket], [], [], self.LISTEN_TIMEOUT)
ready = (len(rd) > 0)
if not ready:
# nothing ready, see if we need to stop the server, if not keep waiting
if self._stop:
break
continue
# connection waiting to be read, accept it
connection = self.server.accept()
self._logger.debug("server accepted connection")
try:
while True:
# test to see if there is data waiting on the connection
if not connection.poll(self.LISTEN_TIMEOUT):
# no data waiting, see if we need to stop the server, if not keep waiting
if self._stop:
break
continue
# data coming over the connection is a tuple of (name, args, kwargs)
(respond, func_name, args, kwargs) = pickle.loads(connection.recv())
self._logger.debug("server calling '%s(%s, %s)'" % (func_name, args, kwargs))
try:
if func_name not in self._functions:
self._logger.error("unknown function call: '%s'" % func_name)
raise ValueError("unknown function call: '%s'" % func_name)
# grab the function from the function table
func = self._functions[func_name]
# execute the function on the main thread. It may do GUI work.
result = self.engine.execute_in_main_thread(func, *args, **kwargs)
# if the client expects the results, send them along
self._logger.debug("server got result '%s'" % result)
if respond:
connection.send(pickle.dumps(result))
except Exception as e:
# if any of the above fails send the exception back to the client
self._logger.error("got exception '%s'" % e)
self._logger.debug(" traceback:\n%s" % traceback.format_exc())
if respond:
connection.send(pickle.dumps(e))
except (EOFError, IOError):
# let these errors go
# just keep serving new connections
pass
finally:
self._logger.debug("server closing")
connection.close()
import multiprocessing.connection
import pickle
from django_load_balancer.load_balancer_factory import LoadBalancerFactory
from django_load_balancer.local_settings import LOAD_BALANCER
if __name__ == "__main__":
load_balancer = LoadBalancerFactory.create_load_balancer()
address = (LOAD_BALANCER['ADDRESS']['HOST'],
LOAD_BALANCER['ADDRESS']['PORT'])
listener = multiprocessing.connection.Listener(
address, authkey=bytes(LOAD_BALANCER['ADDRESS']['AUTHKEY'], 'utf-8'))
while True:
connection = listener.accept()
while True:
payload = connection.recv()
if payload == 'close':
connection.close()
break
else:
query = pickle.loads(payload)
result = load_balancer.run_query(query)
connection.send(result)
