def launch_mpi():
"""Launches the program with MPI
The mpi root host runs in server mode, and others run in client mode.
Note that you need to have more than 1 mpi host for this to work.
"""
try:
from mpi4py import MPI
except ImportError:
logging.fatal('To use launch_mpi, you need mpi4py installed.')
sys.exit(1)
comm = MPI.COMM_WORLD
if comm.Get_size() == 1:
logging.error('You need to specify more than one MPI host.')
sys.exit(1)
# get the server address
address = socket.gethostbyname(socket.gethostname())
address = comm.bcast(address)
if comm.Get_rank() == 0:
# server mode
server = mince.Server()
server.run_server()
# after the server finishes running, tere might be
# some clients still running, and MPI does not exit very elegantly.
# However, with asynchat and the current implementation we have no
# trace of running clients, so this is probably inevitable.
else:
# client mode
client = mince.Client()
client.run_client(address)
return
def launch(argv=None):
"""Launches the program with commandline flag
"""
if argv is None:
argv = sys.argv
process_argv(argv)
if FLAGS.launch == 'local':
launch_local()
elif FLAGS.launch == "server":
# server mode
server = mince.Server()
server.run_server()
elif FLAGS.launch == "client":
# client mode
client = mince.Client()
client.run_client()
elif FLAGS.launch == "mpi":
launch_mpi()
elif FLAGS.launch == "slurm":
launch_slurm(argv)
else:
logging.fatal("Unable to recognize the launch mode: " + FLAGS.launch)
sys.exit(1)
logging.info("Mapreduce terminated.")
return
def launch_local():
""" launches both the server and the clients on the local machine.
The number of clients is defined in FLAGS.num_clients.
"""
server, client = mince.Server(), mince.Client()
serverprocess = Process(target=server.run_server, args=())
serverprocess.start()
clientprocess = []
for i in range(FLAGS.num_clients):
clientprocess.append(Process(target=client.run_client, args=()))
clientprocess[i].start()
serverprocess.join()
logging.info("Waiting for processes to finish.")
# some clients might still be running, and we will wait for them
for i in range(FLAGS.num_clients):
clientprocess[i].join()
return