def launch_mpi(): """Launches the program with MPI The mpi root host runs in server mode, and others run in client mode. Note that you need to have more than 1 mpi host for this to work. """ try: from mpi4py import MPI except ImportError: logging.fatal('To use launch_mpi, you need mpi4py installed.') sys.exit(1) comm = MPI.COMM_WORLD if comm.Get_size() == 1: logging.error('You need to specify more than one MPI host.') sys.exit(1) # get the server address address = socket.gethostbyname(socket.gethostname()) address = comm.bcast(address) if comm.Get_rank() == 0: # server mode server = mince.Server() server.run_server() # after the server finishes running, tere might be # some clients still running, and MPI does not exit very elegantly. # However, with asynchat and the current implementation we have no # trace of running clients, so this is probably inevitable. else: # client mode client = mince.Client() client.run_client(address) return
def launch(argv=None): """Launches the program with commandline flag """ if argv is None: argv = sys.argv process_argv(argv) if FLAGS.launch == 'local': launch_local() elif FLAGS.launch == "server": # server mode server = mince.Server() server.run_server() elif FLAGS.launch == "client": # client mode client = mince.Client() client.run_client() elif FLAGS.launch == "mpi": launch_mpi() elif FLAGS.launch == "slurm": launch_slurm(argv) else: logging.fatal("Unable to recognize the launch mode: " + FLAGS.launch) sys.exit(1) logging.info("Mapreduce terminated.") return
def launch_local(): """ launches both the server and the clients on the local machine. The number of clients is defined in FLAGS.num_clients. """ server, client = mince.Server(), mince.Client() serverprocess = Process(target=server.run_server, args=()) serverprocess.start() clientprocess = [] for i in range(FLAGS.num_clients): clientprocess.append(Process(target=client.run_client, args=())) clientprocess[i].start() serverprocess.join() logging.info("Waiting for processes to finish.") # some clients might still be running, and we will wait for them for i in range(FLAGS.num_clients): clientprocess[i].join() return