def run_std_sim(args, inputfile, usernamespace, optparams=None): """ Run standard simulation - models are run one after another and each model is parallelised using either OpenMP (CPU) or CUDA (GPU) Args: args (dict): Namespace with command line arguments inputfile (object): File object for the input file. usernamespace (dict): Namespace that can be accessed by user in any Python code blocks in input file. optparams (dict): Optional argument. For Taguchi optimisation it provides the parameters to optimise and their values. """ # Set range for number of models to run if args.task: # Job array feeds args.n number of single tasks modelstart = args.task modelend = args.task + 1 elif args.restart: modelstart = args.restart modelend = modelstart + args.n else: modelstart = 1 modelend = modelstart + args.n numbermodelruns = args.n tsimstart = timer() for currentmodelrun in range(modelstart, modelend): # If Taguchi optimistaion, add specific value for each parameter to # optimise for each experiment to user accessible namespace if optparams: tmp = {} tmp.update((key, value[currentmodelrun - 1]) for key, value in optparams.items()) modelusernamespace = usernamespace.copy() modelusernamespace.update({'optparams': tmp}) else: modelusernamespace = usernamespace run_model(args, currentmodelrun, modelend - 1, numbermodelruns, inputfile, modelusernamespace) tsimend = timer() simcompletestr = '\n=== Simulation completed in [HH:MM:SS]: {}'.format( datetime.timedelta(seconds=tsimend - tsimstart)) print('{} {}\n'.format( simcompletestr, '=' * (get_terminal_width() - 1 - len(simcompletestr))))
def run_mpi_no_spawn_sim(args, inputfile, usernamespace, optparams=None): """ Alternate MPI implementation that avoids using the MPI spawn mechanism. This implementation is designed to be used as e.g. 'mpirun -n 5 python -m gprMax user_models/mymodel.in -n 10 --mpi-no-spawn' Run mixed mode MPI/OpenMP simulation - MPI task farm for models with each model parallelised using either OpenMP (CPU) or CUDA (GPU) Args: args (dict): Namespace with command line arguments inputfile (object): File object for the input file. usernamespace (dict): Namespace that can be accessed by user in any Python code blocks in input file. optparams (dict): Optional argument. For Taguchi optimisation it provides the parameters to optimise and their values. """ from mpi4py import MPI # Define MPI message tags tags = Enum('tags', {'READY': 0, 'DONE': 1, 'EXIT': 2, 'START': 3}) # Initializations and preliminaries comm = MPI.COMM_WORLD size = comm.Get_size() # total number of processes rank = comm.Get_rank() # rank of this process status = MPI.Status() # get MPI status object hostname = MPI.Get_processor_name() # get name of processor/host # Set range for number of models to run modelstart = args.restart if args.restart else 1 modelend = modelstart + args.n numbermodelruns = args.n currentmodelrun = modelstart # can use -task argument to start numbering from something other than 1 numworkers = size - 1 ################## # Master process # ################## if rank == 0: tsimstart = timer() mpistartstr = '\n=== MPI task farm (WITHOUT using MPI Spawn)' print('{} {}'.format( mpistartstr, '=' * (get_terminal_width() - 1 - len(mpistartstr)))) print('=== MPI master ({}, rank: {}) on {} using {} workers...'.format( comm.name, comm.Get_rank(), hostname, numworkers)) closedworkers = 0 while closedworkers < numworkers: comm.recv(source=MPI.ANY_SOURCE, tag=MPI.ANY_TAG, status=status) source = status.Get_source() tag = status.Get_tag() # Worker is ready, so send it a task if tag == tags.READY.value: if currentmodelrun < modelend: comm.send(currentmodelrun, dest=source, tag=tags.START.value) currentmodelrun += 1 else: comm.send(None, dest=source, tag=tags.EXIT.value) # Worker has completed a task elif tag == tags.DONE.value: pass # Worker has completed all tasks elif tag == tags.EXIT.value: closedworkers += 1 tsimend = timer() simcompletestr = '\n=== MPI master ({}, rank: {}) on {} completed simulation in [HH:MM:SS]: {}'.format( comm.name, comm.Get_rank(), hostname, datetime.timedelta(seconds=tsimend - tsimstart)) print('{} {}\n'.format( simcompletestr, '=' * (get_terminal_width() - 1 - len(simcompletestr)))) ################## # Worker process # ################## else: # Get info and setup device ID for GPU(s) gpuinfo = '' if args.gpu is not None: # Set device ID based on rank from list of GPUs deviceID = (rank - 1) % len(args.gpu) args.gpu = next(gpu for gpu in args.gpu if gpu.deviceID == deviceID) gpuinfo = ' using {} - {}, {}'.format( args.gpu.deviceID, args.gpu.name, human_size(args.gpu.totalmem, a_kilobyte_is_1024_bytes=True)) while True: comm.send(None, dest=0, tag=tags.READY.value) # Receive a model number to run from the master currentmodelrun = comm.recv(source=0, tag=MPI.ANY_TAG, status=status) tag = status.Get_tag() # Run a model if tag == tags.START.value: # If Taguchi optimistaion, add specific value for each parameter # to optimise for each experiment to user accessible namespace if optparams: tmp = {} tmp.update((key, value[currentmodelrun - 1]) for key, value in optparams.items()) modelusernamespace = usernamespace.copy() modelusernamespace.update({'optparams': tmp}) else: modelusernamespace = usernamespace # Run the model print( 'Starting MPI worker (parent: {}, rank: {}) on {} with model {}/{}{}\n' .format(comm.name, rank, hostname, currentmodelrun, numbermodelruns, gpuinfo)) tsolve = run_model(args, currentmodelrun, modelend - 1, numbermodelruns, inputfile, modelusernamespace) comm.send(None, dest=0, tag=tags.DONE.value) print( 'Completed MPI worker (parent: {}, rank: {}) on {} with model {}/{}{} in [HH:MM:SS]: {}\n' .format(comm.name, rank, hostname, currentmodelrun, numbermodelruns, gpuinfo, datetime.timedelta(seconds=tsolve))) # Break out of loop when work receives exit message elif tag == tags.EXIT.value: break comm.send(None, dest=0, tag=tags.EXIT.value)
def run_mpi_sim(args, inputfile, usernamespace, optparams=None): """ Run mixed mode MPI/OpenMP simulation - MPI task farm for models with each model parallelised using either OpenMP (CPU) or CUDA (GPU) Args: args (dict): Namespace with command line arguments inputfile (object): File object for the input file. usernamespace (dict): Namespace that can be accessed by user in any Python code blocks in input file. optparams (dict): Optional argument. For Taguchi optimisation it provides the parameters to optimise and their values. """ from mpi4py import MPI status = MPI.Status() hostname = MPI.Get_processor_name() # Set range for number of models to run modelstart = args.restart if args.restart else 1 modelend = modelstart + args.n numbermodelruns = args.n # Command line flag used to indicate a spawned worker instance workerflag = '--mpi-worker' numworkers = args.mpi - 1 ################## # Master process # ################## if workerflag not in sys.argv: # N.B Spawned worker flag (--mpi-worker) applied to sys.argv when MPI.Spawn is called # See if the MPI communicator object is being passed as an argument (likely from a MPI.Split) if args.mpicomm is not None: comm = args.mpicomm else: comm = MPI.COMM_WORLD tsimstart = timer() mpistartstr = '\n=== MPI task farm (USING MPI Spawn)' print('{} {}'.format( mpistartstr, '=' * (get_terminal_width() - 1 - len(mpistartstr)))) print('=== MPI master ({}, rank: {}) on {} spawning {} workers...'. format(comm.name, comm.Get_rank(), hostname, numworkers)) # Assemble a sys.argv replacement to pass to spawned worker # N.B This is required as sys.argv not available when gprMax is called via api() # Ignore mpicomm object if it exists as only strings can be passed via spawn myargv = [] for key, value in vars(args).items(): if value: # Input file name always comes first if 'inputfile' in key: myargv.append(value) elif 'gpu' in key: myargv.append('-' + key) # Add GPU device ID(s) from GPU objects for gpu in args.gpu: myargv.append(str(gpu.deviceID)) elif 'mpicomm' in key: pass elif '_' in key: key = key.replace('_', '-') myargv.append('--' + key) else: myargv.append('-' + key) if value is not True: myargv.append(str(value)) # Create a list of work worklist = [] for model in range(modelstart, modelend): workobj = dict() workobj['currentmodelrun'] = model workobj['mpicommname'] = comm.name if optparams: workobj['optparams'] = optparams worklist.append(workobj) # Add stop sentinels worklist += ([StopIteration] * numworkers) # Spawn workers newcomm = comm.Spawn(sys.executable, args=['-m', 'gprMax'] + myargv + [workerflag], maxprocs=numworkers) # Reply to whoever asks until done for work in worklist: newcomm.recv(source=MPI.ANY_SOURCE, status=status) newcomm.send(obj=work, dest=status.Get_source()) # Shutdown communicators newcomm.Disconnect() tsimend = timer() simcompletestr = '\n=== MPI master ({}, rank: {}) on {} completed simulation in [HH:MM:SS]: {}'.format( comm.name, comm.Get_rank(), hostname, datetime.timedelta(seconds=tsimend - tsimstart)) print('{} {}\n'.format( simcompletestr, '=' * (get_terminal_width() - 1 - len(simcompletestr)))) ################## # Worker process # ################## elif workerflag in sys.argv: # Connect to parent to get communicator try: comm = MPI.Comm.Get_parent() rank = comm.Get_rank() except ValueError: raise ValueError('MPI worker could not connect to parent') # Select GPU and get info gpuinfo = '' if args.gpu is not None: # Set device ID based on rank from list of GPUs args.gpu = args.gpu[rank] gpuinfo = ' using {} - {}, {} RAM '.format( args.gpu.deviceID, args.gpu.name, human_size(args.gpu.totalmem, a_kilobyte_is_1024_bytes=True)) # Ask for work until stop sentinel for work in iter(lambda: comm.sendrecv(0, dest=0), StopIteration): currentmodelrun = work['currentmodelrun'] # If Taguchi optimisation, add specific value for each parameter to # optimise for each experiment to user accessible namespace if 'optparams' in work: tmp = {} tmp.update((key, value[currentmodelrun - 1]) for key, value in work['optparams'].items()) modelusernamespace = usernamespace.copy() modelusernamespace.update({'optparams': tmp}) else: modelusernamespace = usernamespace # Run the model print( 'Starting MPI spawned worker (parent: {}, rank: {}) on {} with model {}/{}{}\n' .format(work['mpicommname'], rank, hostname, currentmodelrun, numbermodelruns, gpuinfo)) tsolve = run_model(args, currentmodelrun, modelend - 1, numbermodelruns, inputfile, modelusernamespace) print( 'Completed MPI spawned worker (parent: {}, rank: {}) on {} with model {}/{}{} in [HH:MM:SS]: {}\n' .format(work['mpicommname'], rank, hostname, currentmodelrun, numbermodelruns, gpuinfo, datetime.timedelta(seconds=tsolve))) # Shutdown comm.Disconnect()
def solve_cpu(currentmodelrun, modelend, G): """ Solving using FDTD method on CPU. Parallelised using Cython (OpenMP) for electric and magnetic field updates, and PML updates. Args: currentmodelrun (int): Current model run number. modelend (int): Number of last model to run. G (class): Grid class instance - holds essential parameters describing the model. Returns: tsolve (float): Time taken to execute solving """ tsolvestart = timer() for iteration in tqdm(range(G.iterations), desc='Running simulation, model ' + str(currentmodelrun) + '/' + str(modelend), ncols=get_terminal_width() - 1, file=sys.stdout, disable=not G.progressbars): # Store field component values for every receiver and transmission line store_outputs(iteration, G.Ex, G.Ey, G.Ez, G.Hx, G.Hy, G.Hz, G) # Store any snapshots for snap in G.snapshots: if snap.time == iteration + 1: snap.store(G) # Update magnetic field components update_magnetic(G.nx, G.ny, G.nz, G.nthreads, G.updatecoeffsH, G.ID, G.Ex, G.Ey, G.Ez, G.Hx, G.Hy, G.Hz) # Update magnetic field components with the PML correction for pml in G.pmls: pml.update_magnetic(G) # Update magnetic field components from sources for source in G.transmissionlines + G.magneticdipoles: source.update_magnetic(iteration, G.updatecoeffsH, G.ID, G.Hx, G.Hy, G.Hz, G) # Update electric field components # All materials are non-dispersive so do standard update if Material.maxpoles == 0: update_electric(G.nx, G.ny, G.nz, G.nthreads, G.updatecoeffsE, G.ID, G.Ex, G.Ey, G.Ez, G.Hx, G.Hy, G.Hz) # If there are any dispersive materials do 1st part of dispersive update # (it is split into two parts as it requires present and updated electric field values). elif Material.maxpoles == 1: update_electric_dispersive_1pole_A(G.nx, G.ny, G.nz, G.nthreads, G.updatecoeffsE, G.updatecoeffsdispersive, G.ID, G.Tx, G.Ty, G.Tz, G.Ex, G.Ey, G.Ez, G.Hx, G.Hy, G.Hz) elif Material.maxpoles > 1: update_electric_dispersive_multipole_A(G.nx, G.ny, G.nz, G.nthreads, Material.maxpoles, G.updatecoeffsE, G.updatecoeffsdispersive, G.ID, G.Tx, G.Ty, G.Tz, G.Ex, G.Ey, G.Ez, G.Hx, G.Hy, G.Hz) # Update electric field components with the PML correction for pml in G.pmls: pml.update_electric(G) # Update electric field components from sources (update any Hertzian dipole sources last) for source in G.voltagesources + G.transmissionlines + G.hertziandipoles: source.update_electric(iteration, G.updatecoeffsE, G.ID, G.Ex, G.Ey, G.Ez, G) # If there are any dispersive materials do 2nd part of dispersive update # (it is split into two parts as it requires present and updated electric # field values). Therefore it can only be completely updated after the # electric field has been updated by the PML and source updates. if Material.maxpoles == 1: update_electric_dispersive_1pole_B(G.nx, G.ny, G.nz, G.nthreads, G.updatecoeffsdispersive, G.ID, G.Tx, G.Ty, G.Tz, G.Ex, G.Ey, G.Ez) elif Material.maxpoles > 1: update_electric_dispersive_multipole_B(G.nx, G.ny, G.nz, G.nthreads, Material.maxpoles, G.updatecoeffsdispersive, G.ID, G.Tx, G.Ty, G.Tz, G.Ex, G.Ey, G.Ez) tsolve = timer() - tsolvestart return tsolve