def run_job_array_sim(args,
numbermodelruns,
inputfile,
usernamespace,
optparams=None):
"""Run standard simulation as part of a job array on Open Grid Scheduler/Grid Engine (http://gridscheduler.sourceforge.net/index.html) - each model is parallelised with OpenMP
Args:
args (dict): Namespace with command line arguments
numbermodelruns (int): Total number of model runs.
inputfile (object): File object for the input file.
usernamespace (dict): Namespace that can be accessed by user in any Python code blocks in input file.
optparams (dict): Optional argument. For Taguchi optimisation it provides the parameters to optimise and their values.
"""
currentmodelrun = args.task
tsimstart = perf_counter()
if optparams: # If Taguchi optimistaion, add specific value for each parameter to optimise for each experiment to user accessible namespace
tmp = {}
tmp.update((key, value[currentmodelrun - 1])
for key, value in optparams.items())
modelusernamespace = usernamespace.copy()
modelusernamespace.update({'optparams': tmp})
else:
modelusernamespace = usernamespace
run_model(args, currentmodelrun, numbermodelruns, inputfile,
modelusernamespace)
tsimend = perf_counter()
simcompletestr = '\n=== Simulation completed in [HH:MM:SS]: {}'.format(
datetime.timedelta(seconds=tsimend - tsimstart))
print('{} {}\n'.format(
simcompletestr,
'=' * (get_terminal_width() - 1 - len(simcompletestr))))
def run_std_sim(args, inputfile, usernamespace, optparams=None):
"""
Run standard simulation - models are run one after another and each model
is parallelised using either OpenMP (CPU) or CUDA (GPU)
Args:
args (dict): Namespace with command line arguments
inputfile (object): File object for the input file.
usernamespace (dict): Namespace that can be accessed by user in any
Python code blocks in input file.
optparams (dict): Optional argument. For Taguchi optimisation it
provides the parameters to optimise and their values.
"""
# Set range for number of models to run
if args.task:
# Job array feeds args.n number of single tasks
modelstart = args.task
modelend = args.task + 1
elif args.restart:
modelstart = args.restart
modelend = modelstart + args.n
else:
modelstart = 1
modelend = modelstart + args.n
numbermodelruns = args.n
tsimstart = timer()
for currentmodelrun in range(modelstart, modelend):
# If Taguchi optimistaion, add specific value for each parameter to
# optimise for each experiment to user accessible namespace
if optparams:
tmp = {}
tmp.update((key, value[currentmodelrun - 1])
for key, value in optparams.items())
modelusernamespace = usernamespace.copy()
modelusernamespace.update({'optparams': tmp})
else:
modelusernamespace = usernamespace
run_model(args, currentmodelrun, modelend - 1, numbermodelruns,
inputfile, modelusernamespace)
tsimend = timer()
simcompletestr = '\n=== Simulation completed in [HH:MM:SS]: {}'.format(
datetime.timedelta(seconds=tsimend - tsimstart))
print('{} {}\n'.format(
simcompletestr,
'=' * (get_terminal_width() - 1 - len(simcompletestr))))
def run_benchmark_sim(args, inputfile, usernamespace):
"""Run standard simulation in benchmarking mode - models are run one after another and each model is parallelised with OpenMP
Args:
args (dict): Namespace with command line arguments
inputfile (object): File object for the input file.
usernamespace (dict): Namespace that can be accessed by user in any Python code blocks in input file.
"""
# Get information about host machine
hostinfo = get_host_info()
hyperthreading = ', {} cores with Hyper-Threading'.format(hostinfo['logicalcores']) if hostinfo['hyperthreading'] else ''
machineIDlong = '{}; {} x {} ({} cores{}); {} RAM; {}'.format(hostinfo['machineID'], hostinfo['sockets'], hostinfo['cpuID'], hostinfo['physicalcores'], hyperthreading, human_size(hostinfo['ram'], a_kilobyte_is_1024_bytes=True), hostinfo['osversion'])
# Number of CPU threads to benchmark - start from single thread and double threads until maximum number of physical cores
threads = 1
maxthreads = hostinfo['physicalcores']
maxthreadspersocket = hostinfo['physicalcores'] / hostinfo['sockets']
cputhreads = np.array([], dtype=np.int32)
while threads < maxthreadspersocket:
cputhreads = np.append(cputhreads, int(threads))
threads *= 2
# Check for system with only single thread
if cputhreads.size == 0:
cputhreads = np.append(cputhreads, threads)
# Add maxthreadspersocket and maxthreads if necessary
if cputhreads[-1] != maxthreadspersocket:
cputhreads = np.append(cputhreads, int(maxthreadspersocket))
if cputhreads[-1] != maxthreads:
cputhreads = np.append(cputhreads, int(maxthreads))
cputhreads = cputhreads[::-1]
cputimes = np.zeros(len(cputhreads))
numbermodelruns = len(cputhreads)
modelend = numbermodelruns + 1
usernamespace['number_model_runs'] = numbermodelruns
for currentmodelrun in range(1, modelend):
os.environ['OMP_NUM_THREADS'] = str(cputhreads[currentmodelrun - 1])
cputimes[currentmodelrun - 1] = run_model(args, currentmodelrun, modelend - 1, numbermodelruns, inputfile, usernamespace)
# Get model size (in cells) and number of iterations
if currentmodelrun == 1:
if numbermodelruns == 1:
outputfile = os.path.splitext(args.inputfile)[0] + '.out'
else:
outputfile = os.path.splitext(args.inputfile)[0] + str(currentmodelrun) + '.out'
f = h5py.File(outputfile, 'r')
iterations = f.attrs['Iterations']
numcells = f.attrs['nx, ny, nz']
# Save number of threads and benchmarking times to NumPy archive
np.savez(os.path.splitext(inputfile.name)[0], machineID=machineIDlong, gpuIDs=[], cputhreads=cputhreads, cputimes=cputimes, gputimes=[], iterations=iterations, numcells=numcells, version=__version__)
simcompletestr = '\n=== Simulation completed'
print('{} {}\n'.format(simcompletestr, '=' * (get_terminal_width() - 1 - len(simcompletestr))))
def run_benchmark_sim(args, inputfile, usernamespace):
"""Run standard simulation in benchmarking mode - models are run one after another and each model is parallelised with OpenMP
Args:
args (dict): Namespace with command line arguments
inputfile (object): File object for the input file.
usernamespace (dict): Namespace that can be accessed by user in any Python code blocks in input file.
"""
# Get information about host machine
hostinfo = get_host_info()
machineIDlong = '; '.join(
[hostinfo['machineID'], hostinfo['cpuID'], hostinfo['osversion']])
# Number of threads to test - start from max physical CPU cores and divide in half until 1
minthreads = 1
maxthreads = hostinfo['cpucores']
threads = []
while minthreads < maxthreads:
threads.append(int(minthreads))
minthreads *= 2
threads.append(int(maxthreads))
threads.reverse()
benchtimes = np.zeros(len(threads))
numbermodelruns = len(threads)
usernamespace['number_model_runs'] = numbermodelruns
for currentmodelrun in range(1, numbermodelruns + 1):
os.environ['OMP_NUM_THREADS'] = str(threads[currentmodelrun - 1])
tsolve = run_model(args, currentmodelrun, numbermodelruns, inputfile,
usernamespace)
benchtimes[currentmodelrun - 1] = tsolve
# Save number of threads and benchmarking times to NumPy archive
threads = np.array(threads)
np.savez(os.path.splitext(inputfile.name)[0],
threads=threads,
benchtimes=benchtimes,
machineID=machineIDlong,
version=__version__)
simcompletestr = '\n=== Simulation completed'
print('{} {}\n'.format(
simcompletestr,
'=' * (get_terminal_width() - 1 - len(simcompletestr))))
def run_mpi_sim(args, inputfile, usernamespace, optparams=None):
"""
Run mixed mode MPI/OpenMP simulation - MPI task farm for models with
each model parallelised using either OpenMP (CPU) or CUDA (GPU)
Args:
args (dict): Namespace with command line arguments
inputfile (object): File object for the input file.
usernamespace (dict): Namespace that can be accessed by user in any
Python code blocks in input file.
optparams (dict): Optional argument. For Taguchi optimisation it
provides the parameters to optimise and their values.
"""
from mpi4py import MPI
# Get name of processor/host
name = MPI.Get_processor_name()
# Set range for number of models to run
modelstart = args.restart if args.restart else 1
modelend = modelstart + args.n
numbermodelruns = args.n
# Number of workers and command line flag to indicate a spawned worker
worker = '--mpi-worker'
numberworkers = args.mpi - 1
# Master process
if worker not in sys.argv:
tsimstart = perf_counter()
print('MPI master rank (PID {}) on {} using {} workers'.format(os.getpid(), name, numberworkers))
# Create a list of work
worklist = []
for model in range(modelstart, modelend):
workobj = dict()
workobj['currentmodelrun'] = model
if optparams:
workobj['optparams'] = optparams
worklist.append(workobj)
# Add stop sentinels
worklist += ([StopIteration] * numberworkers)
# Spawn workers
comm = MPI.COMM_WORLD.Spawn(sys.executable, args=['-m', 'gprMax', '-n', str(args.n)] + sys.argv[1::] + [worker], maxprocs=numberworkers)
# Reply to whoever asks until done
status = MPI.Status()
for work in worklist:
comm.recv(source=MPI.ANY_SOURCE, status=status)
comm.send(obj=work, dest=status.Get_source())
# Shutdown
comm.Disconnect()
tsimend = perf_counter()
simcompletestr = '\n=== Simulation completed in [HH:MM:SS]: {}'.format(datetime.timedelta(seconds=tsimend - tsimstart))
print('{} {}\n'.format(simcompletestr, '=' * (get_terminal_width() - 1 - len(simcompletestr))))
# Worker process
elif worker in sys.argv:
# Connect to parent
try:
comm = MPI.Comm.Get_parent() # get MPI communicator object
rank = comm.Get_rank() # rank of this process
except:
raise ValueError('Could not connect to parent')
# Ask for work until stop sentinel
for work in iter(lambda: comm.sendrecv(0, dest=0), StopIteration):
currentmodelrun = work['currentmodelrun']
# Get info and setup device ID for GPU(s)
gpuinfo = ''
if args.gpu is not None:
# Set device ID for multiple GPUs
if isinstance(args.gpu, list):
deviceID = (rank - 1) % len(args.gpu)
args.gpu = next(gpu for gpu in args.gpu if gpu.deviceID == deviceID)
gpuinfo = ' using {} - {}, {} RAM '.format(args.gpu.deviceID, args.gpu.name, human_size(args.gpu.totalmem, a_kilobyte_is_1024_bytes=True))
print('MPI worker rank {} (PID {}) starting model {}/{}{} on {}'.format(rank, os.getpid(), currentmodelrun, numbermodelruns, gpuinfo, name))
# If Taguchi optimistaion, add specific value for each parameter to
# optimise for each experiment to user accessible namespace
if 'optparams' in work:
tmp = {}
tmp.update((key, value[currentmodelrun - 1]) for key, value in work['optparams'].items())
modelusernamespace = usernamespace.copy()
modelusernamespace.update({'optparams': tmp})
else:
modelusernamespace = usernamespace
# Run the model
run_model(args, currentmodelrun, modelend - 1, numbermodelruns, inputfile, modelusernamespace)
# Shutdown
comm.Disconnect()
def run_mpi_no_spawn_sim(args, inputfile, usernamespace, optparams=None):
"""
Alternate MPI implementation that avoids using the MPI spawn mechanism.
This implementation is designed to be used as
e.g. 'mpirun -n 5 python -m gprMax user_models/mymodel.in -n 10 --mpi-no-spawn'
Run mixed mode MPI/OpenMP simulation - MPI task farm for models with
each model parallelised using either OpenMP (CPU) or CUDA (GPU)
Args:
args (dict): Namespace with command line arguments
inputfile (object): File object for the input file.
usernamespace (dict): Namespace that can be accessed by user in any
Python code blocks in input file.
optparams (dict): Optional argument. For Taguchi optimisation it
provides the parameters to optimise and their values.
"""
from mpi4py import MPI
# Define MPI message tags
tags = Enum('tags', {'READY': 0, 'DONE': 1, 'EXIT': 2, 'START': 3})
# Initializations and preliminaries
comm = MPI.COMM_WORLD
size = comm.Get_size() # total number of processes
rank = comm.Get_rank() # rank of this process
status = MPI.Status() # get MPI status object
hostname = MPI.Get_processor_name() # get name of processor/host
# Set range for number of models to run
modelstart = args.restart if args.restart else 1
modelend = modelstart + args.n
numbermodelruns = args.n
currentmodelrun = modelstart # can use -task argument to start numbering from something other than 1
numworkers = size - 1
##################
# Master process #
##################
if rank == 0:
tsimstart = perf_counter()
mpimasterstr = '=== MPI master ({}, rank: {}) on {} using {} workers...\n'.format(
comm.name, comm.Get_rank(), hostname, numworkers)
print('{} {}\n'.format(
mpimasterstr,
'=' * (get_terminal_width() - 1 - len(mpimasterstr))))
closedworkers = 0
while closedworkers < numworkers:
comm.recv(source=MPI.ANY_SOURCE, tag=MPI.ANY_TAG, status=status)
source = status.Get_source()
tag = status.Get_tag()
# Worker is ready, so send it a task
if tag == tags.READY.value:
if currentmodelrun < modelend:
comm.send(currentmodelrun,
dest=source,
tag=tags.START.value)
currentmodelrun += 1
else:
comm.send(None, dest=source, tag=tags.EXIT.value)
# Worker has completed a task
elif tag == tags.DONE.value:
pass
# Worker has completed all tasks
elif tag == tags.EXIT.value:
closedworkers += 1
tsimend = perf_counter()
simcompletestr = '\n=== MPI master ({}, rank: {}) on {} completed simulation in [HH:MM:SS]: {}'.format(
comm.name, comm.Get_rank(), hostname,
datetime.timedelta(seconds=tsimend - tsimstart))
print('{} {}\n'.format(
simcompletestr,
'=' * (get_terminal_width() - 1 - len(simcompletestr))))
##################
# Worker process #
##################
else:
# Get info and setup device ID for GPU(s)
gpuinfo = ''
if args.gpu is not None:
# Set device ID based on rank from list of GPUs
deviceID = (rank - 1) % len(args.gpu)
args.gpu = next(gpu for gpu in args.gpu
if gpu.deviceID == deviceID)
gpuinfo = ' using {} - {}, {}'.format(
args.gpu.deviceID, args.gpu.name,
human_size(args.gpu.totalmem, a_kilobyte_is_1024_bytes=True))
while True:
comm.send(None, dest=0, tag=tags.READY.value)
# Receive a model number to run from the master
currentmodelrun = comm.recv(source=0,
tag=MPI.ANY_TAG,
status=status)
tag = status.Get_tag()
# Run a model
if tag == tags.START.value:
# If Taguchi optimistaion, add specific value for each parameter
# to optimise for each experiment to user accessible namespace
if optparams:
tmp = {}
tmp.update((key, value[currentmodelrun - 1])
for key, value in optparams.items())
modelusernamespace = usernamespace.copy()
modelusernamespace.update({'optparams': tmp})
else:
modelusernamespace = usernamespace
# Run the model
print(
'MPI worker (parent: {}, rank: {}) on {} starting model {}/{}{}\n'
.format(comm.name, rank, hostname, currentmodelrun,
numbermodelruns, gpuinfo))
run_model(args, currentmodelrun, modelend - 1, numbermodelruns,
inputfile, modelusernamespace)
comm.send(None, dest=0, tag=tags.DONE.value)
# Break out of loop when work receives exit message
elif tag == tags.EXIT.value:
break
comm.send(None, dest=0, tag=tags.EXIT.value)
def run_mpi_sim(args, inputfile, usernamespace, optparams=None):
"""
Run mixed mode MPI/OpenMP simulation - MPI task farm for models with
each model parallelised using either OpenMP (CPU) or CUDA (GPU)
Args:
args (dict): Namespace with command line arguments
inputfile (object): File object for the input file.
usernamespace (dict): Namespace that can be accessed by user in any
Python code blocks in input file.
optparams (dict): Optional argument. For Taguchi optimisation it
provides the parameters to optimise and their values.
"""
from mpi4py import MPI
status = MPI.Status()
hostname = MPI.Get_processor_name()
# Set range for number of models to run
modelstart = args.restart if args.restart else 1
modelend = modelstart + args.n
numbermodelruns = args.n
# Command line flag used to indicate a spawned worker instance
workerflag = '--mpi-worker'
numworkers = args.mpi - 1
##################
# Master process #
##################
if workerflag not in sys.argv:
# N.B Spawned worker flag (--mpi-worker) applied to sys.argv when MPI.Spawn is called
# See if the MPI communicator object is being passed as an argument (likely from a MPI.Split)
if hasattr(args, 'mpicomm'):
comm = args.mpicomm
else:
comm = MPI.COMM_WORLD
tsimstart = perf_counter()
mpimasterstr = '=== MPI master ({}, rank: {}) on {} spawning {} workers...'.format(
comm.name, comm.Get_rank(), hostname, numworkers)
print('{} {}\n'.format(
mpimasterstr,
'=' * (get_terminal_width() - 1 - len(mpimasterstr))))
# Assemble a sys.argv replacement to pass to spawned worker
# N.B This is required as sys.argv not available when gprMax is called via api()
# Ignore mpicomm object if it exists as only strings can be passed via spawn
myargv = []
for key, value in vars(args).items():
if value:
# Input file name always comes first
if 'inputfile' in key:
myargv.append(value)
elif 'gpu' in key:
myargv.append('-' + key)
# Add GPU device ID(s) from GPU objects
for gpu in args.gpu:
myargv.append(str(gpu.deviceID))
elif 'mpicomm' in key:
pass
elif '_' in key:
key = key.replace('_', '-')
myargv.append('--' + key)
else:
myargv.append('-' + key)
if value is not True:
myargv.append(str(value))
# Create a list of work
worklist = []
for model in range(modelstart, modelend):
workobj = dict()
workobj['currentmodelrun'] = model
workobj['mpicommname'] = comm.name
if optparams:
workobj['optparams'] = optparams
worklist.append(workobj)
# Add stop sentinels
worklist += ([StopIteration] * numworkers)
# Spawn workers
newcomm = comm.Spawn(sys.executable,
args=['-m', 'gprMax'] + myargv + [workerflag],
maxprocs=numworkers)
# Reply to whoever asks until done
for work in worklist:
newcomm.recv(source=MPI.ANY_SOURCE, status=status)
newcomm.send(obj=work, dest=status.Get_source())
# Shutdown communicators
newcomm.Disconnect()
tsimend = perf_counter()
simcompletestr = '\n=== MPI master ({}, rank: {}) on {} completed simulation in [HH:MM:SS]: {}'.format(
comm.name, comm.Get_rank(), hostname,
datetime.timedelta(seconds=tsimend - tsimstart))
print('{} {}\n'.format(
simcompletestr,
'=' * (get_terminal_width() - 1 - len(simcompletestr))))
##################
# Worker process #
##################
elif workerflag in sys.argv:
# Connect to parent to get communicator
try:
comm = MPI.Comm.Get_parent()
rank = comm.Get_rank()
except ValueError:
raise ValueError('MPI worker could not connect to parent')
# Select GPU and get info
gpuinfo = ''
if args.gpu is not None:
# Set device ID based on rank from list of GPUs
args.gpu = args.gpu[rank]
gpuinfo = ' using {} - {}, {} RAM '.format(
args.gpu.deviceID, args.gpu.name,
human_size(args.gpu.totalmem, a_kilobyte_is_1024_bytes=True))
# Ask for work until stop sentinel
for work in iter(lambda: comm.sendrecv(0, dest=0), StopIteration):
currentmodelrun = work['currentmodelrun']
# If Taguchi optimisation, add specific value for each parameter to
# optimise for each experiment to user accessible namespace
if 'optparams' in work:
tmp = {}
tmp.update((key, value[currentmodelrun - 1])
for key, value in work['optparams'].items())
modelusernamespace = usernamespace.copy()
modelusernamespace.update({'optparams': tmp})
else:
modelusernamespace = usernamespace
# Run the model
print(
'MPI spawned worker (parent: {}, rank: {}) on {} starting model {}/{}{}\n'
.format(work['mpicommname'], rank, hostname, currentmodelrun,
numbermodelruns, gpuinfo))
run_model(args, currentmodelrun, modelend - 1, numbermodelruns,
inputfile, modelusernamespace)
# Shutdown
comm.Disconnect()
def run_mpi_sim(args,
numbermodelruns,
inputfile,
usernamespace,
optparams=None):
"""Run mixed mode MPI/OpenMP simulation - MPI task farm for models with each model parallelised with OpenMP
Args:
args (dict): Namespace with command line arguments
numbermodelruns (int): Total number of model runs.
inputfile (object): File object for the input file.
usernamespace (dict): Namespace that can be accessed by user in any Python code blocks in input file.
optparams (dict): Optional argument. For Taguchi optimisation it provides the parameters to optimise and their values.
"""
from mpi4py import MPI
# Define MPI message tags
tags = Enum('tags', {'READY': 0, 'DONE': 1, 'EXIT': 2, 'START': 3})
# Initializations and preliminaries
comm = MPI.COMM_WORLD # get MPI communicator object
size = comm.Get_size() # total number of processes
rank = comm.Get_rank() # rank of this process
status = MPI.Status() # get MPI status object
name = MPI.Get_processor_name() # get name of processor/host
tsimstart = perf_counter()
# Master process
if rank == 0:
currentmodelrun = 1
numworkers = size - 1
closedworkers = 0
print('Master: PID {} on {} using {} workers.'.format(
os.getpid(), name, numworkers))
while closedworkers < numworkers:
data = comm.recv(source=MPI.ANY_SOURCE,
tag=MPI.ANY_TAG,
status=status)
source = status.Get_source()
tag = status.Get_tag()
if tag == tags.READY.value: # Worker is ready, so send it a task
if currentmodelrun < numbermodelruns + 1:
comm.send(currentmodelrun,
dest=source,
tag=tags.START.value)
print('Master: sending model {} to worker {}.'.format(
currentmodelrun, source))
currentmodelrun += 1
else:
comm.send(None, dest=source, tag=tags.EXIT.value)
elif tag == tags.DONE.value:
print('Worker {}: completed.'.format(source))
elif tag == tags.EXIT.value:
print('Worker {}: exited.'.format(source))
closedworkers += 1
# Worker process
else:
print('Worker {}: PID {} on {}.'.format(rank, os.getpid(), name))
while True:
comm.send(None, dest=0, tag=tags.READY.value)
currentmodelrun = comm.recv(
source=0, tag=MPI.ANY_TAG, status=status
) # Receive a model number to run from the master
tag = status.Get_tag()
# Run a model
if tag == tags.START.value:
if optparams: # If Taguchi optimistaion, add specific value for each parameter to optimise for each experiment to user accessible namespace
tmp = {}
tmp.update((key, value[currentmodelrun - 1])
for key, value in optparams.items())
modelusernamespace = usernamespace.copy()
modelusernamespace.update({'optparams': tmp})
else:
modelusernamespace = usernamespace
run_model(args, currentmodelrun, numbermodelruns, inputfile,
modelusernamespace)
comm.send(None, dest=0, tag=tags.DONE.value)
elif tag == tags.EXIT.value:
break
comm.send(None, dest=0, tag=tags.EXIT.value)
tsimend = perf_counter()
simcompletestr = '\n=== Simulation completed in [HH:MM:SS]: {}'.format(
datetime.timedelta(seconds=tsimend - tsimstart))
print('{} {}\n'.format(
simcompletestr,
'=' * (get_terminal_width() - 1 - len(simcompletestr))))