def run_benchmark_sim(args, inputfile, usernamespace):
"""Run standard simulation in benchmarking mode - models are run one after another and each model is parallelised with OpenMP
Args:
args (dict): Namespace with command line arguments
inputfile (object): File object for the input file.
usernamespace (dict): Namespace that can be accessed by user in any Python code blocks in input file.
"""
# Get information about host machine
hostinfo = get_host_info()
hyperthreading = ', {} cores with Hyper-Threading'.format(hostinfo['logicalcores']) if hostinfo['hyperthreading'] else ''
machineIDlong = '{}; {} x {} ({} cores{}); {} RAM; {}'.format(hostinfo['machineID'], hostinfo['sockets'], hostinfo['cpuID'], hostinfo['physicalcores'], hyperthreading, human_size(hostinfo['ram'], a_kilobyte_is_1024_bytes=True), hostinfo['osversion'])
# Number of CPU threads to benchmark - start from single thread and double threads until maximum number of physical cores
threads = 1
maxthreads = hostinfo['physicalcores']
maxthreadspersocket = hostinfo['physicalcores'] / hostinfo['sockets']
cputhreads = np.array([], dtype=np.int32)
while threads < maxthreadspersocket:
cputhreads = np.append(cputhreads, int(threads))
threads *= 2
# Check for system with only single thread
if cputhreads.size == 0:
cputhreads = np.append(cputhreads, threads)
# Add maxthreadspersocket and maxthreads if necessary
if cputhreads[-1] != maxthreadspersocket:
cputhreads = np.append(cputhreads, int(maxthreadspersocket))
if cputhreads[-1] != maxthreads:
cputhreads = np.append(cputhreads, int(maxthreads))
cputhreads = cputhreads[::-1]
cputimes = np.zeros(len(cputhreads))
numbermodelruns = len(cputhreads)
modelend = numbermodelruns + 1
usernamespace['number_model_runs'] = numbermodelruns
for currentmodelrun in range(1, modelend):
os.environ['OMP_NUM_THREADS'] = str(cputhreads[currentmodelrun - 1])
cputimes[currentmodelrun - 1] = run_model(args, currentmodelrun, modelend - 1, numbermodelruns, inputfile, usernamespace)
# Get model size (in cells) and number of iterations
if currentmodelrun == 1:
if numbermodelruns == 1:
outputfile = os.path.splitext(args.inputfile)[0] + '.out'
else:
outputfile = os.path.splitext(args.inputfile)[0] + str(currentmodelrun) + '.out'
f = h5py.File(outputfile, 'r')
iterations = f.attrs['Iterations']
numcells = f.attrs['nx, ny, nz']
# Save number of threads and benchmarking times to NumPy archive
np.savez(os.path.splitext(inputfile.name)[0], machineID=machineIDlong, gpuIDs=[], cputhreads=cputhreads, cputimes=cputimes, gputimes=[], iterations=iterations, numcells=numcells, version=__version__)
simcompletestr = '\n=== Simulation completed'
print('{} {}\n'.format(simcompletestr, '=' * (get_terminal_width() - 1 - len(simcompletestr))))
def run_main(args):
"""Top-level function that controls what mode of simulation (standard/optimsation/benchmark etc...) is run.
Args:
args (dict): Namespace with input arguments from command line or api.
"""
with open_path_file(args.inputfile) as inputfile:
# Get information about host machine
hostinfo = get_host_info()
hyperthreading = ', {} cores with Hyper-Threading'.format(hostinfo['logicalcores']) if hostinfo['hyperthreading'] else ''
print('\nHost: {}; {} x {} ({} cores{}); {} RAM; {}'.format(hostinfo['machineID'], hostinfo['sockets'], hostinfo['cpuID'], hostinfo['physicalcores'], hyperthreading, human_size(hostinfo['ram'], a_kilobyte_is_1024_bytes=True), hostinfo['osversion']))
# Create a separate namespace that users can access in any Python code blocks in the input file
usernamespace = {'c': c, 'e0': e0, 'm0': m0, 'z0': z0, 'number_model_runs': args.n, 'inputfile': os.path.abspath(inputfile.name)}
#######################################
# Process for benchmarking simulation #
#######################################
if args.benchmark:
if args.mpi or args.opt_taguchi or args.task or args.n > 1:
raise GeneralError('Benchmarking mode cannot be combined with MPI, job array, or Taguchi optimisation modes, or multiple model runs.')
run_benchmark_sim(args, inputfile, usernamespace)
####################################################
# Process for simulation with Taguchi optimisation #
####################################################
elif args.opt_taguchi:
if args.mpi_worker: # Special case for MPI spawned workers - they do not need to enter the Taguchi optimisation mode
run_mpi_sim(args, inputfile, usernamespace)
else:
from gprMax.optimisation_taguchi import run_opt_sim
run_opt_sim(args, inputfile, usernamespace)
################################################
# Process for standard simulation (CPU or GPU) #
################################################
else:
# Mixed mode MPI with OpenMP or CUDA - MPI task farm for models with each model parallelised with OpenMP (CPU) or CUDA (GPU)
if args.mpi:
if args.n == 1:
raise GeneralError('MPI is not beneficial when there is only one model to run')
if args.task:
raise GeneralError('MPI cannot be combined with job array mode')
run_mpi_sim(args, inputfile, usernamespace)
# Standard behaviour - models run serially with each model parallelised with OpenMP (CPU) or CUDA (GPU)
else:
if args.task and args.restart:
raise GeneralError('Job array and restart modes cannot be used together')
run_std_sim(args, inputfile, usernamespace)
def run_benchmark_sim(args, inputfile, usernamespace):
"""Run standard simulation in benchmarking mode - models are run one after another and each model is parallelised with OpenMP
Args:
args (dict): Namespace with command line arguments
inputfile (object): File object for the input file.
usernamespace (dict): Namespace that can be accessed by user in any Python code blocks in input file.
"""
# Get information about host machine
hostinfo = get_host_info()
machineIDlong = '; '.join(
[hostinfo['machineID'], hostinfo['cpuID'], hostinfo['osversion']])
# Number of threads to test - start from max physical CPU cores and divide in half until 1
minthreads = 1
maxthreads = hostinfo['cpucores']
threads = []
while minthreads < maxthreads:
threads.append(int(minthreads))
minthreads *= 2
threads.append(int(maxthreads))
threads.reverse()
benchtimes = np.zeros(len(threads))
numbermodelruns = len(threads)
usernamespace['number_model_runs'] = numbermodelruns
for currentmodelrun in range(1, numbermodelruns + 1):
os.environ['OMP_NUM_THREADS'] = str(threads[currentmodelrun - 1])
tsolve = run_model(args, currentmodelrun, numbermodelruns, inputfile,
usernamespace)
benchtimes[currentmodelrun - 1] = tsolve
# Save number of threads and benchmarking times to NumPy archive
threads = np.array(threads)
np.savez(os.path.splitext(inputfile.name)[0],
threads=threads,
benchtimes=benchtimes,
machineID=machineIDlong,
version=__version__)
simcompletestr = '\n=== Simulation completed'
print('{} {}\n'.format(
simcompletestr,
'=' * (get_terminal_width() - 1 - len(simcompletestr))))
description=
'Plots execution times and speedup factors from benchmarking models run with different numbers of CPU (OpenMP) threads. Can also benchmark GPU(s) if required. Results are read from a NumPy archive.',
usage='cd gprMax; python -m tests.benchmarking.plot_benchmark numpyfile')
parser.add_argument('baseresult',
help='name of NumPy archive file including path')
parser.add_argument('--otherresults',
default=None,
help='list of NumPy archives file including path',
nargs='+')
args = parser.parse_args()
# Load base result
baseresult = dict(np.load(args.baseresult))
# Get machine/CPU/OS details
hostinfo = get_host_info()
try:
machineIDlong = str(baseresult['machineID'])
# machineIDlong = 'Dell PowerEdge R630; Intel(R) Xeon(R) CPU E5-2630 v3 @ 2.40GHz; Linux (3.10.0-327.18.2.el7.x86_64)' # Use to manually describe machine
machineID = machineIDlong.split(';')[0]
cpuID = machineIDlong.split(';')[1]
cpuID = cpuID.split('GHz')[0].split('x')[1][1::] + 'GHz'
except KeyError:
hyperthreading = ', {} cores with Hyper-Threading'.format(
hostinfo['logicalcores']) if hostinfo['hyperthreading'] else ''
machineIDlong = '{}; {} x {} ({} cores{}); {} RAM; {}'.format(
hostinfo['machineID'], hostinfo['sockets'], hostinfo['cpuID'],
hostinfo['physicalcores'], hyperthreading,
human_size(hostinfo['ram'],
a_kilobyte_is_1024_bytes=True), hostinfo['osversion'])
print('Host: {}'.format(machineIDlong))
def run_main(args):
"""
Top-level function that controls what mode of simulation (standard/optimsation/benchmark etc...) is run.
Args:
args (dict): Namespace with input arguments from command line or api.
"""
with open_path_file(args.inputfile) as inputfile:
# Get information about host machine
hostinfo = get_host_info()
hyperthreading = ', {} cores with Hyper-Threading'.format(hostinfo['logicalcores']) if hostinfo['hyperthreading'] else ''
print('\nHost: {}; {} x {} ({} cores{}); {} RAM; {}'.format(hostinfo['machineID'], hostinfo['sockets'], hostinfo['cpuID'], hostinfo['physicalcores'], hyperthreading, human_size(hostinfo['ram'], a_kilobyte_is_1024_bytes=True), hostinfo['osversion']))
# Get information/setup Nvidia GPU(s)
if args.gpu is not None:
# Extract first item of list, either True to automatically determine device ID,
# or an integer to manually specify device ID
args.gpu = args.gpu[0]
gpus = detect_gpus()
# If a device ID is specified check it is valid
if not isinstance(args.gpu, bool):
if args.gpu > len(gpus) - 1:
raise GeneralError('GPU with device ID {} does not exist'.format(args.gpu))
# Set args.gpu to GPU object to access elsewhere
args.gpu = next(gpu for gpu in gpus if gpu.deviceID == args.gpu)
# If no device ID is specified
else:
# If in MPI mode then set args.gpu to list of available GPUs
if args.mpi:
if args.mpi - 1 > len(gpus):
raise GeneralError('Too many MPI tasks requested ({}). The number of MPI tasks requested can only be a maximum of the number of GPU(s) detected plus one, i.e. {} GPU worker tasks + 1 CPU master task'.format(args.mpi, len(gpus)))
args.gpu = gpus
# If benchmarking mode then set args.gpu to list of available GPUs
elif args.benchmark:
args.gpu = gpus
# Otherwise set args.gpu to GPU object with default device ID (0) to access elsewhere
else:
args.gpu = next(gpu for gpu in gpus if gpu.deviceID == 0)
# Create a separate namespace that users can access in any Python code blocks in the input file
usernamespace = {'c': c, 'e0': e0, 'm0': m0, 'z0': z0, 'number_model_runs': args.n, 'inputfile': os.path.abspath(inputfile.name)}
#######################################
# Process for benchmarking simulation #
#######################################
if args.benchmark:
if args.mpi or args.opt_taguchi or args.task or args.n > 1:
raise GeneralError('Benchmarking mode cannot be combined with MPI, job array, or Taguchi optimisation modes, or multiple model runs.')
run_benchmark_sim(args, inputfile, usernamespace)
####################################################
# Process for simulation with Taguchi optimisation #
####################################################
elif args.opt_taguchi:
if args.mpi_worker: # Special case for MPI spawned workers - they do not need to enter the Taguchi optimisation mode
run_mpi_sim(args, inputfile, usernamespace)
else:
from gprMax.optimisation_taguchi import run_opt_sim
run_opt_sim(args, inputfile, usernamespace)
################################################
# Process for standard simulation (CPU or GPU) #
################################################
else:
# Mixed mode MPI with OpenMP or CUDA - MPI task farm for models with each model parallelised with OpenMP (CPU) or CUDA (GPU)
if args.mpi:
if args.n == 1:
raise GeneralError('MPI is not beneficial when there is only one model to run')
if args.task:
raise GeneralError('MPI cannot be combined with job array mode')
run_mpi_sim(args, inputfile, usernamespace)
# Standard behaviour - models run serially with each model parallelised with OpenMP (CPU) or CUDA (GPU)
else:
if args.task and args.restart:
raise GeneralError('Job array and restart modes cannot be used together')
run_std_sim(args, inputfile, usernamespace)
def process_singlecmds(singlecmds, G):
"""Checks the validity of command parameters and creates instances of classes of parameters.
Args:
singlecmds (dict): Commands that can only occur once in the model.
G (class): Grid class instance - holds essential parameters describing the model.
"""
# Check validity of command parameters in order needed
# messages
cmd = '#messages'
if singlecmds[cmd] is not None:
tmp = singlecmds[cmd].split()
if len(tmp) != 1:
raise CmdInputError(cmd + ' requires exactly one parameter')
if singlecmds[cmd].lower() == 'y':
G.messages = True
elif singlecmds[cmd].lower() == 'n':
G.messages = False
else:
raise CmdInputError(cmd +
' requires input values of either y or n')
# Title
cmd = '#title'
if singlecmds[cmd] is not None:
G.title = singlecmds[cmd]
if G.messages:
print('Model title: {}'.format(G.title))
# Get information about host machine
hostinfo = get_host_info()
# Number of threads (OpenMP) to use
cmd = '#num_threads'
if sys.platform == 'darwin':
os.environ[
'OMP_WAIT_POLICY'] = 'ACTIVE' # Should waiting threads consume CPU power (can drastically effect performance)
os.environ[
'OMP_DYNAMIC'] = 'FALSE' # Number of threads may be adjusted by the run time environment to best utilize system resources
os.environ[
'OMP_PLACES'] = 'cores' # Each place corresponds to a single core (having one or more hardware threads)
os.environ['OMP_PROC_BIND'] = 'TRUE' # Bind threads to physical cores
# os.environ['OMP_DISPLAY_ENV'] = 'TRUE' # Prints OMP version and environment variables (useful for debug)
# Catch bug with Windows Subsystem for Linux (https://github.com/Microsoft/BashOnWindows/issues/785)
if 'Microsoft' in hostinfo['osversion']:
os.environ['KMP_AFFINITY'] = 'disabled'
del os.environ['OMP_PLACES']
del os.environ['OMP_PROC_BIND']
if singlecmds[cmd] is not None:
tmp = tuple(int(x) for x in singlecmds[cmd].split())
if len(tmp) != 1:
raise CmdInputError(
cmd +
' requires exactly one parameter to specify the number of threads to use'
)
if tmp[0] < 1:
raise CmdInputError(
cmd + ' requires the value to be an integer not less than one')
G.nthreads = tmp[0]
os.environ['OMP_NUM_THREADS'] = str(G.nthreads)
elif os.environ.get('OMP_NUM_THREADS'):
G.nthreads = int(os.environ.get('OMP_NUM_THREADS'))
else:
# Set number of threads to number of physical CPU cores
G.nthreads = hostinfo['physicalcores']
os.environ['OMP_NUM_THREADS'] = str(G.nthreads)
if G.messages:
print('Number of CPU (OpenMP) threads: {}'.format(G.nthreads))
if G.nthreads > hostinfo['physicalcores']:
print(
Fore.RED +
'WARNING: You have specified more threads ({}) than available physical CPU cores ({}). This may lead to degraded performance.'
.format(G.nthreads, hostinfo['physicalcores']) + Style.RESET_ALL)
# Spatial discretisation
cmd = '#dx_dy_dz'
tmp = [float(x) for x in singlecmds[cmd].split()]
if len(tmp) != 3:
raise CmdInputError(cmd + ' requires exactly three parameters')
if tmp[0] <= 0:
raise CmdInputError(
cmd +
' requires the x-direction spatial step to be greater than zero')
if tmp[1] <= 0:
raise CmdInputError(
cmd +
' requires the y-direction spatial step to be greater than zero')
if tmp[2] <= 0:
raise CmdInputError(
cmd +
' requires the z-direction spatial step to be greater than zero')
G.dx = tmp[0]
G.dy = tmp[1]
G.dz = tmp[2]
if G.messages:
print('Spatial discretisation: {:g} x {:g} x {:g}m'.format(
G.dx, G.dy, G.dz))
# Domain
cmd = '#domain'
tmp = [float(x) for x in singlecmds[cmd].split()]
if len(tmp) != 3:
raise CmdInputError(cmd + ' requires exactly three parameters')
G.nx = round_value(tmp[0] / G.dx)
G.ny = round_value(tmp[1] / G.dy)
G.nz = round_value(tmp[2] / G.dz)
if G.nx == 0 or G.ny == 0 or G.nz == 0:
raise CmdInputError(cmd +
' requires at least one cell in every dimension')
if G.messages:
print(
'Domain size: {:g} x {:g} x {:g}m ({:d} x {:d} x {:d} = {:g} cells)'
.format(tmp[0], tmp[1], tmp[2], G.nx, G.ny, G.nz,
(G.nx * G.ny * G.nz)))
# Estimate memory (RAM) usage
memestimate = memory_usage(G)
# Check if model can be built and/or run on host
if memestimate > hostinfo['ram']:
raise GeneralError(
'Estimated memory (RAM) required ~{} exceeds {} detected!\n'.
format(human_size(memestimate),
human_size(hostinfo['ram'], a_kilobyte_is_1024_bytes=True)))
if G.messages:
print('Estimated memory (RAM) required: ~{}'.format(
human_size(memestimate)))
# Time step CFL limit (use either 2D or 3D) and default PML thickness
if G.nx == 1:
G.dt = 1 / (c * np.sqrt((1 / G.dy) * (1 / G.dy) + (1 / G.dz) *
(1 / G.dz)))
G.dimension = '2D'
G.pmlthickness['x0'] = 0
G.pmlthickness['xmax'] = 0
elif G.ny == 1:
G.dt = 1 / (c * np.sqrt((1 / G.dx) * (1 / G.dx) + (1 / G.dz) *
(1 / G.dz)))
G.dimension = '2D'
G.pmlthickness['y0'] = 0
G.pmlthickness['ymax'] = 0
elif G.nz == 1:
G.dt = 1 / (c * np.sqrt((1 / G.dx) * (1 / G.dx) + (1 / G.dy) *
(1 / G.dy)))
G.dimension = '2D'
G.pmlthickness['z0'] = 0
G.pmlthickness['zmax'] = 0
else:
G.dt = 1 / (c * np.sqrt((1 / G.dx) * (1 / G.dx) + (1 / G.dy) *
(1 / G.dy) + (1 / G.dz) * (1 / G.dz)))
G.dimension = '3D'
# Round down time step to nearest float with precision one less than hardware maximum. Avoids inadvertently exceeding the CFL due to binary representation of floating point number.
G.dt = round_value(G.dt, decimalplaces=d.getcontext().prec - 1)
if G.messages:
print('Time step (at {} CFL limit): {:g} secs'.format(
G.dimension, G.dt))
# Time step stability factor
cmd = '#time_step_stability_factor'
if singlecmds[cmd] is not None:
tmp = tuple(float(x) for x in singlecmds[cmd].split())
if len(tmp) != 1:
raise CmdInputError(cmd + ' requires exactly one parameter')
if tmp[0] <= 0 or tmp[0] > 1:
raise CmdInputError(
cmd +
' requires the value of the time step stability factor to be between zero and one'
)
G.dt = G.dt * tmp[0]
if G.messages:
print('Time step (modified): {:g} secs'.format(G.dt))
# Time window
cmd = '#time_window'
tmp = singlecmds[cmd].split()
if len(tmp) != 1:
raise CmdInputError(
cmd +
' requires exactly one parameter to specify the time window. Either in seconds or number of iterations.'
)
tmp = tmp[0].lower()
# If number of iterations given
try:
tmp = int(tmp)
G.timewindow = (tmp - 1) * G.dt
G.iterations = tmp
# If real floating point value given
except:
tmp = float(tmp)
if tmp > 0:
G.timewindow = tmp
G.iterations = round_value((tmp / G.dt)) + 1
else:
raise CmdInputError(cmd + ' must have a value greater than zero')
if G.messages:
print('Time window: {:g} secs ({} iterations)'.format(
G.timewindow, G.iterations))
# PML
cmd = '#pml_cells'
if singlecmds[cmd] is not None:
tmp = singlecmds[cmd].split()
if len(tmp) != 1 and len(tmp) != 6:
raise CmdInputError(cmd + ' requires either one or six parameters')
if len(tmp) == 1:
for key in G.pmlthickness.keys():
G.pmlthickness[key] = int(tmp[0])
else:
G.pmlthickness['x0'] = int(tmp[0])
G.pmlthickness['y0'] = int(tmp[1])
G.pmlthickness['z0'] = int(tmp[2])
G.pmlthickness['xmax'] = int(tmp[3])
G.pmlthickness['ymax'] = int(tmp[4])
G.pmlthickness['zmax'] = int(tmp[5])
if 2 * G.pmlthickness['x0'] >= G.nx or 2 * G.pmlthickness[
'y0'] >= G.ny or 2 * G.pmlthickness[
'z0'] >= G.nz or 2 * G.pmlthickness[
'xmax'] >= G.nx or 2 * G.pmlthickness[
'ymax'] >= G.ny or 2 * G.pmlthickness['zmax'] >= G.nz:
raise CmdInputError(cmd + ' has too many cells for the domain size')
# src_steps
cmd = '#src_steps'
if singlecmds[cmd] is not None:
tmp = singlecmds[cmd].split()
if len(tmp) != 3:
raise CmdInputError(cmd + ' requires exactly three parameters')
G.srcsteps[0] = round_value(float(tmp[0]) / G.dx)
G.srcsteps[1] = round_value(float(tmp[1]) / G.dy)
G.srcsteps[2] = round_value(float(tmp[2]) / G.dz)
if G.messages:
print(
'Simple sources will step {:g}m, {:g}m, {:g}m for each model run.'
.format(G.srcsteps[0] * G.dx, G.srcsteps[1] * G.dy,
G.srcsteps[2] * G.dz))
# rx_steps
cmd = '#rx_steps'
if singlecmds[cmd] is not None:
tmp = singlecmds[cmd].split()
if len(tmp) != 3:
raise CmdInputError(cmd + ' requires exactly three parameters')
G.rxsteps[0] = round_value(float(tmp[0]) / G.dx)
G.rxsteps[1] = round_value(float(tmp[1]) / G.dy)
G.rxsteps[2] = round_value(float(tmp[2]) / G.dz)
if G.messages:
print(
'All receivers will step {:g}m, {:g}m, {:g}m for each model run.'
.format(G.rxsteps[0] * G.dx, G.rxsteps[1] * G.dy,
G.rxsteps[2] * G.dz))
# Excitation file for user-defined source waveforms
cmd = '#excitation_file'
if singlecmds[cmd] is not None:
tmp = singlecmds[cmd].split()
if len(tmp) != 1:
raise CmdInputError(cmd + ' requires exactly one parameter')
excitationfile = tmp[0]
# See if file exists at specified path and if not try input file directory
if not os.path.isfile(excitationfile):
excitationfile = os.path.abspath(
os.path.join(G.inputdirectory, excitationfile))
# Get waveform names
with open(excitationfile, 'r') as f:
waveformIDs = f.readline().split()
# Read all waveform values into an array
waveformvalues = np.loadtxt(excitationfile,
skiprows=1,
dtype=floattype)
for waveform in range(len(waveformIDs)):
if any(x.ID == waveformIDs[waveform] for x in G.waveforms):
raise CmdInputError(
'Waveform with ID {} already exists'.format(
waveformIDs[waveform]))
w = Waveform()
w.ID = waveformIDs[waveform]
w.type = 'user'
if len(waveformvalues.shape) == 1:
w.uservalues = waveformvalues[:]
else:
w.uservalues = waveformvalues[:, waveform]
if G.messages:
print('User waveform {} created.'.format(w.ID))
G.waveforms.append(w)
def run_main(args):
"""
Top-level function that controls what mode of simulation (standard/optimsation/benchmark etc...) is run.
Args:
args (dict): Namespace with input arguments from command line or api.
"""
# Print gprMax logo, version, and licencing/copyright information
logo(__version__ + ' (' + codename + ')')
with open_path_file(args.inputfile) as inputfile:
# Get information about host machine
hostinfo = get_host_info()
hyperthreading = ', {} cores with Hyper-Threading'.format(
hostinfo['logicalcores']) if hostinfo['hyperthreading'] else ''
print('\nHost: {} | {} | {} x {} ({} cores{}) | {} RAM | {}'.format(
hostinfo['hostname'], hostinfo['machineID'], hostinfo['sockets'],
hostinfo['cpuID'], hostinfo['physicalcores'], hyperthreading,
human_size(hostinfo['ram'],
a_kilobyte_is_1024_bytes=True), hostinfo['osversion']))
# Get information/setup any Nvidia GPU(s)
if args.gpu is not None:
# Flatten a list of lists
if any(isinstance(element, list) for element in args.gpu):
args.gpu = [val for sublist in args.gpu for val in sublist]
gpus, allgpustext = detect_check_gpus(args.gpu)
print('GPU(s) detected: {}'.format(' | '.join(allgpustext)))
# If in MPI mode or benchmarking provide list of GPU objects, otherwise
# provide single GPU object
if args.mpi or args.mpi_no_spawn or args.benchmark:
args.gpu = gpus
else:
args.gpu = gpus[0]
# Create a separate namespace that users can access in any Python code blocks in the input file
usernamespace = {
'c': c,
'e0': e0,
'm0': m0,
'z0': z0,
'number_model_runs': args.n,
'inputfile': os.path.abspath(inputfile.name)
}
#######################################
# Process for benchmarking simulation #
#######################################
if args.benchmark:
if args.mpi or args.opt_taguchi or args.task or args.n > 1:
raise GeneralError(
'Benchmarking mode cannot be combined with MPI, job array, or Taguchi optimisation modes, or multiple model runs.'
)
run_benchmark_sim(args, inputfile, usernamespace)
####################################################
# Process for simulation with Taguchi optimisation #
####################################################
elif args.opt_taguchi:
if args.mpi_worker: # Special case for MPI spawned workers - they do not need to enter the Taguchi optimisation mode
run_mpi_sim(args, inputfile, usernamespace)
else:
from gprMax.optimisation_taguchi import run_opt_sim
run_opt_sim(args, inputfile, usernamespace)
################################################
# Process for standard simulation (CPU or GPU) #
################################################
else:
# Mixed mode MPI with OpenMP or CUDA - MPI task farm for models with each model parallelised with OpenMP (CPU) or CUDA (GPU)
if args.mpi:
if args.n == 1:
raise GeneralError(
'MPI is not beneficial when there is only one model to run'
)
if args.task:
raise GeneralError(
'MPI cannot be combined with job array mode')
run_mpi_sim(args, inputfile, usernamespace)
# Alternate MPI configuration that does not use MPI spawn mechanism
elif args.mpi_no_spawn:
if args.n == 1:
raise GeneralError(
'MPI is not beneficial when there is only one model to run'
)
if args.task:
raise GeneralError(
'MPI cannot be combined with job array mode')
run_mpi_no_spawn_sim(args, inputfile, usernamespace)
# Standard behaviour - models run serially with each model parallelised with OpenMP (CPU) or CUDA (GPU)
else:
if args.task and args.restart:
raise GeneralError(
'Job array and restart modes cannot be used together')
run_std_sim(args, inputfile, usernamespace)
def run_model(args, currentmodelrun, modelend, numbermodelruns, inputfile, usernamespace):
"""Runs a model - processes the input file; builds the Yee cells; calculates update coefficients; runs main FDTD loop.
Args:
args (dict): Namespace with command line arguments
currentmodelrun (int): Current model run number.
modelend (int): Number of last model to run.
numbermodelruns (int): Total number of model runs.
inputfile (object): File object for the input file.
usernamespace (dict): Namespace that can be accessed by user
in any Python code blocks in input file.
Returns:
tsolve (int): Length of time (seconds) of main FDTD calculations
"""
# Monitor memory usage
p = psutil.Process()
# Declare variable to hold FDTDGrid class
global G
# Used for naming geometry and output files
appendmodelnumber = '' if numbermodelruns == 1 and not args.task and not args.restart else '_'+str(currentmodelrun)
appendmodelnumberGeometry = '' if numbermodelruns == 1 and not args.task and not args.restart or args.geometry_fixed else '_'+str(currentmodelrun)
# Normal model reading/building process; bypassed if geometry information to be reused
if 'G' not in globals():
# Initialise an instance of the FDTDGrid class
G = FDTDGrid()
# Get information about host machine
# (need to save this info to FDTDGrid instance after it has been created)
G.hostinfo = get_host_info()
# Single GPU object
if args.gpu:
G.gpu = args.gpu
G.inputfilename = os.path.split(inputfile.name)[1]
G.inputdirectory = os.path.dirname(os.path.abspath(inputfile.name))
inputfilestr = '\n--- Model {}/{}, input file: {}'.format(currentmodelrun, modelend, inputfile.name)
if G.messages:
print(Fore.GREEN + '{} {}\n'.format(inputfilestr, '-' * (get_terminal_width() - 1 - len(inputfilestr))) + Style.RESET_ALL)
# Add the current model run to namespace that can be accessed by
# user in any Python code blocks in input file
usernamespace['current_model_run'] = currentmodelrun
# Read input file and process any Python and include file commands
processedlines = process_python_include_code(inputfile, usernamespace)
# Print constants/variables in user-accessable namespace
uservars = ''
for key, value in sorted(usernamespace.items()):
if key != '__builtins__':
uservars += '{}: {}, '.format(key, value)
if G.messages:
print('Constants/variables used/available for Python scripting: {{{}}}\n'.format(uservars[:-2]))
# Write a file containing the input commands after Python or include file commands have been processed
if args.write_processed:
write_processed_file(processedlines, appendmodelnumber, G)
# Check validity of command names and that essential commands are present
singlecmds, multicmds, geometry = check_cmd_names(processedlines)
# Create built-in materials
m = Material(0, 'pec')
m.se = float('inf')
m.type = 'builtin'
m.averagable = False
G.materials.append(m)
m = Material(1, 'free_space')
m.type = 'builtin'
G.materials.append(m)
# Process parameters for commands that can only occur once in the model
process_singlecmds(singlecmds, G)
# Process parameters for commands that can occur multiple times in the model
if G.messages: print()
process_multicmds(multicmds, G)
# Estimate and check memory (RAM) usage
G.memory_estimate_basic()
#G.memory_check()
#if G.messages:
# if G.gpu is None:
# print('\nMemory (RAM) required: ~{}\n'.format(human_size(G.memoryusage)))
# else:
# print('\nMemory (RAM) required: ~{} host + ~{} GPU\n'.format(human_size(G.memoryusage), human_size(G.memoryusage)))
# Initialise an array for volumetric material IDs (solid), boolean
# arrays for specifying materials not to be averaged (rigid),
# an array for cell edge IDs (ID)
G.initialise_geometry_arrays()
# Initialise arrays for the field components
if G.gpu is None:
G.initialise_field_arrays()
# Process geometry commands in the order they were given
process_geometrycmds(geometry, G)
# Build the PMLs and calculate initial coefficients
if G.messages: print()
if all(value == 0 for value in G.pmlthickness.values()):
if G.messages:
print('PML: switched off')
pass # If all the PMLs are switched off don't need to build anything
else:
# Set default CFS parameters for PML if not given
if not G.cfs:
G.cfs = [CFS()]
if G.messages:
if all(value == G.pmlthickness['x0'] for value in G.pmlthickness.values()):
pmlinfo = str(G.pmlthickness['x0'])
else:
pmlinfo = ''
for key, value in G.pmlthickness.items():
pmlinfo += '{}: {}, '.format(key, value)
pmlinfo = pmlinfo[:-2] + ' cells'
print('PML: formulation: {}, order: {}, thickness: {}'.format(G.pmlformulation, len(G.cfs), pmlinfo))
pbar = tqdm(total=sum(1 for value in G.pmlthickness.values() if value > 0), desc='Building PML boundaries', ncols=get_terminal_width() - 1, file=sys.stdout, disable=not G.progressbars)
build_pmls(G, pbar)
pbar.close()
# Build the model, i.e. set the material properties (ID) for every edge
# of every Yee cell
if G.messages: print()
pbar = tqdm(total=2, desc='Building main grid', ncols=get_terminal_width() - 1, file=sys.stdout, disable=not G.progressbars)
build_electric_components(G.solid, G.rigidE, G.ID, G)
pbar.update()
build_magnetic_components(G.solid, G.rigidH, G.ID, G)
pbar.update()
pbar.close()
# Add PEC boundaries to invariant direction in 2D modes
# N.B. 2D modes are a single cell slice of 3D grid
if '2D TMx' in G.mode:
# Ey & Ez components
G.ID[1, 0, :, :] = 0
G.ID[1, 1, :, :] = 0
G.ID[2, 0, :, :] = 0
G.ID[2, 1, :, :] = 0
elif '2D TMy' in G.mode:
# Ex & Ez components
G.ID[0, :, 0, :] = 0
G.ID[0, :, 1, :] = 0
G.ID[2, :, 0, :] = 0
G.ID[2, :, 1, :] = 0
elif '2D TMz' in G.mode:
# Ex & Ey components
G.ID[0, :, :, 0] = 0
G.ID[0, :, :, 1] = 0
G.ID[1, :, :, 0] = 0
G.ID[1, :, :, 1] = 0
# Process any voltage sources (that have resistance) to create a new
# material at the source location
for voltagesource in G.voltagesources:
voltagesource.create_material(G)
# Initialise arrays of update coefficients to pass to update functions
G.initialise_std_update_coeff_arrays()
# Initialise arrays of update coefficients and temporary values if
# there are any dispersive materials
if Material.maxpoles != 0:
# Update estimated memory (RAM) usage
G.memoryusage += int(3 * Material.maxpoles * (G.nx + 1) * (G.ny + 1) * (G.nz + 1) * np.dtype(complextype).itemsize)
G.memory_check()
if G.messages:
print('\nMemory (RAM) required - updated (dispersive): ~{}\n'.format(human_size(G.memoryusage)))
G.initialise_dispersive_arrays()
# Check there is sufficient memory to store any snapshots
if G.snapshots:
snapsmemsize = 0
for snap in G.snapshots:
# 2 x required to account for electric and magnetic fields
snapsmemsize += (2 * snap.datasizefield)
G.memoryusage += int(snapsmemsize)
G.memory_check(snapsmemsize=int(snapsmemsize))
if G.messages:
print('\nMemory (RAM) required - updated (snapshots): ~{}\n'.format(human_size(G.memoryusage)))
# Process complete list of materials - calculate update coefficients,
# store in arrays, and build text list of materials/properties
materialsdata = process_materials(G)
if G.messages:
print('\nMaterials:')
materialstable = AsciiTable(materialsdata)
materialstable.outer_border = False
materialstable.justify_columns[0] = 'right'
print(materialstable.table)
# Check to see if numerical dispersion might be a problem
results = dispersion_analysis(G)
if results['error'] and G.messages:
print(Fore.RED + "\nWARNING: Numerical dispersion analysis not carried out as {}".format(results['error']) + Style.RESET_ALL)
elif results['N'] < G.mingridsampling:
raise GeneralError("Non-physical wave propagation: Material '{}' has wavelength sampled by {} cells, less than required minimum for physical wave propagation. Maximum significant frequency estimated as {:g}Hz".format(results['material'].ID, results['N'], results['maxfreq']))
elif results['deltavp'] and np.abs(results['deltavp']) > G.maxnumericaldisp and G.messages:
print(Fore.RED + "\nWARNING: Potentially significant numerical dispersion. Estimated largest physical phase-velocity error is {:.2f}% in material '{}' whose wavelength sampled by {} cells. Maximum significant frequency estimated as {:g}Hz".format(results['deltavp'], results['material'].ID, results['N'], results['maxfreq']) + Style.RESET_ALL)
elif results['deltavp'] and G.messages:
print("\nNumerical dispersion analysis: estimated largest physical phase-velocity error is {:.2f}% in material '{}' whose wavelength sampled by {} cells. Maximum significant frequency estimated as {:g}Hz".format(results['deltavp'], results['material'].ID, results['N'], results['maxfreq']))
# If geometry information to be reused between model runs
else:
inputfilestr = '\n--- Model {}/{}, input file (not re-processed, i.e. geometry fixed): {}'.format(currentmodelrun, modelend, inputfile.name)
if G.messages:
print(Fore.GREEN + '{} {}\n'.format(inputfilestr, '-' * (get_terminal_width() - 1 - len(inputfilestr))) + Style.RESET_ALL)
if G.gpu is None:
# Clear arrays for field components
G.initialise_field_arrays()
# Clear arrays for fields in PML
for pml in G.pmls:
pml.initialise_field_arrays()
# Adjust position of simple sources and receivers if required
if G.srcsteps[0] != 0 or G.srcsteps[1] != 0 or G.srcsteps[2] != 0:
for source in itertools.chain(G.hertziandipoles, G.magneticdipoles):
if currentmodelrun == 1:
if source.xcoord + G.srcsteps[0] * modelend < 0 or source.xcoord + G.srcsteps[0] * modelend > G.nx or source.ycoord + G.srcsteps[1] * modelend < 0 or source.ycoord + G.srcsteps[1] * modelend > G.ny or source.zcoord + G.srcsteps[2] * modelend < 0 or source.zcoord + G.srcsteps[2] * modelend > G.nz:
raise GeneralError('Source(s) will be stepped to a position outside the domain.')
source.xcoord = source.xcoordorigin + (currentmodelrun - 1) * G.srcsteps[0]
source.ycoord = source.ycoordorigin + (currentmodelrun - 1) * G.srcsteps[1]
source.zcoord = source.zcoordorigin + (currentmodelrun - 1) * G.srcsteps[2]
if G.rxsteps[0] != 0 or G.rxsteps[1] != 0 or G.rxsteps[2] != 0:
for receiver in G.rxs:
if currentmodelrun == 1:
if receiver.xcoord + G.rxsteps[0] * modelend < 0 or receiver.xcoord + G.rxsteps[0] * modelend > G.nx or receiver.ycoord + G.rxsteps[1] * modelend < 0 or receiver.ycoord + G.rxsteps[1] * modelend > G.ny or receiver.zcoord + G.rxsteps[2] * modelend < 0 or receiver.zcoord + G.rxsteps[2] * modelend > G.nz:
raise GeneralError('Receiver(s) will be stepped to a position outside the domain.')
receiver.xcoord = receiver.xcoordorigin + (currentmodelrun - 1) * G.rxsteps[0]
receiver.ycoord = receiver.ycoordorigin + (currentmodelrun - 1) * G.rxsteps[1]
receiver.zcoord = receiver.zcoordorigin + (currentmodelrun - 1) * G.rxsteps[2]
# Write files for any geometry views and geometry object outputs
if not (G.geometryviews or G.geometryobjectswrite) and args.geometry_only and G.messages:
print(Fore.RED + '\nWARNING: No geometry views or geometry objects to output found.' + Style.RESET_ALL)
if G.geometryviews and (not args.geometry_fixed or currentmodelrun == 1):
if G.messages: print()
for i, geometryview in enumerate(G.geometryviews):
geometryview.set_filename(appendmodelnumberGeometry, G)
pbar = tqdm(total=geometryview.datawritesize, unit='byte', unit_scale=True, desc='Writing geometry view file {}/{}, {}'.format(i + 1, len(G.geometryviews), os.path.split(geometryview.filename)[1]), ncols=get_terminal_width() - 1, file=sys.stdout, disable=not G.progressbars)
geometryview.write_vtk(G, pbar)
pbar.close()
if G.geometryobjectswrite:
for i, geometryobject in enumerate(G.geometryobjectswrite):
pbar = tqdm(total=geometryobject.datawritesize, unit='byte', unit_scale=True, desc='Writing geometry object file {}/{}, {}'.format(i + 1, len(G.geometryobjectswrite), os.path.split(geometryobject.filename)[1]), ncols=get_terminal_width() - 1, file=sys.stdout, disable=not G.progressbars)
geometryobject.write_hdf5(G, pbar)
pbar.close()
# If only writing geometry information
if args.geometry_only:
tsolve = 0
# Run simulation
else:
# Output filename
inputdirectory, inputfilename = os.path.split(os.path.join(G.inputdirectory, G.inputfilename))
if G.outputdirectory is None:
outputdir = inputdirectory
else:
outputdir = G.outputdirectory
# Save current directory
curdir = os.getcwd()
os.chdir(inputdirectory)
outputdir = os.path.abspath(outputdir)
if not os.path.isdir(outputdir):
os.mkdir(outputdir)
if G.messages:
print('\nCreated output directory: {}'.format(outputdir))
# Restore current directory
os.chdir(curdir)
basename, ext = os.path.splitext(inputfilename)
outputfile = os.path.join(outputdir, basename + appendmodelnumber + '.out')
if G.messages:
print('\nOutput file: {}\n'.format(outputfile))
# Main FDTD solving functions for either CPU or GPU
if G.gpu is None:
tsolve = solve_cpu(currentmodelrun, modelend, G)
else:
tsolve, memsolve = solve_gpu(currentmodelrun, modelend, G)
# Write an output file in HDF5 format
write_hdf5_outputfile(outputfile, G)
# Write any snapshots to file
if G.snapshots:
# Create directory and construct filename from user-supplied name and model run number
snapshotdir = os.path.join(G.inputdirectory, os.path.splitext(G.inputfilename)[0] + '_snaps' + appendmodelnumber)
if not os.path.exists(snapshotdir):
os.mkdir(snapshotdir)
if G.messages: print()
for i, snap in enumerate(G.snapshots):
snap.filename = os.path.abspath(os.path.join(snapshotdir, snap.basefilename + '.vti'))
pbar = tqdm(total=snap.vtkdatawritesize, leave=True, unit='byte', unit_scale=True, desc='Writing snapshot file {} of {}, {}'.format(i + 1, len(G.snapshots), os.path.split(snap.filename)[1]), ncols=get_terminal_width() - 1, file=sys.stdout, disable=not G.progressbars)
snap.write_vtk_imagedata(pbar, G)
pbar.close()
if G.messages: print()
if G.messages:
if G.gpu is None:
print('Memory (RAM) used: ~{}'.format(human_size(p.memory_info().rss)))
else:
print('Memory (RAM) used: ~{} host + ~{} GPU'.format(human_size(p.memory_info().rss), human_size(memsolve)))
print('Solving time [HH:MM:SS]: {}'.format(datetime.timedelta(seconds=tsolve)))
# If geometry information to be reused between model runs then FDTDGrid
# class instance must be global so that it persists
if not args.geometry_fixed or currentmodelrun is modelend:
del G
return tsolve
def process_singlecmds(singlecmds, G):
"""Checks the validity of command parameters and creates instances of classes of parameters.
Args:
singlecmds (dict): Commands that can only occur once in the model.
G (class): Grid class instance - holds essential parameters describing the model.
"""
# Check validity of command parameters in order needed
# messages
cmd = '#messages'
if singlecmds[cmd] is not None:
tmp = singlecmds[cmd].split()
if len(tmp) != 1:
raise CmdInputError(cmd + ' requires exactly one parameter')
if singlecmds[cmd].lower() == 'y':
G.messages = True
elif singlecmds[cmd].lower() == 'n':
G.messages = False
else:
raise CmdInputError(cmd + ' requires input values of either y or n')
# Title
cmd = '#title'
if singlecmds[cmd] is not None:
G.title = singlecmds[cmd]
if G.messages:
print('Model title: {}'.format(G.title))
# Get information about host machine
hostinfo = get_host_info()
# Number of threads (OpenMP) to use
cmd = '#num_threads'
if sys.platform == 'darwin':
os.environ['OMP_WAIT_POLICY'] = 'ACTIVE' # Should waiting threads consume CPU power (can drastically effect performance)
os.environ['OMP_DYNAMIC'] = 'FALSE' # Number of threads may be adjusted by the run time environment to best utilize system resources
os.environ['OMP_PLACES'] = 'cores' # Each place corresponds to a single core (having one or more hardware threads)
os.environ['OMP_PROC_BIND'] = 'TRUE' # Bind threads to physical cores
# os.environ['OMP_DISPLAY_ENV'] = 'TRUE' # Prints OMP version and environment variables (useful for debug)
# Catch bug with Windows Subsystem for Linux (https://github.com/Microsoft/BashOnWindows/issues/785)
if 'Microsoft' in G.hostinfo['osversion']:
os.environ['KMP_AFFINITY'] = 'disabled'
del os.environ['OMP_PLACES']
del os.environ['OMP_PROC_BIND']
if singlecmds[cmd] is not None:
tmp = tuple(int(x) for x in singlecmds[cmd].split())
if len(tmp) != 1:
raise CmdInputError(cmd + ' requires exactly one parameter to specify the number of threads to use')
if tmp[0] < 1:
raise CmdInputError(cmd + ' requires the value to be an integer not less than one')
G.nthreads = tmp[0]
os.environ['OMP_NUM_THREADS'] = str(G.nthreads)
elif os.environ.get('OMP_NUM_THREADS'):
G.nthreads = int(os.environ.get('OMP_NUM_THREADS'))
else:
# Set number of threads to number of physical CPU cores
G.nthreads = hostinfo['physicalcores']
os.environ['OMP_NUM_THREADS'] = str(G.nthreads)
if G.messages:
print('Number of CPU (OpenMP) threads: {}'.format(G.nthreads))
if G.nthreads > G.hostinfo['physicalcores']:
print(Fore.RED + 'WARNING: You have specified more threads ({}) than available physical CPU cores ({}). This may lead to degraded performance.'.format(G.nthreads, hostinfo['physicalcores']) + Style.RESET_ALL)
# Print information about any GPU in use
if G.messages:
if G.gpu is not None:
print('GPU solving using: {} - {}'.format(G.gpu.deviceID, G.gpu.name))
# Spatial discretisation
cmd = '#dx_dy_dz'
tmp = [float(x) for x in singlecmds[cmd].split()]
if len(tmp) != 3:
raise CmdInputError(cmd + ' requires exactly three parameters')
if tmp[0] <= 0:
raise CmdInputError(cmd + ' requires the x-direction spatial step to be greater than zero')
if tmp[1] <= 0:
raise CmdInputError(cmd + ' requires the y-direction spatial step to be greater than zero')
if tmp[2] <= 0:
raise CmdInputError(cmd + ' requires the z-direction spatial step to be greater than zero')
G.dx = tmp[0]
G.dy = tmp[1]
G.dz = tmp[2]
if G.messages:
print('Spatial discretisation: {:g} x {:g} x {:g}m'.format(G.dx, G.dy, G.dz))
# Domain
cmd = '#domain'
tmp = [float(x) for x in singlecmds[cmd].split()]
if len(tmp) != 3:
raise CmdInputError(cmd + ' requires exactly three parameters')
G.nx = round_value(tmp[0] / G.dx)
G.ny = round_value(tmp[1] / G.dy)
G.nz = round_value(tmp[2] / G.dz)
if G.nx == 0 or G.ny == 0 or G.nz == 0:
raise CmdInputError(cmd + ' requires at least one cell in every dimension')
if G.messages:
print('Domain size: {:g} x {:g} x {:g}m ({:d} x {:d} x {:d} = {:g} cells)'.format(tmp[0], tmp[1], tmp[2], G.nx, G.ny, G.nz, (G.nx * G.ny * G.nz)))
# Time step CFL limit (either 2D or 3D); switch off appropriate PMLs for 2D
if G.nx == 1:
G.dt = 1 / (c * np.sqrt((1 / G.dy) * (1 / G.dy) + (1 / G.dz) * (1 / G.dz)))
G.mode = '2D TMx'
G.pmlthickness['x0'] = 0
G.pmlthickness['xmax'] = 0
elif G.ny == 1:
G.dt = 1 / (c * np.sqrt((1 / G.dx) * (1 / G.dx) + (1 / G.dz) * (1 / G.dz)))
G.mode = '2D TMy'
G.pmlthickness['y0'] = 0
G.pmlthickness['ymax'] = 0
elif G.nz == 1:
G.dt = 1 / (c * np.sqrt((1 / G.dx) * (1 / G.dx) + (1 / G.dy) * (1 / G.dy)))
G.mode = '2D TMz'
G.pmlthickness['z0'] = 0
G.pmlthickness['zmax'] = 0
else:
G.dt = 1 / (c * np.sqrt((1 / G.dx) * (1 / G.dx) + (1 / G.dy) * (1 / G.dy) + (1 / G.dz) * (1 / G.dz)))
G.mode = '3D'
# Round down time step to nearest float with precision one less than hardware maximum.
# Avoids inadvertently exceeding the CFL due to binary representation of floating point number.
G.dt = round_value(G.dt, decimalplaces=d.getcontext().prec - 1)
if G.messages:
print('Mode: {}'.format(G.mode))
print('Time step (at CFL limit): {:g} secs'.format(G.dt))
# Time step stability factor
cmd = '#time_step_stability_factor'
if singlecmds[cmd] is not None:
tmp = tuple(float(x) for x in singlecmds[cmd].split())
if len(tmp) != 1:
raise CmdInputError(cmd + ' requires exactly one parameter')
if tmp[0] <= 0 or tmp[0] > 1:
raise CmdInputError(cmd + ' requires the value of the time step stability factor to be between zero and one')
G.dt = G.dt * tmp[0]
if G.messages:
print('Time step (modified): {:g} secs'.format(G.dt))
# Time window
cmd = '#time_window'
tmp = singlecmds[cmd].split()
if len(tmp) != 1:
raise CmdInputError(cmd + ' requires exactly one parameter to specify the time window. Either in seconds or number of iterations.')
tmp = tmp[0].lower()
# If number of iterations given
# The +/- 1 used in calculating the number of iterations is to account for
# the fact that the solver (iterations) loop runs from 0 to < G.iterations
try:
tmp = int(tmp)
G.timewindow = (tmp - 1) * G.dt
G.iterations = tmp
# If real floating point value given
except ValueError:
tmp = float(tmp)
if tmp > 0:
G.timewindow = tmp
G.iterations = int(np.ceil(tmp / G.dt)) + 1
else:
raise CmdInputError(cmd + ' must have a value greater than zero')
if G.messages:
print('Time window: {:g} secs ({} iterations)'.format(G.timewindow, G.iterations))
# PML cells
cmd = '#pml_cells'
if singlecmds[cmd] is not None:
tmp = singlecmds[cmd].split()
if len(tmp) != 1 and len(tmp) != 6:
raise CmdInputError(cmd + ' requires either one or six parameter(s)')
if len(tmp) == 1:
for key in G.pmlthickness.keys():
G.pmlthickness[key] = int(tmp[0])
else:
G.pmlthickness['x0'] = int(tmp[0])
G.pmlthickness['y0'] = int(tmp[1])
G.pmlthickness['z0'] = int(tmp[2])
G.pmlthickness['xmax'] = int(tmp[3])
G.pmlthickness['ymax'] = int(tmp[4])
G.pmlthickness['zmax'] = int(tmp[5])
if 2 * G.pmlthickness['x0'] >= G.nx or 2 * G.pmlthickness['y0'] >= G.ny or 2 * G.pmlthickness['z0'] >= G.nz or 2 * G.pmlthickness['xmax'] >= G.nx or 2 * G.pmlthickness['ymax'] >= G.ny or 2 * G.pmlthickness['zmax'] >= G.nz:
raise CmdInputError(cmd + ' has too many cells for the domain size')
# PML formulation
cmd = '#pml_formulation'
if singlecmds[cmd] is not None:
tmp = singlecmds[cmd].split()
if len(tmp) != 1:
raise CmdInputError(cmd + ' requires exactly one parameter')
if singlecmds[cmd].upper() in PML.formulations:
G.pmlformulation = singlecmds[cmd].upper()
else:
raise CmdInputError(cmd + ' PML formulation is not found')
# src_steps
cmd = '#src_steps'
if singlecmds[cmd] is not None:
tmp = singlecmds[cmd].split()
if len(tmp) != 3:
raise CmdInputError(cmd + ' requires exactly three parameters')
G.srcsteps[0] = round_value(float(tmp[0]) / G.dx)
G.srcsteps[1] = round_value(float(tmp[1]) / G.dy)
G.srcsteps[2] = round_value(float(tmp[2]) / G.dz)
if G.messages:
print('Simple sources will step {:g}m, {:g}m, {:g}m for each model run.'.format(G.srcsteps[0] * G.dx, G.srcsteps[1] * G.dy, G.srcsteps[2] * G.dz))
# rx_steps
cmd = '#rx_steps'
if singlecmds[cmd] is not None:
tmp = singlecmds[cmd].split()
if len(tmp) != 3:
raise CmdInputError(cmd + ' requires exactly three parameters')
G.rxsteps[0] = round_value(float(tmp[0]) / G.dx)
G.rxsteps[1] = round_value(float(tmp[1]) / G.dy)
G.rxsteps[2] = round_value(float(tmp[2]) / G.dz)
if G.messages:
print('All receivers will step {:g}m, {:g}m, {:g}m for each model run.'.format(G.rxsteps[0] * G.dx, G.rxsteps[1] * G.dy, G.rxsteps[2] * G.dz))
# Excitation file for user-defined source waveforms
cmd = '#excitation_file'
if singlecmds[cmd] is not None:
tmp = singlecmds[cmd].split()
if len(tmp) != 1 and len(tmp) != 3:
raise CmdInputError(cmd + ' requires either one or three parameter(s)')
excitationfile = tmp[0]
# Optional parameters passed directly to scipy.interpolate.interp1d
kwargs = dict()
if len(tmp) > 1:
kwargs['kind'] = tmp[1]
kwargs['fill_value'] = tmp[2]
else:
args, varargs, keywords, defaults = inspect.getargspec(interpolate.interp1d)
kwargs = dict(zip(reversed(args), reversed(defaults)))
# See if file exists at specified path and if not try input file directory
if not os.path.isfile(excitationfile):
excitationfile = os.path.abspath(os.path.join(G.inputdirectory, excitationfile))
if G.messages:
print('\nExcitation file: {}'.format(excitationfile))
# Get waveform names
with open(excitationfile, 'r') as f:
waveformIDs = f.readline().split()
# Read all waveform values into an array
waveformvalues = np.loadtxt(excitationfile, skiprows=1, dtype=floattype)
# Time array (if specified) for interpolation, otherwise use simulation time
if waveformIDs[0].lower() == 'time':
waveformIDs = waveformIDs[1:]
waveformtime = waveformvalues[:, 0]
waveformvalues = waveformvalues[:, 1:]
timestr = 'user-defined time array'
else:
waveformtime = np.arange(0, G.timewindow + G.dt, G.dt)
timestr = 'simulation time array'
for waveform in range(len(waveformIDs)):
if any(x.ID == waveformIDs[waveform] for x in G.waveforms):
raise CmdInputError('Waveform with ID {} already exists'.format(waveformIDs[waveform]))
w = Waveform()
w.ID = waveformIDs[waveform]
w.type = 'user'
# Select correct column of waveform values depending on array shape
singlewaveformvalues = waveformvalues[:] if len(waveformvalues.shape) == 1 else waveformvalues[:, waveform]
# Truncate waveform array if it is longer than time array
if len(singlewaveformvalues) > len(waveformtime):
singlewaveformvalues = singlewaveformvalues[:len(waveformtime)]
# Zero-pad end of waveform array if it is shorter than time array
elif len(singlewaveformvalues) < len(waveformtime):
tmp = np.zeros(len(waveformtime))
tmp[:len(singlewaveformvalues)] = singlewaveformvalues
singlewaveformvalues = tmp
# Interpolate waveform values
w.userfunc = interpolate.interp1d(waveformtime, singlewaveformvalues, **kwargs)
if G.messages:
print('User waveform {} created using {} and, if required, interpolation parameters (kind: {}, fill value: {}).'.format(w.ID, timestr, kwargs['kind'], kwargs['fill_value']))
G.waveforms.append(w)
# Set the output directory
cmd = '#output_dir'
if singlecmds[cmd] is not None:
outputdir = singlecmds[cmd]
G.outputdirectory = outputdir
def run_main(args):
"""Top-level function that controls what mode of simulation (standard/optimsation/benchmark etc...) is run.
Args:
args (dict): Namespace with input arguments from command line or api.
"""
numbermodelruns = args.n
with open_path_file(args.inputfile) as inputfile:
# Get information about host machine
hostinfo = get_host_info()
print('\nHost: {}; {} ({} cores); {} RAM; {}'.format(
hostinfo['machineID'], hostinfo['cpuID'], hostinfo['cpucores'],
human_size(hostinfo['ram'], a_kilobyte_is_1024_bytes=True),
hostinfo['osversion']))
# Create a separate namespace that users can access in any Python code blocks in the input file
usernamespace = {
'c': c,
'e0': e0,
'm0': m0,
'z0': z0,
'number_model_runs': numbermodelruns,
'input_directory': os.path.dirname(os.path.abspath(inputfile.name))
}
#######################################
# Process for benchmarking simulation #
#######################################
if args.benchmark:
run_benchmark_sim(args, inputfile, usernamespace)
####################################################
# Process for simulation with Taguchi optimisation #
####################################################
elif args.opt_taguchi:
if args.benchmark:
raise GeneralError(
'Taguchi optimisation should not be used with benchmarking mode'
)
from gprMax.optimisation_taguchi import run_opt_sim
run_opt_sim(args, numbermodelruns, inputfile, usernamespace)
################################################
# Process for standard simulation (CPU or GPU) #
################################################
else:
# Mixed mode MPI with OpenMP or CUDA - MPI task farm for models with each model parallelised with OpenMP (CPU) or CUDA (GPU)
if args.mpi:
if args.benchmark:
raise GeneralError(
'MPI should not be used with benchmarking mode')
if numbermodelruns == 1:
raise GeneralError(
'MPI is not beneficial when there is only one model to run'
)
run_mpi_sim(args, numbermodelruns, inputfile, usernamespace)
# Standard behaviour - part of a job array on Open Grid Scheduler/Grid Engine with each model parallelised with OpenMP (CPU) or CUDA (GPU)
elif args.task:
if args.benchmark:
raise GeneralError(
'A job array should not be used with benchmarking mode'
)
run_job_array_sim(args, numbermodelruns, inputfile,
usernamespace)
# Standard behaviour - models run serially with each model parallelised with OpenMP (CPU) or CUDA (GPU)
else:
run_std_sim(args, numbermodelruns, inputfile, usernamespace)