def memory_check(self, snapsmemsize=0):
"""Check if the required amount of memory (RAM) is available on the host and GPU if specified.
Args:
snapsmemsize (int): amount of memory (bytes) required to store all requested snapshots
"""
# Check if model can be built and/or run on host
if self.memoryusage > self.hostinfo['ram']:
raise GeneralError(
'Memory (RAM) required ~{} exceeds {} detected!\n'.format(
human_size(self.memoryusage),
human_size(self.hostinfo['ram'],
a_kilobyte_is_1024_bytes=True)))
# Check if model can be run on specified GPU if required
if self.gpu is not None:
if self.memoryusage - snapsmemsize > self.gpu.totalmem:
raise GeneralError(
'Memory (RAM) required ~{} exceeds {} detected on specified {} - {} GPU!\n'
.format(
human_size(self.memoryusage),
human_size(self.gpu.totalmem,
a_kilobyte_is_1024_bytes=True),
self.gpu.deviceID, self.gpu.name))
# If the required memory without the snapshots will fit on the GPU then transfer and store snaphots on host
if snapsmemsize != 0 and self.memoryusage - snapsmemsize < self.gpu.totalmem:
self.snapsgpu2cpu = True
def min_max_value(filename, args):
"""Minimum value from a response.
Args:
filename (str): Name of output file
args (dict): 'type' key with string 'min', 'max' or 'absmax'; 'outputs' key with a list of names (IDs) of outputs (rxs) from input file
Returns:
value (float): Minimum, maximum, or absolute maximum value from specific outputs
"""
f = h5py.File(filename, 'r')
nrx = f.attrs['nrx']
value = 0
outputsused = False
for rx in range(1, nrx + 1):
output = f['/rxs/rx' + str(rx) + '/']
if output.attrs['Name'] in args['outputs']:
outputname = list(output.keys())[0]
if args['type'] == 'min':
value += np.abs(np.amin(output[outputname]))
elif args['type'] == 'max':
value += np.amax(output[outputname])
elif args['type'] == 'absmax':
value += np.amax(np.abs(output[outputname]))
else:
raise GeneralError('type must be either min, max, or absmax')
outputsused = True
# Check in case no outputs where found
if not outputsused:
raise GeneralError('No outputs matching {} were found'.format(args['outputs']))
return value
def run_main(args):
"""Top-level function that controls what mode of simulation (standard/optimsation/benchmark etc...) is run.
Args:
args (dict): Namespace with input arguments from command line or api.
"""
with open_path_file(args.inputfile) as inputfile:
# Get information about host machine
hostinfo = get_host_info()
hyperthreading = ', {} cores with Hyper-Threading'.format(hostinfo['logicalcores']) if hostinfo['hyperthreading'] else ''
print('\nHost: {}; {} x {} ({} cores{}); {} RAM; {}'.format(hostinfo['machineID'], hostinfo['sockets'], hostinfo['cpuID'], hostinfo['physicalcores'], hyperthreading, human_size(hostinfo['ram'], a_kilobyte_is_1024_bytes=True), hostinfo['osversion']))
# Create a separate namespace that users can access in any Python code blocks in the input file
usernamespace = {'c': c, 'e0': e0, 'm0': m0, 'z0': z0, 'number_model_runs': args.n, 'inputfile': os.path.abspath(inputfile.name)}
#######################################
# Process for benchmarking simulation #
#######################################
if args.benchmark:
if args.mpi or args.opt_taguchi or args.task or args.n > 1:
raise GeneralError('Benchmarking mode cannot be combined with MPI, job array, or Taguchi optimisation modes, or multiple model runs.')
run_benchmark_sim(args, inputfile, usernamespace)
####################################################
# Process for simulation with Taguchi optimisation #
####################################################
elif args.opt_taguchi:
if args.mpi_worker: # Special case for MPI spawned workers - they do not need to enter the Taguchi optimisation mode
run_mpi_sim(args, inputfile, usernamespace)
else:
from gprMax.optimisation_taguchi import run_opt_sim
run_opt_sim(args, inputfile, usernamespace)
################################################
# Process for standard simulation (CPU or GPU) #
################################################
else:
# Mixed mode MPI with OpenMP or CUDA - MPI task farm for models with each model parallelised with OpenMP (CPU) or CUDA (GPU)
if args.mpi:
if args.n == 1:
raise GeneralError('MPI is not beneficial when there is only one model to run')
if args.task:
raise GeneralError('MPI cannot be combined with job array mode')
run_mpi_sim(args, inputfile, usernamespace)
# Standard behaviour - models run serially with each model parallelised with OpenMP (CPU) or CUDA (GPU)
else:
if args.task and args.restart:
raise GeneralError('Job array and restart modes cannot be used together')
run_std_sim(args, inputfile, usernamespace)
def min_sum_diffs(filename, args):
"""Sum of the differences (in dB) between responses and a reference response.
Args:
filename (str): Name of output file
args (dict): 'refresp' key with path & filename of reference response; 'outputs' key with a list of names (IDs) of outputs (rxs) from input file
Returns:
diffdB (float): Sum of the differences (in dB) between responses and a reference response
"""
# Load (from gprMax output file) the reference response. See if file exists at specified path and if not try input file directory
refrespfile = os.path.abspath(args['refresp'])
if not os.path.isfile(refrespfile):
raise GeneralError(
'Cannot load reference response at {}'.format(refrespfile))
f = h5py.File(refrespfile, 'r')
tmp = f['/rxs/rx1/']
fieldname = list(tmp.keys())[0]
refresp = np.array(tmp[fieldname])
# Load (from gprMax output file) the response
f = h5py.File(filename, 'r')
nrx = f.attrs['nrx']
diffdB = 0
outputs = 0
for rx in range(1, nrx + 1):
output = f['/rxs/rx' + str(rx) + '/']
if output.attrs['Name'] in args['outputs']:
outputname = list(output.keys())[0]
modelresp = np.array(output[outputname])
# Calculate sum of differences
with np.errstate(
divide='ignore'
): # Ignore warning from taking a log of any zero values
tmp = 20 * np.log10(
np.abs(modelresp - refresp) / np.amax(np.abs(refresp)))
# Replace any NaNs or Infs from zero division
tmp[np.invert(np.isfinite(tmp))] = 0
tmp = np.abs(np.sum(tmp)) / len(tmp)
diffdB += tmp
outputs += 1
# Check in case no outputs where found
if outputs == 0:
raise GeneralError('No outputs matching {} were found'.format(
args['outputs']))
return diffdB / outputs
def detect_check_gpus(deviceIDs):
"""Get information about Nvidia GPU(s).
Args:
deviceIDs (list): List of integers of device IDs.
Returns:
gpus (list): Detected GPU(s) object(s).
"""
try:
import pycuda.driver as drv
except ImportError:
raise ImportError(
'To use gprMax in GPU mode the pycuda package must be installed, and you must have a NVIDIA CUDA-Enabled GPU (https://developer.nvidia.com/cuda-gpus).'
)
drv.init()
# Check and list any CUDA-Enabled GPUs
if drv.Device.count() == 0:
raise GeneralError(
'No NVIDIA CUDA-Enabled GPUs detected (https://developer.nvidia.com/cuda-gpus)'
)
elif 'CUDA_VISIBLE_DEVICES' in os.environ:
deviceIDsavail = os.environ.get('CUDA_VISIBLE_DEVICES')
deviceIDsavail = [int(s) for s in deviceIDsavail.split(',')]
else:
deviceIDsavail = range(drv.Device.count())
# If no device ID is given use default of 0
if not deviceIDs:
deviceIDs = [0]
# Check if requested device ID(s) exist
for ID in deviceIDs:
if ID not in deviceIDsavail:
raise GeneralError(
'GPU with device ID {} does not exist'.format(ID))
# Gather information about selected/detected GPUs
gpus = []
allgpustext = []
for ID in deviceIDsavail:
gpu = GPU(deviceID=ID)
gpu.get_gpu_info(drv)
if ID in deviceIDs:
gpus.append(gpu)
allgpustext.append('{} - {}, {}'.format(
gpu.deviceID, gpu.name,
human_size(gpu.totalmem, a_kilobyte_is_1024_bytes=True)))
return gpus, allgpustext
def detect_gpus():
"""Get information about Nvidia GPU(s).
Returns:
gpus (list): Detected GPU(s) object(s).
"""
try:
import pycuda.driver as drv
except ImportError:
raise ImportError(
'To use gprMax in GPU mode the pycuda package must be installed, and you must have a NVIDIA CUDA-Enabled GPU (https://developer.nvidia.com/cuda-gpus).'
)
drv.init()
# Check if there are any CUDA-Enabled GPUs
if drv.Device.count() == 0:
raise GeneralError(
'No NVIDIA CUDA-Enabled GPUs detected (https://developer.nvidia.com/cuda-gpus)'
)
# Print information about all detected GPUs
gpus = []
gputext = []
for i in range(drv.Device.count()):
gpu = GPU(deviceID=i)
gpu.get_gpu_info(drv)
gpus.append(gpu)
gputext.append('{} - {}, {} RAM'.format(
gpu.deviceID, gpu.name,
human_size(gpu.totalmem, a_kilobyte_is_1024_bytes=True)))
print('GPU(s) detected: {}'.format('; '.join(gputext)))
return gpus
def compactness(filename, args):
"""A measure of the compactness of a time domain signal.
Args:
filename (str): Name of output file
args (dict): 'outputs' key with a list of names (IDs) of outputs (rxs) from input file
Returns:
compactness (float): Compactness value of signal
"""
f = h5py.File(filename, 'r')
nrx = f.attrs['nrx']
dt = f.attrs['dt']
iterations = f.attrs['Iterations']
time = np.linspace(0, 1, iterations)
time *= (iterations * dt)
outputsused = False
for rx in range(1, nrx + 1):
output = f['/rxs/rx' + str(rx) + '/']
if output.attrs['Name'] in args['outputs']:
outputname = list(output.keys())[0]
outputdata = output[outputname]
# Get absolute maximum value in signal
peak = np.amax([np.amax(outputdata), np.abs(np.amin(outputdata))])
# Get peaks and troughs in signal
delta = peak / 50 # Considered a peak/trough if it has the max/min value, and was preceded (to the left) by a value lower by delta
maxtab, mintab = peakdet(outputdata, delta)
peaks = maxtab + mintab
peaks.sort()
# Remove any peaks smaller than a threshold
thresholdpeak = 1e-3
peaks = [peak for peak in peaks if np.abs(outputdata[peak]) > thresholdpeak]
# Amplitude ratio of the 1st to 3rd peak - hopefully be a measure of a compact envelope
compactness = np.abs(outputdata[peaks[0]]) / np.abs(outputdata[peaks[2]])
# Percentage of maximum value to measure compactness of signal
# durationthreshold = 2
# Check if there is a peak/trough smaller than threshold
# durationthresholdexist = np.where(np.abs(outputdata[peaks]) < (peak * (durationthreshold / 100)))[0]
# if durationthresholdexist.size == 0:
# compactness = time[peaks[-1]]
# else:
# time2threshold = time[peaks[durationthresholdexist[0]]]
# compactness = time2threshold - time[min(peaks)]
# Check in case no outputs where found
if not outputsused:
raise GeneralError('No outputs matching {} were found'.format(args['outputs']))
return compactness
def xcorr(filename, args):
"""Maximum value of a cross-correlation between a response and a reference response.
Args:
filename (str): Name of output file
args (dict): 'refresp' key with path & filename of reference response (time, amp) stored in a text file; 'outputs' key with a list of names (IDs) of outputs (rxs) from input file
Returns:
xcorrmax (float): Maximum value from specific outputs
"""
# Load (from text file) the reference response. See if file exists at specified path and if not try input file directory
refrespfile = os.path.abspath(args['refresp'])
if not os.path.isfile(refrespfile):
raise GeneralError(
'Cannot load reference response from {}'.format(refrespfile))
with open(refrespfile, 'r') as f:
refdata = np.loadtxt(f)
reftime = refdata[:, 0] * 1e-9
refresp = refdata[:, 1]
# Load response from output file
f = h5py.File(filename, 'r')
nrx = f.attrs['nrx']
modeltime = np.arange(0, f.attrs['dt'] * f.attrs['Iterations'],
f.attrs['dt'])
outputsused = False
for rx in range(1, nrx + 1):
output = f['/rxs/rx' + str(rx) + '/']
if output.attrs['Name'] in args['outputs']:
outputname = list(output.keys())[0]
modelresp = output[outputname]
# Convert electric field value (V/m) to voltage (V)
if outputname == 'Ex':
modelresp *= -f.attrs['dx, dy, dz'][0]
elif outputname == 'Ey':
modelresp *= -f.attrs['dx, dy, dz'][1]
elif outputname == 'Ez':
modelresp *= -f.attrs['dx, dy, dz'][2]
outputsused = True
# Check in case no outputs where found
if not outputsused:
raise GeneralError('No outputs matching {} were found'.format(
args['outputs']))
# Normalise reference respose and response from output file
# refresp /= np.amax(np.abs(refresp))
# modelresp /= np.amax(np.abs(modelresp))
# Make both responses the same length in time
# if reftime[-1] > modeltime[-1]:
# reftime = np.arange(0, f.attrs['dt'] * f.attrs['Iterations'], reftime[-1] / len(reftime))
# refresp = refresp[0:len(reftime)]
# elif modeltime[-1] > reftime[-1]:
# modeltime = np.arange(0, reftime[-1], f.attrs['dt'])
# modelresp = modelresp[0:len(modeltime)]
# Downsample the response with the higher sampling rate
# if len(modeltime) < len(reftime):
# refresp = signal.resample(refresp, len(modelresp))
# elif len(reftime) < len(modeltime):
# modelresp = signal.resample(modelresp, len(refresp))
# Prepare data for normalized cross-correlation
refresp = (refresp - np.mean(refresp)) / (np.std(refresp) * len(refresp))
modelresp = (modelresp - np.mean(modelresp)) / np.std(modelresp)
# Plots responses for checking
# fig, ax = plt.subplots(subplot_kw=dict(xlabel='Iterations', ylabel='Voltage [V]'), figsize=(20, 10), facecolor='w', edgecolor='w')
# ax.plot(refresp,'r', lw=2, label='refresp')
# ax.plot(modelresp,'b', lw=2, label='modelresp')
# ax.grid()
# plt.show()
# Calculate cross-correlation
xcorr = signal.correlate(refresp, modelresp)
# Set any NaNs to zero
xcorr = np.nan_to_num(xcorr)
# Plot cross-correlation for checking
# fig, ax = plt.subplots(subplot_kw=dict(xlabel='Iterations', ylabel='Voltage [V]'), figsize=(20, 10), facecolor='w', edgecolor='w')
# ax.plot(xcorr,'r', lw=2, label='xcorr')
# ax.grid()
# plt.show()
xcorrmax = np.amax(xcorr)
return xcorrmax
def process_singlecmds(singlecmds, G):
"""Checks the validity of command parameters and creates instances of classes of parameters.
Args:
singlecmds (dict): Commands that can only occur once in the model.
G (class): Grid class instance - holds essential parameters describing the model.
"""
# Check validity of command parameters in order needed
# messages
cmd = '#messages'
if singlecmds[cmd] is not None:
tmp = singlecmds[cmd].split()
if len(tmp) != 1:
raise CmdInputError(cmd + ' requires exactly one parameter')
if singlecmds[cmd].lower() == 'y':
G.messages = True
elif singlecmds[cmd].lower() == 'n':
G.messages = False
else:
raise CmdInputError(cmd +
' requires input values of either y or n')
# Title
cmd = '#title'
if singlecmds[cmd] is not None:
G.title = singlecmds[cmd]
if G.messages:
print('Model title: {}'.format(G.title))
# Get information about host machine
hostinfo = get_host_info()
# Number of threads (OpenMP) to use
cmd = '#num_threads'
if sys.platform == 'darwin':
os.environ[
'OMP_WAIT_POLICY'] = 'ACTIVE' # Should waiting threads consume CPU power (can drastically effect performance)
os.environ[
'OMP_DYNAMIC'] = 'FALSE' # Number of threads may be adjusted by the run time environment to best utilize system resources
os.environ[
'OMP_PLACES'] = 'cores' # Each place corresponds to a single core (having one or more hardware threads)
os.environ['OMP_PROC_BIND'] = 'TRUE' # Bind threads to physical cores
# os.environ['OMP_DISPLAY_ENV'] = 'TRUE' # Prints OMP version and environment variables (useful for debug)
# Catch bug with Windows Subsystem for Linux (https://github.com/Microsoft/BashOnWindows/issues/785)
if 'Microsoft' in hostinfo['osversion']:
os.environ['KMP_AFFINITY'] = 'disabled'
del os.environ['OMP_PLACES']
del os.environ['OMP_PROC_BIND']
if singlecmds[cmd] is not None:
tmp = tuple(int(x) for x in singlecmds[cmd].split())
if len(tmp) != 1:
raise CmdInputError(
cmd +
' requires exactly one parameter to specify the number of threads to use'
)
if tmp[0] < 1:
raise CmdInputError(
cmd + ' requires the value to be an integer not less than one')
G.nthreads = tmp[0]
os.environ['OMP_NUM_THREADS'] = str(G.nthreads)
elif os.environ.get('OMP_NUM_THREADS'):
G.nthreads = int(os.environ.get('OMP_NUM_THREADS'))
else:
# Set number of threads to number of physical CPU cores
G.nthreads = hostinfo['physicalcores']
os.environ['OMP_NUM_THREADS'] = str(G.nthreads)
if G.messages:
print('Number of CPU (OpenMP) threads: {}'.format(G.nthreads))
if G.nthreads > hostinfo['physicalcores']:
print(
Fore.RED +
'WARNING: You have specified more threads ({}) than available physical CPU cores ({}). This may lead to degraded performance.'
.format(G.nthreads, hostinfo['physicalcores']) + Style.RESET_ALL)
# Spatial discretisation
cmd = '#dx_dy_dz'
tmp = [float(x) for x in singlecmds[cmd].split()]
if len(tmp) != 3:
raise CmdInputError(cmd + ' requires exactly three parameters')
if tmp[0] <= 0:
raise CmdInputError(
cmd +
' requires the x-direction spatial step to be greater than zero')
if tmp[1] <= 0:
raise CmdInputError(
cmd +
' requires the y-direction spatial step to be greater than zero')
if tmp[2] <= 0:
raise CmdInputError(
cmd +
' requires the z-direction spatial step to be greater than zero')
G.dx = tmp[0]
G.dy = tmp[1]
G.dz = tmp[2]
if G.messages:
print('Spatial discretisation: {:g} x {:g} x {:g}m'.format(
G.dx, G.dy, G.dz))
# Domain
cmd = '#domain'
tmp = [float(x) for x in singlecmds[cmd].split()]
if len(tmp) != 3:
raise CmdInputError(cmd + ' requires exactly three parameters')
G.nx = round_value(tmp[0] / G.dx)
G.ny = round_value(tmp[1] / G.dy)
G.nz = round_value(tmp[2] / G.dz)
if G.nx == 0 or G.ny == 0 or G.nz == 0:
raise CmdInputError(cmd +
' requires at least one cell in every dimension')
if G.messages:
print(
'Domain size: {:g} x {:g} x {:g}m ({:d} x {:d} x {:d} = {:g} cells)'
.format(tmp[0], tmp[1], tmp[2], G.nx, G.ny, G.nz,
(G.nx * G.ny * G.nz)))
# Estimate memory (RAM) usage
memestimate = memory_usage(G)
# Check if model can be built and/or run on host
if memestimate > hostinfo['ram']:
raise GeneralError(
'Estimated memory (RAM) required ~{} exceeds {} detected!\n'.
format(human_size(memestimate),
human_size(hostinfo['ram'], a_kilobyte_is_1024_bytes=True)))
if G.messages:
print('Estimated memory (RAM) required: ~{}'.format(
human_size(memestimate)))
# Time step CFL limit (use either 2D or 3D) and default PML thickness
if G.nx == 1:
G.dt = 1 / (c * np.sqrt((1 / G.dy) * (1 / G.dy) + (1 / G.dz) *
(1 / G.dz)))
G.dimension = '2D'
G.pmlthickness['x0'] = 0
G.pmlthickness['xmax'] = 0
elif G.ny == 1:
G.dt = 1 / (c * np.sqrt((1 / G.dx) * (1 / G.dx) + (1 / G.dz) *
(1 / G.dz)))
G.dimension = '2D'
G.pmlthickness['y0'] = 0
G.pmlthickness['ymax'] = 0
elif G.nz == 1:
G.dt = 1 / (c * np.sqrt((1 / G.dx) * (1 / G.dx) + (1 / G.dy) *
(1 / G.dy)))
G.dimension = '2D'
G.pmlthickness['z0'] = 0
G.pmlthickness['zmax'] = 0
else:
G.dt = 1 / (c * np.sqrt((1 / G.dx) * (1 / G.dx) + (1 / G.dy) *
(1 / G.dy) + (1 / G.dz) * (1 / G.dz)))
G.dimension = '3D'
# Round down time step to nearest float with precision one less than hardware maximum. Avoids inadvertently exceeding the CFL due to binary representation of floating point number.
G.dt = round_value(G.dt, decimalplaces=d.getcontext().prec - 1)
if G.messages:
print('Time step (at {} CFL limit): {:g} secs'.format(
G.dimension, G.dt))
# Time step stability factor
cmd = '#time_step_stability_factor'
if singlecmds[cmd] is not None:
tmp = tuple(float(x) for x in singlecmds[cmd].split())
if len(tmp) != 1:
raise CmdInputError(cmd + ' requires exactly one parameter')
if tmp[0] <= 0 or tmp[0] > 1:
raise CmdInputError(
cmd +
' requires the value of the time step stability factor to be between zero and one'
)
G.dt = G.dt * tmp[0]
if G.messages:
print('Time step (modified): {:g} secs'.format(G.dt))
# Time window
cmd = '#time_window'
tmp = singlecmds[cmd].split()
if len(tmp) != 1:
raise CmdInputError(
cmd +
' requires exactly one parameter to specify the time window. Either in seconds or number of iterations.'
)
tmp = tmp[0].lower()
# If number of iterations given
try:
tmp = int(tmp)
G.timewindow = (tmp - 1) * G.dt
G.iterations = tmp
# If real floating point value given
except:
tmp = float(tmp)
if tmp > 0:
G.timewindow = tmp
G.iterations = round_value((tmp / G.dt)) + 1
else:
raise CmdInputError(cmd + ' must have a value greater than zero')
if G.messages:
print('Time window: {:g} secs ({} iterations)'.format(
G.timewindow, G.iterations))
# PML
cmd = '#pml_cells'
if singlecmds[cmd] is not None:
tmp = singlecmds[cmd].split()
if len(tmp) != 1 and len(tmp) != 6:
raise CmdInputError(cmd + ' requires either one or six parameters')
if len(tmp) == 1:
for key in G.pmlthickness.keys():
G.pmlthickness[key] = int(tmp[0])
else:
G.pmlthickness['x0'] = int(tmp[0])
G.pmlthickness['y0'] = int(tmp[1])
G.pmlthickness['z0'] = int(tmp[2])
G.pmlthickness['xmax'] = int(tmp[3])
G.pmlthickness['ymax'] = int(tmp[4])
G.pmlthickness['zmax'] = int(tmp[5])
if 2 * G.pmlthickness['x0'] >= G.nx or 2 * G.pmlthickness[
'y0'] >= G.ny or 2 * G.pmlthickness[
'z0'] >= G.nz or 2 * G.pmlthickness[
'xmax'] >= G.nx or 2 * G.pmlthickness[
'ymax'] >= G.ny or 2 * G.pmlthickness['zmax'] >= G.nz:
raise CmdInputError(cmd + ' has too many cells for the domain size')
# src_steps
cmd = '#src_steps'
if singlecmds[cmd] is not None:
tmp = singlecmds[cmd].split()
if len(tmp) != 3:
raise CmdInputError(cmd + ' requires exactly three parameters')
G.srcsteps[0] = round_value(float(tmp[0]) / G.dx)
G.srcsteps[1] = round_value(float(tmp[1]) / G.dy)
G.srcsteps[2] = round_value(float(tmp[2]) / G.dz)
if G.messages:
print(
'Simple sources will step {:g}m, {:g}m, {:g}m for each model run.'
.format(G.srcsteps[0] * G.dx, G.srcsteps[1] * G.dy,
G.srcsteps[2] * G.dz))
# rx_steps
cmd = '#rx_steps'
if singlecmds[cmd] is not None:
tmp = singlecmds[cmd].split()
if len(tmp) != 3:
raise CmdInputError(cmd + ' requires exactly three parameters')
G.rxsteps[0] = round_value(float(tmp[0]) / G.dx)
G.rxsteps[1] = round_value(float(tmp[1]) / G.dy)
G.rxsteps[2] = round_value(float(tmp[2]) / G.dz)
if G.messages:
print(
'All receivers will step {:g}m, {:g}m, {:g}m for each model run.'
.format(G.rxsteps[0] * G.dx, G.rxsteps[1] * G.dy,
G.rxsteps[2] * G.dz))
# Excitation file for user-defined source waveforms
cmd = '#excitation_file'
if singlecmds[cmd] is not None:
tmp = singlecmds[cmd].split()
if len(tmp) != 1:
raise CmdInputError(cmd + ' requires exactly one parameter')
excitationfile = tmp[0]
# See if file exists at specified path and if not try input file directory
if not os.path.isfile(excitationfile):
excitationfile = os.path.abspath(
os.path.join(G.inputdirectory, excitationfile))
# Get waveform names
with open(excitationfile, 'r') as f:
waveformIDs = f.readline().split()
# Read all waveform values into an array
waveformvalues = np.loadtxt(excitationfile,
skiprows=1,
dtype=floattype)
for waveform in range(len(waveformIDs)):
if any(x.ID == waveformIDs[waveform] for x in G.waveforms):
raise CmdInputError(
'Waveform with ID {} already exists'.format(
waveformIDs[waveform]))
w = Waveform()
w.ID = waveformIDs[waveform]
w.type = 'user'
if len(waveformvalues.shape) == 1:
w.uservalues = waveformvalues[:]
else:
w.uservalues = waveformvalues[:, waveform]
if G.messages:
print('User waveform {} created.'.format(w.ID))
G.waveforms.append(w)
# Arrays for storing time
floattype = filetest[path + outputstest[0]].dtype
timetest = np.zeros((filetest.attrs['Iterations']), dtype=floattype)
timetest = np.arange(
0, filetest.attrs['dt'] * filetest.attrs['Iterations'],
filetest.attrs['dt']) / 1e-9
timeref = timetest
# Arrays for storing field data
datatest = np.zeros((filetest.attrs['Iterations'], len(outputstest)),
dtype=floattype)
for ID, name in enumerate(outputstest):
datatest[:, ID] = filetest[path + str(name)][:]
if np.any(np.isnan(datatest[:, ID])):
raise GeneralError('Test data contains NaNs')
# Tx/Rx position to feed to analytical solution
rxpos = filetest[path].attrs['Position']
txpos = filetest['/srcs/src1/'].attrs['Position']
rxposrelative = ((rxpos[0] - txpos[0]), (rxpos[1] - txpos[1]),
(rxpos[2] - txpos[2]))
# Analytical solution of a dipole in free space
dataref = hertzian_dipole_fs(filetest.attrs['Iterations'],
filetest.attrs['dt'],
filetest.attrs['dx, dy, dz'],
rxposrelative)
filetest.close()
def peakdet(v, delta, x=None):
"""Detect peaks and troughs in a vector (adapted from MATLAB script at http://billauer.co.il/peakdet.html).
A point is considered a maximum peak if it has the maximal value, and was preceded (to the left) by a value lower by delta.
Eli Billauer, 3.4.05 (Explicitly not copyrighted).
This function is released to the public domain; Any use is allowed.
Args:
v (float): 1D array
delta (float): threshold for determining peaks/troughs
Returns:
maxtab, mintab (list): Indices of peak/trough locations
"""
maxtab = []
mintab = []
if x is None:
x = np.arange(len(v))
v = np.asarray(v)
if len(v) != len(x):
raise GeneralError('Input vectors v and x must have same length')
if not np.isscalar(delta):
raise GeneralError('Input argument delta must be a scalar')
if delta <= 0:
raise GeneralError('Input argument delta must be positive')
mn, mx = np.Inf, -np.Inf
mnpos, mxpos = np.NaN, np.NaN
lookformax = True
for i in np.arange(len(v)):
this = v[i]
if this > mx:
mx = this
mxpos = x[i]
if this < mn:
mn = this
mnpos = x[i]
if lookformax:
if this < mx - delta:
if int(mxpos) != 0:
maxtab.append(int(mxpos))
mn = this
mnpos = x[i]
lookformax = False
else:
if this > mn + delta:
if int(mnpos) != 0:
mintab.append(int(mnpos))
mx = this
mxpos = x[i]
lookformax = True
return maxtab, mintab
def run_model(args, currentmodelrun, modelend, numbermodelruns, inputfile, usernamespace):
"""Runs a model - processes the input file; builds the Yee cells; calculates update coefficients; runs main FDTD loop.
Args:
args (dict): Namespace with command line arguments
currentmodelrun (int): Current model run number.
modelend (int): Number of last model to run.
numbermodelruns (int): Total number of model runs.
inputfile (object): File object for the input file.
usernamespace (dict): Namespace that can be accessed by user
in any Python code blocks in input file.
Returns:
tsolve (int): Length of time (seconds) of main FDTD calculations
"""
# Monitor memory usage
p = psutil.Process()
# Declare variable to hold FDTDGrid class
global G
# Used for naming geometry and output files
appendmodelnumber = '' if numbermodelruns == 1 and not args.task and not args.restart else '_'+str(currentmodelrun)
appendmodelnumberGeometry = '' if numbermodelruns == 1 and not args.task and not args.restart or args.geometry_fixed else '_'+str(currentmodelrun)
# Normal model reading/building process; bypassed if geometry information to be reused
if 'G' not in globals():
# Initialise an instance of the FDTDGrid class
G = FDTDGrid()
# Get information about host machine
# (need to save this info to FDTDGrid instance after it has been created)
G.hostinfo = get_host_info()
# Single GPU object
if args.gpu:
G.gpu = args.gpu
G.inputfilename = os.path.split(inputfile.name)[1]
G.inputdirectory = os.path.dirname(os.path.abspath(inputfile.name))
inputfilestr = '\n--- Model {}/{}, input file: {}'.format(currentmodelrun, modelend, inputfile.name)
if G.messages:
print(Fore.GREEN + '{} {}\n'.format(inputfilestr, '-' * (get_terminal_width() - 1 - len(inputfilestr))) + Style.RESET_ALL)
# Add the current model run to namespace that can be accessed by
# user in any Python code blocks in input file
usernamespace['current_model_run'] = currentmodelrun
# Read input file and process any Python and include file commands
processedlines = process_python_include_code(inputfile, usernamespace)
# Print constants/variables in user-accessable namespace
uservars = ''
for key, value in sorted(usernamespace.items()):
if key != '__builtins__':
uservars += '{}: {}, '.format(key, value)
if G.messages:
print('Constants/variables used/available for Python scripting: {{{}}}\n'.format(uservars[:-2]))
# Write a file containing the input commands after Python or include file commands have been processed
if args.write_processed:
write_processed_file(processedlines, appendmodelnumber, G)
# Check validity of command names and that essential commands are present
singlecmds, multicmds, geometry = check_cmd_names(processedlines)
# Create built-in materials
m = Material(0, 'pec')
m.se = float('inf')
m.type = 'builtin'
m.averagable = False
G.materials.append(m)
m = Material(1, 'free_space')
m.type = 'builtin'
G.materials.append(m)
# Process parameters for commands that can only occur once in the model
process_singlecmds(singlecmds, G)
# Process parameters for commands that can occur multiple times in the model
if G.messages: print()
process_multicmds(multicmds, G)
# Estimate and check memory (RAM) usage
G.memory_estimate_basic()
#G.memory_check()
#if G.messages:
# if G.gpu is None:
# print('\nMemory (RAM) required: ~{}\n'.format(human_size(G.memoryusage)))
# else:
# print('\nMemory (RAM) required: ~{} host + ~{} GPU\n'.format(human_size(G.memoryusage), human_size(G.memoryusage)))
# Initialise an array for volumetric material IDs (solid), boolean
# arrays for specifying materials not to be averaged (rigid),
# an array for cell edge IDs (ID)
G.initialise_geometry_arrays()
# Initialise arrays for the field components
if G.gpu is None:
G.initialise_field_arrays()
# Process geometry commands in the order they were given
process_geometrycmds(geometry, G)
# Build the PMLs and calculate initial coefficients
if G.messages: print()
if all(value == 0 for value in G.pmlthickness.values()):
if G.messages:
print('PML: switched off')
pass # If all the PMLs are switched off don't need to build anything
else:
# Set default CFS parameters for PML if not given
if not G.cfs:
G.cfs = [CFS()]
if G.messages:
if all(value == G.pmlthickness['x0'] for value in G.pmlthickness.values()):
pmlinfo = str(G.pmlthickness['x0'])
else:
pmlinfo = ''
for key, value in G.pmlthickness.items():
pmlinfo += '{}: {}, '.format(key, value)
pmlinfo = pmlinfo[:-2] + ' cells'
print('PML: formulation: {}, order: {}, thickness: {}'.format(G.pmlformulation, len(G.cfs), pmlinfo))
pbar = tqdm(total=sum(1 for value in G.pmlthickness.values() if value > 0), desc='Building PML boundaries', ncols=get_terminal_width() - 1, file=sys.stdout, disable=not G.progressbars)
build_pmls(G, pbar)
pbar.close()
# Build the model, i.e. set the material properties (ID) for every edge
# of every Yee cell
if G.messages: print()
pbar = tqdm(total=2, desc='Building main grid', ncols=get_terminal_width() - 1, file=sys.stdout, disable=not G.progressbars)
build_electric_components(G.solid, G.rigidE, G.ID, G)
pbar.update()
build_magnetic_components(G.solid, G.rigidH, G.ID, G)
pbar.update()
pbar.close()
# Add PEC boundaries to invariant direction in 2D modes
# N.B. 2D modes are a single cell slice of 3D grid
if '2D TMx' in G.mode:
# Ey & Ez components
G.ID[1, 0, :, :] = 0
G.ID[1, 1, :, :] = 0
G.ID[2, 0, :, :] = 0
G.ID[2, 1, :, :] = 0
elif '2D TMy' in G.mode:
# Ex & Ez components
G.ID[0, :, 0, :] = 0
G.ID[0, :, 1, :] = 0
G.ID[2, :, 0, :] = 0
G.ID[2, :, 1, :] = 0
elif '2D TMz' in G.mode:
# Ex & Ey components
G.ID[0, :, :, 0] = 0
G.ID[0, :, :, 1] = 0
G.ID[1, :, :, 0] = 0
G.ID[1, :, :, 1] = 0
# Process any voltage sources (that have resistance) to create a new
# material at the source location
for voltagesource in G.voltagesources:
voltagesource.create_material(G)
# Initialise arrays of update coefficients to pass to update functions
G.initialise_std_update_coeff_arrays()
# Initialise arrays of update coefficients and temporary values if
# there are any dispersive materials
if Material.maxpoles != 0:
# Update estimated memory (RAM) usage
G.memoryusage += int(3 * Material.maxpoles * (G.nx + 1) * (G.ny + 1) * (G.nz + 1) * np.dtype(complextype).itemsize)
G.memory_check()
if G.messages:
print('\nMemory (RAM) required - updated (dispersive): ~{}\n'.format(human_size(G.memoryusage)))
G.initialise_dispersive_arrays()
# Check there is sufficient memory to store any snapshots
if G.snapshots:
snapsmemsize = 0
for snap in G.snapshots:
# 2 x required to account for electric and magnetic fields
snapsmemsize += (2 * snap.datasizefield)
G.memoryusage += int(snapsmemsize)
G.memory_check(snapsmemsize=int(snapsmemsize))
if G.messages:
print('\nMemory (RAM) required - updated (snapshots): ~{}\n'.format(human_size(G.memoryusage)))
# Process complete list of materials - calculate update coefficients,
# store in arrays, and build text list of materials/properties
materialsdata = process_materials(G)
if G.messages:
print('\nMaterials:')
materialstable = AsciiTable(materialsdata)
materialstable.outer_border = False
materialstable.justify_columns[0] = 'right'
print(materialstable.table)
# Check to see if numerical dispersion might be a problem
results = dispersion_analysis(G)
if results['error'] and G.messages:
print(Fore.RED + "\nWARNING: Numerical dispersion analysis not carried out as {}".format(results['error']) + Style.RESET_ALL)
elif results['N'] < G.mingridsampling:
raise GeneralError("Non-physical wave propagation: Material '{}' has wavelength sampled by {} cells, less than required minimum for physical wave propagation. Maximum significant frequency estimated as {:g}Hz".format(results['material'].ID, results['N'], results['maxfreq']))
elif results['deltavp'] and np.abs(results['deltavp']) > G.maxnumericaldisp and G.messages:
print(Fore.RED + "\nWARNING: Potentially significant numerical dispersion. Estimated largest physical phase-velocity error is {:.2f}% in material '{}' whose wavelength sampled by {} cells. Maximum significant frequency estimated as {:g}Hz".format(results['deltavp'], results['material'].ID, results['N'], results['maxfreq']) + Style.RESET_ALL)
elif results['deltavp'] and G.messages:
print("\nNumerical dispersion analysis: estimated largest physical phase-velocity error is {:.2f}% in material '{}' whose wavelength sampled by {} cells. Maximum significant frequency estimated as {:g}Hz".format(results['deltavp'], results['material'].ID, results['N'], results['maxfreq']))
# If geometry information to be reused between model runs
else:
inputfilestr = '\n--- Model {}/{}, input file (not re-processed, i.e. geometry fixed): {}'.format(currentmodelrun, modelend, inputfile.name)
if G.messages:
print(Fore.GREEN + '{} {}\n'.format(inputfilestr, '-' * (get_terminal_width() - 1 - len(inputfilestr))) + Style.RESET_ALL)
if G.gpu is None:
# Clear arrays for field components
G.initialise_field_arrays()
# Clear arrays for fields in PML
for pml in G.pmls:
pml.initialise_field_arrays()
# Adjust position of simple sources and receivers if required
if G.srcsteps[0] != 0 or G.srcsteps[1] != 0 or G.srcsteps[2] != 0:
for source in itertools.chain(G.hertziandipoles, G.magneticdipoles):
if currentmodelrun == 1:
if source.xcoord + G.srcsteps[0] * modelend < 0 or source.xcoord + G.srcsteps[0] * modelend > G.nx or source.ycoord + G.srcsteps[1] * modelend < 0 or source.ycoord + G.srcsteps[1] * modelend > G.ny or source.zcoord + G.srcsteps[2] * modelend < 0 or source.zcoord + G.srcsteps[2] * modelend > G.nz:
raise GeneralError('Source(s) will be stepped to a position outside the domain.')
source.xcoord = source.xcoordorigin + (currentmodelrun - 1) * G.srcsteps[0]
source.ycoord = source.ycoordorigin + (currentmodelrun - 1) * G.srcsteps[1]
source.zcoord = source.zcoordorigin + (currentmodelrun - 1) * G.srcsteps[2]
if G.rxsteps[0] != 0 or G.rxsteps[1] != 0 or G.rxsteps[2] != 0:
for receiver in G.rxs:
if currentmodelrun == 1:
if receiver.xcoord + G.rxsteps[0] * modelend < 0 or receiver.xcoord + G.rxsteps[0] * modelend > G.nx or receiver.ycoord + G.rxsteps[1] * modelend < 0 or receiver.ycoord + G.rxsteps[1] * modelend > G.ny or receiver.zcoord + G.rxsteps[2] * modelend < 0 or receiver.zcoord + G.rxsteps[2] * modelend > G.nz:
raise GeneralError('Receiver(s) will be stepped to a position outside the domain.')
receiver.xcoord = receiver.xcoordorigin + (currentmodelrun - 1) * G.rxsteps[0]
receiver.ycoord = receiver.ycoordorigin + (currentmodelrun - 1) * G.rxsteps[1]
receiver.zcoord = receiver.zcoordorigin + (currentmodelrun - 1) * G.rxsteps[2]
# Write files for any geometry views and geometry object outputs
if not (G.geometryviews or G.geometryobjectswrite) and args.geometry_only and G.messages:
print(Fore.RED + '\nWARNING: No geometry views or geometry objects to output found.' + Style.RESET_ALL)
if G.geometryviews and (not args.geometry_fixed or currentmodelrun == 1):
if G.messages: print()
for i, geometryview in enumerate(G.geometryviews):
geometryview.set_filename(appendmodelnumberGeometry, G)
pbar = tqdm(total=geometryview.datawritesize, unit='byte', unit_scale=True, desc='Writing geometry view file {}/{}, {}'.format(i + 1, len(G.geometryviews), os.path.split(geometryview.filename)[1]), ncols=get_terminal_width() - 1, file=sys.stdout, disable=not G.progressbars)
geometryview.write_vtk(G, pbar)
pbar.close()
if G.geometryobjectswrite:
for i, geometryobject in enumerate(G.geometryobjectswrite):
pbar = tqdm(total=geometryobject.datawritesize, unit='byte', unit_scale=True, desc='Writing geometry object file {}/{}, {}'.format(i + 1, len(G.geometryobjectswrite), os.path.split(geometryobject.filename)[1]), ncols=get_terminal_width() - 1, file=sys.stdout, disable=not G.progressbars)
geometryobject.write_hdf5(G, pbar)
pbar.close()
# If only writing geometry information
if args.geometry_only:
tsolve = 0
# Run simulation
else:
# Output filename
inputdirectory, inputfilename = os.path.split(os.path.join(G.inputdirectory, G.inputfilename))
if G.outputdirectory is None:
outputdir = inputdirectory
else:
outputdir = G.outputdirectory
# Save current directory
curdir = os.getcwd()
os.chdir(inputdirectory)
outputdir = os.path.abspath(outputdir)
if not os.path.isdir(outputdir):
os.mkdir(outputdir)
if G.messages:
print('\nCreated output directory: {}'.format(outputdir))
# Restore current directory
os.chdir(curdir)
basename, ext = os.path.splitext(inputfilename)
outputfile = os.path.join(outputdir, basename + appendmodelnumber + '.out')
if G.messages:
print('\nOutput file: {}\n'.format(outputfile))
# Main FDTD solving functions for either CPU or GPU
if G.gpu is None:
tsolve = solve_cpu(currentmodelrun, modelend, G)
else:
tsolve, memsolve = solve_gpu(currentmodelrun, modelend, G)
# Write an output file in HDF5 format
write_hdf5_outputfile(outputfile, G)
# Write any snapshots to file
if G.snapshots:
# Create directory and construct filename from user-supplied name and model run number
snapshotdir = os.path.join(G.inputdirectory, os.path.splitext(G.inputfilename)[0] + '_snaps' + appendmodelnumber)
if not os.path.exists(snapshotdir):
os.mkdir(snapshotdir)
if G.messages: print()
for i, snap in enumerate(G.snapshots):
snap.filename = os.path.abspath(os.path.join(snapshotdir, snap.basefilename + '.vti'))
pbar = tqdm(total=snap.vtkdatawritesize, leave=True, unit='byte', unit_scale=True, desc='Writing snapshot file {} of {}, {}'.format(i + 1, len(G.snapshots), os.path.split(snap.filename)[1]), ncols=get_terminal_width() - 1, file=sys.stdout, disable=not G.progressbars)
snap.write_vtk_imagedata(pbar, G)
pbar.close()
if G.messages: print()
if G.messages:
if G.gpu is None:
print('Memory (RAM) used: ~{}'.format(human_size(p.memory_info().rss)))
else:
print('Memory (RAM) used: ~{} host + ~{} GPU'.format(human_size(p.memory_info().rss), human_size(memsolve)))
print('Solving time [HH:MM:SS]: {}'.format(datetime.timedelta(seconds=tsolve)))
# If geometry information to be reused between model runs then FDTDGrid
# class instance must be global so that it persists
if not args.geometry_fixed or currentmodelrun is modelend:
del G
return tsolve
def run_main(args):
"""
Top-level function that controls what mode of simulation (standard/optimsation/benchmark etc...) is run.
Args:
args (dict): Namespace with input arguments from command line or api.
"""
# Print gprMax logo, version, and licencing/copyright information
logo(__version__ + ' (' + codename + ')')
with open_path_file(args.inputfile) as inputfile:
# Get information about host machine
hostinfo = get_host_info()
hyperthreading = ', {} cores with Hyper-Threading'.format(
hostinfo['logicalcores']) if hostinfo['hyperthreading'] else ''
print('\nHost: {} | {} | {} x {} ({} cores{}) | {} RAM | {}'.format(
hostinfo['hostname'], hostinfo['machineID'], hostinfo['sockets'],
hostinfo['cpuID'], hostinfo['physicalcores'], hyperthreading,
human_size(hostinfo['ram'],
a_kilobyte_is_1024_bytes=True), hostinfo['osversion']))
# Get information/setup any Nvidia GPU(s)
if args.gpu is not None:
# Flatten a list of lists
if any(isinstance(element, list) for element in args.gpu):
args.gpu = [val for sublist in args.gpu for val in sublist]
gpus, allgpustext = detect_check_gpus(args.gpu)
print('GPU(s) detected: {}'.format(' | '.join(allgpustext)))
# If in MPI mode or benchmarking provide list of GPU objects, otherwise
# provide single GPU object
if args.mpi or args.mpi_no_spawn or args.benchmark:
args.gpu = gpus
else:
args.gpu = gpus[0]
# Create a separate namespace that users can access in any Python code blocks in the input file
usernamespace = {
'c': c,
'e0': e0,
'm0': m0,
'z0': z0,
'number_model_runs': args.n,
'inputfile': os.path.abspath(inputfile.name)
}
#######################################
# Process for benchmarking simulation #
#######################################
if args.benchmark:
if args.mpi or args.opt_taguchi or args.task or args.n > 1:
raise GeneralError(
'Benchmarking mode cannot be combined with MPI, job array, or Taguchi optimisation modes, or multiple model runs.'
)
run_benchmark_sim(args, inputfile, usernamespace)
####################################################
# Process for simulation with Taguchi optimisation #
####################################################
elif args.opt_taguchi:
if args.mpi_worker: # Special case for MPI spawned workers - they do not need to enter the Taguchi optimisation mode
run_mpi_sim(args, inputfile, usernamespace)
else:
from gprMax.optimisation_taguchi import run_opt_sim
run_opt_sim(args, inputfile, usernamespace)
################################################
# Process for standard simulation (CPU or GPU) #
################################################
else:
# Mixed mode MPI with OpenMP or CUDA - MPI task farm for models with each model parallelised with OpenMP (CPU) or CUDA (GPU)
if args.mpi:
if args.n == 1:
raise GeneralError(
'MPI is not beneficial when there is only one model to run'
)
if args.task:
raise GeneralError(
'MPI cannot be combined with job array mode')
run_mpi_sim(args, inputfile, usernamespace)
# Alternate MPI configuration that does not use MPI spawn mechanism
elif args.mpi_no_spawn:
if args.n == 1:
raise GeneralError(
'MPI is not beneficial when there is only one model to run'
)
if args.task:
raise GeneralError(
'MPI cannot be combined with job array mode')
run_mpi_no_spawn_sim(args, inputfile, usernamespace)
# Standard behaviour - models run serially with each model parallelised with OpenMP (CPU) or CUDA (GPU)
else:
if args.task and args.restart:
raise GeneralError(
'Job array and restart modes cannot be used together')
run_std_sim(args, inputfile, usernamespace)
def solve_gpu(currentmodelrun, modelend, G):
"""Solving using FDTD method on GPU. Implemented using Nvidia CUDA.
Args:
currentmodelrun (int): Current model run number.
modelend (int): Number of last model to run.
G (class): Grid class instance - holds essential parameters describing the model.
Returns:
tsolve (float): Time taken to execute solving
memsolve (int): memory usage on final iteration in bytes
"""
import pycuda.driver as drv
from pycuda.compiler import SourceModule
drv.init()
# Suppress nvcc warnings on Windows
if sys.platform == 'win32':
compiler_opts = ['-w']
else:
compiler_opts = None
# Create device handle and context on specifc GPU device (and make it current context)
dev = drv.Device(G.gpu.deviceID)
ctx = dev.make_context()
# Electric and magnetic field updates - prepare kernels, and get kernel functions
if Material.maxpoles > 0:
kernels_fields = SourceModule(kernels_template_fields.substitute(REAL=cudafloattype, COMPLEX=cudacomplextype, N_updatecoeffsE=G.updatecoeffsE.size, N_updatecoeffsH=G.updatecoeffsH.size, NY_MATCOEFFS=G.updatecoeffsE.shape[1], NY_MATDISPCOEFFS=G.updatecoeffsdispersive.shape[1], NX_FIELDS=G.nx + 1, NY_FIELDS=G.ny + 1, NZ_FIELDS=G.nz + 1, NX_ID=G.ID.shape[1], NY_ID=G.ID.shape[2], NZ_ID=G.ID.shape[3], NX_T=G.Tx.shape[1], NY_T=G.Tx.shape[2], NZ_T=G.Tx.shape[3]), options=compiler_opts)
else: # Set to one any substitutions for dispersive materials
kernels_fields = SourceModule(kernels_template_fields.substitute(REAL=cudafloattype, COMPLEX=cudacomplextype, N_updatecoeffsE=G.updatecoeffsE.size, N_updatecoeffsH=G.updatecoeffsH.size, NY_MATCOEFFS=G.updatecoeffsE.shape[1], NY_MATDISPCOEFFS=1, NX_FIELDS=G.nx + 1, NY_FIELDS=G.ny + 1, NZ_FIELDS=G.nz + 1, NX_ID=G.ID.shape[1], NY_ID=G.ID.shape[2], NZ_ID=G.ID.shape[3], NX_T=1, NY_T=1, NZ_T=1), options=compiler_opts)
update_e_gpu = kernels_fields.get_function("update_e")
update_h_gpu = kernels_fields.get_function("update_h")
# Copy material coefficient arrays to constant memory of GPU (must be <64KB) for fields kernels
updatecoeffsE = kernels_fields.get_global('updatecoeffsE')[0]
updatecoeffsH = kernels_fields.get_global('updatecoeffsH')[0]
if G.updatecoeffsE.nbytes + G.updatecoeffsH.nbytes > G.gpu.constmem:
raise GeneralError('Too many materials in the model to fit onto constant memory of size {} on {} - {} GPU'.format(human_size(G.gpu.constmem), G.gpu.deviceID, G.gpu.name))
else:
drv.memcpy_htod(updatecoeffsE, G.updatecoeffsE)
drv.memcpy_htod(updatecoeffsH, G.updatecoeffsH)
# Electric and magnetic field updates - dispersive materials - get kernel functions and initialise array on GPU
if Material.maxpoles > 0: # If there are any dispersive materials (updates are split into two parts as they require present and updated electric field values).
update_e_dispersive_A_gpu = kernels_fields.get_function("update_e_dispersive_A")
update_e_dispersive_B_gpu = kernels_fields.get_function("update_e_dispersive_B")
G.gpu_initialise_dispersive_arrays()
# Electric and magnetic field updates - set blocks per grid and initialise field arrays on GPU
G.gpu_set_blocks_per_grid()
G.gpu_initialise_arrays()
# PML updates
if G.pmls:
# Prepare kernels
pmlmodulelectric = 'gprMax.pml_updates.pml_updates_electric_' + G.pmlformulation + '_gpu'
kernelelectricfunc = getattr(import_module(pmlmodulelectric), 'kernels_template_pml_electric_' + G.pmlformulation)
pmlmodulemagnetic = 'gprMax.pml_updates.pml_updates_magnetic_' + G.pmlformulation + '_gpu'
kernelmagneticfunc = getattr(import_module(pmlmodulemagnetic), 'kernels_template_pml_magnetic_' + G.pmlformulation)
kernels_pml_electric = SourceModule(kernelelectricfunc.substitute(REAL=cudafloattype, N_updatecoeffsE=G.updatecoeffsE.size, NY_MATCOEFFS=G.updatecoeffsE.shape[1], NX_FIELDS=G.nx + 1, NY_FIELDS=G.ny + 1, NZ_FIELDS=G.nz + 1, NX_ID=G.ID.shape[1], NY_ID=G.ID.shape[2], NZ_ID=G.ID.shape[3]), options=compiler_opts)
kernels_pml_magnetic = SourceModule(kernelmagneticfunc.substitute(REAL=cudafloattype, N_updatecoeffsH=G.updatecoeffsH.size, NY_MATCOEFFS=G.updatecoeffsH.shape[1], NX_FIELDS=G.nx + 1, NY_FIELDS=G.ny + 1, NZ_FIELDS=G.nz + 1, NX_ID=G.ID.shape[1], NY_ID=G.ID.shape[2], NZ_ID=G.ID.shape[3]), options=compiler_opts)
# Copy material coefficient arrays to constant memory of GPU (must be <64KB) for PML kernels
updatecoeffsE = kernels_pml_electric.get_global('updatecoeffsE')[0]
updatecoeffsH = kernels_pml_magnetic.get_global('updatecoeffsH')[0]
drv.memcpy_htod(updatecoeffsE, G.updatecoeffsE)
drv.memcpy_htod(updatecoeffsH, G.updatecoeffsH)
# Set block per grid, initialise arrays on GPU, and get kernel functions
for pml in G.pmls:
pml.gpu_initialise_arrays()
pml.gpu_get_update_funcs(kernels_pml_electric, kernels_pml_magnetic)
pml.gpu_set_blocks_per_grid(G)
# Receivers
if G.rxs:
# Initialise arrays on GPU
rxcoords_gpu, rxs_gpu = gpu_initialise_rx_arrays(G)
# Prepare kernel and get kernel function
kernel_store_outputs = SourceModule(kernel_template_store_outputs.substitute(REAL=cudafloattype, NY_RXCOORDS=3, NX_RXS=6, NY_RXS=G.iterations, NZ_RXS=len(G.rxs), NX_FIELDS=G.nx + 1, NY_FIELDS=G.ny + 1, NZ_FIELDS=G.nz + 1), options=compiler_opts)
store_outputs_gpu = kernel_store_outputs.get_function("store_outputs")
# Sources - initialise arrays on GPU, prepare kernel and get kernel functions
if G.voltagesources + G.hertziandipoles + G.magneticdipoles:
kernels_sources = SourceModule(kernels_template_sources.substitute(REAL=cudafloattype, N_updatecoeffsE=G.updatecoeffsE.size, N_updatecoeffsH=G.updatecoeffsH.size, NY_MATCOEFFS=G.updatecoeffsE.shape[1], NY_SRCINFO=4, NY_SRCWAVES=G.iterations, NX_FIELDS=G.nx + 1, NY_FIELDS=G.ny + 1, NZ_FIELDS=G.nz + 1, NX_ID=G.ID.shape[1], NY_ID=G.ID.shape[2], NZ_ID=G.ID.shape[3]), options=compiler_opts)
# Copy material coefficient arrays to constant memory of GPU (must be <64KB) for source kernels
updatecoeffsE = kernels_sources.get_global('updatecoeffsE')[0]
updatecoeffsH = kernels_sources.get_global('updatecoeffsH')[0]
drv.memcpy_htod(updatecoeffsE, G.updatecoeffsE)
drv.memcpy_htod(updatecoeffsH, G.updatecoeffsH)
if G.hertziandipoles:
srcinfo1_hertzian_gpu, srcinfo2_hertzian_gpu, srcwaves_hertzian_gpu = gpu_initialise_src_arrays(G.hertziandipoles, G)
update_hertzian_dipole_gpu = kernels_sources.get_function("update_hertzian_dipole")
if G.magneticdipoles:
srcinfo1_magnetic_gpu, srcinfo2_magnetic_gpu, srcwaves_magnetic_gpu = gpu_initialise_src_arrays(G.magneticdipoles, G)
update_magnetic_dipole_gpu = kernels_sources.get_function("update_magnetic_dipole")
if G.voltagesources:
srcinfo1_voltage_gpu, srcinfo2_voltage_gpu, srcwaves_voltage_gpu = gpu_initialise_src_arrays(G.voltagesources, G)
update_voltage_source_gpu = kernels_sources.get_function("update_voltage_source")
# Snapshots - initialise arrays on GPU, prepare kernel and get kernel functions
if G.snapshots:
# Initialise arrays on GPU
snapEx_gpu, snapEy_gpu, snapEz_gpu, snapHx_gpu, snapHy_gpu, snapHz_gpu = gpu_initialise_snapshot_array(G)
# Prepare kernel and get kernel function
kernel_store_snapshot = SourceModule(kernel_template_store_snapshot.substitute(REAL=cudafloattype, NX_SNAPS=Snapshot.nx_max, NY_SNAPS=Snapshot.ny_max, NZ_SNAPS=Snapshot.nz_max, NX_FIELDS=G.nx + 1, NY_FIELDS=G.ny + 1, NZ_FIELDS=G.nz + 1), options=compiler_opts)
store_snapshot_gpu = kernel_store_snapshot.get_function("store_snapshot")
# Iteration loop timer
iterstart = drv.Event()
iterend = drv.Event()
iterstart.record()
for iteration in tqdm(range(G.iterations), desc='Running simulation, model ' + str(currentmodelrun) + '/' + str(modelend), ncols=get_terminal_width() - 1, file=sys.stdout, disable=not G.progressbars):
# Get GPU memory usage on final iteration
if iteration == G.iterations - 1:
memsolve = drv.mem_get_info()[1] - drv.mem_get_info()[0]
# Store field component values for every receiver
if G.rxs:
store_outputs_gpu(np.int32(len(G.rxs)), np.int32(iteration),
rxcoords_gpu.gpudata, rxs_gpu.gpudata,
G.Ex_gpu.gpudata, G.Ey_gpu.gpudata, G.Ez_gpu.gpudata,
G.Hx_gpu.gpudata, G.Hy_gpu.gpudata, G.Hz_gpu.gpudata,
block=(1, 1, 1), grid=(round32(len(G.rxs)), 1, 1))
# Store any snapshots
for i, snap in enumerate(G.snapshots):
if snap.time == iteration + 1:
if not G.snapsgpu2cpu:
store_snapshot_gpu(np.int32(i), np.int32(snap.xs),
np.int32(snap.xf), np.int32(snap.ys),
np.int32(snap.yf), np.int32(snap.zs),
np.int32(snap.zf), np.int32(snap.dx),
np.int32(snap.dy), np.int32(snap.dz),
G.Ex_gpu.gpudata, G.Ey_gpu.gpudata, G.Ez_gpu.gpudata,
G.Hx_gpu.gpudata, G.Hy_gpu.gpudata, G.Hz_gpu.gpudata,
snapEx_gpu.gpudata, snapEy_gpu.gpudata, snapEz_gpu.gpudata,
snapHx_gpu.gpudata, snapHy_gpu.gpudata, snapHz_gpu.gpudata,
block=Snapshot.tpb, grid=Snapshot.bpg)
else:
store_snapshot_gpu(np.int32(0), np.int32(snap.xs),
np.int32(snap.xf), np.int32(snap.ys),
np.int32(snap.yf), np.int32(snap.zs),
np.int32(snap.zf), np.int32(snap.dx),
np.int32(snap.dy), np.int32(snap.dz),
G.Ex_gpu.gpudata, G.Ey_gpu.gpudata, G.Ez_gpu.gpudata,
G.Hx_gpu.gpudata, G.Hy_gpu.gpudata, G.Hz_gpu.gpudata,
snapEx_gpu.gpudata, snapEy_gpu.gpudata, snapEz_gpu.gpudata,
snapHx_gpu.gpudata, snapHy_gpu.gpudata, snapHz_gpu.gpudata,
block=Snapshot.tpb, grid=Snapshot.bpg)
gpu_get_snapshot_array(snapEx_gpu.get(), snapEy_gpu.get(), snapEz_gpu.get(),
snapHx_gpu.get(), snapHy_gpu.get(), snapHz_gpu.get(), 0, snap)
# Update magnetic field components
update_h_gpu(np.int32(G.nx), np.int32(G.ny), np.int32(G.nz),
G.ID_gpu.gpudata, G.Hx_gpu.gpudata, G.Hy_gpu.gpudata,
G.Hz_gpu.gpudata, G.Ex_gpu.gpudata, G.Ey_gpu.gpudata,
G.Ez_gpu.gpudata, block=G.tpb, grid=G.bpg)
# Update magnetic field components with the PML correction
for pml in G.pmls:
pml.gpu_update_magnetic(G)
# Update magnetic field components for magetic dipole sources
if G.magneticdipoles:
update_magnetic_dipole_gpu(np.int32(len(G.magneticdipoles)), np.int32(iteration),
floattype(G.dx), floattype(G.dy), floattype(G.dz),
srcinfo1_magnetic_gpu.gpudata, srcinfo2_magnetic_gpu.gpudata,
srcwaves_magnetic_gpu.gpudata, G.ID_gpu.gpudata,
G.Hx_gpu.gpudata, G.Hy_gpu.gpudata, G.Hz_gpu.gpudata,
block=(1, 1, 1), grid=(round32(len(G.magneticdipoles)), 1, 1))
# Update electric field components
# If all materials are non-dispersive do standard update
if Material.maxpoles == 0:
update_e_gpu(np.int32(G.nx), np.int32(G.ny), np.int32(G.nz), G.ID_gpu.gpudata,
G.Ex_gpu.gpudata, G.Ey_gpu.gpudata, G.Ez_gpu.gpudata,
G.Hx_gpu.gpudata, G.Hy_gpu.gpudata, G.Hz_gpu.gpudata,
block=G.tpb, grid=G.bpg)
# If there are any dispersive materials do 1st part of dispersive update
# (it is split into two parts as it requires present and updated electric field values).
else:
update_e_dispersive_A_gpu(np.int32(G.nx), np.int32(G.ny), np.int32(G.nz),
np.int32(Material.maxpoles), G.updatecoeffsdispersive_gpu.gpudata,
G.Tx_gpu.gpudata, G.Ty_gpu.gpudata, G.Tz_gpu.gpudata, G.ID_gpu.gpudata,
G.Ex_gpu.gpudata, G.Ey_gpu.gpudata, G.Ez_gpu.gpudata,
G.Hx_gpu.gpudata, G.Hy_gpu.gpudata, G.Hz_gpu.gpudata,
block=G.tpb, grid=G.bpg)
# Update electric field components with the PML correction
for pml in G.pmls:
pml.gpu_update_electric(G)
# Update electric field components for voltage sources
if G.voltagesources:
update_voltage_source_gpu(np.int32(len(G.voltagesources)), np.int32(iteration),
floattype(G.dx), floattype(G.dy), floattype(G.dz),
srcinfo1_voltage_gpu.gpudata, srcinfo2_voltage_gpu.gpudata,
srcwaves_voltage_gpu.gpudata, G.ID_gpu.gpudata,
G.Ex_gpu.gpudata, G.Ey_gpu.gpudata, G.Ez_gpu.gpudata,
block=(1, 1, 1), grid=(round32(len(G.voltagesources)), 1, 1))
# Update electric field components for Hertzian dipole sources (update any Hertzian dipole sources last)
if G.hertziandipoles:
update_hertzian_dipole_gpu(np.int32(len(G.hertziandipoles)), np.int32(iteration),
floattype(G.dx), floattype(G.dy), floattype(G.dz),
srcinfo1_hertzian_gpu.gpudata, srcinfo2_hertzian_gpu.gpudata,
srcwaves_hertzian_gpu.gpudata, G.ID_gpu.gpudata,
G.Ex_gpu.gpudata, G.Ey_gpu.gpudata, G.Ez_gpu.gpudata,
block=(1, 1, 1), grid=(round32(len(G.hertziandipoles)), 1, 1))
# If there are any dispersive materials do 2nd part of dispersive update (it is split into two parts as it requires present and updated electric field values). Therefore it can only be completely updated after the electric field has been updated by the PML and source updates.
if Material.maxpoles > 0:
update_e_dispersive_B_gpu(np.int32(G.nx), np.int32(G.ny), np.int32(G.nz),
np.int32(Material.maxpoles), G.updatecoeffsdispersive_gpu.gpudata,
G.Tx_gpu.gpudata, G.Ty_gpu.gpudata, G.Tz_gpu.gpudata, G.ID_gpu.gpudata,
G.Ex_gpu.gpudata, G.Ey_gpu.gpudata, G.Ez_gpu.gpudata,
block=G.tpb, grid=G.bpg)
# Copy output from receivers array back to correct receiver objects
if G.rxs:
gpu_get_rx_array(rxs_gpu.get(), rxcoords_gpu.get(), G)
# Copy data from any snapshots back to correct snapshot objects
if G.snapshots and not G.snapsgpu2cpu:
for i, snap in enumerate(G.snapshots):
gpu_get_snapshot_array(snapEx_gpu.get(), snapEy_gpu.get(), snapEz_gpu.get(),
snapHx_gpu.get(), snapHy_gpu.get(), snapHz_gpu.get(), i, snap)
iterend.record()
iterend.synchronize()
tsolve = iterstart.time_till(iterend) * 1e-3
# Remove context from top of stack and delete
ctx.pop()
del ctx
return tsolve, memsolve
def run_model(args, modelrun, numbermodelruns, inputfile, usernamespace):
"""Runs a model - processes the input file; builds the Yee cells; calculates update coefficients; runs main FDTD loop.
Args:
args (dict): Namespace with command line arguments
modelrun (int): Current model run number.
numbermodelruns (int): Total number of model runs.
inputfile (str): Name of the input file to open.
usernamespace (dict): Namespace that can be accessed by user in any Python code blocks in input file.
"""
# Monitor memory usage
p = psutil.Process()
print('\n{}\n\nModel input file: {}\n'.format(68 * '*', inputfile))
# Add the current model run to namespace that can be accessed by user in any Python code blocks in input file
usernamespace['current_model_run'] = modelrun
print('Constants/variables available for Python scripting: {}\n'.format(
usernamespace))
# Process any user input Python commands
processedlines = python_code_blocks(inputfile, usernamespace)
# Write a file containing the input commands after Python blocks have been processed
if args.write_python:
write_python_processed(inputfile, modelrun, numbermodelruns,
processedlines)
# Check validity of command names & that essential commands are present
singlecmds, multicmds, geometry = check_cmd_names(processedlines)
# Initialise an instance of the FDTDGrid class
G = FDTDGrid()
G.inputdirectory = usernamespace['inputdirectory']
# Process parameters for commands that can only occur once in the model
process_singlecmds(singlecmds, multicmds, G)
# Process parameters for commands that can occur multiple times in the model
process_multicmds(multicmds, G)
# Initialise an array for volumetric material IDs (solid), boolean arrays for specifying materials not to be averaged (rigid),
# an array for cell edge IDs (ID), and arrays for the field components.
G.initialise_std_arrays()
# Process the geometry commands in the order they were given
tinputprocstart = perf_counter()
process_geometrycmds(geometry, G)
tinputprocend = perf_counter()
print('\nInput file processed in [HH:MM:SS]: {}'.format(
datetime.timedelta(seconds=int(tinputprocend - tinputprocstart))))
# Build the PML and calculate initial coefficients
build_pml(G)
# Build the model, i.e. set the material properties (ID) for every edge of every Yee cell
tbuildstart = perf_counter()
build_electric_components(G.solid, G.rigidE, G.ID, G)
build_magnetic_components(G.solid, G.rigidH, G.ID, G)
tbuildend = perf_counter()
print('\nModel built in [HH:MM:SS]: {}'.format(
datetime.timedelta(seconds=int(tbuildend - tbuildstart))))
# Process any voltage sources (that have resistance) to create a new material at the source location
for voltagesource in G.voltagesources:
voltagesource.create_material(G)
# Initialise arrays of update coefficients to pass to update functions
G.initialise_std_updatecoeff_arrays(len(G.materials))
# Initialise arrays of update coefficients and temporary values if there are any dispersive materials
if Material.maxpoles != 0:
G.initialise_dispersive_arrays(len(G.materials))
# Calculate update coefficients, store in arrays, and list materials in model
if G.messages:
print('\nMaterials:\n')
print('ID\tName\t\tProperties')
print('{}'.format('-' * 50))
for material in G.materials:
# Calculate update coefficients for material
material.calculate_update_coeffsE(G)
material.calculate_update_coeffsH(G)
# Store all update coefficients together
G.updatecoeffsE[
material.
numID, :] = material.CA, material.CBx, material.CBy, material.CBz, material.srce
G.updatecoeffsH[
material.
numID, :] = material.DA, material.DBx, material.DBy, material.DBz, material.srcm
# Store coefficients for any dispersive materials
if Material.maxpoles != 0:
z = 0
for pole in range(Material.maxpoles):
G.updatecoeffsdispersive[
material.numID,
z:z + 3] = e0 * material.eqt2[pole], material.eqt[
pole], material.zt[pole]
z += 3
if G.messages:
if material.deltaer and material.tau:
tmp = 'delta_epsr={}, tau={} secs; '.format(
', '.join('{:g}'.format(deltaer)
for deltaer in material.deltaer),
', '.join('{:g}'.format(tau) for tau in material.tau))
else:
tmp = ''
if material.average:
dielectricsmoothing = 'dielectric smoothing permitted.'
else:
dielectricsmoothing = 'dielectric smoothing not permitted.'
print(
'{:3}\t{:12}\tepsr={:g}, sig={:g} S/m; mur={:g}, sig*={:g} S/m; '
.format(material.numID, material.ID, material.er, material.se,
material.mr, material.sm) + tmp + dielectricsmoothing)
# Check to see if numerical dispersion might be a problem
if dispersion_check(G.waveforms, G.materials, G.dx, G.dy, G.dz):
print(
'\nWARNING: Potential numerical dispersion in the simulation. Check the spatial discretisation against the smallest wavelength present.'
)
# Write files for any geometry views
if not G.geometryviews and args.geometry_only:
raise GeneralError('No geometry views found.')
elif G.geometryviews:
tgeostart = perf_counter()
for geometryview in G.geometryviews:
geometryview.write_file(modelrun, numbermodelruns, G)
tgeoend = perf_counter()
print('\nGeometry file(s) written in [HH:MM:SS]: {}'.format(
datetime.timedelta(seconds=int(tgeoend - tgeostart))))
# Run simulation if not doing only geometry
if not args.geometry_only:
# Prepare any snapshot files
for snapshot in G.snapshots:
snapshot.prepare_file(modelrun, numbermodelruns, G)
# Prepare output file
inputfileparts = os.path.splitext(inputfile)
if numbermodelruns == 1:
outputfile = inputfileparts[0] + '.out'
else:
outputfile = inputfileparts[0] + str(modelrun) + '.out'
sys.stdout.write('\nOutput to file: {}\n'.format(outputfile))
sys.stdout.flush()
f = prepare_output_file(outputfile, G)
# Adjust position of sources and receivers if required
if G.srcstepx > 0 or G.srcstepy > 0 or G.srcstepz > 0:
for source in itertools.chain(G.hertziandipoles, G.magneticdipoles,
G.voltagesources,
G.transmissionlines):
source.xcoord += (modelrun - 1) * G.srcstepx
source.ycoord += (modelrun - 1) * G.srcstepy
source.zcoord += (modelrun - 1) * G.srcstepz
if G.rxstepx > 0 or G.rxstepy > 0 or G.rxstepz > 0:
for receiver in G.rxs:
receiver.xcoord += (modelrun - 1) * G.rxstepx
receiver.ycoord += (modelrun - 1) * G.rxstepy
receiver.zcoord += (modelrun - 1) * G.rxstepz
##################################
# Main FDTD calculation loop #
##################################
tsolvestart = perf_counter()
# Absolute time
abstime = 0
for timestep in range(G.iterations):
if timestep == 0:
tstepstart = perf_counter()
# Write field outputs to file
write_output(f, timestep, G.Ex, G.Ey, G.Ez, G.Hx, G.Hy, G.Hz, G)
# Write any snapshots to file
for snapshot in G.snapshots:
if snapshot.time == timestep + 1:
snapshot.write_snapshot(G.Ex, G.Ey, G.Ez, G.Hx, G.Hy, G.Hz,
G)
# Update electric field components
if Material.maxpoles == 0: # All materials are non-dispersive so do standard update
update_electric(G.nx, G.ny, G.nz, G.nthreads, G.updatecoeffsE,
G.ID, G.Ex, G.Ey, G.Ez, G.Hx, G.Hy, G.Hz)
elif Material.maxpoles == 1: # If there are any dispersive materials do 1st part of dispersive update (it is split into two parts as it requires present and updated electric field values).
update_electric_dispersive_1pole_A(G.nx, G.ny, G.nz,
G.nthreads, G.updatecoeffsE,
G.updatecoeffsdispersive,
G.ID, G.Tx, G.Ty, G.Tz,
G.Ex, G.Ey, G.Ez, G.Hx,
G.Hy, G.Hz)
elif Material.maxpoles > 1:
update_electric_dispersive_multipole_A(
G.nx, G.ny, G.nz, G.nthreads, Material.maxpoles,
G.updatecoeffsE, G.updatecoeffsdispersive, G.ID, G.Tx,
G.Ty, G.Tz, G.Ex, G.Ey, G.Ez, G.Hx, G.Hy, G.Hz)
# Update electric field components with the PML correction
update_electric_pml(G)
# Update electric field components from sources
for voltagesource in G.voltagesources:
voltagesource.update_electric(abstime, G.updatecoeffsE, G.ID,
G.Ex, G.Ey, G.Ez, G)
for transmissionline in G.transmissionlines:
transmissionline.update_electric(abstime, G.Ex, G.Ey, G.Ez, G)
for hertziandipole in G.hertziandipoles: # Update any Hertzian dipole sources last
hertziandipole.update_electric(abstime, G.updatecoeffsE, G.ID,
G.Ex, G.Ey, G.Ez, G)
# If there are any dispersive materials do 2nd part of dispersive update (it is split into two parts as it requires present and updated electric field values). Therefore it can only be completely updated after the electric field has been updated by the PML and source updates.
if Material.maxpoles == 1:
update_electric_dispersive_1pole_B(G.nx, G.ny, G.nz,
G.nthreads,
G.updatecoeffsdispersive,
G.ID, G.Tx, G.Ty, G.Tz,
G.Ex, G.Ey, G.Ez)
elif Material.maxpoles > 1:
update_electric_dispersive_multipole_B(
G.nx, G.ny, G.nz, G.nthreads, Material.maxpoles,
G.updatecoeffsdispersive, G.ID, G.Tx, G.Ty, G.Tz, G.Ex,
G.Ey, G.Ez)
# Increment absolute time value
abstime += 0.5 * G.dt
# Update magnetic field components
update_magnetic(G.nx, G.ny, G.nz, G.nthreads, G.updatecoeffsH,
G.ID, G.Ex, G.Ey, G.Ez, G.Hx, G.Hy, G.Hz)
# Update magnetic field components with the PML correction
update_magnetic_pml(G)
# Update magnetic field components from sources
for transmissionline in G.transmissionlines:
transmissionline.update_magnetic(abstime, G.Hx, G.Hy, G.Hz, G)
for magneticdipole in G.magneticdipoles:
magneticdipole.update_magnetic(abstime, G.updatecoeffsH, G.ID,
G.Hx, G.Hy, G.Hz, G)
# Increment absolute time value
abstime += 0.5 * G.dt
# Calculate time for two iterations, used to estimate overall runtime
if timestep == 1:
tstepend = perf_counter()
runtime = datetime.timedelta(
seconds=int((tstepend - tstepstart) / 2 * G.iterations))
sys.stdout.write(
'Estimated runtime [HH:MM:SS]: {}\n'.format(runtime))
sys.stdout.write('Solving for model run {} of {}...\n'.format(
modelrun, numbermodelruns))
sys.stdout.flush()
elif timestep > 1:
update_progress((timestep + 1) / G.iterations)
# Close output file
f.close()
tsolveend = perf_counter()
print('\n\nSolving took [HH:MM:SS]: {}'.format(
datetime.timedelta(seconds=int(tsolveend - tsolvestart))))
print('Peak memory (approx) used: {}'.format(
human_size(p.memory_info().rss)))
def main():
"""This is the main function for gprMax."""
# Print gprMax logo, version, and licencing/copyright information
logo(gprMax.__version__ + ' (Bowmore)')
# Parse command line arguments
parser = argparse.ArgumentParser(
prog='gprMax',
description=
'Electromagnetic modelling software based on the Finite-Difference Time-Domain (FDTD) method'
)
parser.add_argument('inputfile', help='path to and name of inputfile')
parser.add_argument('-n',
default=1,
type=int,
help='number of times to run the input file')
parser.add_argument('-mpi',
action='store_true',
default=False,
help='switch on MPI')
parser.add_argument('--geometry-only',
action='store_true',
default=False,
help='only build model and produce geometry file(s)')
parser.add_argument(
'--write-python',
action='store_true',
default=False,
help=
'write an input file after any Python code blocks in the original input file have been processed'
)
parser.add_argument(
'--opt-taguchi',
action='store_true',
default=False,
help='optimise parameters using the Taguchi optimisation method')
args = parser.parse_args()
numbermodelruns = args.n
inputdirectory = os.path.dirname(os.path.abspath(args.inputfile)) + os.sep
inputfile = inputdirectory + os.path.basename(args.inputfile)
# Create a separate namespace that users can access in any Python code blocks in the input file
usernamespace = {
'c': c,
'e0': e0,
'm0': m0,
'z0': z0,
'number_model_runs': numbermodelruns,
'inputdirectory': inputdirectory
}
# Process for Taguchi optimisation
if args.opt_taguchi:
from gprMax.optimisation_taguchi import run_opt_sim
run_opt_sim(args, numbermodelruns, inputfile, usernamespace)
# Process for standard simulation
else:
if args.mpi: # Mixed mode MPI/OpenMP - MPI task farm for models with each model parallelised with OpenMP
if numbermodelruns == 1:
raise GeneralError(
'MPI is not beneficial when there is only one model to run'
)
run_mpi_sim(args, numbermodelruns, inputfile, usernamespace)
else: # Standard behaviour - models run serially with each model parallelised with OpenMP
run_std_sim(args, numbermodelruns, inputfile, usernamespace)
print('\nSimulation completed.\n{}\n'.format(68 * '*'))
outputstest = list(filetest[path].keys())
# Arrays for storing time
floattype = filetest[path + outputstest[0]].dtype
timetest = np.linspace(
0, (filetest.attrs['Iterations'] - 1) * filetest.attrs['dt'],
num=filetest.attrs['Iterations']) / 1e-9
timeref = timetest
# Arrays for storing field data
datatest = np.zeros((filetest.attrs['Iterations'], len(outputstest)),
dtype=floattype)
for ID, name in enumerate(outputstest):
datatest[:, ID] = filetest[path + str(name)][:]
if np.any(np.isnan(datatest[:, ID])):
raise GeneralError('Test data contains NaNs')
# Tx/Rx position to feed to analytical solution
rxpos = filetest[path].attrs['Position']
txpos = filetest['/srcs/src1/'].attrs['Position']
rxposrelative = ((rxpos[0] - txpos[0]), (rxpos[1] - txpos[1]),
(rxpos[2] - txpos[2]))
# Analytical solution of a dipole in free space
dataref = hertzian_dipole_fs(filetest.attrs['Iterations'],
filetest.attrs['dt'],
filetest.attrs['dx_dy_dz'], rxposrelative)
filetest.close()
else:
def run_model(args, currentmodelrun, numbermodelruns, inputfile,
usernamespace):
"""Runs a model - processes the input file; builds the Yee cells; calculates update coefficients; runs main FDTD loop.
Args:
args (dict): Namespace with command line arguments
currentmodelrun (int): Current model run number.
numbermodelruns (int): Total number of model runs.
inputfile (object): File object for the input file.
usernamespace (dict): Namespace that can be accessed by user in any Python code blocks in input file.
Returns:
tsolve (int): Length of time (seconds) of main FDTD calculations
"""
# Monitor memory usage
p = psutil.Process()
# Declare variable to hold FDTDGrid class
global G
# Normal model reading/building process; bypassed if geometry information to be reused
if 'G' not in globals():
# Initialise an instance of the FDTDGrid class
G = FDTDGrid()
G.inputfilename = os.path.split(inputfile.name)[1]
G.inputdirectory = os.path.dirname(os.path.abspath(inputfile.name))
inputfilestr = '\n--- Model {}/{}, input file: {}'.format(
currentmodelrun, numbermodelruns, inputfile.name)
print(Fore.GREEN + '{} {}\n'.format(
inputfilestr, '-' *
(get_terminal_width() - 1 - len(inputfilestr))) + Style.RESET_ALL)
# Add the current model run to namespace that can be accessed by user in any Python code blocks in input file
usernamespace['current_model_run'] = currentmodelrun
# Read input file and process any Python or include commands
processedlines = process_python_include_code(inputfile, usernamespace)
# Print constants/variables in user-accessable namespace
uservars = ''
for key, value in sorted(usernamespace.items()):
if key != '__builtins__':
uservars += '{}: {}, '.format(key, value)
print(
'Constants/variables used/available for Python scripting: {{{}}}\n'
.format(uservars[:-2]))
# Write a file containing the input commands after Python or include commands have been processed
if args.write_processed:
write_processed_file(
os.path.join(G.inputdirectory, G.inputfilename),
currentmodelrun, numbermodelruns, processedlines)
# Check validity of command names and that essential commands are present
singlecmds, multicmds, geometry = check_cmd_names(processedlines)
# Create built-in materials
m = Material(0, 'pec')
m.se = float('inf')
m.type = 'builtin'
m.averagable = False
G.materials.append(m)
m = Material(1, 'free_space')
m.type = 'builtin'
G.materials.append(m)
# Process parameters for commands that can only occur once in the model
process_singlecmds(singlecmds, G)
# Process parameters for commands that can occur multiple times in the model
print()
process_multicmds(multicmds, G)
# Initialise an array for volumetric material IDs (solid), boolean arrays for specifying materials not to be averaged (rigid),
# an array for cell edge IDs (ID)
G.initialise_geometry_arrays()
# Initialise arrays for the field components
G.initialise_field_arrays()
# Process geometry commands in the order they were given
process_geometrycmds(geometry, G)
# Build the PMLs and calculate initial coefficients
print()
if all(value == 0 for value in G.pmlthickness.values()):
if G.messages:
print('PML boundaries: switched off')
pass # If all the PMLs are switched off don't need to build anything
else:
if G.messages:
if all(value == G.pmlthickness['x0']
for value in G.pmlthickness.values()):
pmlinfo = str(G.pmlthickness['x0']) + ' cells'
else:
pmlinfo = ''
for key, value in G.pmlthickness.items():
pmlinfo += '{}: {} cells, '.format(key, value)
pmlinfo = pmlinfo[:-2]
print('PML boundaries: {}'.format(pmlinfo))
pbar = tqdm(total=sum(1 for value in G.pmlthickness.values()
if value > 0),
desc='Building PML boundaries',
ncols=get_terminal_width() - 1,
file=sys.stdout,
disable=G.tqdmdisable)
build_pmls(G, pbar)
pbar.close()
# Build the model, i.e. set the material properties (ID) for every edge of every Yee cell
print()
pbar = tqdm(total=2,
desc='Building main grid',
ncols=get_terminal_width() - 1,
file=sys.stdout,
disable=G.tqdmdisable)
build_electric_components(G.solid, G.rigidE, G.ID, G)
pbar.update()
build_magnetic_components(G.solid, G.rigidH, G.ID, G)
pbar.update()
pbar.close()
# Process any voltage sources (that have resistance) to create a new material at the source location
for voltagesource in G.voltagesources:
voltagesource.create_material(G)
# Initialise arrays of update coefficients to pass to update functions
G.initialise_std_update_coeff_arrays()
# Initialise arrays of update coefficients and temporary values if there are any dispersive materials
if Material.maxpoles != 0:
G.initialise_dispersive_arrays()
# Process complete list of materials - calculate update coefficients, store in arrays, and build text list of materials/properties
materialsdata = process_materials(G)
if G.messages:
materialstable = AsciiTable(materialsdata)
materialstable.outer_border = False
materialstable.justify_columns[0] = 'right'
print(materialstable.table)
# Check to see if numerical dispersion might be a problem
results = dispersion_analysis(G)
if not results['waveform']:
print(
Fore.RED +
"\nWARNING: Numerical dispersion analysis not carried out as either no waveform detected or waveform does not fit within specified time window and is therefore being truncated."
+ Style.RESET_ALL)
elif results['N'] < G.mingridsampling:
raise GeneralError(
"Non-physical wave propagation: Material '{}' has wavelength sampled by {} cells, less than required minimum for physical wave propagation. Maximum significant frequency estimated as {:g}Hz"
.format(results['material'].ID, results['N'],
results['maxfreq']))
elif results['deltavp'] and np.abs(
results['deltavp']) > G.maxnumericaldisp:
print(
Fore.RED +
"\nWARNING: Potentially significant numerical dispersion. Estimated largest physical phase-velocity error is {:.2f}% in material '{}' whose wavelength sampled by {} cells. Maximum significant frequency estimated as {:g}Hz"
.format(results['deltavp'], results['material'].ID,
results['N'], results['maxfreq']) + Style.RESET_ALL)
elif results['deltavp'] and G.messages:
print(
"\nNumerical dispersion analysis: estimated largest physical phase-velocity error is {:.2f}% in material '{}' whose wavelength sampled by {} cells. Maximum significant frequency estimated as {:g}Hz"
.format(results['deltavp'], results['material'].ID,
results['N'], results['maxfreq']))
# If geometry information to be reused between model runs
else:
inputfilestr = '\n--- Model {}/{}, input file (not re-processed, i.e. geometry fixed): {}'.format(
currentmodelrun, numbermodelruns, inputfile.name)
print(Fore.GREEN + '{} {}\n'.format(
inputfilestr, '-' *
(get_terminal_width() - 1 - len(inputfilestr))) + Style.RESET_ALL)
# Clear arrays for field components
G.initialise_field_arrays()
# Clear arrays for fields in PML
for pml in G.pmls:
pml.initialise_field_arrays()
# Adjust position of simple sources and receivers if required
if G.srcsteps[0] != 0 or G.srcsteps[1] != 0 or G.srcsteps[2] != 0:
for source in itertools.chain(G.hertziandipoles, G.magneticdipoles):
if currentmodelrun == 1:
if source.xcoord + G.srcsteps[0] * (
numbermodelruns -
1) < 0 or source.xcoord + G.srcsteps[0] * (
numbermodelruns -
1) > G.nx or source.ycoord + G.srcsteps[1] * (
numbermodelruns -
1) < 0 or source.ycoord + G.srcsteps[1] * (
numbermodelruns - 1
) > G.ny or source.zcoord + G.srcsteps[2] * (
numbermodelruns -
1) < 0 or source.zcoord + G.srcsteps[2] * (
numbermodelruns - 1) > G.nz:
raise GeneralError(
'Source(s) will be stepped to a position outside the domain.'
)
source.xcoord = source.xcoordorigin + (currentmodelrun -
1) * G.srcsteps[0]
source.ycoord = source.ycoordorigin + (currentmodelrun -
1) * G.srcsteps[1]
source.zcoord = source.zcoordorigin + (currentmodelrun -
1) * G.srcsteps[2]
if G.rxsteps[0] != 0 or G.rxsteps[1] != 0 or G.rxsteps[2] != 0:
for receiver in G.rxs:
if currentmodelrun == 1:
if receiver.xcoord + G.rxsteps[0] * (
numbermodelruns -
1) < 0 or receiver.xcoord + G.rxsteps[0] * (
numbermodelruns -
1) > G.nx or receiver.ycoord + G.rxsteps[1] * (
numbermodelruns -
1) < 0 or receiver.ycoord + G.rxsteps[1] * (
numbermodelruns - 1
) > G.ny or receiver.zcoord + G.rxsteps[2] * (
numbermodelruns - 1
) < 0 or receiver.zcoord + G.rxsteps[2] * (
numbermodelruns - 1) > G.nz:
raise GeneralError(
'Receiver(s) will be stepped to a position outside the domain.'
)
receiver.xcoord = receiver.xcoordorigin + (currentmodelrun -
1) * G.rxsteps[0]
receiver.ycoord = receiver.ycoordorigin + (currentmodelrun -
1) * G.rxsteps[1]
receiver.zcoord = receiver.zcoordorigin + (currentmodelrun -
1) * G.rxsteps[2]
# Write files for any geometry views and geometry object outputs
if not (G.geometryviews or G.geometryobjectswrite) and args.geometry_only:
print(
Fore.RED +
'\nWARNING: No geometry views or geometry objects to output found.'
+ Style.RESET_ALL)
if G.geometryviews:
print()
for i, geometryview in enumerate(G.geometryviews):
geometryview.set_filename(currentmodelrun, numbermodelruns, G)
pbar = tqdm(total=geometryview.datawritesize,
unit='byte',
unit_scale=True,
desc='Writing geometry view file {}/{}, {}'.format(
i + 1, len(G.geometryviews),
os.path.split(geometryview.filename)[1]),
ncols=get_terminal_width() - 1,
file=sys.stdout,
disable=G.tqdmdisable)
geometryview.write_vtk(currentmodelrun, numbermodelruns, G, pbar)
pbar.close()
if G.geometryobjectswrite:
for i, geometryobject in enumerate(G.geometryobjectswrite):
pbar = tqdm(total=geometryobject.datawritesize,
unit='byte',
unit_scale=True,
desc='Writing geometry object file {}/{}, {}'.format(
i + 1, len(G.geometryobjectswrite),
os.path.split(geometryobject.filename)[1]),
ncols=get_terminal_width() - 1,
file=sys.stdout,
disable=G.tqdmdisable)
geometryobject.write_hdf5(G, pbar)
pbar.close()
# Run simulation (if not only looking ar geometry information)
if not args.geometry_only:
# Prepare any snapshot files
for snapshot in G.snapshots:
snapshot.prepare_vtk_imagedata(currentmodelrun, numbermodelruns, G)
# Output filename
inputfileparts = os.path.splitext(
os.path.join(G.inputdirectory, G.inputfilename))
if numbermodelruns == 1:
outputfile = inputfileparts[0] + '.out'
else:
outputfile = inputfileparts[0] + str(currentmodelrun) + '.out'
print('\nOutput file: {}\n'.format(outputfile))
# Main FDTD solving functions for either CPU or GPU
tsolve = solve_cpu(currentmodelrun, numbermodelruns, G)
# Write an output file in HDF5 format
write_hdf5_outputfile(outputfile, G.Ex, G.Ey, G.Ez, G.Hx, G.Hy, G.Hz,
G)
if G.messages:
print('Memory (RAM) used: ~{}'.format(
human_size(p.memory_info().rss)))
print('Solving time [HH:MM:SS]: {}'.format(
datetime.timedelta(seconds=tsolve)))
return tsolve
# If geometry information to be reused between model runs then FDTDGrid class instance must be global so that it persists
if not args.geometry_fixed:
del G
def run_main(args):
"""
Top-level function that controls what mode of simulation (standard/optimsation/benchmark etc...) is run.
Args:
args (dict): Namespace with input arguments from command line or api.
"""
with open_path_file(args.inputfile) as inputfile:
# Get information about host machine
hostinfo = get_host_info()
hyperthreading = ', {} cores with Hyper-Threading'.format(hostinfo['logicalcores']) if hostinfo['hyperthreading'] else ''
print('\nHost: {}; {} x {} ({} cores{}); {} RAM; {}'.format(hostinfo['machineID'], hostinfo['sockets'], hostinfo['cpuID'], hostinfo['physicalcores'], hyperthreading, human_size(hostinfo['ram'], a_kilobyte_is_1024_bytes=True), hostinfo['osversion']))
# Get information/setup Nvidia GPU(s)
if args.gpu is not None:
# Extract first item of list, either True to automatically determine device ID,
# or an integer to manually specify device ID
args.gpu = args.gpu[0]
gpus = detect_gpus()
# If a device ID is specified check it is valid
if not isinstance(args.gpu, bool):
if args.gpu > len(gpus) - 1:
raise GeneralError('GPU with device ID {} does not exist'.format(args.gpu))
# Set args.gpu to GPU object to access elsewhere
args.gpu = next(gpu for gpu in gpus if gpu.deviceID == args.gpu)
# If no device ID is specified
else:
# If in MPI mode then set args.gpu to list of available GPUs
if args.mpi:
if args.mpi - 1 > len(gpus):
raise GeneralError('Too many MPI tasks requested ({}). The number of MPI tasks requested can only be a maximum of the number of GPU(s) detected plus one, i.e. {} GPU worker tasks + 1 CPU master task'.format(args.mpi, len(gpus)))
args.gpu = gpus
# If benchmarking mode then set args.gpu to list of available GPUs
elif args.benchmark:
args.gpu = gpus
# Otherwise set args.gpu to GPU object with default device ID (0) to access elsewhere
else:
args.gpu = next(gpu for gpu in gpus if gpu.deviceID == 0)
# Create a separate namespace that users can access in any Python code blocks in the input file
usernamespace = {'c': c, 'e0': e0, 'm0': m0, 'z0': z0, 'number_model_runs': args.n, 'inputfile': os.path.abspath(inputfile.name)}
#######################################
# Process for benchmarking simulation #
#######################################
if args.benchmark:
if args.mpi or args.opt_taguchi or args.task or args.n > 1:
raise GeneralError('Benchmarking mode cannot be combined with MPI, job array, or Taguchi optimisation modes, or multiple model runs.')
run_benchmark_sim(args, inputfile, usernamespace)
####################################################
# Process for simulation with Taguchi optimisation #
####################################################
elif args.opt_taguchi:
if args.mpi_worker: # Special case for MPI spawned workers - they do not need to enter the Taguchi optimisation mode
run_mpi_sim(args, inputfile, usernamespace)
else:
from gprMax.optimisation_taguchi import run_opt_sim
run_opt_sim(args, inputfile, usernamespace)
################################################
# Process for standard simulation (CPU or GPU) #
################################################
else:
# Mixed mode MPI with OpenMP or CUDA - MPI task farm for models with each model parallelised with OpenMP (CPU) or CUDA (GPU)
if args.mpi:
if args.n == 1:
raise GeneralError('MPI is not beneficial when there is only one model to run')
if args.task:
raise GeneralError('MPI cannot be combined with job array mode')
run_mpi_sim(args, inputfile, usernamespace)
# Standard behaviour - models run serially with each model parallelised with OpenMP (CPU) or CUDA (GPU)
else:
if args.task and args.restart:
raise GeneralError('Job array and restart modes cannot be used together')
run_std_sim(args, inputfile, usernamespace)
def run_main(args):
"""Top-level function that controls what mode of simulation (standard/optimsation/benchmark etc...) is run.
Args:
args (dict): Namespace with input arguments from command line or api.
"""
numbermodelruns = args.n
with open_path_file(args.inputfile) as inputfile:
# Get information about host machine
hostinfo = get_host_info()
print('\nHost: {}; {} ({} cores); {} RAM; {}'.format(
hostinfo['machineID'], hostinfo['cpuID'], hostinfo['cpucores'],
human_size(hostinfo['ram'], a_kilobyte_is_1024_bytes=True),
hostinfo['osversion']))
# Create a separate namespace that users can access in any Python code blocks in the input file
usernamespace = {
'c': c,
'e0': e0,
'm0': m0,
'z0': z0,
'number_model_runs': numbermodelruns,
'input_directory': os.path.dirname(os.path.abspath(inputfile.name))
}
#######################################
# Process for benchmarking simulation #
#######################################
if args.benchmark:
run_benchmark_sim(args, inputfile, usernamespace)
####################################################
# Process for simulation with Taguchi optimisation #
####################################################
elif args.opt_taguchi:
if args.benchmark:
raise GeneralError(
'Taguchi optimisation should not be used with benchmarking mode'
)
from gprMax.optimisation_taguchi import run_opt_sim
run_opt_sim(args, numbermodelruns, inputfile, usernamespace)
################################################
# Process for standard simulation (CPU or GPU) #
################################################
else:
# Mixed mode MPI with OpenMP or CUDA - MPI task farm for models with each model parallelised with OpenMP (CPU) or CUDA (GPU)
if args.mpi:
if args.benchmark:
raise GeneralError(
'MPI should not be used with benchmarking mode')
if numbermodelruns == 1:
raise GeneralError(
'MPI is not beneficial when there is only one model to run'
)
run_mpi_sim(args, numbermodelruns, inputfile, usernamespace)
# Standard behaviour - part of a job array on Open Grid Scheduler/Grid Engine with each model parallelised with OpenMP (CPU) or CUDA (GPU)
elif args.task:
if args.benchmark:
raise GeneralError(
'A job array should not be used with benchmarking mode'
)
run_job_array_sim(args, numbermodelruns, inputfile,
usernamespace)
# Standard behaviour - models run serially with each model parallelised with OpenMP (CPU) or CUDA (GPU)
else:
run_std_sim(args, numbermodelruns, inputfile, usernamespace)