def run_test(numProcs):
# Run the simulation
# Remove existing output files
for filename in glob.glob(runOutput):
os.remove(filename)
print("Running simulation on " + str(numProcs) + " processors")
# Read nxpe (number of processors in x direction) to pass to restart.redistribute, if present in BOUT.inp
options = BoutOptionsFile(os.path.join(runOutputDir, "BOUT.inp"))
try:
nxpe = options["nxpe"]
except KeyError:
nxpe = None
# prepare restart files
restart.redistribute(numProcs,
nxpe=nxpe,
path=os.path.join(runOutputDir, "restart"),
output=runOutputDir)
# run simulation
print("running: " + executable)
s, out = launch(executable, nproc=numProcs, output="test_run.log")
return s
def generate_restarts(numProcs):
# Read nxpe (number of processors in x direction) to pass to restart.redistribute, if present in BOUT.inp
options = BoutOptionsFile(os.path.join(runOutputDir, "BOUT.inp"))
try:
nxpe = options["nxpe"]
except KeyError:
nxpe = None
# prepare restart files
restart.redistribute(numProcs,
nxpe=nxpe,
path=os.path.join(runOutputDir, "restart"),
output=runOutputDir)
#!/usr/bin/env python
from boutdata import restart
from sys import argv, exit
npes = int(argv[1])
try:
restarts_directory = argv[2]
except IndexError:
restarts_directory = "restarts_256x256"
restart.redistribute(npes, path=restarts_directory, output="data", myg=0)
exit(0)
def redistributeProcs(self, oldDir, addType, npes):
self.copyInpFiles(oldDir, addType)
for i in self.scanIDs:
os.chdir('{}/{}'.format(self.runDir, i))
redistribute(npes=npes, path=oldDir, output=addType)
self.nProcs = npes
def redistribute(self,
npes,
nxpe=None,
mxg=2,
myg=2,
include_restarts=True):
"""Create a new set of dump files for npes processors.
Useful for restarting simulations using more or fewer processors.
Existing data and restart files are kept in the directory
"redistribution_backups". redistribute() will fail if this
directory already exists, to avoid overwriting anything
Parameters
----------
npes : int
Number of new files to create
nxpe : int, optional
If nxpe is None (the default), then an 'optimal' number will be
selected automatically
mxg, myg : int, optional
Number of guard cells in x, y (default: 2)
include_restarts : bool, optional
If True, then restart.redistribute will be used to
redistribute the restart files also (default: True)
"""
from boutdata.processor_rearrange import get_processor_layout, create_processor_layout
from os import rename, path, mkdir
# use get_processor_layout to get nx, ny
old_processor_layout = get_processor_layout(DataFile(
self._file_list[0]),
has_t_dimension=True,
mxg=mxg,
myg=myg)
old_nxpe = old_processor_layout.nxpe
old_nype = old_processor_layout.nype
old_npes = old_processor_layout.npes
old_mxsub = old_processor_layout.mxsub
old_mysub = old_processor_layout.mysub
nx = old_processor_layout.nx
ny = old_processor_layout.ny
mz = old_processor_layout.mz
mxg = old_processor_layout.mxg
myg = old_processor_layout.myg
# calculate new processor layout
new_processor_layout = create_processor_layout(old_processor_layout,
npes,
nxpe=nxpe)
nxpe = new_processor_layout.nxpe
nype = new_processor_layout.nype
mxsub = new_processor_layout.mxsub
mysub = new_processor_layout.mysub
# move existing files to backup directory
# don't overwrite backup: os.mkdir will raise exception if directory already exists
backupdir = path.join(self._path, "redistribution_backups")
mkdir(backupdir)
for f in self._file_list:
rename(f, path.join(backupdir, path.basename(f)))
# create new output files
outfile_list = []
this_prefix = self._prefix
if not this_prefix[-1] == '.':
# ensure prefix ends with a '.'
this_prefix = this_prefix + "."
for i in range(npes):
outpath = os.path.join(self._path,
this_prefix + str(i) + "." + self._suffix)
if self._suffix.split(".")[-1] in ["nc", "ncdf", "cdl"]:
# set format option to DataFile explicitly to avoid creating netCDF3 files, which can only contain up to 2GB of data
outfile_list.append(
DataFile(outpath,
write=True,
create=True,
format='NETCDF4'))
else:
outfile_list.append(DataFile(outpath, write=True, create=True))
# Create a DataFileCache, if needed
if self._DataFileCaching:
DataFileCache = create_cache(backupdir, self._prefix)
else:
DataFileCache = None
# read and write the data
for v in self.varNames:
print("processing " + v)
data = collect(v,
path=backupdir,
prefix=self._prefix,
xguards=True,
yguards=True,
info=False,
datafile_cache=DataFileCache)
ndims = len(data.shape)
# write data
for i in range(npes):
ix = i % nxpe
iy = int(i / nxpe)
outfile = outfile_list[i]
if v == "NPES":
outfile.write(v, npes)
elif v == "NXPE":
outfile.write(v, nxpe)
elif v == "NYPE":
outfile.write(v, nype)
elif v == "MXSUB":
outfile.write(v, mxsub)
elif v == "MYSUB":
outfile.write(v, mysub)
elif ndims == 0:
# scalar
outfile.write(v, data)
elif ndims == 1:
# time evolving scalar
outfile.write(v, data)
elif ndims == 2:
# Field2D
if data.shape != (nx + 2 * mxg, ny + 2 * myg):
# FieldPerp?
# check is not perfect, fails if ny=nz
raise ValueError(
"Error: Found FieldPerp '" + v +
"'. This case is not currently handled by BoutOutputs.redistribute()."
)
outfile.write(
v, data[ix * mxsub:(ix + 1) * mxsub + 2 * mxg,
iy * mysub:(iy + 1) * mysub + 2 * myg])
elif ndims == 3:
# Field3D
if data.shape[:2] != (nx + 2 * mxg, ny + 2 * myg):
# evolving Field2D, but this case is not handled
# check is not perfect, fails if ny=nx and nx=nt
raise ValueError(
"Error: Found evolving Field2D '" + v +
"'. This case is not currently handled by BoutOutputs.redistribute()."
)
outfile.write(
v, data[ix * mxsub:(ix + 1) * mxsub + 2 * mxg,
iy * mysub:(iy + 1) * mysub + 2 * myg, :])
elif ndims == 4:
outfile.write(
v, data[:, ix * mxsub:(ix + 1) * mxsub + 2 * mxg,
iy * mysub:(iy + 1) * mysub + 2 * myg, :])
else:
print(
"ERROR: variable found with unexpected number of dimensions,",
ndims)
for outfile in outfile_list:
outfile.close()
if include_restarts:
print("processing restarts")
from boutdata import restart
from glob import glob
restart_prefix = "BOUT.restart"
restarts_list = glob(path.join(self._path, restart_prefix + "*"))
# Move existing restart files to backup directory
for f in restarts_list:
rename(f, path.join(backupdir, path.basename(f)))
# Redistribute restarts
restart.redistribute(npes,
path=backupdir,
nxpe=nxpe,
output=self._path,
mxg=mxg,
myg=myg)
def test_redistribute_connected_doublenull(self, tmp_path):
"""
Check for a connected double-null case using a large number of processes.
'Large' means there is at least one process in each region with no edges
touching another region.
"""
npes_redistributed = 6
grid_info = make_grid_info(nxpe=3,
nype=18,
ixseps1=7,
ixseps2=7,
xpoints=2)
fieldperp_global_yind = 19
fieldperp_yproc_ind = 4
rng = np.random.default_rng(108)
restart_params = [
# inner, lower divertor leg
(0, -1),
(1, -1),
(2, -1),
(3, -1),
(4, -1),
(5, -1),
(6, -1),
(7, -1),
(8, -1),
# inner core
(9, -1),
(10, -1),
(11, -1),
(12, fieldperp_global_yind),
(13, fieldperp_global_yind),
(14, fieldperp_global_yind),
(15, -1),
(16, -1),
(17, -1),
# inner, upper divertor leg
(18, -1),
(19, -1),
(20, -1),
(21, -1),
(22, -1),
(23, -1),
(24, -1),
(25, -1),
(26, -1),
# outer, upper divertor leg
(27, -1),
(28, -1),
(29, -1),
(30, -1),
(31, -1),
(32, -1),
(33, -1),
(34, -1),
(35, -1),
# outer core
(36, -1),
(37, -1),
(38, -1),
(39, -1),
(40, -1),
(41, -1),
(42, -1),
(43, -1),
(44, -1),
# outer, lower divertor leg
(45, -1),
(46, -1),
(47, -1),
(48, -1),
(49, -1),
(50, -1),
(51, -1),
(52, -1),
(53, -1),
]
restarts = []
for i, fieldperp_yind in restart_params:
restarts.append(
create_restart_file(
tmpdir=tmp_path,
rng=rng,
grid_info=grid_info,
i=i,
fieldperp_global_yind=fieldperp_yind,
))
expected = concatenate_data(
restarts,
nxpe=grid_info["NXPE"],
fieldperp_yproc_ind=fieldperp_yproc_ind,
has_t_dim=False,
)
new_path = tmp_path.joinpath("new_restarts")
new_path.mkdir()
restart.redistribute(npes=npes_redistributed,
path=tmp_path,
output=new_path)
# Check the right number of files have been created
new_files = glob(str(new_path.joinpath("*")))
assert len(new_files) == npes_redistributed
# Check data in redistributed restart files
check_redistributed_data(
expected,
fieldperp_global_yind=fieldperp_global_yind,
path=new_path,
)
def redistribute(self, npes, nxpe=None, mxg=2, myg=2, include_restarts=True):
"""Create a new set of dump files for npes processors.
Useful for restarting simulations using more or fewer processors.
Existing data and restart files are kept in the directory
"redistribution_backups". redistribute() will fail if this
directory already exists, to avoid overwriting anything
Parameters
----------
npes : int
Number of new files to create
nxpe : int, optional
If nxpe is None (the default), then an 'optimal' number will be
selected automatically
mxg, myg : int, optional
Number of guard cells in x, y (default: 2)
include_restarts : bool, optional
If True, then restart.redistribute will be used to
redistribute the restart files also (default: True)
"""
from boutdata.processor_rearrange import get_processor_layout, create_processor_layout
from os import rename, path, mkdir
# use get_processor_layout to get nx, ny
old_processor_layout = get_processor_layout(
DataFile(self._file_list[0]), has_t_dimension=True, mxg=mxg, myg=myg)
old_nxpe = old_processor_layout.nxpe
old_nype = old_processor_layout.nype
old_npes = old_processor_layout.npes
old_mxsub = old_processor_layout.mxsub
old_mysub = old_processor_layout.mysub
nx = old_processor_layout.nx
ny = old_processor_layout.ny
mz = old_processor_layout.mz
mxg = old_processor_layout.mxg
myg = old_processor_layout.myg
# calculate new processor layout
new_processor_layout = create_processor_layout(
old_processor_layout, npes, nxpe=nxpe)
nxpe = new_processor_layout.nxpe
nype = new_processor_layout.nype
mxsub = new_processor_layout.mxsub
mysub = new_processor_layout.mysub
# move existing files to backup directory
# don't overwrite backup: os.mkdir will raise exception if directory already exists
backupdir = path.join(self._path, "redistribution_backups")
mkdir(backupdir)
for f in self._file_list:
rename(f, path.join(backupdir, path.basename(f)))
# create new output files
outfile_list = []
this_prefix = self._prefix
if not this_prefix[-1] == '.':
# ensure prefix ends with a '.'
this_prefix = this_prefix + "."
for i in range(npes):
outpath = os.path.join(
self._path, this_prefix+str(i)+"."+self._suffix)
if self._suffix.split(".")[-1] in ["nc", "ncdf", "cdl"]:
# set format option to DataFile explicitly to avoid creating netCDF3 files, which can only contain up to 2GB of data
outfile_list.append(
DataFile(outpath, write=True, create=True, format='NETCDF4'))
else:
outfile_list.append(DataFile(outpath, write=True, create=True))
# Create a DataFileCache, if needed
if self._DataFileCaching:
DataFileCache = create_cache(backupdir, self._prefix)
else:
DataFileCache = None
# read and write the data
for v in self.varNames:
print("processing "+v)
data = collect(v, path=backupdir, prefix=self._prefix, xguards=True,
yguards=True, info=False, datafile_cache=DataFileCache)
ndims = len(data.shape)
# write data
for i in range(npes):
ix = i % nxpe
iy = int(i/nxpe)
outfile = outfile_list[i]
if v == "NPES":
outfile.write(v, npes)
elif v == "NXPE":
outfile.write(v, nxpe)
elif v == "NYPE":
outfile.write(v, nype)
elif v == "MXSUB":
outfile.write(v, mxsub)
elif v == "MYSUB":
outfile.write(v, mysub)
elif ndims == 0:
# scalar
outfile.write(v, data)
elif ndims == 1:
# time evolving scalar
outfile.write(v, data)
elif ndims == 2:
# Field2D
if data.shape != (nx + 2*mxg, ny + 2*myg):
# FieldPerp?
# check is not perfect, fails if ny=nz
raise ValueError(
"Error: Found FieldPerp '"+v+"'. This case is not currently handled by BoutOutputs.redistribute().")
outfile.write(
v, data[ix*mxsub:(ix+1)*mxsub+2*mxg, iy*mysub:(iy+1)*mysub+2*myg])
elif ndims == 3:
# Field3D
if data.shape[:2] != (nx + 2*mxg, ny + 2*myg):
# evolving Field2D, but this case is not handled
# check is not perfect, fails if ny=nx and nx=nt
raise ValueError("Error: Found evolving Field2D '"+v +
"'. This case is not currently handled by BoutOutputs.redistribute().")
outfile.write(
v, data[ix*mxsub:(ix+1)*mxsub+2*mxg, iy*mysub:(iy+1)*mysub+2*myg, :])
elif ndims == 4:
outfile.write(
v, data[:, ix*mxsub:(ix+1)*mxsub+2*mxg, iy*mysub:(iy+1)*mysub+2*myg, :])
else:
print(
"ERROR: variable found with unexpected number of dimensions,", ndims)
for outfile in outfile_list:
outfile.close()
if include_restarts:
print("processing restarts")
from boutdata import restart
from glob import glob
restart_prefix = "BOUT.restart"
restarts_list = glob(path.join(self._path, restart_prefix+"*"))
# Move existing restart files to backup directory
for f in restarts_list:
rename(f, path.join(backupdir, path.basename(f)))
# Redistribute restarts
restart.redistribute(npes, path=backupdir,
nxpe=nxpe, output=self._path, mxg=mxg, myg=myg)