def load(self,
filename,
collective=False,
try_optimise=True,
verbose=False):
"""
Load a swarm from disk. Note that this must be called before any SwarmVariable
members are loaded.
Parameters
----------
filename : str
The filename for the saved file. Relative or absolute paths may be
used.
try_optimise : bool, Default=True
Will speed up the swarm load time but warning - this algorithm assumes the
previously saved swarm data was made on an identical mesh and mesh partitioning
(number of processors) with respect to the current mesh. If this isn't the case then
the reloaded particle ordering will be broken, leading to an invalid swarms.
One can disable this optimisation and revert to a brute force algorithm, much slower,
by setting this option to False.
verbose : bool
Prints a swarm load progress bar.
Notes
-----
This method must be called collectively by all processes.
Example
-------
Refer to example provided for 'save' method.
"""
if not isinstance(filename, str):
raise TypeError("Expected 'filename' to be provided as a string")
# open hdf5 file
h5f = h5py.File(name=filename,
mode="r",
driver='mpio',
comm=MPI.COMM_WORLD)
# get units
try:
units = h5f.attrs["units"]
except KeyError:
units = None
if units and units != "None":
units = u.parse_expression(units)
else:
units = None
dset = h5f.get('data')
if dset == None:
raise RuntimeError(
"Can't find 'data' in file '{0}'.\n".format(filename))
if dset.shape[1] != self.particleCoordinates.data.shape[1]:
raise RuntimeError("Cannot load file data on current swarm. Data in file '{0}', " \
"has {1} components -the particlesCoords has {2} components".format(filename, dset.shape[1], self.particleCoordinates.data.shape[1]))
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
nProcs = comm.Get_size()
if rank == 0 and verbose:
bar = uw.utils._ProgressBar(start=0,
end=dset.shape[0] - 1,
title="loading " + filename)
# try and read the procCount attribute & assume that if nProcs in .h5 file
# is equal to the current no. procs then the particles will be distributed the
# same across the processors. (Danger if different discretisations are used... i think)
# else try and load the whole .h5 file.
# we set the 'offset' & 'size' variables to achieve the above
offset = 0
size = dset.shape[0] # number of particles in h5 file
if try_optimise:
procCount = h5f.attrs.get('proc_offset')
if procCount is not None and nProcs == len(procCount):
for p_i in range(rank):
offset += procCount[p_i]
size = procCount[rank]
valid = np.zeros(0, dtype='i') # array for read in
chunk = int(2e7) # read in this many points at a time
firstChunk = True
(multiples, remainder) = divmod(size, chunk)
for ii in range(multiples + 1):
# setup the points to begin and end reading in
chunkStart = offset + ii * chunk
if ii == multiples:
chunkEnd = chunkStart + remainder
if remainder == 0: # in the case remainder is 0
break
else:
chunkEnd = chunkStart + chunk
# Add particles to swarm, ztmp is the corresponding local array
# non-local particles are not added and their ztmp index is -1.
# Note that for the first chunk, we do collective read, as this
# is the only time that we can guaranteed that all procs will
# take part, and usually most (if not all) particles are loaded
# in this step.
if firstChunk and collective:
with dset.collective:
if units:
vals = non_dimensionalise(dset[chunkStart:chunkEnd] *
units)
ztmp = self.add_particles_with_coordinates(vals)
else:
ztmp = self.add_particles_with_coordinates(
dset[chunkStart:chunkEnd])
firstChunk = False
else:
if units:
vals = non_dimensionalise(dset[chunkStart:chunkEnd] *
units)
ztmp = self.add_particles_with_coordinates(vals)
else:
ztmp = self.add_particles_with_coordinates(
dset[chunkStart:chunkEnd])
tmp = np.copy(ztmp) # copy because ztmp is 'readonly'
# slice out -neg bits and make the local indices global
it = np.nditer(tmp, op_flags=['readwrite'], flags=['f_index'])
while not it.finished:
if it[0] >= 0:
it[0] = chunkStart + it.index # local to global
it.iternext()
# slice out -neg bits
tmp = tmp[tmp[:] >= 0]
# append to valid
valid = np.append(valid, tmp)
if rank == 0 and verbose:
bar.update(chunkEnd)
h5f.close()
self._local2globalMap = valid
# record which swarm state this corresponds to
self._checkpointMapsToState = self.stateId
def load(self, filename):
"""
Load the mesh from disk.
Parameters
----------
filename: str
The filename for the saved file. Relative or absolute paths may be
used, but all directories must exist.
Notes
-----
This method must be called collectively by all processes.
If the file data array is the same length as the current mesh
global size, it is assumed the file contains compatible data. Note that
this may not be the case where for example where you have saved using a
2*8 resolution mesh, then loaded using an 8*2 resolution mesh.
Provided files must be in hdf5 format, and use the correct schema.
Example
-------
Refer to example provided for 'save' method.
"""
self.reset()
if not isinstance(filename, str):
raise TypeError("Expected filename to be provided as a string")
# get field and mesh information
h5f = h5py.File( filename, "r", driver='mpio', comm=MPI.COMM_WORLD );
# get resolution of old mesh
res = h5f.attrs['mesh resolution']
if res is None:
raise RuntimeError("Can't read the 'mesh resolution' for the field hdf5 file,"+
" was it created correctly?")
if (res == self.elementRes).all() == False:
raise RuntimeError("Provided file mesh resolution does not appear to correspond to\n"\
"resolution of mesh object.")
# get units
try:
units = h5f.attrs["units"]
except KeyError:
units = None
if units and units != "None":
units = u.parse_expression(units)
else:
units = None
dset = h5f.get('vertices')
if dset == None:
raise RuntimeError("Can't find the 'vertices' dataset in hdf5 file '{0}'".format(filename) )
dof = dset.shape[1]
if dof != self.data.shape[1]:
raise RuntimeError("Can't load hdf5 '{0}', incompatible data shape".format(filename))
if len(dset) != self.nodesGlobal:
raise RuntimeError("Provided data file appears to be for a different resolution mesh.")
with self.deform_mesh(isRegular=h5f.attrs['regular']):
with dset.collective:
if units:
self.data[0:self.nodesLocal] = non_dimensionalise(
dset[self.data_nodegId[0:self.nodesLocal], :] * units)
else:
self.data[0:self.nodesLocal] = dset[self.data_nodegId[0:self.nodesLocal], :]
h5f.close()
def load(self, filename, collective=False):
"""
Load the swarm variable from disk. This must be called *after* the swarm.load().
Parameters
----------
filename : str
The filename for the saved file. Relative or absolute paths may be
used, but all directories must exist.
Notes
-----
This method must be called collectively by all processes.
Example
-------
Refer to example provided for 'save' method.
"""
if not isinstance(filename, str):
raise TypeError("'filename' parameter must be of type 'str'")
if self.swarm._checkpointMapsToState != self.swarm.stateId:
raise RuntimeError("'Swarm' associate with this 'SwarmVariable' does not appear to be in the correct state.\n" \
"Please ensure that you have loaded the swarm prior to loading any swarm variables.")
gIds = self.swarm._local2globalMap
comm = MPI.COMM_WORLD
rank = comm.rank
# open hdf5 file
h5f = h5py.File(name=filename,
mode="r",
driver='mpio',
comm=MPI.COMM_WORLD)
dset = h5f.get('data')
if dset == None:
raise RuntimeError(
"Can't find 'data' in file '{}'.\n".format(filename))
if dset.shape[1] != self.data.shape[1]:
raise RuntimeError("Cannot load file data on current swarm. Data in file '{0}', " \
"has {1} components -the particlesCoords has {2} components".format(filename, dset.shape[1], self.particleCoordinates.data.shape[1]))
if dset.shape[0] != self.swarm.particleGlobalCount:
raise RuntimeError("It appears that the swarm has {} particles, but provided h5 file has {} data points. Please check that " \
"both the Swarm and the SwarmVariable were saved at the same time, and that you have reloaded using " \
"the correct files.".format(particleGobalCount, dset.shape[0]))
# for efficiency, we want to load swarmvariable data in the largest stride chunks possible.
# we need to determine where required data is contiguous.
# first construct an array of gradients. the required data is contiguous
# where the indices into the array are increasing by 1, ie have a gradient of 1.
gradIds = np.zeros_like(
gIds) # creates array of zeros of same size & type
if len(gIds) > 1:
gradIds[:-1] = gIds[
1:] - gIds[:-1] # forward difference type gradient
# note that we do only the first read into dset collective. this call usually
# does the entire read, but if it doesn't we won't know how many calls will
# be necessary, hence only collective calling the first.
done_collective = False
guy = 0
while guy < len(gIds):
# do contiguous
start_guy = guy
while gradIds[
guy] == 1: # count run of contiguous. note bounds check not required as last element of gradIds is always zero.
guy += 1
# copy contiguous chunk if found.. note that we are copying 'plus 1' items
if guy > start_guy:
if collective and not done_collective:
with dset.collective:
self.data[start_guy:guy +
1] = dset[gIds[start_guy]:gIds[guy] + 1]
done_collective = True
else:
self.data[start_guy:guy +
1] = dset[gIds[start_guy]:gIds[guy] + 1]
guy += 1
# do non-contiguous
start_guy = guy
while guy < len(
gIds) and gradIds[guy] != 1: # count run of non-contiguous
guy += 1
# copy non-contiguous items (if found) using index array slice
if guy > start_guy:
if collective and not done_collective:
with dset.collective:
self.data[start_guy:guy, :] = dset[
gIds[start_guy:guy], :]
done_collective = True
else:
self.data[start_guy:guy, :] = dset[gIds[start_guy:guy], :]
# if we haven't entered a collective call, do so now to
# avoid deadlock. we just do an empty read/write.
if collective and not done_collective:
with dset.collective:
self.data[0:0, :] = dset[0:0, :]
# get units
try:
units = h5f.attrs["units"]
except KeyError:
units = None
if units and units != "None":
units = u.parse_expression(units)
else:
units = None
h5f.close()
if units:
self.data[:] = non_dimensionalise(self.data * units)
def load(self, filename, interpolate=False):
"""
Load the MeshVariable from disk.
Parameters
----------
filename: str
The filename for the saved file. Relative or absolute paths may be
used, but all directories must exist.
interpolate: bool
Set to True to interpolate a file containing different resolution data.
Note that a temporary MeshVariable with the file data will be build
on **each** processor. Also note that the temporary MeshVariable
can only be built if its corresponding mesh file is available.
Also note that the supporting mesh mush be regular.
Notes
-----
This method must be called collectively by all processes.
If the file data array is the same length as the current variable
global size, it is assumed the file contains compatible data. Note that
this may not be the case where for example where you have saved using a
2*8 resolution mesh, then loaded using an 8*2 resolution mesh.
Provided files must be in hdf5 format, and use the correct schema.
Example
-------
Refer to example provided for 'save' method.
"""
if not isinstance(filename, str):
raise TypeError("Expected filename to be provided as a string")
# get field and mesh information
h5f = h5py.File(filename, "r", driver='mpio', comm=MPI.COMM_WORLD)
dset = h5f.get('data')
# get units
try:
units = h5f.attrs["units"]
except KeyError:
units = None
if units and units != "None":
units = u.parse_expression(units)
else:
units = None
if dset == None:
raise RuntimeError(
"Can't find the 'data' in hdf5 file '{0}'".format(filename))
dof = dset.shape[1]
if dof != self.data.shape[1]:
raise RuntimeError(
"Can't load hdf5 '{0}', incompatible data shape".format(
filename))
if len(dset) == self.mesh.nodesGlobal:
# assume dset matches field exactly
mesh = self.mesh
local = mesh.nodesLocal
with dset.collective:
self.data[0:local] = dset[mesh.data_nodegId[0:local], :]
else:
if not interpolate:
raise RuntimeError("Provided data file appears to be for a different resolution MeshVariable.\n"\
"If you would like to interpolate the data to the current variable, please set\n" \
"the 'interpolate' parameter. Check docstring for important caveats of interpolation method.")
# if here then we build a local version of the entire file field and interpolate it's values
# first get file field's mesh
if h5f.get('mesh') == None:
raise RuntimeError(
"The hdf5 field to be loaded with interpolation must have an associated "
+
"'mesh' hdf5 file. Resave the field with its associated mesh."
+ "i.e. myField.save(\"filename.h5\", meshFilename)")
# get resolution of old mesh
res = h5f['mesh'].attrs.get('mesh resolution')
if res is None:
raise RuntimeError(
"Can't read the 'mesh resolution' for the field hdf5 file,"
+ " was it created correctly?")
# get max of old mesh
inputMax = h5f['mesh'].attrs.get('max')
if inputMax is None:
raise RuntimeError(
"Can't read the 'max' for the field hdf5 file," +
" was it created correctly?")
inputMin = h5f['mesh'].attrs.get('min')
if inputMin is None:
raise RuntimeError(
"Can't read the 'min' for the field hdf5 file," +
" was it created correctly?")
regular = h5f['mesh'].attrs.get('regular')
if regular and regular != True:
raise RuntimeError("Saved mesh file appears to correspond to a irregular mesh.\n"\
"Interpolating from irregular mesh not currently supported." )
elType = h5f['mesh'].attrs.get('elementType')
# for backwards compatiblity, the 'elementType' attribute was added Feb2017
if elType == None:
elType = 'Q1'
# build the NON-PARALLEL field and mesh
inputMesh = uw.mesh.FeMesh_Cartesian(
elementType=(elType +
"/DQ0"), # only geometryMesh can be saved
elementRes=tuple(res),
minCoord=tuple(inputMin),
maxCoord=tuple(inputMax),
partitioned=False)
# load data onto MeshVariable
if len(dset) == inputMesh.nodesGlobal:
inputField = uw.mesh.MeshVariable(mesh=inputMesh,
nodeDofCount=dof)
elif dset.shape[0] == inputMesh.subMesh.nodesGlobal:
# load as a subMesh
# assume the dset field belongs to the subMesh
inputField = uw.mesh.MeshVariable(mesh=inputMesh.subMesh,
nodeDofCount=dof)
else:
# raise error
raise RuntimeError("The saved mesh file can't be read onto the interpolation grid.\n" \
"Note: only subMesh variable with elementType 'DQ0' can be used presently used")
# copy hdf5 numpy array onto serial inputField
inputField.data[:] = dset[:]
# interpolate 'inputField' onto the self nodes
self.data[:] = inputField.evaluate(self.mesh.data)
if units:
if units.units == "degC":
units = u.degK
self.data[:] = non_dimensionalise(
(self.data[:] + 273.15) * units)
else:
self.data[:] = non_dimensionalise(self.data[:] * units)
uw.libUnderworld.StgFEM._FeVariable_SyncShadowValues(self._cself)
h5f.close()