def smooth(self, input_path, **kwargs):
"""
Specfem2D requires additional model parameters in directory to perform
the xsmooth_sem task. This function will copy these files into the
directory before performing the base smooth operations.
Kwargs should match arguments of solver.base.smooth()
.. note::
This operation is usually run with run(single=True) so only one
task will be performing these operations.
:type input_path: str
:param input_path: path to data
"""
# Redundant to 'base' class but necessary
if not exists(input_path):
unix.mkdir(input_path)
unix.cd(self.cwd)
unix.cd("DATA")
# Copy over only the files that are required. Won't execute if no match
files = []
for tag in ["jacobian", "NSPEC_ibool", "x", "y", "z"]:
files += glob(f"*_{tag}.bin")
for src in files:
unix.cp(src=src, dst=input_path)
super().smooth(input_path=input_path, **kwargs)
def distribute_databases(self):
"""
A serial task to distrubute the database files outputted by
xgenerate_databases from main solver to all other solver directories
"""
# Copy the database files but ignore any vt? files
src_db = glob(
os.path.join(PATH.SOLVER, self.mainsolver, "OUTPUT_FILES",
"DATABASES_MPI", "*"))
for extension in [".vtu", ".vtk"]:
src_db = [_ for _ in src_db if extension not in _]
# Copy the .h files from the mesher, Specfem needs these as well
src_h = glob(
os.path.join(PATH.SOLVER, self.mainsolver, "OUTPUT_FILES", "*.h"))
for source_name in self.source_names:
# Ensure main solver is skipped
if source_name == self.mainsolver:
continue
# Copy database files to each of the other source directories
dst_db = os.path.join(PATH.SOLVER, source_name, "OUTPUT_FILES",
"DATABASES_MPI", "")
unix.cp(src_db, dst_db)
# Copy mesher h files into the overlying directory
dst_h = os.path.join(PATH.SOLVER, source_name, "OUTPUT_FILES", "")
unix.cp(src_h, dst_h)
def initialize_adjoint_traces(self):
"""
Setup utility: Creates the "adjoint traces" expected by SPECFEM
Note:
Adjoint traces are initialized by writing zeros for all channels.
Channels actually in use during an inversion or migration will be
overwritten with nonzero values later on.
"""
# Initialize adjoint traces as zeroes for all data_filenames
# write to `traces/adj`
super().initialize_adjoint_traces()
# Rename data to work around Specfem naming convetions
self.rename_data()
# Workaround for Specfem3D's requirement that all components exist,
# even ones not in use as adjoint traces
if PAR.FORMAT.upper() == "SU":
unix.cd(os.path.join(self.cwd, "traces", "adj"))
for iproc in range(PAR.NPROC):
for channel in ["x", "y", "z"]:
dst = f"{iproc:d}_d{channel}_SU.adj"
if not exists(dst):
src = f"{iproc:d}_d{PAR.COMPONENTS[0]}_SU.adj"
unix.cp(src, dst)
def save_traces(self):
"""
Save waveform traces into the output directory
"""
src = os.path.join(PATH.SCRATCH, "traces")
dst = PATH.OUTPUT
unix.cp(src, dst)
def generate_mesh(self, model_path, model_name, model_type='gll'):
"""
Performs meshing with internal mesher Meshfem2D and database generation
:type model_path: str
:param model_path: path to the model to be used for mesh generation
:type model_name: str
:param model_name: name of the model to be used as identification
:type model_type: str
:param model_type: available model types to be passed to the Specfem3D
Par_file. See Specfem3D Par_file for available options.
"""
assert (exists(model_path)), f"model {model_path} does not exist"
available_model_types = ["gll"]
assert(model_type in available_model_types), \
f"{model_type} not in available types {available_model_types}"
unix.cd(self.cwd)
# Run mesh generation
if model_type == "gll":
self.check_mesh_properties(model_path)
# Copy the model files (ex: proc000023_vp.bin ...) into DATA
src = glob(os.path.join(model_path, "*"))
dst = self.model_databases
unix.cp(src, dst)
# Export the model into output folder
if self.taskid == 0:
self.export_model(os.path.join(PATH.OUTPUT, model_name))
def save_kernels(self):
"""
Save individual kernels into the output directory
"""
src = os.path.join(PATH.SCRATCH, "kernels")
dst = PATH.OUTPUT
unix.mkdir(dst)
unix.cp(src, dst)
def save_kernels_sum(self):
"""
Same summed kernels into the output directory
"""
src = os.path.join(PATH.SCRATCH, "kernels", "sum")
dst = os.path.join(PATH.OUTPUT, "kernels")
unix.mkdir(dst)
unix.cp(src, dst)
def import_traces(self, path):
"""
File transfer utility. Import traces into the workflow.
:type path: str
:param path: path to traces
"""
src = glob(os.path.join(path, 'traces', self.source_name, '*'))
dst = os.path.join(self.cwd, 'traces', 'obs')
unix.cp(src, dst)
def import_model(self, path):
"""
File transfer utility to move a SPEFEM2D model into the correct location
for a workflow.
:type path: str
:param path: path to the SPECFEM2D model
:return:
"""
unix.cp(src=glob(os.path.join(path, "model", "*")),
dst=os.path.join(self.cwd, "DATA"))
def export_model(self, path):
"""
File transfer utility to move a SPEFEM2D model from the DATA directory
to an external path location
:type path: str
:param path: path to export the SPECFEM2D model
:return:
"""
unix.mkdir(path)
unix.cp(src=glob(os.path.join(self.cwd, "DATA", "*.bin")), dst=path)
def generate_synthetics(self):
"""
Performs forward simulation, and evaluates the objective function
"""
self.logger.info(msg.sub("PREPARING VELOCITY MODEL"))
src = os.path.join(PATH.OUTPUT, "model_init")
dst = os.path.join(PATH.SCRATCH, "model")
assert os.path.exists(src)
unix.cp(src, dst)
self.logger.info(msg.sub("EVALUATE OBJECTIVE FUNCTION"))
system.run("solver",
"eval_func",
path=PATH.SCRATCH,
write_residuals=True)
def initialize_solver_directories(self):
"""
Creates solver directories in serial using a single node.
Should only be run by master job.
Differs from Base initialize_solver_directories() as this serial task
will create directory structures for each source, rather than having
each source create its own. However the internal dir structure is the
same.
"""
for source_name in self.source_names:
cwd = os.path.join(PATH.SOLVER, source_name)
# Remove any existing scratch directory
unix.rm(cwd)
# Create internal directory structure, change into directory to make
# all actions RELATIVE path actions
unix.mkdir(cwd)
unix.cd(cwd)
for cwd_dir in [
"bin", "DATA", "OUTPUT_FILES/DATABASES_MPI", "traces/obs",
"traces/syn", "traces/adj"
]:
unix.mkdir(cwd_dir)
# Copy exectuables
src = glob(os.path.join(PATH.SPECFEM_BIN, "*"))
dst = os.path.join("bin", "")
unix.cp(src, dst)
# Copy all input files except source files
src = glob(os.path.join(PATH.SPECFEM_DATA, "*"))
src = [_ for _ in src if self.source_prefix not in _]
dst = os.path.join("DATA", "")
unix.cp(src, dst)
# symlink event source specifically
src = os.path.join(PATH.SPECFEM_DATA,
f"{self.source_prefix}_{source_name}")
dst = os.path.join("DATA", self.source_prefix)
unix.ln(src, dst)
if source_name == self.mainsolver:
# Symlink taskid_0 as mainsolver in solver directory
unix.ln(source_name, os.path.join(PATH.SOLVER, "mainsolver"))
# Only check the solver parameters once
self.check_solver_parameter_files()
def export_traces(self, path, prefix="traces/obs"):
"""
File transfer utility. Export traces to disk.
:type path: str
:param path: path to save traces
:type prefix: str
:param prefix: location of traces w.r.t self.cwd
"""
if self.taskid == 0:
self.logger.debug("exporting traces to {path} {prefix}")
unix.mkdir(os.path.join(path))
src = os.path.join(self.cwd, prefix)
dst = os.path.join(path, self.source_name)
unix.cp(src, dst)
def initialize_adjoint_traces(self):
"""
Setup utility: Creates the "adjoint traces" expected by SPECFEM.
This is only done for the 'base' the Preprocess class.
Note:
Adjoint traces are initialized by writing zeros for all channels.
Channels actually in use during an inversion or migration will be
overwritten with nonzero values later on.
"""
super().initialize_adjoint_traces()
unix.cd(self.cwd)
unix.cd(os.path.join("traces", "adj"))
# work around SPECFEM2D's use of different name conventions for
# regular traces and 'adjoint' traces
if PAR.FORMAT.upper() == "SU":
files = glob("*SU")
unix.rename(old="_SU", new="_SU.adj", names=files)
elif PAR.FORMAT.upper() == "ASCII":
files = glob("*sem?")
# Get the available extensions, which are named based on unit
extensions = set([os.path.splitext(_)[-1] for _ in files])
for extension in extensions:
unix.rename(old=extension, new=".adj", names=files)
# SPECFEM2D requires that all components exist even if ununsed
components = ["x", "y", "z", "p"]
if PAR.FORMAT.upper() == "SU":
for comp in components:
src = f"U{PAR.COMPONENTS[0]}_file_single.su.adj"
dst = f"U{comp.lower()}s_file_single.su.adj"
if not exists(dst):
unix.cp(src, dst)
elif PAR.FORMAT.upper() == "ASCII":
for fid in glob("*.adj"):
net, sta, cha, ext = fid.split(".")
for comp in components:
# Replace the last value in the channel with new component
cha_check = cha[:-1] + comp.upper()
fid_check = ".".join([net, sta, cha_check, ext])
if not exists(fid_check):
unix.cp(fid, fid_check)
def generate_mesh(self, model_path, model_name, model_type='gll'):
"""
Performs meshing and database generation as a serial task. Differs
slightly from specfem3d class as it only creates database files for
the main solver, which are then copied in serial by the function
distribute_databases()
:type model_path: str
:param model_path: path to the model to be used for mesh generation
:type model_name: str
:param model_name: name of the model to be used as identification
:type model_type: str
:param model_type: available model types to be passed to the Specfem3D
Par_file. See Specfem3D Par_file for available options.
"""
available_model_types = ["gll"]
assert (exists(model_path)), f"model {model_path} does not exist"
model_type = model_type or getpar(key="MODEL", file="DATA/Par_file")
assert(model_type in available_model_types), \
f"{model_type} not in available types {available_model_types}"
# Ensure that we're running on the main solver only
assert (self.taskid == 0)
unix.cd(self.cwd)
# Check that the model parameter falls into the acceptable types
par = getpar("MODEL").strip()
assert(par in available_model_types), \
f"Par_file {par} not in available types {available_model_types}"
if par == "gll":
self.check_mesh_properties(model_path)
# Copy model files and then run xgenerate databases
src = glob(os.path.join(model_path, "*"))
dst = self.model_databases
unix.cp(src, dst)
call_solver(mpiexec=PAR.MPIEXEC,
executable="bin/xgenerate_databases")
self.export_model(os.path.join(PATH.OUTPUT, model_name))
def save_gradient(self):
"""
Save the gradient vector. Allows saving numpy array or standard
Fortran .bin files
Saving as a vector saves on file count, but requires numpy and seisflows
functions to read
"""
dst = os.path.join(PATH.OUTPUT, f"gradient_{optimize.iter:04d}")
if PAR.SAVEAS in ["binary", "both"]:
src = os.path.join(PATH.GRAD, "gradient")
unix.mv(src, dst)
if PAR.SAVEAS in ["vector", "both"]:
src = os.path.join(PATH.OPTIMIZE, optimize.g_old)
unix.cp(src, dst + ".npy")
self.logger.debug(f"saving gradient to path:\n{dst}")
def export_model(self, path, parameters=None):
"""
File transfer utility. Export the model to disk.
Performed by master solver.
:type path: str
:param path: path to save model
:type parameters: list
:param parameters: list of parameters that define the model
"""
if parameters is None:
parameters = self.parameters
if self.taskid == 0:
unix.mkdir(path)
for key in parameters:
files = glob(os.path.join(self.model_databases, f"*{key}.bin"))
unix.cp(files, path)
def setup(self):
"""
Prepares solver for inversion or migration.
Sets up directory structure expected by SPECFEM and copies or generates
seismic data to be inverted or migrated
.. note:;
As input for an inversion or migration, users can choose between
providing data, or providing a target model from which data are
generated on the fly.
In the former case, a value for PATH.DATA must be supplied;
in the latter case, a value for PATH.MODEL_TRUE must be provided.
"""
# Clean up for new inversion
unix.rm(self.cwd)
self.initialize_solver_directories()
# Determine where observation data will come from
if PAR.CASE.upper() == "SYNTHETIC" and PATH.MODEL_TRUE is not None:
if self.taskid == 0:
self.logger.info(
"generating 'data' with MODEL_TRUE synthetics")
# Generate synthetic data on the fly using the true model
self.generate_data(model_path=PATH.MODEL_TRUE,
model_name="model_true",
model_type="gll")
elif PATH.DATA is not None and os.path.exists(PATH.DATA):
# If Data provided by user, copy directly into the solver directory
unix.cp(src=glob(os.path.join(PATH.DATA, self.source_name, "*")),
dst=os.path.join("traces", "obs"))
# Prepare initial model
if self.taskid == 0:
self.logger.info("running mesh generation for MODEL_INIT")
self.generate_mesh(model_path=PATH.MODEL_INIT,
model_name="model_init",
model_type="gll")
# Create blank adjoint traces to be overwritten
self.initialize_adjoint_traces()
def initialize_solver_directories(self):
"""
Creates directory structure expected by SPECFEM3D (bin/, DATA/) copies
executables, and prepares input files. Executables must be supplied
by user as there is no mechanism for automatically compiling from source
Directories will act as completely independent Specfem run directories.
This allows for embarrassing parallelization while avoiding the need
for intra-directory communications, at the cost of redundancy and
extra files.
"""
if self.taskid == 0:
self.logger.info(f"initializing {PAR.NTASK} solver directories")
unix.mkdir(self.cwd)
unix.cd(self.cwd)
# Create directory structure
for cwd_dir in [
"bin", "DATA", "OUTPUT_FILES/DATABASES_MPI", "traces/obs",
"traces/syn", "traces/adj", self.model_databases,
self.kernel_databases
]:
unix.mkdir(cwd_dir)
# Copy exectuables into the bin/ directory
src = glob(os.path.join(PATH.SPECFEM_BIN, "*"))
dst = os.path.join("bin", "")
unix.cp(src, dst)
# Copy all input files except source files
src = glob(os.path.join(PATH.SPECFEM_DATA, "*"))
src = [_ for _ in src if self.source_prefix not in _]
dst = os.path.join("DATA", "")
unix.cp(src, dst)
# Symlink event source specifically, strip the source name as SPECFEM
# just expects `source_name`
src = os.path.join(PATH.SPECFEM_DATA,
f"{self.source_prefix}_{self.source_name}")
dst = os.path.join("DATA", self.source_prefix)
unix.ln(src, dst)
if self.taskid == 0:
mainsolver = os.path.join(PATH.SOLVER, "mainsolver")
# Symlink taskid_0 as mainsolver in solver directory for convenience
if not os.path.exists(mainsolver):
unix.ln(self.cwd, mainsolver)
self.logger.debug(f"source {self.source_name} symlinked as "
f"mainsolver")
else:
# Copy the initial model from mainsolver into current directory
# Avoids the need to run multiple instances of xgenerate_databases
src = os.path.join(PATH.SOLVER, "mainsolver", "OUTPUT_FILES",
"DATABASES_MPI")
dst = os.path.join(self.cwd, "OUTPUT_FILES", "DATABASES_MPI")
unix.cp(src, dst)
self.check_solver_parameter_files()
def _save_quantity(self, filepaths, tag="", path_out=""):
"""
Repeatable convenience function to save quantities from the scratch/
directory to the output/ directory
:type filepaths: list
:param filepaths: full path to files that should be saved to output/
:type tag: str
:param tag: tag for saving the files in PATH.OUTPUT. If not given, will
save directly into the output/ directory
:type path_out: str
:param path_out: overwrite the default output path file naming
"""
if not path_out:
path_out = os.path.join(PATH.OUTPUT, tag)
if not os.path.exists(path_out):
unix.mkdir(path_out)
for src in filepaths:
dst = os.path.join(path_out, os.path.basename(src))
unix.cp(src, dst)
def setup(self):
"""
Create the SeisFlows3 directory structure in preparation for a
SeisFlows3 workflow. Ensure that if any config information is left over
from a previous workflow, that these files are not overwritten by
the new workflow. Should be called by submit()
.. note::
This function is expected to create dirs: SCRATCH, SYSTEM, OUTPUT
and the following log files: output, error
.. note::
Logger is configured here as all workflows, independent of system,
will be calling setup()
:rtype: tuple of str
:return: (path to output log, path to error log)
"""
# Create scratch directories
unix.mkdir(PATH.SCRATCH)
unix.mkdir(PATH.SYSTEM)
# Create output directories
unix.mkdir(PATH.OUTPUT)
log_files = os.path.join(PATH.WORKDIR, CFGPATHS.LOGDIR)
unix.mkdir(log_files)
# If resuming, move old log files to keep them out of the way. Number
# in ascending order, so we don't end up overwriting things
for src in [self.output_log, self.error_log, PATH.PAR_FILE]:
i = 1
if os.path.exists(src):
dst = os.path.join(log_files, number_fid(src, i))
while os.path.exists(dst):
i += 1
dst = os.path.join(log_files, number_fid(src, i))
self.logger.debug(f"copying par/log file to: {dst}")
unix.cp(src=src, dst=dst)
def create_specfem2d_working_directory(self):
"""
Create the working directory where we will generate our initial and
final models using one of the SPECFEM2D examples
"""
assert (os.path.exists(self.sem2d_paths["example"])), (
f"SPECFEM2D/EXAMPLE directory: '{self.sem2d['example']}' "
f"does not exist, please check this path and try again.")
# Incase this example has written before, remove dir. that were created
rm(self.workdir_paths.workdir)
mkdir(self.workdir_paths.workdir)
# Copy the binary executables and DATA from the SPECFEM2D example
cp(self.sem2d_paths.bin, self.workdir_paths.bin)
cp(self.sem2d_paths.example_data, self.workdir_paths.data)
# Make sure that SPECFEM2D can find the expected files in the DATA/ dir
cd(self.workdir_paths.data)
rm("SOURCE")
ln("SOURCE_001", "SOURCE")
rm("Par_file")
ln("Par_file_Tape2007_onerec", "Par_file")
def generate_mesh(self, model_path, model_name, model_type=None):
"""
Performs meshing with internal mesher Meshfem3D and database generation
:type model_path: str
:param model_path: path to the model to be used for mesh generation
:type model_name: str
:param model_name: name of the model to be used as identification
:type model_type: str
:param model_type: available model types to be passed to the Specfem3D
Par_file. See Specfem3D Par_file for available options.
"""
available_model_types = ["gll"]
assert (exists(model_path)), f"model {model_path} does not exist"
model_type = model_type or getpar(key="MODEL", file="DATA/Par_file")
assert(model_type in available_model_types), \
f"{model_type} not in available types {available_model_types}"
unix.cd(self.cwd)
# Run mesh generation
if model_type == "gll":
self.check_mesh_properties(model_path)
src = glob(os.path.join(model_path, "*"))
dst = self.model_databases
unix.cp(src, dst)
call_solver(mpiexec=PAR.MPIEXEC, executable="bin/xmeshfem3D")
call_solver(mpiexec=PAR.MPIEXEC,
executable="bin/xgenerate_databases")
# Export the model for future use in the workflow
if self.taskid == 0:
self.export_model(os.path.join(PATH.OUTPUT, model_name))
def prepare_eval_grad(self, cwd, taskid, filenames, **kwargs):
"""
Prepares solver for gradient evaluation by writing residuals and
adjoint traces. Meant to be called by solver.eval_func().
Reads in observed and synthetic waveforms, applies optional
preprocessing, assesses misfit, and writes out adjoint sources and
STATIONS_ADJOINT file.
.. note::
Meant to be called by solver.eval_func(), may have unused arguments
to keep functions general across subclasses.
:type cwd: str
:param cwd: current specfem working directory containing observed and
synthetic seismic data to be read and processed. Should be defined
by solver.cwd
:type filenames: list of str
:param filenames: list of filenames defining the files in traces
"""
if taskid == 0:
self.logger.debug("preparing files for gradient evaluation")
for filename in filenames:
obs = self.reader(path=os.path.join(cwd, "traces", "obs"),
filename=filename)
syn = self.reader(path=os.path.join(cwd, "traces", "syn"),
filename=filename)
# Process observations and synthetics identically
if PAR.FILTER:
if taskid == 0:
self.logger.debug(f"applying {PAR.FILTER} filter to data")
obs = self._apply_filter(obs)
syn = self._apply_filter(syn)
if PAR.MUTE:
if taskid == 0:
self.logger.debug(f"applying {PAR.MUTE} mutes to data")
obs = self._apply_mute(obs)
syn = self._apply_mute(syn)
if PAR.NORMALIZE:
if taskid == 0:
self.logger.debug(
f"normalizing data with: {PAR.NORMALIZE}")
obs = self._apply_normalize(obs)
syn = self._apply_normalize(syn)
if PAR.MISFIT is not None:
self._write_residuals(cwd, syn, obs)
# Write the adjoint traces. Rename file extension for Specfem
if PAR.FORMAT.upper() == "ASCII":
# Change the extension to '.adj' from whatever it is
ext = os.path.splitext(filename)[-1]
filename_out = filename.replace(ext, ".adj")
elif PAR.FORMAT.upper() == "SU":
# TODO implement this
raise NotImplementedError
self._write_adjoint_traces(path=os.path.join(cwd, "traces", "adj"),
syn=syn,
obs=obs,
filename=filename_out)
# Copy over the STATIONS file to STATIONS_ADJOINT required by Specfem
# ASSUMING that all stations are used in adjoint simulation
src = os.path.join(cwd, "DATA", "STATIONS")
dst = os.path.join(cwd, "DATA", "STATIONS_ADJOINT")
unix.cp(src, dst)
