def test_cmd_configure(tmpdir, setup_par_file, conf_par_file): """ Test configuring a parameter file from a template par file .. note:: I don't know exactly why, but this test needs to be run AFTER any other test which runs seisflows.init(), otherwise the parameters are not instantiated properly (you will hit a KeyError when trying to access PAR). I think this is because of how seisflows.configure() registers a relatively empty parameter file (only modules are defined), and this gets saved into sys modules, affecting subsequent tests which end up accessing sys.modules. I tried flushing sys.modules but it didn't work. This behavior shouldn't get encountered in a real run because we won't need to run init() and configure() in the same python runtime environment, but I leave this warning here wondering if I'll have to fix it at some point... -B """ os.chdir(tmpdir) # Copy in the setup par file so we can configure it src = setup_par_file dst = os.path.join(tmpdir, "parameters.yaml") shutil.copy(src, dst) # run seisflows init with patch.object(sys, "argv", ["seisflows"]): sf = SeisFlows() sf.configure(relative_paths=False) # Simple check that the configuration parameter file has the same number # of lines as the one that has been created by configure lines_conf = open(conf_par_file, "r").readlines() lines_fill = open("parameters.yaml", "r").readlines() assert (len(lines_conf) == len(lines_fill))
def __init__(self, ntask=3, niter=2): """ Set path structure which is used to navigate around SPECFEM repositories and the example working directory :type ntask: int :param ntask: number of events to use in inversion, between 1 and 25. defaults to 3 :type niter: int :param niter: number of iterations to run. defaults to 2 """ specfem2d_repo = input( msg.cli("If you have already downloaded SPECMFE2D, please input " "the full path to the repo. If left blank, this example " "will pull the latest version from GitHub and attempt " "to configure and make the binaries:\n> ")) self.cwd = os.getcwd() self.sem2d_paths, self.workdir_paths = self.define_dir_structures( cwd=self.cwd, specfem2d_repo=specfem2d_repo) self.ntask = ntask assert(1 <= self.ntask <= 25), \ f"number of tasks/events must be between 1 and 25, not {self.ntask}" self.niter = niter assert(1 <= self.niter <= np.inf), \ f"number of iterations must be between 1 and inf, not {self.niter}" # This bool information is provided by the User running 'setup' or 'run' self.run_example = bool(sys.argv[1] == "run") # Command line tool to use $ seisflows <cmd> from inside Python # Zero out sys.argv to ensure that no arguments are given to the CLI sys.argv = [sys.argv[0]] self.sf = SeisFlows()
def test_cmd_par(tmpdir, copy_par_file): """ Make sure the 'par' command can print and edit the parameter file :param tmpdir: :return: """ # Run init first to create a working state os.chdir(tmpdir) copy_par_file parameter = "begin" expected_val = "1" new_val = "2" # testing the get option: seisflows par `parameter` with patch.object(sys, "argv", ["seisflows"]): sf = SeisFlows() # Run this with subprocess so we can capture the print statement cmd_line_arg = ["seisflows", "par", parameter] out = subprocess.run(cmd_line_arg, capture_output=True, universal_newlines=True) # Check that we printed out the correct value par, val = out.stdout.strip().split(":") assert (par.upper() == parameter.upper()) assert (int(val) == int(expected_val)) # testing the set option: seisflows par `parameter` `value` with patch.object(sys, "argv", ["seisflows"]): sf = SeisFlows() # Run this with subprocess so we can capture the print statement cmd_line_arg = ["seisflows", "par", parameter, new_val] out1 = subprocess.run(cmd_line_arg, capture_output=True, universal_newlines=True) # Run this with subprocess so we can capture the print statement cmd_line_arg = ["seisflows", "par", parameter] out2 = subprocess.run(cmd_line_arg, capture_output=True, universal_newlines=True) # Check that the changed print statement works par, vals = out1.stdout.strip().split(":") val_old, val_new = vals.strip().split(" -> ") assert (par.upper() == parameter.upper()) assert (int(val_old) == int(expected_val)) assert (int(val_new) == int(new_val)) # Check that we printed out the correctly changed value par, val = out2.stdout.strip().split(":") assert (par.upper() == parameter.upper()) assert (int(val) == int(new_val))
def sfinit(tmpdir, copy_par_file): """ Re-used function that will initate a SeisFlows3 working environment in sys modules :return: """ copy_par_file os.chdir(tmpdir) with patch.object(sys, "argv", ["seisflows"]): sf = SeisFlows() sf._register(force=True) config.init_seisflows(check=False) return sf
def test_cmd_setup(tmpdir): """ Test setting up the SeisFlows3 working directory """ os.chdir(tmpdir) par_file = os.path.join(tmpdir, "parameters.yaml") # Check with anmd without symlinking as well as with overwriting with patch.object(sys, "argv", ["seisflows"]): # Without symlinking sf = SeisFlows() sf.setup(symlink=False, overwrite=False) assert (os.path.exists(par_file)) os.remove(par_file) # With symlinking sf.setup(symlink=True, overwrite=False) assert (os.path.exists(par_file)) assert (os.path.exists( os.path.join(tmpdir, "source_code", "seisflows3"))) # Edit the current par file in a noticeable way so we can check # if overwriting works in the next step test_phrase = "well this is rather unexpected...\n" with open(par_file, "a") as f: f.write(test_phrase) with open(par_file, "r") as f: assert (test_phrase in f.read()) # With overwriting sf.setup(symlink=False, overwrite=True, force=True) assert (os.path.exists(par_file)) with open(par_file, "r") as f: text = f.read() assert (test_phrase not in text)
def test_call_seisflows(tmpdir, par_file_dict, copy_par_file): """ Test calling the 'par' command from command line and inside a python environemnt. Check that case-insensitivity is also honored Check against the actual value coming from the parameter file Also tests the seisflows 'par' command """ copy_par_file os.chdir(tmpdir) check_val = par_file_dict.LINESEARCH # Mock argv to match the actual scenario in which SeisFlows will be called with patch.object(sys, "argv", ["seisflows"]): for name in ["linesearch", "LINESEARCH", "LineSearch"]: # From the command line cmd_line_arg = ["seisflows", "par", name] out = subprocess.run(cmd_line_arg, capture_output=True, universal_newlines=True) assert (out.stdout.strip() == f"{name.upper()}: {check_val}") # Test from inside a Python environment; we need to redirect stdout # to make sure the print statement is working as expected f = io.StringIO() with contextlib.redirect_stdout(f): sf = SeisFlows() sf(command="par", parameter=name) stdout = f.getvalue() assert (stdout.strip() == f"{name.upper()}: {check_val}")
def test_load_modules(tmpdir, copy_par_file): """ Test if module loading from sys.modules works :param tmpdir: :return: """ # Run init first to create a working state os.chdir(tmpdir) copy_par_file # Create necessary paths to get past some assertion errors parameters = loadyaml("parameters.yaml") paths = parameters.pop("PATHS") for key in ["MODEL_INIT", "MODEL_TRUE"]: os.mkdir(paths[key]) with patch.object(sys, "argv", ["seisflows"]): sf = SeisFlows() sf.init() # Check a random parameter and then set it to something different preprocess = sys.modules["seisflows_preprocess"] assert (preprocess.misfit is None) preprocess.misfit = 1 # See if we can load modules and restore previous working state which # overwrites the previous operation sf._load_modules() assert (sys.modules["seisflows_preprocess"].misfit != 1)
def test_register(tmpdir, par_file_dict, copy_par_file): """ Test that the register function, which reads in PATHS and PARAMETERS works as expected, returning paths and parameters that we can read """ copy_par_file os.chdir(tmpdir) with patch.object(sys, "argv", ["seisflows"]): sf = SeisFlows() assert (sf._paths is None) assert (sf._parameters is None) sf._register(force=True) # Check that paths and parameters have been set in sys.modules paths = sys.modules["seisflows_paths"] parameters = sys.modules["seisflows_parameters"] # Check one or two parameters have been set correctly assert (par_file_dict.LBFGSMAX == parameters.LBFGSMAX) path_check_full = os.path.abspath(par_file_dict.PATHS["SCRATCH"]) assert (path_check_full == paths.SCRATCH)
def test_cmd_init(tmpdir, copy_par_file): """ Test 'seisflows init' command which instantiates a working directory and saves the active working state as pickle files :return: """ os.chdir(tmpdir) copy_par_file # Create necessary paths to get past some assertion errors parameters = loadyaml("parameters.yaml") paths = parameters.pop("PATHS") for key in ["MODEL_INIT", "MODEL_TRUE"]: os.mkdir(paths[key]) with patch.object(sys, "argv", ["seisflows"]): sf = SeisFlows() sf.init() for name in NAMES: assert (os.path.exists( os.path.join(paths["OUTPUT"], f"seisflows_{name}.p")))
def test_cmd_clean(tmpdir): """ :param tmpdir: :return: """ os.chdir(tmpdir) # Create a bunch of files that match what should be deleted. Make them as # directories even though some should be files because we just want to see # if they get deleted or not for path in CFGPATHS.values(): os.mkdir(path) # Symlink the last file to make sure it still exists even if it matches shutil.rmtree(path) os.symlink(src=CFGPATHS.PAR_FILE, dst=path) with patch.object(sys, "argv", ["seisflows"]): sf = SeisFlows() sf.clean(force=True) for fid in [path, CFGPATHS.PAR_FILE]: assert (os.path.exists(fid))
def test_config_logging(tmpdir, copy_par_file): """ Test logging configuration to make sure we can print to file :param tmpdir: :return: """ # Run init first to create a working state os.chdir(tmpdir) copy_par_file msg = "This is an example log that will be checked for test purposes" with patch.object(sys, "argv", ["seisflows"]): sf = SeisFlows() sf._register(force=True) sf._config_logging() logger.debug(msg) # Check that we created the log file and wrote the message in assert (os.path.exists(CFGPATHS.LOGFILE)) with open(CFGPATHS.LOGFILE, "r") as f: lines = f.read() assert (msg in lines) assert ("DEBUG" in lines) # levelname assert ("test_config_logging()" in lines) # funcName
from seisflows3.tools import unix from seisflows3.seisflows import SeisFlows # USER CAN EDIT THE FOLLOWING PATHS: # WORKDIR: points to your own working directory # SPECFEM3D: points to an existing SPECFEM3D repository if available WORKDIR = os.getcwd() SPECFEM3D_ORIGINAL = input("Path to SPECFEM3D respository with compiled " "binaries?: ") assert(os.path.exists(SPECFEM3D_ORIGINAL)), \ f"SPECFEM3D repo doesn't exist: {SPECFEM3D_ORIGINAL}" # Instantiate the SeisFlows class to use its command line arguments sf = SeisFlows() # ============================================================================== # Distribute the necessary file structure of the SPECFEM3D repository that we # will downloaded/reference SPECFEM3D_BIN_ORIGINAL = os.path.join(SPECFEM3D_ORIGINAL, "bin") SPECFEM3D_DATA_ORIGINAL = os.path.join(SPECFEM3D_ORIGINAL, "DATA") TAPE_2007_EXAMPLE = os.path.join(SPECFEM3D_ORIGINAL, "EXAMPLES", "homogeneous_halfspace") # The SPECFEM3D working directory that we will create separate from the # downloaded repo SPECFEM3D_WORKDIR = os.path.join(WORKDIR, "SPECFEM3D_workdir") SPECFEM3D_BIN = os.path.join(SPECFEM3D_WORKDIR, "bin") SPECFEM3D_DATA = os.path.join(SPECFEM3D_WORKDIR, "DATA") SPECFEM3D_OUTPUT = os.path.join(SPECFEM3D_WORKDIR, "OUTPUT_FILES")
from seisflows3.tools import unix from seisflows3.seisflows import SeisFlows # USER CAN EDIT THE FOLLOWING PATHS: # WORKDIR: points to your own working directory # SPECFEM2D: points to an existing specfem2D repository if available WORKDIR = os.getcwd() SPECFEM2D_ORIGINAL = input("Path to SPECFEM2D respository with compiled " "binaries?: ") assert(os.path.exists(SPECFEM2D_ORIGINAL)), \ f"SPECFEM2D repo doesn't exist: {SPECFEM2D_ORIGINAL}" # Instantiate the SeisFlows class to use its command line arguments sf = SeisFlows() # ============================================================================== # Distribute the necessary file structure of the SPECFEM2D repository that we # will downloaded/reference SPECFEM2D_BIN_ORIGINAL = os.path.join(SPECFEM2D_ORIGINAL, "bin") SPECFEM2D_DATA_ORIGINAL = os.path.join(SPECFEM2D_ORIGINAL, "DATA") TAPE_2007_EXAMPLE = os.path.join(SPECFEM2D_ORIGINAL, "EXAMPLES", "Tape2007") # The SPECFEM2D working directory that we will create separate from the # downloaded repo SPECFEM2D_WORKDIR = os.path.join(WORKDIR, "specfem2d_workdir") SPECFEM2D_BIN = os.path.join(SPECFEM2D_WORKDIR, "bin") SPECFEM2D_DATA = os.path.join(SPECFEM2D_WORKDIR, "DATA") SPECFEM2D_OUTPUT = os.path.join(SPECFEM2D_WORKDIR, "OUTPUT_FILES")
class SF3Example2D: """ A class for running SeisFlows3 examples. Simplifies calls structure so that multiple example runs can benefit from the code written here """ def __init__(self, ntask=3, niter=2): """ Set path structure which is used to navigate around SPECFEM repositories and the example working directory :type ntask: int :param ntask: number of events to use in inversion, between 1 and 25. defaults to 3 :type niter: int :param niter: number of iterations to run. defaults to 2 """ specfem2d_repo = input( msg.cli("If you have already downloaded SPECMFE2D, please input " "the full path to the repo. If left blank, this example " "will pull the latest version from GitHub and attempt " "to configure and make the binaries:\n> ")) self.cwd = os.getcwd() self.sem2d_paths, self.workdir_paths = self.define_dir_structures( cwd=self.cwd, specfem2d_repo=specfem2d_repo) self.ntask = ntask assert(1 <= self.ntask <= 25), \ f"number of tasks/events must be between 1 and 25, not {self.ntask}" self.niter = niter assert(1 <= self.niter <= np.inf), \ f"number of iterations must be between 1 and inf, not {self.niter}" # This bool information is provided by the User running 'setup' or 'run' self.run_example = bool(sys.argv[1] == "run") # Command line tool to use $ seisflows <cmd> from inside Python # Zero out sys.argv to ensure that no arguments are given to the CLI sys.argv = [sys.argv[0]] self.sf = SeisFlows() @staticmethod def define_dir_structures(cwd, specfem2d_repo, ex="Tape2007"): """ Define the example directory structure, which will contain abridged versions of the SPECFEM2D working directory :type cwd: str :param cwd: current working directory :type specfem2d_repo: str :param specfem2d_repo: location of the SPECFEM2D repository :type ex: str :type ex: The name of the example problem inside SPECFEM2D/EXAMPLES """ if not specfem2d_repo: print(f"No existing SPECFEM2D repo given, default to: " f"{cwd}/specfem2d") specfem2d_repo = os.path.join(cwd, "specfem2d") else: assert (os.path.exists(specfem2d_repo)), ( f"User supplied SPECFEM2D directory '{specfem2d_repo}' " f"does not exist, please check your path and try again.") # This defines required structures from the SPECFEM2D repository sem2d = { "repo": specfem2d_repo, "bin": os.path.join(specfem2d_repo, "bin"), "data": os.path.join(specfem2d_repo, "DATA"), "example": os.path.join(specfem2d_repo, "EXAMPLES", ex), "example_data": os.path.join(specfem2d_repo, "EXAMPLES", ex, "DATA") } # This defines a working directory structure which we will create working_directory = os.path.join(cwd, "specfem2d_workdir") workdir = { "workdir": os.path.join(working_directory), "bin": os.path.join(working_directory, "bin"), "data": os.path.join(working_directory, "DATA"), "output": os.path.join(working_directory, "OUTPUT_FILES"), "model_init": os.path.join(working_directory, "OUTPUT_FILES_INIT"), "model_true": os.path.join(working_directory, "OUTPUT_FILES_TRUE"), } return Dict(sem2d), Dict(workdir) def download_specfem2d(self): """ Download the latest version of SPECFEM2D from GitHub, devel branch. Last successfully tested 4/28/22 """ if not os.path.exists(self.sem2d_paths.repo): cmd = ("git clone --recursive --branch devel " "https://github.com/geodynamics/specfem2d.git") print(f"Downloading SPECFEM2D with command: {cmd}") subprocess.run(cmd, shell=True, check=True) def configure_specfem2d_and_make_binaries(self): """ Run ./configure within the SPECFEM2D repo directory. This function assumes it is being run from inside the repo. Should guess all the configuration options. Probably the least stable part of the example """ cd(self.sem2d_paths.repo) try: if not os.path.exists("./config.log"): cmd = "./configure" print(f"Configuring SPECFEM2D with command: {cmd}") # Ignore the configure outputs from SPECFEM subprocess.run(cmd, shell=True, check=True, stdout=subprocess.DEVNULL) else: print("SPECFEM2D already configured, skipping 'configure'") except subprocess.CalledProcessError as e: print(f"SPECFEM `configure` step has failed, please check and " f"retry. If this command repeatedly fails, you may need " f"to configure SPECFEM2D manually.\n{e}") sys.exit(-1) try: if not glob.glob("./bin/x*"): cmd = "make all" print(f"Making SPECFEM2D binaries with command: {cmd}") # Ignore the make outputs from SPECFEM subprocess.run(cmd, shell=True, check=True, stdout=subprocess.DEVNULL) else: print("executables found in SPECFEM2D/bin directory, " "skipping 'make'") except subprocess.CalledProcessError as e: print(f"SPECFEM 'make' step has failed, please check and " f"retry. If this command repeatedly fails, you may need " f"to compile SPECFEM2D manually.\n{e}") sys.exit(-1) # Symlink the finished repo into the working directory so that any # subsequent runs won't need to have the user re-type repo location if not os.path.exists(os.path.join(self.cwd, "specfem2d")): print("symlinking existing specfem2D repository to cwd") ln(self.sem2d_paths.repo, os.path.join(self.cwd, "specfem2d")) def create_specfem2d_working_directory(self): """ Create the working directory where we will generate our initial and final models using one of the SPECFEM2D examples """ assert (os.path.exists(self.sem2d_paths["example"])), ( f"SPECFEM2D/EXAMPLE directory: '{self.sem2d['example']}' " f"does not exist, please check this path and try again.") # Incase this example has written before, remove dir. that were created rm(self.workdir_paths.workdir) mkdir(self.workdir_paths.workdir) # Copy the binary executables and DATA from the SPECFEM2D example cp(self.sem2d_paths.bin, self.workdir_paths.bin) cp(self.sem2d_paths.example_data, self.workdir_paths.data) # Make sure that SPECFEM2D can find the expected files in the DATA/ dir cd(self.workdir_paths.data) rm("SOURCE") ln("SOURCE_001", "SOURCE") rm("Par_file") ln("Par_file_Tape2007_onerec", "Par_file") def setup_specfem2d_for_model_init(self): """ Make some adjustments to the original parameter file to. This function assumes it is running from inside the SPECFEM2D/DATA dir """ cd(self.workdir_paths.data) assert (os.path.exists("Par_file")), f"I cannot find the Par_file!" print("> Setting the SPECFEM2D Par_file for SeisFlows3 compatiblility") self.sf.sempar("setup_with_binary_database", 1) # create .bin files self.sf.sempar("save_model", "binary") # output model in .bin format self.sf.sempar("save_ASCII_kernels", ".false.") # kernels also .bin rm(self.workdir_paths.output) mkdir(self.workdir_paths.output) def setup_specfem2d_for_model_true(self): """ Make some adjustments to the parameter file to create the final velocity model. This function assumes it is running from inside the SPECFEM2D/DATA directory """ cd(self.workdir_paths.data) assert (os.path.exists("Par_file")), f"I cannot find the Par_file!" print("> Updating initial homogeneous velocity model values") new_model = "1 1 2600.d0 5900.d0 3550.0d0 0 0 10.d0 10.d0 0 0 0 0 0 0" self.sf.sempar("velocity_model", new_model) def run_xspecfem2d_binaries(self): """ Runs the xmeshfem2d and then xspecfem2d binaries using subprocess and then do some cleanup to get files in the correct locations. Assumes that we can run the binaries directly with './' """ cd(self.workdir_paths.workdir) cmd_mesh = f"./bin/xmeshfem2D > OUTPUT_FILES/mesher.log.txt" cmd_spec = f"./bin/xspecfem2D > OUTPUT_FILES/solver.log.txt" for cmd in [cmd_mesh, cmd_spec]: print(f"Running SPECFEM2D with command: {cmd}") subprocess.run(cmd, shell=True, check=True, stdout=subprocess.DEVNULL) def cleanup_xspecfem2d_run(self, choice=None): """ Do some cleanup after running the SPECFEM2D binaries to make sure files are in the correct locations, and rename the OUTPUT_FILES directory so that it does not get overwritten by subsequent runs :type choice: str :param choice: Rename the OUTPUT_FILES directory with a suffix tag msut be 'INIT' or 'TRUE'. If None, will not rename but the """ cd(self.workdir_paths.workdir) print("> Cleaning up after xspecfem2d, setting up for new run") # SPECFEM2D outputs its models in the DATA/ directory by default, # while SeisFlows3 expects this in the OUTPUT_FILES/ directory (which is # the default in SPECFEM3D) mv(glob.glob("DATA/*bin"), self.workdir_paths.output) if choice == "INIT": mv(self.workdir_paths.output, self.workdir_paths.model_init) # Create a new OUTPUT_FILES/ directory for TRUE run rm(self.workdir_paths.output) mkdir(self.workdir_paths.output) elif choice == "TRUE": mv(self.workdir_paths.output, self.workdir_paths.model_true) def setup_seisflows_working_directory(self): """ Create and set the SeisFlows3 parameter file, making sure all required parameters are set correctly for this example problem """ cd(self.cwd) self.sf.setup(force=True) # Force will delete existing parameter file self.sf.configure() self.sf.par("ntask", self.ntask) # default 3 sources for this example self.sf.par("materials", "elastic") # how velocity model parameterized self.sf.par("density", "constant") # update density or keep constant self.sf.par("nt", 5000) # set by SPECFEM2D Par_file self.sf.par("dt", .06) # set by SPECFEM2D Par_file self.sf.par("f0", 0.084) # set by SOURCE file self.sf.par("format", "ascii") # how to output synthetic seismograms self.sf.par("begin", 1) # first iteration self.sf.par("end", self.niter) # final iteration -- we will run 2 self.sf.par("case", "synthetic") # synthetic-synthetic inversion self.sf.par("attenuation", False) self.sf.par("specfem_bin", self.workdir_paths.bin) self.sf.par("specfem_data", self.workdir_paths.data) self.sf.par("model_init", self.workdir_paths.model_init) self.sf.par("model_true", self.workdir_paths.model_true) def finalize_specfem2d_par_file(self): """ Last minute changes to get the SPECFEM2D Par_file in the correct format for running SeisFlows3. Setting model type to read from .bin GLL files change number of stations etc. """ cd(self.workdir_paths.data) self.sf.sempar("model", "gll") # GLL so SPECFEM reads .bin files def run_sf3_example(self): """ Use subprocess to run the SeisFlows3 example we just set up """ cd(self.cwd) subprocess.run("seisflows submit -f", check=False, shell=True) def main(self): """ Setup the example and then optionally run the actual seisflows workflow """ print(msg.cli("EXAMPLE SETUP", border="=")) # Step 1: Download and configure SPECFEM2D, make binaries. Optional self.download_specfem2d() self.configure_specfem2d_and_make_binaries() # Step 2: Create a working directory and generate initial/final models self.create_specfem2d_working_directory() # Step 2a: Generate MODEL_INIT, rearrange consequent directory structure print(msg.cli("GENERATING INITIAL MODEL", border="=")) self.setup_specfem2d_for_model_init() self.run_xspecfem2d_binaries() self.cleanup_xspecfem2d_run(choice="INIT") # Step 2b: Generate MODEL_INIT, rearrange consequent directory structure print(msg.cli("GENERATING TRUE/TARGET MODEL", border="=")) self.setup_specfem2d_for_model_true() self.run_xspecfem2d_binaries() self.cleanup_xspecfem2d_run(choice="TRUE") # Step 3: Prepare Par_file and directory for MODEL_TRUE generation self.setup_seisflows_working_directory() self.finalize_specfem2d_par_file() print(msg.cli("COMPLETE EXAMPLE SETUP", border="=")) # Step 4: Run the workflwo if self.run_example: print(msg.cli("RUNNING SEISFLOWS3 INVERSION WORKFLOW", border="=")) self.run_sf3_example()