def get_job_status(self,
event: str,
sim_type: str,
par="VS",
iteration="current") -> str:
"""
Check the status of a salvus opt job
:param event: Name of event
:type event: str
:param sim_type: Type of simulation: forward, adjoint or smoothing
:type sim_type: str
:param par: parameter to be smoothed, optional, default VS
:type par: str
:param iteration: Name of iteration. "current" if current iteration
:type iteration: str
:return: status of job
:rtype: str
"""
if iteration == "current":
if sim_type == "forward":
if self.comm.project.forward_job[event]["submitted"]:
job_name = self.comm.project.forward_job[event]["name"]
else:
raise InversionsonError(
f"Forward job for event: {event} has not been "
"submitted")
elif sim_type == "adjoint":
if self.comm.project.adjoint_job[event]["submitted"]:
job_name = self.comm.project.adjoint_job[event]["name"]
else:
raise InversionsonError(
f"Adjoint job for event: {event} has not been "
"submitted")
elif sim_type == "smoothing":
if self.comm.project.smoothing_job[event][par]["submitted"]:
job_name = self.comm.project.smoothing_job[event][par][
"name"]
else:
raise InversionsonError(
f"Smoothing job for event: {event}, param {par} has not been "
"submitted")
else:
it_dict = self.comm.project.get_old_iteration_info(iteration)
job_name = it_dict["events"][event]["jobs"][sim_type]["name"]
job = sapi.get_job(job_name=job_name,
site_name=self.comm.project.site_name)
return job.update_status()
def construct_adjoint_simulation(self, event: str,
adj_src: object) -> object:
"""
Create the adjoint simulation object that salvus flow needs
:param event: Name of event
:type event: str
:param adj_src: List of adjoint source objects
:type adj_src: object
:return: Simulation object
:rtype: object
"""
print("Constructing Adjoint Simulation now")
from salvus.flow.simple_config import simulation
mesh = self.comm.lasif.get_simulation_mesh(event)
forward_job_name = self.comm.project.forward_job[event]["name"]
forward_job = sapi.get_job(site_name=self.comm.project.site_name,
job_name=forward_job_name)
meta = forward_job.output_path / "meta.json"
remote_mesh = forward_job.input_path / "mesh.h5"
# gradient = os.path.join(
# self.comm.lasif.lasif_root,
# "GRADIENTS",
# f"ITERATION_{self.comm.project.current_iteration}",
# event,
# "gradient.h5",
# )
gradient = "gradient.h5"
# if not os.path.exists(os.path.dirname(gradient)):
# os.makedirs(os.path.dirname(gradient))
w = simulation.Waveform(mesh=mesh)
w.adjoint.forward_meta_json_filename = f"REMOTE:{meta}"
if "VPV" in self.comm.project.inversion_params:
parameterization = "tti"
elif "VP" in self.comm.project.inversion_params:
parameterization = "rho-vp-vs"
w.adjoint.gradient.parameterization = parameterization
w.adjoint.gradient.output_filename = gradient
w.adjoint.point_source = adj_src
# Now set a remote mesh
w.set_mesh("REMOTE:" + str(remote_mesh))
w.validate()
return w
def delete_stored_wavefields(self, iteration: str, sim_type: str):
"""
Delete all stored jobs for a certain simulation type of an iteration
:param iteration: Name of iteration
:type iteration: str
:param sim_type: Type of simulation, forward or adjoint
:type sim_type: str
"""
iter_info = self.comm.project.get_old_iteration_info(iteration)
events_in_iteration = list(iter_info["events"].keys())
for event in events_in_iteration:
job_name = iter_info["events"][event]["jobs"][sim_type]["name"]
job = sapi.get_job(site_name=self.comm.project.site_name,
job_name=job_name)
job.delete()
def get_job_file_paths(self, event: str, sim_type: str) -> dict:
"""
Get the output folder for an event
:param event: Name of event
:type event: str
:param sim_type: Forward or adjoint simulation
:type sim_type: str
"""
if sim_type == "forward":
job_name = self.comm.project.forward_job[event]["name"]
elif sim_type == "adjoint":
job_name = self.comm.project.adjoint_job[event]["name"]
else:
raise InversionsonError(f"Don't recognise sim_type {sim_type}")
job = sapi.get_job(job_name=job_name,
site_name=self.comm.project.site_name)
return job.get_output_files()
def retrieve_outputs(self, event_name: str, sim_type: str):
"""
Currently we need to use command line salvus opt to
retrieve the seismograms. There must be some better way
though.
:param event_name: Name of event
:type event_name: str
:param sim_type: Type of simulation, forward, adjoint
:type sim_type: str
"""
job_name = self._get_job_name(
event=event_name, sim_type=sim_type, new=False
)
salvus_job = sapi.get_job(
site_name=self.comm.project.site_name, job_name=job_name
)
if sim_type == "forward":
destination = self.comm.lasif.find_seismograms(
event=event_name, iteration=self.comm.project.current_iteration
)
elif sim_type == "adjoint":
destination = self.comm.lasif.find_gradient(
iteration=self.comm.project.current_iteration,
event=event_name,
smooth=False,
inversion_grid=False,
just_give_path=True,
)
else:
raise InversionsonError(
f"Simulation type {sim_type} not supported in this function"
)
salvus_job.copy_output(
destination=os.path.dirname(destination),
allow_existing_destination_folder=True,
)
def construct_adjoint_simulation(self, event: str,
adj_src: object) -> object:
"""
Create the adjoint simulation object that salvus flow needs
:param event: Name of event
:type event: str
:param adj_src: List of adjoint source objects
:type adj_src: object
:return: Simulation object
:rtype: object
"""
from salvus.flow.simple_config import simulation
mesh = self.comm.lasif.get_simulation_mesh(event)
forward_job_name = self.comm.project.forward_job[event]["name"]
forward_job_path = sapi.get_job(site_name=self.comm.project.site_name,
job_name=forward_job_name).output_path
meta = os.path.join(forward_job_path, "meta.json")
gradient = os.path.join(
self.comm.lasif.lasif_root,
"GRADIENTS",
f"ITERATION_{self.comm.project.current_iteration}",
event,
"gradient.h5",
)
w = simulation.Waveform(mesh=mesh)
w.adjoint.forward_meta_json_filename = f"REMOTE:{meta}"
w.adjoint.gradient.parameterization = "rho-vp-vs"
w.adjoint.gradient.output_filename = gradient
w.adjoint.point_source = adj_src
w.validate()
return w
def retrieve_outputs(self, event_name: str, sim_type: str, par=None):
"""
Currently we need to use command line salvus opt to
retrieve the seismograms. There must be some better way
though.
:param event_name: Name of event
:type event_name: str
:param sim_type: Type of simulation, forward, adjoint, smoothing
:type sim_type: str
:param par: parameter to smooth if sim_type=smoothing, optional
:type par: str
"""
job_name = self._get_job_name(event=event_name,
sim_type=sim_type,
new=False,
par=par)
salvus_job = sapi.get_job(site_name=self.comm.project.site_name,
job_name=job_name)
if sim_type == "forward":
destination = self.comm.lasif.find_seismograms(
event=event_name,
iteration=self.comm.project.current_iteration)
if sim_type == "adjoint":
destination = self.comm.lasif.find_gradient(
iteration=self.comm.project.current_iteration,
event=event_name,
smooth=False,
inversion_grid=False,
just_give_path=True,
)
if sim_type == "smoothing":
if self.comm.project.meshes == "mono-mesh":
destination = self.comm.lasif.find_gradient(
iteration=self.comm.project.current_iteration,
event=None,
smooth=True,
summed=True,
just_give_path=True)
smooth_grad = self.comm.lasif.find_gradient(
iteration=self.comm.project.current_iteration,
event=None,
smooth=True,
summed=True,
just_give_path=True)
else:
destination = self.comm.lasif.find_gradient(
iteration=self.comm.project.current_iteration,
event=event_name,
smooth=True,
inversion_grid=False,
just_give_path=True,
)
smooth_grad = self.comm.lasif.find_gradient(
iteration=self.comm.project.current_iteration,
event=event_name,
smooth=True,
inversion_grid=False,
just_give_path=True,
)
destination = os.path.join(os.path.dirname(destination),
"smoother_output", "smooth_gradient.h5")
salvus_job.copy_output(
destination=os.path.dirname(destination),
allow_existing_destination_folder=True,
)
if sim_type == "smoothing":
self.comm.salvus_mesher.add_field_from_one_mesh_to_another(
from_mesh=destination, to_mesh=smooth_grad, field_name=par)
def get_job(self,
event: str,
sim_type: str,
iteration="current") -> object:
"""
Get Salvus.Flow Job Object, or JobArray Object
:param event: Name of event
:type event: str
:param sim_type: type of simulation
:type sim_type: str
:param iteration: name of iteration, defaults to "current"
:type iteration: str, optional
"""
if (iteration == "current"
or iteration == self.comm.project.current_iteration):
if sim_type == "forward":
if self.comm.project.forward_job[event]["submitted"]:
job_name = self.comm.project.forward_job[event]["name"]
else:
raise InversionsonError(
f"Forward job for event: {event} has not been "
"submitted")
elif sim_type == "adjoint":
if self.comm.project.adjoint_job[event]["submitted"]:
job_name = self.comm.project.adjoint_job[event]["name"]
else:
raise InversionsonError(
f"Adjoint job for event: {event} has not been "
"submitted")
elif sim_type == "smoothing":
if self.comm.project.inversion_mode == "mono-batch":
smoothing_job = self.comm.project.smoothing_job
else:
smoothing_job = self.comm.project.smoothing_job[event]
if smoothing_job["submitted"]:
job_name = smoothing_job["name"]
else:
raise InversionsonError(
f"Smoothing job for event: {event} has not been "
"submitted")
else:
it_dict = self.comm.project.get_old_iteration_info(iteration)
if (sim_type == "smoothing"
and self.comm.project.inversion_mode == "mono-batch"):
job_name = it_dict["smoothing"]["name"]
else:
event_index = self.comm.project.get_key_number_for_event(
event=event, iteration=iteration)
job_name = it_dict["events"][event_index]["job_info"][
sim_type]["name"]
if sim_type == "smoothing":
site_name = self.comm.project.smoothing_site_name
job = sapi.get_job_array(job_array_name=job_name,
site_name=site_name)
else:
site_name = self.comm.project.site_name
job = sapi.get_job(job_name=job_name, site_name=site_name)
return job
def _launch_hpc_processing_job(self, event):
"""
Here, we launch a job to select windows and get adjoint sources
for an event.
"""
submitted, _ = self.__submitted_retrieved(event, "hpc_processing")
if submitted:
return
if not self.comm.project.remote_data_processing:
self.__process_data(event)
iteration = self.comm.project.current_iteration
forward_job = sapi.get_job(
site_name=self.comm.project.site_name,
job_name=self.comm.salvus_flow.get_job_name(event=event,
sim_type="forward"),
)
# Get forward paths
remote_syn_path = str(forward_job.output_path / "receivers.h5")
forward_meta_json_filename = str(forward_job.output_path / "meta.json")
# Get local proc filename
lasif_root = self.comm.project.lasif_root
proc_filename = (f"preprocessed_{int(self.comm.project.min_period)}s_"
f"to_{int(self.comm.project.max_period)}s.h5")
local_proc_file = os.path.join(lasif_root, "PROCESSED_DATA",
"EARTHQUAKES", event, proc_filename)
remote_proc_file_name = f"{event}_{proc_filename}"
hpc_cluster = get_site(self.comm.project.site_name)
remote_processed_dir = os.path.join(
self.comm.project.remote_inversionson_dir, "PROCESSED_DATA")
remote_adj_dir = os.path.join(
self.comm.project.remote_inversionson_dir, "ADJOINT_SOURCES")
remote_receiver_dir = os.path.join(
self.comm.project.remote_inversionson_dir, "RECEIVERS")
for dir_name in [
remote_processed_dir, remote_adj_dir, remote_receiver_dir
]:
if not hpc_cluster.remote_exists(dir_name):
hpc_cluster.remote_mkdir(remote_processed_dir)
remote_proc_path = os.path.join(remote_processed_dir,
remote_proc_file_name)
tmp_remote_path = remote_proc_path + "_tmp"
if not hpc_cluster.remote_exists(remote_proc_path):
hpc_cluster.remote_put(local_proc_file, tmp_remote_path)
hpc_cluster.run_ssh_command(
f"mv {tmp_remote_path} {remote_proc_path}")
remote_adj_dir = os.path.join(
self.comm.project.remote_inversionson_dir, "ADJOINT_SOURCES")
if "VPV" in self.comm.project.inversion_params:
parameterization = "tti"
elif "VP" in self.comm.project.inversion_params:
parameterization = "rho-vp-vs"
info = dict(
processed_filename=remote_proc_path,
synthetic_filename=remote_syn_path,
forward_meta_json_filename=forward_meta_json_filename,
parameterization=parameterization,
event_name=event,
delta=self.comm.project.simulation_dict["time_step"],
npts=self.comm.project.simulation_dict["number_of_time_steps"],
iteration_name=iteration,
minimum_period=self.comm.project.min_period,
maximum_period=self.comm.project.max_period,
start_time_in_s=self.comm.project.simulation_dict["start_time"],
receiver_json_path=os.path.join(remote_receiver_dir,
f"{event}_receivers.json"),
ad_src_type=self.comm.project.ad_src_type,
)
toml_filename = f"{iteration}_{event}_adj_info.toml"
with open(toml_filename, "w") as fh:
toml.dump(info, fh)
# Put info toml on daint and remove local toml
remote_toml = os.path.join(remote_adj_dir, toml_filename)
hpc_cluster.remote_put(toml_filename, remote_toml)
os.remove(toml_filename)
# Copy processing script to hpc
remote_script = os.path.join(remote_adj_dir,
"window_and_calc_adj_src.py")
if not hpc_cluster.remote_exists(remote_script):
hpc_cluster.remote_put(PROCESS_OUTPUT_SCRIPT_PATH, remote_script)
# Now submit the job
description = f"HPC processing of {event} for iteration {iteration}"
# use interp wall time for now
wall_time = self.comm.project.hpc_processing_wall_time
from salvus.flow.sites import job, remote_io_site
commands = [
remote_io_site.site_utils.RemoteCommand(command="mkdir output",
execute_with_mpi=False),
remote_io_site.site_utils.RemoteCommand(
command=f"python {remote_script} {remote_toml}",
execute_with_mpi=False),
]
# Allow to set conda environment first
if self.comm.project.remote_conda_env:
conda_command = [
remote_io_site.site_utils.RemoteCommand(
command=
f"conda activate {self.comm.project.remote_conda_env}",
execute_with_mpi=False,
)
]
commands = conda_command + commands
job = job.Job(
site=sapi.get_site(self.comm.project.interpolation_site),
commands=commands,
job_type="hpc_processing",
job_description=description,
job_info={},
wall_time_in_seconds=wall_time,
no_db=False,
)
self.comm.project.change_attribute(
attribute=f'hpc_processing_job["{event}"]["name"]',
new_value=job.job_name,
)
job.launch()
self.comm.project.change_attribute(
attribute=f'hpc_processing_job["{event}"]["submitted"]',
new_value=True,
)
self.print(f"HPC Processing job for event {event} submitted")
self.comm.project.update_iteration_toml()