def sum_two_fields_on_a_mesh(
self,
mesh: str,
fieldname_1: str,
fieldname_2: str,
newname: str = None,
delete_old_fields: bool = False,
):
"""
Take two fields on a mesh and sum them together. If no newname is
given the summed field will be written into both the old fields.
If newname is given the summed field will be written in there
and if delete_old_fields is true they will be deleted of course.
:param mesh: Path to mesh to be used
:type mesh: str or Path
:param fieldname_1: Name of field to be summed
:type fieldname_1: str
:param fieldname_2: Name of other field to be summed
:type fieldname_2: str
:param newname: Name of field to store summed field, defaults to None
:type newname: str, optional
:param delete_old_fields: Whether old fields should be deleted,
defaults to False. Currently not implemented
:type delete_old_fields: bool, optional
"""
m = UnstructuredMesh.from_h5(mesh)
available_fields = list(m.element_nodal_fields.keys())
if fieldname_1 not in available_fields:
raise InversionsonError(
f"Field {fieldname_1} not available on mesh {mesh}. "
f"Only available fields are: {available_fields}"
)
if fieldname_2 not in available_fields:
raise InversionsonError(
f"Field {fieldname_2} not available on mesh {mesh}. "
f"Only available fields are: {available_fields}"
)
if delete_old_fields:
if newname is not None:
raise InversionsonError(
"If you want to delete old fields you need to write the "
"summed one into a new field"
)
summed_field = np.copy(m.element_nodal_fields[fieldname_1])
summed_field += m.element_nodal_fields[fieldname_2]
if newname is None:
m.attach_field(fieldname_1, summed_field)
m.attach_field(fieldname_2, summed_field)
m.write_h5(mesh)
else:
m.attach_field(newname, summed_field)
def __get_custom_job(self, event: str, sim_type: str):
"""
A get_job function which handles job types which are not of type
salvus.flow.sites.salvus_job.SalvusJob
:param event: Name of event
:type event: str
:param sim_type: Type of simulation
:type sim_type: str
"""
gradient = False
if sim_type == "prepare_forward":
if self.comm.project.prepare_forward_job[event]["submitted"]:
job_name = self.comm.project.prepare_forward_job[event]["name"]
else:
raise InversionsonError(
f"Model interpolation job for event: {event} "
"has not been submitted")
if sim_type == "hpc_processing":
if self.comm.project.hpc_processing_job[event]["submitted"]:
job_name = self.comm.project.hpc_processing_job[event]["name"]
else:
raise InversionsonError(
f"HPC processing job for event: {event} "
"has not been submitted")
elif sim_type == "gradient_interp":
gradient = True
if self.comm.project.gradient_interp_job[event]["submitted"]:
job_name = self.comm.project.gradient_interp_job[event]["name"]
else:
raise InversionsonError(
f"Gradient interpolation job for event: {event} "
"has not been submitted")
site_name = self.comm.project.interpolation_site
db_job = sapi._get_config()["db"].get_jobs(
limit=1,
site_name=site_name,
job_name=job_name,
)[0]
job = s_job.Job(
site=sapi.get_site(site_name=db_job.site.site_name),
commands=self.comm.multi_mesh.get_interp_commands(event, gradient),
job_type=db_job.job_type,
job_info=db_job.info,
jobname=db_job.job_name,
job_description=db_job.description,
wall_time_in_seconds=db_job.wall_time_in_seconds,
working_dir=pathlib.Path(db_job.working_directory),
tmpdir_root=pathlib.Path(db_job.temp_directory_root)
if db_job.temp_directory_root else None,
rundir_root=pathlib.Path(db_job.run_directory_root)
if db_job.run_directory_root else None,
job_groups=[i.group_name for i in db_job.groups],
initialize_on_site=False,
)
return job
def get_job_status(self,
event: str,
sim_type: str,
par="VS",
iteration="current") -> str:
"""
Check the status of a salvus opt job
:param event: Name of event
:type event: str
:param sim_type: Type of simulation: forward, adjoint or smoothing
:type sim_type: str
:param par: parameter to be smoothed, optional, default VS
:type par: str
:param iteration: Name of iteration. "current" if current iteration
:type iteration: str
:return: status of job
:rtype: str
"""
if iteration == "current":
if sim_type == "forward":
if self.comm.project.forward_job[event]["submitted"]:
job_name = self.comm.project.forward_job[event]["name"]
else:
raise InversionsonError(
f"Forward job for event: {event} has not been "
"submitted")
elif sim_type == "adjoint":
if self.comm.project.adjoint_job[event]["submitted"]:
job_name = self.comm.project.adjoint_job[event]["name"]
else:
raise InversionsonError(
f"Adjoint job for event: {event} has not been "
"submitted")
elif sim_type == "smoothing":
if self.comm.project.smoothing_job[event][par]["submitted"]:
job_name = self.comm.project.smoothing_job[event][par][
"name"]
else:
raise InversionsonError(
f"Smoothing job for event: {event}, param {par} has not been "
"submitted")
else:
it_dict = self.comm.project.get_old_iteration_info(iteration)
job_name = it_dict["events"][event]["jobs"][sim_type]["name"]
job = sapi.get_job(job_name=job_name,
site_name=self.comm.project.site_name)
return job.update_status()
def perform_task(self, verbose=False):
"""
Look at which task is the current one and call the function which does it.
"""
task_name = self.task_dict["task"]
self.print(f"Current task is: {task_name}", line_above=True)
if task_name == "prepare_iteration":
if not self.task_dict["finished"]:
if self.comm.lasif.has_iteration(self.iteration_name):
self.print(
f"Iteration {self.iteration_name} exists. Will load its attributes"
)
self.comm.project.get_iteration_attributes()
self.finish_task()
else:
self.prepare_iteration()
else:
self.print("Iteration already prepared")
elif task_name == "compute_gradient":
if not self.task_dict["finished"]:
self.comm.project.get_iteration_attributes()
self.compute_gradient(verbose=verbose)
else:
self.print("Gradient already computed")
elif task_name == "update_model":
if not self.task_dict["finished"]:
self.comm.project.get_iteration_attributes()
self.update_model(verbose=verbose)
else:
self.print("Model already updated")
else:
raise InversionsonError(
f"Task {task_name} is not recognized by AdamOpt")
def perform_smoothing(self):
tasks = self._get_model_smoothing_task()
if max(self.update_smoothing_length) > 0.0:
tasks["smooth_raw_update"] = {
"reference_model": str(self.comm.lasif.get_master_model()),
"model_to_smooth": str(self.raw_update_path),
"smoothing_lengths": self.update_smoothing_length,
"smoothing_parameters": self.parameters,
"output_location": str(self.smooth_update_path),
}
if len(tasks.keys()) > 0:
reg_helper = RegularizationHelper(
comm=self.comm,
iteration_name=self.iteration_name,
tasks=tasks)
reg_helper.monitor_tasks()
else:
raise InversionsonError(
"We require some sort of smoothing in Adam Optimization")
# Write XDFMs
if max(self.update_smoothing_length) > 0.0:
write_xdmf(self.smooth_update_path)
if max(self.roughness_decay_smoothing_length) > 0.0:
write_xdmf(self.smoothed_model_path)
def update_iteration_toml(self, iteration="current", validation=False):
"""
Use iteration parameters to update iteration toml file
:param iteration: Name of iteration
:type iteration: str
"""
if iteration == "current":
iteration = self.current_iteration
if "validation" in iteration:
validation = True
if validation and "validation" not in iteration:
iteration = f"validation_{iteration}"
iteration_toml = os.path.join(self.paths["iteration_tomls"],
iteration + ".toml")
if not os.path.exists(iteration_toml):
raise InversionsonError(
f"Iteration toml for iteration: {iteration} does not exists")
if os.path.exists(self.paths["control_group_toml"]) and not validation:
control_group_dict = toml.load(self.paths["control_group_toml"])
control_group_dict = control_group_dict[iteration]
elif self.inversion_mode == "mini-batch":
control_group_dict = {"old": [], "new": []}
it_dict = {}
it_dict["name"] = iteration
it_dict["events"] = {}
if self.meshes == "mono-mesh":
it_dict["remote_simulation_mesh"] = self.remote_mesh
# I need a way to figure out what the controlgroup is
# This definitely needs improvement
if not validation and self.inversion_mode == "mini-batch":
it_dict["last_control_group"] = control_group_dict["old"]
it_dict["new_control_group"] = control_group_dict["new"]
for _i, event in enumerate(
self.comm.lasif.list_events(iteration=iteration)):
jobs = {"forward": self.forward_job[event]}
if not validation:
jobs["adjoint"] = self.adjoint_job[event]
if self.inversion_mode == "mini-batch":
if not validation:
jobs["smoothing"] = self.smoothing_job[event]
it_dict["events"][str(_i)] = {
"name": event,
"job_info": jobs,
}
else:
it_dict["events"][str(_i)] = {
"job_info": jobs,
}
if not validation:
it_dict["events"][str(_i)]["misfit"] = self.misfits[event]
it_dict["events"][str(
_i)]["usage_updated"] = self.updated[event]
if self.inversion_mode == "mono-batch" and not validation:
it_dict["smoothing"] = self.smoothing_job
with open(iteration_toml, "w") as fh:
toml.dump(it_dict, fh)
def add_fields_for_interpolation_to_mesh(self, gradient=False):
"""
In order to do a layered interpolation, we need some fields to be
present in the model.
:param gradient: We preparing for gradient interpolation?
defaults to False
:type gradient: bool, optional
"""
iteration = self.comm.project.current_iteration
if gradient:
raise InversionsonError("Not yet implemented")
else:
model = self.find_model_file(iteration)
self.comm.salvus_mesher.add_field_from_one_mesh_to_another(
from_mesh=self.comm.project.domain_file,
to_mesh=model,
field_name="layer",
elemental=True,
overwrite=False,
)
self.comm.salvus_mesher.add_field_from_one_mesh_to_another(
from_mesh=self.comm.project.domain_file,
to_mesh=model,
field_name="fluid",
elemental=True,
overwrite=False,
)
self.comm.salvus_mesher.add_field_from_one_mesh_to_another(
from_mesh=self.comm.project.domain_file,
to_mesh=model,
field_name="moho_idx",
global_string=True,
overwrite=False,
)
def monitor_jobs(self, smooth_individual=False):
"""
Takes the job type of the object and monitors the status of
all the events in the object.
:raises InversionsonError: Error if job type not recognized
"""
if self.job_type == "forward":
job_dict = self.comm.project.forward_job
elif self.job_type == "adjoint":
job_dict = self.comm.project.adjoint_job
elif self.job_type == "prepare_forward":
job_dict = self.comm.project.prepare_forward_job
elif self.job_type == "gradient_interp":
job_dict = self.comm.project.gradient_interp_job
elif self.job_type == "hpc_processing":
job_dict = self.comm.project.hpc_processing_job
else:
job_dict = self.comm.project.smoothing_job
if self.job_type in [
"forward",
"adjoint",
"prepare_forward",
"gradient_interp",
"hpc_processing",
]:
self.__monitor_jobs(job_dict=job_dict)
elif self.job_type == "smoothing":
self.__monitor_job_array(job_dict=job_dict,
smooth_individual=smooth_individual)
else:
raise InversionsonError(f"Job type {self.job_type} not recognised")
def change_attribute(self, attribute: str, new_value):
"""
Not possible to change attributes from another class.
This method should take care of it
:param attribute: Name of attribute
:type attribute: str
:param new_value: The new value to assign to the attribute
:type new_value: whatever the attribure needs
"""
if isinstance(new_value, str):
command = f'self.{attribute} = "{new_value}"'
elif isinstance(new_value, list):
command = f"self.{attribute} = {new_value}"
elif isinstance(new_value, bool):
command = f"self.{attribute} = {new_value}"
elif isinstance(new_value, dict):
command = f"self.{attribute} = {new_value}"
elif isinstance(new_value, float):
command = f"self.{attribute} = {new_value}"
elif isinstance(new_value, int):
command = f"self.{attribute} = {new_value}"
else:
raise InversionsonError(
f"Method not implemented for type {new_value.type}")
exec(command)
def __init__(self, comm):
# This init is only called by derived classes
self.current_task = self.read_current_task()
self.comm = comm
self.opt_folder = (
Path(self.comm.project.paths["inversion_root"]) / "OPTIMIZATION"
)
self.parameters = self.comm.project.inversion_params
if not os.path.exists(self.opt_folder):
os.mkdir(self.opt_folder)
# These folders are universally needed
self.model_dir = self.opt_folder / "MODELS"
self.task_dir = self.opt_folder / "TASKS"
self.average_model_dir = self.opt_folder / "AVERAGE_MODELS"
self.raw_gradient_dir = self.opt_folder / "RAW_GRADIENTS"
self.raw_update_dir = self.opt_folder / "RAW_UPDATES"
self.regularization_dir = self.opt_folder / "REGULARIZATION"
self.gradient_norm_dir = self.opt_folder / "GRADIENT_NORMS"
# Do any folder initilization for the derived classes
self._initialize_derived_class_folders()
self.config_file = self.opt_folder / "opt_config.toml"
if not os.path.exists(self.config_file):
self._write_initial_config()
print(
f"Please set config and provide initial model to "
f"{self.optimizer_name} optimizer in {self.config_file} \n"
f"Then reinitialize the {self.optimizer_name} optimizer."
)
sys.exit()
self._read_config()
if self.initial_model == "":
print(
f"Please set config and provide initial model to "
f"{self.optimizer_name} optimizer in {self.config_file} \n"
f"Then reinitialize the {self.optimizer_name} optimizer."
)
sys.exit()
# Initialize folders if needed
if not os.path.exists(self._get_path_for_iteration(0, self.model_path)):
if self.initial_model is None:
raise InversionsonError(
f"{self.optimizer_name} needs to be initialized with a "
"path to an initial model."
)
print(f"Initializing {self.optimizer_name}...")
self._init_directories()
self._issue_first_task()
self.tmp_model_path = self.opt_folder / "tmp_model.h5"
self._read_task_file()
def get_new_task(self):
if self.task_dict["finished"]:
self._write_new_task()
self.print(f"New task is: {self.task_dict['task']}",
line_above=True)
else:
raise InversionsonError(
f"Task: {self.task_dict['task']} is not finished.")
def __init__(self, comm):
self.available_tasks = [
"prepare_iteration",
"compute_gradient",
"update_model",
]
self.comm = comm
self.opt_folder = (Path(self.comm.project.paths["inversion_root"]) /
"OPTIMIZATION")
self.models = self.opt_folder / "MODELS"
if not os.path.exists(self.opt_folder):
os.mkdir(self.opt_folder)
self.config_file = self.opt_folder / "opt_config.toml"
self.model_dir = self.opt_folder / "MODELS"
self.task_dir = self.opt_folder / "TASKS"
if not os.path.exists(self.config_file):
self._write_initial_config()
print(f"Please set config and provide initial model to "
f"Adam optimizer in {self.config_file} \n"
f"Then reinitialize the Adam Optimizer.")
return
self._read_config()
if self.initial_model == "":
raise InversionsonError(
f"Please set config and provide initial model to "
f"Adam optimizer in {self.config_file} \n"
f"Then reinitialize the Adam Optimizer.")
# Initialize folders if needed
if not os.path.exists(self._get_path_for_iteration(0,
self.model_path)):
if self.initial_model is None:
raise InversionsonError(
"AdamOptimizer needs to be initialized with a "
"path to an initial model.")
print("Initializing Adam...")
self._init_directories()
self._issue_first_task()
self.tmp_model_path = self.opt_folder / "tmp_model.h5"
self._read_task_file()
def finish_task(self):
paths = ["raw_update_path", "model", "raw_gradient_path"]
if max(self.update_smoothing_length) > 0.0:
paths.append("smooth_update_path")
if max(self.roughness_decay_smoothing_length) > 0.0:
paths.append("smoothed_model_path")
complete_checks = [
"smoothing_completed",
"gradient_completed",
"iteration_finalized",
"forward_submitted",
"raw_update_completed",
"smooth_update_completed",
"misfit_completed",
"summing_completed",
"validated:",
]
for path in paths:
if path in self.task_dict.keys():
if not os.path.exists(self.task_dict[path]):
raise InversionsonError(
f"Trying to finish task but it can't find {self.task_dict[path]}"
)
for complete_check in complete_checks:
if complete_check in self.task_dict.keys():
if not self.task_dict[complete_check]:
raise InversionsonError(
f"Trying to finish task but {complete_check} is not completed"
)
self.task_dict["finished"] = True
if self.task_dict["task"] == "update_model":
self._update_task_file()
target_location = self._get_path_for_iteration(
self.iteration_number + 1, self.model_path)
# Moving the new model into its place, moves the iteration property to the next one.
shutil.move(
self.tmp_model_path,
target_location,
)
write_xdmf(target_location)
else:
self._update_task_file()
def get_iteration_attributes(self, validation=False):
"""
Save the attributes of the current iteration into memory
:param iteration: Name of iteration
:type iteration: str
"""
iteration = self.comm.salvus_opt.get_newest_iteration_name()
if validation:
iteration = f"validation_{iteration}"
iteration_toml = os.path.join(self.paths["iteration_tomls"],
iteration + ".toml")
if not os.path.exists(iteration_toml):
raise InversionsonError(
f"No toml file exists for iteration: {iteration}")
it_dict = toml.load(iteration_toml)
self.iteration_name = it_dict["name"]
self.current_iteration = self.iteration_name
self.events_in_iteration = self.comm.lasif.list_events(
iteration=iteration)
if not validation:
if self.inversion_mode == "mini-batch":
self.old_control_group = it_dict["last_control_group"]
self.new_control_group = it_dict["new_control_group"]
self.adjoint_job = {}
self.smoothing_job = {}
self.misfits = {}
self.updated = {}
self.forward_job = {}
if self.meshes == "mono-mesh":
if "remote_simulation_mesh" not in it_dict.keys():
self.remote_mesh = None
else:
self.remote_mesh = it_dict["remote_simulation_mesh"]
else:
self.remote_mesh = None
# Not sure if it's worth it to include station misfits
for _i, event in enumerate(self.events_in_iteration):
if not validation:
self.updated[event] = it_dict["events"][str(
_i)]["usage_updated"]
self.misfits[event] = it_dict["events"][str(_i)]["misfit"]
self.adjoint_job[event] = it_dict["events"][str(
_i)]["job_info"]["adjoint"]
if self.inversion_mode == "mini-batch":
self.smoothing_job[event] = it_dict["events"][str(
_i)]["job_info"]["smoothing"]
self.forward_job[event] = it_dict["events"][str(
_i)]["job_info"]["forward"]
if self.inversion_mode == "mono-batch" and not validation:
self.smoothing_job = it_dict["smoothing"]
def get_iteration_attributes(self):
"""
Save the attributes of the current iteration into memory
:param iteration: Name of iteration
:type iteration: str
"""
optimizer = self.get_optimizer()
iteration = optimizer.iteration_name
iteration_toml = os.path.join(self.paths["iteration_tomls"],
iteration + ".toml")
if not os.path.exists(iteration_toml):
raise InversionsonError(
f"No toml file exists for iteration: {iteration}")
it_dict = toml.load(iteration_toml)
self.iteration_name = it_dict["name"]
self.current_iteration = self.iteration_name
self.events_in_iteration = self.comm.lasif.list_events(
iteration=iteration)
self.non_val_events_in_iteration = list(
set(self.events_in_iteration) - set(self.validation_dataset))
self.adjoint_job = {}
self.misfits = {}
self.updated = {}
self.prepare_forward_job = {}
self.forward_job = {}
self.hpc_processing_job = {}
self.gradient_interp_job = {}
# Not sure if it's worth it to include station misfits
for _i, event in enumerate(self.events_in_iteration):
if not self.is_validation_event(event):
self.updated[event] = it_dict["events"][str(
_i)]["usage_updated"]
self.misfits[event] = it_dict["events"][str(_i)]["misfit"]
self.adjoint_job[event] = it_dict["events"][str(
_i)]["job_info"]["adjoint"]
self.forward_job[event] = it_dict["events"][str(
_i)]["job_info"]["forward"]
if self.prepare_forward:
self.prepare_forward_job[event] = it_dict["events"][str(
_i)]["job_info"]["prepare_forward"]
if self.remote_interp:
if not self.is_validation_event(event):
self.gradient_interp_job[event] = it_dict["events"][str(
_i)]["job_info"]["gradient_interp"]
if self.hpc_processing and not self.is_validation_event(event):
self.hpc_processing_job[event] = it_dict["events"][str(
_i)]["job_info"]["hpc_processing"]
def update_iteration_toml(self, iteration="current"):
"""
Use iteration parameters to update iteration toml file
:param iteration: Name of iteration
:type iteration: str
"""
if iteration == "current":
iteration = self.current_iteration
iteration_toml = os.path.join(
self.paths["iteration_tomls"], iteration + ".toml"
)
if not os.path.exists(iteration_toml):
raise InversionsonError(
f"Iteration toml for iteration: {iteration} does not exists"
)
if os.path.exists(self.paths["control_group_toml"]):
control_group_dict = toml.load(self.paths["control_group_toml"])
control_group_dict = control_group_dict[iteration]
else:
control_group_dict = {"old": [], "new": []}
it_dict = {}
it_dict["name"] = iteration
it_dict["events"] = {}
# I need a way to figure out what the controlgroup is
# This definitely needs improvement
it_dict["last_control_group"] = control_group_dict["old"]
it_dict["new_control_group"] = control_group_dict["new"]
for event in self.comm.lasif.list_events(iteration=iteration):
if self.meshes == "multi-mesh":
it_dict["events"][event] = {
"misfit": self.misfits[event],
"usage_updated": self.updated[event],
"jobs": {
"forward": self.forward_job[event],
"adjoint": self.adjoint_job[event],
"smoothing": self.smoothing_job[event],
},
}
else:
it_dict["events"][event] = {
"misfit": self.misfits[event],
"usage_updated": self.updated[event],
"jobs": {
"forward": self.forward_job[event],
"adjoint": self.adjoint_job[event],
},
}
if self.meshes == "mono-mesh":
it_dict["smoothing"] == self.smoothing_job
with open(iteration_toml, "w") as fh:
toml.dump(it_dict, fh)
def read_salvus_opt(self) -> dict:
"""
Read the task that salvus opt has issued into a dictionary
:return: The information contained in the task toml file
:rtype: dictionary
"""
if os.path.exists(os.path.join(self.path, "task.toml")):
task = toml.load(os.path.join(self.path, "task.toml"))
return task
else:
raise InversionsonError("no_task_toml")
def get_optimizer(self):
"""
This creates an instance of the optimization class which is
picked by the user.
"""
if self.optimizer == "adam":
return AdamOpt(comm=self.comm)
if self.optimizer == "sgdm":
return SGDM(comm=self.comm)
else:
raise InversionsonError(
f"Optimization method {self.optimizer} not defined")
def read_salvus_opt_task(self) -> str:
"""
Read the task from salvus opt. See what to do next
:return: task name
:rtype: str
"""
if os.path.exists(os.path.join(self.path, "task.toml")):
task = toml.load(os.path.join(self.path, "task.toml"))
task_type = task["task"][0]["type"]
verbose = task["task"][0]["_meta"]["verbose"]
return task_type, verbose
else:
raise InversionsonError("no task toml")
def generate_smoothing_config(self, event: str) -> dict:
"""
Generate a dictionary which contains smoothing objects for each
parameter to be smoothed.
:param event: Name of event
:type event: str
:return: Dictonary which points each parameter to a smoothing object
:rtype: dict
"""
# The mesh used as diffusion model is the event_mesh with the 1D model
diff_model = self.comm.lasif.find_event_mesh(event=event)
smoothing_config = {}
freq = 1.0 / self.comm.project.min_period
smoothing_lengths = self.comm.project.smoothing_lengths
import toml
# Loop through parameters to assign smoothing objects to parameters.
for param in self.comm.project.inversion_params:
if param.startswith("V"):
reference_velocity = param
elif param == "RHO":
if "VP" in self.comm.project.inversion_params:
reference_velocity = "VP"
elif "VPV" in self.comm.project.inversion_params:
reference_velocity = "VPV"
else:
raise InversionsonError(
f"Unexpected case while smoothing {param}. "
f"Take a closer look")
if self.comm.project.smoothing_mode == "anisotropic":
smooth = smoothing.AnisotropicModelDependent(
reference_frequency_in_hertz=freq,
smoothing_lengths_in_wavelengths=smoothing_lengths,
reference_model=diff_model,
reference_velocity=reference_velocity,
)
elif self.comm.project.smoothing_mode == "isotropic":
smooth = smoothing.IsotropicModelDependent(
reference_frequency_in_hertz=freq,
smoothing_length_in_wavelengths=smoothing_lengths,
reference_model=diff_model,
reference_velocity=reference_velocity,
)
smoothing_config[param] = smooth
with open("./smoothing_config.toml", "w") as fh:
toml.dump(smoothing_config, fh)
return smoothing_config
def preprocess_remote_gradient(comm, gradient_path: str, event: str):
"""
Cut sources and receivers from gradient before smoothing.
We also clip the gradient to some percentile
This can all be configured in information toml.
:param comm inversionson communicator
:param gradient_path: gradient path on remote
:type gradient_path: str
:param event: name of the event
"""
# Connect to daint
daint = get_site(comm.project.site_name)
username = daint.config["ssh_settings"]["username"]
remote_inversionson_dir = os.path.join("/scratch/snx3000", username,
"smoothing_info")
if not daint.remote_exists(remote_inversionson_dir):
daint.remote_mkdir(remote_inversionson_dir)
# copy processing script to daint
remote_script = os.path.join(remote_inversionson_dir, "cut_and_clip.py")
if not daint.remote_exists(remote_script):
daint.remote_put(CUT_SOURCE_SCRIPT_PATH, remote_script)
if comm.project.cut_receiver_radius > 0.0:
raise InversionsonError("Remote receiver cutting not implemented yet.")
info = {}
info["filename"] = str(gradient_path)
info["cutout_radius_in_km"] = comm.project.cut_source_radius
info["source_location"] = comm.lasif.get_source(event_name=event)
info["clipping_percentile"] = comm.project.clip_gradient
info["parameters"] = comm.project.inversion_params
toml_filename = f"{event}_gradient_process.toml"
with open(toml_filename, "w") as fh:
toml.dump(info, fh)
# put toml on daint and remove local toml
remote_toml = os.path.join(remote_inversionson_dir, toml_filename)
daint.remote_put(toml_filename, remote_toml)
os.remove(toml_filename)
# Call script
print(daint.run_ssh_command(f"python {remote_script} {remote_toml}"))
def write_new_opt_fields_to_simulation_mesh(self):
"""
Salvus opt makes a mesh which has the correct velocities but
it does not have everything which is needed to run a simulation.
We will thus write it's fields on to our simulation mesh.
"""
if self.comm.project.meshes == "multi-mesh":
raise InversionsonError(
"Multi-mesh inversion should not use this function. Only "
"Mono-mesh.")
print("Writing new fields to simulation mesh")
iteration = self.comm.project.current_iteration
if "validation" in iteration:
iteration = iteration[11:] # We don't need a special mesh
opt_model = os.path.join(self.comm.salvus_opt.models,
f"{iteration}.h5")
simulation_mesh = self.comm.lasif.get_simulation_mesh(
event_name=None, iteration="current")
sim_mesh_dir = os.path.dirname(simulation_mesh)
success_file = os.path.join(sim_mesh_dir, "success.txt")
if os.path.exists(simulation_mesh) and os.path.exists(success_file):
print("Mesh already exists, will not add fields")
return
else:
shutil.copy(
self.comm.lasif.lasif_comm.project.
lasif_config["domain_settings"]["domain_file"],
simulation_mesh,
)
with h5py.File(simulation_mesh, mode="r+") as f_new:
with h5py.File(opt_model, mode="r") as f:
dim_labels = (f["MODEL/data"].attrs.get("DIMENSION_LABELS")
[1].decode()[1:-1].replace(" ", "").split("|"))
# This assumes the indices are the same in both files,
# which seems to be the case as far as DP could tell.
for param in self.comm.project.inversion_params:
print("Writing field:", param)
i = dim_labels.index(param)
f_new["MODEL/data"][:, i, :] = f["MODEL/data"][:, i, :]
# When all fields are successfully copied write a file to indicate
# success to prevent the issue that we continue
# with the initial model when something here crashes unexpectedly.
with open(success_file, "w") as text_file:
text_file.write("All fields written successfully.")
def get_newest_iteration_name(self):
"""
Get the name of the newest iteration given by salvus opt.
We will look for the iteration with the highest number
and if there are multiple we look at the one with the smallest
trust region.
"""
models = self._get_all_model_names()
iterations = self._parse_model_files(models)
new_it_number = max(iterations)
if len(iterations[new_it_number]) > 4:
raise InversionsonError("Looks like model has been rejected too often")
new_it_tr_region = min(iterations[new_it_number])
return self._create_iteration_name(new_it_number, new_it_tr_region)
def _update_usage_of_events(self):
"""
To keep track of how often events are used.
"""
for event in self.comm.project.events_in_iteration:
if not self.comm.project.updated[event]:
if event not in self.events_used.keys():
self.events_used[event] = 0
if isinstance(self.events_used[event], str):
raise InversionsonError("Events used are strings")
self.events_used[event] += 1
self.comm.project.change_attribute(
attribute=f'updated["{event}"]', new_value=True)
self.comm.project.update_iteration_toml()
with open(self.events_used_toml, "w") as fh:
toml.dump(self.events_used, fh)
def _write_new_task(self):
self._read_task_file()
if not self.task_dict["finished"]:
raise InversionsonError(
f"Task {self.task_dict['task']} does not appear to be finished"
)
if self.task_dict["task"] == "prepare_iteration":
task_dict = {
"task": "compute_gradient",
"model": str(self.model_path),
"forward_submitted": False,
"misfit_completed": False,
"gradient_completed": False,
"validated": False,
"iteration_number": self.iteration_number,
"finished": False,
}
task_file_path = self._increase_task_number()
elif self.task_dict["task"] == "compute_gradient":
task_dict = {
"task": "update_model",
"model": str(self.model_path),
"raw_update_path": str(self.raw_update_path),
"raw_gradient_path": str(self.raw_gradient_path),
"smooth_update_path": str(self.smooth_update_path),
"smoothed_model_path": str(self.smoothed_model_path),
"summing_completed": False,
"raw_update_completed": False,
"smoothing_completed": False,
"smooth_update_completed": False,
"iteration_finalized": False,
"iteration_number": self.iteration_number,
"finished": False,
}
task_file_path = self._increase_task_number()
elif self.task_dict["task"] == "update_model":
task_dict = {
"task": "prepare_iteration",
"model": str(self._model_for_iteration(self.iteration_number)),
"iteration_number": self.iteration_number,
"finished": False,
}
task_file_path = self._increase_iteration_number()
with open(task_file_path, "w+") as fh:
toml.dump(task_dict, fh)
self.task_dict = task_dict
def _update_model(self, raw=True, smooth=False):
"""
Apply an Adam style update to the model
"""
if (raw and smooth) or (not raw and not smooth):
raise InversionsonError(
"Adam updates can be raw or smooth, not both")
if raw:
gradient = (self.comm.lasif.lasif_comm.project.paths["gradients"] /
f"ITERATION_{self.iteration_name}" /
"summed_gradient.h5")
if not os.path.exists(self.raw_gradient_path):
shutil.copy(gradient, self.raw_gradient_path)
if not os.path.exists(self.raw_update_path):
self._compute_raw_update()
if smooth:
self._apply_smooth_update()
def find_event_mesh(self, event: str) -> pathlib.Path:
"""
Find the path for an event mesh
:param event: Name of event
:type event: str
:return: Path to where the mesh is stored.
:rtype: Pathlib.Path
"""
if self.comm.project.meshes == "mono-mesh":
mesh = self.lasif_comm.project.lasif_config["domain_settings"][
"domain_file"]
return mesh
has, mesh = lapi.find_event_mesh(self.lasif_comm, event)
if not has:
raise InversionsonError(
f"Mesh for event: {event} can not be found.")
return pathlib.Path(mesh)
def get_name_for_accepted_iteration_number(self, number: int):
"""
Can be used to get the full name of an iteration with a specific
number. If one wants iteration number 5 the input parameter
should be 5 and the full iteration number will be given. If there
are many iterations with that number, the smallest trust region
will be given.
:param number: Number of iteration
:type number: int
"""
models = self._get_all_model_names()
iterations = self._parse_model_files(models)
if not number in iterations.keys():
raise InversionsonError(f"Iteration number {number} does "
"not exist.")
tr_region = min(iterations[number])
return self._create_iteration_name(it_number, tr_region)
def task_path(self):
task_files = glob.glob(f"{self.task_dir}/task_*_*")
if len(task_files) <= 1:
return self.task_dir / f"task_00000_00.toml"
iteration_numbers = [
int(Path(x).stem.split("_")[-2]) for x in task_files
]
task_nums = glob.glob(
f"{self.task_dir}/task_{max(iteration_numbers):05d}_*")
if len(task_nums) <= 1:
task_nums = [0]
else:
task_nums = [int(Path(x).stem[-2:]) for x in task_nums]
if max(task_nums) >= len(self.available_tasks):
raise InversionsonError(
f"{task_nums}, but also... {max(iteration_numbers)}")
return (self.task_dir /
f"task_{max(iteration_numbers):05d}_{max(task_nums):02d}.toml")
def get_old_iteration_info(self, iteration: str) -> dict:
"""
For getting information about something else than current iteration
:param iteration: Name of iteration
:type iteration: str
:return: Information regarding that iteration
:rtype: dict
"""
iteration_toml = os.path.join(self.paths["iteration_tomls"],
iteration + ".toml")
if not os.path.exists(iteration_toml):
raise InversionsonError(
f"No toml file exists for iteration: {iteration}")
with open(iteration_toml, "r") as fh:
it_dict = toml.load(fh)
return it_dict
