def dispatch_smoothing_tasks(self): dispatching_msg = True for task_name, task_dict in self.tasks.items(): if (not task_dict["submitted"] and task_dict["reposts"] < self.comm.project.max_reposts): if dispatching_msg: self.print("Dispatching Smoothing Tasks") dispatching_msg = False sims = self.comm.smoother.get_sims_for_smoothing_task( reference_model=task_dict["reference_model"], model_to_smooth=task_dict["model_to_smooth"], smoothing_lengths=task_dict["smoothing_lengths"], smoothing_parameters=task_dict["smoothing_parameters"], ) job = sapi.run_many_async( input_files=sims, site_name=self.comm.project.smoothing_site_name, ranks_per_job=self.comm.project.smoothing_ranks, wall_time_in_seconds_per_job=self.comm.project. smoothing_wall_time, ) self.tasks[task_name]["submitted"] = True self.tasks[task_name]["job_name"] = job.job_array_name self._write_tasks(self.tasks) elif task_dict["reposts"] >= self.comm.project.max_reposts: raise Exception( "Too many reposts in smoothing, " "please check the time steps and the inputs." "and reset the number of reposts in the toml file.")
def submit_salvus_simulation( comm: object, simulations: Union[List[object], object], events: Union[List[str], str], iteration: str, sim_type: str, ) -> object: """ Submit a Salvus simulation to the machine defined in config file with details specified in config file :param comm: The Lasif communicator object :type comm: object :param simulations: Simulation object :type simulations: Union[List[object], object] :param events: We need names of events for the corresponding simulations in order to keep tabs on which simulation object corresponds to which event. :type events: Union[List[str], str] :param iteration: Name of iteration, this is needed to know where to download files to when jobs are done. :type iteration: str :param sim_type: can be either forward or adjoint. :type sim_type: str :return: SalvusJob object or an array of them :rtype: object """ from salvus.flow.api import run_async, run_many_async if sim_type not in ["forward", "adjoint"]: raise LASIFError("sim_type needs to be forward or adjoint") array = False if isinstance(simulations, list): array = True if not isinstance(events, list): raise LASIFError( "If simulations are a list, events need to be " "a list aswell, with the corresponding events in the same " "order") else: if isinstance(events, list): raise LASIFError("If there is only one simulation object, " "there should be only one event") iteration = comm.iterations.get_long_iteration_name(iteration) if sim_type == "forward": toml_file = (comm.project.paths["salvus_files"] / iteration / "forward_jobs.toml") elif sim_type == "adjoint": toml_file = (comm.project.paths["salvus_files"] / iteration / "adjoint_jobs.toml") if os.path.exists(toml_file): jobs = toml.load(toml_file) else: jobs = {} site_name = comm.project.salvus_settings["site_name"] ranks = comm.project.salvus_settings["ranks"] wall_time = comm.project.salvus_settings["wall_time_in_s"] if array: job = run_many_async( site_name=site_name, input_files=simulations, ranks_per_job=ranks, wall_time_in_seconds_per_job=wall_time, ) jobs["array_name"] = job.job_array_name for _i, j in enumerate(job.jobs): jobs[events[_i]] = j.job_name else: job = run_async( site_name=site_name, input_file=simulations, ranks=ranks, wall_time_in_seconds=wall_time, ) jobs[events] = job.job_name with open(toml_file, mode="w") as fh: toml.dump(jobs, fh) print(f"Wrote job information into {toml_file}") return job
def run_remote_smoother( self, event: str, ): """ Run the Smoother, the settings are specified in inversion toml. Make sure that the smoothing config has already been generated :param event: Name of event :type event: str """ from salvus.opt import smoothing import salvus.flow.simple_config as sc from salvus.flow.api import get_site from salvus.flow import api as sapi if self.comm.project.meshes == "multi-mesh": mesh = self.comm.lasif.find_event_mesh(event) else: mesh = self.comm.lasif.get_simulation_mesh(event) freq = 1.0 / self.comm.project.min_period smoothing_lengths = self.comm.project.smoothing_lengths # get remote gradient filename job = self.comm.salvus_flow.get_job(event, "adjoint") output_files = job.get_output_files() remote_grad = str(output_files[0][("adjoint", "gradient", "output_filename")]) # make site stuff (hardcoded for now) daint = get_site(self.comm.project.site_name) username = daint.config["ssh_settings"]["username"] remote_diff_dir = os.path.join("/scratch/snx3000", username, "diff_models") local_diff_model_dir = "DIFF_MODELS" if not os.path.exists(local_diff_model_dir): os.mkdir(local_diff_model_dir) if not daint.remote_exists(remote_diff_dir): daint.remote_mkdir(remote_diff_dir) sims = [] for param in self.comm.project.inversion_params: if param.startswith("V"): reference_velocity = param elif param == "RHO": if "VP" in self.comm.project.inversion_params: reference_velocity = "VP" elif "VPV" in self.comm.project.inversion_params: reference_velocity = "VPV" unique_id = ( "_".join([str(i).replace(".", "") for i in smoothing_lengths]) + "_" + str(self.comm.project.min_period)) diff_model_file = unique_id + f"diff_model_{param}.h5" if self.comm.project.meshes == "multi-mesh": diff_model_file = event + "_" + diff_model_file remote_diff_model = os.path.join(remote_diff_dir, diff_model_file) diff_model_file = os.path.join(local_diff_model_dir, diff_model_file) if not os.path.exists(diff_model_file): smooth = smoothing.AnisotropicModelDependent( reference_frequency_in_hertz=freq, smoothing_lengths_in_wavelengths=smoothing_lengths, reference_model=mesh, reference_velocity=reference_velocity, ) diff_model = smooth.get_diffusion_model(mesh) diff_model.write_h5(diff_model_file) if not daint.remote_exists(remote_diff_model): daint.remote_put(diff_model_file, remote_diff_model) sim = sc.simulation.Diffusion(mesh=diff_model_file) if self.comm.project.meshes == "multi-mesh": tensor_order = 4 else: tensor_order = 2 sim.domain.polynomial_order = tensor_order sim.physics.diffusion_equation.time_step_in_seconds = ( self.comm.project.smoothing_timestep) sim.physics.diffusion_equation.courant_number = 0.06 sim.physics.diffusion_equation.initial_values.filename = ( "REMOTE:" + remote_grad) sim.physics.diffusion_equation.initial_values.format = "hdf5" sim.physics.diffusion_equation.initial_values.field = f"{param}" sim.physics.diffusion_equation.final_values.filename = ( f"{param}.h5") sim.domain.mesh.filename = "REMOTE:" + remote_diff_model sim.domain.model.filename = "REMOTE:" + remote_diff_model sim.domain.geometry.filename = "REMOTE:" + remote_diff_model sim.validate() # append sim to array sims.append(sim) job = sapi.run_many_async( input_files=sims, site_name=self.comm.project.smoothing_site_name, ranks_per_job=self.comm.project.smoothing_ranks, wall_time_in_seconds_per_job=self.comm.project.smoothing_wall_time, ) if self.comm.project.inversion_mode == "mini-batch": self.comm.project.change_attribute( f'smoothing_job["{event}"]["name"]', job.job_array_name) self.comm.project.change_attribute( f'smoothing_job["{event}"]["submitted"]', True) else: self.comm.project.change_attribute('smoothing_job["name"]', job.job_array_name) self.comm.project.change_attribute('smoothing_job["submitted"]', True)