def submit_job(
self,
event: str,
simulation: object,
sim_type: str,
site="daint",
wall_time=3600,
ranks=1024,
):
"""
Submit a job with some information. Salvus flow returns an object
which can be used to interact with job.
:param event: Name of event
:type event: str
:param simulation: Simulation object constructed beforehand
:type simulation: object
:param sim_type: Type of simulation, forward or adjoint
:type sim_type: str
:param site: Name of site in salvus flow config file, defaults
to "daint"
:type site: str, optional
:param wall_time: In what time the site kills your job [seconds],
defaults to 3600
:type wall_time: int, optional
:param ranks: How many cores to run on. (A multiple of 12 on daint),
defaults to 1024
:type ranks: int, optional
"""
# iteration = self.comm.project.current_iteration
# output_folder = os.path.join(
# self.comm.lasif.lasif_root,
# "SYNTHETICS",
# "EARTHQUAKES",
# f"ITERATION_{iteration}",
# event)
job = sapi.run_async(site_name=site,
input_file=simulation,
ranks=ranks,
wall_time_in_seconds=self.comm.project.wall_time
# output_folder=output_folder
)
# sapi.run(
# site_name=site,
# input_file=simulation,
# output_folder=output_folder,
# ranks=8,
# overwrite=True)
if sim_type == "forward":
self.comm.project.change_attribute(
f'forward_job["{event}"]["name"]', job.job_name)
self.comm.project.change_attribute(
f'forward_job["{event}"]["submitted"]', True)
elif sim_type == "adjoint":
self.comm.project.change_attribute(
f'adjoint_job["{event}"]["name"]', job.job_name)
self.comm.project.change_attribute(
f'adjoint_job["{event}"]["submitted"]', True)
self.comm.project.update_iteration_toml()
def submit_smoothing_job(self, event: str, simulation, par):
"""
Submit the salvus diffusion equation smoothing job
:param event: name of event
:type event: str
:param simulation: Simulation object required by salvus flow
:type simulation: object
:param par: Parameter to smooth
:type par: str
"""
# output_folder = os.path.join(
# self.comm.lasif.lasif_root,
# "GRADIENTS",
# f"ITERATION_{self.comm.project.current_iteration}",
# event,
# "smoother_output"
# )
# from salvus_mesh.unstructured_mesh import UnstructuredMesh
# if self.comm.project.site_name == "swp":
# for par in simulations.keys():
# sapi.run(
# #site_name="swp_smooth",
# site_name=self.comm.project.site_name,
# input_file=simulations[par],
# output_folder=output_folder,
# overwrite=True,
# ranks=8,
# get_all=True)
#
# smoothed = UnstructuredMesh.from_h5(os.path.join(output_folder, "smooth_gradient.h5"))
# smooth.attach_field(par, smoothed.elemental_fields[par])
# output_folder = os.path.join(
# self.comm.lasif.lasif_root,
# "GRADIENTS",
# f"ITERATION_{self.comm.project.current_iteration}",
# event
# )
# smooth.write_h5(os.path.join(output_folder, "smooth_gradient.h5"))
job = sapi.run_async(
site_name=self.comm.project.smoothing_site_name,
input_file=simulation,
ranks=self.comm.project.smoothing_ranks,
wall_time_in_seconds=self.comm.project.smoothing_wall_time,
)
self.comm.project.change_attribute(
f'smoothing_job["{event}"]["{par}"]["name"]', job.job_name
)
self.comm.project.change_attribute(
f'smoothing_job["{event}"]["{par}"]["submitted"]', True
)
def submit_smoothing_job(self, event: str, simulation, par):
"""
Submit the salvus diffusion equation smoothing job
:param event: name of event
:type event: str
:param simulation: Simulation object required by salvus flow
:type simulation: object
:param par: Parameter to smooth
:type par: str
"""
job = sapi.run_async(
site_name=self.comm.project.smoothing_site_name,
input_file=simulation,
ranks=self.comm.project.smoothing_ranks,
wall_time_in_seconds=self.comm.project.smoothing_wall_time,
)
self.comm.project.change_attribute(
f'smoothing_job["{event}"]["{par}"]["name"]', job.job_name)
self.comm.project.change_attribute(
f'smoothing_job["{event}"]["{par}"]["submitted"]', True)
def submit_job(
self,
event: str,
simulation: object,
sim_type: str,
site="daint",
wall_time=3600,
ranks=1024,
):
"""
Submit a job with some information. Salvus flow returns an object
which can be used to interact with job.
:param event: Name of event
:type event: str
:param simulation: Simulation object constructed beforehand
:type simulation: object
:param sim_type: Type of simulation, forward or adjoint
:type sim_type: str
:param site: Name of site in salvus flow config file, defaults
to "daint"
:type site: str, optional
:param wall_time: In what time the site kills your job [seconds],
defaults to 3600
:type wall_time: int, optional
:param ranks: How many cores to run on. (A multiple of 12 on daint),
defaults to 1024
:type ranks: int, optional
"""
# iteration = self.comm.project.current_iteration
# output_folder = os.path.join(
# self.comm.lasif.lasif_root,
# "SYNTHETICS",
# "EARTHQUAKES",
# f"ITERATION_{iteration}",
# event)
# Adjoint simulation takes longer and seems to be less predictable
# we thus give it a longer wall time.
print("Submitting the job now")
start = time.time()
if sim_type == "adjoint":
wall_time = self.comm.project.wall_time * 2
else:
wall_time = self.comm.project.wall_time
start_sub = time.time()
job = sapi.run_async(
site_name=site,
input_file=simulation,
ranks=ranks,
wall_time_in_seconds=wall_time,
# output_folder=output_folder
)
end_sub = time.time()
print(f"Only submission took: {end_sub - start_sub} seconds")
# sapi.run(
# site_name=site,
# input_file=simulation,
# output_folder=output_folder,
# ranks=8,
# overwrite=True)
if (self.comm.project.remote_mesh is None
and self.comm.project.meshes == "mono-mesh"):
self.comm.project.change_attribute(
"remote_mesh", "REMOTE:" + str(job.input_path / "mesh.h5"))
if sim_type == "forward":
self.comm.project.change_attribute(
f'forward_job["{event}"]["name"]', job.job_name)
self.comm.project.change_attribute(
f'forward_job["{event}"]["submitted"]', True)
elif sim_type == "adjoint":
self.comm.project.change_attribute(
f'adjoint_job["{event}"]["name"]', job.job_name)
self.comm.project.change_attribute(
f'adjoint_job["{event}"]["submitted"]', True)
self.comm.project.update_iteration_toml()
end = time.time()
print(f"Submitting took {end - start} seconds")
def submit_salvus_simulation(
comm: object,
simulations: Union[List[object], object],
events: Union[List[str], str],
iteration: str,
sim_type: str,
) -> object:
"""
Submit a Salvus simulation to the machine defined in config file
with details specified in config file
:param comm: The Lasif communicator object
:type comm: object
:param simulations: Simulation object
:type simulations: Union[List[object], object]
:param events: We need names of events for the corresponding simulations
in order to keep tabs on which simulation object corresponds to which
event.
:type events: Union[List[str], str]
:param iteration: Name of iteration, this is needed to know where to
download files to when jobs are done.
:type iteration: str
:param sim_type: can be either forward or adjoint.
:type sim_type: str
:return: SalvusJob object or an array of them
:rtype: object
"""
from salvus.flow.api import run_async, run_many_async
if sim_type not in ["forward", "adjoint"]:
raise LASIFError("sim_type needs to be forward or adjoint")
array = False
if isinstance(simulations, list):
array = True
if not isinstance(events, list):
raise LASIFError(
"If simulations are a list, events need to be "
"a list aswell, with the corresponding events in the same "
"order")
else:
if isinstance(events, list):
raise LASIFError("If there is only one simulation object, "
"there should be only one event")
iteration = comm.iterations.get_long_iteration_name(iteration)
if sim_type == "forward":
toml_file = (comm.project.paths["salvus_files"] / iteration /
"forward_jobs.toml")
elif sim_type == "adjoint":
toml_file = (comm.project.paths["salvus_files"] / iteration /
"adjoint_jobs.toml")
if os.path.exists(toml_file):
jobs = toml.load(toml_file)
else:
jobs = {}
site_name = comm.project.salvus_settings["site_name"]
ranks = comm.project.salvus_settings["ranks"]
wall_time = comm.project.salvus_settings["wall_time_in_s"]
if array:
job = run_many_async(
site_name=site_name,
input_files=simulations,
ranks_per_job=ranks,
wall_time_in_seconds_per_job=wall_time,
)
jobs["array_name"] = job.job_array_name
for _i, j in enumerate(job.jobs):
jobs[events[_i]] = j.job_name
else:
job = run_async(
site_name=site_name,
input_file=simulations,
ranks=ranks,
wall_time_in_seconds=wall_time,
)
jobs[events] = job.job_name
with open(toml_file, mode="w") as fh:
toml.dump(jobs, fh)
print(f"Wrote job information into {toml_file}")
return job
def submit_job(
self,
event: str,
simulation: object,
sim_type: str,
site="daint",
ranks=1024,
):
"""
Submit a job with some information. Salvus flow returns an object
which can be used to interact with job.
:param event: Name of event
:type event: str
:param simulation: Simulation object constructed beforehand
:type simulation: object
:param sim_type: Type of simulation, forward or adjoint
:type sim_type: str
:param site: Name of site in salvus flow config file, defaults
to "daint"
:type site: str, optional
:param ranks: How many cores to run on. (A multiple of 12 on daint),
defaults to 1024
:type ranks: int, optional
"""
# Adjoint simulation takes longer and seems to be less predictable
# we thus give it a longer wall time.
if sim_type == "adjoint":
wall_time = self.comm.project.wall_time * 1.5
else:
wall_time = self.comm.project.wall_time
start_submit = time.time()
job = sapi.run_async(
site_name=site,
input_file=simulation,
ranks=ranks,
wall_time_in_seconds=wall_time,
)
end_submit = time.time()
self.print(
f"Submitting took {end_submit - start_submit:.3f} seconds",
emoji_alias=":hourglass:",
color="magenta",
)
hpc_cluster = sapi.get_site(self.comm.project.site_name)
if sim_type == "forward":
self.comm.project.change_attribute(
f'forward_job["{event}"]["name"]', job.job_name)
self.comm.project.change_attribute(
f'forward_job["{event}"]["submitted"]', True)
elif sim_type == "adjoint":
self.comm.project.change_attribute(
f'adjoint_job["{event}"]["name"]', job.job_name)
self.comm.project.change_attribute(
f'adjoint_job["{event}"]["submitted"]', True)
self.comm.project.update_iteration_toml()
if hpc_cluster.config["site_type"] == "local":
self.print(f"Running {sim_type} simulation...")
job.wait(
poll_interval_in_seconds=self.comm.project.sleep_time_in_s)